Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

feat(processor): register processing nodes and distribute encrypted configuration

Timothy Yin862f255ee003
diff --git a/.github/workflows/deploy-saas.yml b/.github/workflows/deploy-saas.yml
index beaf3ea2..e156a6a9 100644
--- a/.github/workflows/deploy-saas.yml
+++ b/.github/workflows/deploy-saas.yml
@@ -150,7 +150,6 @@ jobs:
CLOUD_PLATFORM_SECRET: ${{ secrets.CLOUD_PLATFORM_SECRET }}
SITE_PLATFORM_SECRET: ${{ secrets.SITE_PLATFORM_SECRET }}
PLATFORM_STORAGE_SECRET: ${{ secrets.PLATFORM_STORAGE_SECRET }}
- PLATFORM_WORKER_SECRET: ${{ secrets.PLATFORM_WORKER_SECRET }}
PLATFORM_S3_ENDPOINT_HOSTS: ${{ vars.PLATFORM_S3_ENDPOINT_HOSTS }}
run: |
node --input-type=module <<'NODE'
@@ -158,7 +157,6 @@ jobs:
domainDeploymentValues(process.env.DEPLOY_ENVIRONMENT,process.env)
if (process.env.PLATFORM_D1_DATABASE_ID) {
if ((process.env.PLATFORM_STORAGE_SECRET || '').length < 32) throw new Error('Configure PLATFORM_STORAGE_SECRET with at least 32 characters in the selected GitHub Environment secrets')
- if (process.env.ENTRIES.split(/\s+/).includes('cloud') && (process.env.PLATFORM_WORKER_SECRET || '').length < 32) throw new Error('Configure PLATFORM_WORKER_SECRET with at least 32 characters in the selected GitHub Environment secrets')
for (const host of (process.env.PLATFORM_S3_ENDPOINT_HOSTS || '').split(',').filter(Boolean)) {
if (!/^[a-z0-9.-]+$/i.test(host.trim())) throw new Error('PLATFORM_S3_ENDPOINT_HOSTS must contain exact hostnames')
}
@@ -276,7 +274,6 @@ jobs:
CLOUD_PLATFORM_SECRET: ${{ secrets.CLOUD_PLATFORM_SECRET }}
SITE_PLATFORM_SECRET: ${{ secrets.SITE_PLATFORM_SECRET }}
PLATFORM_STORAGE_SECRET: ${{ secrets.PLATFORM_STORAGE_SECRET }}
- PLATFORM_WORKER_SECRET: ${{ secrets.PLATFORM_WORKER_SECRET }}
PLATFORM_S3_ENDPOINT_HOSTS: ${{ vars.PLATFORM_S3_ENDPOINT_HOSTS }}
run: |
set -euo pipefail
@@ -309,7 +306,6 @@ jobs:
const secret = process.env[process.env.ENTRY === 'cloud' ? 'CLOUD_PLATFORM_SECRET' : 'SITE_PLATFORM_SECRET']
const values = secret ? { NUXT_PLATFORM_SECRET: secret } : {}
if (process.env.PLATFORM_STORAGE_SECRET) values.NUXT_PLATFORM_STORAGE_SECRET = process.env.PLATFORM_STORAGE_SECRET
- if (process.env.ENTRY === 'cloud' && process.env.PLATFORM_WORKER_SECRET) values.NUXT_PLATFORM_WORKER_SECRET = process.env.PLATFORM_WORKER_SECRET
if (process.env.ENTRY === 'cloud' && process.env.PLATFORM_DOMAIN_PROVIDER_TOKEN) values.NUXT_PLATFORM_DOMAIN_PROVIDER_TOKEN = process.env.PLATFORM_DOMAIN_PROVIDER_TOKEN
fs.writeFileSync(process.env.WORKER_SECRET_FILE, JSON.stringify(values), { mode: 0o600 })
NODE
diff --git a/.github/workflows/publish-processor.yml b/.github/workflows/publish-processor.yml
new file mode 100644
index 00000000..730a729c
--- /dev/null
+++ b/.github/workflows/publish-processor.yml
@@ -0,0 +1,69 @@
+name: Publish private processing node
+on:
+ workflow_dispatch:
+ push:
+ tags: ['processor-v*']
+permissions:
+ contents: read
+concurrency:
+ group: publish-private-processor
+ cancel-in-progress: false
+jobs:
+ publish:
+ runs-on: ubuntu-24.04
+ environment: processor-release
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - name: Verify registry credential and private visibility
+ id: registry
+ env:
+ GH_TOKEN: ${{ secrets.PROCESSOR_REGISTRY_TOKEN }}
+ run: |
+ set -euo pipefail
+ test -n "$GH_TOKEN" || { echo '::error::Configure PROCESSOR_REGISTRY_TOKEN with chronoframe organization package access'; exit 1; }
+ printf 'username=%s\n' "$(gh api user --jq .login)" >> "$GITHUB_OUTPUT"
+ status=$(gh api --include /orgs/chronoframe/packages/container/processor 2>/dev/null || true)
+ if [[ "$status" == *'"visibility"'* ]]; then
+ visibility=$(gh api /orgs/chronoframe/packages/container/processor --jq .visibility)
+ test "$visibility" = private || { echo '::error::Refusing to publish to a non-private package'; exit 1; }
+ elif [[ "$status" != *'404'* ]]; then
+ echo '::error::Unable to verify private registry access'
+ exit 1
+ fi
+ - uses: docker/setup-qemu-action@v3
+ - uses: docker/setup-buildx-action@v3
+ - uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ steps.registry.outputs.username }}
+ password: ${{ secrets.PROCESSOR_REGISTRY_TOKEN }}
+ - uses: docker/metadata-action@v5
+ id: metadata
+ with:
+ images: ghcr.io/chronoframe/processor
+ tags: |
+ type=sha,format=long
+ type=match,pattern=processor-v(.*),group=1
+ type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/processor-v') }}
+ - uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: apps/worker/Dockerfile
+ platforms: linux/amd64,linux/arm64
+ push: true
+ tags: ${{ steps.metadata.outputs.tags }}
+ labels: ${{ steps.metadata.outputs.labels }}
+ build-args: |
+ PROCESSOR_VERSION=${{ github.sha }}
+ cache-from: type=gha,scope=processor
+ cache-to: type=gha,mode=max,scope=processor
+ provenance: mode=min
+ - name: Confirm package remains private
+ env:
+ GH_TOKEN: ${{ secrets.PROCESSOR_REGISTRY_TOKEN }}
+ run: |
+ set -euo pipefail
+ test "$(gh api /orgs/chronoframe/packages/container/processor --jq .visibility)" = private
+ echo 'Published private image ghcr.io/chronoframe/processor' >> "$GITHUB_STEP_SUMMARY"
diff --git a/apps/cloud/app/components/CloudPhotoLibrary.vue b/apps/cloud/app/components/CloudPhotoLibrary.vue
index 7b0a3654..3d035266 100644
--- a/apps/cloud/app/components/CloudPhotoLibrary.vue
+++ b/apps/cloud/app/components/CloudPhotoLibrary.vue
@@ -11,7 +11,9 @@ import {
} from 'lucide-vue-next'
import { uploadQuotaGate } from '../utils/quota-gate'
import { DASHBOARD_PHOTO_COLUMN_IDS } from '#console/composables/useDashboardPhotoColumnVisibility'
-const { t } = useI18n()
+const { t, locale } = useI18n()
+const processingText = (en: string, zh: string) =>
+ locale.value === 'zh-Hans' ? zh : en
const dayjs = useDayjs()
const locationLabel = useLocationLabel()
@@ -181,6 +183,92 @@ async function addToAlbum() {
selection.value = {}
}
}
+const processingOpen = ref(false),
+ processingPhotos = ref<Photo[]>([]),
+ processingOperation = ref('reindex')
+function openProcessing(photos: Photo[]) {
+ processingPhotos.value = photos
+ processingOperation.value = 'reindex'
+ processingOpen.value = true
+}
+async function processPhotos() {
+ const ok = await action(
+ async () => {
+ for (
+ let offset = 0;
+ offset < processingPhotos.value.length;
+ offset += 100
+ )
+ await api(base.value + '/photos/process-batch', {
+ method: 'POST',
+ body: {
+ photoIds: processingPhotos.value
+ .slice(offset, offset + 100)
+ .map((p) => p.id),
+ operation: processingOperation.value,
+ },
+ })
+ },
+ processingText('Processing tasks queued.', '处理任务已排队。'),
+ )
+ if (ok) processingOpen.value = false
+}
+const liveOpen = ref(false),
+ livePhoto = ref<Photo>(),
+ liveFile = ref<File>(),
+ attaching = ref(false)
+let liveController: AbortController | undefined
+onBeforeUnmount(() => liveController?.abort())
+async function attachVideo() {
+ if (!livePhoto.value || !liveFile.value || attaching.value) return
+ attaching.value = true
+ liveController = new AbortController()
+ const photoId = livePhoto.value.id,
+ file = liveFile.value,
+ signal = liveController.signal
+ try {
+ const ok = await action(
+ async () => {
+ const latest = await api<{ revision: number }>(
+ base.value + '/photos/' + photoId,
+ )
+ const body = {
+ revision: latest.revision,
+ fileName: file.name,
+ bytes: file.size,
+ }
+ const reservation = await api<{ id: string; url: string }>(
+ base.value + '/photos/' + photoId + '/live-video',
+ { method: 'POST', body: { ...body, phase: 'prepare' } },
+ )
+ const response = await fetch(reservation.url, {
+ method: 'PUT',
+ body: file,
+ headers: { 'Content-Type': 'video/quicktime' },
+ credentials: 'omit',
+ redirect: 'error',
+ signal,
+ })
+ if (!response.ok) throw new Error('video_upload_failed')
+ await api(base.value + '/photos/' + photoId + '/live-video', {
+ method: 'POST',
+ body: {
+ phase: 'complete',
+ uploadId: reservation.id,
+ revision: latest.revision,
+ },
+ })
+ },
+ processingText('Live Photo processing queued.', '实况照片处理已排队。'),
+ )
+ if (ok) {
+ liveOpen.value = false
+ liveFile.value = undefined
+ }
+ } finally {
+ attaching.value = false
+ }
+}
function actions(photo: Photo) {
return [
[
@@ -191,6 +279,22 @@ function actions(photo: Photo) {
preview.value = photo
},
},
+ {
+ label: processingText('Attach Live Photo video', '补充实况照片视频'),
+ icon: 'i-lucide-video',
+ disabled: !admin.value,
+ onSelect: () => {
+ livePhoto.value = photo
+ liveFile.value = undefined
+ liveOpen.value = true
+ },
+ },
+ {
+ label: processingText('Process photo', '处理照片'),
+ icon: 'i-lucide-refresh-cw',
+ disabled: !admin.value,
+ onSelect: () => openProcessing([photo]),
+ },
{
label: t('cloud.messages.editDetails'),
icon: 'i-lucide-pencil',
@@ -328,6 +432,13 @@ watch(
@click="selection = {}"
>{{ $t('cloud.messages.clear') }}</CfButton
>
+ <CfButton
+ v-if="admin"
+ size="sm"
+ variant="ghost"
+ @click="openProcessing(chosen)"
+ >{{ processingText('Process selected', '处理所选照片') }}</CfButton
+ >
</ConsoleSelectionToolbar>
<CfTable
class="cf-library-table"
@@ -676,4 +787,97 @@ watch(
></UModal
>
</section>
+ <UModal
+ v-model:open="processingOpen"
+ :title="processingText('Process photos', '处理照片')"
+ ><template #body
+ ><div class="cf-form">
+ <p>
+ {{ processingText('Selected photographs', '已选择照片') }}:
+ {{ processingPhotos.length }}
+ </p>
+ <label class="cf-field"
+ >{{ processingText('Operation', '操作')
+ }}<CfSelect v-model="processingOperation"
+ ><option value="reindex">
+ {{
+ processingText(
+ 'Rebuild EXIF and thumbnails',
+ '重建 EXIF 与缩略图',
+ )
+ }}
+ </option>
+ <option value="geocoding">
+ {{ processingText('Refresh locations', '更新地理位置') }}
+ </option>
+ <option value="erase-location">
+ {{ processingText('Erase image locations', '擦除照片位置') }}
+ </option></CfSelect
+ ></label
+ >
+ <p class="cf-field-hint">
+ {{
+ processingOperation === 'erase-location'
+ ? processingText(
+ 'GPS is removed from metadata and downloadable image files. This cannot be undone after old originals are cleaned up. Companion videos retain their original metadata.',
+ '将从元数据和可下载原图中移除 GPS,旧原图清理后不可撤销。配对视频保留原有元数据。',
+ )
+ : processingText(
+ 'Published photos remain visible during processing. Existing tasks must finish before new processing can be queued.',
+ '处理期间已发布照片保持可见;已有任务需结束后才能重新排队。',
+ )
+ }}
+ </p>
+ </div></template
+ ><template #footer
+ ><CfButton
+ variant="outline"
+ :disabled="busy"
+ @click="processingOpen = false"
+ >{{ t('cloud.messages.cancel') }}</CfButton
+ ><CfButton
+ :disabled="busy"
+ @click="processPhotos"
+ >{{ processingText('Queue processing', '提交处理') }}</CfButton
+ ></template
+ ></UModal
+ >
+ <UModal
+ v-model:open="liveOpen"
+ :title="processingText('Attach Live Photo video', '补充实况照片视频')"
+ :dismissible="!attaching"
+ :close="!attaching"
+ ><template #body
+ ><div class="cf-form">
+ <p>{{ livePhoto?.title }}</p>
+ <label class="cf-field"
+ >{{ processingText('MOV companion video', 'MOV 配对视频')
+ }}<input
+ type="file"
+ accept=".mov,video/quicktime"
+ :disabled="attaching"
+ @change="liveFile = ($event.target as HTMLInputElement).files?.[0]"
+ /></label>
+ <p class="cf-field-hint">
+ {{
+ processingText(
+ 'Upload a companion video for this photo. Processing will publish the new pairing after verification.',
+ '为此照片上传配对视频,核实并完成处理后发布新配对。',
+ )
+ }}
+ </p>
+ </div></template
+ ><template #footer
+ ><CfButton
+ variant="outline"
+ :disabled="attaching"
+ @click="liveOpen = false"
+ >{{ t('cloud.messages.cancel') }}</CfButton
+ ><CfButton
+ :disabled="attaching || !liveFile"
+ @click="attachVideo"
+ >{{ processingText('Upload and process', '上传并处理') }}</CfButton
+ ></template
+ ></UModal
+ >
</template>
diff --git a/apps/cloud/app/components/CloudProcessingSettings.vue b/apps/cloud/app/components/CloudProcessingSettings.vue
new file mode 100644
index 00000000..0090343b
--- /dev/null
+++ b/apps/cloud/app/components/CloudProcessingSettings.vue
@@ -0,0 +1,116 @@
+<script setup lang="ts">
+const props = defineProps<{ siteId: number }>()
+const { locale } = useI18n(),
+ t = (en: string, zh: string) => (locale.value === 'zh-Hans' ? zh : en)
+const endpoint = `/api/sites/${props.siteId}/media/processing-settings`
+const { data, error, refresh } = await useFetch<{
+ eraseLocation: boolean | number
+ revision: number
+}>(endpoint)
+const erase = ref(!!data.value?.eraseLocation),
+ busy = ref(false),
+ message = ref('')
+async function save() {
+ if (!data.value || busy.value) return
+ busy.value = true
+ message.value = ''
+ try {
+ await $fetch(endpoint, {
+ method: 'POST',
+ body: { eraseLocation: erase.value, revision: data.value.revision },
+ })
+ await refresh()
+ message.value = t('Processing settings saved.', '处理设置已保存。')
+ } catch {
+ message.value = t(
+ 'Unable to save; reload and retry.',
+ '保存失败,请重新加载后重试。',
+ )
+ } finally {
+ busy.value = false
+ }
+}
+</script>
+<template>
+ <CfCard class="cf-site-settings"
+ ><div class="cf-panel-heading">
+ <h2>{{ t('Photo processing privacy', '照片处理隐私') }}</h2>
+ <p>
+ {{
+ t(
+ 'Choose whether new uploads retain their location. Erasure removes GPS from the downloadable image as well as its metadata. It does not apply retroactively.',
+ '选择新上传照片是否保留位置。擦除会同时移除可下载原图及元数据中的 GPS,不会自动修改已有照片。',
+ )
+ }}
+ </p>
+ </div>
+ <div class="cf-panel-body">
+ <form
+ class="cf-form"
+ @submit.prevent="save"
+ >
+ <label class="processing-toggle"
+ ><input
+ v-model="erase"
+ type="checkbox"
+ :disabled="busy || !!error"
+ /><span>{{
+ t(
+ 'Automatically erase location on new uploads',
+ '自动擦除新上传照片的位置',
+ )
+ }}</span></label
+ >
+ <p class="cf-field-hint">
+ {{
+ t(
+ 'Use the photo library processing actions to erase locations on existing photos. Live Photo companion videos are retained as uploaded, as in the self-hosted edition.',
+ '已有照片可在照片库的处理操作中擦除位置。与自部署版一致,实况照片的配对视频保持上传内容。',
+ )
+ }}
+ </p>
+ <p
+ v-if="error"
+ role="alert"
+ >
+ {{ t('Unable to load processing settings.', '无法加载处理设置。') }}
+ </p>
+ <p
+ v-if="message"
+ role="status"
+ >
+ {{ message }}
+ </p>
+ <div class="cf-site-actions">
+ <CfButton
+ type="submit"
+ :disabled="busy || !!error"
+ >{{ t('Save settings', '保存设置') }}</CfButton
+ ><CfButton
+ variant="outline"
+ :disabled="busy"
+ @click="
+ refresh().then(() => {
+ erase = !!data?.eraseLocation
+ })
+ "
+ >{{ t('Reload', '重新加载') }}</CfButton
+ >
+ </div>
+ </form>
+ </div></CfCard
+ >
+</template>
+<style scoped>
+.processing-toggle {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+}
+.processing-toggle input {
+ width: 16px;
+ height: 16px;
+ flex: none;
+ accent-color: var(--primary);
+}
+</style>
diff --git a/apps/cloud/app/components/CloudUploadDrawer.vue b/apps/cloud/app/components/CloudUploadDrawer.vue
index d88a58ca..94148881 100644
--- a/apps/cloud/app/components/CloudUploadDrawer.vue
+++ b/apps/cloud/app/components/CloudUploadDrawer.vue
@@ -121,7 +121,7 @@ async function dropFiles(event: DragEvent) {
class="sr-only"
type="file"
multiple
- accept="image/*,.heic,.heif,.mov"
+ accept="image/*,.heic,.heif,.hif,.bmp,.tif,.tiff,.mov"
:aria-label="$t('cloud.messages.choosePhotographs')"
:disabled="uploading || !uploadAllowed || !admin"
@change="upload"
diff --git a/apps/cloud/app/pages/sites/[siteId]/information.vue b/apps/cloud/app/pages/sites/[siteId]/information.vue
index ec1c0083..679b6682 100644
--- a/apps/cloud/app/pages/sites/[siteId]/information.vue
+++ b/apps/cloud/app/pages/sites/[siteId]/information.vue
@@ -21,6 +21,7 @@ const site = useEntryContext().value!
</div>
</div>
<CloudSiteSettings :site-id="site.siteId" />
+ <CloudProcessingSettings :site-id="site.siteId" />
<CloudMapSettings :site-id="site.siteId" />
</div>
</template>
diff --git a/apps/cloud/nuxt.config.ts b/apps/cloud/nuxt.config.ts
index 1393f274..35241298 100644
--- a/apps/cloud/nuxt.config.ts
+++ b/apps/cloud/nuxt.config.ts
@@ -69,7 +69,6 @@ export default defineNuxtConfig({
emailFrom: '',
storageSecret: '',
s3EndpointHosts: '',
- workerSecret: '',
},
},
css: ['~/assets/cloud.css'],
diff --git a/apps/cloud/server/api/internal/worker/[action].post.ts b/apps/cloud/server/api/internal/worker/[action].post.ts
index a9d16724..7ec401cd 100644
--- a/apps/cloud/server/api/internal/worker/[action].post.ts
+++ b/apps/cloud/server/api/internal/worker/[action].post.ts
@@ -1,3 +1,8 @@
+import {
+ authenticateProcessor,
+ processorEnvelope,
+} from '../../../../../../packages/platform/src/processors'
+import { parseProcessorLimits } from '@chronoframe/contracts/processor'
import {
workspaceEntitlements,
workspaceUsageSQL,
@@ -18,36 +23,54 @@ import { readSiteBody } from '../../../utils/site-body'
export default definePlatformHandler(async (event) => {
const config = useRuntimeConfig(event).platform
platformPortalOrigin(event, config.portalOrigin)
- if (
- config.workerSecret.length < 32 ||
- !getHeader(event, 'content-type')?.startsWith('application/json')
- )
- throw createError({ statusCode: 503 })
- const supplied =
- getHeader(event, 'authorization')?.replace(/^Bearer /, '') ?? ''
- if (supplied.length > 256) throw createError({ statusCode: 401 })
- const encoder = new TextEncoder(),
- key = await crypto.subtle.importKey(
- 'raw',
- encoder.encode(config.workerSecret),
- { name: 'HMAC', hash: 'SHA-256' },
- false,
- ['sign', 'verify'],
- )
- const proof = await crypto.subtle.sign(
- 'HMAC',
- key,
- encoder.encode(config.workerSecret),
- )
- if (
- !(await crypto.subtle.verify('HMAC', key, proof, encoder.encode(supplied)))
- )
- throw createError({ statusCode: 401 })
- const body = await readSiteBody(getRequestWebStream(event), 98304),
- action = getRouterParam(event, 'action'),
- now = Date.now()
+ if (!getHeader(event, 'content-type')?.startsWith('application/json'))
+ throw createError({ statusCode: 415 })
+ const token = getHeader(event, 'authorization')?.replace(/^Bearer /, '') ?? ''
const raw = platformDatabase(event),
db = raw as unknown as MediaDatabase
+ const node = await authenticateProcessor(raw, token)
+ if (!node) throw createError({ statusCode: 401 })
+ const body = await readSiteBody(
+ getRequestWebStream(event),
+ getRouterParam(event, 'action') === 'complete' ? 786432 : 98304,
+ ),
+ action = getRouterParam(event, 'action'),
+ now = Date.now()
+ const limits = parseProcessorLimits(JSON.parse(node.limits_json))
+ if (action === 'register' || action === 'status') {
+ const short = (v: unknown, max = 128) =>
+ typeof v === 'string'
+ ? Array.from(v)
+ .filter((c) => c.charCodeAt(0) >= 32 && c.charCodeAt(0) !== 127)
+ .join('')
+ .slice(0, max)
+ : ''
+ const metrics: Record<string, number> = {}
+ for (const k of ['cpuCount', 'load1', 'rss', 'totalMemory', 'freeMemory']) {
+ const v = (body.metrics as Record<string, unknown> | undefined)?.[k]
+ if (typeof v === 'number' && Number.isFinite(v) && v >= 0) metrics[k] = v
+ }
+ await raw
+ .prepare(
+ 'UPDATE platform_processor_nodes SET last_seen_at=?,version=?,instance_id=?,hostname=?,metrics_json=? WHERE id=? AND enabled=1',
+ )
+ .bind(
+ now,
+ short(body.version),
+ short(body.instanceId),
+ short(body.hostname),
+ JSON.stringify(metrics),
+ node.id,
+ )
+ .run()
+ return processorEnvelope(raw, config.storageSecret, token, node, now)
+ }
+ await raw
+ .prepare(
+ 'UPDATE platform_processor_nodes SET last_seen_at=? WHERE id=? AND enabled=1',
+ )
+ .bind(now, node.id)
+ .run()
const allowedHosts = config.s3EndpointHosts
.split(',')
.map((h: string) => h.trim())
@@ -69,33 +92,72 @@ export default definePlatformHandler(async (event) => {
},
)
}
- if (action === 'claim') {
+ if (action === 'geocoding-slot') {
if (
- typeof body.workerId !== 'string' ||
- !/^[a-zA-Z0-9_-]{1,128}$/.test(body.workerId)
+ typeof body.provider !== 'string' ||
+ !['mapbox', 'nominatim'].includes(body.provider)
)
throw createError({ statusCode: 400 })
+ const interval = body.provider === 'nominatim' ? 1000 : 100
+ const result = await raw
+ .prepare(
+ 'INSERT INTO platform_processor_geocoding_slots(provider,next_at) VALUES(?,?) ON CONFLICT(provider) DO UPDATE SET next_at=MAX(next_at,?)+? RETURNING next_at',
+ )
+ .bind(body.provider, now + interval, now, interval)
+ .first<{ next_at: number }>()
+ return { delayMs: Math.max(0, result!.next_at - interval - now) }
+ }
+ if (action === 'claim') {
+ const active = await raw
+ .prepare(
+ "SELECT id FROM cloud_processing_jobs WHERE worker_id=? AND state='running' AND lease_until>? LIMIT 1",
+ )
+ .bind(node.id, now)
+ .first()
+ if (active) return { lease: null }
const candidates = (
await db
.prepare(
- `SELECT j.site_id,MIN(j.created_at) oldest FROM cloud_processing_jobs j JOIN platform_sites s ON s.id=j.site_id AND s.suspended=0 LEFT JOIN cloud_site_dispatch d ON d.site_id=j.site_id WHERE ((state IN ('queued','retry_wait') AND available_at<=?) OR (state='running' AND lease_until<=?)) AND NOT EXISTS(SELECT 1 FROM cloud_processing_jobs active WHERE active.site_id=j.site_id AND active.state='running' AND active.lease_until>?) GROUP BY j.site_id ORDER BY COALESCE(d.last_claim_at,0),oldest,j.site_id LIMIT 10`,
+ `SELECT j.site_id,MIN(j.created_at) oldest FROM cloud_processing_jobs j JOIN platform_sites s ON s.id=j.site_id AND s.suspended=0 LEFT JOIN cloud_site_dispatch d ON d.site_id=j.site_id WHERE ((state IN ('queued','retry_wait') AND available_at<=?) OR (state='running' AND lease_until<=?)) AND j.required_pixels<=? AND j.required_file_bytes<=? AND j.required_metadata_bytes<=? AND EXISTS(SELECT 1 FROM cloud_uploads eligible WHERE eligible.site_id=j.site_id AND eligible.id=j.upload_id AND MAX(eligible.expected_bytes,eligible.expected_video_bytes)<=?) AND NOT EXISTS(SELECT 1 FROM cloud_processing_jobs active WHERE active.site_id=j.site_id AND active.state='running' AND active.lease_until>?) GROUP BY j.site_id ORDER BY COALESCE(d.last_claim_at,0),oldest,j.site_id LIMIT 10`,
+ )
+ .bind(
+ now,
+ now,
+ limits.maxPixels,
+ limits.maxFileBytes,
+ limits.maxMetadataBytes,
+ limits.maxFileBytes,
+ now,
)
- .bind(now, now, now)
.all<{ site_id: number }>()
).results
for (const candidate of candidates) {
- const lease = await createCloudExecutionRepository(db, {
- id: candidate.site_id,
- }).claim(body.workerId, crypto.randomUUID(), now, 120000)
+ const lease = await createCloudExecutionRepository(
+ db,
+ {
+ id: candidate.site_id,
+ },
+ limits,
+ ).claim(node.id, crypto.randomUUID(), now, 120000)
if (!lease) continue
// A deletion mark may race the candidate query. Do not issue fresh transport capabilities.
if (
- !(await db
+ !(await raw
.prepare('SELECT id FROM platform_sites WHERE id=? AND suspended=0')
.bind(lease.siteId)
.first())
)
continue
+ const sanitized = await raw
+ .prepare(
+ "SELECT object_key,bytes FROM cloud_media_assets WHERE site_id=? AND photo_id=? AND source_version=? AND role='sanitized-original' ORDER BY pipeline_version DESC LIMIT 1",
+ )
+ .bind(lease.siteId, lease.photoId, lease.sourceVersion)
+ .first<{ object_key: string; bytes: number }>()
+ if (sanitized) {
+ lease.objectKey = sanitized.object_key
+ lease.expectedBytes = sanitized.bytes
+ }
const s3 = await storage(
lease.siteId,
lease.connectionId,
@@ -114,12 +176,58 @@ export default definePlatformHandler(async (event) => {
video_etag: string | null
expected_video_bytes: number
}>()
+ const attachment = await raw
+ .prepare(
+ "SELECT u.object_key,u.expected_bytes,u.verified_etag FROM cloud_photo_companions c JOIN cloud_uploads u ON u.site_id=c.site_id AND u.id=c.upload_id WHERE c.site_id=? AND c.photo_id=? AND u.source_version=? AND u.state='completed'",
+ )
+ .bind(lease.siteId, lease.photoId, lease.sourceVersion)
+ .first<{
+ object_key: string
+ expected_bytes: number
+ verified_etag: string | null
+ }>()
+ if (attachment && detail) {
+ detail.video_key = attachment.object_key
+ detail.expected_video_bytes = attachment.expected_bytes
+ detail.video_etag = attachment.verified_etag
+ }
+ const options = await raw
+ .prepare(
+ 'SELECT operation,options_json FROM cloud_processing_jobs WHERE site_id=? AND id=?',
+ )
+ .bind(lease.siteId, lease.id)
+ .first<{ operation: string; options_json: string }>()
return {
lease,
sourceUrl: await s3.signGet(lease.objectKey),
fileName: detail?.file_name,
contentType: detail?.content_type,
- etag: detail?.verified_etag,
+ options,
+ priorMetadata: await raw
+ .prepare(
+ 'SELECT metadata_json,title FROM cloud_photos WHERE site_id=? AND id=?',
+ )
+ .bind(lease.siteId, lease.photoId)
+ .first(),
+ eraseLocation:
+ !!sanitized ||
+ (options?.operation === 'photo' &&
+ !!(
+ await raw
+ .prepare(
+ 'SELECT erase_location FROM platform_site_processing WHERE site_id=?',
+ )
+ .bind(lease.siteId)
+ .first<{ erase_location: number }>()
+ )?.erase_location),
+ configuration: await processorEnvelope(
+ raw,
+ config.storageSecret,
+ token,
+ node,
+ now,
+ ),
+ etag: sanitized ? undefined : detail?.verified_etag,
companion: detail?.video_key
? {
key: detail.video_key,
@@ -133,6 +241,67 @@ export default definePlatformHandler(async (event) => {
return { lease: null }
}
if (action === 'cleanup') {
+ // Retire overwritten derivatives and unsanitized originals only after a replacement is published.
+ const obsolete = (
+ await db
+ .prepare(
+ `SELECT a.site_id,a.photo_id,a.pipeline_version,a.role,a.connection_id,a.connection_revision,a.object_key FROM cloud_media_assets a WHERE EXISTS(SELECT 1 FROM cloud_media_assets newer WHERE newer.site_id=a.site_id AND newer.photo_id=a.photo_id AND newer.source_version=a.source_version AND newer.role=a.role AND newer.pipeline_version>a.pipeline_version) LIMIT 20`,
+ )
+ .all<any>()
+ ).results
+ for (const asset of obsolete)
+ await raw.batch([
+ raw
+ .prepare(
+ 'INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,available_at) VALUES(?,?,?,?,?,?) ON CONFLICT DO NOTHING',
+ )
+ .bind(
+ asset.site_id,
+ 'obsolete-' +
+ asset.photo_id +
+ '-' +
+ asset.pipeline_version +
+ '-' +
+ asset.role,
+ asset.connection_id,
+ asset.connection_revision,
+ asset.object_key,
+ now + 360000,
+ ),
+ raw
+ .prepare(
+ 'DELETE FROM cloud_media_assets WHERE site_id=? AND photo_id=? AND pipeline_version=? AND role=? AND object_key=?',
+ )
+ .bind(
+ asset.site_id,
+ asset.photo_id,
+ asset.pipeline_version,
+ asset.role,
+ asset.object_key,
+ ),
+ ])
+ const originals = (
+ await db
+ .prepare(
+ `SELECT u.site_id,u.id,u.connection_id,u.connection_revision,u.object_key FROM cloud_uploads u WHERE u.content_type NOT LIKE 'video/%' AND EXISTS(SELECT 1 FROM cloud_media_assets a WHERE a.site_id=u.site_id AND a.photo_id=u.photo_id AND a.source_version=u.source_version AND a.role='sanitized-original') AND NOT EXISTS(SELECT 1 FROM cloud_cleanup_requests c WHERE c.site_id=u.site_id AND c.id='erased-original-'||u.id) LIMIT 20`,
+ )
+ .all<any>()
+ ).results
+ for (const original of originals)
+ await raw
+ .prepare(
+ 'INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,available_at) VALUES(?,?,?,?,?,?) ON CONFLICT DO NOTHING',
+ )
+ .bind(
+ original.site_id,
+ 'erased-original-' + original.id,
+ original.connection_id,
+ original.connection_revision,
+ original.object_key,
+ now + 360000,
+ )
+ .run()
+
const expired = (
await db
.prepare(
@@ -206,6 +375,7 @@ export default definePlatformHandler(async (event) => {
).results
for (const old of abandoned)
for (const name of [
+ 'sanitized-original',
'thumbnail.webp',
'display.jpeg',
'live-video.mp4',
@@ -287,18 +457,66 @@ export default definePlatformHandler(async (event) => {
typeof lease.id !== 'string'
)
throw createError({ statusCode: 400 })
- const queue = createCloudExecutionRepository(db, { id: lease.siteId })
+ if (lease.workerId !== node.id) throw createError({ statusCode: 403 })
+ const jobPolicy = await raw
+ .prepare(
+ 'SELECT options_json FROM cloud_processing_jobs WHERE site_id=? AND id=?',
+ )
+ .bind(lease.siteId, lease.id)
+ .first<{ options_json: string }>()
+ const publicationRevision = Number(
+ JSON.parse(jobPolicy?.options_json ?? '{}').publicationRevision ?? 0,
+ )
+ const queue = createCloudExecutionRepository(
+ db,
+ { id: lease.siteId },
+ limits,
+ action === 'complete' ? publicationRevision : 0,
+ )
if (action === 'heartbeat')
return { ok: await queue.heartbeat(lease, now, 120000) }
- if (action === 'fail')
- return {
- ok: await queue.fail(
- lease,
- typeof body.code === 'string' ? body.code : 'processing_error',
- body.retryable === true,
- now,
- ),
+ if (action === 'fail') {
+ const code = typeof body.code === 'string' ? body.code : 'processing_error'
+ const ok = await queue.fail(lease, code, body.retryable === true, now)
+ if (
+ ok &&
+ ['node_pixel_limit', 'node_file_limit', 'node_metadata_limit'].includes(
+ code,
+ )
+ ) {
+ const pixels = Number(body.requiredPixels ?? 0),
+ bytes = Number(body.requiredBytes ?? 0),
+ metadata = Number(body.requiredMetadataBytes ?? 0)
+ if (
+ Number.isSafeInteger(pixels) &&
+ pixels >= 0 &&
+ Number.isSafeInteger(bytes) &&
+ bytes >= 0 &&
+ Number.isSafeInteger(metadata) &&
+ metadata >= 0 &&
+ (pixels > limits.maxPixels ||
+ bytes > limits.maxFileBytes ||
+ metadata > limits.maxMetadataBytes)
+ )
+ await raw
+ .prepare(
+ "UPDATE cloud_processing_jobs SET state='queued',attempts=MAX(0,attempts-1),required_pixels=MAX(required_pixels,?),required_file_bytes=MAX(required_file_bytes,?),required_metadata_bytes=MAX(required_metadata_bytes,?),available_at=? WHERE site_id=? AND id=? AND attempt_id=? AND worker_id=? AND state IN('retry_wait','failed') AND error_code=?",
+ )
+ .bind(
+ pixels,
+ bytes,
+ metadata,
+ now + 3000,
+ lease.siteId,
+ lease.id,
+ lease.attemptId,
+ node.id,
+ code,
+ )
+ .run()
}
+ return { ok }
+ }
if (action === 'grant') {
if (!(await queue.heartbeat(lease, now, 120000)))
throw createError({ statusCode: 409 })
@@ -325,7 +543,7 @@ export default definePlatformHandler(async (event) => {
throw createError({ statusCode: 403 })
if (
!Array.isArray(body.assets) ||
- body.assets.length > 3 ||
+ body.assets.length > 4 ||
body.assets.length < 1
)
throw createError({ statusCode: 400 })
@@ -340,7 +558,12 @@ export default definePlatformHandler(async (event) => {
contentType: string
}[]
if (
- declared.some((a) => !Number.isSafeInteger(a.bytes) || a.bytes < 1) ||
+ declared.some(
+ (a) =>
+ !Number.isSafeInteger(a.bytes) ||
+ a.bytes < 1 ||
+ a.bytes > limits.maxFileBytes,
+ ) ||
new Set(declared.map((a) => a.role)).size !== declared.length
)
throw createError({ statusCode: 400 })
@@ -391,6 +614,13 @@ export default definePlatformHandler(async (event) => {
name = 'thumbnail.webp'
else if (asset.role === 'display' && asset.contentType === 'image/jpeg')
name = 'display.jpeg'
+ else if (
+ asset.role === 'sanitized-original' &&
+ /^image\/(jpeg|png|webp|heic|heif|avif|gif|bmp|tiff)$/.test(
+ asset.contentType,
+ )
+ )
+ name = 'sanitized-original'
else if (asset.role === 'live-video' && asset.contentType === 'video/mp4')
name = 'live-video.mp4'
else if (
@@ -453,10 +683,11 @@ export default definePlatformHandler(async (event) => {
const prefix = `chronoframe/${lease.siteId}/derived/${lease.photoId}/${lease.sourceVersion}/${lease.pipelineVersion}/${lease.attemptId}/`
if (
result.assets.length < 1 ||
- result.assets.length > 3 ||
+ result.assets.length > 4 ||
result.assets.some(
(a) =>
![
+ 'sanitized-original',
'thumbnail.webp',
'display.jpeg',
'live-video.mp4',
@@ -475,7 +706,14 @@ export default definePlatformHandler(async (event) => {
if (meta.bytes !== asset.bytes || meta.contentType !== asset.contentType)
throw createError({ statusCode: 400 })
}
- return { ok: await queue.complete(lease, result, now) }
+ const ok = await queue.complete(lease, result, now)
+ if (!ok)
+ await createCloudExecutionRepository(
+ db,
+ { id: lease.siteId },
+ limits,
+ ).fail(lease, 'photo_changed', false, now)
+ return { ok }
}
throw createError({ statusCode: 404 })
})
diff --git a/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts b/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts
index bbf775e8..f49ad650 100644
--- a/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts
+++ b/apps/cloud/server/api/sites/[siteId]/media/[...operation].ts
@@ -1,3 +1,5 @@
+import { cloudLiveAttachment } from '../../../../utils/cloud-live-attachment'
+import { queuePhotoProcessing } from '../../../../utils/cloud-photo-processing'
import {
workspaceEntitlements,
workspaceUsageSQL,
@@ -24,6 +26,10 @@ const types: Record<string, string> = {
heif: 'image/heif',
avif: 'image/avif',
gif: 'image/gif',
+ bmp: 'image/bmp',
+ tif: 'image/tiff',
+ tiff: 'image/tiff',
+ hif: 'image/heif',
}
const text = (v: unknown, max = 128) => {
if (typeof v !== 'string' || !v.trim() || v.length > max)
@@ -177,6 +183,120 @@ export default definePlatformHandler(async (event) => {
throw createError({ statusCode: 409 })
return { ok: true }
}
+ if (
+ kind === 'photos' &&
+ id &&
+ action === 'live-video' &&
+ event.method === 'POST'
+ )
+ return cloudLiveAttachment(ctx, ID(id), body, now)
+ if (kind === 'photos' && id === 'process-batch' && event.method === 'POST') {
+ if (
+ !Array.isArray(body.photoIds) ||
+ !body.photoIds.length ||
+ body.photoIds.length > 100 ||
+ typeof body.operation !== 'string' ||
+ !['reindex', 'geocoding', 'erase-location'].includes(body.operation)
+ )
+ throw createError({ statusCode: 400 })
+ const ids = [...new Set(body.photoIds.map(ID))]
+ const rows = (
+ await db
+ .prepare(
+ `SELECT id,revision FROM cloud_photos WHERE site_id=? AND deleted=0 AND id IN (${ids.map(() => '?').join(',')})`,
+ )
+ .bind(siteId, ...ids)
+ .all<{ id: string; revision: number }>()
+ ).results
+ if (rows.length !== ids.length) throw createError({ statusCode: 404 })
+ const results = []
+ for (const row of rows) {
+ try {
+ await queuePhotoProcessing(
+ raw,
+ siteId,
+ row.id,
+ body.operation,
+ row.revision,
+ )
+ results.push({ id: row.id, queued: true })
+ } catch {
+ results.push({ id: row.id, queued: false })
+ }
+ }
+ if (results.some((r) => !r.queued))
+ throw createError({
+ statusCode: 409,
+ statusMessage:
+ 'Some photos already have processing tasks; refresh before retrying',
+ data: { results },
+ })
+ return { results }
+ }
+ if (kind === 'processing-settings' && !id) {
+ if (event.method === 'GET')
+ return (
+ (await raw
+ .prepare(
+ 'SELECT erase_location AS eraseLocation,revision FROM platform_site_processing WHERE site_id=?',
+ )
+ .bind(siteId)
+ .first()) ?? { eraseLocation: false, revision: 0 }
+ )
+ if (
+ event.method !== 'POST' ||
+ typeof body.eraseLocation !== 'boolean' ||
+ !Number.isSafeInteger(body.revision)
+ )
+ throw createError({ statusCode: 400 })
+ const revision = Number(body.revision)
+ const result =
+ revision === 0
+ ? await raw
+ .prepare(
+ 'INSERT INTO platform_site_processing(site_id,erase_location,revision) VALUES(?,?,1) ON CONFLICT DO NOTHING',
+ )
+ .bind(siteId, body.eraseLocation ? 1 : 0)
+ .run()
+ : await raw
+ .prepare(
+ 'UPDATE platform_site_processing SET erase_location=?,revision=revision+1 WHERE site_id=? AND revision=?',
+ )
+ .bind(body.eraseLocation ? 1 : 0, siteId, revision)
+ .run()
+ if (!result.meta.changes) throw createError({ statusCode: 409 })
+ return { ok: true }
+ }
+ if (
+ kind === 'photos' &&
+ id &&
+ action === 'process' &&
+ event.method === 'POST'
+ ) {
+ const options: Record<string, unknown> = {}
+ if (body.latitude !== undefined || body.longitude !== undefined) {
+ if (
+ typeof body.latitude !== 'number' ||
+ typeof body.longitude !== 'number' ||
+ !Number.isFinite(body.latitude) ||
+ !Number.isFinite(body.longitude) ||
+ Math.abs(body.latitude) > 90 ||
+ Math.abs(body.longitude) > 180 ||
+ (body.latitude === 0 && body.longitude === 0)
+ )
+ throw createError({ statusCode: 400 })
+ options.latitude = body.latitude
+ options.longitude = body.longitude
+ }
+ return queuePhotoProcessing(
+ raw,
+ siteId,
+ ID(id),
+ String(body.operation),
+ number(body.revision),
+ options,
+ )
+ }
if (kind === 'usage' && event.method === 'GET') {
const plan = await workspaceEntitlements(raw, site.workspaceId, 'platform')
const expression = workspaceUsageSQL(siteId)
@@ -420,6 +540,8 @@ export default definePlatformHandler(async (event) => {
return { ok: true }
}
if (event.method === 'POST' && action === 'complete') {
+ if (detail.content_type.startsWith('video/'))
+ throw createError({ statusCode: 400 })
if (upload.state === 'completed') return { jobId: id }
if (upload.state !== 'pending' || upload.expires_at <= now)
throw createError({ statusCode: 409 })
diff --git a/apps/cloud/server/utils/cloud-live-attachment.ts b/apps/cloud/server/utils/cloud-live-attachment.ts
new file mode 100644
index 00000000..dd9ea8c3
--- /dev/null
+++ b/apps/cloud/server/utils/cloud-live-attachment.ts
@@ -0,0 +1,150 @@
+import { createError } from 'h3'
+import type { cloudMedia } from './cloud-media'
+import {
+ workspaceEntitlements,
+ workspaceUsageSQL,
+} from '@chronoframe/platform/entitlements'
+export async function cloudLiveAttachment(
+ ctx: Awaited<ReturnType<typeof cloudMedia>>,
+ photoId: string,
+ body: Record<string, any>,
+ now: number,
+) {
+ const { raw, siteId, userId, site } = ctx
+ if (body.phase === 'complete' && typeof body.uploadId === 'string') {
+ const done = await raw
+ .prepare(
+ "SELECT c.upload_id FROM cloud_photo_companions c JOIN cloud_uploads u ON u.site_id=c.site_id AND u.id=c.upload_id JOIN cloud_photos p ON p.site_id=c.site_id AND p.id=c.photo_id WHERE c.site_id=? AND c.photo_id=? AND c.upload_id=? AND u.owner_id=? AND u.state='completed' AND p.deleted=0 AND p.source_version=u.source_version",
+ )
+ .bind(siteId, photoId, body.uploadId, userId)
+ .first()
+ if (done) return { ok: true }
+ }
+ const source = await raw
+ .prepare(
+ "SELECT u.*,p.revision AS photo_revision FROM cloud_uploads u JOIN cloud_processing_jobs j ON j.site_id=u.site_id AND j.upload_id=u.id JOIN cloud_photos p ON p.site_id=j.site_id AND p.id=j.photo_id AND p.source_version=j.source_version WHERE p.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND j.state IN('succeeded','failed','cancelled')",
+ )
+ .bind(siteId, photoId)
+ .first<any>()
+ if (!source || source.photo_revision !== body.revision)
+ throw createError({ statusCode: 409 })
+ const { s3 } = await ctx.storage(
+ source.connection_id,
+ source.connection_revision,
+ )
+ const stored = await ctx.vault.resolve(
+ source.connection_id,
+ source.connection_revision,
+ )
+ if (!stored) throw createError({ statusCode: 404 })
+ const managed = 'poolId' in stored
+ const plan = await workspaceEntitlements(
+ raw,
+ site.workspaceId,
+ managed ? 'platform' : 'byos',
+ )
+ if (!managed && !plan.limits.byos) throw createError({ statusCode: 403 })
+ if (body.phase === 'prepare') {
+ if (
+ typeof body.fileName !== 'string' ||
+ !body.fileName.toLowerCase().endsWith('.mov') ||
+ body.fileName.length > 255 ||
+ !Number.isSafeInteger(body.bytes) ||
+ body.bytes < 1 ||
+ body.bytes > plan.limits.fileBytes ||
+ body.bytes > 5368709120
+ )
+ throw createError({ statusCode: 400 })
+ const id = crypto.randomUUID(),
+ key = `chronoframe/${siteId}/live-attachments/${id}/source.MOV`,
+ usage = workspaceUsageSQL(siteId)
+ const result = await raw
+ .prepare(
+ `INSERT INTO cloud_uploads(site_id,id,owner_id,photo_id,source_version,connection_id,connection_revision,object_key,expected_bytes,expires_at,content_type,file_name,reserved_storage_bytes) SELECT ?,?,?,?,?,?,?,?,?,?,'video/quicktime',?,? WHERE ${usage.monthly}+?<=? AND (?=0 OR ${usage.storage}+?<=?)`,
+ )
+ .bind(
+ siteId,
+ id,
+ userId,
+ photoId,
+ source.source_version,
+ source.connection_id,
+ source.connection_revision,
+ key,
+ body.bytes,
+ now + 3600000,
+ body.fileName,
+ managed ? body.bytes : 0,
+ plan.quotaPeriod.start,
+ now,
+ body.bytes,
+ plan.limits.monthlyBytes,
+ managed ? 1 : 0,
+ now,
+ body.bytes,
+ plan.limits.storageBytes,
+ )
+ .run()
+ if (!result.meta.changes) throw createError({ statusCode: 429 })
+ return { id, url: await s3.signPut(key, body.bytes, 'video/quicktime') }
+ }
+ if (body.phase !== 'complete' || typeof body.uploadId !== 'string')
+ throw createError({ statusCode: 400 })
+ const upload = await raw
+ .prepare(
+ "SELECT * FROM cloud_uploads WHERE site_id=? AND id=? AND photo_id=? AND owner_id=? AND source_version=? AND content_type='video/quicktime'",
+ )
+ .bind(siteId, body.uploadId, photoId, userId, source.source_version)
+ .first<any>()
+ if (
+ !upload ||
+ upload.state !== 'pending' ||
+ upload.expires_at <= now ||
+ upload.connection_id !== source.connection_id ||
+ upload.connection_revision !== source.connection_revision
+ )
+ throw createError({ statusCode: 409 })
+ const head = await s3.head(upload.object_key)
+ if (
+ head.bytes !== upload.expected_bytes ||
+ head.contentType !== 'video/quicktime'
+ )
+ throw createError({ statusCode: 400 })
+ const pipeline = `p_${String(Number(body.revision) + 1).padStart(16, '0')}_${crypto.randomUUID()}`
+ const result = await raw.batch([
+ raw
+ .prepare(
+ "UPDATE cloud_photos SET revision=revision+1 WHERE site_id=? AND id=? AND revision=? AND deleted=0 AND NOT EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=cloud_photos.site_id AND j.photo_id=cloud_photos.id AND j.state IN('queued','running','retry_wait'))",
+ )
+ .bind(siteId, photoId, body.revision),
+ raw
+ .prepare(
+ "UPDATE cloud_uploads SET state='completed',verified_etag=? WHERE site_id=? AND id=? AND state='pending' AND expires_at>? AND changes()=1",
+ )
+ .bind(head.etag, siteId, upload.id, now),
+ raw
+ .prepare(
+ 'INSERT INTO cloud_usage_events(site_id,upload_id,bytes,created_at) SELECT ?,?,?,? WHERE changes()=1',
+ )
+ .bind(siteId, upload.id, upload.expected_bytes, now),
+ raw
+ .prepare(
+ 'INSERT INTO cloud_photo_companions(site_id,photo_id,upload_id) SELECT ?,?,? WHERE changes()=1 ON CONFLICT(site_id,photo_id) DO UPDATE SET upload_id=excluded.upload_id',
+ )
+ .bind(siteId, photoId, upload.id),
+ raw
+ .prepare(
+ "UPDATE cloud_processing_jobs SET state='queued',operation='live-photo',options_json=?,pipeline_version=?,attempts=0,error_code=NULL,attempt_id=NULL,worker_id=NULL,lease_until=NULL,available_at=?,reserved_output_bytes=0 WHERE site_id=? AND photo_id=? AND source_version=? AND state IN('succeeded','failed','cancelled') AND changes()=1",
+ )
+ .bind(
+ JSON.stringify({ publicationRevision: body.revision + 1 }),
+ pipeline,
+ now,
+ siteId,
+ photoId,
+ source.source_version,
+ ),
+ ])
+ if (!result[4]?.meta.changes) throw createError({ statusCode: 409 })
+ return { ok: true }
+}
diff --git a/apps/cloud/server/utils/cloud-photo-delete.ts b/apps/cloud/server/utils/cloud-photo-delete.ts
index 9d488c99..380d6e1c 100644
--- a/apps/cloud/server/utils/cloud-photo-delete.ts
+++ b/apps/cloud/server/utils/cloud-photo-delete.ts
@@ -49,7 +49,7 @@ export async function deleteCloudPhotos(
.bind(now + 360000, siteId, ids),
db
.prepare(
- `INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,provider_upload_id,available_at) SELECT site_id,'asset-'||photo_id||'-'||role,connection_id,connection_revision,object_key,NULL,? FROM cloud_media_assets WHERE site_id=? AND photo_id ${selection} ON CONFLICT DO NOTHING`,
+ `INSERT INTO cloud_cleanup_requests(site_id,id,connection_id,connection_revision,object_key,provider_upload_id,available_at) SELECT site_id,'asset-'||photo_id||'-'||source_version||'-'||pipeline_version||'-'||role,connection_id,connection_revision,object_key,NULL,? FROM cloud_media_assets WHERE site_id=? AND photo_id ${selection} ON CONFLICT DO NOTHING`,
)
.bind(now + 360000, siteId, ids),
])
diff --git a/apps/cloud/server/utils/cloud-photo-processing.ts b/apps/cloud/server/utils/cloud-photo-processing.ts
new file mode 100644
index 00000000..b0be66f5
--- /dev/null
+++ b/apps/cloud/server/utils/cloud-photo-processing.ts
@@ -0,0 +1,48 @@
+import { createError } from 'h3'
+import type { PlatformD1 } from '../../../../packages/platform/src/index'
+/** Reuse the immutable source and attempt history; old published assets stay visible while rebuilding. */
+export async function queuePhotoProcessing(
+ db: PlatformD1,
+ siteId: number,
+ photoId: string,
+ operation: string,
+ revision: number,
+ options: Record<string, unknown> = {},
+) {
+ if (
+ !['reindex', 'geocoding', 'erase-location', 'live-photo', 'photo'].includes(
+ operation,
+ ) ||
+ !Number.isSafeInteger(revision)
+ )
+ throw createError({ statusCode: 400 })
+ const now = Date.now(),
+ pipeline = `p_${String(revision + 1).padStart(16, '0')}_${crypto.randomUUID()}`
+ const results = await db.batch([
+ db
+ .prepare(
+ "UPDATE cloud_photos SET revision=revision+1 WHERE site_id=? AND id=? AND deleted=0 AND revision=? AND metadata_json IS NOT NULL AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=cloud_photos.site_id AND j.photo_id=cloud_photos.id AND j.source_version=cloud_photos.source_version AND j.state IN('succeeded','failed','cancelled')) AND NOT EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=cloud_photos.site_id AND j.photo_id=cloud_photos.id AND j.state IN('queued','running','retry_wait'))",
+ )
+ .bind(siteId, photoId, revision),
+ db
+ .prepare(
+ "UPDATE cloud_processing_jobs SET state='queued',operation=?,options_json=?,pipeline_version=?,attempts=0,error_code=NULL,attempt_id=NULL,worker_id=NULL,lease_until=NULL,available_at=?,reserved_output_bytes=0 WHERE site_id=? AND photo_id=? AND source_version=(SELECT source_version FROM cloud_photos WHERE site_id=? AND id=?) AND changes()=1",
+ )
+ .bind(
+ operation,
+ JSON.stringify({ ...options, publicationRevision: revision + 1 }),
+ pipeline,
+ now,
+ siteId,
+ photoId,
+ siteId,
+ photoId,
+ ),
+ ])
+ if (!results[1]?.meta.changes)
+ throw createError({
+ statusCode: 409,
+ statusMessage: 'Photo changed or processing already queued',
+ })
+ return { ok: true, revision: revision + 1 }
+}
diff --git a/apps/shared/platform/json.ts b/apps/shared/platform/json.ts
index 2fd4ce96..7c3923db 100644
--- a/apps/shared/platform/json.ts
+++ b/apps/shared/platform/json.ts
@@ -5,7 +5,7 @@ export async function readSiteBody(
stream: ReadableStream<Uint8Array> | null | undefined,
limit = 4096,
): Promise<Record<string, unknown>> {
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > 131072)
+ if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1048576)
throw createError({ statusCode: 400 })
if (!stream) throw createError({ statusCode: 400 })
const reader = stream.getReader()
diff --git a/apps/shared/platform/media.ts b/apps/shared/platform/media.ts
index d3f1c880..36707e23 100644
--- a/apps/shared/platform/media.ts
+++ b/apps/shared/platform/media.ts
@@ -68,7 +68,7 @@ export async function privateMedia(
const raw = platformDatabase(event)
const source = await raw
.prepare(
- `SELECT u.connection_id,u.connection_revision,u.object_key,u.expected_bytes,u.content_type,u.verified_etag FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id AND p.source_version=u.source_version WHERE u.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND u.state='completed' AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=p.site_id AND j.photo_id=p.id AND j.source_version=p.source_version AND j.state='succeeded') LIMIT 1`,
+ `SELECT u.connection_id,u.connection_revision,u.object_key,u.expected_bytes,u.content_type,u.verified_etag FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id AND p.source_version=u.source_version WHERE u.site_id=? AND p.id=? AND p.deleted=0 AND p.metadata_json IS NOT NULL AND u.state='completed' AND u.content_type NOT LIKE 'video/%' AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=p.site_id AND j.photo_id=p.id AND j.source_version=p.source_version AND j.state='succeeded') LIMIT 1`,
)
.bind(siteId, photoId)
.first<{
@@ -87,10 +87,17 @@ export async function privateMedia(
bytes: source.expected_bytes,
content_type: source.content_type,
}
+ const sanitized = await raw
+ .prepare(
+ "SELECT connection_id,connection_revision,object_key,bytes,content_type FROM cloud_media_assets WHERE site_id=? AND photo_id=? AND source_version=(SELECT source_version FROM cloud_photos WHERE site_id=? AND id=?) AND role='sanitized-original' ORDER BY pipeline_version DESC LIMIT 1",
+ )
+ .bind(siteId, photoId, siteId, photoId)
+ .first<typeof object>()
+ if (sanitized) object = sanitized
if (role !== 'original') {
const asset = await raw
.prepare(
- `SELECT connection_id,connection_revision,object_key,bytes,content_type FROM cloud_media_assets a WHERE site_id=? AND photo_id=? AND role=? AND source_version=(SELECT source_version FROM cloud_photos WHERE site_id=? AND id=?) LIMIT 1`,
+ `SELECT connection_id,connection_revision,object_key,bytes,content_type FROM cloud_media_assets a WHERE site_id=? AND photo_id=? AND role=? AND source_version=(SELECT source_version FROM cloud_photos WHERE site_id=? AND id=?) ORDER BY pipeline_version DESC LIMIT 1`,
)
.bind(siteId, photoId, role, siteId, photoId)
.first<typeof object>()
diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile
index bafd733a..833b3292 100644
--- a/apps/worker/Dockerfile
+++ b/apps/worker/Dockerfile
@@ -1,15 +1,24 @@
# syntax=docker/dockerfile:1
-FROM node:22.23.3-alpine
+FROM node:22.23.3-alpine AS base
RUN --mount=type=secret,id=proxy_ca --mount=type=secret,id=ca_bundle \
if [ -f /run/secrets/ca_bundle ]; then export SSL_CERT_FILE=/run/secrets/ca_bundle; fi; \
if [ -f /run/secrets/proxy_ca ]; then export NODE_EXTRA_CA_CERTS=/run/secrets/proxy_ca; fi; \
apk add --no-cache ca-certificates perl exiftool && corepack enable
WORKDIR /app
+FROM base AS dependencies
COPY . .
RUN --mount=type=secret,id=proxy_ca \
if [ -f /run/secrets/proxy_ca ]; then export NODE_EXTRA_CA_CERTS=/run/secrets/proxy_ca; fi; \
NODE_USE_ENV_PROXY=1 pnpm install --frozen-lockfile --prod --ignore-scripts --filter @chronoframe/worker... \
&& chown -R node:node /app
+FROM base
+WORKDIR /app
+COPY --from=dependencies --chown=node:node /app/node_modules ./node_modules
+COPY --from=dependencies --chown=node:node /app/packages ./packages
+COPY --from=dependencies --chown=node:node /app/apps/worker ./apps/worker
+COPY --from=dependencies --chown=node:node /app/package.json ./package.json
+ARG PROCESSOR_VERSION=development
+ENV CHRONOFRAME_PROCESSOR_VERSION=$PROCESSOR_VERSION
USER node
ENV NODE_ENV=production
CMD ["node", "--experimental-strip-types", "apps/worker/src/index.ts"]
diff --git a/apps/worker/package.json b/apps/worker/package.json
index 03515e1c..a2f7cafe 100644
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -11,6 +11,7 @@
"dependencies": {
"@chronoframe/contracts": "workspace:*",
"@chronoframe/core": "workspace:*",
+ "@chronoframe/geocoding": "workspace:*",
"@chronoframe/media": "workspace:*",
"@chronoframe/processing": "workspace:*",
"@chronoframe/storage": "workspace:*",
diff --git a/apps/worker/src/gateway.ts b/apps/worker/src/gateway.ts
index 305579cf..e85acf2a 100644
--- a/apps/worker/src/gateway.ts
+++ b/apps/worker/src/gateway.ts
@@ -1,4 +1,7 @@
type WorkerAction =
+ | 'register'
+ | 'status'
+ | 'geocoding-slot'
| 'claim'
| 'heartbeat'
| 'grant'
@@ -62,9 +65,17 @@ export function createWorkerGateway(
signal?: AbortSignal,
): Promise<T> {
if (
- !['claim', 'heartbeat', 'grant', 'complete', 'fail', 'cleanup'].includes(
- action,
- )
+ ![
+ 'register',
+ 'status',
+ 'geocoding-slot',
+ 'claim',
+ 'heartbeat',
+ 'grant',
+ 'complete',
+ 'fail',
+ 'cleanup',
+ ].includes(action)
)
throw new Error('Invalid worker gateway action')
const response = await transport(
diff --git a/apps/worker/src/geocoding.ts b/apps/worker/src/geocoding.ts
new file mode 100644
index 00000000..eda6fb6d
--- /dev/null
+++ b/apps/worker/src/geocoding.ts
@@ -0,0 +1,46 @@
+import {
+ createGeocodingServices,
+ createGeocodingRateLimiter,
+} from '@chronoframe/geocoding'
+import { withRetry } from '@chronoframe/media'
+import type { ProcessorGeocoding } from '@chronoframe/contracts/processor'
+export function processingGeocoder(
+ config: ProcessorGeocoding,
+ signal: AbortSignal,
+ reserve?: (intervalMs: number) => Promise<void>,
+) {
+ const logger = { info() {}, warn() {}, error() {}, success() {}, debug() {} }
+ const localSlot = createGeocodingRateLimiter()
+ const services = createGeocodingServices({
+ getLanguage: async () => config.language,
+ request: (url, init) =>
+ fetch(url, {
+ ...init,
+ redirect: 'error',
+ signal: init?.signal ? AbortSignal.any([signal, init.signal]) : signal,
+ }),
+ waitForGeocodingSlot: (key, interval, sig) =>
+ reserve ? reserve(interval) : localSlot(key, interval, sig),
+ withRetry,
+ retryOptions: { maxAttempts: 3, timeout: 10000 },
+ logger,
+ })
+ const provider =
+ config.provider === 'mapbox'
+ ? new services.MapboxGeocodingProvider(config.token)
+ : config.provider === 'nominatim'
+ ? new services.NominatimGeocodingProvider(config.endpoint)
+ : null
+ return {
+ parseGPS: services.parseGPSCoordinates,
+ reverseGeocode: async (lat: number, lon: number) => {
+ signal.throwIfAborted()
+ if (!provider) return null
+ const result = await provider.reverseGeocode(lat, lon)
+ signal.throwIfAborted()
+ // Explicitly configured providers must succeed; do not silently publish incomplete locations.
+ if (!result) throw new Error('reverse_geocoding_unavailable')
+ return result
+ },
+ }
+}
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index d5de173d..d0dac377 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -1,12 +1,64 @@
+import os from 'node:os'
+import {
+ processorCipher,
+ parseProcessorLimits,
+ parseProcessorGeocoding,
+ type ProcessorConfiguration,
+} from '@chronoframe/contracts/processor'
import { randomUUID } from 'node:crypto'
import { createWorkerGateway } from './gateway.ts'
import { setTimeout as delay } from 'node:timers/promises'
import { processWork, WorkerProcessingError, type WorkItem } from './process.ts'
import type { CloudMediaResult } from '@chronoframe/core'
-const workerId = process.env.CHRONOFRAME_WORKER_ID ?? randomUUID()
+const instanceId = randomUUID()
+const connectionKey = process.env.CHRONOFRAME_PROCESSOR_KEY ?? ''
+const version = process.env.CHRONOFRAME_PROCESSOR_VERSION ?? 'development'
+let configuration!: ProcessorConfiguration
+let lastStatus = 0
+async function acceptConfiguration(value: {
+ nodeId: string
+ revision: string
+ expiresAt: number
+ envelope: string
+}) {
+ if (
+ !value ||
+ value.nodeId !== connectionKey.split('.')[0]?.slice(3) ||
+ value.expiresAt <= Date.now() ||
+ value.expiresAt > Date.now() + 360000
+ )
+ throw new Error('invalid_configuration')
+ const decoded = await (
+ await processorCipher(connectionKey)
+ ).open<ProcessorConfiguration>(
+ value.envelope,
+ `${value.nodeId}:${value.revision}:${value.expiresAt}`,
+ )
+ configuration = {
+ limits: parseProcessorLimits(decoded.limits),
+ geocoding: parseProcessorGeocoding(decoded.geocoding),
+ }
+}
+async function reportStatus(register = false) {
+ if (!register && Date.now() - lastStatus < 30000) return
+ const result = await rpc<any>(register ? 'register' : 'status', {
+ instanceId,
+ version,
+ hostname: os.hostname(),
+ metrics: {
+ cpuCount: os.cpus().length,
+ load1: os.loadavg()[0],
+ rss: process.memoryUsage().rss,
+ totalMemory: os.totalmem(),
+ freeMemory: os.freemem(),
+ },
+ })
+ await acceptConfiguration(result)
+ lastStatus = Date.now()
+}
const rpc = createWorkerGateway({
origin: process.env.CHRONOFRAME_PORTAL_ORIGIN ?? '',
- secret: process.env.CHRONOFRAME_WORKER_SECRET ?? '',
+ secret: connectionKey,
accessClientId: process.env.CHRONOFRAME_ACCESS_CLIENT_ID,
accessClientSecret: process.env.CHRONOFRAME_ACCESS_CLIENT_SECRET,
})
@@ -19,20 +71,30 @@ for (const signal of ['SIGINT', 'SIGTERM'] as const)
})
while (!stopping) {
try {
- const item = await rpc<WorkItem | { lease: null }>('claim', { workerId })
+ await reportStatus(!configuration)
+ const item = await rpc<
+ (WorkItem & { configuration: any }) | { lease: null }
+ >('claim', {})
if (!item.lease) {
await rpc('cleanup', {})
await delay(3000, undefined, { signal: shutdown.signal }).catch(() => {})
continue
}
+ await acceptConfiguration(
+ (item as WorkItem & { configuration: any }).configuration,
+ )
const work = item as WorkItem,
controller = new AbortController(),
- timeout = setTimeout(() => controller.abort('processing_timeout'), 180000)
+ timeout = setTimeout(
+ () => controller.abort('processing_timeout'),
+ configuration.limits.timeoutMs,
+ )
let heartbeatRunning = false
const timer = setInterval(() => {
if (heartbeatRunning) return
heartbeatRunning = true
- void rpc<{ ok: boolean }>('heartbeat', { lease: work.lease })
+ void reportStatus()
+ .then(() => rpc<{ ok: boolean }>('heartbeat', { lease: work.lease }))
.then((result) => {
if (!result.ok) controller.abort('lease_lost')
})
@@ -42,7 +104,21 @@ while (!stopping) {
})
}, 30000)
try {
- const processed = await processWork(work, controller.signal)
+ const workConfiguration = configuration
+ const processed = await processWork(
+ work,
+ controller.signal,
+ workConfiguration,
+ async () => {
+ const slot = await rpc<{ delayMs: number }>(
+ 'geocoding-slot',
+ { provider: workConfiguration.geocoding.provider },
+ controller.signal,
+ )
+ if (slot.delayMs > 0)
+ await delay(slot.delayMs, undefined, { signal: controller.signal })
+ },
+ )
controller.signal.throwIfAborted()
const grants = await rpc<{
outputs: (CloudMediaResult['assets'][number] & { url: string })[]
@@ -91,6 +167,18 @@ while (!stopping) {
: error instanceof WorkerProcessingError
? error.message
: 'processing_error',
+ requiredPixels:
+ error instanceof WorkerProcessingError
+ ? error.requiredPixels
+ : undefined,
+ requiredMetadataBytes:
+ error instanceof WorkerProcessingError
+ ? error.requiredMetadataBytes
+ : undefined,
+ requiredBytes:
+ error instanceof WorkerProcessingError
+ ? error.requiredBytes
+ : undefined,
retryable: aborted
? controller.signal.reason === 'processing_timeout'
: error instanceof WorkerProcessingError
diff --git a/apps/worker/src/process.ts b/apps/worker/src/process.ts
index f47b37d0..b2ed153a 100644
--- a/apps/worker/src/process.ts
+++ b/apps/worker/src/process.ts
@@ -1,9 +1,18 @@
+import {
+ DEFAULT_PROCESSOR_LIMITS,
+ DEFAULT_PROCESSOR_GEOCODING,
+ type ProcessorConfiguration,
+} from '@chronoframe/contracts/processor'
+import { processingGeocoder } from './geocoding.ts'
import { mkdtemp, rm, readFile, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import sharp from 'sharp'
import { ExifTool } from 'exiftool-vendored'
-import { createPhotoProcessor } from '@chronoframe/processing'
+import {
+ createPhotoProcessor,
+ EXIF_LOCATION_KEYS,
+} from '@chronoframe/processing'
import { createExifService } from '@chronoframe/media'
import type { StorageProvider } from '@chronoframe/storage'
import type { CloudLease, CloudMediaResult } from '@chronoframe/core'
@@ -14,18 +23,33 @@ export interface WorkItem {
contentType: string
companion?: { key: string; url: string; bytes: number; etag?: string } | null
etag?: string
+ eraseLocation?: boolean
+ options?: { operation: string; options_json: string }
+ priorMetadata?: { metadata_json: string | null; title: string }
}
export interface Derivative {
- role: 'thumbnail' | 'live-video' | 'display'
+ role: 'thumbnail' | 'live-video' | 'display' | 'sanitized-original'
bytes: number
contentType: string
buffer: Buffer
}
export class WorkerProcessingError extends Error {
readonly retryable: boolean
- constructor(code: string, retryable: boolean) {
+ readonly requiredPixels?: number
+ readonly requiredBytes?: number
+ readonly requiredMetadataBytes?: number
+ constructor(
+ code: string,
+ retryable: boolean,
+ requiredPixels?: number,
+ requiredBytes?: number,
+ requiredMetadataBytes?: number,
+ ) {
super(code)
this.retryable = retryable
+ this.requiredPixels = requiredPixels
+ this.requiredBytes = requiredBytes
+ this.requiredMetadataBytes = requiredMetadataBytes
}
}
const quiet = { info() {}, warn() {}, error() {}, debug() {}, success() {} }
@@ -35,9 +59,18 @@ export async function boundedDownload(
expected: number,
signal: AbortSignal,
etag?: string,
+ maxBytes = DEFAULT_PROCESSOR_LIMITS.maxFileBytes,
+ sourceModified?: (value: string | null) => void,
) {
- if (!Number.isSafeInteger(expected) || expected < 1 || expected > 1073741824)
+ if (!Number.isSafeInteger(expected) || expected < 1)
throw new WorkerProcessingError('invalid_source_size', false)
+ if (expected > maxBytes)
+ throw new WorkerProcessingError(
+ 'node_file_limit',
+ true,
+ undefined,
+ expected,
+ )
const parsed = new URL(url)
if (parsed.protocol !== 'https:')
throw new WorkerProcessingError('invalid_source_url', false)
@@ -55,6 +88,7 @@ export async function boundedDownload(
: 'storage_unavailable',
response.status >= 500 || response.status === 429,
)
+ sourceModified?.(response.headers.get('last-modified'))
const reader = response.body.getReader(),
parts: Uint8Array[] = []
let size = 0
@@ -82,15 +116,27 @@ export async function boundedDownload(
export async function processWork(
item: WorkItem,
signal: AbortSignal,
+ configuration: ProcessorConfiguration = {
+ limits: DEFAULT_PROCESSOR_LIMITS,
+ geocoding: DEFAULT_PROCESSOR_GEOCODING,
+ },
+ reserveGeocoding?: (intervalMs: number) => Promise<void>,
): Promise<{
result: Omit<CloudMediaResult, 'assets'>
derivatives: Derivative[]
}> {
- const input = await boundedDownload(
+ const { limits } = configuration
+ let lastModified: Date | undefined
+ let input = await boundedDownload(
item.sourceUrl,
item.lease.expectedBytes,
signal,
item.etag,
+ limits.maxFileBytes,
+ (value) => {
+ if (value && Number.isFinite(Date.parse(value)))
+ lastModified = new Date(value)
+ },
)
const companion = item.companion
? await boundedDownload(
@@ -98,16 +144,58 @@ export async function processWork(
item.companion.bytes,
signal,
item.companion.etag,
+ limits.maxFileBytes,
)
: null
- const ext = path.extname(item.lease.objectKey).toLowerCase()
- if (!['.heic', '.heif', '.hif'].includes(ext))
- await sharp(input, { limitInputPixels: 100000000 }).metadata()
+ const ext = path.extname(item.fileName || item.lease.objectKey).toLowerCase()
+ const sourceKey = 'source' + ext
+ const operation = item.options?.operation ?? 'photo'
+ const eraseLocation =
+ item.eraseLocation === true || operation === 'erase-location'
+ const jobOptions = JSON.parse(item.options?.options_json ?? '{}')
+ const previous = item.priorMetadata?.metadata_json
+ ? JSON.parse(item.priorMetadata.metadata_json)
+ : null
const directory = await mkdtemp(path.join(tmpdir(), 'chronoframe-worker-'))
const exiftool = new ExifTool({ maxProcs: 1, taskTimeoutMillis: 30000 })
const derivatives: Derivative[] = []
try {
- if (['.heic', '.heif', '.hif'].includes(ext)) {
+ if (!['.heic', '.heif', '.hif', '.bmp'].includes(ext)) {
+ const dimensions = await sharp(input, {
+ limitInputPixels: false,
+ }).metadata()
+ if (!dimensions.width || !dimensions.height)
+ throw new WorkerProcessingError('invalid_dimensions', false)
+ if (dimensions.width * dimensions.height > limits.maxPixels)
+ throw new WorkerProcessingError(
+ 'node_pixel_limit',
+ true,
+ dimensions.width * dimensions.height,
+ )
+ }
+ if (eraseLocation) {
+ const file = path.join(directory, sourceKey)
+ await writeFile(file, input)
+ const locationTags = await exiftool.read(file)
+ if (
+ EXIF_LOCATION_KEYS.some(
+ (key) => locationTags[key as keyof typeof locationTags] != null,
+ )
+ )
+ await exiftool.write(
+ file,
+ Object.fromEntries(EXIF_LOCATION_KEYS.map((key) => [key, null])),
+ ['-overwrite_original'],
+ )
+ input = await readFile(file)
+ derivatives.push({
+ role: 'sanitized-original',
+ bytes: input.length,
+ contentType: item.contentType,
+ buffer: input,
+ })
+ }
+ if (['.heic', '.heif', '.hif', '.bmp'].includes(ext)) {
const sourceFile = path.join(directory, 'source' + ext)
await writeFile(sourceFile, input)
const tags = await exiftool.read(sourceFile),
@@ -117,23 +205,28 @@ export async function processWork(
!Number.isSafeInteger(width) ||
!Number.isSafeInteger(height) ||
width < 1 ||
- height < 1 ||
- width * height > 100000000
+ height < 1
)
throw new WorkerProcessingError('invalid_dimensions', false)
+ if (width * height > limits.maxPixels)
+ throw new WorkerProcessingError(
+ 'node_pixel_limit',
+ true,
+ width * height,
+ )
}
const storage: StorageProvider = {
async get(key) {
- return key === item.lease.objectKey
+ return key === sourceKey
? input
- : key === item.companion?.key
+ : key === 'source.MOV'
? companion
: null
},
async getFileMeta(key) {
- return key === item.lease.objectKey
- ? { key, size: input.length }
- : key === item.companion?.key && companion
+ return key === sourceKey
+ ? { key, size: input.length, lastModified }
+ : key === 'source.MOV' && companion
? { key, size: companion.length }
: null
},
@@ -147,8 +240,14 @@ export async function processWork(
: contentType === 'video/mp4'
? 'live-video'
: null
- if (!role || buffer.length > 1073741824)
- throw new Error('invalid_derivative')
+ if (!role) throw new Error('invalid_derivative')
+ if (buffer.length > limits.maxFileBytes)
+ throw new WorkerProcessingError(
+ 'node_file_limit',
+ true,
+ undefined,
+ buffer.length,
+ )
derivatives.push({
role,
bytes: buffer.length,
@@ -173,30 +272,46 @@ export async function processWork(
return []
},
}
+ const geocoder = processingGeocoder(
+ configuration.geocoding,
+ signal,
+ reserveGeocoding,
+ )
const process = createPhotoProcessor({
logger: quiet,
exif: createExifService({ exiftool, tempDir: directory, logger: quiet }),
- parseGPS() {
- return {}
- },
- async reverseGeocode() {
- return null
- },
+ ...geocoder,
+ reverseGeocode: ['photo', 'geocoding'].includes(operation)
+ ? geocoder.reverseGeocode
+ : async () => null,
+ parseGPS: (exif) =>
+ operation === 'geocoding' &&
+ typeof jobOptions.latitude === 'number' &&
+ typeof jobOptions.longitude === 'number'
+ ? { latitude: jobOptions.latitude, longitude: jobOptions.longitude }
+ : geocoder.parseGPS(exif),
})
const { photo } = await process({
photoId: item.lease.photoId,
- storageKey: item.lease.objectKey,
- eraseLocation: false,
+ storageKey: sourceKey,
+ eraseLocation,
+ maxPixels: limits.maxPixels,
storage,
async stage() {
signal.throwIfAborted()
},
signal,
})
+ if (operation === 'live-photo' && companion) {
+ for (let i = derivatives.length - 1; i >= 0; i--)
+ if (derivatives[i]?.role === 'live-video') derivatives.splice(i, 1)
+ photo.isLivePhoto = 1
+ photo.livePhotoVideoKey = 'source.MOV'
+ }
if (
photo.isLivePhoto &&
companion &&
- photo.livePhotoVideoKey === item.companion?.key
+ photo.livePhotoVideoKey === 'source.MOV'
)
derivatives.push({
role: 'live-video',
@@ -218,8 +333,52 @@ export async function processWork(
...metadata
} = photo
const cleanMetadata = JSON.parse(JSON.stringify(metadata))
- if (Buffer.byteLength(JSON.stringify(cleanMetadata)) > 60 * 1024)
- throw new WorkerProcessingError('metadata_too_large', false)
+ if (
+ previous &&
+ ['geocoding', 'erase-location', 'live-photo'].includes(operation)
+ ) {
+ cleanMetadata.title = item.priorMetadata!.title
+ cleanMetadata.description = previous.description
+ cleanMetadata.tags = previous.tags
+ }
+ if (previous && operation === 'reindex') {
+ cleanMetadata.description = previous.description
+ for (const key of [
+ 'latitude',
+ 'longitude',
+ 'country',
+ 'city',
+ 'locationName',
+ 'geocoding',
+ ])
+ cleanMetadata[key] = previous[key] ?? null
+ }
+ if (previous && operation === 'live-photo') {
+ for (const [key, value] of Object.entries(previous))
+ if (
+ ![
+ 'isLivePhoto',
+ 'livePhotoVideoKey',
+ 'livePhotoVideoUrl',
+ 'lastModified',
+ ].includes(key)
+ )
+ cleanMetadata[key] = value
+ }
+ if (previous && operation === 'geocoding') {
+ cleanMetadata.exif = previous.exif
+ cleanMetadata.dateTaken = previous.dateTaken
+ }
+ if (eraseLocation) cleanMetadata.locationErased = true
+ const metadataBytes = Buffer.byteLength(JSON.stringify(cleanMetadata))
+ if (metadataBytes > limits.maxMetadataBytes)
+ throw new WorkerProcessingError(
+ 'node_metadata_limit',
+ true,
+ undefined,
+ undefined,
+ metadataBytes,
+ )
return {
result: {
width: photo.width,
diff --git a/packages/contracts/package.json b/packages/contracts/package.json
index 754fb45f..aa5bca7b 100644
--- a/packages/contracts/package.json
+++ b/packages/contracts/package.json
@@ -19,7 +19,8 @@
"./storage-diagnostics": "./src/storage-diagnostics.ts",
"./gallery-settings": "./src/gallery-settings.ts",
"./reactions": "./src/reactions.ts",
- "./domains": "./src/domains.ts"
+ "./domains": "./src/domains.ts",
+ "./processor": "./src/processor.ts"
},
"scripts": {
"type-check": "tsc --noEmit"
diff --git a/packages/contracts/src/processor.ts b/packages/contracts/src/processor.ts
new file mode 100644
index 00000000..df690422
--- /dev/null
+++ b/packages/contracts/src/processor.ts
@@ -0,0 +1,155 @@
+/** Server-owned processing policy; shared by Cloud and Docker processing nodes. */
+export interface ProcessorLimits {
+ maxFileBytes: number
+ maxPixels: number
+ maxMetadataBytes: number
+ timeoutMs: number
+}
+export const DEFAULT_PROCESSOR_LIMITS: ProcessorLimits = {
+ maxFileBytes: 1073741824,
+ maxPixels: 100000000,
+ maxMetadataBytes: 61440,
+ timeoutMs: 180000,
+}
+export function parseProcessorLimits(value: unknown): ProcessorLimits {
+ if (!value || typeof value !== 'object' || Array.isArray(value))
+ throw new Error('Invalid processing limits')
+ const input = value as Record<string, unknown>
+ const result = { ...DEFAULT_PROCESSOR_LIMITS }
+ const ranges = {
+ maxFileBytes: [1048576, 8589934592],
+ maxPixels: [1000000, 1000000000],
+ maxMetadataBytes: [1024, 524288],
+ timeoutMs: [30000, 3600000],
+ } as const
+ if (Object.keys(input).some((key) => !(key in ranges)))
+ throw new Error('Invalid processing limits')
+ for (const key of Object.keys(ranges) as (keyof ProcessorLimits)[]) {
+ const v = input[key] ?? result[key],
+ [min, max] = ranges[key]
+ if (!Number.isSafeInteger(v) || Number(v) < min || Number(v) > max)
+ throw new Error('Invalid processing limits')
+ result[key] = Number(v)
+ }
+ return result
+}
+export interface ProcessorGeocoding {
+ provider: 'none' | 'mapbox' | 'nominatim'
+ token: string
+ endpoint: string
+ language: string
+}
+export const DEFAULT_PROCESSOR_GEOCODING: ProcessorGeocoding = {
+ provider: 'none',
+ token: '',
+ endpoint: 'https://nominatim.openstreetmap.org',
+ language: 'en',
+}
+export function parseProcessorGeocoding(value: unknown): ProcessorGeocoding {
+ if (!value || typeof value !== 'object' || Array.isArray(value))
+ throw new Error('Invalid geocoding settings')
+ const input = value as Record<string, unknown>
+ if (!['none', 'mapbox', 'nominatim'].includes(String(input.provider)))
+ throw new Error('Invalid geocoding provider')
+ const result = {
+ ...DEFAULT_PROCESSOR_GEOCODING,
+ ...input,
+ } as ProcessorGeocoding
+ for (const key of ['token', 'endpoint', 'language'] as const)
+ if (
+ typeof result[key] !== 'string' ||
+ result[key].length > (key === 'language' ? 40 : 2048) ||
+ Array.from(result[key]).some(
+ (c) => c.charCodeAt(0) <= 32 || c.charCodeAt(0) === 127,
+ )
+ )
+ throw new Error('Invalid geocoding settings')
+ if (result.provider === 'mapbox' && !result.token)
+ throw new Error('Mapbox credential required')
+ const url = new URL(result.endpoint)
+ if (
+ url.protocol !== 'https:' ||
+ url.username ||
+ url.password ||
+ url.search ||
+ url.hash ||
+ ['localhost', '0.0.0.0', '127.0.0.1', '[::1]'].includes(url.hostname)
+ )
+ throw new Error('Invalid geocoding endpoint')
+ if (!/^[a-zA-Z0-9,-]+$/.test(result.language))
+ throw new Error('Invalid geocoding language')
+ return {
+ provider: result.provider,
+ token: result.token,
+ endpoint: result.endpoint.replace(/\/$/, ''),
+ language: result.language,
+ }
+}
+export interface ProcessorConfiguration {
+ limits: ProcessorLimits
+ geocoding: ProcessorGeocoding
+}
+const encoder = new TextEncoder()
+const encode = (v: Uint8Array) => btoa(String.fromCharCode(...v))
+const decode = (v: string) => Uint8Array.from(atob(v), (c) => c.charCodeAt(0))
+/** Dedicated HKDF domain; authenticated context prevents cross-node/revision substitution. */
+export async function processorCipher(secret: string) {
+ if (secret.length < 32) throw new Error('Invalid processor key')
+ const material = await crypto.subtle.importKey(
+ 'raw',
+ encoder.encode(secret),
+ 'HKDF',
+ false,
+ ['deriveKey'],
+ )
+ const key = await crypto.subtle.deriveKey(
+ {
+ name: 'HKDF',
+ hash: 'SHA-256',
+ salt: encoder.encode('chronoframe-processor-v1'),
+ info: encoder.encode('configuration'),
+ },
+ material,
+ { name: 'AES-GCM', length: 256 },
+ false,
+ ['encrypt', 'decrypt'],
+ )
+ return {
+ async seal(value: unknown, context: string) {
+ const iv = crypto.getRandomValues(new Uint8Array(12))
+ const body = await crypto.subtle.encrypt(
+ { name: 'AES-GCM', iv, additionalData: encoder.encode(context) },
+ key,
+ encoder.encode(JSON.stringify(value)),
+ )
+ return 'v1.' + encode(iv) + '.' + encode(new Uint8Array(body))
+ },
+ async open<T>(value: string, context: string): Promise<T> {
+ const [version, iv, body, ...rest] = value.split('.')
+ if (
+ version !== 'v1' ||
+ !iv ||
+ !body ||
+ rest.length ||
+ value.length > 16384
+ )
+ throw new Error('Invalid processor envelope')
+ const plain = await crypto.subtle.decrypt(
+ {
+ name: 'AES-GCM',
+ iv: decode(iv),
+ additionalData: encoder.encode(context),
+ },
+ key,
+ decode(body),
+ )
+ return JSON.parse(new TextDecoder().decode(plain))
+ },
+ }
+}
+export async function processorKeyHash(key: string) {
+ return Array.from(
+ new Uint8Array(await crypto.subtle.digest('SHA-256', encoder.encode(key))),
+ (v) => v.toString(16).padStart(2, '0'),
+ ).join('')
+}
diff --git a/packages/core/src/cloud-execution.ts b/packages/core/src/cloud-execution.ts
index 6b751ca2..7170ab96 100644
--- a/packages/core/src/cloud-execution.ts
+++ b/packages/core/src/cloud-execution.ts
@@ -19,7 +19,7 @@ export interface CloudMediaResult {
/** JSON-safe metadata, independently size-limited before persistence. */
metadata: Record<string, unknown>
assets: {
- role: 'thumbnail' | 'live-video' | 'display'
+ role: 'thumbnail' | 'live-video' | 'display' | 'sanitized-original'
objectKey: string
bytes: number
contentType: string
diff --git a/packages/database/src/cloud-execution.ts b/packages/database/src/cloud-execution.ts
index 3f2f8fc4..301d34f3 100644
--- a/packages/database/src/cloud-execution.ts
+++ b/packages/database/src/cloud-execution.ts
@@ -1,3 +1,8 @@
+import {
+ DEFAULT_PROCESSOR_LIMITS,
+ parseProcessorLimits,
+ type ProcessorLimits,
+} from '@chronoframe/contracts/processor'
import {
assertSiteContext,
CoreError,
@@ -32,8 +37,16 @@ const currentSource = `EXISTS(SELECT 1 FROM cloud_photos p WHERE p.site_id=cloud
export function createCloudExecutionRepository(
db: MediaDatabase,
site: SiteContext,
+ nodeLimits?: ProcessorLimits,
+ publicationRevision = 0,
): CloudExecutionRepository {
assertSiteContext(site)
+ const limits = nodeLimits
+ ? parseProcessorLimits(nodeLimits)
+ : DEFAULT_PROCESSOR_LIMITS
+ const capability = nodeLimits
+ ? `required_pixels<=${limits.maxPixels} AND required_file_bytes<=${limits.maxFileBytes} AND required_metadata_bytes<=${limits.maxMetadataBytes} AND `
+ : ''
const siteId = site.id
const stmt = (sql: string, ...values: (string | number | null)[]) =>
db.prepare(sql).bind(...values)
@@ -43,7 +56,12 @@ export function createCloudExecutionRepository(
identifier(lease.attemptId)
identifier(lease.workerId)
}
- const valid = `site_id=? AND id=? AND attempt_id=? AND worker_id=? AND state='running' AND lease_until>? AND ${currentSource} AND ${activeSite}`
+ if (!Number.isSafeInteger(publicationRevision) || publicationRevision < 0)
+ throw new CoreError(400, 'Invalid photo revision')
+ const revisionGuard = publicationRevision
+ ? ` AND EXISTS(SELECT 1 FROM cloud_photos p WHERE p.site_id=cloud_processing_jobs.site_id AND p.id=cloud_processing_jobs.photo_id AND p.revision=${publicationRevision})`
+ : ''
+ const valid = `site_id=? AND id=? AND attempt_id=? AND worker_id=? AND state='running' AND lease_until>? AND ${currentSource} AND ${activeSite}${revisionGuard}`
const validValues = (lease: CloudLease, now: number) =>
[lease.id, lease.attemptId, lease.workerId, positive(now)] as const
async function changed(s: MediaStatement) {
@@ -66,7 +84,7 @@ export function createCloudExecutionRepository(
stmt(
`UPDATE cloud_processing_jobs SET state='running',worker_id=?,attempt_id=?,lease_until=?,attempts=attempts+1,error_code=NULL
WHERE site_id=? AND ${activeSite} AND id=(SELECT id FROM cloud_processing_jobs WHERE site_id=? AND attempts<max_attempts AND ${currentSource}
- AND ((state IN ('queued','retry_wait') AND available_at<=?) OR (state='running' AND lease_until<=?)) AND NOT EXISTS(SELECT 1 FROM cloud_processing_jobs active WHERE active.site_id=? AND active.state='running' AND active.lease_until>?) ORDER BY created_at,id LIMIT 1)`,
+ AND ${capability}((state IN ('queued','retry_wait') AND available_at<=?) OR (state='running' AND lease_until<=?)) AND NOT EXISTS(SELECT 1 FROM cloud_processing_jobs active WHERE (active.site_id=? OR active.worker_id=?) AND active.state='running' AND active.lease_until>?) AND EXISTS(SELECT 1 FROM cloud_uploads eligible WHERE eligible.site_id=cloud_processing_jobs.site_id AND eligible.id=cloud_processing_jobs.upload_id AND MAX(eligible.expected_bytes,eligible.expected_video_bytes)<=?) ORDER BY created_at,id LIMIT 1)`,
workerId,
attemptId,
now + leaseMs,
@@ -75,7 +93,9 @@ export function createCloudExecutionRepository(
now,
now,
siteId,
+ workerId,
now,
+ limits.maxFileBytes,
),
stmt(
`UPDATE cloud_job_attempts SET state='expired',finished_at=? WHERE site_id=? AND state='running' AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=cloud_job_attempts.site_id AND j.id=cloud_job_attempts.job_id AND (j.attempt_id!=cloud_job_attempts.attempt_id OR j.state='failed'))`,
@@ -169,11 +189,7 @@ export function createCloudExecutionRepository(
return false
positive(result.width)
positive(result.height)
- if (
- result.width > 65535 ||
- result.height > 65535 ||
- result.width * result.height > 100_000_000
- )
+ if (result.width * result.height > limits.maxPixels)
throw new CoreError(400, 'Image dimensions exceed limits')
const json = JSON.stringify({
...result.metadata,
@@ -181,33 +197,54 @@ export function createCloudExecutionRepository(
height: result.height,
})
if (
- new TextEncoder().encode(json).length > 65536 ||
+ new TextEncoder().encode(json).length > limits.maxMetadataBytes ||
result.assets.length < 1 ||
- result.assets.length > 3 ||
+ result.assets.length > 4 ||
new Set(result.assets.map((a) => a.role)).size !== result.assets.length
)
throw new CoreError(400, 'Invalid media result')
const prefix = `chronoframe/${siteId}/derived/${lease.photoId}/${lease.sourceVersion}/${lease.pipelineVersion}/${lease.attemptId}/`
for (const asset of result.assets) {
positive(asset.bytes)
+ if (
+ asset.bytes > limits.maxFileBytes ||
+ ![
+ 'image/jpeg',
+ 'image/png',
+ 'image/webp',
+ 'image/heic',
+ 'image/heif',
+ 'image/avif',
+ 'image/gif',
+ 'image/bmp',
+ 'image/tiff',
+ 'video/mp4',
+ 'video/quicktime',
+ ].includes(asset.contentType)
+ )
+ throw new CoreError(400, 'Invalid derivative')
const suffix =
asset.role === 'thumbnail'
? 'thumbnail.webp'
- : asset.role === 'display'
- ? 'display.jpeg'
- : asset.contentType === 'video/mp4'
- ? 'live-video.mp4'
- : 'live-video.mov'
+ : asset.role === 'sanitized-original'
+ ? 'sanitized-original'
+ : asset.role === 'display'
+ ? 'display.jpeg'
+ : asset.contentType === 'video/mp4'
+ ? 'live-video.mp4'
+ : 'live-video.mov'
if (
asset.objectKey !== prefix + suffix ||
asset.contentType !==
- (asset.role === 'thumbnail'
- ? 'image/webp'
- : asset.role === 'display'
- ? 'image/jpeg'
- : asset.objectKey.endsWith('.mp4')
- ? 'video/mp4'
- : 'video/quicktime')
+ (asset.role === 'sanitized-original'
+ ? asset.contentType
+ : asset.role === 'thumbnail'
+ ? 'image/webp'
+ : asset.role === 'display'
+ ? 'image/jpeg'
+ : asset.objectKey.endsWith('.mp4')
+ ? 'video/mp4'
+ : 'video/quicktime')
)
throw new CoreError(400, 'Unexpected media object')
}
diff --git a/packages/database/src/cloud-resources.ts b/packages/database/src/cloud-resources.ts
index 472f39e0..1b1e8fab 100644
--- a/packages/database/src/cloud-resources.ts
+++ b/packages/database/src/cloud-resources.ts
@@ -41,7 +41,7 @@ export function createCloudResourceRepository(
: null,
dateTaken: typeof m.dateTaken === 'string' ? m.dateTaken : null,
fileSize: typeof m.fileSize === 'number' ? m.fileSize : null,
- lastModified: null,
+ lastModified: typeof m.lastModified === 'string' ? m.lastModified : null,
originalUrl: base + '/display',
downloadUrl: base + '/original',
thumbnailUrl: base + '/thumbnail',
@@ -56,13 +56,13 @@ export function createCloudResourceRepository(
country: typeof m.country === 'string' ? m.country : null,
city: typeof m.city === 'string' ? m.city : null,
locationName: typeof m.locationName === 'string' ? m.locationName : null,
- geocoding: null,
+ geocoding: m.geocoding ?? null,
isLivePhoto: row.has_video ? 1 : 0,
livePhotoVideoUrl: row.has_video ? base + '/live-video' : null,
}
}
const columns = `p.*,EXISTS(SELECT 1 FROM cloud_media_assets a WHERE a.site_id=p.site_id AND a.photo_id=p.id AND a.source_version=p.source_version AND a.role='live-video') AS has_video`
- const ready = `p.deleted=0 AND p.metadata_json IS NOT NULL AND EXISTS(SELECT 1 FROM cloud_processing_jobs j WHERE j.site_id=p.site_id AND j.photo_id=p.id AND j.source_version=p.source_version AND j.state='succeeded')`
+ const ready = `p.deleted=0 AND p.metadata_json IS NOT NULL AND EXISTS(SELECT 1 FROM cloud_media_assets a WHERE a.site_id=p.site_id AND a.photo_id=p.id AND a.source_version=p.source_version AND a.role='thumbnail')`
const album = (r: Record<string, unknown>): AlbumReadRecord => ({
resource: {
id: Number(r.id),
diff --git a/packages/platform/migrations/0036_processors.sql b/packages/platform/migrations/0036_processors.sql
new file mode 100644
index 00000000..80c1a663
--- /dev/null
+++ b/packages/platform/migrations/0036_processors.sql
@@ -0,0 +1,37 @@
+CREATE TABLE platform_processor_nodes (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL,
+ secret_hash TEXT NOT NULL UNIQUE,
+ enabled INTEGER NOT NULL DEFAULT 1 CHECK(enabled IN(0,1)),
+ limits_json TEXT NOT NULL,
+ version TEXT NOT NULL DEFAULT '',
+ instance_id TEXT NOT NULL DEFAULT '',
+ hostname TEXT NOT NULL DEFAULT '',
+ last_seen_at INTEGER,
+ metrics_json TEXT NOT NULL DEFAULT '{}',
+ created_at INTEGER NOT NULL,
+ revision INTEGER NOT NULL DEFAULT 1
+);
+CREATE TABLE platform_processor_settings(id INTEGER PRIMARY KEY CHECK(id=1),encrypted_config TEXT,revision INTEGER NOT NULL DEFAULT 1);
+INSERT INTO platform_processor_settings(id) VALUES(1);
+ALTER TABLE cloud_processing_jobs ADD COLUMN operation TEXT NOT NULL DEFAULT 'photo';
+ALTER TABLE cloud_processing_jobs ADD COLUMN options_json TEXT NOT NULL DEFAULT '{}';
+CREATE TABLE platform_site_processing(site_id INTEGER PRIMARY KEY REFERENCES platform_sites(id) ON DELETE CASCADE,erase_location INTEGER NOT NULL DEFAULT 0 CHECK(erase_location IN(0,1)),revision INTEGER NOT NULL DEFAULT 1);
+CREATE INDEX cloud_jobs_worker_active ON cloud_processing_jobs(worker_id,state,lease_until);
+-- Publish sanitized originals atomically without overwriting source objects.
+ALTER TABLE cloud_media_assets RENAME TO cloud_media_assets_old;
+CREATE TABLE cloud_media_assets (
+ site_id INTEGER NOT NULL, photo_id TEXT NOT NULL, source_version TEXT NOT NULL, pipeline_version TEXT NOT NULL,
+ role TEXT NOT NULL CHECK(role IN ('thumbnail','live-video','display','sanitized-original')),
+ connection_id TEXT NOT NULL, object_key TEXT NOT NULL, bytes INTEGER NOT NULL CHECK(bytes>0), content_type TEXT NOT NULL,
+ connection_revision INTEGER NOT NULL DEFAULT 1,
+ PRIMARY KEY(site_id,photo_id,source_version,pipeline_version,role),
+ FOREIGN KEY(site_id,photo_id) REFERENCES cloud_photos(site_id,id)
+);
+INSERT INTO cloud_media_assets SELECT site_id,photo_id,source_version,pipeline_version,role,connection_id,object_key,bytes,content_type,connection_revision FROM cloud_media_assets_old;
+DROP TABLE cloud_media_assets_old;
+CREATE TABLE platform_processor_geocoding_slots(provider TEXT PRIMARY KEY,next_at INTEGER NOT NULL);
+CREATE TABLE cloud_photo_companions(site_id INTEGER NOT NULL,photo_id TEXT NOT NULL,upload_id TEXT NOT NULL,PRIMARY KEY(site_id,photo_id),FOREIGN KEY(site_id,photo_id) REFERENCES cloud_photos(site_id,id),FOREIGN KEY(site_id,upload_id) REFERENCES cloud_uploads(site_id,id));
+ALTER TABLE cloud_processing_jobs ADD COLUMN required_pixels INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE cloud_processing_jobs ADD COLUMN required_file_bytes INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE cloud_processing_jobs ADD COLUMN required_metadata_bytes INTEGER NOT NULL DEFAULT 0;
diff --git a/packages/platform/src/entitlements.ts b/packages/platform/src/entitlements.ts
index 1f806ad0..26aaa9e7 100644
--- a/packages/platform/src/entitlements.ts
+++ b/packages/platform/src/entitlements.ts
@@ -212,9 +212,9 @@ export function workspaceUsageSQL(siteId: number) {
throw new Error('Site required')
const sites = `SELECT id FROM platform_sites WHERE workspace_id=(SELECT workspace_id FROM platform_sites WHERE id=${siteId})`
return {
- photos: `(SELECT COUNT(*) FROM cloud_photos WHERE site_id IN (${sites}) AND deleted=0)+(SELECT COUNT(*) FROM cloud_uploads WHERE site_id IN (${sites}) AND state='pending' AND expires_at>?)`,
+ photos: `(SELECT COUNT(*) FROM cloud_photos WHERE site_id IN (${sites}) AND deleted=0)+(SELECT COUNT(*) FROM cloud_uploads WHERE site_id IN (${sites}) AND state='pending' AND content_type NOT LIKE 'video/%' AND expires_at>?)`,
monthly: `COALESCE((SELECT SUM(bytes) FROM cloud_usage_events WHERE site_id IN (${sites}) AND created_at>=?),0)+COALESCE((SELECT SUM(expected_bytes+expected_video_bytes) FROM cloud_uploads WHERE site_id IN (${sites}) AND state='pending' AND expires_at>?),0)`,
- storage: `COALESCE((SELECT SUM(u.expected_bytes+u.expected_video_bytes) FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id JOIN cloud_storage_connections c ON c.site_id=u.site_id AND c.id=u.connection_id WHERE u.site_id IN (${sites}) AND u.state='completed' AND p.deleted=0 AND c.pool_id IS NOT NULL),0)
+ storage: `COALESCE((SELECT SUM((CASE WHEN EXISTS(SELECT 1 FROM cloud_media_assets sanitized WHERE sanitized.site_id=u.site_id AND sanitized.photo_id=u.photo_id AND sanitized.source_version=u.source_version AND sanitized.role='sanitized-original') AND u.content_type NOT LIKE 'video/%' THEN 0 ELSE u.expected_bytes END)+u.expected_video_bytes) FROM cloud_uploads u JOIN cloud_photos p ON p.site_id=u.site_id AND p.id=u.photo_id JOIN cloud_storage_connections c ON c.site_id=u.site_id AND c.id=u.connection_id WHERE u.site_id IN (${sites}) AND u.state='completed' AND p.deleted=0 AND c.pool_id IS NOT NULL),0)
+COALESCE((SELECT SUM(a.bytes) FROM cloud_media_assets a JOIN cloud_photos p ON p.site_id=a.site_id AND p.id=a.photo_id JOIN cloud_storage_connections c ON c.site_id=a.site_id AND c.id=a.connection_id WHERE a.site_id IN (${sites}) AND p.deleted=0 AND c.pool_id IS NOT NULL),0)
+COALESCE((SELECT SUM(reserved_storage_bytes) FROM cloud_uploads WHERE site_id IN (${sites}) AND state='pending' AND expires_at>?),0)
+COALESCE((SELECT SUM(reserved_output_bytes) FROM cloud_processing_jobs WHERE site_id IN (${sites}) AND state IN('queued','running','retry_wait')),0)`,
diff --git a/packages/platform/src/processors.ts b/packages/platform/src/processors.ts
new file mode 100644
index 00000000..f162d32e
--- /dev/null
+++ b/packages/platform/src/processors.ts
@@ -0,0 +1,66 @@
+import type { PlatformD1 } from './index.ts'
+import {
+ processorCipher,
+ processorKeyHash,
+ parseProcessorLimits,
+ parseProcessorGeocoding,
+ DEFAULT_PROCESSOR_GEOCODING,
+ type ProcessorConfiguration,
+} from '../../contracts/src/processor.ts'
+export async function authenticateProcessor(db: PlatformD1, key: string) {
+ if (!/^pn_[a-f0-9-]{36}\.[A-Za-z0-9_-]{43}$/.test(key)) return null
+ return db
+ .prepare(
+ 'SELECT id,name,limits_json,revision FROM platform_processor_nodes WHERE secret_hash=? AND enabled=1',
+ )
+ .bind(await processorKeyHash(key))
+ .first<{
+ id: string
+ name: string
+ limits_json: string
+ revision: number
+ }>()
+}
+export async function processingGeocoding(db: PlatformD1, secret: string) {
+ const row = await db
+ .prepare(
+ 'SELECT encrypted_config,revision FROM platform_processor_settings WHERE id=1',
+ )
+ .first<{ encrypted_config: string | null; revision: number }>()
+ const value = row?.encrypted_config
+ ? await (
+ await processorCipher(secret)
+ ).open(row.encrypted_config, 'platform-geocoding')
+ : DEFAULT_PROCESSOR_GEOCODING
+ return {
+ config: parseProcessorGeocoding(value),
+ revision: row?.revision ?? 1,
+ }
+}
+export async function processorEnvelope(
+ db: PlatformD1,
+ secret: string,
+ token: string,
+ node: { id: string; limits_json: string; revision: number },
+ now: number,
+) {
+ const geocoding = await processingGeocoding(db, secret)
+ const revision = `${node.revision}:${geocoding.revision}`,
+ expiresAt = now + 300000
+ const config: ProcessorConfiguration = {
+ limits: parseProcessorLimits(JSON.parse(node.limits_json)),
+ geocoding: {
+ ...geocoding.config,
+ token:
+ geocoding.config.provider === 'mapbox' ? geocoding.config.token : '',
+ },
+ }
+ return {
+ nodeId: node.id,
+ revision,
+ expiresAt,
+ envelope: await (
+ await processorCipher(token)
+ ).seal(config, `${node.id}:${revision}:${expiresAt}`),
+ }
+}
diff --git a/packages/processing/src/index.ts b/packages/processing/src/index.ts
index 3ae0059b..3220b9f5 100644
--- a/packages/processing/src/index.ts
+++ b/packages/processing/src/index.ts
@@ -22,6 +22,7 @@ export const EXIF_LOCATION_KEYS = [
'GPSLongitude',
'GPSLongitudeRef',
'GPSPosition',
+ 'GPSCoordinates',
'GPSDateStamp',
'GPSTimeStamp',
'GPSImgDirection',
@@ -41,6 +42,7 @@ export type ProcessedPhoto = Photo & { originalDerivedKey: string | null }
export interface PhotoProcessingInput {
photoId: string
storageKey: string
+ maxPixels?: number
eraseLocation: boolean
storage: StorageProvider
stage(stage: ProcessingStage): Promise<void>
@@ -66,7 +68,10 @@ export function createPhotoProcessor(deps: {
input.signal?.throwIfAborted()
await input.stage(value)
}
- const media = createImageProcessor(deps.logger, { signal: input.signal })
+ const media = createImageProcessor(deps.logger, {
+ signal: input.signal,
+ limitInputPixels: input.maxPixels,
+ })
await stage('preprocessing')
const raw = await storage.get(storageKey)
if (!raw) throw new Error('Source object not found')
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 5aa055e6..91d703b0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -645,6 +645,9 @@ importers:
'@chronoframe/core':
specifier: workspace:*
version: link:../../packages/core
+ '@chronoframe/geocoding':
+ specifier: workspace:*
+ version: link:../../packages/geocoding
'@chronoframe/media':
specifier: workspace:*
version: link:../../packages/media
diff --git a/scripts/setup-cloud-media.mjs b/scripts/setup-cloud-media.mjs
index 1f2df448..28126268 100644
--- a/scripts/setup-cloud-media.mjs
+++ b/scripts/setup-cloud-media.mjs
@@ -1,4 +1,4 @@
-import { randomBytes, randomUUID } from 'node:crypto'
+import { randomBytes } from 'node:crypto'
import { mkdir, readFile, writeFile, realpath, chmod } from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
@@ -55,46 +55,31 @@ try {
} catch (error) {
if (error.code !== 'ENOENT') throw error
}
-if (
- !secrets &&
- existing.some((s) =>
- ['PLATFORM_STORAGE_SECRET', 'PLATFORM_WORKER_SECRET'].includes(s.name),
- )
-)
+if (!secrets && existing.some((s) => s.name === 'PLATFORM_STORAGE_SECRET'))
throw new Error(
'Secrets already exist remotely. Use the original protected output directory; do not silently rotate a storage encryption key',
)
if (!secrets) {
secrets = {
storage: randomBytes(48).toString('base64url'),
- worker: randomBytes(48).toString('base64url'),
- workerId: randomUUID(),
environment,
}
await writeFile(file, JSON.stringify(secrets), { mode: 0o600, flag: 'wx' })
}
-if (
- secrets.environment !== environment ||
- secrets.storage?.length < 32 ||
- secrets.worker?.length < 32
-)
+if (secrets.environment !== environment || secrets.storage?.length < 32)
throw new Error('Invalid protected configuration')
await chmod(file, 0o600)
gh(
['secret', 'set', 'PLATFORM_STORAGE_SECRET', '--env', envName],
secrets.storage,
)
-gh(
- ['secret', 'set', 'PLATFORM_WORKER_SECRET', '--env', envName],
- secrets.worker,
-)
-const workerEnv = path.join(target, environment + '-worker.env')
+const workerEnv = path.join(target, environment + '-processor.env')
await writeFile(
workerEnv,
- `CHRONOFRAME_PORTAL_ORIGIN=${portal}\nCHRONOFRAME_WORKER_SECRET=${secrets.worker}\nCHRONOFRAME_WORKER_ID=${secrets.workerId}\n`,
+ `CHRONOFRAME_PORTAL_ORIGIN=${portal}\n# Register a node in the administration panel and paste its one-time connection key.\nCHRONOFRAME_PROCESSOR_KEY=${process.env.CHRONOFRAME_PROCESSOR_KEY || ''}\n`,
{ mode: 0o600 },
)
await chmod(workerEnv, 0o600)
console.log(
- `Configured ${envName}. Protected worker configuration: ${workerEnv}. Back up the media-secrets.json file securely; losing its storage key makes stored credentials unreadable. No deployment has been started.`,
+ `Configured ${envName}. Protected processing-node configuration: ${workerEnv}. Back up the media-secrets.json file securely; losing its storage key makes stored credentials unreadable. No deployment has been started.`,
)
diff --git a/tests/cloud-live-attachment.test.ts b/tests/cloud-live-attachment.test.ts
new file mode 100644
index 00000000..d74d71f8
--- /dev/null
+++ b/tests/cloud-live-attachment.test.ts
@@ -0,0 +1,203 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { createRequire } from 'node:module'
+import { readFile, readdir } from 'node:fs/promises'
+import ts from 'typescript'
+import { splitSqlQuery } from './helpers/sql.ts'
+import { workspaceUsageSQL } from '../packages/platform/src/entitlements.ts'
+import { createCloudMediaRepository } from '../packages/database/src/cloud-media.ts'
+import { createCloudExecutionRepository } from '../packages/database/src/cloud-execution.ts'
+const require = createRequire(import.meta.url),
+ { Miniflare, convertV4MiniflareOptions } = createRequire(
+ require.resolve('wrangler/package.json'),
+ )('miniflare')
+test('post-upload MOV pairing verifies ownership, object metadata, quota and revisions before queueing once', async () => {
+ const mf = new Miniflare(
+ convertV4MiniflareOptions({
+ modules: true,
+ compatibilityDate: '2026-10-26',
+ script: 'export default {fetch(){return new Response("ok")}}',
+ d1Databases: ['DB'],
+ }),
+ )
+ try {
+ const db = await mf.getD1Database('DB')
+ for (const file of (await readdir('packages/platform/migrations'))
+ .filter((f) => f.endsWith('.sql'))
+ .sort())
+ for (const sql of splitSqlQuery(
+ (
+ await readFile('packages/platform/migrations/' + file, 'utf8')
+ ).replace(/^--.*$/gm, ''),
+ ))
+ if (sql.trim()) await db.prepare(sql).run()
+ await db
+ .prepare(
+ "INSERT INTO platform_users(id,email,created_at) VALUES(1,'[email protected]',1)",
+ )
+ .run()
+ await db
+ .prepare("INSERT INTO platform_workspaces(id,name) VALUES(1,'One')")
+ .run()
+ await db
+ .prepare(
+ "INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'One')",
+ )
+ .run()
+ const media = createCloudMediaRepository(db, { id: 1 }),
+ queue = createCloudExecutionRepository(db, { id: 1 })
+ await media.reserveUpload({
+ id: 'photo',
+ ownerId: 1,
+ photoId: 'photo',
+ sourceVersion: '1',
+ connectionId: 's3',
+ objectKey: 'source.jpg',
+ providerUploadId: null,
+ expectedBytes: 20,
+ expiresAt: 100000,
+ })
+ await media.completeUpload({
+ id: 'photo',
+ ownerId: 1,
+ jobId: 'photo',
+ pipelineVersion: '1',
+ title: 'Photo',
+ now: 1,
+ })
+ const lease = await queue.claim('node', 'attempt', 10, 1000)
+ assert(lease)
+ await queue.complete(
+ lease,
+ {
+ width: 16,
+ height: 12,
+ metadata: {},
+ assets: [
+ {
+ role: 'thumbnail',
+ objectKey: 'chronoframe/1/derived/photo/1/1/attempt/thumbnail.webp',
+ bytes: 5,
+ contentType: 'image/webp',
+ },
+ ],
+ },
+ 20,
+ )
+ const revision = (
+ await db
+ .prepare("SELECT revision FROM cloud_photos WHERE id='photo'")
+ .first()
+ ).revision
+ const source = (
+ await readFile('apps/cloud/server/utils/cloud-live-attachment.ts', 'utf8')
+ ).replace(/^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm, '')
+ const deps = {
+ workspaceUsageSQL,
+ workspaceEntitlements: async () => ({
+ limits: {
+ fileBytes: 1000,
+ monthlyBytes: 10000,
+ storageBytes: 10000,
+ byos: true,
+ },
+ quotaPeriod: { start: 0 },
+ }),
+ crypto,
+ createError: (v: any) => Object.assign(new Error(v.statusMessage), v),
+ }
+ const code = ts.transpileModule(source, {
+ compilerOptions: {
+ module: ts.ModuleKind.CommonJS,
+ target: ts.ScriptTarget.ES2022,
+ },
+ }).outputText
+ const attach = new Function(
+ 'exports',
+ ...Object.keys(deps),
+ code + ';return exports.cloudLiveAttachment',
+ )({}, ...Object.values(deps))
+ let bytes = 99
+ const signed: string[] = []
+ const ctx = {
+ raw: db,
+ siteId: 1,
+ userId: 1,
+ site: { workspaceId: 1 },
+ vault: { resolve: async () => ({ poolId: 'pool' }) },
+ storage: async () => ({
+ s3: {
+ signPut: async (key: string) => {
+ signed.push(key)
+ return 'https://storage.example/' + key
+ },
+ head: async () => ({
+ bytes,
+ contentType: 'video/quicktime',
+ etag: '"verified"',
+ }),
+ },
+ }),
+ }
+ await assert.rejects(
+ attach(
+ ctx,
+ 'photo',
+ { phase: 'prepare', revision, fileName: 'bad.mp4', bytes: 100 },
+ 30,
+ ),
+ { statusCode: 400 },
+ )
+ await assert.rejects(
+ attach(
+ ctx,
+ 'foreign',
+ { phase: 'prepare', revision, fileName: 'live.mov', bytes: 100 },
+ 30,
+ ),
+ { statusCode: 409 },
+ )
+ const prepared = await attach(
+ ctx,
+ 'photo',
+ { phase: 'prepare', revision, fileName: 'live.mov', bytes: 100 },
+ 30,
+ )
+ assert(signed[0].startsWith('chronoframe/1/live-attachments/'))
+ const complete = { phase: 'complete', revision, uploadId: prepared.id }
+ await assert.rejects(attach(ctx, 'photo', complete, 40), {
+ statusCode: 400,
+ })
+ assert.equal(
+ (await db.prepare('SELECT state FROM cloud_processing_jobs').first())
+ .state,
+ 'succeeded',
+ )
+ bytes = 100
+ assert((await attach(ctx, 'photo', complete, 50)).ok)
+ assert.equal(
+ (
+ await db
+ .prepare('SELECT state,operation FROM cloud_processing_jobs')
+ .first()
+ ).operation,
+ 'live-photo',
+ )
+ assert.equal(
+ (await db.prepare('SELECT COUNT(*) n FROM cloud_media_assets').first()).n,
+ 1,
+ )
+ assert.equal(
+ (await db.prepare('SELECT upload_id FROM cloud_photo_companions').first())
+ .upload_id,
+ prepared.id,
+ )
+ assert((await attach(ctx, 'photo', complete, 60)).ok)
+ assert.equal(
+ (await db.prepare('SELECT COUNT(*) n FROM cloud_usage_events').first()).n,
+ 2,
+ )
+ } finally {
+ await mf.dispose()
+ }
+})
diff --git a/tests/cloud-photo-delete.test.ts b/tests/cloud-photo-delete.test.ts
index ab14ac2c..d5cb53fb 100644
--- a/tests/cloud-photo-delete.test.ts
+++ b/tests/cloud-photo-delete.test.ts
@@ -58,7 +58,15 @@ test('bulk deletion rejects foreign or invalid IDs before any site changes', asy
sqlite.prepare('SELECT SUM(deleted) AS n FROM cloud_photos').get().n,
0,
)
+ sqlite.exec(
+ "INSERT INTO cloud_media_assets(site_id,photo_id,source_version,pipeline_version,role,connection_id,object_key,bytes,content_type) VALUES(1,'own','1','1','thumbnail','s3','old.webp',10,'image/webp'),(1,'own','1','2','thumbnail','s3','new.webp',10,'image/webp')",
+ )
await deleteCloudPhotos(db, 1, ['own', 'own'], 1)
+ assert.equal(
+ sqlite.prepare('SELECT COUNT(*) AS n FROM cloud_cleanup_requests').get()
+ .n,
+ 2,
+ )
assert.equal(
sqlite.prepare('SELECT deleted FROM cloud_photos WHERE site_id=1').get()
.deleted,
diff --git a/tests/processor-media.test.ts b/tests/processor-media.test.ts
new file mode 100644
index 00000000..9431fded
--- /dev/null
+++ b/tests/processor-media.test.ts
@@ -0,0 +1,159 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import sharp from 'sharp'
+import { ExifTool } from 'exiftool-vendored'
+import { mkdtemp, writeFile, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { createRequire } from 'node:module'
+import { processWork } from '../apps/worker/src/process.ts'
+import type { CloudLease } from '../packages/core/src/cloud-execution.ts'
+const bmp = createRequire(
+ new URL('../packages/media/src/processor.ts', import.meta.url),
+)('@vingle/bmp-js')
+const lease = {
+ id: 'job',
+ photoId: 'photo',
+ siteId: 1,
+ sourceVersion: 'v1',
+ pipelineVersion: '1',
+ attemptId: 'a',
+ workerId: 'n',
+ attempts: 1,
+ connectionRevision: 1,
+ connectionId: 's3',
+ objectKey: 'source.jpg',
+ expectedBytes: 0,
+} satisfies CloudLease
+
+test('Docker processing path persists actual GPS, creates thumbnails and physically erases location from downloads', async () => {
+ const dir = await mkdtemp(path.join(tmpdir(), 'processor-test-')),
+ tool = new ExifTool({ maxProcs: 1 }),
+ originalFetch = globalThis.fetch
+ try {
+ const file = path.join(dir, 'source.jpg')
+ await writeFile(
+ file,
+ await sharp({
+ create: { width: 32, height: 24, channels: 3, background: '#123456' },
+ })
+ .jpeg()
+ .toBuffer(),
+ )
+ await tool.write(
+ file,
+ {
+ GPSLatitude: 31.2,
+ GPSLongitude: 121.5,
+ GPSLatitudeRef: 'N',
+ GPSLongitudeRef: 'E',
+ },
+ ['-overwrite_original'],
+ )
+ const input = await readFile(file)
+ globalThis.fetch = async () => new Response(new Uint8Array(input))
+ const item = {
+ lease: { ...lease, expectedBytes: input.length },
+ sourceUrl: 'https://storage.example/source',
+ fileName: 'source.jpg',
+ contentType: 'image/jpeg',
+ }
+ const result = await processWork(item, new AbortController().signal)
+ assert.equal(result.result.metadata.latitude, 31.2)
+ assert.equal(result.result.metadata.longitude, 121.5)
+ assert(result.derivatives.some((d) => d.role === 'thumbnail'))
+ const erased = await processWork(
+ { ...item, eraseLocation: true },
+ new AbortController().signal,
+ )
+ assert.equal(erased.result.metadata.latitude, null)
+ assert.equal((erased.result.metadata.exif as any)?.GPSLatitude, undefined)
+ const sanitized = erased.derivatives.find(
+ (d) => d.role === 'sanitized-original',
+ )
+ assert(sanitized)
+ const sanitizedFile = path.join(dir, 'erased.jpg')
+ await writeFile(sanitizedFile, sanitized.buffer)
+ assert.equal((await tool.read(sanitizedFile)).GPSLatitude, undefined)
+ } finally {
+ globalThis.fetch = originalFetch
+ await tool.end()
+ await rm(dir, { recursive: true, force: true })
+ }
+})
+test('BMP passes the processing adapter instead of failing the Sharp preflight', async () => {
+ const originalFetch = globalThis.fetch
+ try {
+ const input = bmp.encode({
+ data: Buffer.alloc(4 * 4 * 4, 255),
+ width: 4,
+ height: 4,
+ }).data
+ globalThis.fetch = async () => new Response(new Uint8Array(input))
+ const result = await processWork(
+ {
+ lease: { ...lease, expectedBytes: input.length },
+ sourceUrl: 'https://storage.example/source',
+ fileName: 'photo.bmp',
+ contentType: 'image/bmp',
+ },
+ new AbortController().signal,
+ )
+ assert.equal(result.result.width, 4)
+ assert.equal(result.result.height, 4)
+ } finally {
+ globalThis.fetch = originalFetch
+ }
+})
+
+test('explicit MOV attachment overrides an embedded motion video and preserves edited metadata', async () => {
+ const originalFetch = globalThis.fetch
+ try {
+ const jpeg = await sharp({
+ create: { width: 16, height: 12, channels: 3, background: '#789abc' },
+ })
+ .jpeg()
+ .toBuffer()
+ const embedded = Buffer.alloc(9000)
+ embedded.writeUInt32BE(24, 0)
+ embedded.write('ftypisom', 4)
+ const input = Buffer.concat([jpeg, embedded]),
+ companion = Buffer.from('external-mov')
+ globalThis.fetch = async (url) =>
+ new Response(
+ new Uint8Array(String(url).endsWith('/video') ? companion : input),
+ )
+ const result = await processWork(
+ {
+ lease: { ...lease, expectedBytes: input.length },
+ sourceUrl: 'https://storage.example/source',
+ fileName: 'source.jpg',
+ contentType: 'image/jpeg',
+ companion: {
+ key: 'video.MOV',
+ url: 'https://storage.example/video',
+ bytes: companion.length,
+ },
+ options: { operation: 'live-photo', options_json: '{}' },
+ priorMetadata: {
+ title: 'Edited',
+ metadata_json: JSON.stringify({
+ description: 'Keep me',
+ tags: ['edited'],
+ dateTaken: '2020-01-01',
+ exif: { Rating: 5 },
+ }),
+ },
+ },
+ new AbortController().signal,
+ )
+ const videos = result.derivatives.filter((d) => d.role === 'live-video')
+ assert.equal(videos.length, 1)
+ assert.equal(videos[0]?.contentType, 'video/quicktime')
+ assert.deepEqual(videos[0]?.buffer, companion)
+ assert.equal(result.result.metadata.title, 'Edited')
+ assert.equal((result.result.metadata.exif as any).Rating, 5)
+ } finally {
+ globalThis.fetch = originalFetch
+ }
+})
diff --git a/tests/processor-migration.test.ts b/tests/processor-migration.test.ts
new file mode 100644
index 00000000..5ae87e1c
--- /dev/null
+++ b/tests/processor-migration.test.ts
@@ -0,0 +1,33 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import Database from 'better-sqlite3'
+import { readFile, readdir } from 'node:fs/promises'
+test('processing node migration preserves existing published assets and their foreign keys', async () => {
+ const db = new Database(':memory:')
+ try {
+ for (const file of (await readdir('packages/platform/migrations'))
+ .filter((f) => f.endsWith('.sql') && f < '0036')
+ .sort())
+ db.exec(await readFile('packages/platform/migrations/' + file, 'utf8'))
+ db.exec(
+ "INSERT INTO platform_workspaces(id,name) VALUES(1,'Existing'); INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'Existing'); INSERT INTO cloud_photos(site_id,id,source_version,title) VALUES(1,'photo','v1','Photo'); INSERT INTO cloud_media_assets(site_id,photo_id,source_version,pipeline_version,role,connection_id,object_key,bytes,content_type,connection_revision) VALUES(1,'photo','v1','1','thumbnail','storage','thumbnail.webp',123,'image/webp',7)",
+ )
+ const before = db.prepare('SELECT * FROM cloud_media_assets').get()
+ db.exec(
+ await readFile(
+ 'packages/platform/migrations/0036_processors.sql',
+ 'utf8',
+ ),
+ )
+ assert.deepEqual(
+ db.prepare('SELECT * FROM cloud_media_assets').get(),
+ before,
+ )
+ assert.deepEqual(db.prepare('PRAGMA foreign_key_check').all(), [])
+ db.prepare(
+ "INSERT INTO cloud_media_assets(site_id,photo_id,source_version,pipeline_version,role,connection_id,object_key,bytes,content_type) VALUES(1,'photo','v1','2','sanitized-original','storage','sanitized.jpg',234,'image/jpeg')",
+ ).run()
+ } finally {
+ db.close()
+ }
+})
diff --git a/tests/processor-nodes.test.ts b/tests/processor-nodes.test.ts
new file mode 100644
index 00000000..2453668f
--- /dev/null
+++ b/tests/processor-nodes.test.ts
@@ -0,0 +1,350 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import ts from 'typescript'
+import { readSiteBody } from '../apps/shared/platform/json.ts'
+import { createRequire } from 'node:module'
+import { readFile, readdir } from 'node:fs/promises'
+import { splitSqlQuery } from './helpers/sql.ts'
+import {
+ processorCipher,
+ processorKeyHash,
+ DEFAULT_PROCESSOR_LIMITS,
+ parseProcessorLimits,
+ parseProcessorGeocoding,
+ DEFAULT_PROCESSOR_GEOCODING,
+} from '../packages/contracts/src/processor.ts'
+import {
+ authenticateProcessor,
+ processorEnvelope,
+} from '../packages/platform/src/processors.ts'
+import { createCloudMediaRepository } from '../packages/database/src/cloud-media.ts'
+import { createCloudExecutionRepository } from '../packages/database/src/cloud-execution.ts'
+import { queuePhotoProcessing } from '../apps/cloud/server/utils/cloud-photo-processing.ts'
+const require = createRequire(import.meta.url)
+const { Miniflare, convertV4MiniflareOptions } = createRequire(
+ require.resolve('wrangler/package.json'),
+)('miniflare')
+
+test('node configuration is authenticated, key-separated and bound to node, revision and expiry', async () => {
+ const key = 'x'.repeat(48),
+ other = 'y'.repeat(48),
+ context = 'node-a:1:12345',
+ cipher = await processorCipher(key)
+ const value = await cipher.seal({ token: 'private-token' }, context)
+ assert(!value.includes('private-token'))
+ assert.deepEqual(await cipher.open(value, context), {
+ token: 'private-token',
+ })
+ await assert.rejects(cipher.open(value, 'node-b:1:12345'))
+ await assert.rejects(cipher.open(value, 'node-a:2:12345'))
+ await assert.rejects((await processorCipher(other)).open(value, context))
+ await assert.rejects(cipher.open(value + 'extra', context))
+ assert.throws(() =>
+ parseProcessorLimits({
+ ...DEFAULT_PROCESSOR_LIMITS,
+ maxFileBytes: Infinity,
+ }),
+ )
+ assert.throws(() =>
+ parseProcessorGeocoding({
+ ...DEFAULT_PROCESSOR_GEOCODING,
+ provider: 'mapbox',
+ }),
+ )
+})
+
+test('registered node auth revokes old keys; full schema preserves assets and fences maintenance publication', async () => {
+ const mf = new Miniflare(
+ convertV4MiniflareOptions({
+ modules: true,
+ compatibilityDate: '2026-10-26',
+ script: 'export default {fetch(){return new Response("ok")}}',
+ d1Databases: ['DB'],
+ }),
+ )
+ try {
+ const db = await mf.getD1Database('DB')
+ for (const file of (await readdir('packages/platform/migrations'))
+ .filter((f) => f.endsWith('.sql'))
+ .sort())
+ for (const sql of splitSqlQuery(
+ (
+ await readFile('packages/platform/migrations/' + file, 'utf8')
+ ).replace(/^--.*$/gm, ''),
+ ))
+ if (sql.trim()) await db.prepare(sql).run()
+ await db
+ .prepare(
+ "INSERT INTO platform_users(id,email,created_at) VALUES(1,'[email protected]',1)",
+ )
+ .run()
+ await db
+ .prepare("INSERT INTO platform_workspaces(id,name) VALUES(1,'One')")
+ .run()
+ await db
+ .prepare(
+ "INSERT INTO platform_sites(id,workspace_id,name) VALUES(1,1,'One'),(2,1,'Two')",
+ )
+ .run()
+ const id = crypto.randomUUID(),
+ key = 'pn_' + id + '.' + 'a'.repeat(43),
+ next = 'pn_' + id + '.' + 'b'.repeat(43)
+ await db
+ .prepare(
+ 'INSERT INTO platform_processor_nodes(id,name,secret_hash,limits_json,created_at) VALUES(?,?,?,?,?)',
+ )
+ .bind(
+ id,
+ 'A',
+ await processorKeyHash(key),
+ JSON.stringify(DEFAULT_PROCESSOR_LIMITS),
+ 1,
+ )
+ .run()
+ const node = await authenticateProcessor(db, key)
+ assert(node)
+ assert.equal(
+ await authenticateProcessor(db, 'old-global-shared-secret'),
+ null,
+ )
+ const source = (
+ await readFile(
+ 'apps/cloud/server/api/internal/worker/[action].post.ts',
+ 'utf8',
+ )
+ ).replace(/^import[\s\S]*?from ['"][^'"]+['"]\s*\n/gm, '')
+ const deps = {
+ authenticateProcessor,
+ processorEnvelope,
+ parseProcessorLimits,
+ createCloudExecutionRepository,
+ definePlatformHandler: (f: any) => f,
+ useRuntimeConfig: () => ({
+ platform: {
+ storageSecret: 's'.repeat(32),
+ portalOrigin: 'https://app.example',
+ s3EndpointHosts: '',
+ },
+ }),
+ platformPortalOrigin: () => {},
+ platformDatabase: () => db,
+ getHeader: (e: any, name: string) =>
+ name === 'content-type'
+ ? 'application/json'
+ : name === 'authorization'
+ ? 'Bearer ' + e.key
+ : undefined,
+ getRouterParam: (e: any) => e.action,
+ getRequestWebStream: (e: any) =>
+ new Response(JSON.stringify(e.body ?? {})).body,
+ readSiteBody,
+ createError: (v: any) => Object.assign(new Error(v.statusMessage), v),
+ }
+ const code = ts.transpileModule(source, {
+ compilerOptions: {
+ module: ts.ModuleKind.CommonJS,
+ target: ts.ScriptTarget.ES2022,
+ },
+ }).outputText
+ const api = new Function(
+ 'exports',
+ ...Object.keys(deps),
+ code + ';return exports.default',
+ )({}, ...Object.values(deps))
+ await assert.rejects(
+ api({ action: 'register', key: 'legacy-shared-key' }),
+ { statusCode: 401 },
+ )
+ const registered = await api({
+ action: 'register',
+ key,
+ body: {
+ version: 'test-build',
+ instanceId: 'instance-a',
+ hostname: 'host',
+ metrics: { rss: 12345, load1: 0.1 },
+ },
+ })
+ assert.equal(registered.nodeId, id)
+ const statusRow = await db
+ .prepare(
+ 'SELECT version,last_seen_at,metrics_json FROM platform_processor_nodes WHERE id=?',
+ )
+ .bind(id)
+ .first()
+ assert.equal(statusRow.version, 'test-build')
+ assert(statusRow.last_seen_at > 0)
+ assert.equal(JSON.parse(statusRow.metrics_json).rss, 12345)
+ assert(!JSON.stringify(registered).includes('token'))
+ const firstSlot = await api({
+ action: 'geocoding-slot',
+ key,
+ body: { provider: 'nominatim' },
+ })
+ const nextSlot = await api({
+ action: 'geocoding-slot',
+ key,
+ body: { provider: 'nominatim' },
+ })
+ assert.equal(firstSlot.delayMs, 0)
+ assert(nextSlot.delayMs > 500)
+ await assert.rejects(
+ api({
+ action: 'heartbeat',
+ key,
+ body: { lease: { siteId: 1, id: 'foreign', workerId: 'another-node' } },
+ }),
+ { statusCode: 403 },
+ )
+ const config = await processorEnvelope(db, 's'.repeat(32), key, node, 100)
+ const decoded = await (
+ await processorCipher(key)
+ ).open<any>(config.envelope, `${id}:${config.revision}:${config.expiresAt}`)
+ assert.equal(decoded.limits.maxPixels, DEFAULT_PROCESSOR_LIMITS.maxPixels)
+ await db
+ .prepare('UPDATE platform_processor_nodes SET secret_hash=? WHERE id=?')
+ .bind(await processorKeyHash(next), id)
+ .run()
+ assert.equal(await authenticateProcessor(db, key), null)
+ assert(await authenticateProcessor(db, next))
+ await db
+ .prepare('UPDATE platform_processor_nodes SET enabled=0 WHERE id=?')
+ .bind(id)
+ .run()
+ assert.equal(await authenticateProcessor(db, next), null)
+ const queue = createCloudExecutionRepository(db, { id: 1 }),
+ media = createCloudMediaRepository(db, { id: 1 })
+ await media.reserveUpload({
+ id: 'photo',
+ ownerId: 1,
+ photoId: 'photo',
+ sourceVersion: 'v1',
+ connectionId: 's3',
+ objectKey: 'original.jpg',
+ providerUploadId: null,
+ expectedBytes: 10,
+ expiresAt: 100000,
+ })
+ await media.completeUpload({
+ id: 'photo',
+ ownerId: 1,
+ jobId: 'photo',
+ pipelineVersion: '1',
+ title: 'Photo',
+ now: 1,
+ })
+ const lease = await queue.claim(id, 'attempt', 10, 1000)
+ assert(lease)
+ const firstResult = {
+ width: 10,
+ height: 10,
+ metadata: {
+ latitude: 31,
+ longitude: 121,
+ geocoding: {
+ version: 1,
+ provider: 'mapbox',
+ language: 'en',
+ components: {},
+ },
+ },
+ assets: [
+ {
+ role: 'thumbnail' as const,
+ objectKey: `chronoframe/1/derived/photo/v1/1/attempt/thumbnail.webp`,
+ bytes: 5,
+ contentType: 'image/webp',
+ },
+ ],
+ }
+ assert(await queue.complete(lease, firstResult, 20))
+ const originalRevision = (
+ await db
+ .prepare("SELECT revision FROM cloud_photos WHERE id='photo'")
+ .first()
+ ).revision
+ await queuePhotoProcessing(db, 1, 'photo', 'geocoding', originalRevision)
+ await assert.rejects(
+ queuePhotoProcessing(db, 1, 'photo', 'reindex', originalRevision + 1),
+ )
+ const maintenance = await queue.claim(id, 'maintenance', Date.now(), 120000)
+ assert(maintenance)
+ // Editing after queueing must invalidate the result without losing the previous publication.
+ await db
+ .prepare(
+ "UPDATE cloud_photos SET revision=revision+1,title='Edited' WHERE id='photo'",
+ )
+ .run()
+ const guarded = createCloudExecutionRepository(
+ db,
+ { id: 1 },
+ DEFAULT_PROCESSOR_LIMITS,
+ originalRevision + 1,
+ )
+ assert.equal(
+ await guarded.complete(
+ maintenance,
+ {
+ ...firstResult,
+ assets: [
+ {
+ ...firstResult.assets[0],
+ objectKey: `chronoframe/1/derived/photo/v1/${maintenance.pipelineVersion}/maintenance/thumbnail.webp`,
+ },
+ ],
+ },
+ Date.now(),
+ ),
+ false,
+ )
+ assert.equal(
+ (
+ await db
+ .prepare("SELECT title FROM cloud_photos WHERE id='photo'")
+ .first()
+ ).title,
+ 'Edited',
+ )
+ assert.equal(
+ (await db.prepare('SELECT COUNT(*) n FROM cloud_media_assets').first()).n,
+ 1,
+ )
+ await db
+ .prepare('UPDATE platform_processor_nodes SET enabled=1 WHERE id=?')
+ .bind(id)
+ .run()
+ const transferred = await api({
+ action: 'fail',
+ key: next,
+ body: {
+ lease: maintenance,
+ code: 'node_pixel_limit',
+ retryable: true,
+ requiredPixels: 200000000,
+ },
+ })
+ assert(transferred.ok)
+ const requirements = await db
+ .prepare('SELECT required_pixels,attempts FROM cloud_processing_jobs')
+ .first()
+ assert.equal(requirements.required_pixels, 200000000)
+ assert.equal(requirements.attempts, 0)
+ assert.equal(
+ await createCloudExecutionRepository(
+ db,
+ { id: 1 },
+ DEFAULT_PROCESSOR_LIMITS,
+ ).claim('small', 'small-attempt', Date.now() + 4000, 120000),
+ null,
+ )
+ assert(
+ await createCloudExecutionRepository(
+ db,
+ { id: 1 },
+ { ...DEFAULT_PROCESSOR_LIMITS, maxPixels: 250000000 },
+ ).claim('large', 'large-attempt', Date.now() + 4000, 120000),
+ )
+ } finally {
+ await mf.dispose()
+ }
+})