Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
refactor: extract processing and fenced embedded jobs
HoshinoSuzumiBot326a15b32dcf
diff --git a/Dockerfile b/Dockerfileindex ff331d6b..1fedbc9f 100644--- a/Dockerfile+++ b/Dockerfile@@ -10,6 +10,8 @@ COPY packages/webgl-image/package.json ./packages/webgl-image/COPY packages/contracts/package.json ./packages/contracts/COPY packages/core/package.json ./packages/core/COPY packages/database/package.json ./packages/database/+COPY packages/jobs/package.json ./packages/jobs/+COPY packages/processing/package.json ./packages/processing/COPY packages/storage/package.json ./packages/storage/COPY packages/media/package.json ./packages/media/COPY packages/geocoding/package.json ./packages/geocoding/diff --git a/nuxt.config.ts b/nuxt.config.tsindex 039d65b3..e893147d 100644--- a/nuxt.config.ts+++ b/nuxt.config.ts@@ -157,6 +157,7 @@ export default defineNuxtConfig({'dayjs/plugin/timezone', // CJS'dayjs/plugin/updateLocale', // CJS'dayjs/plugin/utc', // CJS+ 'driver.js','es-toolkit','file-type','mapbox-gl', // CJSdiff --git a/package.json b/package.jsonindex 06f00432..0b9484f0 100644--- a/package.json+++ b/package.json@@ -33,7 +33,7 @@"docs:dev": "vitepress dev docs","docs:build": "vitepress build docs","docs:preview": "vitepress preview docs",- "check:packages": "node scripts/check-package-boundaries.mjs && pnpm --filter @chronoframe/database --filter @chronoframe/core --filter @chronoframe/contracts --filter @chronoframe/storage --filter @chronoframe/media --filter @chronoframe/geocoding type-check",+ "check:packages": "node scripts/check-package-boundaries.mjs && pnpm --filter @chronoframe/jobs --filter @chronoframe/processing --filter @chronoframe/database --filter @chronoframe/core --filter @chronoframe/contracts --filter @chronoframe/storage --filter @chronoframe/media --filter @chronoframe/geocoding type-check","test": "node --experimental-strip-types --test tests/*.test.ts"},"dependencies": {@@ -89,7 +89,9 @@"@chronoframe/geocoding": "workspace:*","@chronoframe/core": "workspace:*","h3": "^1.15.11",- "@chronoframe/database": "workspace:*"+ "@chronoframe/database": "workspace:*",+ "@chronoframe/jobs": "workspace:*",+ "@chronoframe/processing": "workspace:*"},"packageManager": "[email protected]","devDependencies": {diff --git a/packages/contracts/src/queue.ts b/packages/contracts/src/queue.tsindex 3973a38c..91056613 100644--- a/packages/contracts/src/queue.ts+++ b/packages/contracts/src/queue.ts@@ -1,3 +1,5 @@+import { z } from 'zod'+/*** Warning codes attached to completed pipeline queue tasks.*@@ -15,3 +17,96 @@ export type QueueWarningCode =export const QUEUE_WARNING_CODES: [QueueWarningCode, ...QueueWarningCode[]] = [QueueWarningCode.AlbumMissing,]++export const queuePayloadSchema = z.discriminatedUnion('type', [+ z.object({ type: z.literal('location-migration') }),+ z.object({+ type: z.literal('photo'),+ storageKey: z.string().nonempty(),+ eraseLocation: z.boolean().optional(),+ albumId: z.number().int().positive().optional(),+ }),+ z+ .object({+ type: z.literal('live-photo-video'),+ storageKey: z.string().nonempty(),+ })+ .strict(),+ z.object({+ type: z.literal('photo-reverse-geocoding'),+ photoId: z.string().min(1),+ latitude: z.number().min(-90).max(90).optional(),+ longitude: z.number().min(-180).max(180).optional(),+ }),+ z.object({+ type: z.literal('photo-erase-location'),+ photoId: z.string().min(1),+ }),+])++export type PipelineQueuePayload =+ | { type: 'location-migration' }+ | {+ type: 'photo'+ storageKey: string+ eraseLocation?: boolean+ albumId?: number+ }+ | {+ type: 'live-photo-video'+ storageKey: string+ }+ | {+ type: 'photo-reverse-geocoding'+ migrationOnly?: boolean+ photoId: string+ latitude?: number | null+ longitude?: number | null+ }+ | {+ type: 'photo-erase-location'+ photoId: string+ }++export type ProcessingStage =+ | 'preprocessing'+ | 'metadata'+ | 'thumbnail'+ | 'exif'+ | 'motion-photo'+ | 'reverse-geocoding'+ | 'live-photo'+ | 'location-erase'+/** Persist only operational references; credentials belong to versioned storage connections. */+export const jobPayloadSchema = z.discriminatedUnion('type', [+ z.object({ type: z.literal('location-migration') }).strict(),+ z+ .object({+ type: z.literal('photo'),+ storageKey: z.string().min(1).max(2048),+ eraseLocation: z.boolean().optional(),+ albumId: z.number().int().positive().optional(),+ })+ .strict(),+ z+ .object({+ type: z.literal('live-photo-video'),+ storageKey: z.string().min(1).max(2048),+ })+ .strict(),+ z+ .object({+ type: z.literal('photo-reverse-geocoding'),+ photoId: z.string().min(1).max(256),+ migrationOnly: z.boolean().optional(),+ latitude: z.number().min(-90).max(90).nullable().optional(),+ longitude: z.number().min(-180).max(180).nullable().optional(),+ })+ .strict(),+ z+ .object({+ type: z.literal('photo-erase-location'),+ photoId: z.string().min(1).max(256),+ })+ .strict(),+])diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 419f3129..1e7dbbe7 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -146,3 +146,4 @@ export * from './access.ts'export * from './upload-preparation.ts'export * from './site.ts'+export * from './publication.ts'diff --git a/packages/core/src/mutations.ts b/packages/core/src/mutations.tsindex 9776c43c..0ad28449 100644--- a/packages/core/src/mutations.ts+++ b/packages/core/src/mutations.ts@@ -145,16 +145,26 @@ export type PhotoUpdate = Partial<| 'geocoding'>>+export type PhotoRecord = Photo & {+ originalDerivedKey?: string | null+ storageReference?: { connectionId: number | null; revision: number }+}export interface PhotoWriteRepository {- get(id: string): Promise<Photo | null>- update(id: string, update: PhotoUpdate): Promise<Photo | null>- delete(id: string): Promise<void>+ get(id: string): Promise<PhotoRecord | null>+ beginMutation?(id: string): Promise<void>+ /** Persist a requested follow-up task in the same transaction as the resource update. */+ update(+ id: string,+ update: PhotoUpdate,+ reverseGeocode?: { latitude: number; longitude: number },+ ): Promise<Photo | null>+ delete(id: string): Promise<PhotoRecord | null>}export interface PhotoObjectStorage {delete(key: string): Promise<void>}export interface StorageResolver {- forPhoto(photo: Photo): Promise<PhotoObjectStorage>+ forPhoto(photo: PhotoRecord): Promise<PhotoObjectStorage>}export interface PhotoTaskDispatcher {reverseGeocode(@@ -194,7 +204,6 @@ export function createPhotoWriter(deps: {photo: Photo,input: PhotoMetadataInput,): Promise<Pick<Photo, 'exif' | 'fileSize'>>- tasks: PhotoTaskDispatcherwarn(error: unknown, operation: string): void}) {return {@@ -225,6 +234,7 @@ export function createPhotoWriter(deps: {'dashboard.photos.messages.noStorageKey',)const input = normalizePhotoMetadata(raw)+ await deps.repository.beginMutation?.(id)const edited = await deps.editMetadata(photo, input)const update: PhotoUpdate = {exif: edited.exif,@@ -247,7 +257,11 @@ export function createPhotoWriter(deps: {}let updated: Photo | nulltry {- updated = await deps.repository.update(id, update)+ updated = await deps.repository.update(+ id,+ update,+ input.location ?? undefined,+ )} catch (error) {deps.warn(error, 'update-photo-record')throw new CoreError(@@ -256,27 +270,21 @@ export function createPhotoWriter(deps: {'dashboard.photos.messages.metadataUpdateFailed',)}- if (input.location) {- try {- await deps.tasks.reverseGeocode(id, input.location)- } catch (error) {- deps.warn(error, 'reverse-geocoding')- }- }return { success: true, photo: updated || undefined }},async delete(id: string, access: MutationAccess) {authorize(access)- const photo = await deps.repository.get(id)+ const photo = await deps.repository.delete(id)if (!photo) throw new CoreError(404, 'Photo not found')const keys = [photo.storageKey,photo.thumbnailKey,photo.livePhotoVideoKey,+ photo.originalDerivedKey,]const match =photo.storageKey && /\.(heic|heif|hif)$/i.exec(photo.storageKey)- if (match)+ if (match && !photo.originalDerivedKey)keys.push(photo.storageKey!.slice(0, -match[0].length) + '.jpeg')const storage = await deps.storage.forPhoto(photo)for (const key of new Set(@@ -288,7 +296,6 @@ export function createPhotoWriter(deps: {deps.warn(error, 'delete-photo-object')}}- await deps.repository.delete(id)return { statusCode: 200, statusMessage: 'Photo deleted successfully' }},}diff --git a/packages/core/src/publication.ts b/packages/core/src/publication.tsnew file mode 100644index 00000000..d82ce786--- /dev/null+++ b/packages/core/src/publication.ts@@ -0,0 +1,57 @@+import type { Photo } from '@chronoframe/contracts/resources'+import { CoreError } from './errors.ts'++/** Verify the business identity and object ownership before accepting a processing result. */+export function validatePhotoPublication(+ photo: Photo & { originalDerivedKey?: string | null },+ expected: {+ resourceId: string+ sourceKey: string+ generatedKeys: ReadonlySet<string>+ linkedVideoKeys: ReadonlySet<string>+ publicUrl(key: string): string+ },+) {+ const reject = () => {+ throw new CoreError(409, 'Processing result does not match its resource')+ }+ if (+ photo.id !== expected.resourceId ||+ photo.storageKey !== expected.sourceKey+ )+ reject()+ if (+ !Number.isSafeInteger(photo.width) ||+ !Number.isSafeInteger(photo.height) ||+ photo.width! <= 0 ||+ photo.height! <= 0 ||+ !Number.isFinite(photo.aspectRatio) ||+ Math.abs(photo.aspectRatio! - photo.width! / photo.height!) > 0.000001+ )+ reject()+ if (!Number.isSafeInteger(photo.fileSize) || photo.fileSize! < 0) reject()+ if (!photo.thumbnailKey || !expected.generatedKeys.has(photo.thumbnailKey))+ reject()+ if (+ photo.originalDerivedKey &&+ !expected.generatedKeys.has(photo.originalDerivedKey)+ )+ reject()+ if (+ photo.livePhotoVideoKey &&+ !expected.generatedKeys.has(photo.livePhotoVideoKey) &&+ !expected.linkedVideoKeys.has(photo.livePhotoVideoKey)+ )+ reject()+ if (+ photo.originalUrl !==+ expected.publicUrl(photo.originalDerivedKey ?? expected.sourceKey) ||+ photo.thumbnailUrl !== expected.publicUrl(photo.thumbnailKey!)+ )+ reject()+ if (+ photo.livePhotoVideoKey &&+ photo.livePhotoVideoUrl !== expected.publicUrl(photo.livePhotoVideoKey)+ )+ reject()+}diff --git a/packages/database/README.md b/packages/database/README.mdindex d2409ae1..cccc8686 100644--- a/packages/database/README.md+++ b/packages/database/README.md@@ -5,3 +5,9 @@ SQLite schema, migration history, DTO projection and repository implementationsCore repository methods return promises so callers await persistence; SQLite executes each transaction synchronously and never holds it across asynchronous object I/O. A future PostgreSQL adapter must implement the same atomic repository operations, rather than exposing driver transactions to core. `persistenceErrorCode` maps constraint and contention failures without returning raw SQL messages to clients.Migrations are canonical here. `server/database/migrations` remains a compatibility symlink for existing commands; production images copy these files to the historical runtime path. Do not edit released migrations. Back up the database and its committed WAL with the SQLite backup API before upgrading. Restore a backup using the matching application version when a migration changes constraints.++The default self-hosted Site is fixed by its entry, while every site repository requires an explicit existing Site. Migration creates default Workspace/Site memberships and enforces immutable ownership and same-site relationships. Settings uniqueness and caches include Site. Historical storage connection configurations are retained as immutable revisions for existing object references.++`pnpm db:migrate` and application startup use the same journal-compatible runner. Existing files with pending migrations receive a consistent `*.pre-migration-*.sqlite3` backup before changes. Schema and journal updates share a transaction, foreign keys are checked before commit, and enforcement is restored afterwards. Keep backups with their matching application versions; restoring only an older binary does not reverse schema constraints.++Jobs use source generations, retained deletion tombstones, leases, attempt tokens and input digests. Resource mutation and follow-up task creation share one transaction. Publication guards live only inside the publication transaction and roll back with it; workers never hold database transactions during native processing or storage I/O.diff --git a/packages/database/migrations/0019_job_leases.sql b/packages/database/migrations/0019_job_leases.sqlnew file mode 100644index 00000000..92a8156b--- /dev/null+++ b/packages/database/migrations/0019_job_leases.sql@@ -0,0 +1,61 @@+ALTER TABLE pipeline_queue ADD COLUMN source_hash TEXT;+--> statement-breakpoint+ALTER TABLE pipeline_queue ADD COLUMN resource_id TEXT;+--> statement-breakpoint+ALTER TABLE pipeline_queue ADD COLUMN pipeline_version TEXT NOT NULL DEFAULT '1';+--> statement-breakpoint+ALTER TABLE pipeline_queue ADD COLUMN attempt_id TEXT;+--> statement-breakpoint+ALTER TABLE pipeline_queue ADD COLUMN worker_id TEXT;+--> statement-breakpoint+ALTER TABLE pipeline_queue ADD COLUMN lease_expires_at INTEGER;+--> statement-breakpoint+ALTER TABLE pipeline_queue ADD COLUMN heartbeat_at INTEGER;+--> statement-breakpoint+ALTER TABLE photos ADD COLUMN original_derived_key TEXT;+--> statement-breakpoint+CREATE TABLE resource_versions (+ site_id INTEGER NOT NULL REFERENCES sites(id), resource_id TEXT NOT NULL,+ version INTEGER NOT NULL DEFAULT 1, deleted INTEGER NOT NULL DEFAULT 0,+ storage_connection_id INTEGER, storage_revision INTEGER NOT NULL DEFAULT 1, storage_key TEXT,+ PRIMARY KEY(site_id,resource_id)+);+--> statement-breakpoint+CREATE TABLE job_commit_guards (site_id INTEGER NOT NULL,resource_id TEXT NOT NULL,attempt_id TEXT NOT NULL,PRIMARY KEY(site_id,resource_id));+--> statement-breakpoint+INSERT INTO resource_versions(site_id,resource_id,version,storage_connection_id,storage_revision,storage_key)+ SELECT site_id,id,source_version,storage_connection_id,storage_revision,storage_key FROM photos;+--> statement-breakpoint+UPDATE pipeline_queue SET resource_id=json_extract(payload,'$.photoId') WHERE json_extract(payload,'$.photoId') IS NOT NULL;+--> statement-breakpoint+UPDATE pipeline_queue SET resource_id=(SELECT id FROM photos WHERE photos.site_id=pipeline_queue.site_id AND photos.storage_key=json_extract(pipeline_queue.payload,'$.storageKey') LIMIT 1) WHERE resource_id IS NULL AND json_extract(payload,'$.type')='photo';+--> statement-breakpoint+CREATE INDEX idx_pipeline_queue_lease ON pipeline_queue(site_id,status,lease_expires_at);+--> statement-breakpoint+CREATE INDEX idx_pipeline_queue_resource ON pipeline_queue(site_id,resource_id,source_version);+--> statement-breakpoint+CREATE TRIGGER photo_version_insert AFTER INSERT ON photos BEGIN+ INSERT INTO resource_versions(site_id,resource_id,version,deleted,storage_connection_id,storage_revision,storage_key)+ VALUES(NEW.site_id,NEW.id,NEW.source_version,0,NEW.storage_connection_id,NEW.storage_revision,NEW.storage_key)+ ON CONFLICT(site_id,resource_id) DO UPDATE SET+ version=CASE WHEN EXISTS(SELECT 1 FROM job_commit_guards WHERE site_id=NEW.site_id AND resource_id=NEW.id) THEN resource_versions.version ELSE resource_versions.version+1 END,+ deleted=0, storage_connection_id=NEW.storage_connection_id,storage_revision=NEW.storage_revision,storage_key=NEW.storage_key;+ UPDATE photos SET source_version=(SELECT version FROM resource_versions WHERE site_id=NEW.site_id AND resource_id=NEW.id) WHERE id=NEW.id;+END;+--> statement-breakpoint+CREATE TRIGGER photo_version_update AFTER UPDATE OF title,description,tags,exif,latitude,longitude,country,city,location_name,geocoding,storage_key,storage_connection_id,storage_revision,last_modified,file_size,is_live_photo,live_photo_video_key,original_url,thumbnail_key,original_derived_key ON photos+WHEN NOT EXISTS(SELECT 1 FROM job_commit_guards WHERE site_id=NEW.site_id AND resource_id=NEW.id)+BEGIN+ UPDATE resource_versions SET version=version+1,storage_connection_id=NEW.storage_connection_id,storage_revision=NEW.storage_revision,storage_key=NEW.storage_key WHERE site_id=NEW.site_id AND resource_id=NEW.id;+ UPDATE photos SET source_version=(SELECT version FROM resource_versions WHERE site_id=NEW.site_id AND resource_id=NEW.id) WHERE id=NEW.id;+END;+--> statement-breakpoint+CREATE TRIGGER photo_version_delete AFTER DELETE ON photos BEGIN+ UPDATE resource_versions SET version=version+1,deleted=1 WHERE site_id=OLD.site_id AND resource_id=OLD.id;+ UPDATE pipeline_queue SET status='failed',error_message='Resource deleted',status_stage=NULL,lease_expires_at=NULL+ WHERE site_id=OLD.site_id AND resource_id=OLD.id AND status IN ('pending','in-stages');+END;+--> statement-breakpoint+CREATE TRIGGER resource_reference_insert BEFORE INSERT ON resource_versions WHEN NEW.storage_connection_id IS NOT NULL AND NOT EXISTS(SELECT 1 FROM storage_connection_versions WHERE connection_id=NEW.storage_connection_id AND revision=NEW.storage_revision AND site_id=NEW.site_id) BEGIN SELECT RAISE(ABORT,'cross-site or missing storage revision'); END;+--> statement-breakpoint+CREATE TRIGGER resource_reference_update BEFORE UPDATE ON resource_versions WHEN NEW.storage_connection_id IS NOT NULL AND NOT EXISTS(SELECT 1 FROM storage_connection_versions WHERE connection_id=NEW.storage_connection_id AND revision=NEW.storage_revision AND site_id=NEW.site_id) BEGIN SELECT RAISE(ABORT,'cross-site or missing storage revision'); END;diff --git a/packages/database/migrations/meta/_journal.json b/packages/database/migrations/meta/_journal.jsonindex 48f920b6..8a84461a 100644--- a/packages/database/migrations/meta/_journal.json+++ b/packages/database/migrations/meta/_journal.json@@ -134,6 +134,13 @@"when": 1792307736436,"tag": "0018_site_ownership","breakpoints": true+ },+ {+ "idx": 19,+ "version": "6",+ "when": 1792307737436,+ "tag": "0019_job_leases",+ "breakpoints": true}]}diff --git a/packages/database/package.json b/packages/database/package.jsonindex e05f9d36..98421332 100644--- a/packages/database/package.json+++ b/packages/database/package.json@@ -16,7 +16,8 @@"@chronoframe/contracts": "workspace:*","@chronoframe/core": "workspace:*","better-sqlite3": "^12.11.1",- "drizzle-orm": "^0.45.4"+ "drizzle-orm": "^0.45.4",+ "@chronoframe/jobs": "workspace:*"},"devDependencies": {"@types/better-sqlite3": "^9.6.0",diff --git a/packages/database/src/repositories.ts b/packages/database/src/repositories.tsindex 911dd07e..0efa2860 100644--- a/packages/database/src/repositories.ts+++ b/packages/database/src/repositories.ts@@ -3,3 +3,4 @@ export * from './sqlite-mutations.ts'export * from './sqlite-uploads.ts'export * from './site-repository.ts'export * from './photo-persistence.ts'+export * from './sqlite-jobs.ts'diff --git a/packages/database/src/schema.ts b/packages/database/src/schema.tsindex 1c439b06..e23a326d 100644--- a/packages/database/src/schema.ts+++ b/packages/database/src/schema.ts@@ -9,34 +9,13 @@ import {text,uniqueIndex,} from 'drizzle-orm/sqlite-core'-import { QUEUE_WARNING_CODES } from '@chronoframe/contracts/queue'+import {+ QUEUE_WARNING_CODES,+ type PipelineQueuePayload,+} from '@chronoframe/contracts/queue'import type { NeededExif } from '@chronoframe/contracts/photo'import type { StorageConfig } from '@chronoframe/contracts/storage'-type PipelineQueuePayload =- | { type: 'location-migration' }- | {- type: 'photo'- storageKey: string- eraseLocation?: boolean- albumId?: number- }- | {- type: 'live-photo-video'- storageKey: string- }- | {- type: 'photo-reverse-geocoding'- migrationOnly?: boolean- photoId: string- latitude?: number | null- longitude?: number | null- }- | {- type: 'photo-erase-location'- photoId: string- }-export const users = sqliteTable('users', {id: integer('id').primaryKey({ autoIncrement: true }),username: text('name').notNull().unique(),@@ -80,6 +59,7 @@ const siteColumn = () =>export const photos = sqliteTable('photos',{+ originalDerivedKey: text('original_derived_key'),storageConnectionId: integer('storage_connection_id'),storageRevision: integer('storage_revision').notNull().default(1),sourceVersion: integer('source_version').notNull().default(1),@@ -138,6 +118,13 @@ export const photoAccessKeys = sqliteTable(export const pipelineQueue = sqliteTable('pipeline_queue',{+ resourceId: text('resource_id'),+ sourceHash: text('source_hash'),+ pipelineVersion: text('pipeline_version').notNull().default('1'),+ attemptId: text('attempt_id'),+ workerId: text('worker_id'),+ leaseExpiresAt: integer('lease_expires_at', { mode: 'timestamp_ms' }),+ heartbeatAt: integer('heartbeat_at', { mode: 'timestamp_ms' }),storageConnectionId: integer('storage_connection_id'),storageRevision: integer('storage_revision').notNull().default(1),sourceVersion: integer('source_version').notNull().default(1),@@ -187,6 +174,12 @@ export const pipelineQueue = sqliteTable(},(t) => [index('idx_pipeline_queue_site').on(t.siteId),+ index('idx_pipeline_queue_lease').on(t.siteId, t.status, t.leaseExpiresAt),+ index('idx_pipeline_queue_resource').on(+ t.siteId,+ t.resourceId,+ t.sourceVersion,+ ),index('idx_pipeline_queue_pending').on(sql`${t.priority} DESC`, t.createdAt, t.id).where(sql`${t.status} = 'pending'`),@@ -346,3 +339,26 @@ export const storageConnectionVersions = sqliteTable(},(t) => [primaryKey({ columns: [t.connectionId, t.revision] })],)++export const resourceVersions = sqliteTable(+ 'resource_versions',+ {+ siteId: siteColumn(),+ resourceId: text('resource_id').notNull(),+ version: integer('version').notNull().default(1),+ deleted: integer('deleted', { mode: 'boolean' }).notNull().default(false),+ storageConnectionId: integer('storage_connection_id'),+ storageRevision: integer('storage_revision').notNull().default(1),+ storageKey: text('storage_key'),+ },+ (t) => [primaryKey({ columns: [t.siteId, t.resourceId] })],+)+export const jobCommitGuards = sqliteTable(+ 'job_commit_guards',+ {+ siteId: integer('site_id').notNull(),+ resourceId: text('resource_id').notNull(),+ attemptId: text('attempt_id').notNull(),+ },+ (t) => [primaryKey({ columns: [t.siteId, t.resourceId] })],+)diff --git a/packages/database/src/site-repository.ts b/packages/database/src/site-repository.tsindex dc4127b9..49c67e53 100644--- a/packages/database/src/site-repository.ts+++ b/packages/database/src/site-repository.ts@@ -1,5 +1,9 @@import { and, eq } from 'drizzle-orm'-import { assertSiteContext, type SiteContext } from '@chronoframe/core'+import {+ CoreError,+ assertSiteContext,+ type SiteContext,+} from '@chronoframe/core'import { tables, type AppDatabase } from './connection.ts'/** Fail closed when an entry supplies a missing or nonexistent Site. */@@ -19,6 +23,84 @@ export function requireSite(db: AppDatabase, site: SiteContext) {export function createSiteRepository(db: AppDatabase, site: SiteContext) {requireSite(db, site)return {+ async capturePhoto(id: string) {+ return db.transaction(() => {+ const photo = db+ .select()+ .from(tables.photos)+ .where(+ and(eq(tables.photos.siteId, site.id), eq(tables.photos.id, id)),+ )+ .get()+ if (!photo) return null+ const version = db+ .select()+ .from(tables.resourceVersions)+ .where(+ and(+ eq(tables.resourceVersions.siteId, site.id),+ eq(tables.resourceVersions.resourceId, id),+ ),+ )+ .get()+ if (+ !version ||+ version.deleted ||+ version.storageConnectionId !== photo.storageConnectionId ||+ version.storageRevision !== photo.storageRevision ||+ version.storageKey !== photo.storageKey+ )+ throw new CoreError(409, 'Photo source changed; retry the operation')+ return { photo, version: version.version }+ })+ },+ async updatePhotoSnapshot(+ id: string,+ expectedVersion: number,+ update: Partial<+ Pick<+ typeof tables.photos.$inferSelect,+ | 'exif'+ | 'title'+ | 'dateTaken'+ | 'tags'+ | 'lastModified'+ | 'isLivePhoto'+ | 'livePhotoVideoKey'+ | 'livePhotoVideoUrl'+ >+ >,+ ) {+ return db.transaction(+ () => {+ const version = db+ .select()+ .from(tables.resourceVersions)+ .where(+ and(+ eq(tables.resourceVersions.siteId, site.id),+ eq(tables.resourceVersions.resourceId, id),+ ),+ )+ .get()+ if (+ !version ||+ version.deleted ||+ version.version !== expectedVersion+ )+ throw new CoreError(409, 'Photo changed; retry the operation')+ const changed = db+ .update(tables.photos)+ .set(update)+ .where(+ and(eq(tables.photos.siteId, site.id), eq(tables.photos.id, id)),+ )+ .run().changes+ if (changed !== 1) throw new CoreError(404, 'Photo not found')+ },+ { behavior: 'immediate' },+ )+ },async photoReference(id: string) {return (dbdiff --git a/packages/database/src/sqlite-jobs.ts b/packages/database/src/sqlite-jobs.tsnew file mode 100644index 00000000..6eeb08a3--- /dev/null+++ b/packages/database/src/sqlite-jobs.ts@@ -0,0 +1,567 @@+import { randomUUID } from 'node:crypto'+import { and, asc, desc, eq, isNull, lte, or } from 'drizzle-orm'+import { type SiteContext } from '@chronoframe/core'+import {+ jobPayloadSchema,+ type PipelineQueuePayload,+} from '@chronoframe/contracts/queue'+import {+ LostLeaseError,+ PermanentJobError,+ PIPELINE_VERSION,+ retryDelay,+ type JobLease,+ type JobRepository,+} from '@chronoframe/jobs'+import { tables, type AppDatabase } from './connection.ts'+import { requireSite } from './site-repository.ts'++type Reference = {+ storageConnectionId: number | null+ storageRevision: number+ storageKey?: string | null+}+/** Invalidate before external source writes; tombstones survive deletion of the photo row. */+export function reserveResource(+ db: AppDatabase,+ site: SiteContext,+ id: string,+ reference: Reference,+) {+ requireSite(db, site)+ return db.transaction(+ () => {+ const existing = db+ .select()+ .from(tables.resourceVersions)+ .where(+ and(+ eq(tables.resourceVersions.siteId, site.id),+ eq(tables.resourceVersions.resourceId, id),+ ),+ )+ .get()+ db.insert(tables.resourceVersions)+ .values({+ siteId: site.id,+ resourceId: id,+ version: (existing?.version ?? 0) + 1,+ deleted: false,+ ...reference,+ })+ .onConflictDoUpdate({+ target: [+ tables.resourceVersions.siteId,+ tables.resourceVersions.resourceId,+ ],+ set: {+ version: (existing?.version ?? 0) + 1,+ deleted: false,+ ...reference,+ },+ })+ .run()+ return (existing?.version ?? 0) + 1+ },+ { behavior: 'immediate' },+ )+}++/** SQLite lease/CAS operations are short transactions; processing and object I/O always run outside them. */+export function createSqliteJobRepository(+ db: AppDatabase,+ site: SiteContext,+ resourceId: (payload: PipelineQueuePayload) => string | null,+ uuid: () => string = randomUUID,+) {+ requireSite(db, site)+ const queue = tables.pipelineQueue,+ versions = tables.resourceVersions+ const owned = (id: number) => and(eq(queue.siteId, site.id), eq(queue.id, id))+ const currentVersion = (id: string) =>+ db+ .select()+ .from(versions)+ .where(and(eq(versions.siteId, site.id), eq(versions.resourceId, id)))+ .get()+ const activeReference = (): Reference => {+ const setting = db+ .select({ value: tables.settings.value })+ .from(tables.settings)+ .where(+ and(+ eq(tables.settings.siteId, site.id),+ eq(tables.settings.namespace, 'storage'),+ eq(tables.settings.key, 'provider'),+ ),+ )+ .get()+ const provider = setting?.value+ ? db+ .select()+ .from(tables.settings_storage_providers)+ .where(+ and(+ eq(tables.settings_storage_providers.siteId, site.id),+ eq(tables.settings_storage_providers.id, Number(setting.value)),+ ),+ )+ .get()+ : undefined+ return {+ storageConnectionId: provider?.id ?? null,+ storageRevision: provider?.revision ?? 1,+ }+ }+ const bind = (payload: PipelineQueuePayload, reference?: Reference) => {+ const id = resourceId(payload)+ if (!id)+ return {+ resourceId: null,+ sourceVersion: 1,+ ...(reference ?? activeReference()),+ }+ let version = currentVersion(id)+ if (!version) {+ const photo = db+ .select()+ .from(tables.photos)+ .where(and(eq(tables.photos.siteId, site.id), eq(tables.photos.id, id)))+ .get()+ if ('photoId' in payload && !photo)+ throw new PermanentJobError('Photo not found')+ const ref =+ reference ??+ (photo+ ? {+ storageConnectionId: photo.storageConnectionId,+ storageRevision: photo.storageRevision,+ storageKey: photo.storageKey,+ }+ : activeReference())+ db.insert(versions)+ .values({+ siteId: site.id,+ resourceId: id,+ version: photo?.sourceVersion ?? 1,+ deleted: false,+ ...ref,+ storageKey:+ 'storageKey' in payload ? payload.storageKey : ref.storageKey,+ })+ .onConflictDoNothing()+ .run()+ version = currentVersion(id)!+ }+ if (version.deleted) throw new PermanentJobError('Resource deleted')+ return {+ resourceId: id,+ sourceVersion: version.version,+ storageConnectionId: version.storageConnectionId,+ storageRevision: version.storageRevision,+ }+ }+ const adopt = () => {+ const candidates = db+ .select()+ .from(queue)+ .where(+ and(+ eq(queue.siteId, site.id),+ isNull(queue.resourceId),+ or(eq(queue.status, 'pending'), eq(queue.status, 'in-stages')),+ ),+ )+ .all()+ for (const job of candidates) {+ if (job.payload.type === 'location-migration') continue+ try {+ const payload = jobPayloadSchema.parse(job.payload)+ db.update(queue)+ .set({ ...bind(payload), payload })+ .where(owned(job.id))+ .run()+ } catch {+ db.update(queue)+ .set({+ status: 'failed',+ errorMessage: 'Invalid or missing job resource',+ })+ .where(owned(job.id))+ .run()+ }+ }+ }+ const sameIdentity = (job: typeof queue.$inferSelect, lease: JobLease) =>+ lease.siteId === site.id &&+ job.attemptId === lease.attemptId &&+ job.workerId === lease.workerId &&+ job.resourceId === lease.resourceId &&+ job.sourceVersion === lease.sourceVersion &&+ job.pipelineVersion === lease.pipelineVersion &&+ job.storageConnectionId === lease.storageConnectionId &&+ job.storageRevision === lease.storageRevision &&+ JSON.stringify(job.payload) === JSON.stringify(lease.payload)+ const assertCurrent = (lease: JobLease, now = new Date()) => {+ if (lease.siteId !== site.id)+ throw new LostLeaseError('Job belongs to a different Site')+ const job = db.select().from(queue).where(owned(lease.id)).get()+ if (+ !job ||+ job.status !== 'in-stages' ||+ !sameIdentity(job, lease) ||+ !job.leaseExpiresAt ||+ job.leaseExpiresAt.getTime() <= now.getTime()+ )+ throw new LostLeaseError('Job lease expired or was replaced')+ if (job.pipelineVersion !== PIPELINE_VERSION)+ throw new PermanentJobError('Unsupported pipeline version')+ if (lease.resourceId) {+ const version = currentVersion(lease.resourceId)+ if (+ !version ||+ version.deleted ||+ version.version !== lease.sourceVersion+ )+ throw new LostLeaseError('Source was replaced or deleted')+ }+ return job+ }+ const commit = <T>(+ lease: JobLease,+ operation: () => T,+ options: {+ now?: Date+ resourceUpdated?: boolean+ warningCode?: () => typeof queue.$inferSelect.warningCode+ failure?: string+ } = {},+ ) =>+ db.transaction(+ () => {+ const job = db.select().from(queue).where(owned(lease.id)).get()+ if (job?.status === 'completed' && sameIdentity(job, lease))+ return { published: false as const, result: undefined }+ assertCurrent(lease, options.now)+ if (lease.resourceId)+ db.insert(tables.jobCommitGuards)+ .values({+ siteId: site.id,+ resourceId: lease.resourceId,+ attemptId: lease.attemptId,+ })+ .run()+ const result = operation()+ if (result && typeof (result as { then?: unknown }).then === 'function')+ throw new TypeError('Job commits must be synchronous')+ if (options.resourceUpdated && lease.resourceId) {+ db.update(versions)+ .set({ version: lease.sourceVersion + 1, deleted: false })+ .where(+ and(+ eq(versions.siteId, site.id),+ eq(versions.resourceId, lease.resourceId),+ ),+ )+ .run()+ const photo = db+ .select()+ .from(tables.photos)+ .where(+ and(+ eq(tables.photos.siteId, site.id),+ eq(tables.photos.id, lease.resourceId),+ ),+ )+ .get()+ if (photo) {+ db.update(versions)+ .set({+ storageConnectionId: photo.storageConnectionId,+ storageRevision: photo.storageRevision,+ storageKey: photo.storageKey,+ })+ .where(+ and(+ eq(versions.siteId, site.id),+ eq(versions.resourceId, lease.resourceId),+ ),+ )+ .run()+ db.update(tables.photos)+ .set({ sourceVersion: lease.sourceVersion + 1 })+ .where(+ and(+ eq(tables.photos.siteId, site.id),+ eq(tables.photos.id, lease.resourceId),+ ),+ )+ .run()+ }+ }+ if (lease.resourceId)+ db.delete(tables.jobCommitGuards)+ .where(+ and(+ eq(tables.jobCommitGuards.siteId, site.id),+ eq(tables.jobCommitGuards.resourceId, lease.resourceId),+ eq(tables.jobCommitGuards.attemptId, lease.attemptId),+ ),+ )+ .run()+ db.update(queue)+ .set({+ status: options.failure ? 'failed' : 'completed',+ errorMessage: options.failure ?? null,+ completedAt: options.now ?? new Date(),+ warningCode: options.warningCode?.() ?? null,+ statusStage: null,+ leaseExpiresAt: null,+ })+ .where(owned(lease.id))+ .run()+ return { published: true as const, result }+ },+ { behavior: 'immediate' },+ )+ const repository: JobRepository = {+ async claim(workerId, now, leaseMs) {+ return db.transaction(+ () => {+ adopt()+ const expired = db+ .select()+ .from(queue)+ .where(+ and(+ eq(queue.siteId, site.id),+ eq(queue.status, 'in-stages'),+ or(+ isNull(queue.leaseExpiresAt),+ lte(queue.leaseExpiresAt, now),+ ),+ ),+ )+ .all()+ for (const job of expired)+ db.update(queue)+ .set({+ status: job.attempts >= job.maxAttempts ? 'failed' : 'pending',+ statusStage: null,+ attemptId: null,+ workerId: null,+ leaseExpiresAt: null,+ availableAt: now,+ errorMessage: 'Previous lease expired',+ })+ .where(owned(job.id))+ .run()+ for (;;) {+ const job = db+ .select()+ .from(queue)+ .where(+ and(+ eq(queue.siteId, site.id),+ eq(queue.status, 'pending'),+ lte(queue.availableAt, now),+ ),+ )+ .orderBy(+ desc(queue.priority),+ asc(queue.createdAt),+ asc(queue.id),+ )+ .limit(1)+ .get()+ if (!job) return null+ const version = job.resourceId+ ? currentVersion(job.resourceId)+ : null+ const invalidPayload = !jobPayloadSchema.safeParse(job.payload)+ .success+ if (+ job.attempts >= job.maxAttempts ||+ job.pipelineVersion !== PIPELINE_VERSION ||+ invalidPayload ||+ (job.resourceId &&+ (!version ||+ version.deleted ||+ version.version !== job.sourceVersion))+ ) {+ db.update(queue)+ .set({+ status: 'failed',+ errorMessage: 'Job exhausted, unsupported or source changed',+ })+ .where(owned(job.id))+ .run()+ continue+ }+ const attemptId = uuid(),+ attempts = job.attempts + 1+ db.update(queue)+ .set({+ status: 'in-stages',+ attemptId,+ workerId,+ attempts,+ heartbeatAt: now,+ leaseExpiresAt: new Date(now.getTime() + leaseMs),+ errorMessage: null,+ warningCode: null,+ })+ .where(owned(job.id))+ .run()+ return { ...job, attemptId, workerId, attempts }+ }+ },+ { behavior: 'immediate' },+ )+ },+ async heartbeat(lease, now, leaseMs) {+ const completed = db.select().from(queue).where(owned(lease.id)).get()+ if (completed?.status === 'completed' && sameIdentity(completed, lease))+ return true+ try {+ assertCurrent(lease, now)+ } catch {+ return false+ }+ return (+ db+ .update(queue)+ .set({+ heartbeatAt: now,+ leaseExpiresAt: new Date(now.getTime() + leaseMs),+ })+ .where(+ and(+ owned(lease.id),+ eq(queue.attemptId, lease.attemptId),+ eq(queue.status, 'in-stages'),+ ),+ )+ .run().changes === 1+ )+ },+ async complete(lease, now) {+ const row = db.select().from(queue).where(owned(lease.id)).get()+ if (row?.status === 'completed' && sameIdentity(row, lease)) return true+ try {+ return commit(lease, () => undefined, { now }).published+ } catch (error) {+ if (error instanceof LostLeaseError) return false+ throw error+ }+ },+ async fail(lease, error, retryable, now) {+ const row = db.select().from(queue).where(owned(lease.id)).get()+ if (+ !row ||+ row.status !== 'in-stages' ||+ !sameIdentity(row, lease) ||+ !row.leaseExpiresAt ||+ row.leaseExpiresAt.getTime() <= now.getTime()+ )+ return false+ const retry = retryable && row.attempts < row.maxAttempts+ return (+ db+ .update(queue)+ .set({+ status: retry ? 'pending' : 'failed',+ errorMessage: error.slice(0, 1000),+ statusStage: null,+ warningCode: null,+ leaseExpiresAt: null,+ availableAt: retry+ ? new Date(now.getTime() + retryDelay(row.attempts))+ : row.availableAt,+ })+ .where(and(owned(lease.id), eq(queue.attemptId, lease.attemptId)))+ .run().changes === 1+ )+ },+ }+ return {+ ...repository,+ bindSourceHash(lease: JobLease, hash: string, now = new Date()) {+ if (!/^[a-f0-9]{64}$/.test(hash))+ throw new PermanentJobError('Invalid source digest')+ db.transaction(+ () => {+ const row = assertCurrent(lease, now)+ if (row.sourceHash && row.sourceHash !== hash)+ throw new PermanentJobError(+ 'Source content changed since the first attempt',+ )+ if (!row.sourceHash)+ db.update(queue)+ .set({ sourceHash: hash })+ .where(and(owned(lease.id), eq(queue.attemptId, lease.attemptId)))+ .run()+ },+ { behavior: 'immediate' },+ )+ },+ commit,+ assertCurrent,+ stage(lease: JobLease, stage: typeof queue.$inferSelect.statusStage) {+ assertCurrent(lease)+ db.update(queue)+ .set({ statusStage: stage })+ .where(and(owned(lease.id), eq(queue.attemptId, lease.attemptId)))+ .run()+ },+ enqueue(+ payload: PipelineQueuePayload,+ options: { priority?: number; maxAttempts?: number } = {},+ ) {+ const validated = jobPayloadSchema.parse(payload)+ const priority = options.priority ?? 0,+ maxAttempts = options.maxAttempts ?? 3+ if (+ !Number.isSafeInteger(priority) ||+ priority < -1 ||+ priority > 9 ||+ !Number.isSafeInteger(maxAttempts) ||+ maxAttempts < 1 ||+ maxAttempts > 5+ )+ throw new RangeError('Invalid queue options')+ return db.transaction(+ () => {+ if (+ validated.type === 'photo' &&+ validated.albumId !== undefined &&+ !db+ .select({ id: tables.albums.id })+ .from(tables.albums)+ .where(+ and(+ eq(tables.albums.siteId, site.id),+ eq(tables.albums.id, validated.albumId),+ ),+ )+ .get()+ )+ throw new PermanentJobError('Album not found')+ return db+ .insert(queue)+ .values({+ priority,+ maxAttempts,+ payload: validated,+ siteId: site.id,+ ...bind(validated),+ pipelineVersion: PIPELINE_VERSION,+ })+ .returning({ id: queue.id })+ .get().id+ },+ { behavior: 'immediate' },+ )+ },+ }+}diff --git a/packages/database/src/sqlite-mutations.ts b/packages/database/src/sqlite-mutations.tsindex dc15abfd..323e0700 100644--- a/packages/database/src/sqlite-mutations.ts+++ b/packages/database/src/sqlite-mutations.ts@@ -1,3 +1,4 @@+import { createSqliteJobRepository, reserveResource } from './sqlite-jobs.ts'import { requireSite } from './site-repository.ts'import { type SiteContext } from '@chronoframe/core'import { and as siteAnd, eq as siteEq } from 'drizzle-orm'@@ -212,6 +213,14 @@ export function createSqlitePhotoWriteRepository(site: SiteContext,): PhotoWriteRepository {requireSite(db, site)+ const record = (photo: typeof tables.photos.$inferSelect) => ({+ ...toPhotoDto(photo),+ originalDerivedKey: photo.originalDerivedKey,+ storageReference: {+ connectionId: photo.storageConnectionId,+ revision: photo.storageRevision,+ },+ })return {async get(id) {const photo = db@@ -224,39 +233,72 @@ export function createSqlitePhotoWriteRepository(),).get()- return photo ? toPhotoDto(photo) : null+ return photo ? record(photo) : null},- async update(id, update) {- db.update(tables.photos)- .set(update)- .where(- siteAnd(- eq(tables.photos.id, id),- siteEq(tables.photos.siteId, site.id),- ),- )- .run()+ async beginMutation(id) {const photo = db.select().from(tables.photos)- .where(- siteAnd(- eq(tables.photos.id, id),- siteEq(tables.photos.siteId, site.id),- ),- )+ .where(and(eq(tables.photos.siteId, site.id), eq(tables.photos.id, id))).get()- return photo ? toPhotoDto(photo) : null+ if (photo)+ reserveResource(db, site, id, {+ storageConnectionId: photo.storageConnectionId,+ storageRevision: photo.storageRevision,+ storageKey: photo.storageKey,+ })+ },+ async update(id, update, reverseGeocode) {+ return db.transaction(() => {+ db.update(tables.photos)+ .set(update)+ .where(+ siteAnd(+ eq(tables.photos.id, id),+ siteEq(tables.photos.siteId, site.id),+ ),+ )+ .run()+ const photo = db+ .select()+ .from(tables.photos)+ .where(+ siteAnd(+ eq(tables.photos.id, id),+ siteEq(tables.photos.siteId, site.id),+ ),+ )+ .get()+ if (photo && reverseGeocode)+ createSqliteJobRepository(db, site, (payload) =>+ 'photoId' in payload ? payload.photoId : null,+ ).enqueue(+ { type: 'photo-reverse-geocoding', photoId: id, ...reverseGeocode },+ { priority: 1 },+ )+ return photo ? toPhotoDto(photo) : null+ })},async delete(id) {- db.delete(tables.photos)- .where(- siteAnd(- eq(tables.photos.id, id),- siteEq(tables.photos.siteId, site.id),- ),- )- .run()+ return db.transaction(+ () => {+ const photo = db+ .select()+ .from(tables.photos)+ .where(+ and(eq(tables.photos.siteId, site.id), eq(tables.photos.id, id)),+ )+ .get()+ if (!photo) return null+ db.delete(tables.photos)+ .where(+ and(eq(tables.photos.siteId, site.id), eq(tables.photos.id, id)),+ )+ .run()+ return record(photo)+ },+ { behavior: 'immediate' },+ )},}}diff --git a/packages/jobs/README.md b/packages/jobs/README.mdnew file mode 100644index 00000000..1b72f5db--- /dev/null+++ b/packages/jobs/README.md@@ -0,0 +1,9 @@+# Jobs++Embedded workers use explicit job/Site/resource/source/pipeline/attempt identities. A database adapter owns short claims, lease renewal, retries and compare-and-swap completion. `runOneJob` retains its processing slot until native work settles, even when heartbeat loss aborts the cooperative signal.++Claims count attempts, including crashes. The SQLite adapter recovers only expired leases during polling; startup never resets another live worker. Every result, stage, heartbeat and failure is checked against its attempt. Unsupported pipeline versions remain failed until a compatible worker or deliberate migration handles them.++Publication callbacks run synchronously inside the same transaction as completion. They must contain no object I/O. Source tombstones survive resource deletion. Managed source replacement invalidates the earlier generation before bytes are published, while the first input digest remains attached to retries. Duplicate callbacks cannot reapply domain writes.++Payloads are validated operational references. Storage configuration and credentials remain outside payloads, resolved from an immutable connection revision. Upload transport/session persistence is a separate boundary from job durability.diff --git a/packages/jobs/package.json b/packages/jobs/package.jsonnew file mode 100644index 00000000..2c2f1621--- /dev/null+++ b/packages/jobs/package.json@@ -0,0 +1,19 @@+{+ "name": "@chronoframe/jobs",+ "version": "0.1.0",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "scripts": {+ "type-check": "tsc --noEmit"+ },+ "dependencies": {+ "@chronoframe/core": "workspace:*",+ "@chronoframe/contracts": "workspace:*"+ },+ "devDependencies": {+ "typescript": "^5.9.3"+ }+}diff --git a/packages/jobs/src/index.ts b/packages/jobs/src/index.tsnew file mode 100644index 00000000..a0fd653a--- /dev/null+++ b/packages/jobs/src/index.ts@@ -0,0 +1,98 @@+import type { SiteContext } from '@chronoframe/core'+import type { PipelineQueuePayload } from '@chronoframe/contracts/queue'++export const PIPELINE_VERSION = '1'+export const DEFAULT_LEASE_MS = 60_000+export interface JobLease {+ id: number+ siteId: number+ resourceId: string | null+ payload: PipelineQueuePayload+ sourceVersion: number+ pipelineVersion: string+ attemptId: string+ workerId: string+ attempts: number+ maxAttempts: number+ storageConnectionId: number | null+ storageRevision: number+}+export class LostLeaseError extends Error {}+export class PermanentJobError extends Error {}+export interface JobRepository {+ claim(workerId: string, now: Date, leaseMs: number): Promise<JobLease | null>+ heartbeat(lease: JobLease, now: Date, leaseMs: number): Promise<boolean>+ complete(lease: JobLease, now: Date): Promise<boolean>+ fail(+ lease: JobLease,+ error: string,+ retryable: boolean,+ now: Date,+ ): Promise<boolean>+}+export function retryDelay(attempt: number): number {+ return Math.min(1000 * 2 ** Math.max(0, attempt - 1), 30_000)+}+/** Run one lease, retaining a native processing slot until its work has settled. */+export async function runOneJob(deps: {+ repository: JobRepository+ workerId: string+ site: SiteContext+ process(lease: JobLease, signal: AbortSignal): Promise<void>+ now?: () => Date+ leaseMs?: number+ heartbeatMs?: number+ onError?(error: unknown): void+}) {+ const now = deps.now ?? (() => new Date()),+ leaseMs = deps.leaseMs ?? DEFAULT_LEASE_MS+ const lease = await deps.repository.claim(deps.workerId, now(), leaseMs)+ if (!lease) return 'idle' as const+ if (lease.siteId !== deps.site.id)+ throw new PermanentJobError('Claimed job belongs to a different Site')+ const controller = new AbortController()+ let heartbeatRunning = false+ const timer = setInterval(+ () => {+ if (heartbeatRunning) return+ heartbeatRunning = true+ void deps.repository+ .heartbeat(lease, now(), leaseMs)+ .then((valid) => {+ if (!valid)+ controller.abort(new LostLeaseError('Lease or source changed'))+ })+ .catch((error) => {+ controller.abort(error)+ deps.onError?.(error)+ })+ .finally(() => {+ heartbeatRunning = false+ })+ },+ deps.heartbeatMs ?? Math.max(10, Math.floor(leaseMs / 3)),+ )+ try {+ await deps.process(lease, controller.signal)+ controller.signal.throwIfAborted()+ const completed = await deps.repository.complete(lease, now())+ if (!completed)+ await deps.repository.fail(lease, 'Job result discarded', false, now())+ return completed ? ('completed' as const) : ('discarded' as const)+ } catch (error) {+ await deps.repository.fail(+ lease,+ error instanceof Error ? error.message : 'Job failed',+ !(+ error instanceof PermanentJobError ||+ error instanceof LostLeaseError ||+ controller.signal.aborted+ ),+ now(),+ )+ deps.onError?.(error)+ return 'failed' as const+ } finally {+ clearInterval(timer)+ }+}diff --git a/packages/jobs/tsconfig.json b/packages/jobs/tsconfig.jsonnew file mode 100644index 00000000..5d435f61--- /dev/null+++ b/packages/jobs/tsconfig.json@@ -0,0 +1,15 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "ESNext",+ "moduleResolution": "Bundler",+ "strict": true,+ "noEmit": true,+ "allowImportingTsExtensions": true,+ "types": [],+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true,+ "noUncheckedIndexedAccess": true+ },+ "include": ["src/**/*.ts"]+}diff --git a/packages/processing/README.md b/packages/processing/README.mdnew file mode 100644index 00000000..73294b80--- /dev/null+++ b/packages/processing/README.md@@ -0,0 +1,5 @@+# Processing++`createPhotoProcessor` composes media decoding, HEIC conversion, metadata, WebP/ThumbHash, EXIF, geocoding, Motion Photo and Live Photo discovery. It receives storage, EXIF tooling, geocoding, policy and stage/cancellation callbacks explicitly. It has no framework or database imports and returns a result for core validation and transactional publication.++The embedded adapter supplies a storage wrapper that admits writes only while its lease is valid and writes derived objects under a unique attempt directory. Unpublished attempt outputs are cleaned without touching another attempt or the original source. A final source digest check and database generation check precede publication. Storage providers and native tools retain their existing self-hosted capabilities.diff --git a/packages/processing/package.json b/packages/processing/package.jsonnew file mode 100644index 00000000..cfc2a379--- /dev/null+++ b/packages/processing/package.json@@ -0,0 +1,22 @@+{+ "name": "@chronoframe/processing",+ "version": "0.1.0",+ "private": true,+ "type": "module",+ "exports": {+ ".": "./src/index.ts"+ },+ "scripts": {+ "type-check": "tsc --noEmit"+ },+ "dependencies": {+ "@chronoframe/contracts": "workspace:*",+ "@chronoframe/media": "workspace:*",+ "@chronoframe/storage": "workspace:*",+ "@chronoframe/geocoding": "workspace:*"+ },+ "devDependencies": {+ "typescript": "^5.9.3",+ "@types/node": "^22.20.5"+ }+}diff --git a/packages/processing/src/index.ts b/packages/processing/src/index.tsnew file mode 100644index 00000000..3ae0059b--- /dev/null+++ b/packages/processing/src/index.ts@@ -0,0 +1,188 @@+import path from 'node:path'+import { createHash } from 'node:crypto'+import {+ createImageProcessor,+ generateThumbnailAndHash,+ processMotionPhotoFromXmp,+ type createExifService,+ type MediaLogger,+} from '@chronoframe/media'+import type { StorageProvider } from '@chronoframe/storage'+import { hasValidCoordinates } from '@chronoframe/contracts/location'+import type { NeededExif } from '@chronoframe/contracts/photo'+import type { Photo } from '@chronoframe/contracts/resources'+import type { LocationInfo } from '@chronoframe/geocoding'+import type { ProcessingStage } from '@chronoframe/contracts/queue'++export const EXIF_LOCATION_KEYS = [+ 'GPSAltitude',+ 'GPSAltitudeRef',+ 'GPSLatitude',+ 'GPSLatitudeRef',+ 'GPSLongitude',+ 'GPSLongitudeRef',+ 'GPSPosition',+ 'GPSDateStamp',+ 'GPSTimeStamp',+ 'GPSImgDirection',+ 'GPSImgDirectionRef',+ 'GPSDestBearing',+ 'GPSDestBearingRef',+] as const+export function stripLocationFromExif<+ T extends Record<string, any> | null | undefined,+>(exif: T): T {+ if (!exif) return exif+ const result = { ...exif }+ for (const key of EXIF_LOCATION_KEYS) delete result[key]+ return result as T+}+export type ProcessedPhoto = Photo & { originalDerivedKey: string | null }+export interface PhotoProcessingInput {+ photoId: string+ storageKey: string+ eraseLocation: boolean+ storage: StorageProvider+ stage(stage: ProcessingStage): Promise<void>+ sourceRead?(hash: string): void | Promise<void>+ signal?: AbortSignal+}+/** Compose media capabilities with explicit storage and site-resolved settings; never persist business state here. */+export function createPhotoProcessor(deps: {+ logger: MediaLogger+ exif: ReturnType<typeof createExifService>+ parseGPS(exif: NeededExif): { latitude?: number; longitude?: number }+ reverseGeocode(+ latitude: number,+ longitude: number,+ ): Promise<LocationInfo | null>+ now?: () => Date+}) {+ return async (+ input: PhotoProcessingInput,+ ): Promise<{ photo: ProcessedPhoto; sourceHash: string }> => {+ const { storage, storageKey, photoId } = input+ const stage = async (value: ProcessingStage) => {+ input.signal?.throwIfAborted()+ await input.stage(value)+ }+ const media = createImageProcessor(deps.logger, { signal: input.signal })+ await stage('preprocessing')+ const raw = await storage.get(storageKey)+ if (!raw) throw new Error('Source object not found')+ const sourceHash = createHash('sha256').update(raw).digest('hex')+ await input.sourceRead?.(sourceHash)+ const storageObject = await storage.getFileMeta(storageKey)+ let processed = raw,+ originalDerivedKey: string | null = null+ const ext = path.extname(storageKey).toLowerCase()+ if (['.heic', '.heif', '.hif'].includes(ext)) {+ processed = await media.convertHeicToJpeg(raw)+ originalDerivedKey = (+ await storage.create(+ `${path.basename(storageKey, ext)}.jpeg`,+ processed,+ 'image/jpeg',+ )+ ).key+ }+ await stage('metadata')+ const processedData = await media.processImageMetadataAndSharp(+ processed,+ storageKey,+ )+ if (!processedData) throw new Error('Metadata processing failed')+ const { imageBuffer, metadata } = processedData+ await stage('thumbnail')+ const { thumbnailBuffer, thumbnailHash } = await generateThumbnailAndHash(+ imageBuffer,+ deps.logger,+ )+ const thumbnail = await storage.create(+ `thumbnails/${photoId}.webp`,+ thumbnailBuffer,+ 'image/webp',+ )+ await stage('exif')+ const extracted = await deps.exif.extractExifData(+ imageBuffer,+ raw,+ deps.logger,+ )+ const exif = input.eraseLocation+ ? stripLocationFromExif(extracted)+ : extracted+ const info = deps.exif.extractPhotoInfo(storageKey, exif)+ await stage('reverse-geocoding')+ let coordinates: { latitude: number; longitude: number } | null = null,+ location: LocationInfo | null = null+ if (!input.eraseLocation && exif) {+ const parsed = deps.parseGPS(exif)+ if (hasValidCoordinates(parsed.latitude, parsed.longitude)) {+ coordinates = {+ latitude: parsed.latitude!,+ longitude: parsed.longitude!,+ }+ location = await deps.reverseGeocode(+ coordinates.latitude,+ coordinates.longitude,+ )+ }+ }+ await stage('motion-photo')+ const motion = await processMotionPhotoFromXmp({+ photoId,+ storageKey,+ rawImageBuffer: raw,+ exifData: exif,+ storageProvider: storage,+ logger: deps.logger,+ })+ await stage('live-photo')+ let videoKey = motion?.livePhotoVideoKey ?? null+ if (!motion?.isMotionPhoto) {+ const base = storageKey.slice(0, -path.extname(storageKey).length)+ for (const candidate of [`${base}.MOV`, `${base}.mov`])+ if (await storage.getFileMeta(candidate)) {+ videoKey = candidate+ break+ }+ }+ input.signal?.throwIfAborted()+ return {+ sourceHash,+ photo: {+ id: photoId,+ title: info.title,+ description: info.description,+ dateTaken: info.dateTaken,+ tags: info.tags,+ width: metadata.width,+ height: metadata.height,+ aspectRatio: metadata.width / metadata.height,+ storageKey,+ thumbnailKey: thumbnail.key,+ fileSize: storageObject?.size ?? raw.length,+ lastModified:+ storageObject?.lastModified?.toISOString() ??+ (deps.now?.() ?? new Date()).toISOString(),+ originalUrl: storage.getPublicUrl(originalDerivedKey ?? storageKey),+ originalDerivedKey,+ thumbnailUrl: storage.getPublicUrl(thumbnail.key),+ thumbnailHash: thumbnailHash+ ? Buffer.from(thumbnailHash).toString('hex')+ : null,+ exif,+ latitude: coordinates?.latitude ?? null,+ longitude: coordinates?.longitude ?? null,+ country: location?.country ?? null,+ city: location?.city ?? null,+ locationName: location?.locationName ?? null,+ geocoding: location?.geocoding ?? null,+ isLivePhoto: videoKey ? 1 : 0,+ livePhotoVideoKey: videoKey,+ livePhotoVideoUrl: videoKey ? storage.getPublicUrl(videoKey) : null,+ },+ }+ }+}diff --git a/packages/processing/tsconfig.json b/packages/processing/tsconfig.jsonnew file mode 100644index 00000000..e79cde15--- /dev/null+++ b/packages/processing/tsconfig.json@@ -0,0 +1,15 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "ESNext",+ "moduleResolution": "Bundler",+ "strict": true,+ "noEmit": true,+ "allowImportingTsExtensions": true,+ "types": ["node"],+ "lib": ["ES2022", "DOM"],+ "skipLibCheck": true,+ "noUncheckedIndexedAccess": true+ },+ "include": ["src/**/*.ts"]+}diff --git a/packages/storage/src/providers/s3.ts b/packages/storage/src/providers/s3.tsindex 2abc14c8..f486914c 100644--- a/packages/storage/src/providers/s3.ts+++ b/packages/storage/src/providers/s3.ts@@ -77,11 +77,19 @@ export class S3StorageProvider implements StorageProvider {contentType?: string,): Promise<StorageObject> {try {+ const configuredPrefix = this.config.prefix ?? ''+ let start = 0,+ end = configuredPrefix.length,+ keyStart = 0+ while (start < end && configuredPrefix[start] === '/') start+++ while (end > start && configuredPrefix[end - 1] === '/') end--+ while (keyStart < key.length && key[keyStart] === '/') keyStart+++ const prefix = configuredPrefix.slice(start, end),+ cleanKey = key.slice(keyStart)const absoluteKey =- `${(this.config.prefix || '').replace(/\/+$/, '')}/${key}`.replace(- /^\/+/,- '',- )+ prefix && !cleanKey.startsWith(prefix + '/')+ ? prefix + '/' + cleanKey+ : cleanKeyconst cmd = new PutObjectCommand({Bucket: this.config.bucket,Key: absoluteKey,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex abfb162e..3c8700c7 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -29,6 +29,12 @@ importers:'@chronoframe/geocoding':specifier: workspace:*version: link:packages/geocoding+ '@chronoframe/jobs':+ specifier: workspace:*+ version: link:packages/jobs+ '@chronoframe/processing':+ specifier: workspace:*+ version: link:packages/processing'@chronoframe/media':specifier: workspace:*version: link:packages/media@@ -266,6 +272,9 @@ importers:packages/database:dependencies:+ '@chronoframe/jobs':+ specifier: workspace:*+ version: link:../jobs'@chronoframe/contracts':specifier: workspace:*version: link:../contracts@@ -358,6 +367,41 @@ importers:specifier: ^5.9.3version: 5.9.3+ packages/jobs:+ dependencies:+ '@chronoframe/core':+ specifier: workspace:*+ version: link:../core+ '@chronoframe/contracts':+ specifier: workspace:*+ version: link:../contracts+ devDependencies:+ typescript:+ specifier: ^5.9.3+ version: 5.9.3++ packages/processing:+ dependencies:+ '@chronoframe/contracts':+ specifier: workspace:*+ version: link:../contracts+ '@chronoframe/media':+ specifier: workspace:*+ version: link:../media+ '@chronoframe/storage':+ specifier: workspace:*+ version: link:../storage+ '@chronoframe/geocoding':+ specifier: workspace:*+ version: link:../geocoding+ devDependencies:+ typescript:+ specifier: ^5.9.3+ version: 5.9.3+ '@types/node':+ specifier: ^22.20.5+ version: 22.20.5+packages/webgl-image:dependencies:vue:diff --git a/scripts/check-package-boundaries.mjs b/scripts/check-package-boundaries.mjsindex 6ffa79e0..84dde586 100644--- a/scripts/check-package-boundaries.mjs+++ b/scripts/check-package-boundaries.mjs@@ -7,6 +7,8 @@ const packageNames = ['contracts','core','database',+ 'jobs',+ 'processing','storage','media','geocoding',diff --git a/server/api/photos/exif/reindex.post.ts b/server/api/photos/exif/reindex.post.tsindex c68a251a..6e7b42fe 100644--- a/server/api/photos/exif/reindex.post.ts+++ b/server/api/photos/exif/reindex.post.ts@@ -1,240 +1,97 @@-import { and as siteAnd, eq as siteEq } from 'drizzle-orm'-import { DEFAULT_SITE } from '@chronoframe/core'-import { eq, isNotNull, inArray, and } from 'drizzle-orm'-import {- extractExifData,- extractPhotoInfo,-} from '~~/server/services/image/exif'+import { and, eq, inArray, isNotNull } from 'drizzle-orm'+import { CoreError, DEFAULT_SITE } from '@chronoframe/core'+import { createSiteRepository } from '@chronoframe/database'+import { extractExifData, extractPhotoInfo } from '../../../services/image/exif'+import { storageForPhoto } from '../../../utils/object-storage'export default eventHandler(async (event) => {await requireUserSession(event)-- const body = await readBody(event)- const { action, photoId } = body-+ const { action, photoId, photoIds } = await readBody(event)+ const db = useDB(),+ repository = createSiteRepository(db, DEFAULT_SITE)+ const reindex = async (id: string) => {+ const { storageProvider: fallback } = useStorageProvider(event)+ const snapshot = await repository.capturePhoto(id)+ if (!snapshot?.photo.storageKey) throw new CoreError(404, 'Photo not found')+ const provider = await storageForPhoto(id, fallback)+ const buffer = await provider.get(snapshot.photo.storageKey)+ if (!buffer) throw new CoreError(404, 'File not found in storage')+ const exif = await extractExifData(buffer, undefined, logger.chrono)+ const info = extractPhotoInfo(snapshot.photo.storageKey, exif)+ await repository.updatePhotoSnapshot(id, snapshot.version, {+ exif,+ title: info.title,+ dateTaken: info.dateTaken,+ tags: info.tags,+ lastModified: new Date().toISOString(),+ })+ }try {if (action === 'single-reindex' && photoId) {- // 处理单个照片的 EXIF 重新索引- const photo = await useDB()- .select()- .from(tables.photos)- .where(- siteAnd(- eq(tables.photos.id, photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )- .limit(1)- .then((rows) => rows[0])-- if (!photo) {- throw createError({- statusCode: 404,- statusMessage: 'Photo not found',- })- }-- logger.chrono.info(`Reindexing EXIF for photo ${photoId}`)-- // 重新提取文件的元数据- const { storageProvider } = useStorageProvider(event)- const fileBuffer = await storageProvider.get(photo.storageKey!)-- if (!fileBuffer) {- throw createError({- statusCode: 404,- statusMessage: 'File not found in storage',- })- }-- // 提取新的 EXIF 数据- const exifData = await extractExifData(- fileBuffer,- undefined,- logger.chrono,+ await reindex(photoId)+ return { success: true, message: 'EXIF 数据已成功重新索引', photoId }+ }+ if (action !== 'batch-reindex')+ throw new CoreError(400, 'Invalid action parameter')+ const photos = db+ .select({ id: tables.photos.id })+ .from(tables.photos)+ .where(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ isNotNull(tables.photos.storageKey),+ Array.isArray(photoIds) && photoIds.length+ ? inArray(tables.photos.id, photoIds)+ : undefined,+ ),)- const photoInfo = extractPhotoInfo(photo.storageKey!, exifData)-- // 更新数据库中的 EXIF 数据- await useDB()- .update(tables.photos)- .set({- exif: exifData,- title: photoInfo.title,- dateTaken: photoInfo.dateTaken,- tags: photoInfo.tags,- lastModified: new Date().toISOString(),- thumbnailKey: `${storageProvider.config?.prefix?.replace(/\/$/, '')}/thumbnails/${photoId}.webp`,- })- .where(- siteAnd(- eq(tables.photos.id, photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )-- logger.chrono.success(`Successfully reindexed EXIF for photo ${photoId}`)-+ .all()+ if (!photos.length)return {- success: true,- message: 'EXIF 数据已成功重新索引',- photoId,+ message: '没有找到需要重新索引的照片',+ results: { total: 0, processed: 0, updated: 0, errors: [] },}- } else if (action === 'batch-reindex') {- // 批量重新索引所有照片的 EXIF- const { photoIds } = body- let photos-- if (photoIds && Array.isArray(photoIds) && photoIds.length > 0) {- // 只处理指定的照片- photos = await useDB()- .select({- id: tables.photos.id,- storageKey: tables.photos.storageKey,- })- .from(tables.photos)- .where(- siteAnd(- and(- isNotNull(tables.photos.storageKey),- inArray(tables.photos.id, photoIds),- ),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )- } else {- // 处理所有照片- photos = await useDB()- .select({- id: tables.photos.id,- storageKey: tables.photos.storageKey,- })- .from(tables.photos)- .where(- siteAnd(- isNotNull(tables.photos.storageKey),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )- }-- if (photos.length === 0) {- return {- message: '没有找到需要重新索引的照片',- results: {- total: 0,- processed: 0,- updated: 0,- errors: [],- },- }- }-- logger.chrono.info(- `Starting batch EXIF reindexing for ${photos.length} photos`,- )-- let processed = 0- let updated = 0- const errors: Array<{ photoId: string; error: string }> = []- const { storageProvider } = useStorageProvider(event)-- for (const photo of photos) {- try {- processed++-- logger.chrono.info(- `Processing photo ${photo.id} (${processed}/${photos.length})`,- )-- // 获取文件- const fileBuffer = await storageProvider.get(photo.storageKey!)-- if (!fileBuffer) {- errors.push({- photoId: photo.id,- error: 'File not found in storage',- })- continue- }-- // 提取元数据和照片信息- const exifData = await extractExifData(- fileBuffer,- undefined,- logger.chrono,- )- const photoInfo = extractPhotoInfo(photo.storageKey!, exifData)-- // 更新数据库- await useDB()- .update(tables.photos)- .set({- exif: exifData,- title: photoInfo.title,- dateTaken: photoInfo.dateTaken,- tags: photoInfo.tags,- lastModified: new Date().toISOString(),- thumbnailKey: `${storageProvider.config?.prefix?.replace(/\/$/, '')}/thumbnails/${photo.id}.webp`,- })- .where(- siteAnd(- eq(tables.photos.id, photo.id),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )-- updated++-- logger.chrono.success(- `Updated EXIF for photo ${photo.id} (${updated}/${processed})`,- )-- // 添加小延迟以避免过度负载- if (processed % 10 === 0) {- await new Promise((resolve) => setTimeout(resolve, 100))- }- } catch (error) {- const errorMessage =- error instanceof Error ? error.message : 'Unknown error'- logger.chrono.error(`Failed to process photo ${photo.id}:`, error)- errors.push({- photoId: photo.id,- error: errorMessage,- })- }+ let processed = 0,+ updated = 0+ const errors: Array<{ photoId: string; error: string }> = []+ for (const photo of photos) {+ processed+++ try {+ await reindex(photo.id)+ updated+++ } catch (error) {+ errors.push({+ photoId: photo.id,+ error:+ error instanceof CoreError+ ? error.message+ : 'Failed to reindex photo',+ })}-- const result = {- message: `EXIF 重新索引完成`,- results: {- total: photos.length,- processed,- updated,- errors: errors.length > 0 ? errors : undefined,- statistics: {- successRate:- processed > 0- ? (((processed - errors.length) / processed) * 100).toFixed(1) +- '%'- : '0%',- },+ if (processed % 10 === 0)+ await new Promise((resolve) => setTimeout(resolve, 100))+ }+ return {+ message: 'EXIF 重新索引完成',+ results: {+ total: photos.length,+ processed,+ updated,+ errors: errors.length ? errors : undefined,+ statistics: {+ successRate: processed+ ? (((processed - errors.length) / processed) * 100).toFixed(1) + '%'+ : '0%',},- }-- logger.chrono.success(- `EXIF batch reindexing completed: ${updated} photos updated out of ${processed} processed`,- )-- return result- } else {- throw createError({- statusCode: 400,- statusMessage: 'Invalid action parameter',- })+ },}} catch (error) {- logger.chrono.error('Failed to reindex EXIF data:', error)+ if (error instanceof CoreError)+ throw createError({+ statusCode: error.statusCode,+ statusMessage: error.message,+ })+ logger.chrono.error('Failed to reindex EXIF data', error)throw createError({statusCode: 500,statusMessage: 'Failed to reindex EXIF data',diff --git a/server/api/photos/index.post.ts b/server/api/photos/index.post.tsindex d62b1e44..c15a61b7 100644--- a/server/api/photos/index.post.ts+++ b/server/api/photos/index.post.ts@@ -1,3 +1,7 @@+import {+ reserveUploadSource,+ sourceAwareUploadStorage,+} from '../../utils/upload-source'import { DEFAULT_SITE } from '@chronoframe/core'import type {UploadPreparationResponse,@@ -29,7 +33,14 @@ export default eventHandler(// Self-hosted has no account-wide quota; the per-file policy still applies.quota: { assertAllowed: async () => {} },prepare: (owner, key, size, contentType, chunked) =>- prepareUpload(owner, key, size, contentType, storageProvider, chunked),+ prepareUpload(+ owner,+ key,+ size,+ contentType,+ sourceAwareUploadStorage(storageProvider),+ chunked,+ ),})try {const result = await preparer(String(session.user.id), body, {@@ -60,6 +71,7 @@ export default eventHandler('upload.duplicateCheck.mode',)) ?? 'skip',})+ if (!result.skipped) reserveUploadSource(result.fileKey, storageProvider)const fileName = body.fileNameif (result.skipped) {return {diff --git a/server/api/photos/livephoto/manage.post.ts b/server/api/photos/livephoto/manage.post.tsindex bf8b9210..d48539de 100644--- a/server/api/photos/livephoto/manage.post.ts+++ b/server/api/photos/livephoto/manage.post.ts@@ -1,10 +1,10 @@-import { and as siteAnd, eq as siteEq } from 'drizzle-orm'+import { createSiteRepository } from '@chronoframe/database'+import { storageForPhoto } from '../../../utils/object-storage'import { DEFAULT_SITE } from '@chronoframe/core'import {processSpecificLivePhotoVideo,scanAndProcessExistingLivePhotos,} from '~~/server/services/video/scanner'-import { eq } from 'drizzle-orm'import { findLivePhotoVideoForImage } from '~~/server/services/video/livephoto'import { getStorageManager } from '~~/server/plugins/3.storage'import { batchTestLivePhotoDetection } from '~~/server/services/video/test-utils'@@ -70,49 +70,27 @@ export default eventHandler(async (event) => {})}- const db = useDB()- const photos = await db- .select()- .from(tables.photos)- .where(- siteAnd(- eq(tables.photos.id, photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )- .limit(1)-- if (photos.length === 0) {+ const repository = createSiteRepository(useDB(), DEFAULT_SITE)+ const snapshot = await repository.capturePhoto(photoId)+ if (!snapshot?.photo.storageKey)throw createError({statusCode: 404,statusMessage: 'Photo not found',})- }-- const photo = photos[0]+ const provider = await storageForPhoto(+ photoId,+ getStorageManager().getProvider(),+ )const livePhotoVideo = await findLivePhotoVideoForImage(- photo.storageKey!,+ snapshot.photo.storageKey,+ provider,)-if (livePhotoVideo) {- const storageProvider = getStorageManager().getProvider()-- await db- .update(tables.photos)- .set({- isLivePhoto: 1,- livePhotoVideoUrl: storageProvider.getPublicUrl(- livePhotoVideo.videoKey,- ),- livePhotoVideoKey: livePhotoVideo.videoKey,- })- .where(- siteAnd(- eq(tables.photos.id, photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )-+ await repository.updatePhotoSnapshot(photoId, snapshot.version, {+ isLivePhoto: 1,+ livePhotoVideoUrl: provider.getPublicUrl(livePhotoVideo.videoKey),+ livePhotoVideoKey: livePhotoVideo.videoKey,+ })return {message: 'Photo updated to LivePhoto successfully',success: true,diff --git a/server/api/photos/upload.put.ts b/server/api/photos/upload.put.tsindex f7829252..9b2eb85e 100644--- a/server/api/photos/upload.put.ts+++ b/server/api/photos/upload.put.ts@@ -1,3 +1,7 @@+import {+ providerForUpload,+ sourceAwareUploadStorage,+} from '../../utils/upload-source'import { mkdtemp, open, rm } from 'node:fs/promises'import { tmpdir } from 'node:os'import path from 'node:path'@@ -8,7 +12,7 @@ import { settingsManager } from '~~/server/services/settings/settingsManager'export default eventHandler(async (event) => {await requireUserSession(event)- const { storageProvider } = useStorageProvider(event)+ let { storageProvider } = useStorageProvider(event)const key = getQuery(event).key as string | undefinedconst t = await useTranslation(event)@@ -23,6 +27,10 @@ export default eventHandler(async (event) => {})}+ storageProvider = sourceAwareUploadStorage(+ await providerForUpload(key, storageProvider),+ )+const contentType =getHeader(event, 'content-type') || 'application/octet-stream'diff --git a/server/plugins/4.pipeline-queue.ts b/server/plugins/4.pipeline-queue.tsindex 27f5ce12..28741fa8 100644--- a/server/plugins/4.pipeline-queue.ts+++ b/server/plugins/4.pipeline-queue.ts@@ -1,3 +1,4 @@+import { exiftool } from 'exiftool-vendored'import { closeDB } from '../utils/db'import type {} from '../types/worker'import { WorkerPool } from '../services/pipeline-queue'@@ -23,7 +24,11 @@ export default defineNitroPlugin(async (nitroApp) => {globalThis.__workerPool = workerPoollet shutdown: Promise<void> | undefined- const stop = () => (shutdown ||= workerPool.stop().finally(closeDB))+ const stop = () =>+ (shutdown ||= workerPool.stop().finally(async () => {+ await exiftool.end()+ closeDB()+ }))nitroApp.hooks.hook('close', stop)const exitHandler = () => {diff --git a/server/services/pipeline-queue/manager.ts b/server/services/pipeline-queue/manager.tsindex 841157ac..d07e6093 100644--- a/server/services/pipeline-queue/manager.ts+++ b/server/services/pipeline-queue/manager.ts@@ -1,1093 +1,555 @@-import { storageForReference } from '../../utils/object-storage'-import { and as siteAnd, eq as siteEq } from 'drizzle-orm'-import { DEFAULT_SITE } from '@chronoframe/core'-import {- enqueueLocationMigration,- enqueueLocationMigrationRequest,-} from './location-migration'-import { matchesLocationRequest, updateGeocodedPhoto } from './location-update'-import { hasValidCoordinates } from '../../../shared/utils/location'import type { ConsolaInstance } from 'consola'-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'+import { createHash } from 'node:crypto'+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'-import path from 'path'-import { and, asc, desc, eq, lte, sql } from 'drizzle-orm'+import path from 'node:path'+import { and, eq, sql } from 'drizzle-orm'import { exiftool } from 'exiftool-vendored'-import type {- NewPipelineQueueItem,- PipelineQueueItem,- Photo,-} from '~~/server/utils/db'-import { QueueWarningCode } from '~~/shared/utils/queue-warning'-import { compressUint8Array } from '~~/shared/utils/u8array'+import { DEFAULT_SITE, validatePhotoPublication } from '@chronoframe/core'+import {+ createSqliteJobRepository,+ createPhotoPersistenceRepository,+} from '@chronoframe/database'+import {+ runOneJob,+ LostLeaseError,+ PermanentJobError,+ type JobLease,+} from '@chronoframe/jobs'import {- preprocessImageWithJpegUpload,- processImageMetadataAndSharp,-} from '../image/processor'-import { generateThumbnailAndHash } from '../image/thumbnail'-import { extractExifData, extractPhotoInfo } from '../image/exif'+ createPhotoProcessor,+ EXIF_LOCATION_KEYS,+ stripLocationFromExif,+} from '@chronoframe/processing'+import { createExifService } from '@chronoframe/media'+import type { PipelineQueuePayload } from '@chronoframe/contracts/queue'+import { QueueWarningCode } from '@chronoframe/contracts/queue'+import {+ useDB,+ tables,+ type NewPipelineQueueItem,+ type PipelineQueueItem,+} from '../../utils/db'+import { generateSafePhotoId } from '../../utils/file-utils'+import { logger } from '../../utils/logger'+import { storageForReference } from '../../utils/object-storage'+import { getStorageManager } from '../../plugins/3.storage'+import { settingsManager } from '../settings/settingsManager'import {extractLocationFromGPS,parseGPSCoordinates,} from '../location/geocoding'-import { settingsManager } from '../settings/settingsManager'-import { findLivePhotoVideoForImage } from '../video/livephoto'-import { processMotionPhotoFromXmp } from '../video/motion-photo'-import { getStorageManager } from '~~/server/plugins/3.storage'-import { persistPhotoWithAlbum } from '~~/server/utils/photo-album'--const EXIF_LOCATION_KEYS = [- 'GPSAltitude',- 'GPSAltitudeRef',- 'GPSLatitude',- 'GPSLatitudeRef',- 'GPSLongitude',- 'GPSLongitudeRef',- 'GPSPosition',- 'GPSDateStamp',- 'GPSTimeStamp',- 'GPSImgDirection',- 'GPSImgDirectionRef',- 'GPSDestBearing',- 'GPSDestBearingRef',-] as const--class PermanentTaskError extends Error {}--const stripLocationFromExif = <- T extends Record<string, any> | null | undefined,->(- exif: T,-): T => {- if (!exif || typeof exif !== 'object') {- return exif- }-- const cloned = { ...exif }- for (const key of EXIF_LOCATION_KEYS) {- delete cloned[key]- }+import { hasValidCoordinates } from '../../../shared/utils/location'+import {+ enqueueLocationMigration,+ enqueueLocationMigrationRequest,+} from './location-migration'+import { matchesLocationRequest, updateGeocodedPhoto } from './location-update'- return cloned as T-}+type Repository = ReturnType<typeof createSqliteJobRepository>+/** Embedded scheduling adapter. Packages own leases and processing; this class assembles application capabilities. */export class QueueManager {- private static instances: Map<string, QueueManager> = new Map()- private workerId: string- private logger: ConsolaInstance- private isProcessing: boolean = false- private processingInterval: NodeJS.Timeout | null = null+ private static instances = new Map<string, QueueManager>()+ private processingInterval: ReturnType<typeof setInterval> | null = nullprivate activeTask: Promise<void> | null = null- private processedCount: number = 0- private errorCount: number = 0- private startTime: Date-- static getInstance(- workerId: string = 'default',- logger?: ConsolaInstance,- ): QueueManager {- if (!QueueManager.instances.has(workerId)) {- QueueManager.instances.set(workerId, new QueueManager(workerId, logger))- }- return QueueManager.instances.get(workerId)!+ private isProcessing = false+ private processedCount = 0+ private errorCount = 0+ private readonly startTime = new Date()+ private logger: ConsolaInstance+ private constructor(+ private workerId: string,+ serviceLogger?: ConsolaInstance,+ ) {+ this.logger = serviceLogger+ ? serviceLogger.withTag(workerId)+ : logger.dynamic(`queue-${workerId}`)}-- static getAllInstances(): QueueManager[] {- return Array.from(QueueManager.instances.values())+ static getInstance(workerId = 'default', serviceLogger?: ConsolaInstance) {+ let worker = this.instances.get(workerId)+ if (!worker) {+ worker = new QueueManager(workerId, serviceLogger)+ this.instances.set(workerId, worker)+ }+ return worker}-- private constructor(workerId: string, _logger?: ConsolaInstance) {- this.workerId = workerId- this.logger = _logger- ? _logger.withTag(`${workerId}`)- : logger.dynamic(`queue-${workerId}`)- this.startTime = new Date()+ static getAllInstances() {+ return Array.from(this.instances.values())}-- getWorkerId(): string {+ getWorkerId() {return this.workerId}-getStats() {- const uptime = Date.now() - this.startTime.getTime()+ const attempts = this.processedCount + this.errorCountreturn {workerId: this.workerId,isProcessing: this.isProcessing,processedCount: this.processedCount,errorCount: this.errorCount,- uptime: Math.floor(uptime / 1000), // seconds- successRate:- this.processedCount > 0- ? (this.processedCount / (this.processedCount + this.errorCount)) *- 100- : 0,+ uptime: Math.floor((Date.now() - this.startTime.getTime()) / 1000),+ successRate: attempts ? (this.processedCount / attempts) * 100 : 0,}}-- /**- * 插入新任务到队列- * @param payload 任务负荷- * @param options 可选的任务设置- * @returns 新创建任务的 ID- */- async addTask(- payload: any,- options?: Partial<NewPipelineQueueItem>,- ): Promise<number> {+ private repository() {const db = useDB()- if (payload.type === 'location-migration')- return enqueueLocationMigrationRequest(db, options)- const active = await settingsManager.storage.getActiveProvider()- const photoId = 'photoId' in payload ? payload.photoId : undefined- const photo = photoId- ? db- .select()+ return createSqliteJobRepository(db, DEFAULT_SITE, (payload) => {+ if ('photoId' in payload) return payload.photoId+ if ('storageKey' in payload) {+ const existing = db+ .select({ id: tables.photos.id }).from(tables.photos).where(and(- eq(tables.photos.id, photoId),eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.storageKey, payload.storageKey),),).get()- : undefined- if (photoId && !photo) throw new PermanentTaskError('Photo not found')- const result = db- .insert(tables.pipelineQueue)- .values({- payload,- ...options,- siteId: DEFAULT_SITE.id,- storageConnectionId: photo?.storageConnectionId ?? active?.id ?? null,- storageRevision: photo?.storageRevision ?? active?.revision ?? 1,- sourceVersion: photo?.sourceVersion ?? 1,- })- .returning({ id: tables.pipelineQueue.id })- .get()- return result.id- }-- /**- * 获取任务状态- * @param taskId 任务ID- * @returns 任务状态信息- */- async getTaskStatus(taskId: number) {- const db = useDB()- const task = await db- .select()- .from(tables.pipelineQueue)- .where(- siteAnd(- eq(tables.pipelineQueue.id, taskId),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )- .get()- return task- }-- /**- * 获取并锁定下一个待处理任务- * @returns 下一个待处理任务- */- async getNextTask(): Promise<PipelineQueueItem | null> {- const db = useDB()-- // 使用同步事务防止竞态条件- const task = db.transaction((tx) => {- const highestPriorityPendingTask = tx- .select()- .from(tables.pipelineQueue)- .where(- siteAnd(- and(- eq(tables.pipelineQueue.status, 'pending'),- lte(tables.pipelineQueue.availableAt, new Date()),- ),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )- // 优先处理高优先级和较早创建的任务- .orderBy(- desc(tables.pipelineQueue.priority),- asc(tables.pipelineQueue.createdAt),- asc(tables.pipelineQueue.id),- )- .limit(1)- .get()-- if (!highestPriorityPendingTask) return null-- const task = highestPriorityPendingTask- tx.update(tables.pipelineQueue)- .set({ status: 'in-stages' })- .where(- siteAnd(- eq(tables.pipelineQueue.id, task.id),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )- .run()-- return { ...task, status: 'in-stages' as const }+ return existing?.id ?? generateSafePhotoId(payload.storageKey)+ }+ return null})-- return task}-- /**- * 更新任务阶段- * @param taskId 任务ID- * @param stage 新的任务阶段- */- async updateTaskStage(- taskId: number,- stage: PipelineQueueItem['statusStage'],- ): Promise<void> {- const db = useDB()- await db- .update(tables.pipelineQueue)- .set({ statusStage: stage })- .where(- siteAnd(- eq(tables.pipelineQueue.id, taskId),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )- }-- /**- * 标记任务为已完成- * @param taskId 任务ID- */- async markTaskCompleted(- taskId: number,- warningCode: PipelineQueueItem['warningCode'] = null,- ): Promise<void> {- const db = useDB()- await db- .update(tables.pipelineQueue)- .set({- status: 'completed',- completedAt: sql`(unixepoch())`,- warningCode,+ async addTask(+ payload: PipelineQueuePayload,+ options?: Partial<NewPipelineQueueItem>,+ ) {+ if (payload.type === 'location-migration')+ return enqueueLocationMigrationRequest(useDB(), {+ priority: options?.priority,+ maxAttempts: options?.maxAttempts,})- .where(- siteAnd(- eq(tables.pipelineQueue.id, taskId),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )+ return this.repository().enqueue(payload, {+ priority: options?.priority,+ maxAttempts: options?.maxAttempts,+ })}-- /**- * 标记任务为失败- * @param taskId 任务ID- * @param errorMessage 错误信息- */- async markTaskFailed(- taskId: number,- errorMessage?: string,- retryable = true,- ): Promise<void> {- const db = useDB()- const task = await db+ async getTaskStatus(id: number) {+ return useDB().select().from(tables.pipelineQueue).where(- siteAnd(- eq(tables.pipelineQueue.id, taskId),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),+ and(+ eq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),+ eq(tables.pipelineQueue.id, id),),).get()-- if (!task) return-- const newAttempts = task.attempts + 1- const shouldRetry = retryable && newAttempts < task.maxAttempts-- // 计算重试延迟(指数退避)- const retryDelay = shouldRetry- ? Math.min(1000 * Math.pow(2, newAttempts - 1), 30000)- : 0-- await db- .update(tables.pipelineQueue)- .set({- status: shouldRetry ? 'pending' : 'failed',- attempts: newAttempts,- errorMessage: errorMessage || 'Unknown error',- warningCode: null,- // 如果重试,设置延迟重试时间- statusStage: null,- ...(shouldRetry- ? {- availableAt: new Date(- Date.now() + retryDelay + Math.floor(Math.random() * 500),- ),- }- : {}),+ }+ async getQueueStats() {+ const stats = useDB()+ .select({+ status: tables.pipelineQueue.status,+ count: sql<number>`COUNT(*)`,})- .where(- siteAnd(- eq(tables.pipelineQueue.id, taskId),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )-- if (shouldRetry) {- this.logger.warn(- `Task ${taskId} failed (attempt ${newAttempts}/${task.maxAttempts}), will retry in ${retryDelay}ms: ${errorMessage}`,- )- } else {- this.logger.error(- `Task ${taskId} failed permanently after ${newAttempts} attempts: ${errorMessage}`,- )- }+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.siteId, DEFAULT_SITE.id))+ .groupBy(tables.pipelineQueue.status)+ .all()+ return Object.fromEntries(stats.map((row) => [row.status, row.count]))}-- /** 任务处理器 */- private processors = (() => {- return {- photo: async (task: PipelineQueueItem) => {- const { id: taskId, payload } = task- if (payload.type !== 'photo') {- throw new Error(- `Invalid payload type for photo task: ${payload.type}`,- )- }- const { storageKey } = payload- const storageProvider = await storageForReference(- task,- getStorageManager().getProvider(),- )- const photoId = generateSafePhotoId(storageKey)-- try {- this.logger.info(`Start processing task ${taskId}: ${storageKey}`)-- let storageObject = await storageProvider.getFileMeta(storageKey)- if (!storageObject) {- // Fallback: try read the file directly to confirm existence (e.g., local provider)- const maybeBuffer = await storageProvider.get(storageKey)- if (maybeBuffer) {- storageObject = {- key: storageKey,- size: maybeBuffer.length,- lastModified: new Date(),- }- }- }- if (!storageObject) {- throw new PermanentTaskError(`Storage object not found`)- }-- // STEP 1: 预处理 - 转换 HEIC 到 JPEG 并上传- await this.updateTaskStage(taskId, 'preprocessing')- this.logger.info(`[${taskId}:in-stage] preprocessing`)- const imageBuffers = await preprocessImageWithJpegUpload(- storageKey,- storageProvider,- )- if (!imageBuffers) {- throw new Error('Preprocessing failed')- }-- // STEP 2: 元数据处理 - 使用 Sharp 处理图片元数据- await this.updateTaskStage(taskId, 'metadata')- this.logger.info(`[${taskId}:in-stage] metadata extraction`)- const processedData = await processImageMetadataAndSharp(- imageBuffers.processed,- storageKey,- )- if (!processedData) {- throw new Error('Metadata processing failed')- }-- const { imageBuffer, metadata } = processedData-- // STEP 3: 生成缩略图- await this.updateTaskStage(taskId, 'thumbnail')- this.logger.info(`[${taskId}:in-stage] thumbnail generation`)- const { thumbnailBuffer, thumbnailHash } =- await generateThumbnailAndHash(imageBuffer, this.logger)-- // 上传缩略图到存储服务- const thumbnailObject = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await storageProvider.create(- `thumbnails/${photoId}.webp`,- thumbnailBuffer,- 'image/webp',- )- resolve(result)- } catch (error) {- reject(error)- }- })- })-- // STEP 4: 提取 EXIF 数据- await this.updateTaskStage(taskId, 'exif')- this.logger.info(`[${taskId}:in-stage] exif extraction`)- const exifData = await extractExifData(- imageBuffer,- imageBuffers.raw,- this.logger,- )- const systemAutoEraseLocationOnUpload =- (await settingsManager.get<boolean>(- 'privacy',- 'upload.autoEraseLocation',- )) ?? false- const shouldAutoEraseLocationOnUpload =- typeof payload.eraseLocation === 'boolean'- ? payload.eraseLocation- : systemAutoEraseLocationOnUpload- const normalizedExifData = shouldAutoEraseLocationOnUpload- ? stripLocationFromExif(exifData)- : exifData-- // 提取照片基本信息- const photoInfo = extractPhotoInfo(storageKey, normalizedExifData)-- // STEP 5: 地理位置反向解析- // 这里逆编码失败不报错,宽容处理- await this.updateTaskStage(taskId, 'reverse-geocoding')- this.logger.info(`[${taskId}:in-stage] reverse geocoding`)-- let coordinates = null- let locationInfo = null- if (!shouldAutoEraseLocationOnUpload && normalizedExifData) {- const { latitude, longitude } =- parseGPSCoordinates(normalizedExifData)- if (hasValidCoordinates(latitude, longitude)) {- coordinates = { latitude, longitude }- locationInfo = await extractLocationFromGPS(latitude, longitude)- }- }-- // STEP 6: Motion Photo (XMP) 支持- await this.updateTaskStage(taskId, 'motion-photo')- this.logger.info(`[${taskId}:in-stage] motion photo detection`)- const motionPhotoInfo = imageBuffers.raw- ? await processMotionPhotoFromXmp({- photoId,- storageKey,- rawImageBuffer: imageBuffers.raw,- exifData: normalizedExifData,- storageProvider,- logger: this.logger,- })- : null-- if (!imageBuffers.raw) {- this.logger.warn(- `[${taskId}:in-stage] motion photo detection skipped: missing raw buffer for ${storageKey}`,- )- }-- // STEP 7: LivePhoto 视频配对(独立 MOV 文件)- await this.updateTaskStage(taskId, 'live-photo')- this.logger.info(`[${taskId}:in-stage] live photo detection`)- let livePhotoInfo = null- if (!motionPhotoInfo?.isMotionPhoto) {- const livePhotoVideo = await findLivePhotoVideoForImage(- storageKey,- storageProvider,+ private async process(+ lease: JobLease,+ signal: AbortSignal,+ repository: Repository,+ ) {+ const db = useDB(),+ payload = lease.payload+ if (payload.type === 'location-migration') {+ repository.commit(lease, () => enqueueLocationMigration(db))+ return+ }+ const provider = await storageForReference(+ lease,+ getStorageManager().getProvider(),+ )+ const generated = new Set<string>(),+ linked = new Set<string>()+ const namespace = `.generated/${lease.siteId}/${lease.id}/${lease.sourceVersion}/${lease.attemptId}/`+ let published = false+ const staged = new Proxy(provider, {+ get(target, property) {+ if (property === 'create')+ return async (key: string, buffer: Buffer, contentType?: string) => {+ repository.assertCurrent(lease)+ signal.throwIfAborted()+ const safe = key+ .replace(/\\/g, '/')+ .split('/')+ .filter((part) => part && part !== '.' && part !== '..')+ .join('/')+ const object = await target.create(+ namespace + safe,+ buffer,+ contentType,)- if (livePhotoVideo) {- livePhotoInfo = {- isLivePhoto: 1,- livePhotoVideoUrl: storageProvider.getPublicUrl(- livePhotoVideo.videoKey,- ),- livePhotoVideoKey: livePhotoVideo.videoKey,- }- this.logger.info(- `[${taskId}:in-stage] found LivePhoto video: ${livePhotoVideo.videoKey}`,- )- }- } else {- livePhotoInfo = {- isLivePhoto: 1,- livePhotoVideoUrl: motionPhotoInfo.livePhotoVideoUrl || null,- livePhotoVideoKey: motionPhotoInfo.livePhotoVideoKey || null,- }+ generated.add(object.key)+ return object}-- // 构建最终的 Photo 对象- const result: typeof tables.photos.$inferInsert = {- siteId: task.siteId,- storageConnectionId: task.storageConnectionId,- storageRevision: task.storageRevision,- sourceVersion: task.sourceVersion,- id: photoId,- title: photoInfo.title,- description: photoInfo.description,- dateTaken: photoInfo.dateTaken,- tags: photoInfo.tags,- width: metadata.width,- height: metadata.height,- aspectRatio: metadata.width / metadata.height,- storageKey: storageKey,- thumbnailKey: thumbnailObject.key,- fileSize: storageObject.size || null,- lastModified:- storageObject.lastModified?.toISOString() ||- new Date().toISOString(),- originalUrl: imageBuffers.jpegKey- ? storageProvider.getPublicUrl(imageBuffers.jpegKey) // 使用 JPEG 版本作为 originalUrl- : storageProvider.getPublicUrl(storageKey),- thumbnailUrl: storageProvider.getPublicUrl(thumbnailObject.key),- thumbnailHash: thumbnailHash- ? compressUint8Array(thumbnailHash)- : null,- exif: normalizedExifData,- // 地理位置信息- latitude: coordinates?.latitude ?? null,- longitude: coordinates?.longitude ?? null,- country: locationInfo?.country || null,- city: locationInfo?.city || null,- locationName: locationInfo?.locationName || null,- geocoding: locationInfo?.geocoding || null,- // LivePhoto 相关字段- isLivePhoto:- motionPhotoInfo?.isMotionPhoto || livePhotoInfo?.isLivePhoto- ? 1- : 0,- livePhotoVideoUrl:- motionPhotoInfo?.livePhotoVideoUrl ||- livePhotoInfo?.livePhotoVideoUrl ||- null,- livePhotoVideoKey:- motionPhotoInfo?.livePhotoVideoKey ||- livePhotoInfo?.livePhotoVideoKey ||- null,- }-- const assignment = persistPhotoWithAlbum(- useDB(),- result,- payload.albumId,- )- if (assignment === QueueWarningCode.AlbumMissing) {- this.logger.warn(- `Task ${taskId}: album ${payload.albumId} was deleted before photo ${photoId} could be added`,+ if (property === 'delete' || property === 'createFromFile')+ return () =>+ Promise.reject(+ new Error('Processing must publish versioned objects'),)- }-- if (shouldAutoEraseLocationOnUpload) {- try {- await this.addTask(- {- type: 'photo-erase-location',- photoId,- },- {- priority: 2,- maxAttempts: 3,- },- )- } catch (enqueueError) {- this.logger.warn(- `[${taskId}:location-erase] failed to enqueue location erase task for ${photoId}`,- enqueueError,- )- }- }-- this.logger.success(`Task ${taskId} processed successfully`)- return assignment === QueueWarningCode.AlbumMissing- ? QueueWarningCode.AlbumMissing- : null- } catch (error) {- this.logger.error(`Task ${taskId} processing failed`, error)- throw error- }+ const value = Reflect.get(target, property, target)+ return typeof value === 'function' ? value.bind(target) : value},- reverseGeocoding: async (task: PipelineQueueItem) => {- const db = useDB()- const { id: taskId, payload } = task-- if (payload.type !== 'photo-reverse-geocoding') {- throw new Error(- `Invalid payload type for reverse geocoding task: ${payload.type}`,+ })+ const stage = async (value: PipelineQueueItem['statusStage']) => {+ signal.throwIfAborted()+ repository.stage(lease, value)+ }+ const verifySource = async (key: string, hash: string) => {+ repository.assertCurrent(lease)+ signal.throwIfAborted()+ const fresh = await provider.get(key)+ if (!fresh || createHash('sha256').update(fresh).digest('hex') !== hash)+ throw new LostLeaseError('Source object changed during processing')+ }+ try {+ if (payload.type === 'photo') {+ if (!lease.resourceId)+ throw new PermanentJobError('Missing resource identity')+ const process = createPhotoProcessor({+ logger: this.logger,+ exif: createExifService({+ exiftool,+ tempDir: tmpdir(),+ logger: this.logger,+ }),+ parseGPS: parseGPSCoordinates,+ reverseGeocode: extractLocationFromGPS,+ })+ const erase =+ payload.eraseLocation ??+ (await settingsManager.get<boolean>(+ 'privacy',+ 'upload.autoEraseLocation',+ )) ??+ false+ const { photo, sourceHash } = await process({+ photoId: lease.resourceId,+ storageKey: payload.storageKey,+ eraseLocation: erase,+ storage: staged,+ sourceRead: (hash) => repository.bindSourceHash(lease, hash),+ stage,+ signal,+ })+ if (+ photo.livePhotoVideoKey &&+ !generated.has(photo.livePhotoVideoKey)+ ) {+ const base = payload.storageKey.slice(+ 0,+ -path.extname(payload.storageKey).length,)+ if ([`${base}.MOV`, `${base}.mov`].includes(photo.livePhotoVideoKey))+ linked.add(photo.livePhotoVideoKey)}-- const { photoId } = payload-- try {- await this.updateTaskStage(taskId, 'reverse-geocoding')- this.logger.info(- `[${taskId}:in-stage] reverse geocoding for photo ${photoId}`,- )-- const photo = await db- .select()- .from(tables.photos)- .where(- siteAnd(- eq(tables.photos.id, photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )- .get()-- if (!photo) {- this.logger.warn(- `[${taskId}:reverse-geocoding] photo ${photoId} not found`,- )- throw new PermanentTaskError(`Photo ${photoId} not found`)- }-- if (- payload.migrationOnly &&- (photo.geocoding ||- !hasValidCoordinates(photo.latitude, photo.longitude))- )- return-- // An explicit queued coordinate must still belong to this photo. In- // particular, a refresh queued before erasure must not restore it.- if (!matchesLocationRequest(photo, payload)) {- throw new PermanentTaskError(- `Location changed for photo ${photoId}`,- )- }-- let latitude = payload.latitude ?? photo.latitude ?? undefined- let longitude = payload.longitude ?? photo.longitude ?? undefined-- if (- latitude === undefined ||- latitude === null ||- longitude === undefined ||- longitude === null- ) {- if (photo.exif) {- const coords = parseGPSCoordinates(photo.exif)- if (latitude === undefined || latitude === null) {- latitude = coords.latitude- }- if (longitude === undefined || longitude === null) {- longitude = coords.longitude- }- }- }-- const hasLatitude = latitude !== undefined && latitude !== null- const hasLongitude = longitude !== undefined && longitude !== null-- if (!hasLatitude || !hasLongitude) {- this.logger.warn(- `[${taskId}:reverse-geocoding] missing coordinates for photo ${photoId}`,- )- await db- .update(tables.photos)- .set({- latitude: null,- longitude: null,- country: null,- city: null,- locationName: null,- geocoding: null,- })- .where(- siteAnd(- eq(tables.photos.id, photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )- throw new PermanentTaskError(- `Missing coordinates for photo ${photoId}`,- )- }-- if (!hasValidCoordinates(latitude, longitude)) {- throw new PermanentTaskError(- `Invalid coordinates for photo ${photoId}`,- )- }-- const locationInfo = await extractLocationFromGPS(- latitude!,- longitude!,- )-- if (!locationInfo) {- throw new Error(- `Failed to extract location from GPS coordinates (${latitude}, ${longitude}), maybe network issue?`,+ validatePhotoPublication(photo, {+ resourceId: lease.resourceId,+ sourceKey: payload.storageKey,+ generatedKeys: generated,+ linkedVideoKeys: linked,+ publicUrl: (key) => provider.getPublicUrl(key),+ })+ await verifySource(payload.storageKey, sourceHash)+ let warning: QueueWarningCode | null = null+ const publication = repository.commit(+ lease,+ () => {+ const assignment = createPhotoPersistenceRepository(+ db,+ DEFAULT_SITE,+ ).persist(+ {+ ...photo,+ siteId: lease.siteId,+ storageConnectionId: lease.storageConnectionId,+ storageRevision: lease.storageRevision,+ sourceVersion: lease.sourceVersion,+ },+ payload.albumId,)- }-- const changes = updateGeocodedPhoto(- db,- photo,- locationInfo,- Boolean(payload.migrationOnly),- )- if (changes === 0) {- throw new PermanentTaskError(- `Location changed for photo ${photoId}; result discarded`,- )- }-- this.logger.success(- `[${taskId}:reverse-geocoding] updated location for photo ${photoId}`,- )- } catch (error) {- this.logger.error(- `[${taskId}:reverse-geocoding] failed for photo ${photoId}`,- error,+ warning =+ assignment === QueueWarningCode.AlbumMissing ? assignment : null+ },+ { resourceUpdated: true, warningCode: () => warning },+ )+ published = publication.published+ return+ }+ if (payload.type === 'photo-reverse-geocoding') {+ await stage('reverse-geocoding')+ const photo = db+ .select()+ .from(tables.photos)+ .where(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.id, payload.photoId),+ ),)- throw error- }- },- eraseLocation: async (task: PipelineQueueItem) => {- const db = useDB()- const { id: taskId, payload } = task- const storageProvider = await storageForReference(- task,- getStorageManager().getProvider(),+ .get()+ if (!photo) throw new PermanentJobError('Photo not found')+ if (+ payload.migrationOnly &&+ (photo.geocoding ||+ !hasValidCoordinates(photo.latitude, photo.longitude)))-- if (payload.type !== 'photo-erase-location') {- throw new Error(- `Invalid payload type for erase location task: ${payload.type}`,+ return+ if (!matchesLocationRequest(photo, payload))+ throw new PermanentJobError('Location changed')+ const parsed = photo.exif ? parseGPSCoordinates(photo.exif) : {}+ const latitude = payload.latitude ?? photo.latitude ?? parsed.latitude,+ longitude = payload.longitude ?? photo.longitude ?? parsed.longitude+ if (latitude == null || longitude == null) {+ repository.commit(+ lease,+ () => {+ db.update(tables.photos)+ .set({+ latitude: null,+ longitude: null,+ country: null,+ city: null,+ locationName: null,+ geocoding: null,+ })+ .where(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.id, photo.id),+ ),+ )+ .run()+ },+ { resourceUpdated: true, failure: 'Missing coordinates' },)+ throw new PermanentJobError('Missing coordinates')}-- await this.updateTaskStage(taskId, 'location-erase')- this.logger.info(- `[${taskId}:in-stage] erase location info for photo ${payload.photoId}`,+ if (!hasValidCoordinates(latitude, longitude))+ throw new PermanentJobError('Invalid coordinates')+ const location = await extractLocationFromGPS(latitude, longitude)+ if (!location) throw new Error('Reverse geocoding unavailable')+ repository.commit(+ lease,+ () => {+ if (+ !updateGeocodedPhoto(+ db,+ photo,+ location,+ Boolean(payload.migrationOnly),+ )+ )+ throw new LostLeaseError('Location changed')+ },+ { resourceUpdated: true },)-- const photo = await db+ published = true+ return+ }+ if (payload.type === 'photo-erase-location') {+ await stage('location-erase')+ const photo = db.select().from(tables.photos).where(- siteAnd(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),eq(tables.photos.id, payload.photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),),).get()-- if (!photo) {- throw new PermanentTaskError(`Photo ${payload.photoId} not found`)- }-- if (!photo.storageKey) {- throw new PermanentTaskError(- `Photo ${payload.photoId} has no storage key`,- )- }-- const originalBuffer = await storageProvider.get(photo.storageKey)- if (!originalBuffer) {- throw new PermanentTaskError(- `Photo file ${photo.storageKey} not found in storage`,- )- }-- const tempRoot = tmpdir()- await mkdir(tempRoot, { recursive: true })- const tempDir = await mkdtemp(path.join(tempRoot, 'cframe-location-'))- const ext = path.extname(photo.storageKey) || '.jpg'- const tempFile = path.join(tempDir, `erase-location${ext}`)-+ if (!photo?.storageKey)+ throw new PermanentJobError('Photo source not found')+ const original = await provider.get(photo.storageKey)+ if (!original) throw new PermanentJobError('Source object not found')+ const hash = createHash('sha256').update(original).digest('hex')+ const directory = await mkdtemp(path.join(tmpdir(), 'cframe-location-'))try {- await writeFile(tempFile, originalBuffer)-- const exifLocationNullMap = EXIF_LOCATION_KEYS.reduce(- (acc, key) => {- acc[key] = null- return acc- },- {} as Record<string, null>,+ const file = path.join(+ directory,+ 'source' + (path.extname(photo.storageKey) || '.jpg'),)-- await exiftool.write(tempFile, exifLocationNullMap, [- '-overwrite_original',- ])-- const updatedBuffer = await readFile(tempFile)-- await storageProvider.create(photo.storageKey, updatedBuffer)-- const exifData = stripLocationFromExif(- await extractExifData(updatedBuffer),+ await writeFile(file, original)+ const tags = Object.fromEntries(+ EXIF_LOCATION_KEYS.map((key) => [key, null]),)-- await db- .update(tables.photos)- .set({- exif: exifData,- fileSize: updatedBuffer.length,- lastModified: new Date().toISOString(),- latitude: null,- longitude: null,- country: null,- city: null,- locationName: null,- geocoding: null,- })- .where(- siteAnd(- eq(tables.photos.id, payload.photoId),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )-- this.logger.success(- `[${taskId}:location-erase] erased location info for photo ${payload.photoId}`,+ await exiftool.write(file, tags, ['-overwrite_original'])+ const buffer = await readFile(file)+ const object = await staged.create(+ 'source' + (path.extname(photo.storageKey) || '.jpg'),+ buffer,)- } finally {- await rm(tempDir, { recursive: true, force: true })- }- },- livePhotoDetect: async (task: PipelineQueueItem) => {- const db = useDB()- const storageProvider = await storageForReference(- task,- getStorageManager().getProvider(),- )-- const { id: taskId, payload } = task- if (payload.type !== 'live-photo-video') {- throw new Error(- `Invalid payload type for live-photo task: ${payload.type}`,+ let derived = photo.originalDerivedKey+ if (derived) {+ const oldDerived = await provider.get(derived)+ if (!oldDerived)+ throw new PermanentJobError('Derived source not found')+ const jpeg = path.join(directory, 'derived.jpeg')+ await writeFile(jpeg, oldDerived)+ await exiftool.write(jpeg, tags, ['-overwrite_original'])+ derived = (+ await staged.create(+ 'original.jpeg',+ await readFile(jpeg),+ 'image/jpeg',+ )+ ).key+ }+ const exif = stripLocationFromExif(+ await createExifService({+ exiftool,+ tempDir: tmpdir(),+ logger: this.logger,+ }).extractExifData(buffer),)+ await verifySource(photo.storageKey, hash)+ published = repository.commit(+ lease,+ () => {+ db.update(tables.photos)+ .set({+ storageKey: object.key,+ originalDerivedKey: derived,+ originalUrl: provider.getPublicUrl(derived ?? object.key),+ exif,+ fileSize: buffer.length,+ lastModified: new Date().toISOString(),+ latitude: null,+ longitude: null,+ country: null,+ city: null,+ locationName: null,+ geocoding: null,+ })+ .where(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.id, photo.id),+ ),+ )+ .run()+ },+ { resourceUpdated: true },+ ).published+ if (published)+ for (const key of [photo.storageKey, photo.originalDerivedKey])+ if (key && !generated.has(key))+ await provider+ .delete(key)+ .catch((error) =>+ this.logger.warn(+ 'Old source cleanup failed',+ error instanceof Error ? error.message : 'storage error',+ ),+ )+ } finally {+ await rm(directory, { recursive: true, force: true })}- const { storageKey: videoKey } = payload-- try {- this.logger.info(- `Start processing LivePhoto detection task ${taskId}: ${videoKey}`,+ return+ }+ if (payload.type === 'live-photo-video') {+ await stage('live-photo')+ if (!lease.resourceId)+ throw new PermanentJobError('Missing photo identity')+ const photo = db+ .select()+ .from(tables.photos)+ .where(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.id, lease.resourceId),+ ),)-- let storageObject = await storageProvider.getFileMeta(videoKey)- if (!storageObject) {- const maybeBuffer = await storageProvider.get(videoKey)- if (maybeBuffer) {- storageObject = {- key: videoKey,- size: maybeBuffer.length,- lastModified: new Date(),- }- }- }- if (!storageObject) {- throw new PermanentTaskError(`Storage object not found`)- }-- // 寻找是否有同名的照片文件- const videoDir = path.dirname(videoKey)- const videoBaseName = path.basename(videoKey, path.extname(videoKey))-- const possiblePhotoKeys = [- path.join(videoDir, `${videoBaseName}.HEIC`).replace(/\\/g, '/'),- path.join(videoDir, `${videoBaseName}.heic`).replace(/\\/g, '/'),- path.join(videoDir, `${videoBaseName}.JPG`).replace(/\\/g, '/'),- path.join(videoDir, `${videoBaseName}.jpg`).replace(/\\/g, '/'),- path.join(videoDir, `${videoBaseName}.JPEG`).replace(/\\/g, '/'),- path.join(videoDir, `${videoBaseName}.jpeg`).replace(/\\/g, '/'),- ]-- let matchedPhoto: Photo | null = null- for (const photoKey of possiblePhotoKeys) {- const photos = await db- .select()- .from(tables.photos)+ .get()+ if (!photo) throw new Error('Matching photo is not ready')+ const video = await provider.get(payload.storageKey)+ if (!video) throw new PermanentJobError('Video source not found')+ await verifySource(+ payload.storageKey,+ createHash('sha256').update(video).digest('hex'),+ )+ repository.commit(+ lease,+ () => {+ db.update(tables.photos)+ .set({+ isLivePhoto: 1,+ livePhotoVideoKey: payload.storageKey,+ livePhotoVideoUrl: provider.getPublicUrl(payload.storageKey),+ }).where(- siteAnd(- eq(tables.photos.storageKey, photoKey),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.id, photo.id),),)- .limit(1)-- const matched = photos[0]- if (matched) {- matchedPhoto = matched- this.logger.info(- `Found matching photo for LivePhoto video: ${photoKey}`,- )- break- }- }-- if (!matchedPhoto) {- this.logger.warn(- `No matching photo found for LivePhoto video: ${videoKey}`,- )- throw new Error(- `No matching photo found for LivePhoto video: ${videoKey}`,- )- }-- const livePhotoVideoUrl = storageProvider.getPublicUrl(videoKey)- await db- .update(tables.photos)- .set({- isLivePhoto: 1,- livePhotoVideoUrl,- livePhotoVideoKey: videoKey,- })- .where(- siteAnd(- eq(tables.photos.id, matchedPhoto.id),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),+ .run()+ },+ { resourceUpdated: true },+ )+ published = true+ }+ } finally {+ if (!published)+ for (const key of generated)+ await provider+ .delete(key)+ .catch((error) =>+ this.logger.warn(+ 'Unpublished attempt cleanup failed',+ error instanceof Error ? error.message : 'storage error',),)-- this.logger.success(- `LivePhoto detection task ${taskId} processed successfully, updated photo ${matchedPhoto.id}`,- )- } catch (error) {- this.logger.error(- `LivePhoto detection task ${taskId} processing failed`,- error,- )- throw error- }- },}- })()-- /**- * 处理下一个待处理任务- */- private async processNextTask(): Promise<void> {- if (this.isProcessing) {- this.logger.debug('Task is already processing, skipping this poll')- return- }-+ }+ private async processNextTask() {+ if (this.isProcessing) returnthis.isProcessing = true-try {- const task = await this.getNextTask()- if (!task) {- this.logger.debug('No tasks to process at the moment')- return- }-- try {- const { type } = task.payload-- let warningCode: PipelineQueueItem['warningCode'] = null- switch (type) {- case 'location-migration':- enqueueLocationMigration(useDB())- break- case 'live-photo-video':- await this.processors.livePhotoDetect(task)- break- case 'photo':- warningCode = await this.processors.photo(task)- break- case 'photo-reverse-geocoding':- await this.processors.reverseGeocoding(task)- break- case 'photo-erase-location':- await this.processors.eraseLocation(task)- break- default:- throw new PermanentTaskError(`Unknown task type: ${type}`)- }-- await this.markTaskCompleted(task.id, warningCode)- this.processedCount++- this.logger.success(- `[${this.workerId}] Task ${task.id} processed successfully (Total: ${this.processedCount})`,- )- } catch (error) {- const errorMessage =- error instanceof Error ? error.message : String(error)- await this.markTaskFailed(- task.id,- errorMessage,- !(error instanceof PermanentTaskError),- )- this.errorCount++- this.logger.error(- `[${this.workerId}] Task ${task.id} processing failed (Error: ${this.errorCount}):`,- errorMessage,- )- }+ const repository = this.repository()+ const outcome = await runOneJob({+ repository,+ workerId: this.workerId,+ site: DEFAULT_SITE,+ process: (lease, signal) => this.process(lease, signal, repository),+ onError: (error) =>+ this.logger.warn(+ error instanceof Error ? error.message : 'Queue operation failed',+ ),+ })+ if (outcome === 'completed') this.processedCount+++ if (outcome === 'failed') this.errorCount++} catch (error) {- this.logger.error('Error occurred while fetching the next task:', error)+ this.logger.error(+ 'Queue execution failed',+ error instanceof Error ? error.message : 'unknown error',+ )} finally {this.isProcessing = false}}-- /**- * 开始处理队列- * @param intervalMs 轮询间隔- */- startProcessing(intervalMs: number = 3000): void {+ startProcessing(intervalMs = 3000) {if (this.processingInterval) return-- this.processingInterval = setInterval(() => {- this.runTask()- }, intervalMs)-- this.logger.success(- `Queue processing started with interval: ${intervalMs}ms`,- )-+ this.processingInterval = setInterval(() => this.runTask(), intervalMs)this.runTask()}-- private runTask(): void {+ private runTask() {if (this.activeTask) return- this.activeTask = this.processNextTask()- .catch((error) => this.logger.error('Error processing queue:', error))- .finally(() => {- this.activeTask = null- })+ this.activeTask = this.processNextTask().finally(() => {+ this.activeTask = null+ })}-- /**- * 停止处理队列- */- async stopProcessing(): Promise<void> {+ async stopProcessing() {if (this.processingInterval) {clearInterval(this.processingInterval)this.processingInterval = null- this.logger.warn('Queue processing stopped')}await this.activeTask}-- /**- * 获取队列统计信息- * @returns 队列统计信息- */- async getQueueStats() {- const db = useDB()- const stats = await db- .select({- status: tables.pipelineQueue.status,- count: sql<number>`COUNT(*)`,- })- .from(tables.pipelineQueue)- .where(siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id))- .groupBy(tables.pipelineQueue.status)-- return stats.reduce(- (acc, stat) => {- acc[stat.status] = stat.count- return acc- },- {} as Record<string, number>,- )- }}diff --git a/server/services/pipeline-queue/worker-pool.ts b/server/services/pipeline-queue/worker-pool.tsindex ccf11124..4c5db006 100644--- a/server/services/pipeline-queue/worker-pool.ts+++ b/server/services/pipeline-queue/worker-pool.ts@@ -1,9 +1,5 @@-import { and as siteAnd, eq as siteEq } from 'drizzle-orm'-import { DEFAULT_SITE } from '@chronoframe/core'import type { ConsolaInstance } from 'consola'import { consola } from 'consola'-import { eq, sql } from 'drizzle-orm'-import { useDB, tables } from '~~/server/utils/db'import { QueueManager } from './manager'import {DEFAULT_QUEUE_WORKER_COUNT,@@ -81,7 +77,7 @@ export class WorkerPool {this.stopping = null// 在启动工作器之前,清理死任务- await this.cleanupDeadTasks()+ // Expired leases are recovered by the jobs adapter during each claim.// 创建工作器for (let i = 1; i <= this.config.workerCount; i++) {@@ -254,56 +250,6 @@ export class WorkerPool {/*** 清理死任务 - 重置为 pending,迁移任务保留后台优先级*/- private async cleanupDeadTasks(): Promise<void> {- try {- const db = useDB()-- // 查找所有 in-stages 状态的任务(这些可能是死任务)- const deadTasks = await db- .select()- .from(tables.pipelineQueue)- .where(- siteAnd(- eq(tables.pipelineQueue.status, 'in-stages'),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )-- if (deadTasks.length > 0) {- this.logger.warn(- `Found ${deadTasks.length} dead tasks in 'in-stages' status, resetting to pending with migration priorities preserved`,- )-- // 重置为 pending,清除状态阶段;迁移任务保持低优先级- await db- .update(tables.pipelineQueue)- .set({- status: 'pending',- priority: sql`CASE- WHEN json_extract(${tables.pipelineQueue.payload}, '$.migrationOnly') = 1- AND json_extract(${tables.pipelineQueue.payload}, '$.type') = 'photo-reverse-geocoding'- THEN -1 ELSE 1 END`,- statusStage: null,- })- .where(- siteAnd(- eq(tables.pipelineQueue.status, 'in-stages'),- siteEq(tables.pipelineQueue.siteId, DEFAULT_SITE.id),- ),- )-- this.logger.success(- `Successfully reset ${deadTasks.length} dead tasks to pending status with migration priorities preserved`,- )- } else {- this.logger.info('No dead tasks found in queue')- }- } catch (error) {- this.logger.error('Failed to cleanup dead tasks:', error)- // 不抛出错误,允许工作器池继续启动- }- }-/*** 检查池是否正在运行*/@@ -355,7 +301,9 @@ export class WorkerPool {/*** 添加任务到队列(通过任意一个工作器)*/- async addTask(payload: any, options?: any): Promise<number> {+ async addTask(+ ...[payload, options]: Parameters<QueueManager['addTask']>+ ): Promise<number> {const worker = this.getFirstWorker()if (!worker) {throw new Error('No workers available')diff --git a/server/services/video/livephoto.ts b/server/services/video/livephoto.tsindex ae424db6..6c60653e 100644--- a/server/services/video/livephoto.ts+++ b/server/services/video/livephoto.ts@@ -1,3 +1,5 @@+import { createSiteRepository } from '@chronoframe/database'+import { storageForPhoto } from '../../utils/object-storage'import type { StorageProvider } from '@chronoframe/storage'import { and as siteAnd, eq as siteEq } from 'drizzle-orm'import { DEFAULT_SITE } from '@chronoframe/core'@@ -13,7 +15,7 @@ export const processLivePhotoVideo = async (videoKey: string,_videoSize: number,): Promise<boolean> => {- const storageProvider = getStorageManager().getProvider()+ let storageProvider = getStorageManager().getProvider()const db = useDB()try {@@ -64,23 +66,16 @@ export const processLivePhotoVideo = async (return false}- // 获取视频的公共 URL- const videoUrl = storageProvider.getPublicUrl(videoKey)-- // 更新照片记录,设置 LivePhoto 信息- await db- .update(tables.photos)- .set({- isLivePhoto: 1,- livePhotoVideoUrl: videoUrl,- livePhotoVideoKey: videoKey,- })- .where(- siteAnd(- eq(tables.photos.id, matchedPhoto.id),- siteEq(tables.photos.siteId, DEFAULT_SITE.id),- ),- )+ const repository = createSiteRepository(db, DEFAULT_SITE)+ const snapshot = await repository.capturePhoto(matchedPhoto.id)+ if (!snapshot) return false+ storageProvider = await storageForPhoto(matchedPhoto.id, storageProvider)+ if (!(await storageProvider.getFileMeta(videoKey))) return false+ await repository.updatePhotoSnapshot(matchedPhoto.id, snapshot.version, {+ isLivePhoto: 1,+ livePhotoVideoUrl: storageProvider.getPublicUrl(videoKey),+ livePhotoVideoKey: videoKey,+ })logger.chrono.success(`Successfully processed LivePhoto: ${matchedPhoto.id}, video: ${videoKey}`,diff --git a/server/utils/queue-payload.ts b/server/utils/queue-payload.tsindex 19217e5a..03d6f14a 100644--- a/server/utils/queue-payload.ts+++ b/server/utils/queue-payload.ts@@ -1,35 +1,12 @@import { and as siteAnd, eq as siteEq } from 'drizzle-orm'import { DEFAULT_SITE } from '@chronoframe/core'-import { z } from 'zod'+import type { z } from 'zod'import { eq } from 'drizzle-orm'import * as tables from '../database/schema.ts'import { useDB } from './db.ts'-export const queuePayloadSchema = z.discriminatedUnion('type', [- z.object({ type: z.literal('location-migration') }),- z.object({- type: z.literal('photo'),- storageKey: z.string().nonempty(),- eraseLocation: z.boolean().optional(),- albumId: z.number().int().positive().optional(),- }),- z- .object({- type: z.literal('live-photo-video'),- storageKey: z.string().nonempty(),- })- .strict(),- z.object({- type: z.literal('photo-reverse-geocoding'),- photoId: z.string().min(1),- latitude: z.number().min(-90).max(90).optional(),- longitude: z.number().min(-180).max(180).optional(),- }),- z.object({- type: z.literal('photo-erase-location'),- photoId: z.string().min(1),- }),-])+export { queuePayloadSchema } from '@chronoframe/contracts/queue'+import { queuePayloadSchema } from '@chronoframe/contracts/queue'export const queuePayloadRequiresAdmin = (payload: z.infer<typeof queuePayloadSchema>,diff --git a/server/utils/resource-writer.ts b/server/utils/resource-writer.tsindex f706ad01..cdaf8934 100644--- a/server/utils/resource-writer.ts+++ b/server/utils/resource-writer.ts@@ -1,5 +1,5 @@import { persistenceErrorCode } from '@chronoframe/database'-import { storageForPhoto } from './object-storage'+import { storageForPhoto, storageForReference } from './object-storage'import { DEFAULT_SITE } from '@chronoframe/core'import type {} from '../types/worker'import { createError, type H3Event } from 'h3'@@ -34,7 +34,16 @@ export function usePhotoWriter(event: H3Event) {repository: createSqlitePhotoWriteRepository(useDB(), DEFAULT_SITE),clock: { now: () => new Date() },storage: {- forPhoto: (photo) => storageForPhoto(photo.id, storageProvider),+ forPhoto: (photo) =>+ photo.storageReference+ ? storageForReference(+ {+ storageConnectionId: photo.storageReference.connectionId,+ storageRevision: photo.storageReference.revision,+ },+ storageProvider,+ )+ : storageForPhoto(photo.id, storageProvider),},editMetadata: async (photo, input) => {if (!photo.storageKey)@@ -45,20 +54,6 @@ export function usePhotoWriter(event: H3Event) {await storageForPhoto(photo.id, storageProvider),)},- tasks: {- async reverseGeocode(photoId, coordinates) {- if (!globalThis.__workerPool) {- logger.location.warn(- `Worker pool not initialized, skipping reverse geocoding enqueue for photo ${photoId}`,- )- return- }- await globalThis.__workerPool.addTask(- { type: 'photo-reverse-geocoding', photoId, ...coordinates },- { priority: 1 },- )- },- },warn(error, operation) {logger.image.warn(`Photo operation failed: ${operation}`, error)},diff --git a/server/utils/upload-source.ts b/server/utils/upload-source.tsnew file mode 100644index 00000000..230e4b08--- /dev/null+++ b/server/utils/upload-source.ts@@ -0,0 +1,148 @@+import { isDeepStrictEqual } from 'node:util'+import { and, desc, eq } from 'drizzle-orm'+import { DEFAULT_SITE } from '@chronoframe/core'+import { reserveResource } from '@chronoframe/database'+import type { StorageProvider } from '@chronoframe/storage'+import { useDB, tables } from './db'+import { generateSafePhotoId } from './file-utils'+import { storageForReference } from './object-storage'++const references = new WeakMap<+ StorageProvider,+ { storageConnectionId: number; storageRevision: number }+>()+function referenceForProvider(provider: StorageProvider) {+ const cached = references.get(provider)+ if (cached) return cached+ const db = useDB()+ const selected = db+ .select({ value: tables.settings.value })+ .from(tables.settings)+ .where(+ and(+ eq(tables.settings.siteId, DEFAULT_SITE.id),+ eq(tables.settings.namespace, 'storage'),+ eq(tables.settings.key, 'provider'),+ ),+ )+ .get()+ const current = selected?.value+ ? db+ .select()+ .from(tables.settings_storage_providers)+ .where(+ and(+ eq(tables.settings_storage_providers.siteId, DEFAULT_SITE.id),+ eq(tables.settings_storage_providers.id, Number(selected.value)),+ ),+ )+ .get()+ : undefined+ let reference:+ | { storageConnectionId: number; storageRevision: number }+ | undefined+ if (current && isDeepStrictEqual(current.config, provider.config))+ reference = {+ storageConnectionId: current.id,+ storageRevision: current.revision,+ }+ else {+ const history = db+ .select()+ .from(tables.storageConnectionVersions)+ .where(eq(tables.storageConnectionVersions.siteId, DEFAULT_SITE.id))+ .orderBy(+ desc(tables.storageConnectionVersions.connectionId),+ desc(tables.storageConnectionVersions.revision),+ )+ .all()+ .find((row) => isDeepStrictEqual(row.config, provider.config))+ if (history)+ reference = {+ storageConnectionId: history.connectionId,+ storageRevision: history.revision,+ }+ }+ if (!reference) throw new Error('Upload storage connection is not registered')+ references.set(provider, reference)+ return reference+}+function idForKey(key: string) {+ return (+ useDB()+ .select({ id: tables.photos.id })+ .from(tables.photos)+ .where(+ and(+ eq(tables.photos.siteId, DEFAULT_SITE.id),+ eq(tables.photos.storageKey, key),+ ),+ )+ .get()?.id ?? generateSafePhotoId(key)+ )+}+/** Invalidate the prior generation before a managed upload can replace its source. */+export function reserveUploadSource(key: string, provider: StorageProvider) {+ return reserveResource(useDB(), DEFAULT_SITE, idForKey(key), {+ ...referenceForProvider(provider),+ storageKey: key,+ })+}++/** Keep an issued Local upload on its original connection when settings change. */+export async function providerForUpload(+ key: string,+ fallback: StorageProvider,+) {+ const reference = useDB()+ .select()+ .from(tables.resourceVersions)+ .where(+ and(+ eq(tables.resourceVersions.siteId, DEFAULT_SITE.id),+ eq(tables.resourceVersions.resourceId, idForKey(key)),+ eq(tables.resourceVersions.storageKey, key),+ ),+ )+ .get()+ return reference ? storageForReference(reference, fallback) : fallback+}++/** Completion callbacks reserve a new generation immediately before publishing bytes. */+export function sourceAwareUploadStorage(+ provider: StorageProvider,+): StorageProvider {+ return new Proxy(provider, {+ get(target, property) {+ if (property === 'create')+ return (...args: Parameters<StorageProvider['create']>) => {+ reserveUploadSource(args[0], provider)+ return target.create(...args)+ }+ if (property === 'createFromFile')+ return (...args: Parameters<StorageProvider['createFromFile']>) => {+ reserveUploadSource(args[0], provider)+ return target.createFromFile(...args)+ }+ if (property === 'prepareDirectUpload' && target.prepareDirectUpload)+ return async (+ ...args: Parameters<+ NonNullable<StorageProvider['prepareDirectUpload']>+ >+ ) => {+ const plan = await target.prepareDirectUpload!(...args)+ if (plan?.mode === 'multipart')+ return {+ ...plan,+ complete: async (parts: Parameters<typeof plan.complete>[0]) => {+ reserveUploadSource(args[0], provider)+ await plan.complete(parts)+ },+ }+ return plan+ }+ const value = Reflect.get(target, property, target)+ return typeof value === 'function' ? value.bind(target) : value+ },+ })+}diff --git a/shared/utils/queue-warning.ts b/shared/utils/queue-warning.tsindex 01d4f371..1ccff995 100644--- a/shared/utils/queue-warning.ts+++ b/shared/utils/queue-warning.ts@@ -1 +1,4 @@-export * from '@chronoframe/contracts/queue'+export {+ QueueWarningCode,+ QUEUE_WARNING_CODES,+} from '@chronoframe/contracts/queue'diff --git a/tests/core-mutations.test.ts b/tests/core-mutations.test.tsindex 2334ae06..86f99c6b 100644--- a/tests/core-mutations.test.ts+++ b/tests/core-mutations.test.ts@@ -130,7 +130,7 @@ test('album mutations preserve ordering, password semantics, cover membership an}})-test('photo mutations keep normalized EXIF and database edits aligned and tolerate queue or cleanup failures', async () => {+test('photo mutations keep normalized EXIF and database edits aligned and persist follow-up tasks atomically and tolerate cleanup failures', async () => {const { sqlite, db } = setup()const deleted: string[] = []const edits: unknown[] = []@@ -162,11 +162,6 @@ test('photo mutations keep normalized EXIF and database edits aligned and toleraedits.push(input)return { exif: null, fileSize: 100 }},- tasks: {- reverseGeocode: async () => {- throw new Error('queue failure')- },- },warn: (_error, operation) => warnings.push(operation),})await assert.rejects(@@ -192,7 +187,15 @@ test('photo mutations keep normalized EXIF and database edits aligned and toleratags: ['Tag', 'Other'],location: { latitude: 32, longitude: 120 },})- assert.deepEqual(warnings, ['reverse-geocoding'])+ assert.deepEqual(warnings, [])+ const queued = db.select().from(schema.pipelineQueue).all()+ assert.equal(queued.length, 1)+ assert.deepEqual(queued[0]!.payload, {+ type: 'photo-reverse-geocoding',+ photoId: 'p',+ latitude: 32,+ longitude: 120,+ })const erased = await writer.update('p',{ title: ' ', location: null },diff --git a/tests/job-leases.test.ts b/tests/job-leases.test.tsnew file mode 100644index 00000000..8a800d7d--- /dev/null+++ b/tests/job-leases.test.ts@@ -0,0 +1,449 @@+import assert from 'node:assert/strict'+import test from 'node:test'+import { mkdtemp, rm } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { eq, sql } from 'drizzle-orm'+import { DEFAULT_SITE } from '@chronoframe/core'+import {+ LostLeaseError,+ PermanentJobError,+ runOneJob,+ type JobLease,+} from '@chronoframe/jobs'+import {+ openDatabase,+ tables,+ createSqliteJobRepository,+ createSqlitePhotoWriteRepository,+ reserveResource,+} from '@chronoframe/database'++const migrationsFolder = 'packages/database/migrations'+const clock = new Date('2026-10-20T00:00:00Z')+const resolveId = (+ payload: typeof tables.pipelineQueue.$inferSelect.payload,+) =>+ 'photoId' in payload+ ? payload.photoId+ : 'storageKey' in payload+ ? 'photo'+ : null++test('independent connections preserve live leases, recover expired attempts and fence duplicate callbacks', async () => {+ const root = await mkdtemp(join(tmpdir(), 'chronoframe-leases-'))+ const filename = join(root, 'app.sqlite3'),+ first = openDatabase({ filename, migrationsFolder }),+ second = openDatabase({ filename, migrationsFolder })+ try {+ first.db+ .insert(tables.photos)+ .values({ id: 'photo', storageKey: 'original.jpg', title: 'before' })+ .run()+ const a = createSqliteJobRepository(first.db, DEFAULT_SITE, resolveId),+ b = createSqliteJobRepository(second.db, DEFAULT_SITE, resolveId)+ const id = a.enqueue({ type: 'photo', storageKey: 'original.jpg' })+ const duplicate = a.enqueue({ type: 'photo', storageKey: 'original.jpg' })+ const old = await a.claim('first', clock, 1000)+ assert.equal(old!.id, id)+ const other = await b.claim('second', clock, 1000)+ assert.equal(other!.id, duplicate)+ assert.equal(+ await b.claim('restarted', new Date(clock.getTime() + 200), 1000),+ null,+ )+ assert.equal(+ await a.heartbeat(old!, new Date(clock.getTime() + 500), 1000),+ true,+ )+ const recovered = await b.claim(+ 'second-restarted',+ new Date(clock.getTime() + 1501),+ 1000,+ )+ assert.equal(recovered!.id, id)+ assert.equal(recovered!.attempts, 2)+ assert.notEqual(recovered!.attemptId, old!.attemptId)+ const now = new Date(clock.getTime() + 1600)+ assert.throws(+ () =>+ a.commit(+ old!,+ () =>+ first.db+ .update(tables.photos)+ .set({ title: 'stale' })+ .where(eq(tables.photos.id, 'photo'))+ .run(),+ { now, resourceUpdated: true },+ ),+ LostLeaseError,+ )+ const publication = b.commit(+ recovered!,+ () =>+ second.db+ .update(tables.photos)+ .set({ title: 'published' })+ .where(eq(tables.photos.id, 'photo'))+ .run(),+ { now, resourceUpdated: true },+ )+ assert.equal(publication.published, true)+ assert.equal(+ b.commit(+ recovered!,+ () => {+ throw new Error('duplicate callback ran')+ },+ { now },+ ).published,+ false,+ )+ assert.equal(await b.complete(recovered!, now), true)+ assert.equal(await a.complete(old!, now), false)+ assert.equal(+ first.db.select().from(tables.photos).get()!.title,+ 'published',+ )+ assert.equal(first.db.select().from(tables.jobCommitGuards).all().length, 0)+ assert.equal(+ await b.claim('next', new Date(clock.getTime() + 3000), 1000),+ null,+ )+ assert.equal(+ first.db+ .select()+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.id, duplicate))+ .get()!.status,+ 'failed',+ )+ } finally {+ first.close()+ second.close()+ await rm(root, { recursive: true, force: true })+ }+})++test('deletion and source replacement reject late results, while a deliberate reupload can publish', async () => {+ const connection = openDatabase({ filename: ':memory:', migrationsFolder }),+ { db } = connection+ try {+ db.insert(tables.photos)+ .values({ id: 'photo', storageKey: 'original.jpg', title: 'original' })+ .run()+ const repository = createSqliteJobRepository(db, DEFAULT_SITE, resolveId)+ repository.enqueue({ type: 'photo', storageKey: 'original.jpg' })+ const old = await repository.claim('worker', clock, 10_000)+ db.delete(tables.photos).where(eq(tables.photos.id, 'photo')).run()+ assert.throws(+ () =>+ repository.commit(+ old!,+ () =>+ db+ .insert(tables.photos)+ .values({ id: 'photo', title: 'resurrected' })+ .run(),+ { now: clock, resourceUpdated: true },+ ),+ LostLeaseError,+ )+ assert.equal(db.select().from(tables.photos).all().length, 0)+ reserveResource(db, DEFAULT_SITE, 'photo', {+ storageConnectionId: null,+ storageRevision: 1,+ storageKey: 'replacement.jpg',+ })+ repository.enqueue({ type: 'photo', storageKey: 'replacement.jpg' })+ const replacement = await repository.claim('worker-new', clock, 10_000)+ repository.commit(+ replacement!,+ () =>+ db+ .insert(tables.photos)+ .values({+ id: 'photo',+ title: 'replacement',+ storageKey: 'replacement.jpg',+ })+ .run(),+ { now: clock, resourceUpdated: true },+ )+ repository.enqueue({ type: 'photo', storageKey: 'replacement.jpg' })+ const changed = await repository.claim('worker-change', clock, 10_000)+ reserveResource(db, DEFAULT_SITE, 'photo', {+ storageConnectionId: null,+ storageRevision: 1,+ storageKey: 'newer.jpg',+ })+ assert.throws(+ () =>+ repository.commit(+ changed!,+ () =>+ db+ .update(tables.photos)+ .set({ title: 'stale source' })+ .where(eq(tables.photos.id, 'photo'))+ .run(),+ { now: clock, resourceUpdated: true },+ ),+ LostLeaseError,+ )+ assert.equal(db.select().from(tables.photos).get()!.title, 'replacement')+ assert.equal(+ await repository.fail(changed!, 'Source replaced', false, clock),+ true,+ )+ } finally {+ connection.close()+ }+})++test('business result, job completion and commit guard roll back together', async () => {+ const connection = openDatabase({ filename: ':memory:', migrationsFolder }),+ { db } = connection+ try {+ db.insert(tables.photos).values({ id: 'photo', title: 'before' }).run()+ const repository = createSqliteJobRepository(db, DEFAULT_SITE, resolveId)+ const id = repository.enqueue({ type: 'photo', storageKey: 'original.jpg' })+ const lease = await repository.claim('worker', clock, 1000)+ assert.throws(() =>+ repository.commit(+ lease!,+ () => {+ db.update(tables.photos)+ .set({ title: 'uncommitted' })+ .where(eq(tables.photos.id, 'photo'))+ .run()+ throw new Error('injected commit failure')+ },+ { now: clock, resourceUpdated: true },+ ),+ )+ assert.equal(db.select().from(tables.photos).get()!.title, 'before')+ assert.equal(db.select().from(tables.jobCommitGuards).all().length, 0)+ assert.equal(+ db+ .select()+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.id, id))+ .get()!.status,+ 'in-stages',+ )+ assert.throws(+ () =>+ repository.commit({ ...lease!, sourceVersion: 999 }, () => undefined, {+ now: clock,+ }),+ LostLeaseError,+ )+ assert.throws(+ () =>+ repository.commit(+ { ...lease!, payload: { type: 'photo', storageKey: 'forged.jpg' } },+ () => undefined,+ { now: clock },+ ),+ LostLeaseError,+ )+ assert.equal(await repository.fail(lease!, 'retry', true, clock), true)+ assert.equal(await repository.claim('retry-too-early', clock, 1000), null)+ const retry = await repository.claim(+ 'retry',+ new Date(clock.getTime() + 1001),+ 1000,+ )+ assert.equal(retry!.attempts, 2)+ } finally {+ connection.close()+ }+})++test('metadata and follow-up task share a transaction and payloads contain no credentials', async () => {+ const connection = openDatabase({ filename: ':memory:', migrationsFolder }),+ { db } = connection+ try {+ db.insert(tables.photos).values({ id: 'photo', title: 'before' }).run()+ const repository = createSqlitePhotoWriteRepository(db, DEFAULT_SITE)+ const before = db.select().from(tables.photos).get()+ db.run(+ sql.raw(+ "CREATE TRIGGER injected_enqueue_failure BEFORE INSERT ON pipeline_queue BEGIN SELECT RAISE(ABORT,'injected enqueue failure'); END",+ ),+ )+ await assert.rejects(+ repository.update(+ 'photo',+ { title: 'rollback' },+ { latitude: 31, longitude: 121 },+ ),+ )+ assert.deepEqual(db.select().from(tables.photos).get(), before)+ assert.equal(db.select().from(tables.pipelineQueue).all().length, 0)+ db.run(sql.raw('DROP TRIGGER injected_enqueue_failure'))+ await repository.update(+ 'photo',+ { title: 'updated' },+ { latitude: 31, longitude: 121 },+ )+ assert.equal(db.select().from(tables.pipelineQueue).all().length, 1)+ const jobs = createSqliteJobRepository(db, DEFAULT_SITE, resolveId)+ assert.throws(() =>+ jobs.enqueue({+ type: 'photo',+ storageKey: 'image.jpg',+ accessKeyId: 'fixture-never-persist',+ } as never),+ )+ db.insert(tables.sites)+ .values({ id: 2, workspaceId: 1, name: 'second' })+ .run()+ const other = createSqliteJobRepository(db, { id: 2 }, resolveId)+ assert.throws(() =>+ other.enqueue({ type: 'photo-reverse-geocoding', photoId: 'photo' }),+ )+ assert.equal(await other.claim('other', clock, 1000), null)+ } finally {+ connection.close()+ }+})++test('heartbeat loss aborts cooperatively and keeps the native slot until processing settles', async () => {+ let release!: () => void,+ started = false,+ finished = false,+ failed = false+ const gate = new Promise<void>((resolve) => {+ release = resolve+ })+ const lease: JobLease = {+ id: 1,+ siteId: 1,+ resourceId: null,+ payload: { type: 'location-migration' },+ sourceVersion: 1,+ pipelineVersion: '1',+ attemptId: 'attempt',+ workerId: 'worker',+ attempts: 1,+ maxAttempts: 3,+ storageConnectionId: null,+ storageRevision: 1,+ }+ const run = runOneJob({+ site: DEFAULT_SITE,+ workerId: 'worker',+ leaseMs: 100,+ heartbeatMs: 10,+ repository: {+ claim: async () => lease,+ heartbeat: async () => false,+ complete: async () => {+ throw new Error('lost lease completed')+ },+ fail: async () => {+ failed = true+ return true+ },+ },+ process: async (_lease, signal) => {+ started = true+ await gate+ assert.equal(signal.aborted, true)+ finished = true+ },+ })+ await new Promise((resolve) => setTimeout(resolve, 40))+ assert.equal(started, true)+ assert.equal(finished, false)+ assert.equal(failed, false)+ release()+ assert.equal(await run, 'failed')+ assert.equal(finished, true)+ assert.equal(failed, true)+})++test('source digests survive retries and unknown pipelines are never executed', async () => {+ const connection = openDatabase({ filename: ':memory:', migrationsFolder }),+ { db } = connection+ try {+ const repository = createSqliteJobRepository(db, DEFAULT_SITE, resolveId)+ repository.enqueue({ type: 'photo', storageKey: 'source.jpg' })+ const lease = await repository.claim('first', clock, 1000)+ repository.bindSourceHash(lease!, 'a'.repeat(64), clock)+ await repository.fail(lease!, 'transient', true, clock)+ const retryNow = new Date(clock.getTime() + 1001)+ const retry = await repository.claim('second', retryNow, 1000)+ assert.throws(+ () => repository.bindSourceHash(retry!, 'b'.repeat(64), retryNow),+ PermanentJobError,+ )+ assert.equal(+ db.select().from(tables.pipelineQueue).get()!.sourceHash,+ 'a'.repeat(64),+ )+ await repository.fail(retry!, 'source changed', false, retryNow)+ const unsupported = repository.enqueue({+ type: 'photo',+ storageKey: 'new.jpg',+ })+ db.update(tables.pipelineQueue)+ .set({ pipelineVersion: 'future' })+ .where(eq(tables.pipelineQueue.id, unsupported))+ .run()+ assert.equal(await repository.claim('old-worker', retryNow, 1000), null)+ assert.equal(+ db+ .select()+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.id, unsupported))+ .get()!.status,+ 'failed',+ )+ } finally {+ connection.close()+ }+})++test('maintenance snapshots reject stale writes and preserve published derived object keys', async () => {+ const connection = openDatabase({ filename: ':memory:', migrationsFolder }),+ { db } = connection+ try {+ const { createSiteRepository } = await import('@chronoframe/database')+ db.insert(tables.photos)+ .values({+ id: 'photo',+ storageKey: 'original.heic',+ thumbnailKey: 'library/.generated/thumb.webp',+ originalDerivedKey: 'library/.generated/original.jpeg',+ title: 'before',+ })+ .run()+ const repository = createSiteRepository(db, DEFAULT_SITE)+ const snapshot = await repository.capturePhoto('photo')+ db.update(tables.photos)+ .set({ title: 'newer' })+ .where(eq(tables.photos.id, 'photo'))+ .run()+ await assert.rejects(+ repository.updatePhotoSnapshot('photo', snapshot!.version, {+ title: 'stale',+ }),+ )+ const fresh = await repository.capturePhoto('photo')+ await repository.updatePhotoSnapshot('photo', fresh!.version, {+ title: 'reindexed',+ exif: null,+ })+ const row = db.select().from(tables.photos).get()!+ assert.equal(row.title, 'reindexed')+ assert.equal(row.thumbnailKey, 'library/.generated/thumb.webp')+ assert.equal(row.originalDerivedKey, 'library/.generated/original.jpeg')+ assert.equal(await repository.capturePhoto('missing'), null)+ } finally {+ connection.close()+ }+})diff --git a/tests/location-migration.test.ts b/tests/location-migration.test.tsindex 5b17a789..841fc539 100644--- a/tests/location-migration.test.ts+++ b/tests/location-migration.test.ts@@ -30,7 +30,7 @@ test('migration selects only old geocoded photos and deduplicates queue entries'),)sqlite.exec(- 'ALTER TABLE photos ADD COLUMN site_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN site_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN storage_connection_id INTEGER; ALTER TABLE pipeline_queue ADD COLUMN storage_revision INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN source_version INTEGER NOT NULL DEFAULT 1;',+ 'ALTER TABLE photos ADD COLUMN site_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN site_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN storage_connection_id INTEGER; ALTER TABLE pipeline_queue ADD COLUMN storage_revision INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN source_version INTEGER NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN resource_id TEXT; ALTER TABLE pipeline_queue ADD COLUMN source_hash TEXT; ALTER TABLE pipeline_queue ADD COLUMN pipeline_version TEXT NOT NULL DEFAULT 1; ALTER TABLE pipeline_queue ADD COLUMN attempt_id TEXT; ALTER TABLE pipeline_queue ADD COLUMN worker_id TEXT; ALTER TABLE pipeline_queue ADD COLUMN lease_expires_at INTEGER; ALTER TABLE pipeline_queue ADD COLUMN heartbeat_at INTEGER;',)const db = drizzle(sqlite) as Parameters<typeof enqueueLocationMigration>[0]const insert = sqlite.prepare(diff --git a/tests/processing-package.test.ts b/tests/processing-package.test.tsnew file mode 100644index 00000000..9d52ceb4--- /dev/null+++ b/tests/processing-package.test.ts@@ -0,0 +1,124 @@+import assert from 'node:assert/strict'+import test from 'node:test'+import { mkdtemp, readFile, rm } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { createHash } from 'node:crypto'+import sharp from 'sharp'+import { createPhotoProcessor } from '@chronoframe/processing'+import { LocalStorageProvider } from '@chronoframe/storage'+import { validatePhotoPublication } from '@chronoframe/core'++const logger = { info() {}, warn() {}, error() {}, debug() {}, success() {} }+const exif = {+ extractExifData: async () => null,+ extractPhotoInfo: () => ({+ title: 'fixture',+ description: '',+ dateTaken: '2026-10-20',+ tags: [],+ }),+}++for (const format of ['jpeg', 'heic'] as const)+ test(+ `processing composes actual ${format} decoding, metadata, thumbnails and publication ownership`,+ { skip: format === 'heic' && !process.env.CHRONOFRAME_HEIC_FIXTURE },+ async () => {+ const root = await mkdtemp(join(tmpdir(), 'chronoframe-processing-'))+ try {+ const storage = new LocalStorageProvider({+ provider: 'local',+ basePath: root,+ baseUrl: '/storage',+ prefix: 'library',+ })+ const key = `library/fixture.${format}`+ const input =+ format === 'heic'+ ? await readFile(process.env.CHRONOFRAME_HEIC_FIXTURE!)+ : await sharp({+ create: {+ width: 320,+ height: 240,+ channels: 3,+ background: '#5799aa',+ },+ })+ .jpeg()+ .toBuffer()+ await storage.create(key, input)+ const stages: string[] = []+ const processor = createPhotoProcessor({+ logger,+ exif,+ parseGPS: () => ({}),+ reverseGeocode: async () => null,+ })+ const result = await processor({+ photoId: 'fixture',+ storageKey: key,+ eraseLocation: false,+ storage,+ stage: async (stage) => {+ stages.push(stage)+ },+ })+ assert.equal(+ result.sourceHash,+ createHash('sha256').update(input).digest('hex'),+ )+ assert.ok(result.photo.width! > 0 && result.photo.height! > 0)+ assert.equal(+ (+ await sharp(+ (await storage.get(result.photo.thumbnailKey!))!,+ ).metadata()+ ).format,+ 'webp',+ )+ if (format === 'heic') {+ assert.ok(result.photo.originalDerivedKey)+ assert.equal(+ (+ await sharp(+ (await storage.get(result.photo.originalDerivedKey!))!,+ ).metadata()+ ).format,+ 'jpeg',+ )+ }+ assert.deepEqual(stages, [+ 'preprocessing',+ 'metadata',+ 'thumbnail',+ 'exif',+ 'reverse-geocoding',+ 'motion-photo',+ 'live-photo',+ ])+ const expected = {+ resourceId: 'fixture',+ sourceKey: key,+ generatedKeys: new Set([+ result.photo.thumbnailKey!,+ ...(result.photo.originalDerivedKey+ ? [result.photo.originalDerivedKey]+ : []),+ ]),+ linkedVideoKeys: new Set<string>(),+ publicUrl: (objectKey: string) => storage.getPublicUrl(objectKey),+ }+ validatePhotoPublication(result.photo, expected)+ for (const invalid of [+ { ...result.photo, id: 'other-site' },+ { ...result.photo, thumbnailKey: 'foreign.webp' },+ { ...result.photo, fileSize: NaN },+ { ...result.photo, originalUrl: 'https://foreign.invalid/photo' },+ ])+ assert.throws(() => validatePhotoPublication(invalid, expected))+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ },+ )diff --git a/tests/storage-package.test.ts b/tests/storage-package.test.tsindex ce500e50..43198733 100644--- a/tests/storage-package.test.ts+++ b/tests/storage-package.test.ts@@ -251,6 +251,19 @@ test(assert.ok((await provider.listAll()).some((object) => object.key === 'small.jpg'),)+ const prefixed = new S3StorageProvider({+ ...provider.config,+ prefix: '/library/',+ })+ const first = await prefixed.create('edit.jpg', image, 'image/jpeg')+ assert.equal(first.key, 'library/edit.jpg')+ const replacement = Buffer.from('updated-exif-fixture')+ assert.equal(+ (await prefixed.create(first.key, replacement, 'image/jpeg')).key,+ first.key,+ )+ assert.deepEqual(await prefixed.get(first.key), replacement)+ assert.equal(objects.has('library/library/edit.jpg'), false)const size = 8 * 1024 * 1024 + 5const plan = await provider.prepareDirectUpload('large.jpg',