Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

feat!: 照片处理队列和 WorkerPool 管理器,队列管理 API,优化上传处理进度展示

Timothy Yin01cf41a0500e
diff --git a/app/pages/dashboard/photos.vue b/app/pages/dashboard/photos.vue
index 8259f9c4..fdfd5e79 100644
--- a/app/pages/dashboard/photos.vue
+++ b/app/pages/dashboard/photos.vue
@@ -1,6 +1,6 @@
<script lang="ts" setup>
import type { TableColumn } from '@nuxt/ui'
-import type { Photo } from '~~/server/utils/db'
+import type { Photo, PipelineQueueItem } from '~~/server/utils/db'
import { h, resolveComponent } from 'vue'
import { Icon, UBadge } from '#components'
import ThumbImage from '~/components/ThumbImage.vue'
@@ -16,20 +16,27 @@ useHead({
})
const dayjs = useDayjs()
-const { localizeExif } = useExifLocalization()
-const { photos: data, status, refresh } = usePhotos()
-const { filteredPhotos, activeFilters } = usePhotoFilters()
+const { status, refresh } = usePhotos()
+const { filteredPhotos, selectedCounts, hasActiveFilters } = usePhotoFilters()
+
+const totalSelectedFilters = computed(() => {
+ return Object.values(selectedCounts.value).reduce(
+ (total, count) => total + count,
+ 0,
+ )
+})
-// 当前上传的文件信息
interface UploadingFile {
file: File
fileName: string
+ fileId: string
status: 'preparing' | 'uploading' | 'processing' | 'completed' | 'error'
+ stage?: PipelineQueueItem['statusStage'] | null
progress?: number
error?: string
+ taskId?: number
signedUrlResponse?: { signedUrl: string; fileKey: string; expiresIn: number }
- // 添加详细进度信息
uploadProgress?: {
loaded: number
total: number
@@ -39,31 +46,30 @@ interface UploadingFile {
speedText?: string
timeRemainingText?: string
}
- // 添加上传管理器的关键方法
canAbort?: boolean
abortUpload?: () => void
}
const uploadingFiles = ref<Map<string, UploadingFile>>(new Map())
-// 使用预签名 URL 上传文件函数
+// 重写的文件上传和处理函数
const uploadImage = async (file: File) => {
const fileName = file.name
const fileId = `${Date.now()}-${fileName}`
- // 为每个文件创建独立的上传管理器
const uploadManager = useUpload({
timeout: 10 * 60 * 1000, // 10分钟超时
})
- // 添加到上传队列
const uploadingFile: UploadingFile = {
file,
fileName,
+ fileId,
status: 'preparing',
canAbort: false,
abortUpload: () => uploadManager.abortUpload(),
}
+
uploadingFiles.value.set(fileId, uploadingFile)
try {
@@ -80,12 +86,12 @@ const uploadImage = async (file: File) => {
uploadingFile.signedUrlResponse = signedUrlResponse
uploadingFile.status = 'uploading'
uploadingFile.canAbort = true
+ uploadingFile.progress = 0
+ uploadingFiles.value = new Map(uploadingFiles.value)
- // 第二步:使用 composable 上传文件到 S3
+ // 第二步:使用 composable 上传文件到存储
await uploadManager.uploadFile(file, signedUrlResponse.signedUrl, {
onProgress: (progress: UploadProgress) => {
- // 更新文件特定的进度信息
- console.log(`文件 ${fileName} 上传进度:`, progress.percentage + '%')
uploadingFile.progress = progress.percentage
uploadingFile.uploadProgress = {
loaded: progress.loaded,
@@ -98,48 +104,53 @@ const uploadImage = async (file: File) => {
? dayjs.duration(progress.timeRemaining, 'seconds').humanize()
: '',
}
-
- // 触发响应式更新
uploadingFiles.value = new Map(uploadingFiles.value)
},
onStatusChange: (status: string) => {
- console.log(`文件 ${fileName} 上传状态变更:`, status)
- // 根据状态更新 canAbort
uploadingFile.canAbort = status === 'uploading'
uploadingFiles.value = new Map(uploadingFiles.value)
},
- onSuccess: async (xhr: XMLHttpRequest) => {
- console.log(`文件 ${fileName} 上传成功`, xhr.status)
-
- // 第三步:通知服务器开始处理照片
+ onSuccess: async (_xhr: XMLHttpRequest) => {
+ // 第三步:上传完成,提交到队列任务
uploadingFile.status = 'processing'
+ uploadingFile.progress = 100
uploadingFile.canAbort = false
+ uploadingFile.stage = null // 重置 stage,准备显示任务状态
uploadingFiles.value = new Map(uploadingFiles.value)
try {
- await $fetch('/api/photos/process', {
- method: 'POST' as any,
+ const resp = await $fetch('/api/queue/add-task', {
+ method: 'POST',
body: {
- fileKey: signedUrlResponse.fileKey,
- fileName: file.name,
- fileSize: file.size,
+ payload: {
+ type:
+ file.type === 'video/quicktime'
+ ? 'live-photo-video'
+ : 'photo',
+ storageKey: signedUrlResponse.fileKey,
+ },
+ priority: 0,
+ maxAttempts: 3,
},
})
- uploadingFile.status = 'completed'
- uploadingFile.canAbort = false
- uploadingFiles.value = new Map(uploadingFiles.value)
-
- // 开始状态检查
- if (!statusInterval) {
- statusInterval = setInterval(checkProcessingStatus, 2000)
+ if (resp.success) {
+ uploadingFile.taskId = resp.taskId
+ uploadingFile.status = 'processing'
+ uploadingFiles.value = new Map(uploadingFiles.value)
+
+ // 开始任务状态检查
+ startTaskStatusCheck(resp.taskId, fileId)
+ } else {
+ uploadingFile.status = 'error'
+ uploadingFile.error = '提交处理任务失败'
+ uploadingFiles.value = new Map(uploadingFiles.value)
}
} catch (processError: any) {
uploadingFile.status = 'error'
- uploadingFile.error = `处理失败: ${processError.message}`
+ uploadingFile.error = `提交处理任务失败: ${processError.message}`
uploadingFile.canAbort = false
uploadingFiles.value = new Map(uploadingFiles.value)
- throw processError
}
},
onError: (error: string) => {
@@ -147,7 +158,6 @@ const uploadImage = async (file: File) => {
uploadingFile.error = error
uploadingFile.canAbort = false
uploadingFiles.value = new Map(uploadingFiles.value)
- console.error(`文件 ${fileName} 上传失败:`, error)
},
})
} catch (error: any) {
@@ -156,52 +166,39 @@ const uploadImage = async (file: File) => {
uploadingFile.canAbort = false
uploadingFiles.value = new Map(uploadingFiles.value)
- console.error('上传请求失败:', error)
-
// 提供更详细的错误信息
if (error.response?.status === 401) {
- throw new Error('未授权,请重新登录')
+ uploadingFile.error = '未授权,请重新登录'
} else if (error.message?.includes('CORS')) {
- throw new Error('跨域请求失败,请检查存储服务的 CORS 配置')
+ uploadingFile.error = '跨域请求失败,请检查存储服务的 CORS 配置'
} else if (
error.message?.includes('NetworkError') ||
error.name === 'TypeError'
) {
- throw new Error('网络连接失败或 CORS 错误,请检查网络连接和存储配置')
+ uploadingFile.error = '网络连接失败或 CORS 错误,请检查网络连接和存储配置'
} else if (error.message?.includes('上传到存储失败')) {
- throw new Error('文件上传到云存储失败,请重试')
- } else {
- throw new Error(error.message || '上传失败,请重试')
+ uploadingFile.error = '文件上传到云存储失败,请重试'
}
- } finally {
- // 上传完成后从队列中移除(延迟移除以便用户看到结果)
- setTimeout(() => {
- uploadingFiles.value.delete(fileId)
- uploadingFiles.value = new Map(uploadingFiles.value)
- }, 5000)
+
+ uploadingFiles.value = new Map(uploadingFiles.value)
}
}
-// 处理状态
-const processingFiles = ref<Set<string>>(new Set())
const toast = useToast()
const selectedFiles = ref<File[]>([])
-// 表格多选状态 - 使用官方推荐的方式
+// 表格多选状态
const rowSelection = ref({})
const table: any = useTemplateRef('table')
-// 计算选中的行数
const selectedRowsCount = computed((): number => {
return table.value?.tableApi?.getFilteredSelectedRowModel().rows.length || 0
})
-// 计算总行数
const totalRowsCount = computed((): number => {
return table.value?.tableApi?.getFilteredRowModel().rows.length || 0
})
-// 计算 LivePhoto 统计
const livePhotoStats = computed(() => {
if (!filteredPhotos.value) return { total: 0, livePhotos: 0, staticPhotos: 0 }
@@ -214,10 +211,8 @@ const livePhotoStats = computed(() => {
return { total, livePhotos, staticPhotos }
})
-// 照片过滤器
const photoFilter = ref<'all' | 'livephoto' | 'static'>('all')
-// 过滤后的数据
const filteredData = computed(() => {
if (!filteredPhotos.value) return []
@@ -231,8 +226,144 @@ const filteredData = computed(() => {
}
})
-// 状态检查间隔
-let statusInterval: NodeJS.Timeout | null = null
+// 状态检查间隔 Map,每个任务对应一个定时器
+const statusIntervals = ref<Map<number, NodeJS.Timeout>>(new Map())
+
+// 启动任务状态检查
+const startTaskStatusCheck = (taskId: number, fileId: string) => {
+ const intervalId = setInterval(async () => {
+ try {
+ const response = await $fetch(`/api/queue/stats/${taskId}`)
+ const uploadingFile = uploadingFiles.value.get(fileId)
+
+ if (!uploadingFile) {
+ clearInterval(intervalId)
+ statusIntervals.value.delete(taskId)
+ return
+ }
+
+ // 更新任务状态
+ uploadingFile.stage =
+ response.status === 'in-stages' ? response.statusStage : null
+ uploadingFiles.value = new Map(uploadingFiles.value)
+
+ if (response.status === 'completed') {
+ // 任务完成
+ uploadingFile.status = 'completed'
+ uploadingFile.stage = null
+ uploadingFiles.value = new Map(uploadingFiles.value)
+
+ // 停止状态检查
+ clearInterval(intervalId)
+ statusIntervals.value.delete(taskId)
+
+ // 显示成功提示
+ toast.add({
+ title: '照片处理完成',
+ description: `照片 ${uploadingFile.fileName} 已完成处理`,
+ color: 'success',
+ })
+
+ // 刷新照片列表
+ await refresh()
+
+ // 2秒后从界面移除成功的任务
+ setTimeout(() => {
+ uploadingFiles.value.delete(fileId)
+ uploadingFiles.value = new Map(uploadingFiles.value)
+ }, 2000)
+ } else if (response.status === 'failed') {
+ // 任务失败
+ uploadingFile.status = 'error'
+ uploadingFile.error = `处理失败: ${response.errorMessage || '未知错误'}`
+ uploadingFile.stage = null
+ uploadingFiles.value = new Map(uploadingFiles.value)
+
+ // 停止状态检查
+ clearInterval(intervalId)
+ statusIntervals.value.delete(taskId)
+
+ // 显示错误提示
+ toast.add({
+ title: '照片处理失败',
+ description: `照片 ${uploadingFile.fileName} 处理失败: ${response.errorMessage || '未知错误'}`,
+ color: 'error',
+ })
+ // 失败的任务不自动移除,让用户查看错误信息
+ }
+ } catch (error) {
+ console.error('检查任务状态失败:', error)
+
+ // 如果检查状态失败,清理定时器
+ clearInterval(intervalId)
+ statusIntervals.value.delete(taskId)
+
+ const uploadingFile = uploadingFiles.value.get(fileId)
+ if (uploadingFile) {
+ uploadingFile.status = 'error'
+ uploadingFile.error = '状态检查失败,请刷新页面重试'
+ uploadingFiles.value = new Map(uploadingFiles.value)
+ }
+ }
+ }, 1000) // 每秒检查一次
+
+ statusIntervals.value.set(taskId, intervalId)
+}
+
+// 手动移除上传任务
+const removeUploadingFile = (fileId: string) => {
+ const uploadingFile = uploadingFiles.value.get(fileId)
+
+ // 如果任务还在进行中,先清理定时器
+ if (uploadingFile?.taskId) {
+ const intervalId = statusIntervals.value.get(uploadingFile.taskId)
+ if (intervalId) {
+ clearInterval(intervalId)
+ statusIntervals.value.delete(uploadingFile.taskId)
+ }
+ }
+
+ // 从列表中移除
+ uploadingFiles.value.delete(fileId)
+ uploadingFiles.value = new Map(uploadingFiles.value)
+}
+
+// 批量清除已完成和错误的任务
+const clearCompletedTasks = () => {
+ const toRemove: string[] = []
+
+ for (const [fileId, uploadingFile] of uploadingFiles.value) {
+ if (
+ uploadingFile.status === 'completed' ||
+ uploadingFile.status === 'error'
+ ) {
+ toRemove.push(fileId)
+
+ // 清理可能存在的定时器
+ if (uploadingFile.taskId) {
+ const intervalId = statusIntervals.value.get(uploadingFile.taskId)
+ if (intervalId) {
+ clearInterval(intervalId)
+ statusIntervals.value.delete(uploadingFile.taskId)
+ }
+ }
+ }
+ }
+
+ toRemove.forEach((fileId) => {
+ uploadingFiles.value.delete(fileId)
+ })
+
+ uploadingFiles.value = new Map(uploadingFiles.value)
+
+ if (toRemove.length > 0) {
+ toast.add({
+ title: '任务清理完成',
+ description: `已清除 ${toRemove.length} 个已完成的任务`,
+ color: 'info',
+ })
+ }
+}
const columns: TableColumn<Photo>[] = [
{
@@ -386,9 +517,8 @@ const columns: TableColumn<Photo>[] = [
cell: (info) => formatBytes(info.getValue() as number),
},
{
- accessorKey: 'exif.ColorSpace',
+ accessorFn: (row) => row.exif?.ColorSpace,
header: '颜色空间',
- cell: (info) => localizeExif('colorSpace', info.getValue() as string),
},
{
accessorKey: 'actions',
@@ -396,46 +526,6 @@ const columns: TableColumn<Photo>[] = [
},
]
-const checkProcessingStatus = async () => {
- if (processingFiles.value.size === 0) {
- // 如果没有文件在处理,停止检查
- if (statusInterval) {
- clearInterval(statusInterval)
- statusInterval = null
- }
- return
- }
-
- try {
- const response = await $fetch('/api/photos/status')
- const currentPhotoCount = data.value?.length || 0
-
- // 简单检查:如果照片数量增加了,说明有照片处理完成
- if (response.recentPhotos?.length > currentPhotoCount) {
- // 刷新照片列表
- await refresh()
-
- // 清空处理中的文件列表
- processingFiles.value.clear()
-
- // 停止状态检查
- if (statusInterval) {
- clearInterval(statusInterval)
- statusInterval = null
- }
-
- toast.add({
- title: '照片处理完成',
- description: '您的照片已成功处理并上传',
- color: 'success',
- })
- }
- } catch (error) {
- console.error('检查状态失败:', error)
- // 不显示错误给用户,避免频繁弹窗
- }
-}
-
// 文件验证函数
const validateFile = (file: File): { valid: boolean; error?: string } => {
// 检查文件类型
@@ -489,36 +579,10 @@ const handleUpload = async () => {
continue
}
- const fileName = file.name
- const isVideoFile = fileName.toLowerCase().endsWith('.mov')
- processingFiles.value.add(fileName)
-
- toast.add({
- title: isVideoFile ? '开始上传 LivePhoto 视频' : '开始上传照片',
- description: `正在上传 ${fileName} 到云存储...`,
- color: 'info',
- })
-
try {
await uploadImage(file)
-
- toast.add({
- title: '照片上传成功',
- description: `${fileName} 已上传,正在后台处理...`,
- color: 'success',
- })
-
- // 开始状态检查
- if (!statusInterval) {
- statusInterval = setInterval(checkProcessingStatus, 2000)
- }
} catch (error: any) {
- processingFiles.value.delete(fileName)
- toast.add({
- title: '上传失败',
- description: `${fileName}: ${error.message}`,
- color: 'error',
- })
+ // 错误已经在 uploadImage 函数内部处理了
console.error('上传错误:', error)
}
}
@@ -565,59 +629,74 @@ const handleViewLivePhoto = async (photoId: string) => {
}
}
-// LivePhoto 模态框相关状态
-const isLivePhotoModalOpen = ref(false)
-const selectedLivePhoto = ref<{
- id: string
- title: string | null
- originalUrl: string
- videoUrl: string
-} | null>(null)
-
-// 打开链接的辅助函数
-const openInNewTab = (url: string) => {
- if (typeof window !== 'undefined') {
- window.open(url, '_blank')
- }
-}
-
-const handleUpdateLivePhoto = async (photoId: string) => {
+// 重新处理单张照片
+const handleReprocessSingle = async (photo: Photo) => {
try {
- const updateToast = toast.add({
- title: '正在检查 LivePhoto',
- description: '正在检查是否有对应的视频文件...',
+ if (!photo || !photo.storageKey) {
+ toast.add({
+ title: '处理失败',
+ description: '无法找到照片的存储键',
+ color: 'error',
+ })
+ return
+ }
+
+ const reprocessToast = toast.add({
+ title: '正在处理照片',
+ description: '正在添加到任务队列...',
color: 'info',
})
- const result = (await $fetch('/api/photos/livephoto/manage', {
+ const result = await $fetch('/api/queue/add-task', {
method: 'POST',
body: {
- action: 'update-photo',
- photoId: photoId,
+ payload: {
+ type: 'photo',
+ storageKey: photo.storageKey,
+ },
+ priority: 0,
+ maxAttempts: 3,
},
- })) as any
-
- const isSuccess = 'success' in result && result.success
-
- toast.update(updateToast.id, {
- title: isSuccess ? 'LivePhoto 更新成功' : 'LivePhoto 更新失败',
- description: result.message,
- color: isSuccess ? 'success' : 'warning',
})
- if (isSuccess) {
- await refresh()
+ if (result.success) {
+ toast.update(reprocessToast.id, {
+ title: '处理任务已添加',
+ description: `已添加到任务队列,任务ID: ${result.taskId}`,
+ color: 'success',
+ })
+ } else {
+ toast.update(reprocessToast.id, {
+ title: '添加任务失败',
+ description: '无法添加处理任务',
+ color: 'error',
+ })
}
} catch (error: any) {
- console.error('LivePhoto 更新失败:', error)
+ console.error('处理照片失败:', error)
toast.add({
- title: 'LivePhoto 更新失败',
- description: error.message || '更新过程中发生错误',
+ title: '处理失败',
+ description: error.message || '添加任务时发生错误',
color: 'error',
})
}
}
+// LivePhoto Modal
+const isLivePhotoModalOpen = ref(false)
+const selectedLivePhoto = ref<{
+ id: string
+ title: string | null
+ originalUrl: string
+ videoUrl: string
+} | null>(null)
+
+const openInNewTab = (url: string) => {
+ if (typeof window !== 'undefined') {
+ window.open(url, '_blank')
+ }
+}
+
// 批量删除功能
const handleBatchDelete = async () => {
const selectedRowModel = table.value?.tableApi?.getFilteredSelectedRowModel()
@@ -669,8 +748,8 @@ const handleBatchDelete = async () => {
}
}
-// 批量检测 LivePhoto 功能(只对选中的照片)
-const handleBatchDetectLivePhoto = async () => {
+// 批量重新处理照片功能
+const handleBatchReprocess = async () => {
const selectedRowModel = table.value?.tableApi?.getFilteredSelectedRowModel()
const selectedPhotos =
selectedRowModel?.rows.map((row: any) => row.original) || []
@@ -678,138 +757,67 @@ const handleBatchDetectLivePhoto = async () => {
if (selectedPhotos.length === 0) {
toast.add({
title: '请选择照片',
- description: '请至少选择一张照片进行 LivePhoto 检测',
+ description: '请至少选择一张照片进行处理',
color: 'warning',
})
return
}
- try {
- const detectToast = toast.add({
- title: '正在检测 LivePhoto',
- description: `正在检查所选 ${selectedPhotos.length} 张照片的 LivePhoto 状态...`,
- color: 'info',
- })
-
- const photoIds = selectedPhotos.map((photo: any) => photo.id)
-
- const result = (await $fetch('/api/photos/livephoto/manage', {
- method: 'POST',
- body: {
- action: 'detect',
- photoIds: photoIds, // 传递选中的照片ID列表
- },
- })) as any
-
- const foundCount = result.results?.found || 0
- const processedCount = result.results?.processed || 0
-
- toast.update(detectToast.id, {
- title: 'LivePhoto 检测完成',
- description: `检测了 ${processedCount} 张照片,发现 ${foundCount} 个潜在的 LivePhoto`,
- color: foundCount > 0 ? 'success' : 'info',
- })
-
- if (foundCount > 0) {
- await refresh()
- }
-
- // 清空选中状态
- rowSelection.value = {}
- } catch (error: any) {
- console.error('批量检测 LivePhoto 失败:', error)
+ // 检查所有选中照片是否都有 storageKey
+ const photosWithStorageKey = selectedPhotos.filter(
+ (photo: Photo) => photo.storageKey,
+ )
+ if (photosWithStorageKey.length !== selectedPhotos.length) {
toast.add({
- title: '批量检测失败',
- description: error.message || '检测过程中发生错误',
+ title: '处理失败',
+ description: `${selectedPhotos.length - photosWithStorageKey.length} 张照片缺少存储信息,无法处理`,
color: 'error',
})
- }
-}
-
-// 批量重新索引 EXIF 功能(只对选中的照片)
-const handleBatchReindexExif = async () => {
- const selectedRowModel = table.value?.tableApi?.getFilteredSelectedRowModel()
- const selectedPhotos =
- selectedRowModel?.rows.map((row: any) => row.original) || []
-
- if (selectedPhotos.length === 0) {
- toast.add({
- title: '请选择照片',
- description: '请至少选择一张照片进行 EXIF 重新索引',
- color: 'warning',
- })
return
}
try {
- const reindexToast = toast.add({
- title: '正在重新索引 EXIF',
- description: `正在重新提取所选 ${selectedPhotos.length} 张照片的 EXIF 数据...`,
+ const reprocessToast = toast.add({
+ title: '正在添加处理任务',
+ description: `正在为所选 ${photosWithStorageKey.length} 张照片添加处理任务...`,
color: 'info',
})
- const photoIds = selectedPhotos.map((photo: any) => photo.id)
-
- const result = (await $fetch('/api/photos/exif/reindex', {
+ const result = await $fetch('/api/queue/add-tasks', {
method: 'POST',
body: {
- action: 'batch-reindex',
- photoIds: photoIds, // 传递选中的照片ID列表
+ tasks: photosWithStorageKey.map((photo: Photo) => ({
+ payload: {
+ type: 'photo',
+ storageKey: photo.storageKey,
+ },
+ priority: 0,
+ maxAttempts: 3,
+ })),
},
- })) as any
-
- const processedCount = result.results?.processed || 0
- const updatedCount = result.results?.updated || 0
-
- toast.update(reindexToast.id, {
- title: 'EXIF 重新索引完成',
- description: `处理了 ${processedCount} 张照片,更新了 ${updatedCount} 张照片的 EXIF 数据`,
- color: 'success',
})
- await refresh()
+ if (result.success) {
+ toast.update(reprocessToast.id, {
+ title: '任务已添加',
+ description: `已将 ${photosWithStorageKey.length} 张照片添加到处理队列`,
+ color: 'success',
+ })
+ } else {
+ toast.update(reprocessToast.id, {
+ title: '添加任务失败',
+ description: '无法添加批量重新处理任务',
+ color: 'error',
+ })
+ }
// 清空选中状态
rowSelection.value = {}
} catch (error: any) {
- console.error('批量重新索引 EXIF 失败:', error)
+ console.error('批量处理失败:', error)
toast.add({
- title: 'EXIF 重新索引失败',
- description: error.message || '重新索引过程中发生错误',
- color: 'error',
- })
- }
-}
-
-// 单个照片重新索引 EXIF 功能
-const handleReindexSingleExif = async (photoId: string) => {
- try {
- const reindexToast = toast.add({
- title: '正在重新索引 EXIF',
- description: '正在重新提取照片的 EXIF 数据...',
- color: 'info',
- })
-
- const result = (await $fetch('/api/photos/exif/reindex', {
- method: 'POST',
- body: {
- action: 'single-reindex',
- photoId: photoId,
- },
- })) as any
-
- toast.update(reindexToast.id, {
- title: 'EXIF 重新索引完成',
- description: result.message || '照片 EXIF 数据已更新',
- color: 'success',
- })
-
- await refresh()
- } catch (error: any) {
- console.error('重新索引 EXIF 失败:', error)
- toast.add({
- title: 'EXIF 重新索引失败',
- description: error.message || '重新索引过程中发生错误',
+ title: '批量处理失败',
+ description: error.message || '添加任务时发生错误',
color: 'error',
})
}
@@ -817,9 +825,11 @@ const handleReindexSingleExif = async (photoId: string) => {
// 清理定时器
onUnmounted(() => {
- if (statusInterval) {
- clearInterval(statusInterval)
- }
+ // 清理所有状态检查定时器
+ statusIntervals.value.forEach((intervalId) => {
+ clearInterval(intervalId)
+ })
+ statusIntervals.value.clear()
})
</script>
@@ -830,7 +840,18 @@ onUnmounted(() => {
v-if="uploadingFiles.size > 0"
class="space-y-3"
>
- <h3 class="text-lg font-semibold">上传进度</h3>
+ <div class="flex items-center justify-between">
+ <h3 class="text-lg font-semibold">上传进度</h3>
+ <UButton
+ size="sm"
+ color="neutral"
+ variant="ghost"
+ icon="tabler:trash"
+ @click="clearCompletedTasks"
+ >
+ 清除已完成任务
+ </UButton>
+ </div>
<div class="space-y-2">
<div
v-for="[fileId, uploadingFile] of uploadingFiles"
@@ -862,33 +883,66 @@ onUnmounted(() => {
uploadingFile.status === 'preparing'
? '准备中'
: uploadingFile.status === 'uploading'
- ? '上传中'
+ ? `上传中 ${uploadingFile.progress || 0}%`
: uploadingFile.status === 'processing'
- ? '处理中'
+ ? uploadingFile.stage
+ ? uploadingFile.stage === 'preprocessing'
+ ? '预处理中'
+ : uploadingFile.stage === 'metadata'
+ ? '提取元数据'
+ : uploadingFile.stage === 'thumbnail'
+ ? '生成缩略图'
+ : uploadingFile.stage === 'exif'
+ ? '处理EXIF'
+ : uploadingFile.stage === 'reverse-geocoding'
+ ? '地理解析'
+ : uploadingFile.stage === 'live-photo'
+ ? '处理LivePhoto'
+ : '处理中'
+ : '等待处理'
: uploadingFile.status === 'completed'
? '完成'
: '错误'
}}
</UBadge>
- <!-- 中止按钮 -->
- <UButton
- v-if="
- uploadingFile.status === 'uploading' && uploadingFile.canAbort
- "
- size="xs"
- color="error"
- variant="soft"
- icon="tabler:file-x"
- @click="uploadingFile.abortUpload?.()"
- >
- <span class="hidden sm:inline">中止上传</span>
- <span class="sm:hidden">中止</span>
- </UButton>
+ <!-- 操作按钮 -->
+ <div class="flex items-center gap-2">
+ <!-- 中止上传按钮 -->
+ <UButton
+ v-if="
+ uploadingFile.status === 'uploading' &&
+ uploadingFile.canAbort
+ "
+ size="xs"
+ color="error"
+ variant="soft"
+ icon="tabler:file-x"
+ @click="uploadingFile.abortUpload?.()"
+ >
+ <span class="hidden sm:inline">中止上传</span>
+ <span class="sm:hidden">中止</span>
+ </UButton>
+
+ <!-- 清除任务按钮 -->
+ <UButton
+ v-if="
+ uploadingFile.status === 'completed' ||
+ uploadingFile.status === 'error'
+ "
+ size="xs"
+ color="neutral"
+ variant="ghost"
+ icon="tabler:x"
+ @click="removeUploadingFile(fileId)"
+ >
+ <span class="hidden sm:inline">清除</span>
+ </UButton>
+ </div>
</div>
</div>
- <!-- 进度条 -->
+ <!-- 上传进度条 -->
<div
v-if="
uploadingFile.status === 'uploading' &&
@@ -899,7 +953,7 @@ onUnmounted(() => {
<div
class="flex justify-between text-xs sm:text-sm text-neutral-600 dark:text-neutral-400 mb-1"
>
- <span>{{ uploadingFile.progress }}%</span>
+ <span>上传进度: {{ uploadingFile.progress }}%</span>
<span
v-if="uploadingFile.uploadProgress?.speedText"
class="hidden sm:inline"
@@ -915,11 +969,45 @@ onUnmounted(() => {
v-if="uploadingFile.uploadProgress?.timeRemainingText"
class="text-xs text-neutral-500 mt-1"
>
- <span class="hidden sm:inline">ETA: </span
- >{{ uploadingFile.uploadProgress.timeRemainingText }}
+ <span class="hidden sm:inline">预计剩余时间: </span>
+ {{ uploadingFile.uploadProgress.timeRemainingText }}
</div>
</div>
+ <!-- 处理状态进度条 -->
+ <div
+ v-if="uploadingFile.status === 'processing'"
+ class="mb-2"
+ >
+ <div
+ class="flex justify-between text-xs sm:text-sm text-neutral-600 dark:text-neutral-400 mb-1"
+ >
+ <span>
+ 处理状态:
+ {{
+ uploadingFile.stage === 'preprocessing'
+ ? '预处理中'
+ : uploadingFile.stage === 'metadata'
+ ? '提取元数据'
+ : uploadingFile.stage === 'thumbnail'
+ ? '生成缩略图'
+ : uploadingFile.stage === 'exif'
+ ? '处理EXIF信息'
+ : uploadingFile.stage === 'reverse-geocoding'
+ ? '地理位置解析'
+ : uploadingFile.stage === 'live-photo'
+ ? '处理LivePhoto'
+ : '等待处理...'
+ }}
+ </span>
+ </div>
+ <UProgress
+ :model-value="uploadingFile.stage ? undefined : null"
+ animation="carousel"
+ class="h-2"
+ />
+ </div>
+
<!-- 错误信息 -->
<UAlert
v-if="uploadingFile.status === 'error' && uploadingFile.error"
@@ -968,15 +1056,15 @@ onUnmounted(() => {
<!-- 工具栏 -->
<div
- class="flex flex-row sm:items-center justify-between gap-3 sm:gap-0 p-3 sm:p-4 bg-gray-50 dark:bg-gray-900/50 border border-gray-200 dark:border-gray-700 rounded-lg"
+ class="flex flex-row sm:items-center justify-between gap-3 sm:gap-0 p-3 sm:p-4 bg-neutral-50 dark:bg-neutral-900/50 border border-neutral-200 dark:border-neutral-700 rounded-lg"
>
<div class="flex items-center gap-2">
<UIcon
name="tabler:photo"
- class="text-gray-500"
+ class="text-neutral-500"
/>
<span
- class="font-medium text-gray-700 dark:text-gray-300 hidden sm:inline"
+ class="font-medium text-neutral-700 dark:text-neutral-300 hidden sm:inline"
>
照片管理
</span>
@@ -1007,6 +1095,30 @@ onUnmounted(() => {
</div>
<div class="flex items-center gap-2">
+ <UPopover>
+ <UTooltip :text="$t('ui.action.filter.tooltip')">
+ <UChip
+ inset
+ size="sm"
+ color="info"
+ :show="totalSelectedFilters > 0"
+ >
+ <UButton
+ variant="soft"
+ :color="hasActiveFilters ? 'info' : 'neutral'"
+ class="bg-transparent rounded-full cursor-pointer relative"
+ icon="tabler:filter"
+ size="sm"
+ />
+ </UChip>
+ </UTooltip>
+
+ <template #content>
+ <UCard variant="glassmorphism">
+ <OverlayFilterPanel />
+ </UCard>
+ </template>
+ </UPopover>
<!-- 过滤器 -->
<USelectMenu
v-model="photoFilter"
@@ -1026,15 +1138,6 @@ onUnmounted(() => {
>
</USelectMenu>
- <!-- 模糊搜索 -->
- <UInput
- v-model="activeFilters.search"
- size="sm"
- icon="tabler:search"
- placeholder="搜索照片..."
- class="w-32"
- />
-
<!-- 刷新按钮 -->
<UButton
variant="soft"
@@ -1048,68 +1151,6 @@ onUnmounted(() => {
</div>
</div>
- <!-- 批量操作栏 -->
- <div
- v-if="selectedRowsCount > 0"
- class="flex flex-col sm:flex-row sm:items-center justify-between gap-3 sm:gap-0 p-3 sm:p-4 bg-blue-50 dark:bg-blue-950/20 border border-blue-200 dark:border-blue-800 rounded-lg"
- >
- <div class="flex items-center gap-2">
- <UIcon
- name="tabler:check-circle"
- class="text-blue-500"
- />
- <span class="font-medium text-blue-700 dark:text-blue-300">
- <span>已选择</span> {{ selectedRowsCount }} <span>张照片</span>
- </span>
- <UButton
- variant="ghost"
- size="sm"
- @click="rowSelection = {}"
- >
- <span>取消选择</span>
- </UButton>
- </div>
- <div class="flex flex-wrap sm:flex-nowrap items-center gap-1 sm:gap-2">
- <!-- 重新索引 EXIF 按钮 -->
- <UButton
- variant="soft"
- color="success"
- size="sm"
- icon="tabler:refresh"
- class="flex-1 sm:flex-none"
- @click="handleBatchReindexExif"
- >
- <span class="hidden sm:inline">重新索引 EXIF</span>
- <span class="sm:hidden">索引</span>
- </UButton>
-
- <!-- 实况配对按钮 -->
- <UButton
- variant="soft"
- color="info"
- size="sm"
- icon="tabler:live-photo"
- class="flex-1 sm:flex-none"
- @click="handleBatchDetectLivePhoto"
- >
- <span class="hidden sm:inline">实况配对</span>
- <span class="sm:hidden">配对</span>
- </UButton>
-
- <UButton
- color="error"
- variant="soft"
- size="sm"
- icon="tabler:trash"
- class="flex-1 sm:flex-none"
- @click="handleBatchDelete"
- >
- <span class="hidden sm:inline">批量删除</span>
- <span class="sm:hidden">删除</span>
- </UButton>
- </div>
- </div>
-
<!-- 照片列表 -->
<div
class="border border-neutral-300 dark:border-neutral-800 rounded overflow-hidden"
@@ -1124,7 +1165,7 @@ onUnmounted(() => {
:columns="columns"
:loading="status === 'pending'"
sticky
- class="h-[500px] sm:h-[600px]"
+ class="h-[calc(100vh-25rem)] sm:h-[calc(100vh-24.5rem)]"
:ui="{
separator:
'bg-(--ui-color-neutral-200) dark:bg-(--ui-color-neutral-700)',
@@ -1141,22 +1182,16 @@ onUnmounted(() => {
[
{
color: row.original.isLivePhoto ? 'warning' : 'info',
- label: row.original.isLivePhoto ? '实况预览' : '实况配对',
+ label: row.original.isLivePhoto ? '实况预览' : '重新处理',
icon: row.original.isLivePhoto
? 'tabler:live-photo'
: 'tabler:refresh',
onSelect() {
row.original.isLivePhoto
? handleViewLivePhoto(row.original.id)
- : handleUpdateLivePhoto(row.original.id)
+ : handleReprocessSingle(row.original)
},
},
- {
- color: 'success',
- label: '重新索引 EXIF',
- icon: 'tabler:refresh',
- onSelect: () => handleReindexSingleExif(row.original.id),
- },
],
[
{
@@ -1179,11 +1214,43 @@ onUnmounted(() => {
</template>
</UTable>
- <!-- 选择状态信息 -->
+ <!-- 选择状态信息和批量操作 -->
<div
- class="px-4 py-3.5 border-t border-neutral-200 dark:border-neutral-700 text-sm text-neutral-600 dark:text-neutral-400"
+ class="px-4 py-4 border-t border-neutral-200 dark:border-neutral-700"
>
- {{ selectedRowsCount }} / {{ totalRowsCount }} 行已选择
+ <div
+ class="text-sm text-neutral-600 dark:text-neutral-400 flex items-center gap-2"
+ >
+ <div class="leading-6">
+ {{ selectedRowsCount }} / {{ totalRowsCount }} 行已选择
+ </div>
+ <div
+ v-if="selectedRowsCount > 0"
+ class="flex items-center gap-1 sm:gap-2"
+ >
+ <UButton
+ variant="soft"
+ color="info"
+ size="xs"
+ icon="tabler:refresh"
+ class="flex-1 sm:flex-none"
+ @click="handleBatchReprocess"
+ >
+ <span>重新处理</span>
+ </UButton>
+
+ <UButton
+ color="error"
+ variant="soft"
+ size="xs"
+ icon="tabler:trash"
+ class="flex-1 sm:flex-none"
+ @click="handleBatchDelete"
+ >
+ <span>批量删除</span>
+ </UButton>
+ </div>
+ </div>
</div>
</div>
diff --git a/docs/queue-api.md b/docs/queue-api.md
new file mode 100644
index 00000000..ee679491
--- /dev/null
+++ b/docs/queue-api.md
@@ -0,0 +1,228 @@
+# 队列系统 API 使用指南
+
+支持多个并发消费者的队列处理系统。以下是通过 API 创建和管理任务的方法:
+
+## API 概览
+
+### 1. 添加单个任务
+**POST** `/api/queue/add-task`
+
+### 2. 批量添加任务
+**POST** `/api/queue/add-tasks`
+
+### 3. 查看队列状态
+**GET** `/api/queue/status`
+
+## 使用示例
+
+### 1. 添加单个任务
+
+```javascript
+// 添加单个照片处理任务
+const response = await fetch('/api/queue/add-task', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ payload: {
+ storageKey: 'photos/2024/IMG_1234.HEIC'
+ },
+ priority: 5, // 可选,优先级 0-10,默认 0
+ maxAttempts: 3 // 可选,最大重试次数 1-5,默认 3
+ })
+})
+
+const result = await response.json()
+console.log('任务已添加:', result)
+// 输出:
+// {
+// "success": true,
+// "taskId": 123,
+// "message": "Task added to queue successfully",
+// "payload": {
+// "storageKey": "photos/2024/IMG_1234.HEIC",
+// "priority": 5,
+// "maxAttempts": 3
+// }
+// }
+```
+
+### 2. 批量添加任务
+
+```javascript
+// 批量添加多个照片处理任务
+const response = await fetch('/api/queue/add-tasks', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ tasks: [
+ {
+ payload: { storageKey: 'photos/2024/IMG_1234.HEIC' },
+ priority: 8
+ },
+ {
+ payload: { storageKey: 'photos/2024/IMG_1235.HEIC' },
+ priority: 5
+ },
+ {
+ payload: { storageKey: 'photos/2024/IMG_1236.HEIC' }
+ // 使用默认优先级和重试次数
+ }
+ ],
+ defaultPriority: 3, // 可选,默认优先级
+ defaultMaxAttempts: 3 // 可选,默认最大重试次数
+ })
+})
+
+const result = await response.json()
+console.log('批量任务结果:', result)
+// 输出:
+// {
+// "success": true,
+// "totalTasks": 3,
+// "successCount": 3,
+// "errorCount": 0,
+// "results": [
+// { "index": 0, "taskId": 124, "storageKey": "photos/2024/IMG_1234.HEIC", "success": true },
+// { "index": 1, "taskId": 125, "storageKey": "photos/2024/IMG_1235.HEIC", "success": true },
+// { "index": 2, "taskId": 126, "storageKey": "photos/2024/IMG_1236.HEIC", "success": true }
+// ],
+// "message": "Processed 3 tasks: 3 successful, 0 failed"
+// }
+```
+
+### 3. 查看队列状态
+
+```javascript
+// 获取实时队列和工作器状态
+const response = await fetch('/api/queue/status')
+const status = await response.json()
+
+console.log('队列状态:', status)
+// 输出:
+// {
+// "timestamp": "2024-09-24T10:30:00.000Z",
+// "pool": {
+// "isActive": true,
+// "workerCount": 3,
+// "totalWorkers": 3,
+// "activeWorkers": 2,
+// "totalProcessed": 45,
+// "totalErrors": 2,
+// "averageSuccessRate": 95.74,
+// "workers": [
+// {
+// "workerId": "worker-1",
+// "isProcessing": true,
+// "processedCount": 15,
+// "errorCount": 1,
+// "uptime": 3600,
+// "successRate": 93.75
+// },
+// {
+// "workerId": "worker-2",
+// "isProcessing": true,
+// "processedCount": 18,
+// "errorCount": 0,
+// "uptime": 3600,
+// "successRate": 100
+// },
+// {
+// "workerId": "worker-3",
+// "isProcessing": false,
+// "processedCount": 12,
+// "errorCount": 1,
+// "uptime": 3600,
+// "successRate": 92.31
+// }
+// ]
+// },
+// "queue": {
+// "pending": 8,
+// "in-stages": 2,
+// "completed": 45,
+// "failed": 2
+// }
+// }
+```
+
+## 系统特性
+
+### 多消费者并发处理
+- **3个工作器**:同时处理不同的任务
+- **错开轮询**:工作器使用不同的轮询间隔,减少数据库竞争
+- **负载均衡**:自动检测和重启有问题的工作器
+
+### 任务优先级
+- 优先级范围:0-9(9为最高优先级)
+- 高优先级任务会被优先处理
+- 相同优先级按照创建时间排序
+
+### 错误处理和重试
+- 自动重试机制:任务失败时会自动重试
+- 可配置重试次数:1-5次
+- 渐进式退避:重试间隔逐渐增加
+
+### 实时监控
+- **工作器状态**:查看每个工作器的处理统计
+- **队列统计**:查看待处理、处理中、已完成、失败的任务数量
+- **性能指标**:成功率、处理速度、运行时间等
+
+## 在现有代码中集成
+
+### 在照片上传时自动添加任务
+
+```typescript
+// 在 server/api/photos/process.post.ts 中
+export default defineEventHandler(async (event) => {
+ const { storageKey } = await readBody(event)
+
+ // 获取工作器池并添加任务
+ const workerPool = (globalThis as any).__workerPool
+ if (workerPool) {
+ const taskId = await workerPool.addTask(
+ { storageKey },
+ { priority: 5 } // 用户上传的照片给予中等优先级
+ )
+
+ return {
+ message: 'Photo processing started',
+ taskId
+ }
+ }
+})
+```
+
+### 程序内部直接使用
+
+```typescript
+import { QueueManager } from '~~/server/services/pipeline-queue'
+
+// 获取特定工作器实例
+const worker = QueueManager.getInstance('worker-1')
+
+// 添加任务
+const taskId = await worker.addTask({
+ storageKey: 'photos/example.jpg'
+}, {
+ priority: 8,
+ maxAttempts: 5
+})
+```
+
+## 配置说明
+
+工作器池在 `server/plugins/pipeline-queue.ts` 中配置:
+
+```typescript
+const workerPoolConfig = {
+ workerCount: 3, // 消费者数量
+ intervalMs: 2000, // 基础轮询间隔
+ intervalOffset: 667, // 错开时间
+ enableLoadBalancing: true, // 启用负载均衡
+ statsReportInterval: 60000, // 统计报告间隔
+}
+```
\ No newline at end of file
diff --git a/server/api/photos/process.post.ts b/server/api/photos/process.post.ts
deleted file mode 100644
index 1474d6a4..00000000
--- a/server/api/photos/process.post.ts
+++ /dev/null
@@ -1,109 +0,0 @@
-import { execPhotoPipelineAsync } from '~~/server/services/photo/pipeline-async'
-import { useStorageProvider } from '~~/server/utils/useStorageProvider'
-import { isLivePhotoVideo, processLivePhotoVideo } from '~~/server/services/video/livephoto'
-
-export default eventHandler(async (event) => {
- await requireUserSession(event)
-
- const { storageProvider } = useStorageProvider(event)
- const body = await readBody(event)
- const { fileKey, fileName, fileSize } = body
-
- if (!fileKey) {
- throw createError({
- statusCode: 400,
- statusMessage: 'fileKey is required',
- })
- }
-
- try {
- // 验证文件是否存在于存储中
- const fileBuffer = await storageProvider.get(fileKey)
- if (!fileBuffer) {
- throw createError({
- statusCode: 404,
- statusMessage: 'File not found in storage',
- })
- }
-
- // 创建存储对象信息
- const originalObject = {
- key: fileKey,
- size: fileSize || fileBuffer.length,
- lastModified: new Date(),
- }
-
- logger.chrono.info('Starting file processing for', fileKey)
-
- // 检查是否为 LivePhoto 视频文件
- if (fileName && isLivePhotoVideo(fileName, fileSize || fileBuffer.length)) {
- logger.chrono.info('LivePhoto video detected:', fileName)
-
- // 异步处理 LivePhoto 视频
- processLivePhotoVideoInBackground(fileKey, fileSize || fileBuffer.length)
-
- return {
- message: 'LivePhoto video processing started',
- fileKey,
- status: 'processing',
- type: 'livephoto-video'
- }
- }
-
- // 异步处理照片
- processPhotoInBackground(fileKey, originalObject, fileName)
-
- return {
- message: 'Photo processing started',
- fileKey,
- status: 'processing'
- }
- } catch (error) {
- logger.chrono.error('Failed to start photo processing:', error)
- throw createError({
- statusCode: 500,
- statusMessage: 'Failed to start photo processing',
- })
- }
-})
-
-// 后台处理函数
-async function processPhotoInBackground(s3key: string, originalObject: any, originalFileName?: string) {
- try {
- logger.chrono.info('Starting background photo processing for', s3key)
-
- const photo = await execPhotoPipelineAsync(s3key, originalObject)
-
- if (!photo) {
- logger.chrono.error('Photo processing failed for', s3key)
- return
- }
-
- // 如果有原始文件名,使用它作为标题
- if (originalFileName && !photo.title) {
- photo.title = originalFileName
- }
-
- await useDB().insert(tables.photos).values(photo).execute()
- logger.chrono.success('Background photo processing completed for', photo.title || photo.storageKey)
- } catch (error) {
- logger.chrono.error('Background photo processing error for', s3key, error)
- }
-}
-
-// LivePhoto 视频后台处理函数
-async function processLivePhotoVideoInBackground(videoKey: string, videoSize: number) {
- try {
- logger.chrono.info('Starting background LivePhoto video processing for', videoKey)
-
- const success = await processLivePhotoVideo(videoKey, videoSize)
-
- if (success) {
- logger.chrono.success('LivePhoto video processing completed for', videoKey)
- } else {
- logger.chrono.warn('LivePhoto video processing failed - no matching photo found for', videoKey)
- }
- } catch (error) {
- logger.chrono.error('LivePhoto video processing error for', videoKey, error)
- }
-}
diff --git a/server/api/queue/add-task.post.ts b/server/api/queue/add-task.post.ts
new file mode 100644
index 00000000..a13a9019
--- /dev/null
+++ b/server/api/queue/add-task.post.ts
@@ -0,0 +1,54 @@
+import { z } from 'zod'
+
+export default defineEventHandler(async (event) => {
+ await requireUserSession(event)
+
+ try {
+ const { payload, priority, maxAttempts } = await readValidatedBody(
+ event,
+ z.object({
+ payload: z.object({
+ type: z.enum(['photo', 'live-photo-video']).default('photo'),
+ storageKey: z.string().nonempty(),
+ }),
+ priority: z.number().min(0).max(9).optional().default(0),
+ maxAttempts: z.number().min(1).max(5).optional().default(3),
+ }).parse,
+ )
+
+ const workerPool = globalThis.__workerPool
+
+ if (!workerPool) {
+ throw createError({
+ statusCode: 503,
+ statusMessage: 'Worker pool not initialized',
+ })
+ }
+
+ const taskId = await workerPool.addTask(payload, {
+ priority,
+ maxAttempts,
+ })
+
+ return {
+ success: true,
+ taskId,
+ message: 'Task added to queue successfully',
+ payload: {
+ storageKey: payload.storageKey,
+ priority,
+ maxAttempts,
+ },
+ }
+ } catch (error: any) {
+ if (error.statusCode) {
+ throw error
+ }
+
+ throw createError({
+ statusCode: 500,
+ statusMessage:
+ error instanceof Error ? error.message : 'Failed to add task to queue',
+ })
+ }
+})
diff --git a/server/api/queue/add-tasks.post.ts b/server/api/queue/add-tasks.post.ts
new file mode 100644
index 00000000..ff654b7e
--- /dev/null
+++ b/server/api/queue/add-tasks.post.ts
@@ -0,0 +1,88 @@
+import { z } from 'zod'
+
+export default defineEventHandler(async (event) => {
+ await requireUserSession(event)
+
+ try {
+ const { tasks, defaultPriority, defaultMaxAttempts } =
+ await readValidatedBody(
+ event,
+ z.object({
+ tasks: z
+ .array(
+ z.object({
+ payload: z.object({
+ type: z.enum(['photo', 'live-photo-video']).default('photo'),
+ storageKey: z.string().nonempty(),
+ }),
+ priority: z.number().min(0).max(9).optional(),
+ maxAttempts: z.number().min(1).max(5).optional(),
+ }),
+ )
+ .min(1, 'At least one task is required')
+ .max(1000, 'Too many tasks: maximum 1000 tasks per batch'),
+ defaultPriority: z.number().min(0).max(9).optional().default(0),
+ defaultMaxAttempts: z.number().min(1).max(5).optional().default(3),
+ }).parse,
+ )
+
+ const workerPool = globalThis.__workerPool
+
+ if (!workerPool) {
+ throw createError({
+ statusCode: 503,
+ statusMessage: 'Worker pool not initialized',
+ })
+ }
+
+ const results = []
+ const errors = []
+
+ for (let i = 0; i < tasks.length; i++) {
+ const task = tasks[i]
+
+ try {
+ const taskId = await workerPool.addTask(task.payload, {
+ priority: task.priority ?? defaultPriority,
+ maxAttempts: task.maxAttempts ?? defaultMaxAttempts,
+ })
+
+ results.push({
+ index: i,
+ taskId,
+ storageKey: task.payload.storageKey,
+ success: true,
+ })
+ } catch (error) {
+ const errorMessage =
+ error instanceof Error ? error.message : `Task ${i}: Unknown error`
+ errors.push({
+ index: i,
+ storageKey: task.payload?.storageKey || 'unknown',
+ error: errorMessage,
+ success: false,
+ })
+ }
+ }
+
+ return {
+ success: errors.length === 0,
+ totalTasks: tasks.length,
+ successCount: results.length,
+ errorCount: errors.length,
+ results,
+ errors: errors.length > 0 ? errors : undefined,
+ message: `Processed ${tasks.length} tasks: ${results.length} successful, ${errors.length} failed`,
+ }
+ } catch (error: any) {
+ if (error.statusCode) {
+ throw error
+ }
+
+ throw createError({
+ statusCode: 500,
+ statusMessage:
+ error instanceof Error ? error.message : 'Failed to add tasks to queue',
+ })
+ }
+})
diff --git a/server/api/queue/stats/[taskId].get.ts b/server/api/queue/stats/[taskId].get.ts
new file mode 100644
index 00000000..839d5930
--- /dev/null
+++ b/server/api/queue/stats/[taskId].get.ts
@@ -0,0 +1,38 @@
+import z from 'zod'
+
+export default defineEventHandler(async (event) => {
+ await requireUserSession(event)
+
+ try {
+ const { taskId } = await getValidatedRouterParams(
+ event,
+ z.object({
+ taskId: z.string().nonempty(),
+ }).parse,
+ )
+
+ const workerPool = globalThis.__workerPool
+ if (!workerPool) {
+ throw createError({
+ statusCode: 503,
+ statusMessage: 'Worker pool not initialized',
+ })
+ }
+
+ const taskStats = await workerPool.getTaskStatus(Number(taskId))
+ if (!taskStats) {
+ throw createError({
+ statusCode: 404,
+ statusMessage: 'Task not found',
+ })
+ }
+
+ return taskStats
+ } catch (error) {
+ throw createError({
+ statusCode: 500,
+ statusMessage:
+ error instanceof Error ? error.message : 'Failed to get queue status',
+ })
+ }
+})
diff --git a/server/api/queue/stats/index.ts b/server/api/queue/stats/index.ts
new file mode 100644
index 00000000..234c7c87
--- /dev/null
+++ b/server/api/queue/stats/index.ts
@@ -0,0 +1,33 @@
+export default defineEventHandler(async (event) => {
+ await requireUserSession(event)
+
+ try {
+ const workerPool = globalThis.__workerPool
+
+ if (!workerPool) {
+ throw createError({
+ statusCode: 503,
+ statusMessage: 'Worker pool not initialized',
+ })
+ }
+
+ const poolStats = workerPool.getPoolStats()
+ const queueStats = await workerPool.getQueueStats()
+
+ return {
+ timestamp: new Date().toISOString(),
+ pool: {
+ isActive: workerPool.isActive(),
+ workerCount: workerPool.getWorkerCount(),
+ ...poolStats,
+ },
+ queue: queueStats,
+ }
+ } catch (error) {
+ throw createError({
+ statusCode: 500,
+ statusMessage:
+ error instanceof Error ? error.message : 'Failed to get queue status',
+ })
+ }
+})
diff --git a/server/database/migrations/0004_absent_lockheed.sql b/server/database/migrations/0004_absent_lockheed.sql
new file mode 100644
index 00000000..2fe31f40
--- /dev/null
+++ b/server/database/migrations/0004_absent_lockheed.sql
@@ -0,0 +1,11 @@
+CREATE TABLE `pipeline_queue` (
+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+ `payload` text NOT NULL,
+ `priority` integer DEFAULT 0 NOT NULL,
+ `attempts` integer DEFAULT 0 NOT NULL,
+ `max_attempts` integer DEFAULT 3 NOT NULL,
+ `status` text DEFAULT 'pending' NOT NULL,
+ `status_stage` text,
+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,
+ `completed_at` integer
+);
diff --git a/server/database/migrations/0005_light_stranger.sql b/server/database/migrations/0005_light_stranger.sql
new file mode 100644
index 00000000..714fcf79
--- /dev/null
+++ b/server/database/migrations/0005_light_stranger.sql
@@ -0,0 +1 @@
+ALTER TABLE `pipeline_queue` ADD `error_message` text;
\ No newline at end of file
diff --git a/server/database/migrations/0006_flaky_the_enforcers.sql b/server/database/migrations/0006_flaky_the_enforcers.sql
new file mode 100644
index 00000000..c3a173ad
--- /dev/null
+++ b/server/database/migrations/0006_flaky_the_enforcers.sql
@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=OFF;--> statement-breakpoint
+CREATE TABLE `__new_pipeline_queue` (
+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+ `payload` text DEFAULT '{"type":"photo","storageKey":""}' NOT NULL,
+ `priority` integer DEFAULT 0 NOT NULL,
+ `attempts` integer DEFAULT 0 NOT NULL,
+ `max_attempts` integer DEFAULT 3 NOT NULL,
+ `status` text DEFAULT 'pending' NOT NULL,
+ `status_stage` text,
+ `error_message` text,
+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,
+ `completed_at` integer
+);
+--> statement-breakpoint
+INSERT INTO `__new_pipeline_queue`("id", "payload", "priority", "attempts", "max_attempts", "status", "status_stage", "error_message", "created_at", "completed_at") SELECT "id", "payload", "priority", "attempts", "max_attempts", "status", "status_stage", "error_message", "created_at", "completed_at" FROM `pipeline_queue`;--> statement-breakpoint
+DROP TABLE `pipeline_queue`;--> statement-breakpoint
+ALTER TABLE `__new_pipeline_queue` RENAME TO `pipeline_queue`;--> statement-breakpoint
+PRAGMA foreign_keys=ON;
\ No newline at end of file
diff --git a/server/database/migrations/meta/0004_snapshot.json b/server/database/migrations/meta/0004_snapshot.json
new file mode 100644
index 00000000..26185166
--- /dev/null
+++ b/server/database/migrations/meta/0004_snapshot.json
@@ -0,0 +1,358 @@
+{
+ "version": "6",
+ "dialect": "sqlite",
+ "id": "b9e6f5a8-2bca-465d-8578-62b3b35c6af3",
+ "prevId": "09eda2ca-cacb-4ef5-a4b9-e17c1326fa19",
+ "tables": {
+ "photos": {
+ "name": "photos",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "width": {
+ "name": "width",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "height": {
+ "name": "height",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "aspect_ratio": {
+ "name": "aspect_ratio",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "date_taken": {
+ "name": "date_taken",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "storage_key": {
+ "name": "storage_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_key": {
+ "name": "thumbnail_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "file_size": {
+ "name": "file_size",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_modified": {
+ "name": "last_modified",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "original_url": {
+ "name": "original_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_url": {
+ "name": "thumbnail_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_hash": {
+ "name": "thumbnail_hash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "tags": {
+ "name": "tags",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "exif": {
+ "name": "exif",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "latitude": {
+ "name": "latitude",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "longitude": {
+ "name": "longitude",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "country": {
+ "name": "country",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "city": {
+ "name": "city",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "location_name": {
+ "name": "location_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_live_photo": {
+ "name": "is_live_photo",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "live_photo_video_url": {
+ "name": "live_photo_video_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "live_photo_video_key": {
+ "name": "live_photo_video_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "photos_id_unique": {
+ "name": "photos_id_unique",
+ "columns": [
+ "id"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "pipeline_queue": {
+ "name": "pipeline_queue",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "payload": {
+ "name": "payload",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "attempts": {
+ "name": "attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "max_attempts": {
+ "name": "max_attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 3
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "status_stage": {
+ "name": "status_stage",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(unixepoch())"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "users": {
+ "name": "users",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "avatar": {
+ "name": "avatar",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "is_admin": {
+ "name": "is_admin",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ }
+ },
+ "indexes": {
+ "users_name_unique": {
+ "name": "users_name_unique",
+ "columns": [
+ "name"
+ ],
+ "isUnique": true
+ },
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "columns": [
+ "email"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ }
+ },
+ "views": {},
+ "enums": {},
+ "_meta": {
+ "schemas": {},
+ "tables": {},
+ "columns": {}
+ },
+ "internal": {
+ "indexes": {}
+ }
+}
\ No newline at end of file
diff --git a/server/database/migrations/meta/0005_snapshot.json b/server/database/migrations/meta/0005_snapshot.json
new file mode 100644
index 00000000..732ebb60
--- /dev/null
+++ b/server/database/migrations/meta/0005_snapshot.json
@@ -0,0 +1,365 @@
+{
+ "version": "6",
+ "dialect": "sqlite",
+ "id": "f819221e-a365-4562-81d1-ec510aa6de94",
+ "prevId": "b9e6f5a8-2bca-465d-8578-62b3b35c6af3",
+ "tables": {
+ "photos": {
+ "name": "photos",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "width": {
+ "name": "width",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "height": {
+ "name": "height",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "aspect_ratio": {
+ "name": "aspect_ratio",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "date_taken": {
+ "name": "date_taken",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "storage_key": {
+ "name": "storage_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_key": {
+ "name": "thumbnail_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "file_size": {
+ "name": "file_size",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_modified": {
+ "name": "last_modified",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "original_url": {
+ "name": "original_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_url": {
+ "name": "thumbnail_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_hash": {
+ "name": "thumbnail_hash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "tags": {
+ "name": "tags",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "exif": {
+ "name": "exif",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "latitude": {
+ "name": "latitude",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "longitude": {
+ "name": "longitude",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "country": {
+ "name": "country",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "city": {
+ "name": "city",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "location_name": {
+ "name": "location_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_live_photo": {
+ "name": "is_live_photo",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "live_photo_video_url": {
+ "name": "live_photo_video_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "live_photo_video_key": {
+ "name": "live_photo_video_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "photos_id_unique": {
+ "name": "photos_id_unique",
+ "columns": [
+ "id"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "pipeline_queue": {
+ "name": "pipeline_queue",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "payload": {
+ "name": "payload",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "attempts": {
+ "name": "attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "max_attempts": {
+ "name": "max_attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 3
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "status_stage": {
+ "name": "status_stage",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "error_message": {
+ "name": "error_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(unixepoch())"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "users": {
+ "name": "users",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "avatar": {
+ "name": "avatar",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "is_admin": {
+ "name": "is_admin",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ }
+ },
+ "indexes": {
+ "users_name_unique": {
+ "name": "users_name_unique",
+ "columns": [
+ "name"
+ ],
+ "isUnique": true
+ },
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "columns": [
+ "email"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ }
+ },
+ "views": {},
+ "enums": {},
+ "_meta": {
+ "schemas": {},
+ "tables": {},
+ "columns": {}
+ },
+ "internal": {
+ "indexes": {}
+ }
+}
\ No newline at end of file
diff --git a/server/database/migrations/meta/0006_snapshot.json b/server/database/migrations/meta/0006_snapshot.json
new file mode 100644
index 00000000..00ac497c
--- /dev/null
+++ b/server/database/migrations/meta/0006_snapshot.json
@@ -0,0 +1,366 @@
+{
+ "version": "6",
+ "dialect": "sqlite",
+ "id": "f395c220-c4ec-46f8-a0bf-4fda6e692f7c",
+ "prevId": "f819221e-a365-4562-81d1-ec510aa6de94",
+ "tables": {
+ "photos": {
+ "name": "photos",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "width": {
+ "name": "width",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "height": {
+ "name": "height",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "aspect_ratio": {
+ "name": "aspect_ratio",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "date_taken": {
+ "name": "date_taken",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "storage_key": {
+ "name": "storage_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_key": {
+ "name": "thumbnail_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "file_size": {
+ "name": "file_size",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_modified": {
+ "name": "last_modified",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "original_url": {
+ "name": "original_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_url": {
+ "name": "thumbnail_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "thumbnail_hash": {
+ "name": "thumbnail_hash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "tags": {
+ "name": "tags",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "exif": {
+ "name": "exif",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "latitude": {
+ "name": "latitude",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "longitude": {
+ "name": "longitude",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "country": {
+ "name": "country",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "city": {
+ "name": "city",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "location_name": {
+ "name": "location_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_live_photo": {
+ "name": "is_live_photo",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "live_photo_video_url": {
+ "name": "live_photo_video_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "live_photo_video_key": {
+ "name": "live_photo_video_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "photos_id_unique": {
+ "name": "photos_id_unique",
+ "columns": [
+ "id"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "pipeline_queue": {
+ "name": "pipeline_queue",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "payload": {
+ "name": "payload",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'{\"type\":\"photo\",\"storageKey\":\"\"}'"
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "attempts": {
+ "name": "attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "max_attempts": {
+ "name": "max_attempts",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 3
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "status_stage": {
+ "name": "status_stage",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "error_message": {
+ "name": "error_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(unixepoch())"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "users": {
+ "name": "users",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "avatar": {
+ "name": "avatar",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "is_admin": {
+ "name": "is_admin",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ }
+ },
+ "indexes": {
+ "users_name_unique": {
+ "name": "users_name_unique",
+ "columns": [
+ "name"
+ ],
+ "isUnique": true
+ },
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "columns": [
+ "email"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ }
+ },
+ "views": {},
+ "enums": {},
+ "_meta": {
+ "schemas": {},
+ "tables": {},
+ "columns": {}
+ },
+ "internal": {
+ "indexes": {}
+ }
+}
\ No newline at end of file
diff --git a/server/database/migrations/meta/_journal.json b/server/database/migrations/meta/_journal.json
index 4cc48681..d3b16d87 100644
--- a/server/database/migrations/meta/_journal.json
+++ b/server/database/migrations/meta/_journal.json
@@ -29,6 +29,27 @@
"when": 1758511705153,
"tag": "0003_nifty_hellcat",
"breakpoints": true
+ },
+ {
+ "idx": 4,
+ "version": "6",
+ "when": 1758650003665,
+ "tag": "0004_absent_lockheed",
+ "breakpoints": true
+ },
+ {
+ "idx": 5,
+ "version": "6",
+ "when": 1758651965719,
+ "tag": "0005_light_stranger",
+ "breakpoints": true
+ },
+ {
+ "idx": 6,
+ "version": "6",
+ "when": 1758723057975,
+ "tag": "0006_flaky_the_enforcers",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/server/database/schema.ts b/server/database/schema.ts
index 8016fa34..7108f0ce 100644
--- a/server/database/schema.ts
+++ b/server/database/schema.ts
@@ -1,3 +1,4 @@
+import { sql } from 'drizzle-orm'
import { sqliteTable, text, integer, real } from 'drizzle-orm/sqlite-core'
import type { NeededExif } from '~~/shared/types/photo'
@@ -39,3 +40,45 @@ export const photos = sqliteTable('photos', {
livePhotoVideoUrl: text('live_photo_video_url'),
livePhotoVideoKey: text('live_photo_video_key'),
})
+
+export const pipelineQueue = sqliteTable('pipeline_queue', {
+ id: integer('id').primaryKey({ autoIncrement: true }),
+ payload: text('payload', { mode: 'json' })
+ .$type<{
+ type: 'photo' | 'live-photo-video'
+ storageKey: string
+ }>()
+ .notNull()
+ .default({
+ type: 'photo',
+ storageKey: '',
+ }),
+ priority: integer('priority').default(0).notNull(),
+ attempts: integer('attempts').default(0).notNull(),
+ maxAttempts: integer('max_attempts').default(3).notNull(),
+ status: text('status', {
+ enum: [
+ 'pending', // Waiting to be processed
+ 'in-stages', // Currently being processed
+ 'completed', // Successfully processed
+ 'failed', // Processing failed
+ ],
+ })
+ .notNull()
+ .default('pending'),
+ statusStage: text('status_stage', {
+ enum: [
+ 'preprocessing',
+ 'metadata',
+ 'thumbnail',
+ 'exif',
+ 'reverse-geocoding',
+ 'live-photo',
+ ],
+ }),
+ errorMessage: text('error_message'),
+ createdAt: integer('created_at', { mode: 'timestamp' })
+ .notNull()
+ .default(sql`(unixepoch())`),
+ completedAt: integer('completed_at', { mode: 'timestamp' }),
+})
diff --git a/server/plugins/pipeline-queue.ts b/server/plugins/pipeline-queue.ts
new file mode 100644
index 00000000..341231f8
--- /dev/null
+++ b/server/plugins/pipeline-queue.ts
@@ -0,0 +1,40 @@
+import { WorkerPool } from '../services/pipeline-queue'
+
+export default defineNitroPlugin(async (_nitroApp) => {
+ const _logger = logger.dynamic('queue')
+
+ const workerPool = new WorkerPool(
+ {
+ workerCount: 3,
+ intervalMs: 1000,
+ intervalOffset: Math.floor(1000 / 3),
+ enableLoadBalancing: true,
+ statsReportInterval: 60000 * 10,
+ },
+ _logger,
+ )
+ await workerPool.start()
+
+ globalThis.__workerPool = workerPool
+
+ const exitHandler = async () => {
+ _logger.info('Shutting down worker pool...')
+ await workerPool.stop()
+ process.exit(0)
+ }
+
+ process.on('SIGINT', exitHandler)
+ process.on('SIGTERM', exitHandler)
+
+ // 每 5 分钟进行一次负载均衡检查
+ setInterval(
+ async () => {
+ try {
+ await workerPool.rebalance()
+ } catch (error) {
+ _logger.error('Rebalance failed:', error)
+ }
+ },
+ 5 * 60 * 1000,
+ )
+})
diff --git a/server/services/location/geocoding.ts b/server/services/location/geocoding.ts
index 7513d4f9..349b1d11 100644
--- a/server/services/location/geocoding.ts
+++ b/server/services/location/geocoding.ts
@@ -13,6 +13,7 @@ export interface GeocodingProvider {
/**
* OpenStreetMap Nominatim API 地理编码提供者
* 免费的地理编码服务,适合开发和小规模使用
+ * TODO: 实现 MapBox 逆编码提供者
*/
class NominatimGeocodingProvider implements GeocodingProvider {
private readonly baseUrl = 'https://nominatim.openstreetmap.org'
diff --git a/server/services/photo/pipeline-async.ts b/server/services/photo/pipeline-async.ts
deleted file mode 100644
index 40dd32cd..00000000
--- a/server/services/photo/pipeline-async.ts
+++ /dev/null
@@ -1,274 +0,0 @@
-import {
- preprocessImageWithJpegUpload,
- processImageMetadataAndSharp,
-} from '../image/processor'
-import { generateThumbnailAndHash } from '../image/thumbnail'
-import type { Photo } from '~~/server/utils/db'
-import { getStorageManager } from '~~/server/plugins/storage'
-import { compressUint8Array } from '~~/shared/utils/u8array'
-import type { StorageObject } from '../storage'
-import { extractExifData, extractPhotoInfo } from '../image/exif'
-import {
- extractLocationFromGPS,
- parseGPSCoordinates,
-} from '../location/geocoding'
-import { findLivePhotoVideoForImage } from '../video/livephoto'
-import { generateSafePhotoId } from '~~/server/utils/file-utils'
-
-/**
- * 异步执行照片处理管道 - 不阻塞主线程
- */
-export const execPhotoPipelineAsync = async (
- s3key: string,
- storageObject: StorageObject,
-): Promise<Photo | null> => {
- return new Promise((resolve, reject) => {
- // 使用 setImmediate 确保处理在下一个事件循环中执行
- setImmediate(async () => {
- try {
- const result = await processPhotoInternal(s3key, storageObject)
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
-}
-
-/**
- * 内部照片处理函数
- */
-async function processPhotoInternal(
- s3key: string,
- storageObject: StorageObject,
-): Promise<Photo | null> {
- const log = logger.image
- const storageProvider = getStorageManager().getProvider()
- const photoId = generateSafePhotoId(s3key)
-
- try {
- log.info(`Starting async processing for ${s3key}`)
-
- // 步骤1: 预处理图片并上传JPEG版本
- const imageBuffers = await new Promise<any>((resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await preprocessImageWithJpegUpload(s3key)
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
-
- if (!imageBuffers) {
- log.error(`Image preprocessing failed for ${s3key}`)
- return null
- }
-
- // 步骤2: 处理图片元数据和Sharp处理
- const processedData = await new Promise<any>((resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await processImageMetadataAndSharp(
- imageBuffers.processed,
- s3key,
- )
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
-
- if (!processedData) {
- log.error(`Image processing failed for ${s3key}`)
- return null
- }
-
- const { imageBuffer, metadata } = processedData
-
- // 步骤3: 生成缩略图和哈希值
- const { thumbnailBuffer, thumbnailHash } = await new Promise<any>(
- (resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await generateThumbnailAndHash(imageBuffer, logger.image)
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- },
- )
-
- // TODO 步骤4: 直方图和影调分析
- // const histogramData = await new Promise<any>((resolve, reject) => {
- // setImmediate(async () => {
- // try {
- // const result = await calculateHistogram(processedData.sharpInst)
- // resolve(result)
- // } catch (error) {
- // reject(error)
- // }
- // })
- // })
-
- // 步骤5: 提取EXIF数据
- const exifData = await new Promise<any>((resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await extractExifData(
- imageBuffer,
- imageBuffers.raw,
- logger.image,
- )
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
-
- // 步骤6: 提取照片基本信息
- const photoInfo = extractPhotoInfo(s3key, exifData)
-
- // 步骤7: 处理地理位置信息
- let locationInfo = null
- if (exifData) {
- const { latitude, longitude } = parseGPSCoordinates(exifData)
- if (latitude && longitude) {
- locationInfo = await new Promise<any>((resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await extractLocationFromGPS(latitude, longitude)
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
- }
- }
-
- // 步骤8: 配对 LivePhoto 视频文件
- let livePhotoInfo = null
- const livePhotoVideo = await new Promise<any>((resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await findLivePhotoVideoForImage(s3key)
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
-
- if (livePhotoVideo) {
- livePhotoInfo = {
- isLivePhoto: 1,
- livePhotoVideoUrl: storageProvider.getPublicUrl(
- livePhotoVideo.videoKey,
- ),
- livePhotoVideoKey: livePhotoVideo.videoKey,
- }
- log.info(`LivePhoto video found for ${s3key}: ${livePhotoVideo.videoKey}`)
- }
-
- // 步骤9: 上传缩略图到存储服务
- const thumbnailObject = await new Promise<any>((resolve, reject) => {
- setImmediate(async () => {
- try {
- const result = await storageProvider.create(
- `thumbnails/${photoId}.webp`,
- thumbnailBuffer,
- 'image/webp',
- )
- resolve(result)
- } catch (error) {
- reject(error)
- }
- })
- })
-
- // 步骤10: 构建最终的Photo对象
- const result: Photo = {
- id: photoId,
- title: photoInfo.title,
- description: photoInfo.description,
- dateTaken: photoInfo.dateTaken,
- tags: photoInfo.tags,
- width: metadata.width,
- height: metadata.height,
- aspectRatio: metadata.width / metadata.height,
- storageKey: s3key,
- thumbnailKey: thumbnailObject.key,
- fileSize: storageObject.size || null,
- lastModified:
- storageObject.lastModified?.toISOString() || new Date().toISOString(),
- originalUrl: imageBuffers.jpegKey
- ? storageProvider.getPublicUrl(imageBuffers.jpegKey) // 使用 JPEG 版本作为 originalUrl
- : storageProvider.getPublicUrl(s3key),
- thumbnailUrl: storageProvider.getPublicUrl(thumbnailObject.key),
- thumbnailHash: thumbnailHash ? compressUint8Array(thumbnailHash) : null,
- exif: exifData,
- // 地理位置信息
- latitude: locationInfo?.latitude || null,
- longitude: locationInfo?.longitude || null,
- country: locationInfo?.country || null,
- city: locationInfo?.city || null,
- locationName: locationInfo?.locationName || null,
- // LivePhoto 相关字段
- isLivePhoto: livePhotoInfo?.isLivePhoto || 0,
- livePhotoVideoUrl: livePhotoInfo?.livePhotoVideoUrl || null,
- livePhotoVideoKey: livePhotoInfo?.livePhotoVideoKey || null,
- }
-
- log.info(`Async processing completed for ${s3key}`)
- return result
- } catch (err) {
- log.error(`Photo pipeline failed for ${s3key}`, err)
- return null
- }
-}
-
-/**
- * 批量异步处理照片
- */
-export const execBatchPhotoPipelineAsync = async (
- photos: Array<{ s3key: string; storageObject: StorageObject }>,
- maxConcurrent = 3,
-): Promise<Array<{ s3key: string; photo: Photo | null; error?: string }>> => {
- const results: Array<{ s3key: string; photo: Photo | null; error?: string }> =
- []
-
- // 分批处理以限制并发数
- for (let i = 0; i < photos.length; i += maxConcurrent) {
- const batch = photos.slice(i, i + maxConcurrent)
- const batchPromises = batch.map(async ({ s3key, storageObject }) => {
- try {
- const photo = await execPhotoPipelineAsync(s3key, storageObject)
- return {
- s3key,
- photo,
- error: photo ? undefined : 'Processing failed',
- }
- } catch (error) {
- return {
- s3key,
- photo: null,
- error: error instanceof Error ? error.message : String(error),
- }
- }
- })
-
- const batchResults = await Promise.all(batchPromises)
- results.push(...batchResults)
-
- logger.image.info(
- `Processed async batch ${Math.floor(i / maxConcurrent) + 1} of ${Math.ceil(photos.length / maxConcurrent)}`,
- )
- }
-
- return results
-}
diff --git a/server/services/pipeline-queue/index.ts b/server/services/pipeline-queue/index.ts
new file mode 100644
index 00000000..4391b11e
--- /dev/null
+++ b/server/services/pipeline-queue/index.ts
@@ -0,0 +1,7 @@
+export { QueueManager } from './manager'
+export { WorkerPool } from './worker-pool'
+export type {
+ WorkerPoolConfig,
+ WorkerStats,
+ PoolStats,
+} from './worker-pool'
diff --git a/server/services/pipeline-queue/manager.ts b/server/services/pipeline-queue/manager.ts
new file mode 100644
index 00000000..52a8454e
--- /dev/null
+++ b/server/services/pipeline-queue/manager.ts
@@ -0,0 +1,558 @@
+import type { ConsolaInstance } from 'consola'
+import path from 'path'
+import { asc, desc, eq, sql } from 'drizzle-orm'
+import type {
+ NewPipelineQueueItem,
+ PipelineQueueItem,
+ Photo,
+} from '~~/server/utils/db'
+import {
+ preprocessImageWithJpegUpload,
+ processImageMetadataAndSharp,
+} from '../image/processor'
+import { generateThumbnailAndHash } from '../image/thumbnail'
+import { extractExifData, extractPhotoInfo } from '../image/exif'
+import {
+ extractLocationFromGPS,
+ parseGPSCoordinates,
+} from '../location/geocoding'
+import { findLivePhotoVideoForImage } from '../video/livephoto'
+import { getStorageManager } from '~~/server/plugins/storage'
+
+export class QueueManager {
+ private static instances: Map<string, QueueManager> = new Map()
+ private workerId: string
+ private logger: ConsolaInstance
+ private isProcessing: boolean = false
+ private processingInterval: NodeJS.Timeout | null = null
+ private processedCount: number = 0
+ private errorCount: number = 0
+ private startTime: Date
+
+ static getInstance(
+ workerId: string = 'default',
+ logger?: ConsolaInstance,
+ ): QueueManager {
+ if (!QueueManager.instances.has(workerId)) {
+ QueueManager.instances.set(workerId, new QueueManager(workerId, logger))
+ }
+ return QueueManager.instances.get(workerId)!
+ }
+
+ static getAllInstances(): QueueManager[] {
+ return Array.from(QueueManager.instances.values())
+ }
+
+ private constructor(workerId: string, _logger?: ConsolaInstance) {
+ this.workerId = workerId
+ this.logger = _logger
+ ? _logger.withTag(`${workerId}`)
+ : logger.dynamic(`queue-${workerId}`)
+ this.startTime = new Date()
+ }
+
+ getWorkerId(): string {
+ return this.workerId
+ }
+
+ getStats() {
+ const uptime = Date.now() - this.startTime.getTime()
+ return {
+ workerId: this.workerId,
+ isProcessing: this.isProcessing,
+ processedCount: this.processedCount,
+ errorCount: this.errorCount,
+ uptime: Math.floor(uptime / 1000), // seconds
+ successRate:
+ this.processedCount > 0
+ ? (this.processedCount / (this.processedCount + this.errorCount)) *
+ 100
+ : 0,
+ }
+ }
+
+ /**
+ * 插入新任务到队列
+ * @param payload 任务负荷
+ * @param options 可选的任务设置
+ * @returns 新创建任务的 ID
+ */
+ async addTask(
+ payload: any,
+ options?: Partial<NewPipelineQueueItem>,
+ ): Promise<number> {
+ const db = useDB()
+ const result = db
+ .insert(tables.pipelineQueue)
+ .values({
+ payload,
+ ...options,
+ })
+ .returning({ id: tables.pipelineQueue.id })
+ .get()
+ return result.id
+ }
+
+ /**
+ * 获取任务状态
+ * @param taskId 任务ID
+ * @returns 任务状态信息
+ */
+ async getTaskStatus(taskId: number) {
+ const db = useDB()
+ const task = await db
+ .select()
+ .from(tables.pipelineQueue)
+ .where(eq(tables.pipelineQueue.id, taskId))
+ .get()
+ return task
+ }
+
+ /**
+ * 获取并锁定下一个待处理任务
+ * @returns 下一个待处理任务
+ */
+ async getNextTask(): Promise<PipelineQueueItem | null> {
+ const db = useDB()
+
+ // 使用同步事务防止竞态条件
+ const task = db.transaction((tx) => {
+ const highestPriorityPendingTask = tx
+ .select()
+ .from(tables.pipelineQueue)
+ .where(eq(tables.pipelineQueue.status, 'pending'))
+ // 优先处理高优先级和较早创建的任务
+ .orderBy(
+ desc(tables.pipelineQueue.priority),
+ asc(tables.pipelineQueue.createdAt),
+ )
+ .limit(1)
+ .get()
+
+ if (!highestPriorityPendingTask) return null
+
+ const task = highestPriorityPendingTask
+ tx.update(tables.pipelineQueue)
+ .set({ status: 'in-stages' })
+ .where(eq(tables.pipelineQueue.id, task.id))
+ .run()
+
+ return { ...task, status: 'in-stages' as const }
+ })
+
+ return task
+ }
+
+ /**
+ * 更新任务阶段
+ * @param taskId 任务ID
+ * @param stage 新的任务阶段
+ */
+ async updateTaskStage(
+ taskId: number,
+ stage: PipelineQueueItem['statusStage'],
+ ): Promise<void> {
+ const db = useDB()
+ await db
+ .update(tables.pipelineQueue)
+ .set({ statusStage: stage })
+ .where(eq(tables.pipelineQueue.id, taskId))
+ }
+
+ /**
+ * 标记任务为已完成
+ * @param taskId 任务ID
+ */
+ async markTaskCompleted(taskId: number): Promise<void> {
+ const db = useDB()
+ await db
+ .update(tables.pipelineQueue)
+ .set({
+ status: 'completed',
+ completedAt: sql`(unixepoch())`,
+ })
+ .where(eq(tables.pipelineQueue.id, taskId))
+ }
+
+ /**
+ * 标记任务为失败
+ * @param taskId 任务ID
+ * @param errorMessage 错误信息
+ */
+ async markTaskFailed(taskId: number, errorMessage?: string): Promise<void> {
+ const db = useDB()
+ const task = await db
+ .select()
+ .from(tables.pipelineQueue)
+ .where(eq(tables.pipelineQueue.id, taskId))
+ .get()
+
+ if (!task) return
+
+ const newAttempts = task.attempts + 1
+ const shouldRetry = newAttempts < task.maxAttempts
+
+ await db
+ .update(tables.pipelineQueue)
+ .set({
+ status: shouldRetry ? 'pending' : 'failed',
+ attempts: newAttempts,
+ errorMessage: errorMessage || 'Unknown error',
+ })
+ .where(eq(tables.pipelineQueue.id, taskId))
+ }
+
+ /** 任务处理器 */
+ private processors = (() => {
+ return {
+ photo: async (task: PipelineQueueItem) => {
+ const { id: taskId, payload } = task
+ const { storageKey } = payload
+ const storageProvider = getStorageManager().getProvider()
+ const photoId = generateSafePhotoId(storageKey)
+
+ try {
+ this.logger.info(`Start processing task ${taskId}: ${storageKey}`)
+
+ const storageObject = await storageProvider.getFileMeta(storageKey)
+
+ if (!storageObject) {
+ throw new Error(`Storage object not found`)
+ }
+
+ // STEP 1: 预处理 - 转换 HEIC 到 JPEG 并上传
+ await this.updateTaskStage(taskId, 'preprocessing')
+ this.logger.info(`[${taskId}:in-stage] preprocessing`)
+ const imageBuffers = await preprocessImageWithJpegUpload(storageKey)
+ if (!imageBuffers) {
+ throw new Error('Preprocessing failed')
+ }
+
+ // STEP 2: 元数据处理 - 使用 Sharp 处理图片元数据
+ await this.updateTaskStage(taskId, 'metadata')
+ this.logger.info(`[${taskId}:in-stage] metadata extraction`)
+ const processedData = await processImageMetadataAndSharp(
+ imageBuffers.processed,
+ storageKey,
+ )
+ if (!processedData) {
+ throw new Error('Metadata processing failed')
+ }
+
+ const { imageBuffer, metadata } = processedData
+
+ // STEP 3: 生成缩略图
+ await this.updateTaskStage(taskId, 'thumbnail')
+ this.logger.info(`[${taskId}:in-stage] thumbnail generation`)
+ const { thumbnailBuffer, thumbnailHash } =
+ await generateThumbnailAndHash(imageBuffer, this.logger)
+
+ // 上传缩略图到存储服务
+ const thumbnailObject = await new Promise<any>((resolve, reject) => {
+ setImmediate(async () => {
+ try {
+ const result = await storageProvider.create(
+ `thumbnails/${photoId}.webp`,
+ thumbnailBuffer,
+ 'image/webp',
+ )
+ resolve(result)
+ } catch (error) {
+ reject(error)
+ }
+ })
+ })
+
+ // STEP 4: 提取 EXIF 数据
+ await this.updateTaskStage(taskId, 'exif')
+ this.logger.info(`[${taskId}:in-stage] exif extraction`)
+ const exifData = await extractExifData(
+ imageBuffer,
+ imageBuffers.raw,
+ this.logger,
+ )
+
+ // 提取照片基本信息
+ const photoInfo = extractPhotoInfo(storageKey, exifData)
+
+ // STEP 5: 地理位置反向解析
+ // 这里逆编码失败不报错,宽容处理
+ await this.updateTaskStage(taskId, 'reverse-geocoding')
+ this.logger.info(`[${taskId}:in-stage] reverse geocoding`)
+
+ let locationInfo = null
+ if (exifData) {
+ const { latitude, longitude } = parseGPSCoordinates(exifData)
+ if (latitude && longitude) {
+ locationInfo = await extractLocationFromGPS(latitude, longitude)
+ }
+ }
+
+ // STEP 6: LivePhoto 视频配对
+ await this.updateTaskStage(taskId, 'live-photo')
+ this.logger.info(`[${taskId}:in-stage] live photo detection`)
+ let livePhotoInfo = null
+ const livePhotoVideo = await findLivePhotoVideoForImage(storageKey)
+ if (livePhotoVideo) {
+ livePhotoInfo = {
+ isLivePhoto: 1,
+ livePhotoVideoUrl: storageProvider.getPublicUrl(
+ livePhotoVideo.videoKey,
+ ),
+ livePhotoVideoKey: livePhotoVideo.videoKey,
+ }
+ this.logger.info(
+ `[${taskId}:in-stage] found LivePhoto video: ${livePhotoVideo.videoKey}`,
+ )
+ }
+
+ // 构建最终的 Photo 对象
+ const result: Photo = {
+ id: photoId,
+ title: photoInfo.title,
+ description: photoInfo.description,
+ dateTaken: photoInfo.dateTaken,
+ tags: photoInfo.tags,
+ width: metadata.width,
+ height: metadata.height,
+ aspectRatio: metadata.width / metadata.height,
+ storageKey: storageKey,
+ thumbnailKey: thumbnailObject.key,
+ fileSize: storageObject.size || null,
+ lastModified:
+ storageObject.lastModified?.toISOString() ||
+ new Date().toISOString(),
+ originalUrl: imageBuffers.jpegKey
+ ? storageProvider.getPublicUrl(imageBuffers.jpegKey) // 使用 JPEG 版本作为 originalUrl
+ : storageProvider.getPublicUrl(storageKey),
+ thumbnailUrl: storageProvider.getPublicUrl(thumbnailObject.key),
+ thumbnailHash: thumbnailHash
+ ? compressUint8Array(thumbnailHash)
+ : null,
+ exif: exifData,
+ // 地理位置信息
+ latitude: locationInfo?.latitude || null,
+ longitude: locationInfo?.longitude || null,
+ country: locationInfo?.country || null,
+ city: locationInfo?.city || null,
+ locationName: locationInfo?.locationName || null,
+ // LivePhoto 相关字段
+ isLivePhoto: livePhotoInfo?.isLivePhoto || 0,
+ livePhotoVideoUrl: livePhotoInfo?.livePhotoVideoUrl || null,
+ livePhotoVideoKey: livePhotoInfo?.livePhotoVideoKey || null,
+ }
+
+ const db = useDB()
+ await db.insert(tables.photos).values(result).onConflictDoUpdate({
+ target: tables.photos.id,
+ set: result,
+ })
+
+ this.logger.success(`Task ${taskId} processed successfully`)
+ return result
+ } catch (error) {
+ this.logger.error(`Task ${taskId} processing failed`, error)
+ throw error
+ }
+ },
+ livePhotoDetect: async (task: PipelineQueueItem) => {
+ const db = useDB()
+ const storageProvider = getStorageManager().getProvider()
+
+ const { id: taskId, payload } = task
+ const { storageKey: videoKey } = payload
+
+ try {
+ this.logger.info(
+ `Start processing LivePhoto detection task ${taskId}: ${videoKey}`,
+ )
+
+ const storageObject = await storageProvider.getFileMeta(videoKey)
+
+ if (!storageObject) {
+ throw new Error(`Storage object not found`)
+ }
+
+ // 寻找是否有同名的照片文件
+ const videoDir = path.dirname(videoKey)
+ const videoBaseName = path.basename(videoKey, path.extname(videoKey))
+
+ const possiblePhotoKeys = [
+ path.join(videoDir, `${videoBaseName}.HEIC`).replace(/\\/g, '/'),
+ path.join(videoDir, `${videoBaseName}.heic`).replace(/\\/g, '/'),
+ path.join(videoDir, `${videoBaseName}.JPG`).replace(/\\/g, '/'),
+ path.join(videoDir, `${videoBaseName}.jpg`).replace(/\\/g, '/'),
+ path.join(videoDir, `${videoBaseName}.JPEG`).replace(/\\/g, '/'),
+ path.join(videoDir, `${videoBaseName}.jpeg`).replace(/\\/g, '/'),
+ ]
+
+ let matchedPhoto: Photo | null = null
+ for (const photoKey of possiblePhotoKeys) {
+ const photos = await db
+ .select()
+ .from(tables.photos)
+ .where(eq(tables.photos.storageKey, photoKey))
+ .limit(1)
+
+ if (photos.length > 0) {
+ matchedPhoto = photos[0]
+ this.logger.info(
+ `Found matching photo for LivePhoto video: ${photoKey}`,
+ )
+ break
+ }
+ }
+
+ if (!matchedPhoto) {
+ this.logger.warn(
+ `No matching photo found for LivePhoto video: ${videoKey}`,
+ )
+ return null
+ }
+
+ const livePhotoVideoUrl = storageProvider.getPublicUrl(videoKey)
+ await db
+ .update(tables.photos)
+ .set({
+ isLivePhoto: 1,
+ livePhotoVideoUrl,
+ livePhotoVideoKey: videoKey,
+ })
+ .where(eq(tables.photos.id, matchedPhoto.id))
+
+ this.logger.success(
+ `LivePhoto detection task ${taskId} processed successfully, updated photo ${matchedPhoto.id}`,
+ )
+ } catch (error) {
+ this.logger.error(
+ `LivePhoto detection task ${taskId} processing failed`,
+ error,
+ )
+ throw error
+ }
+ },
+ }
+ })()
+
+ /**
+ * 处理下一个待处理任务
+ */
+ private async processNextTask(): Promise<void> {
+ if (this.isProcessing) {
+ this.logger.debug('Task is already processing, skipping this poll')
+ return
+ }
+
+ this.isProcessing = true
+
+ try {
+ const task = await this.getNextTask()
+ if (!task) {
+ this.logger.debug('No tasks to process at the moment')
+ return
+ }
+
+ try {
+ const { type } = task.payload
+
+ switch (type) {
+ case 'live-photo-video':
+ await this.processors.livePhotoDetect(task)
+ break
+ case 'photo':
+ await this.processors.photo(task)
+ break
+ default:
+ throw new Error(`Unknown task type: ${type}`)
+ }
+
+ await this.markTaskCompleted(task.id)
+ this.processedCount++
+ this.logger.success(
+ `[${this.workerId}] Task ${task.id} processed successfully (Total: ${this.processedCount})`,
+ )
+
+ // const result = await this.processTask(task)
+ // if (result) {
+ // await this.markTaskCompleted(task.id)
+ // this.processedCount++
+ // this.logger.success(
+ // `[${this.workerId}] Task ${task.id} processed successfully (Total: ${this.processedCount})`,
+ // )
+ // } else {
+ // await this.markTaskFailed(task.id, 'Processing result is empty')
+ // this.errorCount++
+ // }
+ } catch (error) {
+ const errorMessage =
+ error instanceof Error ? error.message : String(error)
+ await this.markTaskFailed(task.id, errorMessage)
+ this.errorCount++
+ this.logger.error(
+ `[${this.workerId}] Task ${task.id} processing failed (Error: ${this.errorCount}):`,
+ errorMessage,
+ )
+ }
+ } catch (error) {
+ this.logger.error('Error occurred while fetching the next task:', error)
+ } finally {
+ this.isProcessing = false
+ }
+ }
+
+ /**
+ * 开始处理队列
+ * @param intervalMs 轮询间隔
+ */
+ startProcessing(intervalMs: number = 3000): void {
+ if (this.processingInterval) return
+
+ this.processingInterval = setInterval(() => {
+ this.processNextTask().catch((error) => {
+ this.logger.error('Error occurred while processing the queue:', error)
+ })
+ }, intervalMs)
+
+ this.logger.success(
+ `Queue processing started with interval: ${intervalMs}ms`,
+ )
+
+ this.processNextTask().catch((error) => {
+ this.logger.error('Error occurred while processing the queue:', error)
+ })
+ }
+
+ /**
+ * 停止处理队列
+ */
+ stopProcessing(): void {
+ if (this.processingInterval) {
+ clearInterval(this.processingInterval)
+ this.processingInterval = null
+ this.logger.warn('Queue processing stopped')
+ }
+ }
+
+ /**
+ * 获取队列统计信息
+ * @returns 队列统计信息
+ */
+ async getQueueStats() {
+ const db = useDB()
+ const stats = await db
+ .select({
+ status: tables.pipelineQueue.status,
+ count: sql<number>`COUNT(*)`,
+ })
+ .from(tables.pipelineQueue)
+ .groupBy(tables.pipelineQueue.status)
+
+ return stats.reduce(
+ (acc, stat) => {
+ acc[stat.status] = stat.count
+ return acc
+ },
+ {} as Record<string, number>,
+ )
+ }
+}
diff --git a/server/services/pipeline-queue/worker-pool.ts b/server/services/pipeline-queue/worker-pool.ts
new file mode 100644
index 00000000..00319645
--- /dev/null
+++ b/server/services/pipeline-queue/worker-pool.ts
@@ -0,0 +1,288 @@
+import type { ConsolaInstance } from 'consola'
+import { consola } from 'consola'
+import { QueueManager } from './manager'
+
+export interface WorkerPoolConfig {
+ /** 工作器数量 */
+ workerCount: number
+ /** 轮询间隔(毫秒) */
+ intervalMs: number
+ /** 工作器间隔偏移(毫秒),用于错开轮询时间 */
+ intervalOffset: number
+ /** 是否启用负载均衡 */
+ enableLoadBalancing: boolean
+ /** 统计报告间隔(毫秒) */
+ statsReportInterval: number
+}
+
+export interface WorkerStats {
+ workerId: string
+ isProcessing: boolean
+ processedCount: number
+ errorCount: number
+ uptime: number
+ successRate: number
+}
+
+export interface PoolStats {
+ totalWorkers: number
+ activeWorkers: number
+ totalProcessed: number
+ totalErrors: number
+ averageSuccessRate: number
+ workers: WorkerStats[]
+}
+
+/**
+ * 工作器池
+ */
+export class WorkerPool {
+ private workers: QueueManager[] = []
+ private logger: ConsolaInstance
+ private config: WorkerPoolConfig
+ private statsInterval: NodeJS.Timeout | null = null
+ private isRunning: boolean = false
+
+ constructor(config: Partial<WorkerPoolConfig>, logger?: ConsolaInstance) {
+ this.config = {
+ workerCount: 3,
+ intervalMs: 2000,
+ intervalOffset: 500,
+ enableLoadBalancing: true,
+ statsReportInterval: 30000, // 30秒
+ ...config,
+ }
+ this.logger = logger || consola.withTag('worker-pool')
+ }
+
+ /**
+ * 启动工作器池
+ */
+ async start(): Promise<void> {
+ if (this.isRunning) {
+ this.logger.warn('WorkerPool is already running')
+ return
+ }
+
+ this.logger.info(
+ `Starting WorkerPool with ${this.config.workerCount} workers`,
+ )
+
+ // 创建工作器
+ for (let i = 1; i <= this.config.workerCount; i++) {
+ const workerId = `worker-${i}`
+ const worker = QueueManager.getInstance(workerId, this.logger)
+
+ // 错开时间以减少数据库竞争
+ const workerInterval =
+ this.config.intervalMs + (i - 1) * this.config.intervalOffset
+
+ this.workers.push(worker)
+ worker.startProcessing(workerInterval)
+ }
+
+ this.isRunning = true
+
+ // 启动统计报告
+ if (this.config.statsReportInterval > 0) {
+ this.startStatsReporting()
+ }
+
+ this.logger.success(
+ `WorkerPool started successfully with ${this.workers.length} workers`,
+ )
+ }
+
+ /**
+ * 停止工作器池
+ */
+ async stop(): Promise<void> {
+ if (!this.isRunning) {
+ this.logger.warn('WorkerPool is not running')
+ return
+ }
+
+ this.logger.info('Stopping WorkerPool...')
+
+ // 停止统计报告
+ if (this.statsInterval) {
+ clearInterval(this.statsInterval)
+ this.statsInterval = null
+ }
+
+ // 停止所有工作器
+ const stopPromises = this.workers.map(async (worker, _index) => {
+ try {
+ worker.stopProcessing()
+ this.logger.info(`worker [${worker.getWorkerId()}] stopped`)
+ } catch (error) {
+ this.logger.error(
+ `Error stopping worker [${worker.getWorkerId()}]:`,
+ error,
+ )
+ }
+ })
+
+ await Promise.allSettled(stopPromises)
+
+ this.workers = []
+ this.isRunning = false
+
+ this.logger.success('WorkerPool stopped successfully')
+ }
+
+ /**
+ * 获取池统计信息
+ */
+ getPoolStats(): PoolStats {
+ const workerStats = this.workers.map((worker) => worker.getStats())
+
+ const totalProcessed = workerStats.reduce(
+ (sum, stats) => sum + stats.processedCount,
+ 0,
+ )
+ const totalErrors = workerStats.reduce(
+ (sum, stats) => sum + stats.errorCount,
+ 0,
+ )
+ const activeWorkers = workerStats.filter(
+ (stats) => stats.isProcessing,
+ ).length
+
+ // 只计算已执行过操作的worker的平均成功率
+ const workersWithActivity = workerStats.filter(
+ (stats) => stats.processedCount > 0,
+ )
+ const averageSuccessRate =
+ workersWithActivity.length > 0
+ ? workersWithActivity.reduce(
+ (sum, stats) => sum + stats.successRate,
+ 0,
+ ) / workersWithActivity.length
+ : 0
+
+ return {
+ totalWorkers: this.workers.length,
+ activeWorkers,
+ totalProcessed,
+ totalErrors,
+ averageSuccessRate: Math.round(averageSuccessRate * 100) / 100,
+ workers: workerStats,
+ }
+ }
+
+ /**
+ * 启动控制台统计报告
+ */
+ private startStatsReporting(): void {
+ this.statsInterval = setInterval(() => {
+ const stats = this.getPoolStats()
+
+ this.logger.info('=== WorkerPool Stats ===')
+ this.logger.info(
+ `Total Workers: ${stats.totalWorkers}, Active: ${stats.activeWorkers}`,
+ )
+ this.logger.info(
+ `Total Processed: ${stats.totalProcessed}, Total Errors: ${stats.totalErrors}`,
+ )
+ this.logger.info(`Average Success Rate: ${stats.averageSuccessRate}%`)
+
+ stats.workers.forEach((worker) => {
+ const status = worker.isProcessing ? 'Processing' : 'Idle'
+ this.logger.info(
+ ` ${worker.workerId}: ${status} | Processed: ${worker.processedCount} | Errors: ${worker.errorCount} | Success Rate: ${worker.successRate.toFixed(1)}% | Uptime: ${worker.uptime}s`,
+ )
+ })
+ }, this.config.statsReportInterval)
+ }
+
+ /**
+ * 重新平衡工作器负载
+ */
+ async rebalance(): Promise<void> {
+ if (!this.config.enableLoadBalancing) {
+ return
+ }
+
+ const stats = this.getPoolStats()
+
+ // 如果某个工作器错误率过高,重启它
+ const problematicWorkers = stats.workers.filter(
+ (worker) => worker.errorCount > 10 && worker.successRate < 50,
+ )
+
+ for (const workerStat of problematicWorkers) {
+ this.logger.warn(
+ `Worker ${workerStat.workerId} has high error rate, preparing to restart`,
+ )
+
+ const worker = this.workers.find(
+ (w) => w.getWorkerId() === workerStat.workerId,
+ )
+ if (worker) {
+ worker.stopProcessing()
+
+ setTimeout(() => {
+ worker.startProcessing(this.config.intervalMs)
+ this.logger.info(`Worker ${workerStat.workerId} has been restarted`)
+ }, 5000)
+ }
+ }
+ }
+
+ /**
+ * 获取队列统计信息(从任意一个工作器获取)
+ */
+ async getQueueStats() {
+ if (this.workers.length === 0) {
+ return {}
+ }
+
+ return await this.workers[0].getQueueStats()
+ }
+
+ /**
+ * 检查池是否正在运行
+ */
+ isActive(): boolean {
+ return this.isRunning
+ }
+
+ /**
+ * 获取工作器数量
+ */
+ getWorkerCount(): number {
+ return this.workers.length
+ }
+
+ /**
+ * 获取第一个工作器(用于添加任务)
+ */
+ getFirstWorker(): QueueManager | null {
+ return this.workers.length > 0 ? this.workers[0] : null
+ }
+
+ /**
+ * 添加任务到队列(通过任意一个工作器)
+ */
+ async addTask(payload: any, options?: any): Promise<number> {
+ const worker = this.getFirstWorker()
+ if (!worker) {
+ throw new Error('No workers available')
+ }
+ return await worker.addTask(payload, options)
+ }
+
+ /**
+ * 获取任务状态
+ * @param taskId 任务ID
+ * @returns 任务状态信息
+ */
+ async getTaskStatus(taskId: number) {
+ const worker = this.getFirstWorker()
+ if (!worker) {
+ throw new Error('No workers available')
+ }
+ return await worker.getTaskStatus(taskId)
+ }
+}
diff --git a/server/services/storage/interfaces.ts b/server/services/storage/interfaces.ts
index 2faa342d..bfb1d962 100644
--- a/server/services/storage/interfaces.ts
+++ b/server/services/storage/interfaces.ts
@@ -61,6 +61,7 @@ export interface StorageProvider {
expiresIn?: number,
options?: UploadOptions,
): Promise<string>
+ getFileMeta(key: string): Promise<StorageObject | null>
listAll(): Promise<StorageObject[]>
listImages(): Promise<StorageObject[]>
}
diff --git a/server/services/storage/providers/s3.ts b/server/services/storage/providers/s3.ts
index 9cfb738b..d6ae2e61 100644
--- a/server/services/storage/providers/s3.ts
+++ b/server/services/storage/providers/s3.ts
@@ -1,12 +1,10 @@
-import type {
- _Object,
- S3ClientConfig} from '@aws-sdk/client-s3';
+import type { _Object, S3ClientConfig } from '@aws-sdk/client-s3'
import {
DeleteObjectCommand,
GetObjectCommand,
ListObjectsCommand,
PutObjectCommand,
- S3Client
+ S3Client,
} from '@aws-sdk/client-s3'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
import type {
@@ -182,6 +180,34 @@ export class S3StorageProvider implements StorageProvider {
return url
}
+ async getFileMeta(key: string): Promise<StorageObject | null> {
+ try {
+ const cmd = new GetObjectCommand({
+ Bucket: this.config.bucket,
+ Key: key,
+ })
+
+ const resp = await this.client.send(cmd)
+
+ if (!resp.ETag) {
+ return null
+ }
+
+ return {
+ key,
+ size: resp.ContentLength || 0,
+ lastModified: resp.LastModified,
+ etag: resp.ETag,
+ }
+ } catch (error) {
+ if ((error as any).$metadata?.httpStatusCode === 404) {
+ return null
+ }
+ this.logger?.error(`Failed to get metadata for key: ${key}`, error)
+ throw error
+ }
+ }
+
async listAll(): Promise<StorageObject[]> {
const cmd = new ListObjectsCommand({
Bucket: this.config.bucket,
diff --git a/server/utils/db.ts b/server/utils/db.ts
index f7151e49..1331f160 100644
--- a/server/utils/db.ts
+++ b/server/utils/db.ts
@@ -15,3 +15,6 @@ export function useDB() {
export type User = typeof schema.users.$inferSelect
export type Photo = typeof schema.photos.$inferSelect
+
+export type PipelineQueueItem = typeof schema.pipelineQueue.$inferSelect
+export type NewPipelineQueueItem = typeof schema.pipelineQueue.$inferInsert
diff --git a/shared/types/global.d.ts b/shared/types/global.d.ts
new file mode 100644
index 00000000..08489f94
--- /dev/null
+++ b/shared/types/global.d.ts
@@ -0,0 +1,7 @@
+import type { WorkerPool } from '../../server/services/pipeline-queue'
+
+declare global {
+ var __workerPool: WorkerPool | undefined
+}
+
+export {}
\ No newline at end of file