Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
feat!: 照片处理队列和 WorkerPool 管理器,队列管理 API,优化上传处理进度展示
Timothy Yin01cf41a0500e
diff --git a/app/pages/dashboard/photos.vue b/app/pages/dashboard/photos.vueindex 8259f9c4..fdfd5e79 100644--- a/app/pages/dashboard/photos.vue+++ b/app/pages/dashboard/photos.vue@@ -1,6 +1,6 @@<script lang="ts" setup>import type { TableColumn } from '@nuxt/ui'-import type { Photo } from '~~/server/utils/db'+import type { Photo, PipelineQueueItem } from '~~/server/utils/db'import { h, resolveComponent } from 'vue'import { Icon, UBadge } from '#components'import ThumbImage from '~/components/ThumbImage.vue'@@ -16,20 +16,27 @@ useHead({})const dayjs = useDayjs()-const { localizeExif } = useExifLocalization()-const { photos: data, status, refresh } = usePhotos()-const { filteredPhotos, activeFilters } = usePhotoFilters()+const { status, refresh } = usePhotos()+const { filteredPhotos, selectedCounts, hasActiveFilters } = usePhotoFilters()++const totalSelectedFilters = computed(() => {+ return Object.values(selectedCounts.value).reduce(+ (total, count) => total + count,+ 0,+ )+})-// 当前上传的文件信息interface UploadingFile {file: FilefileName: string+ fileId: stringstatus: 'preparing' | 'uploading' | 'processing' | 'completed' | 'error'+ stage?: PipelineQueueItem['statusStage'] | nullprogress?: numbererror?: string+ taskId?: numbersignedUrlResponse?: { signedUrl: string; fileKey: string; expiresIn: number }- // 添加详细进度信息uploadProgress?: {loaded: numbertotal: number@@ -39,31 +46,30 @@ interface UploadingFile {speedText?: stringtimeRemainingText?: string}- // 添加上传管理器的关键方法canAbort?: booleanabortUpload?: () => void}const uploadingFiles = ref<Map<string, UploadingFile>>(new Map())-// 使用预签名 URL 上传文件函数+// 重写的文件上传和处理函数const uploadImage = async (file: File) => {const fileName = file.nameconst fileId = `${Date.now()}-${fileName}`- // 为每个文件创建独立的上传管理器const uploadManager = useUpload({timeout: 10 * 60 * 1000, // 10分钟超时})- // 添加到上传队列const uploadingFile: UploadingFile = {file,fileName,+ fileId,status: 'preparing',canAbort: false,abortUpload: () => uploadManager.abortUpload(),}+uploadingFiles.value.set(fileId, uploadingFile)try {@@ -80,12 +86,12 @@ const uploadImage = async (file: File) => {uploadingFile.signedUrlResponse = signedUrlResponseuploadingFile.status = 'uploading'uploadingFile.canAbort = true+ uploadingFile.progress = 0+ uploadingFiles.value = new Map(uploadingFiles.value)- // 第二步:使用 composable 上传文件到 S3+ // 第二步:使用 composable 上传文件到存储await uploadManager.uploadFile(file, signedUrlResponse.signedUrl, {onProgress: (progress: UploadProgress) => {- // 更新文件特定的进度信息- console.log(`文件 ${fileName} 上传进度:`, progress.percentage + '%')uploadingFile.progress = progress.percentageuploadingFile.uploadProgress = {loaded: progress.loaded,@@ -98,48 +104,53 @@ const uploadImage = async (file: File) => {? dayjs.duration(progress.timeRemaining, 'seconds').humanize(): '',}-- // 触发响应式更新uploadingFiles.value = new Map(uploadingFiles.value)},onStatusChange: (status: string) => {- console.log(`文件 ${fileName} 上传状态变更:`, status)- // 根据状态更新 canAbortuploadingFile.canAbort = status === 'uploading'uploadingFiles.value = new Map(uploadingFiles.value)},- onSuccess: async (xhr: XMLHttpRequest) => {- console.log(`文件 ${fileName} 上传成功`, xhr.status)-- // 第三步:通知服务器开始处理照片+ onSuccess: async (_xhr: XMLHttpRequest) => {+ // 第三步:上传完成,提交到队列任务uploadingFile.status = 'processing'+ uploadingFile.progress = 100uploadingFile.canAbort = false+ uploadingFile.stage = null // 重置 stage,准备显示任务状态uploadingFiles.value = new Map(uploadingFiles.value)try {- await $fetch('/api/photos/process', {- method: 'POST' as any,+ const resp = await $fetch('/api/queue/add-task', {+ method: 'POST',body: {- fileKey: signedUrlResponse.fileKey,- fileName: file.name,- fileSize: file.size,+ payload: {+ type:+ file.type === 'video/quicktime'+ ? 'live-photo-video'+ : 'photo',+ storageKey: signedUrlResponse.fileKey,+ },+ priority: 0,+ maxAttempts: 3,},})- uploadingFile.status = 'completed'- uploadingFile.canAbort = false- uploadingFiles.value = new Map(uploadingFiles.value)-- // 开始状态检查- if (!statusInterval) {- statusInterval = setInterval(checkProcessingStatus, 2000)+ if (resp.success) {+ uploadingFile.taskId = resp.taskId+ uploadingFile.status = 'processing'+ uploadingFiles.value = new Map(uploadingFiles.value)++ // 开始任务状态检查+ startTaskStatusCheck(resp.taskId, fileId)+ } else {+ uploadingFile.status = 'error'+ uploadingFile.error = '提交处理任务失败'+ uploadingFiles.value = new Map(uploadingFiles.value)}} catch (processError: any) {uploadingFile.status = 'error'- uploadingFile.error = `处理失败: ${processError.message}`+ uploadingFile.error = `提交处理任务失败: ${processError.message}`uploadingFile.canAbort = falseuploadingFiles.value = new Map(uploadingFiles.value)- throw processError}},onError: (error: string) => {@@ -147,7 +158,6 @@ const uploadImage = async (file: File) => {uploadingFile.error = erroruploadingFile.canAbort = falseuploadingFiles.value = new Map(uploadingFiles.value)- console.error(`文件 ${fileName} 上传失败:`, error)},})} catch (error: any) {@@ -156,52 +166,39 @@ const uploadImage = async (file: File) => {uploadingFile.canAbort = falseuploadingFiles.value = new Map(uploadingFiles.value)- console.error('上传请求失败:', error)-// 提供更详细的错误信息if (error.response?.status === 401) {- throw new Error('未授权,请重新登录')+ uploadingFile.error = '未授权,请重新登录'} else if (error.message?.includes('CORS')) {- throw new Error('跨域请求失败,请检查存储服务的 CORS 配置')+ uploadingFile.error = '跨域请求失败,请检查存储服务的 CORS 配置'} else if (error.message?.includes('NetworkError') ||error.name === 'TypeError') {- throw new Error('网络连接失败或 CORS 错误,请检查网络连接和存储配置')+ uploadingFile.error = '网络连接失败或 CORS 错误,请检查网络连接和存储配置'} else if (error.message?.includes('上传到存储失败')) {- throw new Error('文件上传到云存储失败,请重试')- } else {- throw new Error(error.message || '上传失败,请重试')+ uploadingFile.error = '文件上传到云存储失败,请重试'}- } finally {- // 上传完成后从队列中移除(延迟移除以便用户看到结果)- setTimeout(() => {- uploadingFiles.value.delete(fileId)- uploadingFiles.value = new Map(uploadingFiles.value)- }, 5000)++ uploadingFiles.value = new Map(uploadingFiles.value)}}-// 处理状态-const processingFiles = ref<Set<string>>(new Set())const toast = useToast()const selectedFiles = ref<File[]>([])-// 表格多选状态 - 使用官方推荐的方式+// 表格多选状态const rowSelection = ref({})const table: any = useTemplateRef('table')-// 计算选中的行数const selectedRowsCount = computed((): number => {return table.value?.tableApi?.getFilteredSelectedRowModel().rows.length || 0})-// 计算总行数const totalRowsCount = computed((): number => {return table.value?.tableApi?.getFilteredRowModel().rows.length || 0})-// 计算 LivePhoto 统计const livePhotoStats = computed(() => {if (!filteredPhotos.value) return { total: 0, livePhotos: 0, staticPhotos: 0 }@@ -214,10 +211,8 @@ const livePhotoStats = computed(() => {return { total, livePhotos, staticPhotos }})-// 照片过滤器const photoFilter = ref<'all' | 'livephoto' | 'static'>('all')-// 过滤后的数据const filteredData = computed(() => {if (!filteredPhotos.value) return []@@ -231,8 +226,144 @@ const filteredData = computed(() => {}})-// 状态检查间隔-let statusInterval: NodeJS.Timeout | null = null+// 状态检查间隔 Map,每个任务对应一个定时器+const statusIntervals = ref<Map<number, NodeJS.Timeout>>(new Map())++// 启动任务状态检查+const startTaskStatusCheck = (taskId: number, fileId: string) => {+ const intervalId = setInterval(async () => {+ try {+ const response = await $fetch(`/api/queue/stats/${taskId}`)+ const uploadingFile = uploadingFiles.value.get(fileId)++ if (!uploadingFile) {+ clearInterval(intervalId)+ statusIntervals.value.delete(taskId)+ return+ }++ // 更新任务状态+ uploadingFile.stage =+ response.status === 'in-stages' ? response.statusStage : null+ uploadingFiles.value = new Map(uploadingFiles.value)++ if (response.status === 'completed') {+ // 任务完成+ uploadingFile.status = 'completed'+ uploadingFile.stage = null+ uploadingFiles.value = new Map(uploadingFiles.value)++ // 停止状态检查+ clearInterval(intervalId)+ statusIntervals.value.delete(taskId)++ // 显示成功提示+ toast.add({+ title: '照片处理完成',+ description: `照片 ${uploadingFile.fileName} 已完成处理`,+ color: 'success',+ })++ // 刷新照片列表+ await refresh()++ // 2秒后从界面移除成功的任务+ setTimeout(() => {+ uploadingFiles.value.delete(fileId)+ uploadingFiles.value = new Map(uploadingFiles.value)+ }, 2000)+ } else if (response.status === 'failed') {+ // 任务失败+ uploadingFile.status = 'error'+ uploadingFile.error = `处理失败: ${response.errorMessage || '未知错误'}`+ uploadingFile.stage = null+ uploadingFiles.value = new Map(uploadingFiles.value)++ // 停止状态检查+ clearInterval(intervalId)+ statusIntervals.value.delete(taskId)++ // 显示错误提示+ toast.add({+ title: '照片处理失败',+ description: `照片 ${uploadingFile.fileName} 处理失败: ${response.errorMessage || '未知错误'}`,+ color: 'error',+ })+ // 失败的任务不自动移除,让用户查看错误信息+ }+ } catch (error) {+ console.error('检查任务状态失败:', error)++ // 如果检查状态失败,清理定时器+ clearInterval(intervalId)+ statusIntervals.value.delete(taskId)++ const uploadingFile = uploadingFiles.value.get(fileId)+ if (uploadingFile) {+ uploadingFile.status = 'error'+ uploadingFile.error = '状态检查失败,请刷新页面重试'+ uploadingFiles.value = new Map(uploadingFiles.value)+ }+ }+ }, 1000) // 每秒检查一次++ statusIntervals.value.set(taskId, intervalId)+}++// 手动移除上传任务+const removeUploadingFile = (fileId: string) => {+ const uploadingFile = uploadingFiles.value.get(fileId)++ // 如果任务还在进行中,先清理定时器+ if (uploadingFile?.taskId) {+ const intervalId = statusIntervals.value.get(uploadingFile.taskId)+ if (intervalId) {+ clearInterval(intervalId)+ statusIntervals.value.delete(uploadingFile.taskId)+ }+ }++ // 从列表中移除+ uploadingFiles.value.delete(fileId)+ uploadingFiles.value = new Map(uploadingFiles.value)+}++// 批量清除已完成和错误的任务+const clearCompletedTasks = () => {+ const toRemove: string[] = []++ for (const [fileId, uploadingFile] of uploadingFiles.value) {+ if (+ uploadingFile.status === 'completed' ||+ uploadingFile.status === 'error'+ ) {+ toRemove.push(fileId)++ // 清理可能存在的定时器+ if (uploadingFile.taskId) {+ const intervalId = statusIntervals.value.get(uploadingFile.taskId)+ if (intervalId) {+ clearInterval(intervalId)+ statusIntervals.value.delete(uploadingFile.taskId)+ }+ }+ }+ }++ toRemove.forEach((fileId) => {+ uploadingFiles.value.delete(fileId)+ })++ uploadingFiles.value = new Map(uploadingFiles.value)++ if (toRemove.length > 0) {+ toast.add({+ title: '任务清理完成',+ description: `已清除 ${toRemove.length} 个已完成的任务`,+ color: 'info',+ })+ }+}const columns: TableColumn<Photo>[] = [{@@ -386,9 +517,8 @@ const columns: TableColumn<Photo>[] = [cell: (info) => formatBytes(info.getValue() as number),},{- accessorKey: 'exif.ColorSpace',+ accessorFn: (row) => row.exif?.ColorSpace,header: '颜色空间',- cell: (info) => localizeExif('colorSpace', info.getValue() as string),},{accessorKey: 'actions',@@ -396,46 +526,6 @@ const columns: TableColumn<Photo>[] = [},]-const checkProcessingStatus = async () => {- if (processingFiles.value.size === 0) {- // 如果没有文件在处理,停止检查- if (statusInterval) {- clearInterval(statusInterval)- statusInterval = null- }- return- }-- try {- const response = await $fetch('/api/photos/status')- const currentPhotoCount = data.value?.length || 0-- // 简单检查:如果照片数量增加了,说明有照片处理完成- if (response.recentPhotos?.length > currentPhotoCount) {- // 刷新照片列表- await refresh()-- // 清空处理中的文件列表- processingFiles.value.clear()-- // 停止状态检查- if (statusInterval) {- clearInterval(statusInterval)- statusInterval = null- }-- toast.add({- title: '照片处理完成',- description: '您的照片已成功处理并上传',- color: 'success',- })- }- } catch (error) {- console.error('检查状态失败:', error)- // 不显示错误给用户,避免频繁弹窗- }-}-// 文件验证函数const validateFile = (file: File): { valid: boolean; error?: string } => {// 检查文件类型@@ -489,36 +579,10 @@ const handleUpload = async () => {continue}- const fileName = file.name- const isVideoFile = fileName.toLowerCase().endsWith('.mov')- processingFiles.value.add(fileName)-- toast.add({- title: isVideoFile ? '开始上传 LivePhoto 视频' : '开始上传照片',- description: `正在上传 ${fileName} 到云存储...`,- color: 'info',- })-try {await uploadImage(file)-- toast.add({- title: '照片上传成功',- description: `${fileName} 已上传,正在后台处理...`,- color: 'success',- })-- // 开始状态检查- if (!statusInterval) {- statusInterval = setInterval(checkProcessingStatus, 2000)- }} catch (error: any) {- processingFiles.value.delete(fileName)- toast.add({- title: '上传失败',- description: `${fileName}: ${error.message}`,- color: 'error',- })+ // 错误已经在 uploadImage 函数内部处理了console.error('上传错误:', error)}}@@ -565,59 +629,74 @@ const handleViewLivePhoto = async (photoId: string) => {}}-// LivePhoto 模态框相关状态-const isLivePhotoModalOpen = ref(false)-const selectedLivePhoto = ref<{- id: string- title: string | null- originalUrl: string- videoUrl: string-} | null>(null)--// 打开链接的辅助函数-const openInNewTab = (url: string) => {- if (typeof window !== 'undefined') {- window.open(url, '_blank')- }-}--const handleUpdateLivePhoto = async (photoId: string) => {+// 重新处理单张照片+const handleReprocessSingle = async (photo: Photo) => {try {- const updateToast = toast.add({- title: '正在检查 LivePhoto',- description: '正在检查是否有对应的视频文件...',+ if (!photo || !photo.storageKey) {+ toast.add({+ title: '处理失败',+ description: '无法找到照片的存储键',+ color: 'error',+ })+ return+ }++ const reprocessToast = toast.add({+ title: '正在处理照片',+ description: '正在添加到任务队列...',color: 'info',})- const result = (await $fetch('/api/photos/livephoto/manage', {+ const result = await $fetch('/api/queue/add-task', {method: 'POST',body: {- action: 'update-photo',- photoId: photoId,+ payload: {+ type: 'photo',+ storageKey: photo.storageKey,+ },+ priority: 0,+ maxAttempts: 3,},- })) as any-- const isSuccess = 'success' in result && result.success-- toast.update(updateToast.id, {- title: isSuccess ? 'LivePhoto 更新成功' : 'LivePhoto 更新失败',- description: result.message,- color: isSuccess ? 'success' : 'warning',})- if (isSuccess) {- await refresh()+ if (result.success) {+ toast.update(reprocessToast.id, {+ title: '处理任务已添加',+ description: `已添加到任务队列,任务ID: ${result.taskId}`,+ color: 'success',+ })+ } else {+ toast.update(reprocessToast.id, {+ title: '添加任务失败',+ description: '无法添加处理任务',+ color: 'error',+ })}} catch (error: any) {- console.error('LivePhoto 更新失败:', error)+ console.error('处理照片失败:', error)toast.add({- title: 'LivePhoto 更新失败',- description: error.message || '更新过程中发生错误',+ title: '处理失败',+ description: error.message || '添加任务时发生错误',color: 'error',})}}+// LivePhoto Modal+const isLivePhotoModalOpen = ref(false)+const selectedLivePhoto = ref<{+ id: string+ title: string | null+ originalUrl: string+ videoUrl: string+} | null>(null)++const openInNewTab = (url: string) => {+ if (typeof window !== 'undefined') {+ window.open(url, '_blank')+ }+}+// 批量删除功能const handleBatchDelete = async () => {const selectedRowModel = table.value?.tableApi?.getFilteredSelectedRowModel()@@ -669,8 +748,8 @@ const handleBatchDelete = async () => {}}-// 批量检测 LivePhoto 功能(只对选中的照片)-const handleBatchDetectLivePhoto = async () => {+// 批量重新处理照片功能+const handleBatchReprocess = async () => {const selectedRowModel = table.value?.tableApi?.getFilteredSelectedRowModel()const selectedPhotos =selectedRowModel?.rows.map((row: any) => row.original) || []@@ -678,138 +757,67 @@ const handleBatchDetectLivePhoto = async () => {if (selectedPhotos.length === 0) {toast.add({title: '请选择照片',- description: '请至少选择一张照片进行 LivePhoto 检测',+ description: '请至少选择一张照片进行处理',color: 'warning',})return}- try {- const detectToast = toast.add({- title: '正在检测 LivePhoto',- description: `正在检查所选 ${selectedPhotos.length} 张照片的 LivePhoto 状态...`,- color: 'info',- })-- const photoIds = selectedPhotos.map((photo: any) => photo.id)-- const result = (await $fetch('/api/photos/livephoto/manage', {- method: 'POST',- body: {- action: 'detect',- photoIds: photoIds, // 传递选中的照片ID列表- },- })) as any-- const foundCount = result.results?.found || 0- const processedCount = result.results?.processed || 0-- toast.update(detectToast.id, {- title: 'LivePhoto 检测完成',- description: `检测了 ${processedCount} 张照片,发现 ${foundCount} 个潜在的 LivePhoto`,- color: foundCount > 0 ? 'success' : 'info',- })-- if (foundCount > 0) {- await refresh()- }-- // 清空选中状态- rowSelection.value = {}- } catch (error: any) {- console.error('批量检测 LivePhoto 失败:', error)+ // 检查所有选中照片是否都有 storageKey+ const photosWithStorageKey = selectedPhotos.filter(+ (photo: Photo) => photo.storageKey,+ )+ if (photosWithStorageKey.length !== selectedPhotos.length) {toast.add({- title: '批量检测失败',- description: error.message || '检测过程中发生错误',+ title: '处理失败',+ description: `${selectedPhotos.length - photosWithStorageKey.length} 张照片缺少存储信息,无法处理`,color: 'error',})- }-}--// 批量重新索引 EXIF 功能(只对选中的照片)-const handleBatchReindexExif = async () => {- const selectedRowModel = table.value?.tableApi?.getFilteredSelectedRowModel()- const selectedPhotos =- selectedRowModel?.rows.map((row: any) => row.original) || []-- if (selectedPhotos.length === 0) {- toast.add({- title: '请选择照片',- description: '请至少选择一张照片进行 EXIF 重新索引',- color: 'warning',- })return}try {- const reindexToast = toast.add({- title: '正在重新索引 EXIF',- description: `正在重新提取所选 ${selectedPhotos.length} 张照片的 EXIF 数据...`,+ const reprocessToast = toast.add({+ title: '正在添加处理任务',+ description: `正在为所选 ${photosWithStorageKey.length} 张照片添加处理任务...`,color: 'info',})- const photoIds = selectedPhotos.map((photo: any) => photo.id)-- const result = (await $fetch('/api/photos/exif/reindex', {+ const result = await $fetch('/api/queue/add-tasks', {method: 'POST',body: {- action: 'batch-reindex',- photoIds: photoIds, // 传递选中的照片ID列表+ tasks: photosWithStorageKey.map((photo: Photo) => ({+ payload: {+ type: 'photo',+ storageKey: photo.storageKey,+ },+ priority: 0,+ maxAttempts: 3,+ })),},- })) as any-- const processedCount = result.results?.processed || 0- const updatedCount = result.results?.updated || 0-- toast.update(reindexToast.id, {- title: 'EXIF 重新索引完成',- description: `处理了 ${processedCount} 张照片,更新了 ${updatedCount} 张照片的 EXIF 数据`,- color: 'success',})- await refresh()+ if (result.success) {+ toast.update(reprocessToast.id, {+ title: '任务已添加',+ description: `已将 ${photosWithStorageKey.length} 张照片添加到处理队列`,+ color: 'success',+ })+ } else {+ toast.update(reprocessToast.id, {+ title: '添加任务失败',+ description: '无法添加批量重新处理任务',+ color: 'error',+ })+ }// 清空选中状态rowSelection.value = {}} catch (error: any) {- console.error('批量重新索引 EXIF 失败:', error)+ console.error('批量处理失败:', error)toast.add({- title: 'EXIF 重新索引失败',- description: error.message || '重新索引过程中发生错误',- color: 'error',- })- }-}--// 单个照片重新索引 EXIF 功能-const handleReindexSingleExif = async (photoId: string) => {- try {- const reindexToast = toast.add({- title: '正在重新索引 EXIF',- description: '正在重新提取照片的 EXIF 数据...',- color: 'info',- })-- const result = (await $fetch('/api/photos/exif/reindex', {- method: 'POST',- body: {- action: 'single-reindex',- photoId: photoId,- },- })) as any-- toast.update(reindexToast.id, {- title: 'EXIF 重新索引完成',- description: result.message || '照片 EXIF 数据已更新',- color: 'success',- })-- await refresh()- } catch (error: any) {- console.error('重新索引 EXIF 失败:', error)- toast.add({- title: 'EXIF 重新索引失败',- description: error.message || '重新索引过程中发生错误',+ title: '批量处理失败',+ description: error.message || '添加任务时发生错误',color: 'error',})}@@ -817,9 +825,11 @@ const handleReindexSingleExif = async (photoId: string) => {// 清理定时器onUnmounted(() => {- if (statusInterval) {- clearInterval(statusInterval)- }+ // 清理所有状态检查定时器+ statusIntervals.value.forEach((intervalId) => {+ clearInterval(intervalId)+ })+ statusIntervals.value.clear()})</script>@@ -830,7 +840,18 @@ onUnmounted(() => {v-if="uploadingFiles.size > 0"class="space-y-3">- <h3 class="text-lg font-semibold">上传进度</h3>+ <div class="flex items-center justify-between">+ <h3 class="text-lg font-semibold">上传进度</h3>+ <UButton+ size="sm"+ color="neutral"+ variant="ghost"+ icon="tabler:trash"+ @click="clearCompletedTasks"+ >+ 清除已完成任务+ </UButton>+ </div><div class="space-y-2"><divv-for="[fileId, uploadingFile] of uploadingFiles"@@ -862,33 +883,66 @@ onUnmounted(() => {uploadingFile.status === 'preparing'? '准备中': uploadingFile.status === 'uploading'- ? '上传中'+ ? `上传中 ${uploadingFile.progress || 0}%`: uploadingFile.status === 'processing'- ? '处理中'+ ? uploadingFile.stage+ ? uploadingFile.stage === 'preprocessing'+ ? '预处理中'+ : uploadingFile.stage === 'metadata'+ ? '提取元数据'+ : uploadingFile.stage === 'thumbnail'+ ? '生成缩略图'+ : uploadingFile.stage === 'exif'+ ? '处理EXIF'+ : uploadingFile.stage === 'reverse-geocoding'+ ? '地理解析'+ : uploadingFile.stage === 'live-photo'+ ? '处理LivePhoto'+ : '处理中'+ : '等待处理': uploadingFile.status === 'completed'? '完成': '错误'}}</UBadge>- <!-- 中止按钮 -->- <UButton- v-if="- uploadingFile.status === 'uploading' && uploadingFile.canAbort- "- size="xs"- color="error"- variant="soft"- icon="tabler:file-x"- @click="uploadingFile.abortUpload?.()"- >- <span class="hidden sm:inline">中止上传</span>- <span class="sm:hidden">中止</span>- </UButton>+ <!-- 操作按钮 -->+ <div class="flex items-center gap-2">+ <!-- 中止上传按钮 -->+ <UButton+ v-if="+ uploadingFile.status === 'uploading' &&+ uploadingFile.canAbort+ "+ size="xs"+ color="error"+ variant="soft"+ icon="tabler:file-x"+ @click="uploadingFile.abortUpload?.()"+ >+ <span class="hidden sm:inline">中止上传</span>+ <span class="sm:hidden">中止</span>+ </UButton>++ <!-- 清除任务按钮 -->+ <UButton+ v-if="+ uploadingFile.status === 'completed' ||+ uploadingFile.status === 'error'+ "+ size="xs"+ color="neutral"+ variant="ghost"+ icon="tabler:x"+ @click="removeUploadingFile(fileId)"+ >+ <span class="hidden sm:inline">清除</span>+ </UButton>+ </div></div></div>- <!-- 进度条 -->+ <!-- 上传进度条 --><divv-if="uploadingFile.status === 'uploading' &&@@ -899,7 +953,7 @@ onUnmounted(() => {<divclass="flex justify-between text-xs sm:text-sm text-neutral-600 dark:text-neutral-400 mb-1">- <span>{{ uploadingFile.progress }}%</span>+ <span>上传进度: {{ uploadingFile.progress }}%</span><spanv-if="uploadingFile.uploadProgress?.speedText"class="hidden sm:inline"@@ -915,11 +969,45 @@ onUnmounted(() => {v-if="uploadingFile.uploadProgress?.timeRemainingText"class="text-xs text-neutral-500 mt-1">- <span class="hidden sm:inline">ETA: </span- >{{ uploadingFile.uploadProgress.timeRemainingText }}+ <span class="hidden sm:inline">预计剩余时间: </span>+ {{ uploadingFile.uploadProgress.timeRemainingText }}</div></div>+ <!-- 处理状态进度条 -->+ <div+ v-if="uploadingFile.status === 'processing'"+ class="mb-2"+ >+ <div+ class="flex justify-between text-xs sm:text-sm text-neutral-600 dark:text-neutral-400 mb-1"+ >+ <span>+ 处理状态:+ {{+ uploadingFile.stage === 'preprocessing'+ ? '预处理中'+ : uploadingFile.stage === 'metadata'+ ? '提取元数据'+ : uploadingFile.stage === 'thumbnail'+ ? '生成缩略图'+ : uploadingFile.stage === 'exif'+ ? '处理EXIF信息'+ : uploadingFile.stage === 'reverse-geocoding'+ ? '地理位置解析'+ : uploadingFile.stage === 'live-photo'+ ? '处理LivePhoto'+ : '等待处理...'+ }}+ </span>+ </div>+ <UProgress+ :model-value="uploadingFile.stage ? undefined : null"+ animation="carousel"+ class="h-2"+ />+ </div>+<!-- 错误信息 --><UAlertv-if="uploadingFile.status === 'error' && uploadingFile.error"@@ -968,15 +1056,15 @@ onUnmounted(() => {<!-- 工具栏 --><div- class="flex flex-row sm:items-center justify-between gap-3 sm:gap-0 p-3 sm:p-4 bg-gray-50 dark:bg-gray-900/50 border border-gray-200 dark:border-gray-700 rounded-lg"+ class="flex flex-row sm:items-center justify-between gap-3 sm:gap-0 p-3 sm:p-4 bg-neutral-50 dark:bg-neutral-900/50 border border-neutral-200 dark:border-neutral-700 rounded-lg"><div class="flex items-center gap-2"><UIconname="tabler:photo"- class="text-gray-500"+ class="text-neutral-500"/><span- class="font-medium text-gray-700 dark:text-gray-300 hidden sm:inline"+ class="font-medium text-neutral-700 dark:text-neutral-300 hidden sm:inline">照片管理</span>@@ -1007,6 +1095,30 @@ onUnmounted(() => {</div><div class="flex items-center gap-2">+ <UPopover>+ <UTooltip :text="$t('ui.action.filter.tooltip')">+ <UChip+ inset+ size="sm"+ color="info"+ :show="totalSelectedFilters > 0"+ >+ <UButton+ variant="soft"+ :color="hasActiveFilters ? 'info' : 'neutral'"+ class="bg-transparent rounded-full cursor-pointer relative"+ icon="tabler:filter"+ size="sm"+ />+ </UChip>+ </UTooltip>++ <template #content>+ <UCard variant="glassmorphism">+ <OverlayFilterPanel />+ </UCard>+ </template>+ </UPopover><!-- 过滤器 --><USelectMenuv-model="photoFilter"@@ -1026,15 +1138,6 @@ onUnmounted(() => {></USelectMenu>- <!-- 模糊搜索 -->- <UInput- v-model="activeFilters.search"- size="sm"- icon="tabler:search"- placeholder="搜索照片..."- class="w-32"- />-<!-- 刷新按钮 --><UButtonvariant="soft"@@ -1048,68 +1151,6 @@ onUnmounted(() => {</div></div>- <!-- 批量操作栏 -->- <div- v-if="selectedRowsCount > 0"- class="flex flex-col sm:flex-row sm:items-center justify-between gap-3 sm:gap-0 p-3 sm:p-4 bg-blue-50 dark:bg-blue-950/20 border border-blue-200 dark:border-blue-800 rounded-lg"- >- <div class="flex items-center gap-2">- <UIcon- name="tabler:check-circle"- class="text-blue-500"- />- <span class="font-medium text-blue-700 dark:text-blue-300">- <span>已选择</span> {{ selectedRowsCount }} <span>张照片</span>- </span>- <UButton- variant="ghost"- size="sm"- @click="rowSelection = {}"- >- <span>取消选择</span>- </UButton>- </div>- <div class="flex flex-wrap sm:flex-nowrap items-center gap-1 sm:gap-2">- <!-- 重新索引 EXIF 按钮 -->- <UButton- variant="soft"- color="success"- size="sm"- icon="tabler:refresh"- class="flex-1 sm:flex-none"- @click="handleBatchReindexExif"- >- <span class="hidden sm:inline">重新索引 EXIF</span>- <span class="sm:hidden">索引</span>- </UButton>-- <!-- 实况配对按钮 -->- <UButton- variant="soft"- color="info"- size="sm"- icon="tabler:live-photo"- class="flex-1 sm:flex-none"- @click="handleBatchDetectLivePhoto"- >- <span class="hidden sm:inline">实况配对</span>- <span class="sm:hidden">配对</span>- </UButton>-- <UButton- color="error"- variant="soft"- size="sm"- icon="tabler:trash"- class="flex-1 sm:flex-none"- @click="handleBatchDelete"- >- <span class="hidden sm:inline">批量删除</span>- <span class="sm:hidden">删除</span>- </UButton>- </div>- </div>-<!-- 照片列表 --><divclass="border border-neutral-300 dark:border-neutral-800 rounded overflow-hidden"@@ -1124,7 +1165,7 @@ onUnmounted(() => {:columns="columns":loading="status === 'pending'"sticky- class="h-[500px] sm:h-[600px]"+ class="h-[calc(100vh-25rem)] sm:h-[calc(100vh-24.5rem)]":ui="{separator:'bg-(--ui-color-neutral-200) dark:bg-(--ui-color-neutral-700)',@@ -1141,22 +1182,16 @@ onUnmounted(() => {[{color: row.original.isLivePhoto ? 'warning' : 'info',- label: row.original.isLivePhoto ? '实况预览' : '实况配对',+ label: row.original.isLivePhoto ? '实况预览' : '重新处理',icon: row.original.isLivePhoto? 'tabler:live-photo': 'tabler:refresh',onSelect() {row.original.isLivePhoto? handleViewLivePhoto(row.original.id)- : handleUpdateLivePhoto(row.original.id)+ : handleReprocessSingle(row.original)},},- {- color: 'success',- label: '重新索引 EXIF',- icon: 'tabler:refresh',- onSelect: () => handleReindexSingleExif(row.original.id),- },],[{@@ -1179,11 +1214,43 @@ onUnmounted(() => {</template></UTable>- <!-- 选择状态信息 -->+ <!-- 选择状态信息和批量操作 --><div- class="px-4 py-3.5 border-t border-neutral-200 dark:border-neutral-700 text-sm text-neutral-600 dark:text-neutral-400"+ class="px-4 py-4 border-t border-neutral-200 dark:border-neutral-700">- {{ selectedRowsCount }} / {{ totalRowsCount }} 行已选择+ <div+ class="text-sm text-neutral-600 dark:text-neutral-400 flex items-center gap-2"+ >+ <div class="leading-6">+ {{ selectedRowsCount }} / {{ totalRowsCount }} 行已选择+ </div>+ <div+ v-if="selectedRowsCount > 0"+ class="flex items-center gap-1 sm:gap-2"+ >+ <UButton+ variant="soft"+ color="info"+ size="xs"+ icon="tabler:refresh"+ class="flex-1 sm:flex-none"+ @click="handleBatchReprocess"+ >+ <span>重新处理</span>+ </UButton>++ <UButton+ color="error"+ variant="soft"+ size="xs"+ icon="tabler:trash"+ class="flex-1 sm:flex-none"+ @click="handleBatchDelete"+ >+ <span>批量删除</span>+ </UButton>+ </div>+ </div></div></div>diff --git a/docs/queue-api.md b/docs/queue-api.mdnew file mode 100644index 00000000..ee679491--- /dev/null+++ b/docs/queue-api.md@@ -0,0 +1,228 @@+# 队列系统 API 使用指南++支持多个并发消费者的队列处理系统。以下是通过 API 创建和管理任务的方法:++## API 概览++### 1. 添加单个任务+**POST** `/api/queue/add-task`++### 2. 批量添加任务+**POST** `/api/queue/add-tasks`++### 3. 查看队列状态+**GET** `/api/queue/status`++## 使用示例++### 1. 添加单个任务++```javascript+// 添加单个照片处理任务+const response = await fetch('/api/queue/add-task', {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ },+ body: JSON.stringify({+ payload: {+ storageKey: 'photos/2024/IMG_1234.HEIC'+ },+ priority: 5, // 可选,优先级 0-10,默认 0+ maxAttempts: 3 // 可选,最大重试次数 1-5,默认 3+ })+})++const result = await response.json()+console.log('任务已添加:', result)+// 输出:+// {+// "success": true,+// "taskId": 123,+// "message": "Task added to queue successfully",+// "payload": {+// "storageKey": "photos/2024/IMG_1234.HEIC",+// "priority": 5,+// "maxAttempts": 3+// }+// }+```++### 2. 批量添加任务++```javascript+// 批量添加多个照片处理任务+const response = await fetch('/api/queue/add-tasks', {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ },+ body: JSON.stringify({+ tasks: [+ {+ payload: { storageKey: 'photos/2024/IMG_1234.HEIC' },+ priority: 8+ },+ {+ payload: { storageKey: 'photos/2024/IMG_1235.HEIC' },+ priority: 5+ },+ {+ payload: { storageKey: 'photos/2024/IMG_1236.HEIC' }+ // 使用默认优先级和重试次数+ }+ ],+ defaultPriority: 3, // 可选,默认优先级+ defaultMaxAttempts: 3 // 可选,默认最大重试次数+ })+})++const result = await response.json()+console.log('批量任务结果:', result)+// 输出:+// {+// "success": true,+// "totalTasks": 3,+// "successCount": 3,+// "errorCount": 0,+// "results": [+// { "index": 0, "taskId": 124, "storageKey": "photos/2024/IMG_1234.HEIC", "success": true },+// { "index": 1, "taskId": 125, "storageKey": "photos/2024/IMG_1235.HEIC", "success": true },+// { "index": 2, "taskId": 126, "storageKey": "photos/2024/IMG_1236.HEIC", "success": true }+// ],+// "message": "Processed 3 tasks: 3 successful, 0 failed"+// }+```++### 3. 查看队列状态++```javascript+// 获取实时队列和工作器状态+const response = await fetch('/api/queue/status')+const status = await response.json()++console.log('队列状态:', status)+// 输出:+// {+// "timestamp": "2024-09-24T10:30:00.000Z",+// "pool": {+// "isActive": true,+// "workerCount": 3,+// "totalWorkers": 3,+// "activeWorkers": 2,+// "totalProcessed": 45,+// "totalErrors": 2,+// "averageSuccessRate": 95.74,+// "workers": [+// {+// "workerId": "worker-1",+// "isProcessing": true,+// "processedCount": 15,+// "errorCount": 1,+// "uptime": 3600,+// "successRate": 93.75+// },+// {+// "workerId": "worker-2",+// "isProcessing": true,+// "processedCount": 18,+// "errorCount": 0,+// "uptime": 3600,+// "successRate": 100+// },+// {+// "workerId": "worker-3",+// "isProcessing": false,+// "processedCount": 12,+// "errorCount": 1,+// "uptime": 3600,+// "successRate": 92.31+// }+// ]+// },+// "queue": {+// "pending": 8,+// "in-stages": 2,+// "completed": 45,+// "failed": 2+// }+// }+```++## 系统特性++### 多消费者并发处理+- **3个工作器**:同时处理不同的任务+- **错开轮询**:工作器使用不同的轮询间隔,减少数据库竞争+- **负载均衡**:自动检测和重启有问题的工作器++### 任务优先级+- 优先级范围:0-9(9为最高优先级)+- 高优先级任务会被优先处理+- 相同优先级按照创建时间排序++### 错误处理和重试+- 自动重试机制:任务失败时会自动重试+- 可配置重试次数:1-5次+- 渐进式退避:重试间隔逐渐增加++### 实时监控+- **工作器状态**:查看每个工作器的处理统计+- **队列统计**:查看待处理、处理中、已完成、失败的任务数量+- **性能指标**:成功率、处理速度、运行时间等++## 在现有代码中集成++### 在照片上传时自动添加任务++```typescript+// 在 server/api/photos/process.post.ts 中+export default defineEventHandler(async (event) => {+ const { storageKey } = await readBody(event)++ // 获取工作器池并添加任务+ const workerPool = (globalThis as any).__workerPool+ if (workerPool) {+ const taskId = await workerPool.addTask(+ { storageKey },+ { priority: 5 } // 用户上传的照片给予中等优先级+ )++ return {+ message: 'Photo processing started',+ taskId+ }+ }+})+```++### 程序内部直接使用++```typescript+import { QueueManager } from '~~/server/services/pipeline-queue'++// 获取特定工作器实例+const worker = QueueManager.getInstance('worker-1')++// 添加任务+const taskId = await worker.addTask({+ storageKey: 'photos/example.jpg'+}, {+ priority: 8,+ maxAttempts: 5+})+```++## 配置说明++工作器池在 `server/plugins/pipeline-queue.ts` 中配置:++```typescript+const workerPoolConfig = {+ workerCount: 3, // 消费者数量+ intervalMs: 2000, // 基础轮询间隔+ intervalOffset: 667, // 错开时间+ enableLoadBalancing: true, // 启用负载均衡+ statsReportInterval: 60000, // 统计报告间隔+}+```\ No newline at end of filediff --git a/server/api/photos/process.post.ts b/server/api/photos/process.post.tsdeleted file mode 100644index 1474d6a4..00000000--- a/server/api/photos/process.post.ts+++ /dev/null@@ -1,109 +0,0 @@-import { execPhotoPipelineAsync } from '~~/server/services/photo/pipeline-async'-import { useStorageProvider } from '~~/server/utils/useStorageProvider'-import { isLivePhotoVideo, processLivePhotoVideo } from '~~/server/services/video/livephoto'--export default eventHandler(async (event) => {- await requireUserSession(event)-- const { storageProvider } = useStorageProvider(event)- const body = await readBody(event)- const { fileKey, fileName, fileSize } = body-- if (!fileKey) {- throw createError({- statusCode: 400,- statusMessage: 'fileKey is required',- })- }-- try {- // 验证文件是否存在于存储中- const fileBuffer = await storageProvider.get(fileKey)- if (!fileBuffer) {- throw createError({- statusCode: 404,- statusMessage: 'File not found in storage',- })- }-- // 创建存储对象信息- const originalObject = {- key: fileKey,- size: fileSize || fileBuffer.length,- lastModified: new Date(),- }-- logger.chrono.info('Starting file processing for', fileKey)-- // 检查是否为 LivePhoto 视频文件- if (fileName && isLivePhotoVideo(fileName, fileSize || fileBuffer.length)) {- logger.chrono.info('LivePhoto video detected:', fileName)-- // 异步处理 LivePhoto 视频- processLivePhotoVideoInBackground(fileKey, fileSize || fileBuffer.length)-- return {- message: 'LivePhoto video processing started',- fileKey,- status: 'processing',- type: 'livephoto-video'- }- }-- // 异步处理照片- processPhotoInBackground(fileKey, originalObject, fileName)-- return {- message: 'Photo processing started',- fileKey,- status: 'processing'- }- } catch (error) {- logger.chrono.error('Failed to start photo processing:', error)- throw createError({- statusCode: 500,- statusMessage: 'Failed to start photo processing',- })- }-})--// 后台处理函数-async function processPhotoInBackground(s3key: string, originalObject: any, originalFileName?: string) {- try {- logger.chrono.info('Starting background photo processing for', s3key)-- const photo = await execPhotoPipelineAsync(s3key, originalObject)-- if (!photo) {- logger.chrono.error('Photo processing failed for', s3key)- return- }-- // 如果有原始文件名,使用它作为标题- if (originalFileName && !photo.title) {- photo.title = originalFileName- }-- await useDB().insert(tables.photos).values(photo).execute()- logger.chrono.success('Background photo processing completed for', photo.title || photo.storageKey)- } catch (error) {- logger.chrono.error('Background photo processing error for', s3key, error)- }-}--// LivePhoto 视频后台处理函数-async function processLivePhotoVideoInBackground(videoKey: string, videoSize: number) {- try {- logger.chrono.info('Starting background LivePhoto video processing for', videoKey)-- const success = await processLivePhotoVideo(videoKey, videoSize)-- if (success) {- logger.chrono.success('LivePhoto video processing completed for', videoKey)- } else {- logger.chrono.warn('LivePhoto video processing failed - no matching photo found for', videoKey)- }- } catch (error) {- logger.chrono.error('LivePhoto video processing error for', videoKey, error)- }-}diff --git a/server/api/queue/add-task.post.ts b/server/api/queue/add-task.post.tsnew file mode 100644index 00000000..a13a9019--- /dev/null+++ b/server/api/queue/add-task.post.ts@@ -0,0 +1,54 @@+import { z } from 'zod'++export default defineEventHandler(async (event) => {+ await requireUserSession(event)++ try {+ const { payload, priority, maxAttempts } = await readValidatedBody(+ event,+ z.object({+ payload: z.object({+ type: z.enum(['photo', 'live-photo-video']).default('photo'),+ storageKey: z.string().nonempty(),+ }),+ priority: z.number().min(0).max(9).optional().default(0),+ maxAttempts: z.number().min(1).max(5).optional().default(3),+ }).parse,+ )++ const workerPool = globalThis.__workerPool++ if (!workerPool) {+ throw createError({+ statusCode: 503,+ statusMessage: 'Worker pool not initialized',+ })+ }++ const taskId = await workerPool.addTask(payload, {+ priority,+ maxAttempts,+ })++ return {+ success: true,+ taskId,+ message: 'Task added to queue successfully',+ payload: {+ storageKey: payload.storageKey,+ priority,+ maxAttempts,+ },+ }+ } catch (error: any) {+ if (error.statusCode) {+ throw error+ }++ throw createError({+ statusCode: 500,+ statusMessage:+ error instanceof Error ? error.message : 'Failed to add task to queue',+ })+ }+})diff --git a/server/api/queue/add-tasks.post.ts b/server/api/queue/add-tasks.post.tsnew file mode 100644index 00000000..ff654b7e--- /dev/null+++ b/server/api/queue/add-tasks.post.ts@@ -0,0 +1,88 @@+import { z } from 'zod'++export default defineEventHandler(async (event) => {+ await requireUserSession(event)++ try {+ const { tasks, defaultPriority, defaultMaxAttempts } =+ await readValidatedBody(+ event,+ z.object({+ tasks: z+ .array(+ z.object({+ payload: z.object({+ type: z.enum(['photo', 'live-photo-video']).default('photo'),+ storageKey: z.string().nonempty(),+ }),+ priority: z.number().min(0).max(9).optional(),+ maxAttempts: z.number().min(1).max(5).optional(),+ }),+ )+ .min(1, 'At least one task is required')+ .max(1000, 'Too many tasks: maximum 1000 tasks per batch'),+ defaultPriority: z.number().min(0).max(9).optional().default(0),+ defaultMaxAttempts: z.number().min(1).max(5).optional().default(3),+ }).parse,+ )++ const workerPool = globalThis.__workerPool++ if (!workerPool) {+ throw createError({+ statusCode: 503,+ statusMessage: 'Worker pool not initialized',+ })+ }++ const results = []+ const errors = []++ for (let i = 0; i < tasks.length; i++) {+ const task = tasks[i]++ try {+ const taskId = await workerPool.addTask(task.payload, {+ priority: task.priority ?? defaultPriority,+ maxAttempts: task.maxAttempts ?? defaultMaxAttempts,+ })++ results.push({+ index: i,+ taskId,+ storageKey: task.payload.storageKey,+ success: true,+ })+ } catch (error) {+ const errorMessage =+ error instanceof Error ? error.message : `Task ${i}: Unknown error`+ errors.push({+ index: i,+ storageKey: task.payload?.storageKey || 'unknown',+ error: errorMessage,+ success: false,+ })+ }+ }++ return {+ success: errors.length === 0,+ totalTasks: tasks.length,+ successCount: results.length,+ errorCount: errors.length,+ results,+ errors: errors.length > 0 ? errors : undefined,+ message: `Processed ${tasks.length} tasks: ${results.length} successful, ${errors.length} failed`,+ }+ } catch (error: any) {+ if (error.statusCode) {+ throw error+ }++ throw createError({+ statusCode: 500,+ statusMessage:+ error instanceof Error ? error.message : 'Failed to add tasks to queue',+ })+ }+})diff --git a/server/api/queue/stats/[taskId].get.ts b/server/api/queue/stats/[taskId].get.tsnew file mode 100644index 00000000..839d5930--- /dev/null+++ b/server/api/queue/stats/[taskId].get.ts@@ -0,0 +1,38 @@+import z from 'zod'++export default defineEventHandler(async (event) => {+ await requireUserSession(event)++ try {+ const { taskId } = await getValidatedRouterParams(+ event,+ z.object({+ taskId: z.string().nonempty(),+ }).parse,+ )++ const workerPool = globalThis.__workerPool+ if (!workerPool) {+ throw createError({+ statusCode: 503,+ statusMessage: 'Worker pool not initialized',+ })+ }++ const taskStats = await workerPool.getTaskStatus(Number(taskId))+ if (!taskStats) {+ throw createError({+ statusCode: 404,+ statusMessage: 'Task not found',+ })+ }++ return taskStats+ } catch (error) {+ throw createError({+ statusCode: 500,+ statusMessage:+ error instanceof Error ? error.message : 'Failed to get queue status',+ })+ }+})diff --git a/server/api/queue/stats/index.ts b/server/api/queue/stats/index.tsnew file mode 100644index 00000000..234c7c87--- /dev/null+++ b/server/api/queue/stats/index.ts@@ -0,0 +1,33 @@+export default defineEventHandler(async (event) => {+ await requireUserSession(event)++ try {+ const workerPool = globalThis.__workerPool++ if (!workerPool) {+ throw createError({+ statusCode: 503,+ statusMessage: 'Worker pool not initialized',+ })+ }++ const poolStats = workerPool.getPoolStats()+ const queueStats = await workerPool.getQueueStats()++ return {+ timestamp: new Date().toISOString(),+ pool: {+ isActive: workerPool.isActive(),+ workerCount: workerPool.getWorkerCount(),+ ...poolStats,+ },+ queue: queueStats,+ }+ } catch (error) {+ throw createError({+ statusCode: 500,+ statusMessage:+ error instanceof Error ? error.message : 'Failed to get queue status',+ })+ }+})diff --git a/server/database/migrations/0004_absent_lockheed.sql b/server/database/migrations/0004_absent_lockheed.sqlnew file mode 100644index 00000000..2fe31f40--- /dev/null+++ b/server/database/migrations/0004_absent_lockheed.sql@@ -0,0 +1,11 @@+CREATE TABLE `pipeline_queue` (+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,+ `payload` text NOT NULL,+ `priority` integer DEFAULT 0 NOT NULL,+ `attempts` integer DEFAULT 0 NOT NULL,+ `max_attempts` integer DEFAULT 3 NOT NULL,+ `status` text DEFAULT 'pending' NOT NULL,+ `status_stage` text,+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,+ `completed_at` integer+);diff --git a/server/database/migrations/0005_light_stranger.sql b/server/database/migrations/0005_light_stranger.sqlnew file mode 100644index 00000000..714fcf79--- /dev/null+++ b/server/database/migrations/0005_light_stranger.sql@@ -0,0 +1 @@+ALTER TABLE `pipeline_queue` ADD `error_message` text;\ No newline at end of filediff --git a/server/database/migrations/0006_flaky_the_enforcers.sql b/server/database/migrations/0006_flaky_the_enforcers.sqlnew file mode 100644index 00000000..c3a173ad--- /dev/null+++ b/server/database/migrations/0006_flaky_the_enforcers.sql@@ -0,0 +1,18 @@+PRAGMA foreign_keys=OFF;--> statement-breakpoint+CREATE TABLE `__new_pipeline_queue` (+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,+ `payload` text DEFAULT '{"type":"photo","storageKey":""}' NOT NULL,+ `priority` integer DEFAULT 0 NOT NULL,+ `attempts` integer DEFAULT 0 NOT NULL,+ `max_attempts` integer DEFAULT 3 NOT NULL,+ `status` text DEFAULT 'pending' NOT NULL,+ `status_stage` text,+ `error_message` text,+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,+ `completed_at` integer+);+--> statement-breakpoint+INSERT INTO `__new_pipeline_queue`("id", "payload", "priority", "attempts", "max_attempts", "status", "status_stage", "error_message", "created_at", "completed_at") SELECT "id", "payload", "priority", "attempts", "max_attempts", "status", "status_stage", "error_message", "created_at", "completed_at" FROM `pipeline_queue`;--> statement-breakpoint+DROP TABLE `pipeline_queue`;--> statement-breakpoint+ALTER TABLE `__new_pipeline_queue` RENAME TO `pipeline_queue`;--> statement-breakpoint+PRAGMA foreign_keys=ON;\ No newline at end of filediff --git a/server/database/migrations/meta/0004_snapshot.json b/server/database/migrations/meta/0004_snapshot.jsonnew file mode 100644index 00000000..26185166--- /dev/null+++ b/server/database/migrations/meta/0004_snapshot.json@@ -0,0 +1,358 @@+{+ "version": "6",+ "dialect": "sqlite",+ "id": "b9e6f5a8-2bca-465d-8578-62b3b35c6af3",+ "prevId": "09eda2ca-cacb-4ef5-a4b9-e17c1326fa19",+ "tables": {+ "photos": {+ "name": "photos",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "title": {+ "name": "title",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "description": {+ "name": "description",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "width": {+ "name": "width",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "height": {+ "name": "height",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "aspect_ratio": {+ "name": "aspect_ratio",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "date_taken": {+ "name": "date_taken",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "storage_key": {+ "name": "storage_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_key": {+ "name": "thumbnail_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "file_size": {+ "name": "file_size",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "last_modified": {+ "name": "last_modified",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "original_url": {+ "name": "original_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_url": {+ "name": "thumbnail_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_hash": {+ "name": "thumbnail_hash",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "tags": {+ "name": "tags",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "exif": {+ "name": "exif",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "latitude": {+ "name": "latitude",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "longitude": {+ "name": "longitude",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "country": {+ "name": "country",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "city": {+ "name": "city",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "location_name": {+ "name": "location_name",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "is_live_photo": {+ "name": "is_live_photo",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "live_photo_video_url": {+ "name": "live_photo_video_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "live_photo_video_key": {+ "name": "live_photo_video_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {+ "photos_id_unique": {+ "name": "photos_id_unique",+ "columns": [+ "id"+ ],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "pipeline_queue": {+ "name": "pipeline_queue",+ "columns": {+ "id": {+ "name": "id",+ "type": "integer",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": true+ },+ "payload": {+ "name": "payload",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "priority": {+ "name": "priority",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "attempts": {+ "name": "attempts",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "max_attempts": {+ "name": "max_attempts",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 3+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'pending'"+ },+ "status_stage": {+ "name": "status_stage",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "(unixepoch())"+ },+ "completed_at": {+ "name": "completed_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {},+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "users": {+ "name": "users",+ "columns": {+ "id": {+ "name": "id",+ "type": "integer",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": true+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "email": {+ "name": "email",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "password": {+ "name": "password",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "avatar": {+ "name": "avatar",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "is_admin": {+ "name": "is_admin",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ }+ },+ "indexes": {+ "users_name_unique": {+ "name": "users_name_unique",+ "columns": [+ "name"+ ],+ "isUnique": true+ },+ "users_email_unique": {+ "name": "users_email_unique",+ "columns": [+ "email"+ ],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ }+ },+ "views": {},+ "enums": {},+ "_meta": {+ "schemas": {},+ "tables": {},+ "columns": {}+ },+ "internal": {+ "indexes": {}+ }+}\ No newline at end of filediff --git a/server/database/migrations/meta/0005_snapshot.json b/server/database/migrations/meta/0005_snapshot.jsonnew file mode 100644index 00000000..732ebb60--- /dev/null+++ b/server/database/migrations/meta/0005_snapshot.json@@ -0,0 +1,365 @@+{+ "version": "6",+ "dialect": "sqlite",+ "id": "f819221e-a365-4562-81d1-ec510aa6de94",+ "prevId": "b9e6f5a8-2bca-465d-8578-62b3b35c6af3",+ "tables": {+ "photos": {+ "name": "photos",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "title": {+ "name": "title",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "description": {+ "name": "description",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "width": {+ "name": "width",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "height": {+ "name": "height",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "aspect_ratio": {+ "name": "aspect_ratio",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "date_taken": {+ "name": "date_taken",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "storage_key": {+ "name": "storage_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_key": {+ "name": "thumbnail_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "file_size": {+ "name": "file_size",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "last_modified": {+ "name": "last_modified",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "original_url": {+ "name": "original_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_url": {+ "name": "thumbnail_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_hash": {+ "name": "thumbnail_hash",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "tags": {+ "name": "tags",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "exif": {+ "name": "exif",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "latitude": {+ "name": "latitude",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "longitude": {+ "name": "longitude",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "country": {+ "name": "country",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "city": {+ "name": "city",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "location_name": {+ "name": "location_name",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "is_live_photo": {+ "name": "is_live_photo",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "live_photo_video_url": {+ "name": "live_photo_video_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "live_photo_video_key": {+ "name": "live_photo_video_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {+ "photos_id_unique": {+ "name": "photos_id_unique",+ "columns": [+ "id"+ ],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "pipeline_queue": {+ "name": "pipeline_queue",+ "columns": {+ "id": {+ "name": "id",+ "type": "integer",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": true+ },+ "payload": {+ "name": "payload",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "priority": {+ "name": "priority",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "attempts": {+ "name": "attempts",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "max_attempts": {+ "name": "max_attempts",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 3+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'pending'"+ },+ "status_stage": {+ "name": "status_stage",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "error_message": {+ "name": "error_message",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "(unixepoch())"+ },+ "completed_at": {+ "name": "completed_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {},+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "users": {+ "name": "users",+ "columns": {+ "id": {+ "name": "id",+ "type": "integer",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": true+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "email": {+ "name": "email",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "password": {+ "name": "password",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "avatar": {+ "name": "avatar",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "is_admin": {+ "name": "is_admin",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ }+ },+ "indexes": {+ "users_name_unique": {+ "name": "users_name_unique",+ "columns": [+ "name"+ ],+ "isUnique": true+ },+ "users_email_unique": {+ "name": "users_email_unique",+ "columns": [+ "email"+ ],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ }+ },+ "views": {},+ "enums": {},+ "_meta": {+ "schemas": {},+ "tables": {},+ "columns": {}+ },+ "internal": {+ "indexes": {}+ }+}\ No newline at end of filediff --git a/server/database/migrations/meta/0006_snapshot.json b/server/database/migrations/meta/0006_snapshot.jsonnew file mode 100644index 00000000..00ac497c--- /dev/null+++ b/server/database/migrations/meta/0006_snapshot.json@@ -0,0 +1,366 @@+{+ "version": "6",+ "dialect": "sqlite",+ "id": "f395c220-c4ec-46f8-a0bf-4fda6e692f7c",+ "prevId": "f819221e-a365-4562-81d1-ec510aa6de94",+ "tables": {+ "photos": {+ "name": "photos",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "title": {+ "name": "title",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "description": {+ "name": "description",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "width": {+ "name": "width",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "height": {+ "name": "height",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "aspect_ratio": {+ "name": "aspect_ratio",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "date_taken": {+ "name": "date_taken",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "storage_key": {+ "name": "storage_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_key": {+ "name": "thumbnail_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "file_size": {+ "name": "file_size",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "last_modified": {+ "name": "last_modified",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "original_url": {+ "name": "original_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_url": {+ "name": "thumbnail_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "thumbnail_hash": {+ "name": "thumbnail_hash",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "tags": {+ "name": "tags",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "exif": {+ "name": "exif",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "latitude": {+ "name": "latitude",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "longitude": {+ "name": "longitude",+ "type": "real",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "country": {+ "name": "country",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "city": {+ "name": "city",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "location_name": {+ "name": "location_name",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "is_live_photo": {+ "name": "is_live_photo",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "live_photo_video_url": {+ "name": "live_photo_video_url",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "live_photo_video_key": {+ "name": "live_photo_video_key",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {+ "photos_id_unique": {+ "name": "photos_id_unique",+ "columns": [+ "id"+ ],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "pipeline_queue": {+ "name": "pipeline_queue",+ "columns": {+ "id": {+ "name": "id",+ "type": "integer",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": true+ },+ "payload": {+ "name": "payload",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'{\"type\":\"photo\",\"storageKey\":\"\"}'"+ },+ "priority": {+ "name": "priority",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "attempts": {+ "name": "attempts",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "max_attempts": {+ "name": "max_attempts",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 3+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'pending'"+ },+ "status_stage": {+ "name": "status_stage",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "error_message": {+ "name": "error_message",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "(unixepoch())"+ },+ "completed_at": {+ "name": "completed_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {},+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "users": {+ "name": "users",+ "columns": {+ "id": {+ "name": "id",+ "type": "integer",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": true+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "email": {+ "name": "email",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "password": {+ "name": "password",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "avatar": {+ "name": "avatar",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "is_admin": {+ "name": "is_admin",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ }+ },+ "indexes": {+ "users_name_unique": {+ "name": "users_name_unique",+ "columns": [+ "name"+ ],+ "isUnique": true+ },+ "users_email_unique": {+ "name": "users_email_unique",+ "columns": [+ "email"+ ],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ }+ },+ "views": {},+ "enums": {},+ "_meta": {+ "schemas": {},+ "tables": {},+ "columns": {}+ },+ "internal": {+ "indexes": {}+ }+}\ No newline at end of filediff --git a/server/database/migrations/meta/_journal.json b/server/database/migrations/meta/_journal.jsonindex 4cc48681..d3b16d87 100644--- a/server/database/migrations/meta/_journal.json+++ b/server/database/migrations/meta/_journal.json@@ -29,6 +29,27 @@"when": 1758511705153,"tag": "0003_nifty_hellcat","breakpoints": true+ },+ {+ "idx": 4,+ "version": "6",+ "when": 1758650003665,+ "tag": "0004_absent_lockheed",+ "breakpoints": true+ },+ {+ "idx": 5,+ "version": "6",+ "when": 1758651965719,+ "tag": "0005_light_stranger",+ "breakpoints": true+ },+ {+ "idx": 6,+ "version": "6",+ "when": 1758723057975,+ "tag": "0006_flaky_the_enforcers",+ "breakpoints": true}]}\ No newline at end of filediff --git a/server/database/schema.ts b/server/database/schema.tsindex 8016fa34..7108f0ce 100644--- a/server/database/schema.ts+++ b/server/database/schema.ts@@ -1,3 +1,4 @@+import { sql } from 'drizzle-orm'import { sqliteTable, text, integer, real } from 'drizzle-orm/sqlite-core'import type { NeededExif } from '~~/shared/types/photo'@@ -39,3 +40,45 @@ export const photos = sqliteTable('photos', {livePhotoVideoUrl: text('live_photo_video_url'),livePhotoVideoKey: text('live_photo_video_key'),})++export const pipelineQueue = sqliteTable('pipeline_queue', {+ id: integer('id').primaryKey({ autoIncrement: true }),+ payload: text('payload', { mode: 'json' })+ .$type<{+ type: 'photo' | 'live-photo-video'+ storageKey: string+ }>()+ .notNull()+ .default({+ type: 'photo',+ storageKey: '',+ }),+ priority: integer('priority').default(0).notNull(),+ attempts: integer('attempts').default(0).notNull(),+ maxAttempts: integer('max_attempts').default(3).notNull(),+ status: text('status', {+ enum: [+ 'pending', // Waiting to be processed+ 'in-stages', // Currently being processed+ 'completed', // Successfully processed+ 'failed', // Processing failed+ ],+ })+ .notNull()+ .default('pending'),+ statusStage: text('status_stage', {+ enum: [+ 'preprocessing',+ 'metadata',+ 'thumbnail',+ 'exif',+ 'reverse-geocoding',+ 'live-photo',+ ],+ }),+ errorMessage: text('error_message'),+ createdAt: integer('created_at', { mode: 'timestamp' })+ .notNull()+ .default(sql`(unixepoch())`),+ completedAt: integer('completed_at', { mode: 'timestamp' }),+})diff --git a/server/plugins/pipeline-queue.ts b/server/plugins/pipeline-queue.tsnew file mode 100644index 00000000..341231f8--- /dev/null+++ b/server/plugins/pipeline-queue.ts@@ -0,0 +1,40 @@+import { WorkerPool } from '../services/pipeline-queue'++export default defineNitroPlugin(async (_nitroApp) => {+ const _logger = logger.dynamic('queue')++ const workerPool = new WorkerPool(+ {+ workerCount: 3,+ intervalMs: 1000,+ intervalOffset: Math.floor(1000 / 3),+ enableLoadBalancing: true,+ statsReportInterval: 60000 * 10,+ },+ _logger,+ )+ await workerPool.start()++ globalThis.__workerPool = workerPool++ const exitHandler = async () => {+ _logger.info('Shutting down worker pool...')+ await workerPool.stop()+ process.exit(0)+ }++ process.on('SIGINT', exitHandler)+ process.on('SIGTERM', exitHandler)++ // 每 5 分钟进行一次负载均衡检查+ setInterval(+ async () => {+ try {+ await workerPool.rebalance()+ } catch (error) {+ _logger.error('Rebalance failed:', error)+ }+ },+ 5 * 60 * 1000,+ )+})diff --git a/server/services/location/geocoding.ts b/server/services/location/geocoding.tsindex 7513d4f9..349b1d11 100644--- a/server/services/location/geocoding.ts+++ b/server/services/location/geocoding.ts@@ -13,6 +13,7 @@ export interface GeocodingProvider {/*** OpenStreetMap Nominatim API 地理编码提供者* 免费的地理编码服务,适合开发和小规模使用+ * TODO: 实现 MapBox 逆编码提供者*/class NominatimGeocodingProvider implements GeocodingProvider {private readonly baseUrl = 'https://nominatim.openstreetmap.org'diff --git a/server/services/photo/pipeline-async.ts b/server/services/photo/pipeline-async.tsdeleted file mode 100644index 40dd32cd..00000000--- a/server/services/photo/pipeline-async.ts+++ /dev/null@@ -1,274 +0,0 @@-import {- preprocessImageWithJpegUpload,- processImageMetadataAndSharp,-} from '../image/processor'-import { generateThumbnailAndHash } from '../image/thumbnail'-import type { Photo } from '~~/server/utils/db'-import { getStorageManager } from '~~/server/plugins/storage'-import { compressUint8Array } from '~~/shared/utils/u8array'-import type { StorageObject } from '../storage'-import { extractExifData, extractPhotoInfo } from '../image/exif'-import {- extractLocationFromGPS,- parseGPSCoordinates,-} from '../location/geocoding'-import { findLivePhotoVideoForImage } from '../video/livephoto'-import { generateSafePhotoId } from '~~/server/utils/file-utils'--/**- * 异步执行照片处理管道 - 不阻塞主线程- */-export const execPhotoPipelineAsync = async (- s3key: string,- storageObject: StorageObject,-): Promise<Photo | null> => {- return new Promise((resolve, reject) => {- // 使用 setImmediate 确保处理在下一个事件循环中执行- setImmediate(async () => {- try {- const result = await processPhotoInternal(s3key, storageObject)- resolve(result)- } catch (error) {- reject(error)- }- })- })-}--/**- * 内部照片处理函数- */-async function processPhotoInternal(- s3key: string,- storageObject: StorageObject,-): Promise<Photo | null> {- const log = logger.image- const storageProvider = getStorageManager().getProvider()- const photoId = generateSafePhotoId(s3key)-- try {- log.info(`Starting async processing for ${s3key}`)-- // 步骤1: 预处理图片并上传JPEG版本- const imageBuffers = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await preprocessImageWithJpegUpload(s3key)- resolve(result)- } catch (error) {- reject(error)- }- })- })-- if (!imageBuffers) {- log.error(`Image preprocessing failed for ${s3key}`)- return null- }-- // 步骤2: 处理图片元数据和Sharp处理- const processedData = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await processImageMetadataAndSharp(- imageBuffers.processed,- s3key,- )- resolve(result)- } catch (error) {- reject(error)- }- })- })-- if (!processedData) {- log.error(`Image processing failed for ${s3key}`)- return null- }-- const { imageBuffer, metadata } = processedData-- // 步骤3: 生成缩略图和哈希值- const { thumbnailBuffer, thumbnailHash } = await new Promise<any>(- (resolve, reject) => {- setImmediate(async () => {- try {- const result = await generateThumbnailAndHash(imageBuffer, logger.image)- resolve(result)- } catch (error) {- reject(error)- }- })- },- )-- // TODO 步骤4: 直方图和影调分析- // const histogramData = await new Promise<any>((resolve, reject) => {- // setImmediate(async () => {- // try {- // const result = await calculateHistogram(processedData.sharpInst)- // resolve(result)- // } catch (error) {- // reject(error)- // }- // })- // })-- // 步骤5: 提取EXIF数据- const exifData = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await extractExifData(- imageBuffer,- imageBuffers.raw,- logger.image,- )- resolve(result)- } catch (error) {- reject(error)- }- })- })-- // 步骤6: 提取照片基本信息- const photoInfo = extractPhotoInfo(s3key, exifData)-- // 步骤7: 处理地理位置信息- let locationInfo = null- if (exifData) {- const { latitude, longitude } = parseGPSCoordinates(exifData)- if (latitude && longitude) {- locationInfo = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await extractLocationFromGPS(latitude, longitude)- resolve(result)- } catch (error) {- reject(error)- }- })- })- }- }-- // 步骤8: 配对 LivePhoto 视频文件- let livePhotoInfo = null- const livePhotoVideo = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await findLivePhotoVideoForImage(s3key)- resolve(result)- } catch (error) {- reject(error)- }- })- })-- if (livePhotoVideo) {- livePhotoInfo = {- isLivePhoto: 1,- livePhotoVideoUrl: storageProvider.getPublicUrl(- livePhotoVideo.videoKey,- ),- livePhotoVideoKey: livePhotoVideo.videoKey,- }- log.info(`LivePhoto video found for ${s3key}: ${livePhotoVideo.videoKey}`)- }-- // 步骤9: 上传缩略图到存储服务- const thumbnailObject = await new Promise<any>((resolve, reject) => {- setImmediate(async () => {- try {- const result = await storageProvider.create(- `thumbnails/${photoId}.webp`,- thumbnailBuffer,- 'image/webp',- )- resolve(result)- } catch (error) {- reject(error)- }- })- })-- // 步骤10: 构建最终的Photo对象- const result: Photo = {- id: photoId,- title: photoInfo.title,- description: photoInfo.description,- dateTaken: photoInfo.dateTaken,- tags: photoInfo.tags,- width: metadata.width,- height: metadata.height,- aspectRatio: metadata.width / metadata.height,- storageKey: s3key,- thumbnailKey: thumbnailObject.key,- fileSize: storageObject.size || null,- lastModified:- storageObject.lastModified?.toISOString() || new Date().toISOString(),- originalUrl: imageBuffers.jpegKey- ? storageProvider.getPublicUrl(imageBuffers.jpegKey) // 使用 JPEG 版本作为 originalUrl- : storageProvider.getPublicUrl(s3key),- thumbnailUrl: storageProvider.getPublicUrl(thumbnailObject.key),- thumbnailHash: thumbnailHash ? compressUint8Array(thumbnailHash) : null,- exif: exifData,- // 地理位置信息- latitude: locationInfo?.latitude || null,- longitude: locationInfo?.longitude || null,- country: locationInfo?.country || null,- city: locationInfo?.city || null,- locationName: locationInfo?.locationName || null,- // LivePhoto 相关字段- isLivePhoto: livePhotoInfo?.isLivePhoto || 0,- livePhotoVideoUrl: livePhotoInfo?.livePhotoVideoUrl || null,- livePhotoVideoKey: livePhotoInfo?.livePhotoVideoKey || null,- }-- log.info(`Async processing completed for ${s3key}`)- return result- } catch (err) {- log.error(`Photo pipeline failed for ${s3key}`, err)- return null- }-}--/**- * 批量异步处理照片- */-export const execBatchPhotoPipelineAsync = async (- photos: Array<{ s3key: string; storageObject: StorageObject }>,- maxConcurrent = 3,-): Promise<Array<{ s3key: string; photo: Photo | null; error?: string }>> => {- const results: Array<{ s3key: string; photo: Photo | null; error?: string }> =- []-- // 分批处理以限制并发数- for (let i = 0; i < photos.length; i += maxConcurrent) {- const batch = photos.slice(i, i + maxConcurrent)- const batchPromises = batch.map(async ({ s3key, storageObject }) => {- try {- const photo = await execPhotoPipelineAsync(s3key, storageObject)- return {- s3key,- photo,- error: photo ? undefined : 'Processing failed',- }- } catch (error) {- return {- s3key,- photo: null,- error: error instanceof Error ? error.message : String(error),- }- }- })-- const batchResults = await Promise.all(batchPromises)- results.push(...batchResults)-- logger.image.info(- `Processed async batch ${Math.floor(i / maxConcurrent) + 1} of ${Math.ceil(photos.length / maxConcurrent)}`,- )- }-- return results-}diff --git a/server/services/pipeline-queue/index.ts b/server/services/pipeline-queue/index.tsnew file mode 100644index 00000000..4391b11e--- /dev/null+++ b/server/services/pipeline-queue/index.ts@@ -0,0 +1,7 @@+export { QueueManager } from './manager'+export { WorkerPool } from './worker-pool'+export type {+ WorkerPoolConfig,+ WorkerStats,+ PoolStats,+} from './worker-pool'diff --git a/server/services/pipeline-queue/manager.ts b/server/services/pipeline-queue/manager.tsnew file mode 100644index 00000000..52a8454e--- /dev/null+++ b/server/services/pipeline-queue/manager.ts@@ -0,0 +1,558 @@+import type { ConsolaInstance } from 'consola'+import path from 'path'+import { asc, desc, eq, sql } from 'drizzle-orm'+import type {+ NewPipelineQueueItem,+ PipelineQueueItem,+ Photo,+} from '~~/server/utils/db'+import {+ preprocessImageWithJpegUpload,+ processImageMetadataAndSharp,+} from '../image/processor'+import { generateThumbnailAndHash } from '../image/thumbnail'+import { extractExifData, extractPhotoInfo } from '../image/exif'+import {+ extractLocationFromGPS,+ parseGPSCoordinates,+} from '../location/geocoding'+import { findLivePhotoVideoForImage } from '../video/livephoto'+import { getStorageManager } from '~~/server/plugins/storage'++export class QueueManager {+ private static instances: Map<string, QueueManager> = new Map()+ private workerId: string+ private logger: ConsolaInstance+ private isProcessing: boolean = false+ private processingInterval: NodeJS.Timeout | null = null+ private processedCount: number = 0+ private errorCount: number = 0+ private startTime: Date++ static getInstance(+ workerId: string = 'default',+ logger?: ConsolaInstance,+ ): QueueManager {+ if (!QueueManager.instances.has(workerId)) {+ QueueManager.instances.set(workerId, new QueueManager(workerId, logger))+ }+ return QueueManager.instances.get(workerId)!+ }++ static getAllInstances(): QueueManager[] {+ return Array.from(QueueManager.instances.values())+ }++ private constructor(workerId: string, _logger?: ConsolaInstance) {+ this.workerId = workerId+ this.logger = _logger+ ? _logger.withTag(`${workerId}`)+ : logger.dynamic(`queue-${workerId}`)+ this.startTime = new Date()+ }++ getWorkerId(): string {+ return this.workerId+ }++ getStats() {+ const uptime = Date.now() - this.startTime.getTime()+ return {+ workerId: this.workerId,+ isProcessing: this.isProcessing,+ processedCount: this.processedCount,+ errorCount: this.errorCount,+ uptime: Math.floor(uptime / 1000), // seconds+ successRate:+ this.processedCount > 0+ ? (this.processedCount / (this.processedCount + this.errorCount)) *+ 100+ : 0,+ }+ }++ /**+ * 插入新任务到队列+ * @param payload 任务负荷+ * @param options 可选的任务设置+ * @returns 新创建任务的 ID+ */+ async addTask(+ payload: any,+ options?: Partial<NewPipelineQueueItem>,+ ): Promise<number> {+ const db = useDB()+ const result = db+ .insert(tables.pipelineQueue)+ .values({+ payload,+ ...options,+ })+ .returning({ id: tables.pipelineQueue.id })+ .get()+ return result.id+ }++ /**+ * 获取任务状态+ * @param taskId 任务ID+ * @returns 任务状态信息+ */+ async getTaskStatus(taskId: number) {+ const db = useDB()+ const task = await db+ .select()+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.id, taskId))+ .get()+ return task+ }++ /**+ * 获取并锁定下一个待处理任务+ * @returns 下一个待处理任务+ */+ async getNextTask(): Promise<PipelineQueueItem | null> {+ const db = useDB()++ // 使用同步事务防止竞态条件+ const task = db.transaction((tx) => {+ const highestPriorityPendingTask = tx+ .select()+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.status, 'pending'))+ // 优先处理高优先级和较早创建的任务+ .orderBy(+ desc(tables.pipelineQueue.priority),+ asc(tables.pipelineQueue.createdAt),+ )+ .limit(1)+ .get()++ if (!highestPriorityPendingTask) return null++ const task = highestPriorityPendingTask+ tx.update(tables.pipelineQueue)+ .set({ status: 'in-stages' })+ .where(eq(tables.pipelineQueue.id, task.id))+ .run()++ return { ...task, status: 'in-stages' as const }+ })++ return task+ }++ /**+ * 更新任务阶段+ * @param taskId 任务ID+ * @param stage 新的任务阶段+ */+ async updateTaskStage(+ taskId: number,+ stage: PipelineQueueItem['statusStage'],+ ): Promise<void> {+ const db = useDB()+ await db+ .update(tables.pipelineQueue)+ .set({ statusStage: stage })+ .where(eq(tables.pipelineQueue.id, taskId))+ }++ /**+ * 标记任务为已完成+ * @param taskId 任务ID+ */+ async markTaskCompleted(taskId: number): Promise<void> {+ const db = useDB()+ await db+ .update(tables.pipelineQueue)+ .set({+ status: 'completed',+ completedAt: sql`(unixepoch())`,+ })+ .where(eq(tables.pipelineQueue.id, taskId))+ }++ /**+ * 标记任务为失败+ * @param taskId 任务ID+ * @param errorMessage 错误信息+ */+ async markTaskFailed(taskId: number, errorMessage?: string): Promise<void> {+ const db = useDB()+ const task = await db+ .select()+ .from(tables.pipelineQueue)+ .where(eq(tables.pipelineQueue.id, taskId))+ .get()++ if (!task) return++ const newAttempts = task.attempts + 1+ const shouldRetry = newAttempts < task.maxAttempts++ await db+ .update(tables.pipelineQueue)+ .set({+ status: shouldRetry ? 'pending' : 'failed',+ attempts: newAttempts,+ errorMessage: errorMessage || 'Unknown error',+ })+ .where(eq(tables.pipelineQueue.id, taskId))+ }++ /** 任务处理器 */+ private processors = (() => {+ return {+ photo: async (task: PipelineQueueItem) => {+ const { id: taskId, payload } = task+ const { storageKey } = payload+ const storageProvider = getStorageManager().getProvider()+ const photoId = generateSafePhotoId(storageKey)++ try {+ this.logger.info(`Start processing task ${taskId}: ${storageKey}`)++ const storageObject = await storageProvider.getFileMeta(storageKey)++ if (!storageObject) {+ throw new Error(`Storage object not found`)+ }++ // STEP 1: 预处理 - 转换 HEIC 到 JPEG 并上传+ await this.updateTaskStage(taskId, 'preprocessing')+ this.logger.info(`[${taskId}:in-stage] preprocessing`)+ const imageBuffers = await preprocessImageWithJpegUpload(storageKey)+ if (!imageBuffers) {+ throw new Error('Preprocessing failed')+ }++ // STEP 2: 元数据处理 - 使用 Sharp 处理图片元数据+ await this.updateTaskStage(taskId, 'metadata')+ this.logger.info(`[${taskId}:in-stage] metadata extraction`)+ const processedData = await processImageMetadataAndSharp(+ imageBuffers.processed,+ storageKey,+ )+ if (!processedData) {+ throw new Error('Metadata processing failed')+ }++ const { imageBuffer, metadata } = processedData++ // STEP 3: 生成缩略图+ await this.updateTaskStage(taskId, 'thumbnail')+ this.logger.info(`[${taskId}:in-stage] thumbnail generation`)+ const { thumbnailBuffer, thumbnailHash } =+ await generateThumbnailAndHash(imageBuffer, this.logger)++ // 上传缩略图到存储服务+ const thumbnailObject = await new Promise<any>((resolve, reject) => {+ setImmediate(async () => {+ try {+ const result = await storageProvider.create(+ `thumbnails/${photoId}.webp`,+ thumbnailBuffer,+ 'image/webp',+ )+ resolve(result)+ } catch (error) {+ reject(error)+ }+ })+ })++ // STEP 4: 提取 EXIF 数据+ await this.updateTaskStage(taskId, 'exif')+ this.logger.info(`[${taskId}:in-stage] exif extraction`)+ const exifData = await extractExifData(+ imageBuffer,+ imageBuffers.raw,+ this.logger,+ )++ // 提取照片基本信息+ const photoInfo = extractPhotoInfo(storageKey, exifData)++ // STEP 5: 地理位置反向解析+ // 这里逆编码失败不报错,宽容处理+ await this.updateTaskStage(taskId, 'reverse-geocoding')+ this.logger.info(`[${taskId}:in-stage] reverse geocoding`)++ let locationInfo = null+ if (exifData) {+ const { latitude, longitude } = parseGPSCoordinates(exifData)+ if (latitude && longitude) {+ locationInfo = await extractLocationFromGPS(latitude, longitude)+ }+ }++ // STEP 6: LivePhoto 视频配对+ await this.updateTaskStage(taskId, 'live-photo')+ this.logger.info(`[${taskId}:in-stage] live photo detection`)+ let livePhotoInfo = null+ const livePhotoVideo = await findLivePhotoVideoForImage(storageKey)+ if (livePhotoVideo) {+ livePhotoInfo = {+ isLivePhoto: 1,+ livePhotoVideoUrl: storageProvider.getPublicUrl(+ livePhotoVideo.videoKey,+ ),+ livePhotoVideoKey: livePhotoVideo.videoKey,+ }+ this.logger.info(+ `[${taskId}:in-stage] found LivePhoto video: ${livePhotoVideo.videoKey}`,+ )+ }++ // 构建最终的 Photo 对象+ const result: Photo = {+ id: photoId,+ title: photoInfo.title,+ description: photoInfo.description,+ dateTaken: photoInfo.dateTaken,+ tags: photoInfo.tags,+ width: metadata.width,+ height: metadata.height,+ aspectRatio: metadata.width / metadata.height,+ storageKey: storageKey,+ thumbnailKey: thumbnailObject.key,+ fileSize: storageObject.size || null,+ lastModified:+ storageObject.lastModified?.toISOString() ||+ new Date().toISOString(),+ originalUrl: imageBuffers.jpegKey+ ? storageProvider.getPublicUrl(imageBuffers.jpegKey) // 使用 JPEG 版本作为 originalUrl+ : storageProvider.getPublicUrl(storageKey),+ thumbnailUrl: storageProvider.getPublicUrl(thumbnailObject.key),+ thumbnailHash: thumbnailHash+ ? compressUint8Array(thumbnailHash)+ : null,+ exif: exifData,+ // 地理位置信息+ latitude: locationInfo?.latitude || null,+ longitude: locationInfo?.longitude || null,+ country: locationInfo?.country || null,+ city: locationInfo?.city || null,+ locationName: locationInfo?.locationName || null,+ // LivePhoto 相关字段+ isLivePhoto: livePhotoInfo?.isLivePhoto || 0,+ livePhotoVideoUrl: livePhotoInfo?.livePhotoVideoUrl || null,+ livePhotoVideoKey: livePhotoInfo?.livePhotoVideoKey || null,+ }++ const db = useDB()+ await db.insert(tables.photos).values(result).onConflictDoUpdate({+ target: tables.photos.id,+ set: result,+ })++ this.logger.success(`Task ${taskId} processed successfully`)+ return result+ } catch (error) {+ this.logger.error(`Task ${taskId} processing failed`, error)+ throw error+ }+ },+ livePhotoDetect: async (task: PipelineQueueItem) => {+ const db = useDB()+ const storageProvider = getStorageManager().getProvider()++ const { id: taskId, payload } = task+ const { storageKey: videoKey } = payload++ try {+ this.logger.info(+ `Start processing LivePhoto detection task ${taskId}: ${videoKey}`,+ )++ const storageObject = await storageProvider.getFileMeta(videoKey)++ if (!storageObject) {+ throw new Error(`Storage object not found`)+ }++ // 寻找是否有同名的照片文件+ const videoDir = path.dirname(videoKey)+ const videoBaseName = path.basename(videoKey, path.extname(videoKey))++ const possiblePhotoKeys = [+ path.join(videoDir, `${videoBaseName}.HEIC`).replace(/\\/g, '/'),+ path.join(videoDir, `${videoBaseName}.heic`).replace(/\\/g, '/'),+ path.join(videoDir, `${videoBaseName}.JPG`).replace(/\\/g, '/'),+ path.join(videoDir, `${videoBaseName}.jpg`).replace(/\\/g, '/'),+ path.join(videoDir, `${videoBaseName}.JPEG`).replace(/\\/g, '/'),+ path.join(videoDir, `${videoBaseName}.jpeg`).replace(/\\/g, '/'),+ ]++ let matchedPhoto: Photo | null = null+ for (const photoKey of possiblePhotoKeys) {+ const photos = await db+ .select()+ .from(tables.photos)+ .where(eq(tables.photos.storageKey, photoKey))+ .limit(1)++ if (photos.length > 0) {+ matchedPhoto = photos[0]+ this.logger.info(+ `Found matching photo for LivePhoto video: ${photoKey}`,+ )+ break+ }+ }++ if (!matchedPhoto) {+ this.logger.warn(+ `No matching photo found for LivePhoto video: ${videoKey}`,+ )+ return null+ }++ const livePhotoVideoUrl = storageProvider.getPublicUrl(videoKey)+ await db+ .update(tables.photos)+ .set({+ isLivePhoto: 1,+ livePhotoVideoUrl,+ livePhotoVideoKey: videoKey,+ })+ .where(eq(tables.photos.id, matchedPhoto.id))++ this.logger.success(+ `LivePhoto detection task ${taskId} processed successfully, updated photo ${matchedPhoto.id}`,+ )+ } catch (error) {+ this.logger.error(+ `LivePhoto detection task ${taskId} processing failed`,+ error,+ )+ throw error+ }+ },+ }+ })()++ /**+ * 处理下一个待处理任务+ */+ private async processNextTask(): Promise<void> {+ if (this.isProcessing) {+ this.logger.debug('Task is already processing, skipping this poll')+ return+ }++ this.isProcessing = true++ try {+ const task = await this.getNextTask()+ if (!task) {+ this.logger.debug('No tasks to process at the moment')+ return+ }++ try {+ const { type } = task.payload++ switch (type) {+ case 'live-photo-video':+ await this.processors.livePhotoDetect(task)+ break+ case 'photo':+ await this.processors.photo(task)+ break+ default:+ throw new Error(`Unknown task type: ${type}`)+ }++ await this.markTaskCompleted(task.id)+ this.processedCount+++ this.logger.success(+ `[${this.workerId}] Task ${task.id} processed successfully (Total: ${this.processedCount})`,+ )++ // const result = await this.processTask(task)+ // if (result) {+ // await this.markTaskCompleted(task.id)+ // this.processedCount+++ // this.logger.success(+ // `[${this.workerId}] Task ${task.id} processed successfully (Total: ${this.processedCount})`,+ // )+ // } else {+ // await this.markTaskFailed(task.id, 'Processing result is empty')+ // this.errorCount+++ // }+ } catch (error) {+ const errorMessage =+ error instanceof Error ? error.message : String(error)+ await this.markTaskFailed(task.id, errorMessage)+ this.errorCount+++ this.logger.error(+ `[${this.workerId}] Task ${task.id} processing failed (Error: ${this.errorCount}):`,+ errorMessage,+ )+ }+ } catch (error) {+ this.logger.error('Error occurred while fetching the next task:', error)+ } finally {+ this.isProcessing = false+ }+ }++ /**+ * 开始处理队列+ * @param intervalMs 轮询间隔+ */+ startProcessing(intervalMs: number = 3000): void {+ if (this.processingInterval) return++ this.processingInterval = setInterval(() => {+ this.processNextTask().catch((error) => {+ this.logger.error('Error occurred while processing the queue:', error)+ })+ }, intervalMs)++ this.logger.success(+ `Queue processing started with interval: ${intervalMs}ms`,+ )++ this.processNextTask().catch((error) => {+ this.logger.error('Error occurred while processing the queue:', error)+ })+ }++ /**+ * 停止处理队列+ */+ stopProcessing(): void {+ if (this.processingInterval) {+ clearInterval(this.processingInterval)+ this.processingInterval = null+ this.logger.warn('Queue processing stopped')+ }+ }++ /**+ * 获取队列统计信息+ * @returns 队列统计信息+ */+ async getQueueStats() {+ const db = useDB()+ const stats = await db+ .select({+ status: tables.pipelineQueue.status,+ count: sql<number>`COUNT(*)`,+ })+ .from(tables.pipelineQueue)+ .groupBy(tables.pipelineQueue.status)++ return stats.reduce(+ (acc, stat) => {+ acc[stat.status] = stat.count+ return acc+ },+ {} as Record<string, number>,+ )+ }+}diff --git a/server/services/pipeline-queue/worker-pool.ts b/server/services/pipeline-queue/worker-pool.tsnew file mode 100644index 00000000..00319645--- /dev/null+++ b/server/services/pipeline-queue/worker-pool.ts@@ -0,0 +1,288 @@+import type { ConsolaInstance } from 'consola'+import { consola } from 'consola'+import { QueueManager } from './manager'++export interface WorkerPoolConfig {+ /** 工作器数量 */+ workerCount: number+ /** 轮询间隔(毫秒) */+ intervalMs: number+ /** 工作器间隔偏移(毫秒),用于错开轮询时间 */+ intervalOffset: number+ /** 是否启用负载均衡 */+ enableLoadBalancing: boolean+ /** 统计报告间隔(毫秒) */+ statsReportInterval: number+}++export interface WorkerStats {+ workerId: string+ isProcessing: boolean+ processedCount: number+ errorCount: number+ uptime: number+ successRate: number+}++export interface PoolStats {+ totalWorkers: number+ activeWorkers: number+ totalProcessed: number+ totalErrors: number+ averageSuccessRate: number+ workers: WorkerStats[]+}++/**+ * 工作器池+ */+export class WorkerPool {+ private workers: QueueManager[] = []+ private logger: ConsolaInstance+ private config: WorkerPoolConfig+ private statsInterval: NodeJS.Timeout | null = null+ private isRunning: boolean = false++ constructor(config: Partial<WorkerPoolConfig>, logger?: ConsolaInstance) {+ this.config = {+ workerCount: 3,+ intervalMs: 2000,+ intervalOffset: 500,+ enableLoadBalancing: true,+ statsReportInterval: 30000, // 30秒+ ...config,+ }+ this.logger = logger || consola.withTag('worker-pool')+ }++ /**+ * 启动工作器池+ */+ async start(): Promise<void> {+ if (this.isRunning) {+ this.logger.warn('WorkerPool is already running')+ return+ }++ this.logger.info(+ `Starting WorkerPool with ${this.config.workerCount} workers`,+ )++ // 创建工作器+ for (let i = 1; i <= this.config.workerCount; i++) {+ const workerId = `worker-${i}`+ const worker = QueueManager.getInstance(workerId, this.logger)++ // 错开时间以减少数据库竞争+ const workerInterval =+ this.config.intervalMs + (i - 1) * this.config.intervalOffset++ this.workers.push(worker)+ worker.startProcessing(workerInterval)+ }++ this.isRunning = true++ // 启动统计报告+ if (this.config.statsReportInterval > 0) {+ this.startStatsReporting()+ }++ this.logger.success(+ `WorkerPool started successfully with ${this.workers.length} workers`,+ )+ }++ /**+ * 停止工作器池+ */+ async stop(): Promise<void> {+ if (!this.isRunning) {+ this.logger.warn('WorkerPool is not running')+ return+ }++ this.logger.info('Stopping WorkerPool...')++ // 停止统计报告+ if (this.statsInterval) {+ clearInterval(this.statsInterval)+ this.statsInterval = null+ }++ // 停止所有工作器+ const stopPromises = this.workers.map(async (worker, _index) => {+ try {+ worker.stopProcessing()+ this.logger.info(`worker [${worker.getWorkerId()}] stopped`)+ } catch (error) {+ this.logger.error(+ `Error stopping worker [${worker.getWorkerId()}]:`,+ error,+ )+ }+ })++ await Promise.allSettled(stopPromises)++ this.workers = []+ this.isRunning = false++ this.logger.success('WorkerPool stopped successfully')+ }++ /**+ * 获取池统计信息+ */+ getPoolStats(): PoolStats {+ const workerStats = this.workers.map((worker) => worker.getStats())++ const totalProcessed = workerStats.reduce(+ (sum, stats) => sum + stats.processedCount,+ 0,+ )+ const totalErrors = workerStats.reduce(+ (sum, stats) => sum + stats.errorCount,+ 0,+ )+ const activeWorkers = workerStats.filter(+ (stats) => stats.isProcessing,+ ).length++ // 只计算已执行过操作的worker的平均成功率+ const workersWithActivity = workerStats.filter(+ (stats) => stats.processedCount > 0,+ )+ const averageSuccessRate =+ workersWithActivity.length > 0+ ? workersWithActivity.reduce(+ (sum, stats) => sum + stats.successRate,+ 0,+ ) / workersWithActivity.length+ : 0++ return {+ totalWorkers: this.workers.length,+ activeWorkers,+ totalProcessed,+ totalErrors,+ averageSuccessRate: Math.round(averageSuccessRate * 100) / 100,+ workers: workerStats,+ }+ }++ /**+ * 启动控制台统计报告+ */+ private startStatsReporting(): void {+ this.statsInterval = setInterval(() => {+ const stats = this.getPoolStats()++ this.logger.info('=== WorkerPool Stats ===')+ this.logger.info(+ `Total Workers: ${stats.totalWorkers}, Active: ${stats.activeWorkers}`,+ )+ this.logger.info(+ `Total Processed: ${stats.totalProcessed}, Total Errors: ${stats.totalErrors}`,+ )+ this.logger.info(`Average Success Rate: ${stats.averageSuccessRate}%`)++ stats.workers.forEach((worker) => {+ const status = worker.isProcessing ? 'Processing' : 'Idle'+ this.logger.info(+ ` ${worker.workerId}: ${status} | Processed: ${worker.processedCount} | Errors: ${worker.errorCount} | Success Rate: ${worker.successRate.toFixed(1)}% | Uptime: ${worker.uptime}s`,+ )+ })+ }, this.config.statsReportInterval)+ }++ /**+ * 重新平衡工作器负载+ */+ async rebalance(): Promise<void> {+ if (!this.config.enableLoadBalancing) {+ return+ }++ const stats = this.getPoolStats()++ // 如果某个工作器错误率过高,重启它+ const problematicWorkers = stats.workers.filter(+ (worker) => worker.errorCount > 10 && worker.successRate < 50,+ )++ for (const workerStat of problematicWorkers) {+ this.logger.warn(+ `Worker ${workerStat.workerId} has high error rate, preparing to restart`,+ )++ const worker = this.workers.find(+ (w) => w.getWorkerId() === workerStat.workerId,+ )+ if (worker) {+ worker.stopProcessing()++ setTimeout(() => {+ worker.startProcessing(this.config.intervalMs)+ this.logger.info(`Worker ${workerStat.workerId} has been restarted`)+ }, 5000)+ }+ }+ }++ /**+ * 获取队列统计信息(从任意一个工作器获取)+ */+ async getQueueStats() {+ if (this.workers.length === 0) {+ return {}+ }++ return await this.workers[0].getQueueStats()+ }++ /**+ * 检查池是否正在运行+ */+ isActive(): boolean {+ return this.isRunning+ }++ /**+ * 获取工作器数量+ */+ getWorkerCount(): number {+ return this.workers.length+ }++ /**+ * 获取第一个工作器(用于添加任务)+ */+ getFirstWorker(): QueueManager | null {+ return this.workers.length > 0 ? this.workers[0] : null+ }++ /**+ * 添加任务到队列(通过任意一个工作器)+ */+ async addTask(payload: any, options?: any): Promise<number> {+ const worker = this.getFirstWorker()+ if (!worker) {+ throw new Error('No workers available')+ }+ return await worker.addTask(payload, options)+ }++ /**+ * 获取任务状态+ * @param taskId 任务ID+ * @returns 任务状态信息+ */+ async getTaskStatus(taskId: number) {+ const worker = this.getFirstWorker()+ if (!worker) {+ throw new Error('No workers available')+ }+ return await worker.getTaskStatus(taskId)+ }+}diff --git a/server/services/storage/interfaces.ts b/server/services/storage/interfaces.tsindex 2faa342d..bfb1d962 100644--- a/server/services/storage/interfaces.ts+++ b/server/services/storage/interfaces.ts@@ -61,6 +61,7 @@ export interface StorageProvider {expiresIn?: number,options?: UploadOptions,): Promise<string>+ getFileMeta(key: string): Promise<StorageObject | null>listAll(): Promise<StorageObject[]>listImages(): Promise<StorageObject[]>}diff --git a/server/services/storage/providers/s3.ts b/server/services/storage/providers/s3.tsindex 9cfb738b..d6ae2e61 100644--- a/server/services/storage/providers/s3.ts+++ b/server/services/storage/providers/s3.ts@@ -1,12 +1,10 @@-import type {- _Object,- S3ClientConfig} from '@aws-sdk/client-s3';+import type { _Object, S3ClientConfig } from '@aws-sdk/client-s3'import {DeleteObjectCommand,GetObjectCommand,ListObjectsCommand,PutObjectCommand,- S3Client+ S3Client,} from '@aws-sdk/client-s3'import { getSignedUrl } from '@aws-sdk/s3-request-presigner'import type {@@ -182,6 +180,34 @@ export class S3StorageProvider implements StorageProvider {return url}+ async getFileMeta(key: string): Promise<StorageObject | null> {+ try {+ const cmd = new GetObjectCommand({+ Bucket: this.config.bucket,+ Key: key,+ })++ const resp = await this.client.send(cmd)++ if (!resp.ETag) {+ return null+ }++ return {+ key,+ size: resp.ContentLength || 0,+ lastModified: resp.LastModified,+ etag: resp.ETag,+ }+ } catch (error) {+ if ((error as any).$metadata?.httpStatusCode === 404) {+ return null+ }+ this.logger?.error(`Failed to get metadata for key: ${key}`, error)+ throw error+ }+ }+async listAll(): Promise<StorageObject[]> {const cmd = new ListObjectsCommand({Bucket: this.config.bucket,diff --git a/server/utils/db.ts b/server/utils/db.tsindex f7151e49..1331f160 100644--- a/server/utils/db.ts+++ b/server/utils/db.ts@@ -15,3 +15,6 @@ export function useDB() {export type User = typeof schema.users.$inferSelectexport type Photo = typeof schema.photos.$inferSelect++export type PipelineQueueItem = typeof schema.pipelineQueue.$inferSelect+export type NewPipelineQueueItem = typeof schema.pipelineQueue.$inferInsertdiff --git a/shared/types/global.d.ts b/shared/types/global.d.tsnew file mode 100644index 00000000..08489f94--- /dev/null+++ b/shared/types/global.d.ts@@ -0,0 +1,7 @@+import type { WorkerPool } from '../../server/services/pipeline-queue'++declare global {+ var __workerPool: WorkerPool | undefined+}++export {}\ No newline at end of file