Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
feat: 增强图片上传功能,支持MOV视频文件类型;优化数据库连接,增加worker数量和减少轮询间隔以提升性能
cuijianzhuang8562bbf7ec5a
diff --git a/app/pages/dashboard/photos.vue b/app/pages/dashboard/photos.vueindex da264d80..ea287e4a 100644--- a/app/pages/dashboard/photos.vue+++ b/app/pages/dashboard/photos.vue@@ -120,14 +120,16 @@ const uploadImage = async (file: File) => {uploadingFiles.value = new Map(uploadingFiles.value)try {+ // 检查是否为MOV视频文件(通过MIME类型或文件扩展名)+ const isMovFile = file.type === 'video/quicktime' ||+ file.type === 'video/mp4' ||+ file.name.toLowerCase().endsWith('.mov')+const resp = await $fetch('/api/queue/add-task', {method: 'POST',body: {payload: {- type:- file.type === 'video/quicktime'- ? 'live-photo-video'- : 'photo',+ type: isMovFile ? 'live-photo-video' : 'photo',storageKey: signedUrlResponse.fileKey,},priority: 0,@@ -569,24 +571,98 @@ const handleUpload = async () => {return}+ // 显示批量上传进度+ const uploadToast = toast.add({+ title: '开始批量上传',+ description: `正在上传 ${fileList.length} 个文件...`,+ color: 'info',+ })++ let successCount = 0+ let errorCount = 0+ const errors: string[] = []++ // 先验证所有文件+ const validFiles: File[] = []for (const file of fileList) {- // 验证文件const validation = validateFile(file)if (!validation.valid) {- toast.add({- title: '文件验证失败',- description: validation.error,- color: 'error',- })- continue+ errors.push(`${file.name}: ${validation.error}`)+ errorCount+++ } else {+ validFiles.push(file)}+ }- try {- await uploadImage(file)- } catch (error: any) {- // 错误已经在 uploadImage 函数内部处理了- console.error('上传错误:', error)- }+ if (validFiles.length === 0) {+ toast.update(uploadToast.id, {+ title: '批量上传失败',+ description: '所有文件验证失败',+ color: 'error',+ })+ selectedFiles.value = []+ return+ }++ // 并发上传,但限制并发数量以避免资源竞争+ const CONCURRENT_LIMIT = 3 // 限制同时上传的文件数量+ const uploadPromises: Promise<void>[] = []++ for (let i = 0; i < validFiles.length; i += CONCURRENT_LIMIT) {+ const batch = validFiles.slice(i, i + CONCURRENT_LIMIT)++ const batchPromises = batch.map(async (file) => {+ try {+ await uploadImage(file)+ successCount+++ } catch (error: any) {+ errorCount+++ errors.push(`${file.name}: ${error.message || '上传失败'}`)+ console.error('上传错误:', error)+ }+ })++ uploadPromises.push(...batchPromises)++ // 等待当前批次完成再处理下一批次+ await Promise.allSettled(batchPromises)++ // 更新进度+ const processed = Math.min(i + CONCURRENT_LIMIT, validFiles.length)+ toast.update(uploadToast.id, {+ title: '批量上传进行中',+ description: `已处理 ${processed}/${validFiles.length} 个文件`,+ color: 'info',+ })+ }++ // 等待所有上传完成+ await Promise.allSettled(uploadPromises)++ // 显示最终结果+ if (errorCount === 0) {+ toast.update(uploadToast.id, {+ title: '批量上传成功',+ description: `成功上传 ${successCount} 个文件`,+ color: 'success',+ })+ } else if (successCount === 0) {+ toast.update(uploadToast.id, {+ title: '批量上传失败',+ description: `所有 ${errorCount} 个文件上传失败`,+ color: 'error',+ })+ } else {+ toast.update(uploadToast.id, {+ title: '批量上传部分成功',+ description: `成功: ${successCount}, 失败: ${errorCount}`,+ color: 'warning',+ })+ }++ // 如果有错误,显示详细错误信息+ if (errors.length > 0) {+ console.error('批量上传错误详情:', errors)}// 清空选中的文件diff --git a/server/services/image/exif.ts b/server/services/image/exif.tsindex c7d46912..7d573b8f 100644--- a/server/services/image/exif.ts+++ b/server/services/image/exif.ts@@ -274,46 +274,84 @@ export const extractExifData = async (rawImageBuffer?: Buffer,logger?: Logger[keyof Logger],): Promise<NeededExif | null> => {- try {- let metadata = await sharp(imageBuffer).metadata()+ const maxRetries = 3+ let lastError: Error | null = null- if (!metadata.exif && rawImageBuffer) {- try {- metadata = await sharp(rawImageBuffer).metadata()- } catch (err) {- logger?.warn('Error extracting EXIF data from raw image buffer:', err)+ for (let attempt = 1; attempt <= maxRetries; attempt++) {+ let tempImagePath: string | null = null++ try {+ logger?.info(`Extracting EXIF data (attempt ${attempt}/${maxRetries})...`)++ let metadata = await Promise.race([+ sharp(imageBuffer).metadata(),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('Metadata extraction timeout')), 5000)+ )+ ])++ if (!metadata.exif && rawImageBuffer) {+ try {+ metadata = await Promise.race([+ sharp(rawImageBuffer).metadata(),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('Raw metadata extraction timeout')), 5000)+ )+ ])+ } catch (err) {+ logger?.warn('Error extracting EXIF data from raw image buffer:', err)+ }}- }-- if (!metadata.exif) {- logger?.warn('No EXIF data found in image metadata')- return null- }-- logger?.info('Extracting EXIF data using exiftool...')- const tempDir = path.resolve(process.cwd(), 'data/.exif_workdir')- await mkdir(tempDir, { recursive: true })- const tempImagePath = path.resolve(tempDir, `${crypto.randomUUID()}.jpg`)+ if (!metadata.exif) {+ logger?.warn('No EXIF data found in image metadata')+ return null+ }- await writeFile(tempImagePath, rawImageBuffer || imageBuffer)- const exifData = await exiftool.read(tempImagePath)- const result = processExifData(exifData, metadata)+ logger?.info('Extracting EXIF data using exiftool...')++ const tempDir = path.resolve(process.cwd(), 'data/.exif_workdir')+ await mkdir(tempDir, { recursive: true })+ tempImagePath = path.resolve(tempDir, `${crypto.randomUUID()}.jpg`)++ await writeFile(tempImagePath, rawImageBuffer || imageBuffer)++ const exifData = await Promise.race([+ exiftool.read(tempImagePath),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('EXIF tool timeout')), 10000)+ )+ ])++ const result = processExifData(exifData, metadata)++ // 记录颜色空间信息的来源+ if (result.ColorSpace) {+ logger?.success(`Inferred ColorSpace: ${result.ColorSpace}`)+ } else {+ logger?.info('ColorSpace could not be determined')+ }- // 记录颜色空间信息的来源- if (result.ColorSpace) {- logger?.success(`Inferred ColorSpace: ${result.ColorSpace}`)- } else {- logger?.info('ColorSpace could not be determined')+ // 清理临时文件+ await unlink(tempImagePath).catch(noop)+ return result+ } catch (err) {+ lastError = err as Error+ logger?.warn(`EXIF extraction attempt ${attempt} failed:`, err)++ // 清理临时文件+ if (tempImagePath) {+ await unlink(tempImagePath).catch(noop)+ }++ if (attempt < maxRetries) {+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt))+ }}-- // 清理临时文件- await unlink(tempImagePath).catch(noop)- return result- } catch (err) {- logger?.error('EXIF extraction failed:', err)- return null}++ logger?.error('All EXIF extraction attempts failed')+ return null}export const extractPhotoInfo = (diff --git a/server/services/image/processor.ts b/server/services/image/processor.tsindex da0b75b4..1fcb54fa 100644--- a/server/services/image/processor.ts+++ b/server/services/image/processor.ts@@ -25,48 +25,95 @@ const isBitmap = (buffer: Buffer) => {const getMetadataWithSharp = async (sharpInst: sharp.Sharp,): Promise<ImageMeta | null> => {- try {- const metadata = await sharpInst.metadata()+ const maxRetries = 3+ let lastError: Error | null = null- if (!metadata.height || !metadata.width || !metadata.format) {- return null- }+ for (let attempt = 1; attempt <= maxRetries; attempt++) {+ try {+ logger.image.info(`Extracting metadata (attempt ${attempt}/${maxRetries})...`)++ const metadata = await Promise.race([+ sharpInst.metadata(),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('Metadata extraction timeout')), 10000)+ )+ ])++ if (!metadata.height || !metadata.width || !metadata.format) {+ logger.image.warn('Incomplete metadata received:', {+ hasHeight: !!metadata.height,+ hasWidth: !!metadata.width,+ hasFormat: !!metadata.format+ })+ return null+ }- const { orientation } = metadata- let { width, height } = metadata+ const { orientation } = metadata+ let { width, height } = metadata- if (orientation && [5, 6, 7, 8].includes(orientation)) {- ;[width, height] = [height, width]- }+ if (orientation && [5, 6, 7, 8].includes(orientation)) {+ ;[width, height] = [height, width]+ }- return {- width: width,- height: height,- format: metadata.format,+ logger.image.info(`Successfully extracted metadata: ${width}x${height} ${metadata.format}`)+ return {+ width: width,+ height: height,+ format: metadata.format,+ }+ } catch (err) {+ lastError = err as Error+ logger.image.warn(`Metadata extraction attempt ${attempt} failed:`, err)++ if (attempt < maxRetries) {+ await new Promise(resolve => setTimeout(resolve, 500 * attempt))+ }}- } catch (err) {- logger.image.error('Failed to get image metadata', err)- return null}++ logger.image.error('All metadata extraction attempts failed')+ return null}export const convertHeicToJpeg = async (heicBuffer: Buffer) => {- try {- logger.image.info('Converting HEIC to JPEG...')-- const jpegBuffer = await heicConvert({- // @ts-expect-error idk why there is a type error here- buffer: heicBuffer,- format: 'JPEG',- quality: 0.95,- })+ const maxRetries = 3+ let lastError: Error | null = null- logger.image.info('Successfully converted HEIC to JPEG')- return Buffer.from(jpegBuffer)- } catch (err) {- logger.image.error('Failed to convert HEIC to JPEG', err)- throw err+ for (let attempt = 1; attempt <= maxRetries; attempt++) {+ try {+ logger.image.info(`Converting HEIC to JPEG (attempt ${attempt}/${maxRetries})...`)++ // 检查文件大小,如果太大则降低质量+ const fileSizeMB = heicBuffer.length / (1024 * 1024)+ const quality = fileSizeMB > 10 ? 0.8 : 0.95++ const jpegBuffer = await Promise.race([+ heicConvert({+ // @ts-expect-error idk why there is a type error here+ buffer: heicBuffer,+ format: 'JPEG',+ quality,+ }),+ new Promise((_, reject) =>+ setTimeout(() => reject(new Error('HEIC conversion timeout')), 30000)+ )+ ])++ logger.image.info(`Successfully converted HEIC to JPEG (quality: ${quality})`)+ return Buffer.from(jpegBuffer as ArrayBuffer)+ } catch (err) {+ lastError = err as Error+ logger.image.warn(`HEIC conversion attempt ${attempt} failed:`, err)++ if (attempt < maxRetries) {+ // 等待一段时间后重试+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt))+ }+ }}++ logger.image.error('All HEIC conversion attempts failed')+ throw lastError || new Error('HEIC conversion failed after all retries')}export const convertBitmapToSharpInst = async (bitmapBuffer: Buffer) => {diff --git a/server/services/image/thumbnail.ts b/server/services/image/thumbnail.tsindex eaf8dd72..c36e1208 100644--- a/server/services/image/thumbnail.ts+++ b/server/services/image/thumbnail.ts@@ -5,11 +5,48 @@ export const generateThumbnailAndHash = async (buffer: Buffer,logger?: Logger[keyof Logger],) => {- const sharpInst = sharp(buffer).rotate()- const thumbnailBuffer = await sharpInst- .resize(600, null, { withoutEnlargement: true })- .webp({ quality: 100 })- .toBuffer()- const thumbnailHash = await generateBlurHash(thumbnailBuffer, logger)- return { thumbnailBuffer, thumbnailHash }+ const maxRetries = 3+ let lastError: Error | null = null++ for (let attempt = 1; attempt <= maxRetries; attempt++) {+ try {+ logger?.info(`Generating thumbnail (attempt ${attempt}/${maxRetries})...`)++ const sharpInst = sharp(buffer).rotate()++ // 根据文件大小调整缩略图质量+ const fileSizeMB = buffer.length / (1024 * 1024)+ const quality = fileSizeMB > 5 ? 85 : 100++ const thumbnailBuffer = await Promise.race([+ sharpInst+ .resize(600, null, {+ withoutEnlargement: true,+ fastShrinkOnLoad: false // 提高质量+ })+ .webp({ quality })+ .toBuffer(),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('Thumbnail generation timeout')), 15000)+ )+ ])++ logger?.info(`Successfully generated thumbnail (quality: ${quality})`)++ // 生成BlurHash+ const thumbnailHash = await generateBlurHash(thumbnailBuffer, logger)++ return { thumbnailBuffer, thumbnailHash }+ } catch (err) {+ lastError = err as Error+ logger?.warn(`Thumbnail generation attempt ${attempt} failed:`, err)++ if (attempt < maxRetries) {+ await new Promise(resolve => setTimeout(resolve, 500 * attempt))+ }+ }+ }++ logger?.error('All thumbnail generation attempts failed')+ throw lastError || new Error('Thumbnail generation failed after all retries')}diff --git a/server/services/location/geocoding.ts b/server/services/location/geocoding.tsindex 2bd1b841..6118c89f 100644--- a/server/services/location/geocoding.ts+++ b/server/services/location/geocoding.ts@@ -25,65 +25,87 @@ export class MapboxGeocodingProvider implements GeocodingProvider {}async reverseGeocode(lat: number, lon: number): Promise<LocationInfo | null> {- try {- // 应用速率限制- await this.applyRateLimit()-- const url = new URL('/search/geocode/v6/reverse', this.baseUrl)- url.searchParams.set('access_token', this.accessToken)- url.searchParams.set('longitude', lon.toString())- url.searchParams.set('latitude', lat.toString())- url.searchParams.set('types', 'address,place,district,region,country')- url.searchParams.set('language', 'zh-Hants')-- logger.location.info(`Mapbox API URL: ${url.toString()}`)-- const response = await fetch(url.toString())-- if (!response.ok) {- logger.location.warn(- `Mapbox API responded with status ${response.status}`,- )- return null+ const maxRetries = 3+ let lastError: Error | null = null++ for (let attempt = 1; attempt <= maxRetries; attempt++) {+ try {+ // 应用速率限制+ await this.applyRateLimit()++ const url = new URL('/search/geocode/v6/reverse', this.baseUrl)+ url.searchParams.set('access_token', this.accessToken)+ url.searchParams.set('longitude', lon.toString())+ url.searchParams.set('latitude', lat.toString())+ url.searchParams.set('types', 'address,place,district,region,country')+ url.searchParams.set('language', 'zh-Hants')++ logger.location.info(`Mapbox API URL (attempt ${attempt}): ${url.toString()}`)++ const response = await Promise.race([+ fetch(url.toString()),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error('Mapbox API timeout')), 10000)+ )+ ])++ if (!response.ok) {+ logger.location.warn(+ `Mapbox API responded with status ${response.status} (attempt ${attempt})`,+ )+ if (attempt < maxRetries) {+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt))+ continue+ }+ return null+ }++ const data = await response.json()++ if (!data || !data.features || data.features.length === 0) {+ logger.location.warn('Mapbox API returned no features')+ return null+ }++ // 取第一个最相关的结果+ const feature = data.features[0]+ const properties = feature.properties || {}+ const context = properties.context || {}++ // 提取国家信息+ const country = context.country?.name++ // 提取城市信息(优先级:locality > place > locality > district > region)+ const city =+ context.locality?.name ||+ context.place?.name ||+ context.locality?.name ||+ context.district?.name ||+ context.region?.name++ // 构建位置名称+ const locationName = properties.place_formatted || properties.name++ logger.location.success(`Successfully geocoded location: ${city}, ${country}`)+ return {+ latitude: lat,+ longitude: lon,+ country,+ city,+ locationName,+ }+ } catch (error) {+ lastError = error as Error+ logger.location.warn(`Mapbox reverse geocoding attempt ${attempt} failed:`, error)++ if (attempt < maxRetries) {+ await new Promise(resolve => setTimeout(resolve, 2000 * attempt))+ }}-- const data = await response.json()-- if (!data || !data.features || data.features.length === 0) {- logger.location.warn('Mapbox API returned no features')- return null- }-- // 取第一个最相关的结果- const feature = data.features[0]- const properties = feature.properties || {}- const context = properties.context || {}-- // 提取国家信息- const country = context.country?.name-- // 提取城市信息(优先级:locality > place > locality > district > region)- const city =- context.locality?.name ||- context.place?.name ||- context.locality?.name ||- context.district?.name ||- context.region?.name-- // 构建位置名称- const locationName = properties.place_formatted || properties.name-- return {- latitude: lat,- longitude: lon,- country,- city,- locationName,- }- } catch (error) {- logger.location.error('Mapbox reverse geocoding failed:', error)- return null}++ logger.location.error('All Mapbox reverse geocoding attempts failed')+ return null}private async applyRateLimit(): Promise<void> {diff --git a/server/services/pipeline-queue/manager.ts b/server/services/pipeline-queue/manager.tsindex 52a8454e..4893a812 100644--- a/server/services/pipeline-queue/manager.ts+++ b/server/services/pipeline-queue/manager.ts@@ -192,14 +192,31 @@ export class QueueManager {const newAttempts = task.attempts + 1const shouldRetry = newAttempts < task.maxAttempts+ // 计算重试延迟(指数退避)+ const retryDelay = shouldRetry ? Math.min(1000 * Math.pow(2, newAttempts - 1), 30000) : 0+await db.update(tables.pipelineQueue).set({status: shouldRetry ? 'pending' : 'failed',attempts: newAttempts,errorMessage: errorMessage || 'Unknown error',+ // 如果重试,设置延迟重试时间+ ...(shouldRetry && retryDelay > 0 ? {+ createdAt: new Date(Date.now() + retryDelay).toISOString()+ } : {})}).where(eq(tables.pipelineQueue.id, taskId))++ if (shouldRetry) {+ this.logger.warn(+ `Task ${taskId} failed (attempt ${newAttempts}/${task.maxAttempts}), will retry in ${retryDelay}ms: ${errorMessage}`+ )+ } else {+ this.logger.error(+ `Task ${taskId} failed permanently after ${newAttempts} attempts: ${errorMessage}`+ )+ }}/** 任务处理器 */diff --git a/server/services/pipeline-queue/worker-pool.ts b/server/services/pipeline-queue/worker-pool.tsindex 00319645..afed7627 100644--- a/server/services/pipeline-queue/worker-pool.ts+++ b/server/services/pipeline-queue/worker-pool.ts@@ -45,9 +45,9 @@ export class WorkerPool {constructor(config: Partial<WorkerPoolConfig>, logger?: ConsolaInstance) {this.config = {- workerCount: 3,- intervalMs: 2000,- intervalOffset: 500,+ workerCount: 5, // 增加worker数量以提高并发处理能力+ intervalMs: 1500, // 减少轮询间隔以提高响应速度+ intervalOffset: 300, // 减少偏移时间enableLoadBalancing: true,statsReportInterval: 30000, // 30秒...config,diff --git a/server/utils/db.ts b/server/utils/db.tsindex 1331f160..432ddb97 100644--- a/server/utils/db.ts+++ b/server/utils/db.ts@@ -7,10 +7,36 @@ export { sql, eq, and, or } from 'drizzle-orm'export const tables = schema+// 创建单例数据库连接+let dbInstance: ReturnType<typeof drizzle> | null = null+let sqliteInstance: Database.Database | null = null+export function useDB() {- // return drizzle(hubDatabase(), { schema })- const sqlite = new Database('data/app.sqlite3')- return drizzle(sqlite, { schema })+ if (!dbInstance || !sqliteInstance) {+ // 创建数据库连接,启用WAL模式以提高并发性能+ sqliteInstance = new Database('data/app.sqlite3', {+ verbose: process.env.NODE_ENV === 'development' ? console.log : undefined,+ })++ // 启用WAL模式以提高并发性能+ sqliteInstance.pragma('journal_mode = WAL')+ sqliteInstance.pragma('synchronous = NORMAL')+ sqliteInstance.pragma('cache_size = 1000')+ sqliteInstance.pragma('temp_store = MEMORY')++ dbInstance = drizzle(sqliteInstance, { schema })+ }++ return dbInstance+}++// 优雅关闭数据库连接+export function closeDB() {+ if (sqliteInstance) {+ sqliteInstance.close()+ sqliteInstance = null+ dbInstance = null+ }}export type User = typeof schema.users.$inferSelect