Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

feat: 增强图片上传功能,支持MOV视频文件类型;优化数据库连接,增加worker数量和减少轮询间隔以提升性能

cuijianzhuang8562bbf7ec5a
diff --git a/app/pages/dashboard/photos.vue b/app/pages/dashboard/photos.vue
index da264d80..ea287e4a 100644
--- a/app/pages/dashboard/photos.vue
+++ b/app/pages/dashboard/photos.vue
@@ -120,14 +120,16 @@ const uploadImage = async (file: File) => {
uploadingFiles.value = new Map(uploadingFiles.value)
try {
+ // 检查是否为MOV视频文件(通过MIME类型或文件扩展名)
+ const isMovFile = file.type === 'video/quicktime' ||
+ file.type === 'video/mp4' ||
+ file.name.toLowerCase().endsWith('.mov')
+
const resp = await $fetch('/api/queue/add-task', {
method: 'POST',
body: {
payload: {
- type:
- file.type === 'video/quicktime'
- ? 'live-photo-video'
- : 'photo',
+ type: isMovFile ? 'live-photo-video' : 'photo',
storageKey: signedUrlResponse.fileKey,
},
priority: 0,
@@ -569,24 +571,98 @@ const handleUpload = async () => {
return
}
+ // 显示批量上传进度
+ const uploadToast = toast.add({
+ title: '开始批量上传',
+ description: `正在上传 ${fileList.length} 个文件...`,
+ color: 'info',
+ })
+
+ let successCount = 0
+ let errorCount = 0
+ const errors: string[] = []
+
+ // 先验证所有文件
+ const validFiles: File[] = []
for (const file of fileList) {
- // 验证文件
const validation = validateFile(file)
if (!validation.valid) {
- toast.add({
- title: '文件验证失败',
- description: validation.error,
- color: 'error',
- })
- continue
+ errors.push(`${file.name}: ${validation.error}`)
+ errorCount++
+ } else {
+ validFiles.push(file)
}
+ }
- try {
- await uploadImage(file)
- } catch (error: any) {
- // 错误已经在 uploadImage 函数内部处理了
- console.error('上传错误:', error)
- }
+ if (validFiles.length === 0) {
+ toast.update(uploadToast.id, {
+ title: '批量上传失败',
+ description: '所有文件验证失败',
+ color: 'error',
+ })
+ selectedFiles.value = []
+ return
+ }
+
+ // 并发上传,但限制并发数量以避免资源竞争
+ const CONCURRENT_LIMIT = 3 // 限制同时上传的文件数量
+ const uploadPromises: Promise<void>[] = []
+
+ for (let i = 0; i < validFiles.length; i += CONCURRENT_LIMIT) {
+ const batch = validFiles.slice(i, i + CONCURRENT_LIMIT)
+
+ const batchPromises = batch.map(async (file) => {
+ try {
+ await uploadImage(file)
+ successCount++
+ } catch (error: any) {
+ errorCount++
+ errors.push(`${file.name}: ${error.message || '上传失败'}`)
+ console.error('上传错误:', error)
+ }
+ })
+
+ uploadPromises.push(...batchPromises)
+
+ // 等待当前批次完成再处理下一批次
+ await Promise.allSettled(batchPromises)
+
+ // 更新进度
+ const processed = Math.min(i + CONCURRENT_LIMIT, validFiles.length)
+ toast.update(uploadToast.id, {
+ title: '批量上传进行中',
+ description: `已处理 ${processed}/${validFiles.length} 个文件`,
+ color: 'info',
+ })
+ }
+
+ // 等待所有上传完成
+ await Promise.allSettled(uploadPromises)
+
+ // 显示最终结果
+ if (errorCount === 0) {
+ toast.update(uploadToast.id, {
+ title: '批量上传成功',
+ description: `成功上传 ${successCount} 个文件`,
+ color: 'success',
+ })
+ } else if (successCount === 0) {
+ toast.update(uploadToast.id, {
+ title: '批量上传失败',
+ description: `所有 ${errorCount} 个文件上传失败`,
+ color: 'error',
+ })
+ } else {
+ toast.update(uploadToast.id, {
+ title: '批量上传部分成功',
+ description: `成功: ${successCount}, 失败: ${errorCount}`,
+ color: 'warning',
+ })
+ }
+
+ // 如果有错误,显示详细错误信息
+ if (errors.length > 0) {
+ console.error('批量上传错误详情:', errors)
}
// 清空选中的文件
diff --git a/server/services/image/exif.ts b/server/services/image/exif.ts
index c7d46912..7d573b8f 100644
--- a/server/services/image/exif.ts
+++ b/server/services/image/exif.ts
@@ -274,46 +274,84 @@ export const extractExifData = async (
rawImageBuffer?: Buffer,
logger?: Logger[keyof Logger],
): Promise<NeededExif | null> => {
- try {
- let metadata = await sharp(imageBuffer).metadata()
+ const maxRetries = 3
+ let lastError: Error | null = null
- if (!metadata.exif && rawImageBuffer) {
- try {
- metadata = await sharp(rawImageBuffer).metadata()
- } catch (err) {
- logger?.warn('Error extracting EXIF data from raw image buffer:', err)
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
+ let tempImagePath: string | null = null
+
+ try {
+ logger?.info(`Extracting EXIF data (attempt ${attempt}/${maxRetries})...`)
+
+ let metadata = await Promise.race([
+ sharp(imageBuffer).metadata(),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error('Metadata extraction timeout')), 5000)
+ )
+ ])
+
+ if (!metadata.exif && rawImageBuffer) {
+ try {
+ metadata = await Promise.race([
+ sharp(rawImageBuffer).metadata(),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error('Raw metadata extraction timeout')), 5000)
+ )
+ ])
+ } catch (err) {
+ logger?.warn('Error extracting EXIF data from raw image buffer:', err)
+ }
}
- }
-
- if (!metadata.exif) {
- logger?.warn('No EXIF data found in image metadata')
- return null
- }
-
- logger?.info('Extracting EXIF data using exiftool...')
- const tempDir = path.resolve(process.cwd(), 'data/.exif_workdir')
- await mkdir(tempDir, { recursive: true })
- const tempImagePath = path.resolve(tempDir, `${crypto.randomUUID()}.jpg`)
+ if (!metadata.exif) {
+ logger?.warn('No EXIF data found in image metadata')
+ return null
+ }
- await writeFile(tempImagePath, rawImageBuffer || imageBuffer)
- const exifData = await exiftool.read(tempImagePath)
- const result = processExifData(exifData, metadata)
+ logger?.info('Extracting EXIF data using exiftool...')
+
+ const tempDir = path.resolve(process.cwd(), 'data/.exif_workdir')
+ await mkdir(tempDir, { recursive: true })
+ tempImagePath = path.resolve(tempDir, `${crypto.randomUUID()}.jpg`)
+
+ await writeFile(tempImagePath, rawImageBuffer || imageBuffer)
+
+ const exifData = await Promise.race([
+ exiftool.read(tempImagePath),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error('EXIF tool timeout')), 10000)
+ )
+ ])
+
+ const result = processExifData(exifData, metadata)
+
+ // 记录颜色空间信息的来源
+ if (result.ColorSpace) {
+ logger?.success(`Inferred ColorSpace: ${result.ColorSpace}`)
+ } else {
+ logger?.info('ColorSpace could not be determined')
+ }
- // 记录颜色空间信息的来源
- if (result.ColorSpace) {
- logger?.success(`Inferred ColorSpace: ${result.ColorSpace}`)
- } else {
- logger?.info('ColorSpace could not be determined')
+ // 清理临时文件
+ await unlink(tempImagePath).catch(noop)
+ return result
+ } catch (err) {
+ lastError = err as Error
+ logger?.warn(`EXIF extraction attempt ${attempt} failed:`, err)
+
+ // 清理临时文件
+ if (tempImagePath) {
+ await unlink(tempImagePath).catch(noop)
+ }
+
+ if (attempt < maxRetries) {
+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt))
+ }
}
-
- // 清理临时文件
- await unlink(tempImagePath).catch(noop)
- return result
- } catch (err) {
- logger?.error('EXIF extraction failed:', err)
- return null
}
+
+ logger?.error('All EXIF extraction attempts failed')
+ return null
}
export const extractPhotoInfo = (
diff --git a/server/services/image/processor.ts b/server/services/image/processor.ts
index da0b75b4..1fcb54fa 100644
--- a/server/services/image/processor.ts
+++ b/server/services/image/processor.ts
@@ -25,48 +25,95 @@ const isBitmap = (buffer: Buffer) => {
const getMetadataWithSharp = async (
sharpInst: sharp.Sharp,
): Promise<ImageMeta | null> => {
- try {
- const metadata = await sharpInst.metadata()
+ const maxRetries = 3
+ let lastError: Error | null = null
- if (!metadata.height || !metadata.width || !metadata.format) {
- return null
- }
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
+ try {
+ logger.image.info(`Extracting metadata (attempt ${attempt}/${maxRetries})...`)
+
+ const metadata = await Promise.race([
+ sharpInst.metadata(),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error('Metadata extraction timeout')), 10000)
+ )
+ ])
+
+ if (!metadata.height || !metadata.width || !metadata.format) {
+ logger.image.warn('Incomplete metadata received:', {
+ hasHeight: !!metadata.height,
+ hasWidth: !!metadata.width,
+ hasFormat: !!metadata.format
+ })
+ return null
+ }
- const { orientation } = metadata
- let { width, height } = metadata
+ const { orientation } = metadata
+ let { width, height } = metadata
- if (orientation && [5, 6, 7, 8].includes(orientation)) {
- ;[width, height] = [height, width]
- }
+ if (orientation && [5, 6, 7, 8].includes(orientation)) {
+ ;[width, height] = [height, width]
+ }
- return {
- width: width,
- height: height,
- format: metadata.format,
+ logger.image.info(`Successfully extracted metadata: ${width}x${height} ${metadata.format}`)
+ return {
+ width: width,
+ height: height,
+ format: metadata.format,
+ }
+ } catch (err) {
+ lastError = err as Error
+ logger.image.warn(`Metadata extraction attempt ${attempt} failed:`, err)
+
+ if (attempt < maxRetries) {
+ await new Promise(resolve => setTimeout(resolve, 500 * attempt))
+ }
}
- } catch (err) {
- logger.image.error('Failed to get image metadata', err)
- return null
}
+
+ logger.image.error('All metadata extraction attempts failed')
+ return null
}
export const convertHeicToJpeg = async (heicBuffer: Buffer) => {
- try {
- logger.image.info('Converting HEIC to JPEG...')
-
- const jpegBuffer = await heicConvert({
- // @ts-expect-error idk why there is a type error here
- buffer: heicBuffer,
- format: 'JPEG',
- quality: 0.95,
- })
+ const maxRetries = 3
+ let lastError: Error | null = null
- logger.image.info('Successfully converted HEIC to JPEG')
- return Buffer.from(jpegBuffer)
- } catch (err) {
- logger.image.error('Failed to convert HEIC to JPEG', err)
- throw err
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
+ try {
+ logger.image.info(`Converting HEIC to JPEG (attempt ${attempt}/${maxRetries})...`)
+
+ // 检查文件大小,如果太大则降低质量
+ const fileSizeMB = heicBuffer.length / (1024 * 1024)
+ const quality = fileSizeMB > 10 ? 0.8 : 0.95
+
+ const jpegBuffer = await Promise.race([
+ heicConvert({
+ // @ts-expect-error idk why there is a type error here
+ buffer: heicBuffer,
+ format: 'JPEG',
+ quality,
+ }),
+ new Promise((_, reject) =>
+ setTimeout(() => reject(new Error('HEIC conversion timeout')), 30000)
+ )
+ ])
+
+ logger.image.info(`Successfully converted HEIC to JPEG (quality: ${quality})`)
+ return Buffer.from(jpegBuffer as ArrayBuffer)
+ } catch (err) {
+ lastError = err as Error
+ logger.image.warn(`HEIC conversion attempt ${attempt} failed:`, err)
+
+ if (attempt < maxRetries) {
+ // 等待一段时间后重试
+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt))
+ }
+ }
}
+
+ logger.image.error('All HEIC conversion attempts failed')
+ throw lastError || new Error('HEIC conversion failed after all retries')
}
export const convertBitmapToSharpInst = async (bitmapBuffer: Buffer) => {
diff --git a/server/services/image/thumbnail.ts b/server/services/image/thumbnail.ts
index eaf8dd72..c36e1208 100644
--- a/server/services/image/thumbnail.ts
+++ b/server/services/image/thumbnail.ts
@@ -5,11 +5,48 @@ export const generateThumbnailAndHash = async (
buffer: Buffer,
logger?: Logger[keyof Logger],
) => {
- const sharpInst = sharp(buffer).rotate()
- const thumbnailBuffer = await sharpInst
- .resize(600, null, { withoutEnlargement: true })
- .webp({ quality: 100 })
- .toBuffer()
- const thumbnailHash = await generateBlurHash(thumbnailBuffer, logger)
- return { thumbnailBuffer, thumbnailHash }
+ const maxRetries = 3
+ let lastError: Error | null = null
+
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
+ try {
+ logger?.info(`Generating thumbnail (attempt ${attempt}/${maxRetries})...`)
+
+ const sharpInst = sharp(buffer).rotate()
+
+ // 根据文件大小调整缩略图质量
+ const fileSizeMB = buffer.length / (1024 * 1024)
+ const quality = fileSizeMB > 5 ? 85 : 100
+
+ const thumbnailBuffer = await Promise.race([
+ sharpInst
+ .resize(600, null, {
+ withoutEnlargement: true,
+ fastShrinkOnLoad: false // 提高质量
+ })
+ .webp({ quality })
+ .toBuffer(),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error('Thumbnail generation timeout')), 15000)
+ )
+ ])
+
+ logger?.info(`Successfully generated thumbnail (quality: ${quality})`)
+
+ // 生成BlurHash
+ const thumbnailHash = await generateBlurHash(thumbnailBuffer, logger)
+
+ return { thumbnailBuffer, thumbnailHash }
+ } catch (err) {
+ lastError = err as Error
+ logger?.warn(`Thumbnail generation attempt ${attempt} failed:`, err)
+
+ if (attempt < maxRetries) {
+ await new Promise(resolve => setTimeout(resolve, 500 * attempt))
+ }
+ }
+ }
+
+ logger?.error('All thumbnail generation attempts failed')
+ throw lastError || new Error('Thumbnail generation failed after all retries')
}
diff --git a/server/services/location/geocoding.ts b/server/services/location/geocoding.ts
index 2bd1b841..6118c89f 100644
--- a/server/services/location/geocoding.ts
+++ b/server/services/location/geocoding.ts
@@ -25,65 +25,87 @@ export class MapboxGeocodingProvider implements GeocodingProvider {
}
async reverseGeocode(lat: number, lon: number): Promise<LocationInfo | null> {
- try {
- // 应用速率限制
- await this.applyRateLimit()
-
- const url = new URL('/search/geocode/v6/reverse', this.baseUrl)
- url.searchParams.set('access_token', this.accessToken)
- url.searchParams.set('longitude', lon.toString())
- url.searchParams.set('latitude', lat.toString())
- url.searchParams.set('types', 'address,place,district,region,country')
- url.searchParams.set('language', 'zh-Hants')
-
- logger.location.info(`Mapbox API URL: ${url.toString()}`)
-
- const response = await fetch(url.toString())
-
- if (!response.ok) {
- logger.location.warn(
- `Mapbox API responded with status ${response.status}`,
- )
- return null
+ const maxRetries = 3
+ let lastError: Error | null = null
+
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
+ try {
+ // 应用速率限制
+ await this.applyRateLimit()
+
+ const url = new URL('/search/geocode/v6/reverse', this.baseUrl)
+ url.searchParams.set('access_token', this.accessToken)
+ url.searchParams.set('longitude', lon.toString())
+ url.searchParams.set('latitude', lat.toString())
+ url.searchParams.set('types', 'address,place,district,region,country')
+ url.searchParams.set('language', 'zh-Hants')
+
+ logger.location.info(`Mapbox API URL (attempt ${attempt}): ${url.toString()}`)
+
+ const response = await Promise.race([
+ fetch(url.toString()),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error('Mapbox API timeout')), 10000)
+ )
+ ])
+
+ if (!response.ok) {
+ logger.location.warn(
+ `Mapbox API responded with status ${response.status} (attempt ${attempt})`,
+ )
+ if (attempt < maxRetries) {
+ await new Promise(resolve => setTimeout(resolve, 1000 * attempt))
+ continue
+ }
+ return null
+ }
+
+ const data = await response.json()
+
+ if (!data || !data.features || data.features.length === 0) {
+ logger.location.warn('Mapbox API returned no features')
+ return null
+ }
+
+ // 取第一个最相关的结果
+ const feature = data.features[0]
+ const properties = feature.properties || {}
+ const context = properties.context || {}
+
+ // 提取国家信息
+ const country = context.country?.name
+
+ // 提取城市信息(优先级:locality > place > locality > district > region)
+ const city =
+ context.locality?.name ||
+ context.place?.name ||
+ context.locality?.name ||
+ context.district?.name ||
+ context.region?.name
+
+ // 构建位置名称
+ const locationName = properties.place_formatted || properties.name
+
+ logger.location.success(`Successfully geocoded location: ${city}, ${country}`)
+ return {
+ latitude: lat,
+ longitude: lon,
+ country,
+ city,
+ locationName,
+ }
+ } catch (error) {
+ lastError = error as Error
+ logger.location.warn(`Mapbox reverse geocoding attempt ${attempt} failed:`, error)
+
+ if (attempt < maxRetries) {
+ await new Promise(resolve => setTimeout(resolve, 2000 * attempt))
+ }
}
-
- const data = await response.json()
-
- if (!data || !data.features || data.features.length === 0) {
- logger.location.warn('Mapbox API returned no features')
- return null
- }
-
- // 取第一个最相关的结果
- const feature = data.features[0]
- const properties = feature.properties || {}
- const context = properties.context || {}
-
- // 提取国家信息
- const country = context.country?.name
-
- // 提取城市信息(优先级:locality > place > locality > district > region)
- const city =
- context.locality?.name ||
- context.place?.name ||
- context.locality?.name ||
- context.district?.name ||
- context.region?.name
-
- // 构建位置名称
- const locationName = properties.place_formatted || properties.name
-
- return {
- latitude: lat,
- longitude: lon,
- country,
- city,
- locationName,
- }
- } catch (error) {
- logger.location.error('Mapbox reverse geocoding failed:', error)
- return null
}
+
+ logger.location.error('All Mapbox reverse geocoding attempts failed')
+ return null
}
private async applyRateLimit(): Promise<void> {
diff --git a/server/services/pipeline-queue/manager.ts b/server/services/pipeline-queue/manager.ts
index 52a8454e..4893a812 100644
--- a/server/services/pipeline-queue/manager.ts
+++ b/server/services/pipeline-queue/manager.ts
@@ -192,14 +192,31 @@ export class QueueManager {
const newAttempts = task.attempts + 1
const shouldRetry = newAttempts < task.maxAttempts
+ // 计算重试延迟(指数退避)
+ const retryDelay = shouldRetry ? Math.min(1000 * Math.pow(2, newAttempts - 1), 30000) : 0
+
await db
.update(tables.pipelineQueue)
.set({
status: shouldRetry ? 'pending' : 'failed',
attempts: newAttempts,
errorMessage: errorMessage || 'Unknown error',
+ // 如果重试,设置延迟重试时间
+ ...(shouldRetry && retryDelay > 0 ? {
+ createdAt: new Date(Date.now() + retryDelay).toISOString()
+ } : {})
})
.where(eq(tables.pipelineQueue.id, taskId))
+
+ if (shouldRetry) {
+ this.logger.warn(
+ `Task ${taskId} failed (attempt ${newAttempts}/${task.maxAttempts}), will retry in ${retryDelay}ms: ${errorMessage}`
+ )
+ } else {
+ this.logger.error(
+ `Task ${taskId} failed permanently after ${newAttempts} attempts: ${errorMessage}`
+ )
+ }
}
/** 任务处理器 */
diff --git a/server/services/pipeline-queue/worker-pool.ts b/server/services/pipeline-queue/worker-pool.ts
index 00319645..afed7627 100644
--- a/server/services/pipeline-queue/worker-pool.ts
+++ b/server/services/pipeline-queue/worker-pool.ts
@@ -45,9 +45,9 @@ export class WorkerPool {
constructor(config: Partial<WorkerPoolConfig>, logger?: ConsolaInstance) {
this.config = {
- workerCount: 3,
- intervalMs: 2000,
- intervalOffset: 500,
+ workerCount: 5, // 增加worker数量以提高并发处理能力
+ intervalMs: 1500, // 减少轮询间隔以提高响应速度
+ intervalOffset: 300, // 减少偏移时间
enableLoadBalancing: true,
statsReportInterval: 30000, // 30秒
...config,
diff --git a/server/utils/db.ts b/server/utils/db.ts
index 1331f160..432ddb97 100644
--- a/server/utils/db.ts
+++ b/server/utils/db.ts
@@ -7,10 +7,36 @@ export { sql, eq, and, or } from 'drizzle-orm'
export const tables = schema
+// 创建单例数据库连接
+let dbInstance: ReturnType<typeof drizzle> | null = null
+let sqliteInstance: Database.Database | null = null
+
export function useDB() {
- // return drizzle(hubDatabase(), { schema })
- const sqlite = new Database('data/app.sqlite3')
- return drizzle(sqlite, { schema })
+ if (!dbInstance || !sqliteInstance) {
+ // 创建数据库连接,启用WAL模式以提高并发性能
+ sqliteInstance = new Database('data/app.sqlite3', {
+ verbose: process.env.NODE_ENV === 'development' ? console.log : undefined,
+ })
+
+ // 启用WAL模式以提高并发性能
+ sqliteInstance.pragma('journal_mode = WAL')
+ sqliteInstance.pragma('synchronous = NORMAL')
+ sqliteInstance.pragma('cache_size = 1000')
+ sqliteInstance.pragma('temp_store = MEMORY')
+
+ dbInstance = drizzle(sqliteInstance, { schema })
+ }
+
+ return dbInstance
+}
+
+// 优雅关闭数据库连接
+export function closeDB() {
+ if (sqliteInstance) {
+ sqliteInstance.close()
+ sqliteInstance = null
+ dbInstance = null
+ }
}
export type User = typeof schema.users.$inferSelect