Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

perf(worker-pool): add dead tasks cleanup when startup, modified pool configuration, workers up to 5

Timothy Yinbee3aabef5de
diff --git a/server/plugins/pipeline-queue.ts b/server/plugins/pipeline-queue.ts
index 341231f8..abaed8db 100644
--- a/server/plugins/pipeline-queue.ts
+++ b/server/plugins/pipeline-queue.ts
@@ -5,9 +5,9 @@ export default defineNitroPlugin(async (_nitroApp) => {
const workerPool = new WorkerPool(
{
- workerCount: 3,
- intervalMs: 1000,
- intervalOffset: Math.floor(1000 / 3),
+ workerCount: 5,
+ intervalMs: 1500,
+ intervalOffset: 300,
enableLoadBalancing: true,
statsReportInterval: 60000 * 10,
},
diff --git a/server/services/pipeline-queue/worker-pool.ts b/server/services/pipeline-queue/worker-pool.ts
index afed7627..6f46a3fd 100644
--- a/server/services/pipeline-queue/worker-pool.ts
+++ b/server/services/pipeline-queue/worker-pool.ts
@@ -1,5 +1,7 @@
import type { ConsolaInstance } from 'consola'
import { consola } from 'consola'
+import { eq } from 'drizzle-orm'
+import { useDB, tables } from '~~/server/utils/db'
import { QueueManager } from './manager'
export interface WorkerPoolConfig {
@@ -45,11 +47,11 @@ export class WorkerPool {
constructor(config: Partial<WorkerPoolConfig>, logger?: ConsolaInstance) {
this.config = {
- workerCount: 5, // 增加worker数量以提高并发处理能力
- intervalMs: 1500, // 减少轮询间隔以提高响应速度
- intervalOffset: 300, // 减少偏移时间
+ workerCount: 3,
+ intervalMs: 2000,
+ intervalOffset: 300,
enableLoadBalancing: true,
- statsReportInterval: 30000, // 30秒
+ statsReportInterval: 30000,
...config,
}
this.logger = logger || consola.withTag('worker-pool')
@@ -68,6 +70,9 @@ export class WorkerPool {
`Starting WorkerPool with ${this.config.workerCount} workers`,
)
+ // 在启动工作器之前,清理死任务
+ await this.cleanupDeadTasks()
+
// 创建工作器
for (let i = 1; i <= this.config.workerCount; i++) {
const workerId = `worker-${i}`
@@ -241,6 +246,46 @@ export class WorkerPool {
return await this.workers[0].getQueueStats()
}
+ /**
+ * 清理死任务 - 将 in-stages 状态的任务重置为 pending,并设置高优先级
+ */
+ private async cleanupDeadTasks(): Promise<void> {
+ try {
+ const db = useDB()
+
+ // 查找所有 in-stages 状态的任务(这些可能是死任务)
+ const deadTasks = await db
+ .select()
+ .from(tables.pipelineQueue)
+ .where(eq(tables.pipelineQueue.status, 'in-stages'))
+
+ if (deadTasks.length > 0) {
+ this.logger.warn(
+ `Found ${deadTasks.length} dead tasks in 'in-stages' status, resetting to pending with priority 1`,
+ )
+
+ // 将这些任务重置为 pending 状态,设置优先级为 1,清除状态阶段
+ await db
+ .update(tables.pipelineQueue)
+ .set({
+ status: 'pending',
+ priority: 1,
+ statusStage: null,
+ })
+ .where(eq(tables.pipelineQueue.status, 'in-stages'))
+
+ this.logger.success(
+ `Successfully reset ${deadTasks.length} dead tasks to pending status with priority 1`,
+ )
+ } else {
+ this.logger.info('No dead tasks found in queue')
+ }
+ } catch (error) {
+ this.logger.error('Failed to cleanup dead tasks:', error)
+ // 不抛出错误,允许工作器池继续启动
+ }
+ }
+
/**
* 检查池是否正在运行
*/