Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(worker): handle stale queued task params

kapil971390254cd028906e
diff --git a/app/controllers/manager/base_manager.py b/app/controllers/manager/base_manager.py
index ac5bf710..ab214643 100644
--- a/app/controllers/manager/base_manager.py
+++ b/app/controllers/manager/base_manager.py
@@ -70,6 +70,12 @@ class TaskManager:
and not self.is_queue_empty()
):
task_info = self.dequeue()
+ if task_info is None:
+ # dequeue() may skip and discard queue entries that no longer
+ # pass current validation (see RedisTaskManager.dequeue) and
+ # return None once nothing usable is left, even though
+ # is_queue_empty() was False a moment earlier.
+ return
func = task_info["func"]
args = task_info.get("args", ())
kwargs = task_info.get("kwargs", {})
diff --git a/app/controllers/manager/redis_manager.py b/app/controllers/manager/redis_manager.py
index db108cc2..2f9568aa 100644
--- a/app/controllers/manager/redis_manager.py
+++ b/app/controllers/manager/redis_manager.py
@@ -2,9 +2,13 @@ import json
from typing import Dict
import redis
+from loguru import logger
+from pydantic import ValidationError
from app.controllers.manager.base_manager import TaskManager
+from app.models import const
from app.models.schema import VideoParams
+from app.services import state as sm
from app.services import task as tm
FUNC_MAP = {
@@ -44,8 +48,17 @@ class RedisTaskManager(TaskManager):
self.redis_client.rpush(self.queue, json.dumps(task_with_serializable_params))
def dequeue(self):
- task_json = self.redis_client.lpop(self.queue)
- if task_json:
+ # 循环而非单次弹出:某个任务在入队时可能满足当时的 VideoParams 校验规则,
+ # 但校验规则本身在两次部署之间收紧了(例如新增 ge=1 约束)。lpop 是破坏性
+ # 操作,一旦弹出就不能放回原位;如果重建 VideoParams 时才发现校验失败,
+ # 这条任务已经从队列中永久移除了,不能再假装它还在。与其让异常从这里往上
+ # 抛、把这条已经丢失的任务的 lock 持有者带崩,不如原地丢弃并继续尝试队列
+ # 里的下一条,把"拿到一条可用任务或者队列确实空了"这个约定维持住。
+ while True:
+ task_json = self.redis_client.lpop(self.queue)
+ if not task_json:
+ return None
+
task_info = json.loads(task_json)
# 将函数名称转换回函数对象
task_info["func"] = FUNC_MAP[task_info["func"]]
@@ -53,12 +66,31 @@ class RedisTaskManager(TaskManager):
if "params" in task_info["kwargs"] and isinstance(
task_info["kwargs"]["params"], dict
):
- task_info["kwargs"]["params"] = VideoParams(
- **task_info["kwargs"]["params"]
- )
+ try:
+ task_info["kwargs"]["params"] = VideoParams(
+ **task_info["kwargs"]["params"]
+ )
+ except ValidationError as e:
+ logger.error(
+ "dropping queued task with params that fail current "
+ f"VideoParams validation (queued under an older, more "
+ f"permissive schema, or corrupted): {e}"
+ )
+ # 任务状态记录在入队前就已创建,且默认是 processing;如果只是
+ # 丢弃这条队列项而不动状态记录,API/WebUI 会一直显示任务在
+ # 运行,永远不会变成失败。用 patch_task 而不是 update_task,
+ # 这样如果用户已经删除了这个任务,我们不会又把它建回来。
+ task_id = task_info["kwargs"].get("task_id")
+ if task_id:
+ sm.state.patch_task(
+ task_id,
+ state=const.TASK_STATE_FAILED,
+ failed_stage="dequeue",
+ error=f"discarded stale queued task: {e}",
+ )
+ continue
return task_info
- return None
def is_queue_empty(self):
return self.redis_client.llen(self.queue) == 0
diff --git a/test/services/test_task_manager.py b/test/services/test_task_manager.py
index 428ae02a..3ec25594 100644
--- a/test/services/test_task_manager.py
+++ b/test/services/test_task_manager.py
@@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch
from app.controllers.manager.base_manager import TaskQueueFullError
from app.controllers.manager.memory_manager import InMemoryTaskManager
from app.controllers.manager.redis_manager import RedisTaskManager
+from app.models import const
from app.models.schema import VideoParams
from app.services import task as task_service
@@ -103,6 +104,22 @@ class TestInMemoryTaskManager(unittest.TestCase):
self.assertEqual(manager.current_tasks, 1)
task_done.assert_called_once_with()
+ def test_check_queue_handles_dequeue_returning_none(self):
+ """
+ dequeue() 可能在内部跳过所有已不满足当前校验的排队任务后返回 None,
+ 即使调用 check_queue 之前 is_queue_empty() 曾经是 False。check_queue
+ 不能假设 dequeue 一定能拿到可用任务,否则会在 task_info["func"] 上崩溃。
+ """
+ manager = InMemoryTaskManager(max_concurrent_tasks=1, max_queued_tasks=1)
+
+ with patch.object(manager, "is_queue_empty", return_value=False), patch.object(
+ manager, "dequeue", return_value=None
+ ), patch.object(manager, "execute_task") as execute_task:
+ manager.check_queue()
+
+ execute_task.assert_not_called()
+ self.assertEqual(manager.current_tasks, 0)
+
def test_execute_task_starts_background_thread(self):
"""任务执行入口必须启动线程,并把函数参数完整传给 run_task。"""
manager = InMemoryTaskManager(max_concurrent_tasks=1)
@@ -190,6 +207,116 @@ class TestRedisTaskManager(unittest.TestCase):
self.assertTrue(self.manager.is_queue_empty())
self.assertEqual(self.manager.queue_size(), 2)
+ def test_dequeue_skips_task_that_fails_current_validation(self):
+ """
+ 一条任务可能是在校验规则收紧前入队的(例如 video_count 曾允许为 0)。
+ lpop 是破坏性操作,重建 VideoParams 失败时这条任务已经从 Redis 里
+ 永久移除了,不能再假装它还在;dequeue 不应该把校验异常抛给调用方
+ (那样会让持锁的调用方崩溃且丢失这条任务却不打日志),而应该跳过它,
+ 继续尝试队列里的下一条,直到取到一条可用任务或者队列确实空了。
+ """
+ stale_payload = {
+ "func": "start",
+ "args": [],
+ "kwargs": {
+ "task_id": "task-stale",
+ "params": {**VideoParams(video_subject="Coffee").model_dump(
+ warnings=False
+ ), "video_count": 0},
+ },
+ }
+ valid_payload = {
+ "func": "start",
+ "args": [],
+ "kwargs": {
+ "task_id": "task-valid",
+ "params": VideoParams(video_subject="Tea").model_dump(
+ warnings=False
+ ),
+ },
+ }
+ self.redis_client.lpop.side_effect = [
+ json.dumps(stale_payload),
+ json.dumps(valid_payload),
+ ]
+
+ task = self.manager.dequeue()
+
+ self.assertEqual(self.redis_client.lpop.call_count, 2)
+ self.assertEqual(task["kwargs"]["task_id"], "task-valid")
+ self.assertIsInstance(task["kwargs"]["params"], VideoParams)
+ self.assertEqual(task["kwargs"]["params"].video_subject, "Tea")
+
+ def test_dequeue_returns_none_when_every_queued_task_is_stale(self):
+ """全部剩余任务都因当前校验规则被丢弃时,应返回 None 而不是抛出异常。"""
+ stale_payload = {
+ "func": "start",
+ "args": [],
+ "kwargs": {
+ "task_id": "task-stale",
+ "params": {**VideoParams(video_subject="Coffee").model_dump(
+ warnings=False
+ ), "video_count": -1},
+ },
+ }
+ self.redis_client.lpop.side_effect = [json.dumps(stale_payload), None]
+
+ self.assertIsNone(self.manager.dequeue())
+ self.assertEqual(self.redis_client.lpop.call_count, 2)
+
+ def test_dequeue_marks_stale_task_failed_instead_of_leaving_it_processing(self):
+ """
+ 任务状态记录在入队前就已创建,默认是 processing。仅仅在 dequeue 里跳过
+ 并丢弃这条队列项而不更新状态记录,会让这个任务在 API/WebUI 里永远显示
+ 为运行中。应该用 patch_task(而不是 update_task)把它标记为失败,
+ 这样如果任务已经被用户删除,我们不会又把它的状态记录建回来。
+ """
+ stale_payload = {
+ "func": "start",
+ "args": [],
+ "kwargs": {
+ "task_id": "task-stale",
+ "params": {**VideoParams(video_subject="Coffee").model_dump(
+ warnings=False
+ ), "video_count": 0},
+ },
+ }
+ self.redis_client.lpop.side_effect = [json.dumps(stale_payload), None]
+
+ with patch("app.controllers.manager.redis_manager.sm.state") as state:
+ state.patch_task.return_value = True
+ result = self.manager.dequeue()
+
+ self.assertIsNone(result)
+ state.patch_task.assert_called_once()
+ call_args = state.patch_task.call_args
+ self.assertEqual(call_args.args[0], "task-stale")
+ self.assertEqual(call_args.kwargs["state"], const.TASK_STATE_FAILED)
+ self.assertEqual(call_args.kwargs["failed_stage"], "dequeue")
+ self.assertIn("video_count", call_args.kwargs["error"])
+
+ def test_dequeue_does_not_recreate_state_for_already_deleted_task(self):
+ """patch_task 在任务已被删除时返回 False;dequeue 不应把它当成错误处理。"""
+ stale_payload = {
+ "func": "start",
+ "args": [],
+ "kwargs": {
+ "task_id": "task-deleted",
+ "params": {**VideoParams(video_subject="Coffee").model_dump(
+ warnings=False
+ ), "video_count": 0},
+ },
+ }
+ self.redis_client.lpop.side_effect = [json.dumps(stale_payload), None]
+
+ with patch("app.controllers.manager.redis_manager.sm.state") as state:
+ state.patch_task.return_value = False
+ result = self.manager.dequeue()
+
+ self.assertIsNone(result)
+ state.patch_task.assert_called_once()
+ state.update_task.assert_not_called()
+
if __name__ == "__main__":
unittest.main()