利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix(queue): skip unusable queued tasks (#1383)
XiaoChen633dbb0cf686
diff --git a/app/controllers/manager/redis_manager.py b/app/controllers/manager/redis_manager.pyindex 2f9568aa..fff077f2 100644--- a/app/controllers/manager/redis_manager.py+++ b/app/controllers/manager/redis_manager.py@@ -48,28 +48,64 @@ class RedisTaskManager(TaskManager):self.redis_client.rpush(self.queue, json.dumps(task_with_serializable_params))def dequeue(self):- # 循环而非单次弹出:某个任务在入队时可能满足当时的 VideoParams 校验规则,- # 但校验规则本身在两次部署之间收紧了(例如新增 ge=1 约束)。lpop 是破坏性- # 操作,一旦弹出就不能放回原位;如果重建 VideoParams 时才发现校验失败,- # 这条任务已经从队列中永久移除了,不能再假装它还在。与其让异常从这里往上- # 抛、把这条已经丢失的任务的 lock 持有者带崩,不如原地丢弃并继续尝试队列- # 里的下一条,把"拿到一条可用任务或者队列确实空了"这个约定维持住。+ # 循环而非单次弹出:某个任务在入队时可能满足当时的校验规则,但校验规则与+ # FUNC_MAP 成员会随部署变化(例如 VideoParams 新增 ge=1 约束、某个入口+ # 函数被移除),队列里因此可能残留按旧 schema 写入、或已无法解析的条目。+ # lpop 是破坏性操作,一旦弹出就不能放回原位;这条任务已经从队列中永久+ # 移除了,不能再假装它还在。与其让异常从这里往上抛(check_queue 持锁调用+ # 本方法,异常会顺着 task_done → run_task 的 finally 把工作线程带崩;此后+ # 没有任务在跑,就再也不会有人调用 check_queue,队列里后面的任务会永久+ # 停在 processing),不如原地丢弃并继续尝试下一条,把"拿到一条可用任务+ # 或者队列确实空了"这个约定维持住。while True:task_json = self.redis_client.lpop(self.queue)- if not task_json:+ # 只有 lpop 什么都没弹出来才代表队列空了。空字符串(或空 bytes)同样+ # 是一条不可用条目,它后面可能还排着可用的任务,所以要走下面的丢弃+ # 路径,而不是当成"队列结束"直接返回。+ if task_json is None:return None- task_info = json.loads(task_json)- # 将函数名称转换回函数对象- task_info["func"] = FUNC_MAP[task_info["func"]]+ task_info = None+ try:+ task_info = json.loads(task_json)+ # 将函数名称转换回函数对象。名称缺失、或已不在 FUNC_MAP 中时不能+ # 直接索引,否则 KeyError 会绕过下面针对 params 的丢弃策略。+ task_info["func"] = FUNC_MAP[task_info["func"]]+ task_kwargs = task_info["kwargs"]+ if not isinstance(task_kwargs, dict):+ raise ValueError("queued task has no keyword argument mapping")+ # args 整体缺失时沿用 check_queue 的默认值;写成 null 或其它不是+ # 数组的形态则会让 check_queue 展开 `*args` 时抛 TypeError,那里+ # 会把条目重新入队并让异常逃出工作线程,必须在这里先拦下。+ if not isinstance(task_info.get("args", []), list):+ raise ValueError("queued task positional arguments are not a list")+ except (TypeError, ValueError, KeyError) as e:+ logger.error(f"dropping unusable queued task: {e}")+ # 与下面的 params 校验失败路径一致:只要能读出可用的 task_id,就把+ # 这条已经永久离开队列的任务收敛为失败,否则 API/WebUI 会一直显示+ # 它在 processing。payload 本身没法解析、或 task_id 不是字符串+ # (例如 JSON 数组)时则没有可回写的记录,只能丢弃 —— 把非字符串+ # 直接交给 patch_task 会让 redis 抛 DataError,反过来打断丢弃循环。+ stale_kwargs = (+ task_info.get("kwargs") if isinstance(task_info, dict) else None+ )+ task_id = (+ stale_kwargs.get("task_id")+ if isinstance(stale_kwargs, dict)+ else None+ )+ if isinstance(task_id, str) and task_id:+ sm.state.patch_task(+ task_id,+ state=const.TASK_STATE_FAILED,+ failed_stage="dequeue",+ error=f"discarded stale queued task: {e}",+ )+ continue- if "params" in task_info["kwargs"] and isinstance(- task_info["kwargs"]["params"], dict- ):+ if "params" in task_kwargs and isinstance(task_kwargs["params"], dict):try:- task_info["kwargs"]["params"] = VideoParams(- **task_info["kwargs"]["params"]- )+ task_kwargs["params"] = VideoParams(**task_kwargs["params"])except ValidationError as e:logger.error("dropping queued task with params that fail current "@@ -80,8 +116,9 @@ class RedisTaskManager(TaskManager):# 丢弃这条队列项而不动状态记录,API/WebUI 会一直显示任务在# 运行,永远不会变成失败。用 patch_task 而不是 update_task,# 这样如果用户已经删除了这个任务,我们不会又把它建回来。- task_id = task_info["kwargs"].get("task_id")- if task_id:+ # task_id 不是字符串时同上:没有可回写的记录,跳过状态更新。+ task_id = task_kwargs.get("task_id")+ if isinstance(task_id, str) and task_id:sm.state.patch_task(task_id,state=const.TASK_STATE_FAILED,diff --git a/test/services/test_task_manager.py b/test/services/test_task_manager.pyindex 01a2f552..90dfd516 100644--- a/test/services/test_task_manager.py+++ b/test/services/test_task_manager.py@@ -10,6 +10,16 @@ from app.models.schema import VideoParamsfrom app.services import task as task_service+def _queued_payload(func: str, **kwargs) -> str:+ """按 RedisTaskManager.enqueue 的落盘格式构造一条队列条目。"""+ return json.dumps({"func": func, "args": [], "kwargs": kwargs})+++def _video_params() -> dict:+ """返回一份能通过当前校验的 VideoParams 序列化结果。"""+ return VideoParams(video_subject="Tea").model_dump(warnings=False)++class TestInMemoryTaskManager(unittest.TestCase):def test_queue_operations_preserve_task_payload(self):"""内存队列应保持函数、位置参数和关键字参数,不得改变任务内容。"""@@ -379,6 +389,195 @@ class TestRedisTaskManager(unittest.TestCase):state.patch_task.assert_called_once()state.update_task.assert_not_called()+ def test_dequeue_skips_task_with_unknown_function_name(self):+ """+ FUNC_MAP 的成员会随部署变化(本文件里就保留着被注释掉的第二个入口),+ 队列里可能残留旧入口函数的任务。对不在 FUNC_MAP 中的名称直接索引会抛+ KeyError,绕过 dequeue 自己的丢弃策略,因此必须和校验失败路径一样跳过它。+ """+ self.redis_client.lpop.side_effect = [+ _queued_payload("start_test", task_id="task-unknown-func"),+ _queued_payload("start", task_id="task-valid", params=_video_params()),+ ]++ with patch("app.controllers.manager.redis_manager.sm.state") as state:+ state.patch_task.return_value = True+ task = self.manager.dequeue()++ self.assertEqual(self.redis_client.lpop.call_count, 2)+ self.assertIs(task["func"], task_service.start)+ self.assertEqual(task["kwargs"]["task_id"], "task-valid")+ state.patch_task.assert_called_once()+ call_args = state.patch_task.call_args+ self.assertEqual(call_args.args[0], "task-unknown-func")+ self.assertEqual(call_args.kwargs["state"], const.TASK_STATE_FAILED)+ self.assertIn("start_test", call_args.kwargs["error"])++ def test_dequeue_skips_every_unusable_payload_shape(self):+ """+ 队列里可能残留三种不可用条目:写入被截断的 JSON、不是对象的 JSON、以及+ 引用了已移除入口函数(FUNC_MAP 里只留下注释)的任务。它们都不能把异常+ 抛出 dequeue,否则排在后面的可用任务再也不会被调度;其中能读出 task_id+ 的条目还要把已经永久离开队列的任务收敛为失败。+ """+ self.redis_client.lpop.side_effect = [+ '{"func": "start", "args": [',+ json.dumps(["not", "a", "mapping"]),+ _queued_payload("retired-entry", task_id="task-retired"),+ _queued_payload("start", task_id="task-valid", params=_video_params()),+ ]++ with patch("app.controllers.manager.redis_manager.sm.state") as state:+ state.patch_task.return_value = True+ task = self.manager.dequeue()++ self.assertEqual(self.redis_client.lpop.call_count, 4)+ self.assertIs(task["func"], task_service.start)+ self.assertEqual(task["kwargs"]["task_id"], "task-valid")+ state.patch_task.assert_called_once()+ self.assertEqual(state.patch_task.call_args.args[0], "task-retired")+ self.assertEqual(+ state.patch_task.call_args.kwargs["state"], const.TASK_STATE_FAILED+ )++ def test_dequeue_returns_none_when_only_unusable_entries_remain(self):+ """剩余条目全部不可用时,应返回 None 而不是抛出异常。"""+ self.redis_client.lpop.side_effect = [+ _queued_payload("retired-entry"),+ None,+ ]++ with patch("app.controllers.manager.redis_manager.sm.state") as state:+ result = self.manager.dequeue()++ self.assertIsNone(result)+ self.assertEqual(self.redis_client.lpop.call_count, 2)+ state.patch_task.assert_not_called()++ def test_task_done_keeps_draining_queue_when_entry_is_unusable(self):+ """+ 跑完的任务释放名额后由 task_done 触发 check_queue 调度下一个任务。如果+ 不可用条目让 check_queue 抛异常,异常会顺着 run_task 的 finally 把工作+ 线程带崩;此后已没有任务在运行,也就不会有人再调用 check_queue,队列里+ 后面的任务会永久停在 processing。因此 task_done 必须能继续调度下一个+ 可用任务,而不是把整个队列留在原地。+ """+ self.manager.current_tasks = 1+ self.redis_client.lpop.side_effect = [+ _queued_payload("retired-entry"),+ _queued_payload("start", task_id="task-next", params=_video_params()),+ ]++ with patch.object(self.manager, "execute_task") as execute_task:+ self.manager.task_done()++ self.assertEqual(self.manager.current_tasks, 1)+ self.assertEqual(execute_task.call_args.kwargs["task_id"], "task-next")+ self.assertIs(execute_task.call_args.args[0], task_service.start)++ def test_dequeue_discards_payloads_that_cannot_be_dispatched(self):+ """+ 三种能通过解析、却无法派发的条目:lpop 返回空值、args 写成 null / 对象 /+ 数字 / 字符串。它们都不能让 dequeue 抛异常、也不能被当成"队列空了"——+ 否则 check_queue 会重新入队或直接早退,后面排着的可用任务再也调度不到。+ """+ cases = (+ ("empty bytes", b"", None),+ ("empty string", "", None),+ ("args null", {"args": None}, "t-1"),+ ("args mapping", {"args": {}}, "t-2"),+ ("args number", {"args": 5}, "t-3"),+ ("args string", {"args": "x"}, "t-4"),+ )++ for case, broken, expected_task_id in cases:+ with self.subTest(case=case):+ if isinstance(broken, dict):+ broken = json.dumps(+ {+ "func": "start",+ "kwargs": {"task_id": expected_task_id},+ **broken,+ }+ )+ self.redis_client.reset_mock()+ self.redis_client.lpop.side_effect = [+ broken,+ _queued_payload(+ "start", task_id="task-valid", params=_video_params()+ ),+ ]++ with patch("app.controllers.manager.redis_manager.sm.state") as state:+ state.patch_task.return_value = True+ task = self.manager.dequeue()++ self.assertEqual(self.redis_client.lpop.call_count, 2)+ self.assertEqual(task["kwargs"]["task_id"], "task-valid")+ self.assertIs(task["func"], task_service.start)+ if expected_task_id is None:+ state.patch_task.assert_not_called()+ else:+ self.assertEqual(+ state.patch_task.call_args.args[0], expected_task_id+ )+ self.assertIn(+ "positional arguments",+ state.patch_task.call_args.kwargs["error"],+ )++ def test_dequeue_dispatches_payload_without_args_field(self):+ """args 字段整体缺失是既有格式(check_queue 取默认空元组),不能被丢掉。"""+ self.redis_client.lpop.return_value = json.dumps(+ {+ "func": "start",+ "kwargs": {"task_id": "task-no-args", "params": _video_params()},+ }+ )++ with patch("app.controllers.manager.redis_manager.sm.state") as state:+ task = self.manager.dequeue()++ self.assertEqual(task["kwargs"]["task_id"], "task-no-args")+ self.assertIs(task["func"], task_service.start)+ state.patch_task.assert_not_called()++ def test_dequeue_ignores_non_string_task_id_in_every_discard_path(self):+ """+ task_id 是 JSON 数组 / 对象这类非字符串值时,交给 patch_task 会让 redis+ 抛 DataError,把丢弃循环打断在一条坏条目上,后面排着的可用任务因此起不来。+ 两条丢弃路径都要跳过状态回写,只丢弃条目本身。+ """+ cases = (+ ("unknown function", _queued_payload("retired-entry", task_id=["task-1"])),+ (+ "stale params",+ _queued_payload(+ "start",+ task_id={"nested": "task-2"},+ params={**_video_params(), "video_count": 0},+ ),+ ),+ )++ for case, payload in cases:+ with self.subTest(case=case):+ self.redis_client.reset_mock()+ self.redis_client.lpop.side_effect = [+ payload,+ _queued_payload(+ "start", task_id="task-valid", params=_video_params()+ ),+ ]++ with patch("app.controllers.manager.redis_manager.sm.state") as state:+ task = self.manager.dequeue()++ self.assertEqual(self.redis_client.lpop.call_count, 2)+ self.assertEqual(task["kwargs"]["task_id"], "task-valid")+ state.patch_task.assert_not_called()+ state.update_task.assert_not_called()+if __name__ == "__main__":unittest.main()