Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(redis): admit queued requests atomically across API workers

RudyCelekli41a6c2a97822
diff --git a/app/controllers/manager/base_manager.py b/app/controllers/manager/base_manager.py
index 079767de..a2187df4 100644
--- a/app/controllers/manager/base_manager.py
+++ b/app/controllers/manager/base_manager.py
@@ -81,7 +81,7 @@ class TaskManager:
f"enqueue task: {func.__name__}, current_tasks: {self.current_tasks}, "
f"queue_size: {queue_size}"
)
- self.enqueue({"func": func, "args": args, "kwargs": kwargs})
+ self.enqueue_new_task({"func": func, "args": args, "kwargs": kwargs})
def execute_task(self, func: Callable, *args: Any, **kwargs: Any):
thread = threading.Thread(
@@ -132,6 +132,10 @@ class TaskManager:
def enqueue(self, task: Dict):
raise NotImplementedError()
+ def enqueue_new_task(self, task: Dict):
+ """Admit new work; shared queues may need their own atomic limit check."""
+ self.enqueue(task)
+
def dequeue(self):
raise NotImplementedError()
diff --git a/app/controllers/manager/redis_manager.py b/app/controllers/manager/redis_manager.py
index 84476b18..af7b7681 100644
--- a/app/controllers/manager/redis_manager.py
+++ b/app/controllers/manager/redis_manager.py
@@ -6,7 +6,7 @@ import redis
from loguru import logger
from pydantic import ValidationError
-from app.controllers.manager.base_manager import TaskManager
+from app.controllers.manager.base_manager import TaskManager, TaskQueueFullError
from app.models import const
from app.models.schema import VideoParams
from app.services import state as sm
@@ -17,6 +17,14 @@ FUNC_MAP = {
# 'start_test': tm.start_test
}
+_ADMIT_QUEUED_TASK_SCRIPT = """
+if redis.call("LLEN", KEYS[1]) >= tonumber(ARGV[1]) then
+ return 0
+end
+redis.call("RPUSH", KEYS[1], ARGV[2])
+return 1
+"""
+
class RedisTaskManager(TaskManager):
def __init__(
@@ -42,6 +50,23 @@ class RedisTaskManager(TaskManager):
self.check_queue()
def enqueue(self, task: Dict):
+ self.redis_client.rpush(self.queue, self._serialize_task(task))
+
+ def enqueue_new_task(self, task: Dict):
+ # Manager locks are process-local. Keep capacity validation and admission
+ # in one Redis operation so other API workers cannot claim the same slot.
+ admitted = self.redis_client.eval(
+ _ADMIT_QUEUED_TASK_SCRIPT,
+ 1,
+ self.queue,
+ self.max_queued_tasks,
+ self._serialize_task(task),
+ )
+ if not admitted:
+ raise TaskQueueFullError("task queue is full, please try again later")
+
+ @staticmethod
+ def _serialize_task(task: Dict) -> str:
task_with_serializable_params = task.copy()
# task.copy() 只复制最外层字典;如果直接改写嵌套 kwargs,会把调用方
# 持有的 VideoParams 同步替换成 dict。后续日志或重试仍可能读取原任务,
@@ -56,7 +81,7 @@ class RedisTaskManager(TaskManager):
# 将函数对象转换为其名称
task_with_serializable_params["func"] = task["func"].__name__
- self.redis_client.rpush(self.queue, json.dumps(task_with_serializable_params))
+ return json.dumps(task_with_serializable_params)
def dequeue(self):
# 循环而非单次弹出:某个任务在入队时可能满足当时的校验规则,但校验规则与
diff --git a/test/services/test_redis_queue_admission.py b/test/services/test_redis_queue_admission.py
new file mode 100644
index 00000000..1809e1e8
--- /dev/null
+++ b/test/services/test_redis_queue_admission.py
@@ -0,0 +1,87 @@
+"""Queue capacity is shared by every manager using the Redis list."""
+
+import os
+import threading
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from unittest.mock import patch
+from uuid import uuid4
+
+import redis
+
+from app.controllers.manager.base_manager import TaskQueueFullError
+from app.controllers.manager.redis_manager import RedisTaskManager
+from app.models.schema import VideoParams
+from app.services import task as tm
+
+
+class TestRedisQueueAdmission(unittest.TestCase):
+ def setUp(self):
+ host = os.getenv("MPT_TEST_REDIS_HOST", "127.0.0.1")
+ port = int(os.getenv("MPT_TEST_REDIS_PORT", "6379"))
+ db = int(os.getenv("MPT_TEST_REDIS_DB", "15"))
+ self.managers = [
+ RedisTaskManager(0, f"redis://{host}:{port}/{db}", max_queued_tasks=1)
+ for _ in range(2)
+ ]
+ try:
+ self.managers[0].redis_client.ping()
+ except redis.RedisError as exc:
+ self.skipTest(f"Redis is unavailable: {exc}")
+ queue = f"test-admission:{uuid4()}"
+ for manager in self.managers:
+ manager.queue = queue
+ self.addCleanup(manager.redis_client.close)
+ self.addCleanup(self.managers[0].redis_client.delete, queue)
+
+ def test_simultaneous_managers_cannot_exceed_shared_capacity(self):
+ # Hold both callers after their real LLEN reads. Their local locks are
+ # independent, as they are in separate API processes sharing Redis.
+ barrier = threading.Barrier(2)
+
+ def add(manager):
+ original = manager.queue_size
+
+ def synchronized_size():
+ size = original()
+ barrier.wait(timeout=5)
+ return size
+
+ with patch.object(manager, "queue_size", synchronized_size):
+ try:
+ manager.add_task(
+ tm.start,
+ task_id=str(uuid4()),
+ params=VideoParams(video_subject="Coffee"),
+ )
+ except TaskQueueFullError:
+ return "rejected"
+ return "queued"
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(pool.map(add, self.managers))
+ self.assertCountEqual(results, ["queued", "rejected"])
+ self.assertEqual(self.managers[0].queue_size(), 1)
+
+ def test_worker_start_failure_can_restore_previously_admitted_task(self):
+ manager = self.managers[0]
+ manager.max_concurrent_tasks = 1
+ manager.enqueue({"func": tm.start, "args": (), "kwargs": {
+ "task_id": "original", "params": VideoParams(video_subject="Coffee")
+ }})
+
+ def failed_start(*args, **kwargs):
+ # A second process admits a fresh task after the first pop. Restoring
+ # the old accepted work must remain possible even at the new limit.
+ self.managers[1].add_task(
+ tm.start, task_id="new", params=VideoParams(video_subject="Tea")
+ )
+ raise RuntimeError("can't start new thread")
+
+ with patch.object(manager, "execute_task", side_effect=failed_start):
+ with self.assertRaisesRegex(RuntimeError, "can't start new thread"):
+ manager.check_queue()
+ self.assertEqual(manager.current_tasks, 0)
+ self.assertEqual(manager.queue_size(), 2)
+ restored = [manager.dequeue()["kwargs"]["task_id"] for _ in range(2)]
+ self.assertCountEqual(restored, ["original", "new"])