利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix(redis): admit queued requests atomically across API workers
RudyCelekli41a6c2a97822
diff --git a/app/controllers/manager/base_manager.py b/app/controllers/manager/base_manager.pyindex 079767de..a2187df4 100644--- a/app/controllers/manager/base_manager.py+++ b/app/controllers/manager/base_manager.py@@ -81,7 +81,7 @@ class TaskManager:f"enqueue task: {func.__name__}, current_tasks: {self.current_tasks}, "f"queue_size: {queue_size}")- self.enqueue({"func": func, "args": args, "kwargs": kwargs})+ self.enqueue_new_task({"func": func, "args": args, "kwargs": kwargs})def execute_task(self, func: Callable, *args: Any, **kwargs: Any):thread = threading.Thread(@@ -132,6 +132,10 @@ class TaskManager:def enqueue(self, task: Dict):raise NotImplementedError()+ def enqueue_new_task(self, task: Dict):+ """Admit new work; shared queues may need their own atomic limit check."""+ self.enqueue(task)+def dequeue(self):raise NotImplementedError()diff --git a/app/controllers/manager/redis_manager.py b/app/controllers/manager/redis_manager.pyindex 84476b18..af7b7681 100644--- a/app/controllers/manager/redis_manager.py+++ b/app/controllers/manager/redis_manager.py@@ -6,7 +6,7 @@ import redisfrom loguru import loggerfrom pydantic import ValidationError-from app.controllers.manager.base_manager import TaskManager+from app.controllers.manager.base_manager import TaskManager, TaskQueueFullErrorfrom app.models import constfrom app.models.schema import VideoParamsfrom app.services import state as sm@@ -17,6 +17,14 @@ FUNC_MAP = {# 'start_test': tm.start_test}+_ADMIT_QUEUED_TASK_SCRIPT = """+if redis.call("LLEN", KEYS[1]) >= tonumber(ARGV[1]) then+ return 0+end+redis.call("RPUSH", KEYS[1], ARGV[2])+return 1+"""+class RedisTaskManager(TaskManager):def __init__(@@ -42,6 +50,23 @@ class RedisTaskManager(TaskManager):self.check_queue()def enqueue(self, task: Dict):+ self.redis_client.rpush(self.queue, self._serialize_task(task))++ def enqueue_new_task(self, task: Dict):+ # Manager locks are process-local. Keep capacity validation and admission+ # in one Redis operation so other API workers cannot claim the same slot.+ admitted = self.redis_client.eval(+ _ADMIT_QUEUED_TASK_SCRIPT,+ 1,+ self.queue,+ self.max_queued_tasks,+ self._serialize_task(task),+ )+ if not admitted:+ raise TaskQueueFullError("task queue is full, please try again later")++ @staticmethod+ def _serialize_task(task: Dict) -> str:task_with_serializable_params = task.copy()# task.copy() 只复制最外层字典;如果直接改写嵌套 kwargs,会把调用方# 持有的 VideoParams 同步替换成 dict。后续日志或重试仍可能读取原任务,@@ -56,7 +81,7 @@ class RedisTaskManager(TaskManager):# 将函数对象转换为其名称task_with_serializable_params["func"] = task["func"].__name__- self.redis_client.rpush(self.queue, json.dumps(task_with_serializable_params))+ return json.dumps(task_with_serializable_params)def dequeue(self):# 循环而非单次弹出:某个任务在入队时可能满足当时的校验规则,但校验规则与diff --git a/test/services/test_redis_queue_admission.py b/test/services/test_redis_queue_admission.pynew file mode 100644index 00000000..1809e1e8--- /dev/null+++ b/test/services/test_redis_queue_admission.py@@ -0,0 +1,87 @@+"""Queue capacity is shared by every manager using the Redis list."""++import os+import threading+import unittest+from concurrent.futures import ThreadPoolExecutor+from unittest.mock import patch+from uuid import uuid4++import redis++from app.controllers.manager.base_manager import TaskQueueFullError+from app.controllers.manager.redis_manager import RedisTaskManager+from app.models.schema import VideoParams+from app.services import task as tm+++class TestRedisQueueAdmission(unittest.TestCase):+ def setUp(self):+ host = os.getenv("MPT_TEST_REDIS_HOST", "127.0.0.1")+ port = int(os.getenv("MPT_TEST_REDIS_PORT", "6379"))+ db = int(os.getenv("MPT_TEST_REDIS_DB", "15"))+ self.managers = [+ RedisTaskManager(0, f"redis://{host}:{port}/{db}", max_queued_tasks=1)+ for _ in range(2)+ ]+ try:+ self.managers[0].redis_client.ping()+ except redis.RedisError as exc:+ self.skipTest(f"Redis is unavailable: {exc}")+ queue = f"test-admission:{uuid4()}"+ for manager in self.managers:+ manager.queue = queue+ self.addCleanup(manager.redis_client.close)+ self.addCleanup(self.managers[0].redis_client.delete, queue)++ def test_simultaneous_managers_cannot_exceed_shared_capacity(self):+ # Hold both callers after their real LLEN reads. Their local locks are+ # independent, as they are in separate API processes sharing Redis.+ barrier = threading.Barrier(2)++ def add(manager):+ original = manager.queue_size++ def synchronized_size():+ size = original()+ barrier.wait(timeout=5)+ return size++ with patch.object(manager, "queue_size", synchronized_size):+ try:+ manager.add_task(+ tm.start,+ task_id=str(uuid4()),+ params=VideoParams(video_subject="Coffee"),+ )+ except TaskQueueFullError:+ return "rejected"+ return "queued"++ with ThreadPoolExecutor(max_workers=2) as pool:+ results = list(pool.map(add, self.managers))+ self.assertCountEqual(results, ["queued", "rejected"])+ self.assertEqual(self.managers[0].queue_size(), 1)++ def test_worker_start_failure_can_restore_previously_admitted_task(self):+ manager = self.managers[0]+ manager.max_concurrent_tasks = 1+ manager.enqueue({"func": tm.start, "args": (), "kwargs": {+ "task_id": "original", "params": VideoParams(video_subject="Coffee")+ }})++ def failed_start(*args, **kwargs):+ # A second process admits a fresh task after the first pop. Restoring+ # the old accepted work must remain possible even at the new limit.+ self.managers[1].add_task(+ tm.start, task_id="new", params=VideoParams(video_subject="Tea")+ )+ raise RuntimeError("can't start new thread")++ with patch.object(manager, "execute_task", side_effect=failed_start):+ with self.assertRaisesRegex(RuntimeError, "can't start new thread"):+ manager.check_queue()+ self.assertEqual(manager.current_tasks, 0)+ self.assertEqual(manager.queue_size(), 2)+ restored = [manager.dequeue()["kwargs"]["task_id"] for _ in range(2)]+ self.assertCountEqual(restored, ["original", "new"])