利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix(state): stabilize Redis task traversal
harry07038e259e9f072c
diff --git a/app/services/state.py b/app/services/state.pyindex 30470963..79a6a312 100644--- a/app/services/state.py+++ b/app/services/state.py@@ -34,6 +34,11 @@ class BaseState(ABC):def get_all_tasks(self, page: int, page_size: int):pass+ @abstractmethod+ def list_task_ids(self, scan_count: int = 100) -> list[str]:+ """获取一次遍历中的任务 ID,供启动恢复等全量操作使用。"""+ pass+@abstractmethoddef patch_task(self, task_id: str, **kwargs) -> bool:"""只更新已有任务的指定字段;任务不存在时返回 False。"""@@ -54,6 +59,10 @@ class MemoryState(BaseState):total = len(tasks)return tasks[start:end], total+ def list_task_ids(self, scan_count: int = 100) -> list[str]:+ with self._lock:+ return list(self._tasks)+def update_task(self,task_id: str,@@ -114,41 +123,35 @@ class RedisState(BaseState):def get_all_tasks(self, page: int, page_size: int):start = (page - 1) * page_sizeend = start + page_size+ # 每一页都以同一套确定性顺序切片,而不是依赖 SCAN 的返回顺序。+ # 这仍不是并发增删任务时的事务快照;无索引时优先保证静态任务集+ # 的分页正确性,并让 total 统计去重后的任务键。+ task_ids = self.list_task_ids(scan_count=page_size)tasks = []+ for task_id in task_ids[start:end]:+ task = self.get_task(task_id)+ if task is not None:+ tasks.append(task)+ return tasks, len(task_ids)++ def list_task_ids(self, scan_count: int = 100) -> list[str]:+ """一次完整扫描中按任务键去重,避免 SCAN 重复返回同一条记录。"""+ task_keys = set()cursor = 0- total = 0while True:# Redis 数据库中除了任务 Hash,还可能存在 RedisTaskManager 使用的# List 队列。只扫描 Hash 可以避免对队列执行 HGETALL 时触发- # WRONGTYPE,同时保证 total 只统计真正的任务记录。+ # WRONGTYPE。COUNT 是扫描工作量提示,不保证每批返回的数量。cursor, keys = self._redis.scan(cursor,- count=page_size,+ count=scan_count,_type="HASH",)- batch_start = total- batch_size = len(keys)- total += batch_size-- # Redis SCAN 是分批返回 key。分页切片必须基于“当前批次起始索引”- # 计算,而不能用累积后的 total 反推,否则第一页会切到空数组,- # 第二页也可能只返回部分数据。- if batch_start < end and total > start:- slice_start = max(0, start - batch_start)- slice_end = min(batch_size, end - batch_start)- for key in keys[slice_start:slice_end]:- task_data = self._redis.hgetall(key)- task = {- k.decode("utf-8"): self._convert_to_original_type(v)- for k, v in task_data.items()- }- tasks.append(task)-- # 即使当前页已经取满,也要继续 SCAN 到 cursor=0,- # 因为调用方需要准确 total 来渲染分页信息。+ task_keys.update(keys)if cursor == 0:break- return tasks, total+ # 按任务键排序不依赖 Hash 扫描顺序;不额外维护索引,也不改变旧任务。+ return [key.decode("utf-8") for key in sorted(task_keys)]def update_task(self,diff --git a/app/services/task.py b/app/services/task.pyindex 93ef3901..29b01a59 100644--- a/app/services/task.py+++ b/app/services/task.py@@ -1104,23 +1104,19 @@ def recover_interrupted_cross_posts(page_size: int = 100) -> int | None:跨平台发布使用当前进程内的线程池,不是持久化任务队列。进程启动时,Redis 中残留的 pending/processing 不会自动继续执行;如果继续把它们视为- 运行中,用户将永久无法删除任务。这里分页扫描状态,只处理当前进程没有- 对应 Future 的活动记录,并保留已经生成的视频结果。+ 运行中,用户将永久无法删除任务。这里仅收集一次任务 ID,再读取当前状态;+ 不能跨多次 Redis SCAN 按页码推进,否则扫描顺序变化会漏掉遗留任务。+ 只处理当前进程没有对应 Future 的活动记录,并保留已经生成的视频结果。"""recovered = 0- page = 1-- while True:- try:- tasks, total = sm.state.get_all_tasks(page, page_size)- except Exception as exc:- logger.exception(f"failed to recover interrupted cross-post tasks: {exc}")- return None-- for task in tasks:- task_id = str(task.get("task_id") or "")+ try:+ task_ids = sm.state.list_task_ids(scan_count=page_size)+ for task_id in task_ids:+ # 扫描之后任务可能已被删除或转为终态;以最新状态决定是否恢复,+ # 不能使用扫描时的旧快照覆盖已经完成的发布结果。+ task = sm.state.get_task(task_id)if (- not task_id+ not taskor task.get("cross_post_state") not in _ACTIVE_CROSS_POST_STATESor _is_cross_post_active_in_process(task_id)or _is_cross_post_owner_alive(task.get("cross_post_owner"))@@ -1135,10 +1131,9 @@ def recover_interrupted_cross_posts(page_size: int = 100) -> int | None:)if updated is True:recovered += 1-- if page * page_size >= total or not tasks:- break- page += 1+ except Exception as exc:+ logger.exception(f"failed to recover interrupted cross-post tasks: {exc}")+ return Noneif recovered:logger.warning(f"recovered interrupted cross-post tasks: {recovered}")diff --git a/test/services/test_state.py b/test/services/test_state.pyindex 05347b01..cf07f4d5 100644--- a/test/services/test_state.py+++ b/test/services/test_state.py@@ -163,11 +163,10 @@ class TestRedisState(unittest.TestCase):def test_get_all_tasks_paginates_across_scan_batches(self):"""- Redis SCAN 分批返回 key 时,分页切片必须按当前批次起始位置计算。+ Redis SCAN 分批返回 key 时,分页必须按任务键的稳定顺序切片。这个用例复现 PR #890 描述的 18 条任务、page_size=10 场景:- 第一批 10 条,第二批 8 条。旧逻辑第一页会返回空列表,第二页- 只返回 2 条;修复后第一页返回 10 条,第二页返回剩余 8 条。+ 第一批 10 条,第二批 8 条;两页合起来应完整覆盖全部任务。"""state = self._build_state([10, 8])@@ -178,17 +177,42 @@ class TestRedisState(unittest.TestCase):self.assertEqual(second_total, 18)self.assertEqual(len(first_page), 10)self.assertEqual(len(second_page), 8)+ expected_ids = sorted(f"task:{i}" for i in range(18))self.assertEqual([task["task_id"] for task in first_page],- [f"task:{i}" for i in range(10)],+ expected_ids[:10],)self.assertEqual([task["task_id"] for task in second_page],- [f"task:{i}" for i in range(10, 18)],+ expected_ids[10:],)self.assertTrue(state._redis.scan_types)self.assertEqual(set(state._redis.scan_types), {"HASH"})+ def test_get_all_tasks_deduplicates_and_stabilizes_scan_order(self):+ """两次独立扫描顺序不同、单次扫描重复返回键时仍不重不漏。"""+ state = self._build_state([3])+ state._redis.batches = [+ [b"task:1", b"task:0"],+ [],+ [b"task:2", b"task:1"],+ ]+ first_page, first_total = state.get_all_tasks(page=1, page_size=2)++ state._redis.batches = [+ [b"task:2", b"task:1"],+ [b"task:0", b"task:2"],+ ]+ second_page, second_total = state.get_all_tasks(page=2, page_size=2)++ self.assertEqual(first_total, 3)+ self.assertEqual(second_total, 3)+ self.assertEqual(+ [task["task_id"] for task in first_page + second_page],+ ["task:0", "task:1", "task:2"],+ )+ self.assertEqual(state.list_task_ids(scan_count=1), ["task:0", "task:1", "task:2"])+@unittest.skipUnless(os.getenv("MPT_TEST_REDIS_HOST"),"MPT_TEST_REDIS_HOST not set",@@ -221,6 +245,38 @@ class TestRedisState(unittest.TestCase):finally:state._redis.delete(queue_key, *task_ids)+ @unittest.skipUnless(+ os.getenv("MPT_TEST_REDIS_HOST"),+ "MPT_TEST_REDIS_HOST not set",+ )+ def test_real_redis_pagination_keeps_a_stable_task_order(self):+ """真实 Redis 的多批 SCAN 不应让跨页任务重复或丢失。"""+ state = RedisState(+ host=os.environ["MPT_TEST_REDIS_HOST"],+ port=int(os.getenv("MPT_TEST_REDIS_PORT", "6379")),+ db=int(os.getenv("MPT_TEST_REDIS_DB", "15")),+ )+ task_ids = [f"ci-page-{uuid.uuid4()}-{index}" for index in range(13)]+ try:+ for task_id in reversed(task_ids):+ state.update_task(task_id, state=const.TASK_STATE_COMPLETE)++ all_tasks = []+ page = 1+ while True:+ tasks, total = state.get_all_tasks(page=page, page_size=3)+ all_tasks.extend(tasks)+ if page * 3 >= total:+ break+ page += 1++ returned_ids = [task["task_id"] for task in all_tasks if "task_id" in task]+ self.assertEqual(len(returned_ids), len(set(returned_ids)))+ self.assertEqual(returned_ids, sorted(returned_ids))+ self.assertTrue(set(task_ids).issubset(returned_ids))+ finally:+ state._redis.delete(*task_ids)+def test_patch_task_updates_only_existing_redis_task(self):state = self._build_state([1])diff --git a/test/services/test_task.py b/test/services/test_task.pyindex 256e3645..fc960403 100644--- a/test/services/test_task.py+++ b/test/services/test_task.py@@ -2021,6 +2021,43 @@ class TestTaskService(unittest.TestCase):)active_future.set_result(None)+ def test_recover_interrupted_cross_posts_scans_ids_only_once(self):+ """启动恢复不能依赖多次独立扫描的分页顺序。"""+ state = MemoryState()+ for task_id, cross_post_state in (+ ("stale-pending", tm.const.CROSS_POST_STATE_PENDING),+ ("stale-processing", tm.const.CROSS_POST_STATE_PROCESSING),+ ("already-complete", tm.const.CROSS_POST_STATE_COMPLETE),+ ):+ state.update_task(+ task_id,+ state=tm.const.TASK_STATE_COMPLETE,+ progress=100,+ videos=["final.mp4"],+ cross_post_state=cross_post_state,+ )++ with (+ patch.object(tm.sm, "state", state),+ patch.object(state, "list_task_ids", wraps=state.list_task_ids) as list_ids,+ patch.object(+ state, "get_all_tasks", side_effect=AssertionError("pagination used")+ ) as get_all_tasks,+ ):+ recovered = tm.recover_interrupted_cross_posts(page_size=1)++ self.assertEqual(recovered, 2)+ list_ids.assert_called_once_with(scan_count=1)+ get_all_tasks.assert_not_called()+ for task_id in ("stale-pending", "stale-processing"):+ task = state.get_task(task_id)+ self.assertEqual(task["cross_post_state"], tm.const.CROSS_POST_STATE_FAILED)+ self.assertEqual(task["videos"], ["final.mp4"])+ self.assertEqual(+ state.get_task("already-complete")["cross_post_state"],+ tm.const.CROSS_POST_STATE_COMPLETE,+ )+def test_cross_post_owner_uses_future_registry_for_current_process(self):"""当前进程无活动 Future 时,同 PID 的新旧 owner 都应视为中断。"""stale_owner = f"{tm.socket.gethostname()}:{tm.os.getpid()}:old-instance"@@ -2093,7 +2130,7 @@ class TestTaskService(unittest.TestCase):def test_cross_post_recovery_reports_state_backend_failure(self):"""启动恢复读取状态失败时应返回 None,允许 WebUI 后续 rerun 重试。"""state = MagicMock()- state.get_all_tasks.side_effect = RuntimeError("redis unavailable")+ state.list_task_ids.side_effect = RuntimeError("redis unavailable")with (patch.object(tm.sm, "state", state),@@ -2135,33 +2172,49 @@ class TestTaskService(unittest.TestCase):"MPT_TEST_REDIS_HOST not set",)def test_real_redis_recovers_interrupted_cross_post_state(self):- """真实 Redis 中的遗留发布状态必须在恢复后保留视频并进入失败终态。"""+ """真实 Redis 的多批扫描应恢复全部遗留发布状态并保留视频。"""state = RedisState(host=os.environ["MPT_TEST_REDIS_HOST"],port=int(os.getenv("MPT_TEST_REDIS_PORT", "6379")),db=int(os.getenv("MPT_TEST_REDIS_DB", "15")),)- task_id = f"ci-cross-post-recovery-{uuid4()}"- state.update_task(- task_id,- state=tm.const.TASK_STATE_COMPLETE,- progress=100,- videos=["final.mp4"],- cross_post_state=tm.const.CROSS_POST_STATE_PROCESSING,- cross_post_owner="",- )+ task_ids = [f"ci-cross-post-recovery-{uuid4()}" for _ in range(3)]+ for task_id, cross_post_state in zip(+ task_ids,+ (+ tm.const.CROSS_POST_STATE_PENDING,+ tm.const.CROSS_POST_STATE_PROCESSING,+ tm.const.CROSS_POST_STATE_COMPLETE,+ ),+ ):+ state.update_task(+ task_id,+ state=tm.const.TASK_STATE_COMPLETE,+ progress=100,+ videos=["final.mp4"],+ cross_post_state=cross_post_state,+ cross_post_owner="",+ )try:with patch.object(tm.sm, "state", state):- recovered = tm.recover_interrupted_cross_posts(page_size=10)+ recovered = tm.recover_interrupted_cross_posts(page_size=1)- self.assertGreaterEqual(recovered, 1)- task = state.get_task(task_id)- self.assertEqual(task["videos"], ["final.mp4"])- self.assertEqual(task["cross_post_state"], tm.const.CROSS_POST_STATE_FAILED)- self.assertEqual(task["cross_post_error"], tm._INTERRUPTED_CROSS_POST_ERROR)+ self.assertGreaterEqual(recovered, 2)+ for task_id in task_ids[:2]:+ task = state.get_task(task_id)+ self.assertEqual(task["videos"], ["final.mp4"])+ self.assertEqual(task["cross_post_state"], tm.const.CROSS_POST_STATE_FAILED)+ self.assertEqual(+ task["cross_post_error"], tm._INTERRUPTED_CROSS_POST_ERROR+ )+ self.assertEqual(+ state.get_task(task_ids[2])["cross_post_state"],+ tm.const.CROSS_POST_STATE_COMPLETE,+ )finally:- state.delete_task(task_id)+ for task_id in task_ids:+ state.delete_task(task_id)def test_cross_post_future_exception_is_observed(self):"""线程池自身抛出的异常必须进入日志,不能留在无人读取的 Future 中。"""