Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(state): stabilize Redis task traversal

harry07038e259e9f072c
diff --git a/app/services/state.py b/app/services/state.py
index 30470963..79a6a312 100644
--- a/app/services/state.py
+++ b/app/services/state.py
@@ -34,6 +34,11 @@ class BaseState(ABC):
def get_all_tasks(self, page: int, page_size: int):
pass
+ @abstractmethod
+ def list_task_ids(self, scan_count: int = 100) -> list[str]:
+ """获取一次遍历中的任务 ID,供启动恢复等全量操作使用。"""
+ pass
+
@abstractmethod
def patch_task(self, task_id: str, **kwargs) -> bool:
"""只更新已有任务的指定字段;任务不存在时返回 False。"""
@@ -54,6 +59,10 @@ class MemoryState(BaseState):
total = len(tasks)
return tasks[start:end], total
+ def list_task_ids(self, scan_count: int = 100) -> list[str]:
+ with self._lock:
+ return list(self._tasks)
+
def update_task(
self,
task_id: str,
@@ -114,41 +123,35 @@ class RedisState(BaseState):
def get_all_tasks(self, page: int, page_size: int):
start = (page - 1) * page_size
end = start + page_size
+ # 每一页都以同一套确定性顺序切片,而不是依赖 SCAN 的返回顺序。
+ # 这仍不是并发增删任务时的事务快照;无索引时优先保证静态任务集
+ # 的分页正确性,并让 total 统计去重后的任务键。
+ task_ids = self.list_task_ids(scan_count=page_size)
tasks = []
+ for task_id in task_ids[start:end]:
+ task = self.get_task(task_id)
+ if task is not None:
+ tasks.append(task)
+ return tasks, len(task_ids)
+
+ def list_task_ids(self, scan_count: int = 100) -> list[str]:
+ """一次完整扫描中按任务键去重,避免 SCAN 重复返回同一条记录。"""
+ task_keys = set()
cursor = 0
- total = 0
while True:
# Redis 数据库中除了任务 Hash,还可能存在 RedisTaskManager 使用的
# List 队列。只扫描 Hash 可以避免对队列执行 HGETALL 时触发
- # WRONGTYPE,同时保证 total 只统计真正的任务记录。
+ # WRONGTYPE。COUNT 是扫描工作量提示,不保证每批返回的数量。
cursor, keys = self._redis.scan(
cursor,
- count=page_size,
+ count=scan_count,
_type="HASH",
)
- batch_start = total
- batch_size = len(keys)
- total += batch_size
-
- # Redis SCAN 是分批返回 key。分页切片必须基于“当前批次起始索引”
- # 计算,而不能用累积后的 total 反推,否则第一页会切到空数组,
- # 第二页也可能只返回部分数据。
- if batch_start < end and total > start:
- slice_start = max(0, start - batch_start)
- slice_end = min(batch_size, end - batch_start)
- for key in keys[slice_start:slice_end]:
- task_data = self._redis.hgetall(key)
- task = {
- k.decode("utf-8"): self._convert_to_original_type(v)
- for k, v in task_data.items()
- }
- tasks.append(task)
-
- # 即使当前页已经取满,也要继续 SCAN 到 cursor=0,
- # 因为调用方需要准确 total 来渲染分页信息。
+ task_keys.update(keys)
if cursor == 0:
break
- return tasks, total
+ # 按任务键排序不依赖 Hash 扫描顺序;不额外维护索引,也不改变旧任务。
+ return [key.decode("utf-8") for key in sorted(task_keys)]
def update_task(
self,
diff --git a/app/services/task.py b/app/services/task.py
index 93ef3901..29b01a59 100644
--- a/app/services/task.py
+++ b/app/services/task.py
@@ -1104,23 +1104,19 @@ def recover_interrupted_cross_posts(page_size: int = 100) -> int | None:
跨平台发布使用当前进程内的线程池,不是持久化任务队列。进程启动时,
Redis 中残留的 pending/processing 不会自动继续执行;如果继续把它们视为
- 运行中,用户将永久无法删除任务。这里分页扫描状态,只处理当前进程没有
- 对应 Future 的活动记录,并保留已经生成的视频结果。
+ 运行中,用户将永久无法删除任务。这里仅收集一次任务 ID,再读取当前状态;
+ 不能跨多次 Redis SCAN 按页码推进,否则扫描顺序变化会漏掉遗留任务。
+ 只处理当前进程没有对应 Future 的活动记录,并保留已经生成的视频结果。
"""
recovered = 0
- page = 1
-
- while True:
- try:
- tasks, total = sm.state.get_all_tasks(page, page_size)
- except Exception as exc:
- logger.exception(f"failed to recover interrupted cross-post tasks: {exc}")
- return None
-
- for task in tasks:
- task_id = str(task.get("task_id") or "")
+ try:
+ task_ids = sm.state.list_task_ids(scan_count=page_size)
+ for task_id in task_ids:
+ # 扫描之后任务可能已被删除或转为终态;以最新状态决定是否恢复,
+ # 不能使用扫描时的旧快照覆盖已经完成的发布结果。
+ task = sm.state.get_task(task_id)
if (
- not task_id
+ not task
or task.get("cross_post_state") not in _ACTIVE_CROSS_POST_STATES
or _is_cross_post_active_in_process(task_id)
or _is_cross_post_owner_alive(task.get("cross_post_owner"))
@@ -1135,10 +1131,9 @@ def recover_interrupted_cross_posts(page_size: int = 100) -> int | None:
)
if updated is True:
recovered += 1
-
- if page * page_size >= total or not tasks:
- break
- page += 1
+ except Exception as exc:
+ logger.exception(f"failed to recover interrupted cross-post tasks: {exc}")
+ return None
if recovered:
logger.warning(f"recovered interrupted cross-post tasks: {recovered}")
diff --git a/test/services/test_state.py b/test/services/test_state.py
index 05347b01..cf07f4d5 100644
--- a/test/services/test_state.py
+++ b/test/services/test_state.py
@@ -163,11 +163,10 @@ class TestRedisState(unittest.TestCase):
def test_get_all_tasks_paginates_across_scan_batches(self):
"""
- Redis SCAN 分批返回 key 时,分页切片必须按当前批次起始位置计算。
+ Redis SCAN 分批返回 key 时,分页必须按任务键的稳定顺序切片。
这个用例复现 PR #890 描述的 18 条任务、page_size=10 场景:
- 第一批 10 条,第二批 8 条。旧逻辑第一页会返回空列表,第二页
- 只返回 2 条;修复后第一页返回 10 条,第二页返回剩余 8 条。
+ 第一批 10 条,第二批 8 条;两页合起来应完整覆盖全部任务。
"""
state = self._build_state([10, 8])
@@ -178,17 +177,42 @@ class TestRedisState(unittest.TestCase):
self.assertEqual(second_total, 18)
self.assertEqual(len(first_page), 10)
self.assertEqual(len(second_page), 8)
+ expected_ids = sorted(f"task:{i}" for i in range(18))
self.assertEqual(
[task["task_id"] for task in first_page],
- [f"task:{i}" for i in range(10)],
+ expected_ids[:10],
)
self.assertEqual(
[task["task_id"] for task in second_page],
- [f"task:{i}" for i in range(10, 18)],
+ expected_ids[10:],
)
self.assertTrue(state._redis.scan_types)
self.assertEqual(set(state._redis.scan_types), {"HASH"})
+ def test_get_all_tasks_deduplicates_and_stabilizes_scan_order(self):
+ """两次独立扫描顺序不同、单次扫描重复返回键时仍不重不漏。"""
+ state = self._build_state([3])
+ state._redis.batches = [
+ [b"task:1", b"task:0"],
+ [],
+ [b"task:2", b"task:1"],
+ ]
+ first_page, first_total = state.get_all_tasks(page=1, page_size=2)
+
+ state._redis.batches = [
+ [b"task:2", b"task:1"],
+ [b"task:0", b"task:2"],
+ ]
+ second_page, second_total = state.get_all_tasks(page=2, page_size=2)
+
+ self.assertEqual(first_total, 3)
+ self.assertEqual(second_total, 3)
+ self.assertEqual(
+ [task["task_id"] for task in first_page + second_page],
+ ["task:0", "task:1", "task:2"],
+ )
+ self.assertEqual(state.list_task_ids(scan_count=1), ["task:0", "task:1", "task:2"])
+
@unittest.skipUnless(
os.getenv("MPT_TEST_REDIS_HOST"),
"MPT_TEST_REDIS_HOST not set",
@@ -221,6 +245,38 @@ class TestRedisState(unittest.TestCase):
finally:
state._redis.delete(queue_key, *task_ids)
+ @unittest.skipUnless(
+ os.getenv("MPT_TEST_REDIS_HOST"),
+ "MPT_TEST_REDIS_HOST not set",
+ )
+ def test_real_redis_pagination_keeps_a_stable_task_order(self):
+ """真实 Redis 的多批 SCAN 不应让跨页任务重复或丢失。"""
+ state = RedisState(
+ host=os.environ["MPT_TEST_REDIS_HOST"],
+ port=int(os.getenv("MPT_TEST_REDIS_PORT", "6379")),
+ db=int(os.getenv("MPT_TEST_REDIS_DB", "15")),
+ )
+ task_ids = [f"ci-page-{uuid.uuid4()}-{index}" for index in range(13)]
+ try:
+ for task_id in reversed(task_ids):
+ state.update_task(task_id, state=const.TASK_STATE_COMPLETE)
+
+ all_tasks = []
+ page = 1
+ while True:
+ tasks, total = state.get_all_tasks(page=page, page_size=3)
+ all_tasks.extend(tasks)
+ if page * 3 >= total:
+ break
+ page += 1
+
+ returned_ids = [task["task_id"] for task in all_tasks if "task_id" in task]
+ self.assertEqual(len(returned_ids), len(set(returned_ids)))
+ self.assertEqual(returned_ids, sorted(returned_ids))
+ self.assertTrue(set(task_ids).issubset(returned_ids))
+ finally:
+ state._redis.delete(*task_ids)
+
def test_patch_task_updates_only_existing_redis_task(self):
state = self._build_state([1])
diff --git a/test/services/test_task.py b/test/services/test_task.py
index 256e3645..fc960403 100644
--- a/test/services/test_task.py
+++ b/test/services/test_task.py
@@ -2021,6 +2021,43 @@ class TestTaskService(unittest.TestCase):
)
active_future.set_result(None)
+ def test_recover_interrupted_cross_posts_scans_ids_only_once(self):
+ """启动恢复不能依赖多次独立扫描的分页顺序。"""
+ state = MemoryState()
+ for task_id, cross_post_state in (
+ ("stale-pending", tm.const.CROSS_POST_STATE_PENDING),
+ ("stale-processing", tm.const.CROSS_POST_STATE_PROCESSING),
+ ("already-complete", tm.const.CROSS_POST_STATE_COMPLETE),
+ ):
+ state.update_task(
+ task_id,
+ state=tm.const.TASK_STATE_COMPLETE,
+ progress=100,
+ videos=["final.mp4"],
+ cross_post_state=cross_post_state,
+ )
+
+ with (
+ patch.object(tm.sm, "state", state),
+ patch.object(state, "list_task_ids", wraps=state.list_task_ids) as list_ids,
+ patch.object(
+ state, "get_all_tasks", side_effect=AssertionError("pagination used")
+ ) as get_all_tasks,
+ ):
+ recovered = tm.recover_interrupted_cross_posts(page_size=1)
+
+ self.assertEqual(recovered, 2)
+ list_ids.assert_called_once_with(scan_count=1)
+ get_all_tasks.assert_not_called()
+ for task_id in ("stale-pending", "stale-processing"):
+ task = state.get_task(task_id)
+ self.assertEqual(task["cross_post_state"], tm.const.CROSS_POST_STATE_FAILED)
+ self.assertEqual(task["videos"], ["final.mp4"])
+ self.assertEqual(
+ state.get_task("already-complete")["cross_post_state"],
+ tm.const.CROSS_POST_STATE_COMPLETE,
+ )
+
def test_cross_post_owner_uses_future_registry_for_current_process(self):
"""当前进程无活动 Future 时,同 PID 的新旧 owner 都应视为中断。"""
stale_owner = f"{tm.socket.gethostname()}:{tm.os.getpid()}:old-instance"
@@ -2093,7 +2130,7 @@ class TestTaskService(unittest.TestCase):
def test_cross_post_recovery_reports_state_backend_failure(self):
"""启动恢复读取状态失败时应返回 None,允许 WebUI 后续 rerun 重试。"""
state = MagicMock()
- state.get_all_tasks.side_effect = RuntimeError("redis unavailable")
+ state.list_task_ids.side_effect = RuntimeError("redis unavailable")
with (
patch.object(tm.sm, "state", state),
@@ -2135,33 +2172,49 @@ class TestTaskService(unittest.TestCase):
"MPT_TEST_REDIS_HOST not set",
)
def test_real_redis_recovers_interrupted_cross_post_state(self):
- """真实 Redis 中的遗留发布状态必须在恢复后保留视频并进入失败终态。"""
+ """真实 Redis 的多批扫描应恢复全部遗留发布状态并保留视频。"""
state = RedisState(
host=os.environ["MPT_TEST_REDIS_HOST"],
port=int(os.getenv("MPT_TEST_REDIS_PORT", "6379")),
db=int(os.getenv("MPT_TEST_REDIS_DB", "15")),
)
- task_id = f"ci-cross-post-recovery-{uuid4()}"
- state.update_task(
- task_id,
- state=tm.const.TASK_STATE_COMPLETE,
- progress=100,
- videos=["final.mp4"],
- cross_post_state=tm.const.CROSS_POST_STATE_PROCESSING,
- cross_post_owner="",
- )
+ task_ids = [f"ci-cross-post-recovery-{uuid4()}" for _ in range(3)]
+ for task_id, cross_post_state in zip(
+ task_ids,
+ (
+ tm.const.CROSS_POST_STATE_PENDING,
+ tm.const.CROSS_POST_STATE_PROCESSING,
+ tm.const.CROSS_POST_STATE_COMPLETE,
+ ),
+ ):
+ state.update_task(
+ task_id,
+ state=tm.const.TASK_STATE_COMPLETE,
+ progress=100,
+ videos=["final.mp4"],
+ cross_post_state=cross_post_state,
+ cross_post_owner="",
+ )
try:
with patch.object(tm.sm, "state", state):
- recovered = tm.recover_interrupted_cross_posts(page_size=10)
+ recovered = tm.recover_interrupted_cross_posts(page_size=1)
- self.assertGreaterEqual(recovered, 1)
- task = state.get_task(task_id)
- self.assertEqual(task["videos"], ["final.mp4"])
- self.assertEqual(task["cross_post_state"], tm.const.CROSS_POST_STATE_FAILED)
- self.assertEqual(task["cross_post_error"], tm._INTERRUPTED_CROSS_POST_ERROR)
+ self.assertGreaterEqual(recovered, 2)
+ for task_id in task_ids[:2]:
+ task = state.get_task(task_id)
+ self.assertEqual(task["videos"], ["final.mp4"])
+ self.assertEqual(task["cross_post_state"], tm.const.CROSS_POST_STATE_FAILED)
+ self.assertEqual(
+ task["cross_post_error"], tm._INTERRUPTED_CROSS_POST_ERROR
+ )
+ self.assertEqual(
+ state.get_task(task_ids[2])["cross_post_state"],
+ tm.const.CROSS_POST_STATE_COMPLETE,
+ )
finally:
- state.delete_task(task_id)
+ for task_id in task_ids:
+ state.delete_task(task_id)
def test_cross_post_future_exception_is_observed(self):
"""线程池自身抛出的异常必须进入日志,不能留在无人读取的 Future 中。"""