Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(state): guard Redis task deletion atomically

harry07037c80ca0b1440
diff --git a/app/services/state.py b/app/services/state.py
index 3916e70c..ca736366 100644
--- a/app/services/state.py
+++ b/app/services/state.py
@@ -28,6 +28,18 @@ return 1
"""
+_DELETE_OWNED_TASK_SCRIPT = """
+if redis.call("TYPE", KEYS[1]).ok ~= "hash" then
+ return 0
+end
+if redis.call("HGET", KEYS[1], "task_id") ~= KEYS[1] then
+ return 0
+end
+
+return redis.call("DEL", KEYS[1])
+"""
+
+
# Base class for state management
class BaseState(ABC):
@abstractmethod
@@ -258,7 +270,9 @@ class RedisState(BaseState):
return bool(updated)
def delete_task(self, task_id: str):
- self._redis.delete(task_id)
+ # 调用方读取任务后还会清理目录,同名键可能在这段时间被其它服务复用。
+ # 在一次 Lua 操作内检查类型、任务标记并删除,避免检查与删除之间的竞态。
+ self._redis.eval(_DELETE_OWNED_TASK_SCRIPT, 1, task_id)
@staticmethod
def _serialize_field(field, value):
diff --git a/test/services/test_redis_delete_ownership.py b/test/services/test_redis_delete_ownership.py
new file mode 100644
index 00000000..901f12e3
--- /dev/null
+++ b/test/services/test_redis_delete_ownership.py
@@ -0,0 +1,232 @@
+"""Redis 任务删除必须保留被复用的键,且不改变正常删除与错误传播。"""
+
+import os
+import shutil
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from threading import Barrier
+from unittest.mock import Mock, patch
+from uuid import uuid4
+
+import pytest
+from fastapi import FastAPI
+from fastapi.responses import JSONResponse
+from fastapi.testclient import TestClient
+from redis.exceptions import ResponseError
+
+from app.config import config
+from app.controllers.v1 import video
+from app.models import const
+from app.models.exception import HttpException
+from app.services.state import RedisState, _DELETE_OWNED_TASK_SCRIPT
+
+
+def test_delete_checks_ownership_and_deletes_in_one_command():
+ state = RedisState.__new__(RedisState)
+ state._redis = Mock()
+
+ assert state.delete_task("task-1") is None
+
+ state._redis.eval.assert_called_once_with(_DELETE_OWNED_TASK_SCRIPT, 1, "task-1")
+ state._redis.delete.assert_not_called()
+ state._redis.type.assert_not_called()
+ state._redis.hget.assert_not_called()
+
+
[email protected]("error", ["NOPERM no permissions", "NOAUTH Authentication required"])
+def test_delete_does_not_hide_errors_or_fall_back_to_unsafe_delete(error):
+ state = RedisState.__new__(RedisState)
+ state._redis = Mock()
+ state._redis.eval.side_effect = ResponseError(error)
+
+ with pytest.raises(ResponseError, match=error):
+ state.delete_task("task-1")
+
+ state._redis.delete.assert_not_called()
+
+
+def native_state():
+ host = os.getenv("MPT_TEST_REDIS_HOST")
+ if not host:
+ pytest.skip("MPT_TEST_REDIS_HOST not set")
+ state = RedisState(
+ host=host,
+ port=int(os.getenv("MPT_TEST_REDIS_PORT", "6379")),
+ db=int(os.getenv("MPT_TEST_REDIS_DB", "15")),
+ )
+ key = f"ci-delete-owner-{uuid4()}"
+ try:
+ yield state, key
+ finally:
+ state._redis.delete(key)
+ state._redis.close()
+
+
+REPLACEMENT_KINDS = ["foreign_hash", "mismatched_task", "queue", "string", "set", "sorted_set", "stream"]
+
+
+def _replace_task_key(state, key, kind):
+ client = state._redis
+ client.delete(key)
+ if kind == "foreign_hash":
+ client.hset(key, mapping={"owner": "other-service", "value": "retained"})
+ elif kind == "mismatched_task":
+ client.hset(key, mapping={"task_id": "different-task", "value": "retained"})
+ elif kind == "queue":
+ client.rpush(key, "queued-task")
+ elif kind == "string":
+ client.set(key, "retained")
+ elif kind == "set":
+ client.sadd(key, "retained")
+ elif kind == "sorted_set":
+ client.zadd(key, {"retained": 1})
+ elif kind == "stream":
+ client.xadd(key, {"value": "retained"})
+ else:
+ raise AssertionError(f"unexpected replacement kind: {kind}")
+ client.pexpire(key, 60000)
+ return client.dump(key)
+
+
[email protected]("kind", REPLACEMENT_KINDS)
+def test_delete_preserves_repurposed_key_and_expiry(native_state, kind):
+ state, key = native_state
+ state.update_task(key, state=const.TASK_STATE_COMPLETE, progress=100)
+ assert state.get_task(key)["state"] == const.TASK_STATE_COMPLETE
+ before = _replace_task_key(state, key, kind)
+ assert state.get_task(key) is None
+
+ state.delete_task(key)
+ state.delete_task(key)
+
+ assert state._redis.dump(key) == before
+ assert 0 < state._redis.pttl(key) <= 60000
+
+
[email protected]("present", [False, True])
+def test_delete_owned_and_absent_tasks_is_idempotent(native_state, present):
+ state, key = native_state
+ if present:
+ state.update_task(key, state=const.TASK_STATE_COMPLETE, progress=100)
+ state.delete_task(key)
+ state.delete_task(key)
+ assert not state._redis.exists(key)
+
+
+def test_delete_accepts_matching_unicode_task_marker(native_state):
+ state, key = native_state
+ unicode_key = f"{key}-任务"
+ try:
+ state.update_task(unicode_key, state=const.TASK_STATE_COMPLETE)
+ assert state.get_task(unicode_key)["task_id"] == unicode_key
+ state.delete_task(unicode_key)
+ assert not state._redis.exists(unicode_key)
+ finally:
+ state._redis.delete(unicode_key)
+
+
+def test_concurrent_replacement_and_delete_always_preserve_foreign_key(native_state):
+ state, key = native_state
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ for _ in range(50):
+ state._redis.delete(key)
+ state.update_task(key, state=const.TASK_STATE_COMPLETE)
+ barrier = Barrier(2)
+
+ def replace():
+ barrier.wait(timeout=5)
+ # 整个替换在 Redis 事务中执行;无论它先于还是后于删除,外部键都应保留。
+ with state._redis.pipeline() as transaction:
+ transaction.delete(key)
+ transaction.set(key, "other-service")
+ transaction.execute()
+
+ def delete():
+ barrier.wait(timeout=5)
+ state.delete_task(key)
+
+ futures = [executor.submit(replace), executor.submit(delete)]
+ for future in futures:
+ future.result(timeout=5)
+ assert state._redis.get(key) == b"other-service"
+
+
+def deletion_client(native_state, tmp_path):
+ state, _ = native_state
+ application = FastAPI()
+ application.include_router(video.router)
+
+ @application.exception_handler(HttpException)
+ async def handle_error(request, exc):
+ return JSONResponse(status_code=exc.status_code, content={"message": exc.message})
+
+ with (
+ patch.object(video.sm, "state", state),
+ patch.object(video.utils, "task_dir", return_value=str(tmp_path)),
+ patch.object(config, "app", dict(config.app, api_key="test-delete-api-key")),
+ TestClient(application) as client,
+ ):
+ yield client
+
+
[email protected]("kind", REPLACEMENT_KINDS)
+def test_http_delete_preserves_key_repurposed_after_directory_cleanup(
+ native_state, deletion_client, tmp_path, kind
+):
+ state, key = native_state
+ state.update_task(key, state=const.TASK_STATE_COMPLETE, progress=100)
+ task_dir = tmp_path / key
+ task_dir.mkdir()
+ (task_dir / "artifact.txt").write_text("generated artifact", encoding="utf-8")
+ original_rmtree = shutil.rmtree
+ retained = []
+
+ def cleanup_then_repurpose(path, *args, **kwargs):
+ assert Path(path) == task_dir
+ original_rmtree(path, *args, **kwargs)
+ # 在真实路由读完任务并清理目录后、删除 Redis 记录前,确定性插入键复用。
+ retained.append(_replace_task_key(state, key, kind))
+
+ with patch.object(video.shutil, "rmtree", side_effect=cleanup_then_repurpose):
+ response = deletion_client.delete(
+ f"/api/v1/tasks/{key}", headers={"x-api-key": "test-delete-api-key"}
+ )
+
+ assert response.status_code == 200
+ assert not task_dir.exists()
+ assert retained
+ assert state._redis.dump(key) == retained[0]
+ assert state._redis.pttl(key) > 0
+
+
+ "control,expected", [("foreign", 404), ("busy", 409), ("unauthorized", 401), ("owned", 200)]
+)
+def test_http_deletion_keeps_existing_guards(native_state, deletion_client, tmp_path, control, expected):
+ state, key = native_state
+ if control == "foreign":
+ state._redis.set(key, "other-service")
+ else:
+ state.update_task(
+ key,
+ state=const.TASK_STATE_PROCESSING if control == "busy" else const.TASK_STATE_COMPLETE,
+ progress=100,
+ )
+ task_dir = tmp_path / key
+ task_dir.mkdir()
+ before = state._redis.dump(key)
+
+ response = deletion_client.delete(
+ f"/api/v1/tasks/{key}",
+ headers={} if control == "unauthorized" else {"x-api-key": "test-delete-api-key"},
+ )
+
+ assert response.status_code == expected
+ if expected == 200:
+ assert not state._redis.exists(key)
+ assert not task_dir.exists()
+ else:
+ assert state._redis.dump(key) == before
+ assert task_dir.exists()