Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(redis): isolate non-hash keys during task lookup

Rudy Celeklie98d1a59632d
diff --git a/app/services/state.py b/app/services/state.py
index c6ccb5aa..48bfee37 100644
--- a/app/services/state.py
+++ b/app/services/state.py
@@ -2,6 +2,8 @@ import ast
import copy
import threading
from abc import ABC, abstractmethod
+
+from redis.exceptions import ResponseError
from itertools import islice
from app.config import config
@@ -163,12 +165,16 @@ class RedisState(BaseState):
with self._redis.pipeline(transaction=False) as pipeline:
for key in candidates:
pipeline.hget(key, "task_id")
- embedded_ids = pipeline.execute()
- task_keys.update(
- key
- for key, embedded_id in zip(candidates, embedded_ids)
- if embedded_id == key
- )
+ # A key can change type after SCAN. Isolate that row instead
+ # of letting one WRONGTYPE abort the whole task listing.
+ embedded_ids = pipeline.execute(raise_on_error=False)
+ for key, embedded_id in zip(candidates, embedded_ids):
+ if isinstance(embedded_id, ResponseError):
+ if str(embedded_id).startswith("WRONGTYPE"):
+ continue
+ raise embedded_id
+ if embedded_id == key:
+ task_keys.add(key)
if cursor == 0:
break
# 按任务键排序不依赖 Hash 扫描顺序;不额外维护索引,也不改变旧任务。
@@ -204,7 +210,14 @@ class RedisState(BaseState):
)
def get_task(self, task_id: str):
- task_data = self._redis.hgetall(task_id)
+ try:
+ task_data = self._redis.hgetall(task_id)
+ except ResponseError as exc:
+ # Arbitrary task IDs can name the application's List queue or
+ # another service's non-hash key. Those are not task records.
+ if str(exc).startswith("WRONGTYPE"):
+ return None
+ raise
# An API caller may ask for any Redis key by name. Require the same
# marker as list_task_ids before returning a hash's contents.
if not task_data or task_data.get(b"task_id") != task_id.encode("utf-8"):
diff --git a/test/services/test_state.py b/test/services/test_state.py
index b8daa4e2..e4bb7ece 100644
--- a/test/services/test_state.py
+++ b/test/services/test_state.py
@@ -28,7 +28,7 @@ class _FakeRedisPipeline:
self.keys.append((key, field))
return self
- def execute(self):
+ def execute(self, raise_on_error=True):
return [self.redis.data.get(key, {}).get(field.encode("utf-8")) for key, field in self.keys]
diff --git a/test/services/test_state_wrongtype.py b/test/services/test_state_wrongtype.py
new file mode 100644
index 00000000..51cf9b9a
--- /dev/null
+++ b/test/services/test_state_wrongtype.py
@@ -0,0 +1,68 @@
+import os
+from unittest.mock import MagicMock, Mock
+from uuid import uuid4
+
+import pytest
+from redis.exceptions import ResponseError
+
+from app.services.state import RedisState
+
+
+WRONGTYPE = "WRONGTYPE Operation against a key holding the wrong kind of value"
+
+
+def test_non_hash_task_lookup_returns_absent():
+ state = RedisState.__new__(RedisState)
+ state._redis = Mock()
+ state._redis.hgetall.side_effect = ResponseError(WRONGTYPE)
+ assert state.get_task("task_queue") is None
+
+
+def test_task_lookup_does_not_hide_unrelated_redis_errors():
+ state = RedisState.__new__(RedisState)
+ state._redis = Mock()
+ state._redis.hgetall.side_effect = ResponseError("NOAUTH Authentication required")
+ with pytest.raises(ResponseError, match="NOAUTH"):
+ state.get_task("task-id")
+
+
+def test_scan_skips_key_that_changes_type_before_hash_probe():
+ state = RedisState.__new__(RedisState)
+ state._redis = MagicMock()
+ state._redis.scan.return_value = (0, [b"changed", b"task-id"])
+ pipeline = state._redis.pipeline.return_value.__enter__.return_value
+ pipeline.execute.return_value = [ResponseError(WRONGTYPE), b"task-id"]
+ assert state.list_task_ids() == ["task-id"]
+ pipeline.execute.assert_called_once_with(raise_on_error=False)
+
+
+def test_scan_does_not_hide_unrelated_probe_errors():
+ state = RedisState.__new__(RedisState)
+ state._redis = MagicMock()
+ state._redis.scan.return_value = (0, [b"task-id"])
+ pipeline = state._redis.pipeline.return_value.__enter__.return_value
+ pipeline.execute.return_value = [ResponseError("NOPERM no permissions")]
+ with pytest.raises(ResponseError, match="NOPERM"):
+ state.list_task_ids()
+
+
[email protected](not os.getenv("MPT_TEST_REDIS_HOST"), reason="MPT_TEST_REDIS_HOST not set")
+def test_real_redis_non_hash_lookup_and_scan_type_change():
+ state = RedisState(host=os.environ["MPT_TEST_REDIS_HOST"], port=int(os.getenv("MPT_TEST_REDIS_PORT", "6379")), db=int(os.getenv("MPT_TEST_REDIS_DB", "15")))
+ key = f"ci-type-change-{uuid4()}"
+ client = state._redis
+ try:
+ client.rpush(key, "queue-item")
+ assert state.get_task(key) is None
+ client.delete(key)
+ state.update_task(key)
+ def scan_then_change_type(*args, **kwargs):
+ client.delete(key)
+ client.rpush(key, "now-a-list")
+ return 0, [key.encode()]
+ proxy = Mock(wraps=client)
+ proxy.scan.side_effect = scan_then_change_type
+ state._redis = proxy
+ assert state.list_task_ids() == []
+ finally:
+ client.delete(key)