Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(cache): preserve refreshed search files during stale cleanup

Rudy Celekli5de533f6f49b
diff --git a/app/services/material_cache.py b/app/services/material_cache.py
index 18ea3e83..4426ae0e 100644
--- a/app/services/material_cache.py
+++ b/app/services/material_cache.py
@@ -7,6 +7,7 @@ import json
import math
import os
import re
+import stat
import tempfile
import threading
import time
@@ -173,9 +174,24 @@ def get_material_search_cache_lock(
return _CACHE_LOCKS[int(digest[:8], 16) % len(_CACHE_LOCKS)]
-def _remove_invalid_cache(cache_path: Path) -> None:
+def _is_same_cache_file(cache_path: Path, expected: os.stat_result) -> bool:
+ """Avoid removing a file replaced since the scan/read snapshot."""
+ try:
+ current = cache_path.stat(follow_symlinks=False)
+ except FileNotFoundError:
+ return False
+ return stat.S_ISREG(current.st_mode) and (
+ current.st_dev, current.st_ino, current.st_mtime_ns, current.st_size
+ ) == (expected.st_dev, expected.st_ino, expected.st_mtime_ns, expected.st_size)
+
+
+def _remove_invalid_cache(
+ cache_path: Path, expected: os.stat_result | None = None
+) -> None:
"""删除已经过期或无法解析的单个缓存文件,失败时不影响素材搜索主流程。"""
try:
+ if expected is not None and not _is_same_cache_file(cache_path, expected):
+ return
cache_path.unlink(missing_ok=True)
except OSError as exc:
logger.warning(
@@ -248,7 +264,7 @@ def load_material_search_cache(
# 系统时间回拨或文件从其它机器复制后,mtime 可能落在未来。此时不能把
# 缓存长期视为新鲜数据,直接失效并重新请求远端更可靠。
if cache_age < 0 or cache_age >= MATERIAL_SEARCH_CACHE_TTL_SECONDS:
- _remove_invalid_cache(cache_path)
+ _remove_invalid_cache(cache_path, expected=stat_result)
return None
try:
@@ -298,7 +314,7 @@ def load_material_search_cache(
logger.warning(
f"failed to load material search cache: file={cache_path.name}, error={exc}"
)
- _remove_invalid_cache(cache_path)
+ _remove_invalid_cache(cache_path, expected=stat_result)
return None
logger.info(
@@ -436,8 +452,16 @@ def cleanup_expired_material_search_cache(
try:
if not entry.is_file(follow_symlinks=False):
continue
- cache_age = current_time - entry.stat(follow_symlinks=False).st_mtime
- if 0 <= cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:
+ expected = entry.stat(follow_symlinks=False)
+ cache_age = current_time - expected.st_mtime
+ if _CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name):
+ # An active write may look future-dated after a clock
+ # rollback. Only remove temp files demonstrably old enough.
+ if cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:
+ continue
+ elif 0 <= cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:
+ continue
+ if not _is_same_cache_file(Path(entry.path), expected):
continue
os.unlink(entry.path)
deleted_count += 1
diff --git a/test/services/test_material_cache_replacement.py b/test/services/test_material_cache_replacement.py
new file mode 100644
index 00000000..27b9ed2b
--- /dev/null
+++ b/test/services/test_material_cache_replacement.py
@@ -0,0 +1,97 @@
+import os
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from app.models.schema import VideoAspect
+from app.services import material_cache
+
+
+_NOW = 2_000_000_000
+_PAYLOAD = (
+ '{"version":2,"items":[{"provider":"pixabay","url":"https://example.com/clip.mp4",'
+ '"duration":12,"source_info":{"provider":"pixabay"}}]}'
+)
+
+
+def _replacement(path):
+ replacement = path.with_suffix('.replacement')
+ replacement.write_text(_PAYLOAD, encoding='utf-8')
+ os.utime(replacement, (_NOW, _NOW))
+ os.replace(replacement, path)
+
+
+def test_expired_reader_preserves_replacement_published_after_stat():
+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):
+ cache_path = material_cache._cache_path('pixabay', 'nature', 5, VideoAspect.portrait)
+ cache_path.write_text('expired', encoding='utf-8')
+ old = _NOW - material_cache.MATERIAL_SEARCH_CACHE_TTL_SECONDS - 1
+ os.utime(cache_path, (old, old))
+ stat = Path.stat
+ published = False
+
+ def replace_after_stat(path, *args, **kwargs):
+ nonlocal published
+ result = stat(path, *args, **kwargs)
+ if path == cache_path and not published:
+ published = True
+ _replacement(cache_path)
+ return result
+
+ with patch.object(Path, 'stat', replace_after_stat):
+ assert material_cache.load_material_search_cache('pixabay', 'nature', 5, VideoAspect.portrait, now=_NOW) is None
+ assert cache_path.read_text() == _PAYLOAD
+
+
+def test_corrupt_reader_preserves_replacement_published_after_parse():
+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):
+ cache_path = material_cache._cache_path('pixabay', 'nature', 5, VideoAspect.portrait)
+ cache_path.write_text('{broken', encoding='utf-8')
+ os.utime(cache_path, (_NOW, _NOW))
+ remove_invalid = material_cache._remove_invalid_cache
+
+ def publish_before_removal(path, *args, **kwargs):
+ _replacement(path)
+ return remove_invalid(path, *args, **kwargs)
+
+ with patch.object(material_cache, '_remove_invalid_cache', side_effect=publish_before_removal):
+ assert material_cache.load_material_search_cache('pixabay', 'nature', 5, VideoAspect.portrait, now=_NOW) is None
+ assert cache_path.read_text() == _PAYLOAD
+
+
+def test_ttl_cleanup_preserves_replacement_since_directory_scan():
+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):
+ cache_path = Path(directory, 'a' * 64 + '.json')
+ cache_path.write_text('expired', encoding='utf-8')
+ old = _NOW - material_cache.MATERIAL_SEARCH_CACHE_TTL_SECONDS - 1
+ os.utime(cache_path, (old, old))
+ scan = os.scandir
+
+ class ScannedEntries:
+ def __enter__(self):
+ with scan(directory) as entries:
+ self.entries = list(entries)
+ for entry in self.entries:
+ entry.stat(follow_symlinks=False)
+ _replacement(cache_path)
+ return iter(self.entries)
+
+ def __iter__(self):
+ return iter(self.entries)
+
+ def __exit__(self, *args):
+ return False
+
+ with patch.object(material_cache.os, 'scandir', return_value=ScannedEntries()):
+ assert material_cache.cleanup_expired_material_search_cache(now=_NOW, force=True) == 0
+ assert cache_path.exists()
+
+
+def test_cleanup_keeps_future_dated_active_temporary_file():
+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):
+ temporary = Path(directory, '.' + 'a' * 64 + '-worker.tmp')
+ temporary.write_text('active incomplete upload', encoding='utf-8')
+ # A clock rollback can leave an actively written file in the future.
+ os.utime(temporary, (_NOW + 60, _NOW + 60))
+ assert material_cache.cleanup_expired_material_search_cache(now=_NOW, force=True) == 0
+ assert temporary.exists()