利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix(cache): preserve refreshed search files during stale cleanup
Rudy Celekli5de533f6f49b
diff --git a/app/services/material_cache.py b/app/services/material_cache.pyindex 18ea3e83..4426ae0e 100644--- a/app/services/material_cache.py+++ b/app/services/material_cache.py@@ -7,6 +7,7 @@ import jsonimport mathimport osimport re+import statimport tempfileimport threadingimport time@@ -173,9 +174,24 @@ def get_material_search_cache_lock(return _CACHE_LOCKS[int(digest[:8], 16) % len(_CACHE_LOCKS)]-def _remove_invalid_cache(cache_path: Path) -> None:+def _is_same_cache_file(cache_path: Path, expected: os.stat_result) -> bool:+ """Avoid removing a file replaced since the scan/read snapshot."""+ try:+ current = cache_path.stat(follow_symlinks=False)+ except FileNotFoundError:+ return False+ return stat.S_ISREG(current.st_mode) and (+ current.st_dev, current.st_ino, current.st_mtime_ns, current.st_size+ ) == (expected.st_dev, expected.st_ino, expected.st_mtime_ns, expected.st_size)+++def _remove_invalid_cache(+ cache_path: Path, expected: os.stat_result | None = None+) -> None:"""删除已经过期或无法解析的单个缓存文件,失败时不影响素材搜索主流程。"""try:+ if expected is not None and not _is_same_cache_file(cache_path, expected):+ returncache_path.unlink(missing_ok=True)except OSError as exc:logger.warning(@@ -248,7 +264,7 @@ def load_material_search_cache(# 系统时间回拨或文件从其它机器复制后,mtime 可能落在未来。此时不能把# 缓存长期视为新鲜数据,直接失效并重新请求远端更可靠。if cache_age < 0 or cache_age >= MATERIAL_SEARCH_CACHE_TTL_SECONDS:- _remove_invalid_cache(cache_path)+ _remove_invalid_cache(cache_path, expected=stat_result)return Nonetry:@@ -298,7 +314,7 @@ def load_material_search_cache(logger.warning(f"failed to load material search cache: file={cache_path.name}, error={exc}")- _remove_invalid_cache(cache_path)+ _remove_invalid_cache(cache_path, expected=stat_result)return Nonelogger.info(@@ -436,8 +452,16 @@ def cleanup_expired_material_search_cache(try:if not entry.is_file(follow_symlinks=False):continue- cache_age = current_time - entry.stat(follow_symlinks=False).st_mtime- if 0 <= cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:+ expected = entry.stat(follow_symlinks=False)+ cache_age = current_time - expected.st_mtime+ if _CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name):+ # An active write may look future-dated after a clock+ # rollback. Only remove temp files demonstrably old enough.+ if cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:+ continue+ elif 0 <= cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:+ continue+ if not _is_same_cache_file(Path(entry.path), expected):continueos.unlink(entry.path)deleted_count += 1diff --git a/test/services/test_material_cache_replacement.py b/test/services/test_material_cache_replacement.pynew file mode 100644index 00000000..27b9ed2b--- /dev/null+++ b/test/services/test_material_cache_replacement.py@@ -0,0 +1,97 @@+import os+from pathlib import Path+from tempfile import TemporaryDirectory+from unittest.mock import patch++from app.models.schema import VideoAspect+from app.services import material_cache+++_NOW = 2_000_000_000+_PAYLOAD = (+ '{"version":2,"items":[{"provider":"pixabay","url":"https://example.com/clip.mp4",'+ '"duration":12,"source_info":{"provider":"pixabay"}}]}'+)+++def _replacement(path):+ replacement = path.with_suffix('.replacement')+ replacement.write_text(_PAYLOAD, encoding='utf-8')+ os.utime(replacement, (_NOW, _NOW))+ os.replace(replacement, path)+++def test_expired_reader_preserves_replacement_published_after_stat():+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):+ cache_path = material_cache._cache_path('pixabay', 'nature', 5, VideoAspect.portrait)+ cache_path.write_text('expired', encoding='utf-8')+ old = _NOW - material_cache.MATERIAL_SEARCH_CACHE_TTL_SECONDS - 1+ os.utime(cache_path, (old, old))+ stat = Path.stat+ published = False++ def replace_after_stat(path, *args, **kwargs):+ nonlocal published+ result = stat(path, *args, **kwargs)+ if path == cache_path and not published:+ published = True+ _replacement(cache_path)+ return result++ with patch.object(Path, 'stat', replace_after_stat):+ assert material_cache.load_material_search_cache('pixabay', 'nature', 5, VideoAspect.portrait, now=_NOW) is None+ assert cache_path.read_text() == _PAYLOAD+++def test_corrupt_reader_preserves_replacement_published_after_parse():+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):+ cache_path = material_cache._cache_path('pixabay', 'nature', 5, VideoAspect.portrait)+ cache_path.write_text('{broken', encoding='utf-8')+ os.utime(cache_path, (_NOW, _NOW))+ remove_invalid = material_cache._remove_invalid_cache++ def publish_before_removal(path, *args, **kwargs):+ _replacement(path)+ return remove_invalid(path, *args, **kwargs)++ with patch.object(material_cache, '_remove_invalid_cache', side_effect=publish_before_removal):+ assert material_cache.load_material_search_cache('pixabay', 'nature', 5, VideoAspect.portrait, now=_NOW) is None+ assert cache_path.read_text() == _PAYLOAD+++def test_ttl_cleanup_preserves_replacement_since_directory_scan():+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):+ cache_path = Path(directory, 'a' * 64 + '.json')+ cache_path.write_text('expired', encoding='utf-8')+ old = _NOW - material_cache.MATERIAL_SEARCH_CACHE_TTL_SECONDS - 1+ os.utime(cache_path, (old, old))+ scan = os.scandir++ class ScannedEntries:+ def __enter__(self):+ with scan(directory) as entries:+ self.entries = list(entries)+ for entry in self.entries:+ entry.stat(follow_symlinks=False)+ _replacement(cache_path)+ return iter(self.entries)++ def __iter__(self):+ return iter(self.entries)++ def __exit__(self, *args):+ return False++ with patch.object(material_cache.os, 'scandir', return_value=ScannedEntries()):+ assert material_cache.cleanup_expired_material_search_cache(now=_NOW, force=True) == 0+ assert cache_path.exists()+++def test_cleanup_keeps_future_dated_active_temporary_file():+ with TemporaryDirectory() as directory, patch.object(material_cache.utils, 'storage_dir', return_value=directory):+ temporary = Path(directory, '.' + 'a' * 64 + '-worker.tmp')+ temporary.write_text('active incomplete upload', encoding='utf-8')+ # A clock rollback can leave an actively written file in the future.+ os.utime(temporary, (_NOW + 60, _NOW + 60))+ assert material_cache.cleanup_expired_material_search_cache(now=_NOW, force=True) == 0+ assert temporary.exists()