Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

perf(material): cache provider search results

harry070395dd03ed0255
diff --git a/app/services/material.py b/app/services/material.py
index fec110a7..d73ff43f 100644
--- a/app/services/material.py
+++ b/app/services/material.py
@@ -1,7 +1,7 @@
import os
import random
import threading
-from typing import List
+from typing import Callable, List
from urllib.parse import quote_plus, urlencode
import requests
@@ -10,6 +10,7 @@ from moviepy.video.io.VideoFileClip import VideoFileClip
from app.config import config
from app.models.schema import MaterialInfo, VideoAspect, VideoConcatMode
+from app.services import material_cache
from app.utils import utils
# Thread-safe counter for API key rotation
@@ -388,6 +389,70 @@ def save_video(video_url: str, save_dir: str = "") -> str:
return ""
+def _search_videos_with_cache(
+ provider: str,
+ search_videos: Callable[..., List[MaterialInfo]],
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect,
+) -> List[MaterialInfo]:
+ """
+ 统一处理三个在线素材源的 24 小时搜索缓存。
+
+ 缓存只包裹搜索 API,不改变后续视频下载与去重逻辑。远端返回空列表时不写
+ 缓存,因为现有 provider 接口使用空列表同时表示“没有结果”和“请求失败”;
+ 在两者尚未拆分为明确结果类型前,宁可下次重试,也不能把临时故障缓存一天。
+ """
+ cache_args = {
+ "provider": provider,
+ "search_term": search_term,
+ "minimum_duration": minimum_duration,
+ "video_aspect": video_aspect,
+ }
+
+ def load_cache_safely() -> List[MaterialInfo] | None:
+ try:
+ return material_cache.load_material_search_cache(**cache_args)
+ except Exception as exc:
+ # 缓存是可选优化,任何缓存实现异常都必须按未命中处理,不能阻断
+ # Pexels、Pixabay 或 Coverr 的正常远端搜索。
+ logger.warning(
+ "material search cache read failed, continue with remote search: "
+ f"provider={provider}, error={type(exc).__name__}, detail={exc}"
+ )
+ return None
+
+ cached_items = load_cache_safely()
+ if cached_items is not None:
+ return cached_items
+
+ cache_lock = material_cache.get_material_search_cache_lock(**cache_args)
+ with cache_lock:
+ # 等待相同搜索条件的线程完成后再次读取,避免多个 API 任务在首次缓存
+ # 未命中时同时请求远端,降低第三方接口限流和风控触发概率。
+ cached_items = load_cache_safely()
+ if cached_items is not None:
+ return cached_items
+
+ items = search_videos(
+ search_term=search_term,
+ minimum_duration=minimum_duration,
+ video_aspect=video_aspect,
+ )
+ if items:
+ try:
+ material_cache.save_material_search_cache(
+ **cache_args,
+ items=items,
+ )
+ except Exception as exc:
+ logger.warning(
+ "material search cache write failed, use remote results: "
+ f"provider={provider}, error={type(exc).__name__}, detail={exc}"
+ )
+ return items
+
+
def download_videos(
task_id: str,
search_terms: List[str],
@@ -398,11 +463,27 @@ def download_videos(
max_clip_duration: int = 5,
match_script_order: bool = False,
) -> List[str]:
- search_videos = search_videos_pexels
+ provider = "pexels"
+ remote_search_videos = search_videos_pexels
if source == "pixabay":
- search_videos = search_videos_pixabay
+ provider = "pixabay"
+ remote_search_videos = search_videos_pixabay
elif source == "coverr":
- search_videos = search_videos_coverr
+ provider = "coverr"
+ remote_search_videos = search_videos_coverr
+
+ def search_videos(
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect,
+ ) -> List[MaterialInfo]:
+ return _search_videos_with_cache(
+ provider=provider,
+ search_videos=remote_search_videos,
+ search_term=search_term,
+ minimum_duration=minimum_duration,
+ video_aspect=video_aspect,
+ )
material_directory = config.app.get("material_directory", "").strip()
if material_directory == "task":
diff --git a/app/services/material_cache.py b/app/services/material_cache.py
new file mode 100644
index 00000000..edea3c4a
--- /dev/null
+++ b/app/services/material_cache.py
@@ -0,0 +1,346 @@
+"""在线素材搜索结果的磁盘缓存。"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import tempfile
+import threading
+import time
+from pathlib import Path
+from typing import Iterable
+
+from loguru import logger
+
+from app.models.schema import MaterialInfo, VideoAspect
+from app.utils import utils
+
+
+MATERIAL_SEARCH_CACHE_TTL_SECONDS = 24 * 60 * 60
+_CACHE_FORMAT_VERSION = 1
+_CACHE_CLEANUP_INTERVAL_SECONDS = 60 * 60
+_CACHE_FILE_PATTERN = re.compile(r"^[0-9a-f]{64}\.json$")
+
+# API 默认允许多个视频任务并发执行。固定数量的锁分片可以让相同搜索条件共用
+# 一个锁,同时避免按关键词永久保存 Lock 导致内存持续增长。它只负责合并当前
+# 进程内的并发请求;跨进程写入仍由临时文件和 os.replace 保证完整性。
+_CACHE_LOCKS = tuple(threading.Lock() for _ in range(256))
+_cleanup_state_lock = threading.Lock()
+_last_cleanup_monotonic: float | None = None
+
+
+def _cache_dir() -> Path:
+ """
+ 返回所有运行入口共用的素材搜索缓存目录。
+
+ 缓存必须位于 ``storage`` 下,而不是 WebUI session 或进程内存中,才能让
+ WebUI、API、CLI 以及 Docker 重启后的任务复用同一份结果。
+ """
+ return Path(utils.storage_dir("cache_material_search", create=True))
+
+
+def _cache_key(
+ provider: str,
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect | str,
+) -> str:
+ """
+ 根据会影响搜索结果的业务参数生成稳定文件名。
+
+ API Key 只负责鉴权,不影响公开搜索结果,因此不能写入缓存键或缓存内容。
+ 使用 SHA-256 可以避免关键词直接出现在文件名中,同时保持路径长度固定。
+ """
+ aspect_value = getattr(video_aspect, "value", video_aspect)
+ cache_key = json.dumps(
+ {
+ "provider": str(provider).strip().lower(),
+ "search_term": str(search_term).strip(),
+ "minimum_duration": int(minimum_duration),
+ "video_aspect": str(aspect_value),
+ },
+ ensure_ascii=False,
+ sort_keys=True,
+ separators=(",", ":"),
+ )
+ return hashlib.sha256(cache_key.encode("utf-8")).hexdigest()
+
+
+def _cache_path(
+ provider: str,
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect | str,
+) -> Path:
+ digest = _cache_key(
+ provider=provider,
+ search_term=search_term,
+ minimum_duration=minimum_duration,
+ video_aspect=video_aspect,
+ )
+ return _cache_dir() / f"{digest}.json"
+
+
+def get_material_search_cache_lock(
+ provider: str,
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect | str,
+) -> threading.Lock:
+ """返回当前搜索条件对应的进程内锁分片。"""
+ digest = _cache_key(
+ provider=provider,
+ search_term=search_term,
+ minimum_duration=minimum_duration,
+ video_aspect=video_aspect,
+ )
+ return _CACHE_LOCKS[int(digest[:8], 16) % len(_CACHE_LOCKS)]
+
+
+def _remove_invalid_cache(cache_path: Path) -> None:
+ """删除已经过期或无法解析的单个缓存文件,失败时不影响素材搜索主流程。"""
+ try:
+ cache_path.unlink(missing_ok=True)
+ except OSError as exc:
+ logger.warning(
+ f"failed to remove invalid material search cache: "
+ f"file={cache_path.name}, error={exc}"
+ )
+
+
+def load_material_search_cache(
+ provider: str,
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect | str,
+ *,
+ now: float | None = None,
+) -> list[MaterialInfo] | None:
+ """
+ 读取仍在 24 小时有效期内的素材搜索结果。
+
+ ``None`` 表示缓存未命中,需要请求远端 API;空列表不作为有效缓存返回,
+ 避免网络错误或上游异常被误缓存后持续阻断后续任务。
+ """
+ try:
+ cache_path = _cache_path(
+ provider=provider,
+ search_term=search_term,
+ minimum_duration=minimum_duration,
+ video_aspect=video_aspect,
+ )
+ except Exception as exc:
+ # 缓存目录创建、路径解析等异常不能阻断远端素材搜索。这里保留完整异常
+ # 类型和信息,便于定位权限或挂载问题,同时按缓存未命中继续主流程。
+ logger.warning(
+ "failed to prepare material search cache: "
+ f"operation=read, error={type(exc).__name__}, detail={exc}"
+ )
+ return None
+ try:
+ stat_result = cache_path.stat()
+ except FileNotFoundError:
+ return None
+ except OSError as exc:
+ logger.warning(
+ f"failed to inspect material search cache: "
+ f"file={cache_path.name}, error={exc}"
+ )
+ return None
+
+ current_time = time.time() if now is None else now
+ cache_age = current_time - stat_result.st_mtime
+ # 系统时间回拨或文件从其它机器复制后,mtime 可能落在未来。此时不能把
+ # 缓存长期视为新鲜数据,直接失效并重新请求远端更可靠。
+ if cache_age < 0 or cache_age >= MATERIAL_SEARCH_CACHE_TTL_SECONDS:
+ _remove_invalid_cache(cache_path)
+ return None
+
+ try:
+ with cache_path.open("r", encoding="utf-8") as cache_file:
+ payload = json.load(cache_file)
+
+ if (
+ not isinstance(payload, dict)
+ or payload.get("version") != _CACHE_FORMAT_VERSION
+ or not isinstance(payload.get("items"), list)
+ or not payload["items"]
+ ):
+ raise ValueError("invalid cache payload")
+
+ items = []
+ for raw_item in payload["items"]:
+ if not isinstance(raw_item, dict):
+ raise ValueError("invalid material item")
+ item_provider = raw_item.get("provider")
+ item_url = raw_item.get("url")
+ item_duration = raw_item.get("duration")
+ if (
+ not isinstance(item_provider, str)
+ or not item_provider
+ or not isinstance(item_url, str)
+ or not item_url
+ or isinstance(item_duration, bool)
+ or not isinstance(item_duration, (int, float))
+ or item_duration <= 0
+ ):
+ raise ValueError("invalid material fields")
+ items.append(
+ MaterialInfo(
+ provider=item_provider,
+ url=item_url,
+ duration=int(item_duration),
+ )
+ )
+ except (OSError, ValueError, TypeError) as exc:
+ logger.warning(
+ f"failed to load material search cache: "
+ f"file={cache_path.name}, error={exc}"
+ )
+ _remove_invalid_cache(cache_path)
+ return None
+
+ logger.info(
+ f"material search cache hit: provider={provider}, "
+ f"term={search_term!r}, items={len(items)}"
+ )
+ return items
+
+
+def save_material_search_cache(
+ provider: str,
+ search_term: str,
+ minimum_duration: int,
+ video_aspect: VideoAspect | str,
+ items: Iterable[MaterialInfo],
+) -> bool:
+ """
+ 原子保存一次成功的非空素材搜索结果。
+
+ 多个任务可能并发搜索相同关键词。先写入同目录唯一临时文件,再通过
+ ``os.replace`` 发布,可以保证读进程只会看到完整旧文件或完整新文件;
+ 即使两个写进程同时完成,最终内容也都是同一缓存键对应的合法结果。
+ """
+ temp_path = None
+ try:
+ serialized_items = [
+ {
+ "provider": item.provider,
+ "url": item.url,
+ "duration": int(item.duration),
+ }
+ for item in items
+ if item.url and item.duration > 0
+ ]
+ if not serialized_items:
+ return False
+
+ cache_path = _cache_path(
+ provider=provider,
+ search_term=search_term,
+ minimum_duration=minimum_duration,
+ video_aspect=video_aspect,
+ )
+ cleanup_expired_material_search_cache()
+ payload = {
+ "version": _CACHE_FORMAT_VERSION,
+ "items": serialized_items,
+ }
+ with tempfile.NamedTemporaryFile(
+ mode="w",
+ encoding="utf-8",
+ dir=cache_path.parent,
+ prefix=f".{cache_path.stem}-",
+ suffix=".tmp",
+ delete=False,
+ ) as temp_file:
+ temp_path = Path(temp_file.name)
+ json.dump(
+ payload,
+ temp_file,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+ temp_file.flush()
+ os.fsync(temp_file.fileno())
+
+ os.replace(temp_path, cache_path)
+ return True
+ except Exception as exc:
+ logger.warning(
+ "failed to save material search cache: "
+ f"error={type(exc).__name__}, detail={exc}"
+ )
+ if temp_path is not None:
+ try:
+ temp_path.unlink(missing_ok=True)
+ except OSError:
+ pass
+ return False
+
+
+def cleanup_expired_material_search_cache(
+ *,
+ now: float | None = None,
+ force: bool = False,
+) -> int:
+ """
+ 低频清理没有再次被查询到的过期搜索缓存。
+
+ 正常写入路径每小时最多扫描一次目录,避免每次搜索都产生线性目录遍历;
+ ``force`` 仅供测试或显式维护调用。只删除 SHA-256 命名的 JSON 文件,不会
+ 触碰用户放入目录的其它文件。
+ """
+ global _last_cleanup_monotonic
+
+ monotonic_now = time.monotonic()
+ with _cleanup_state_lock:
+ if (
+ not force
+ and _last_cleanup_monotonic is not None
+ and monotonic_now - _last_cleanup_monotonic
+ < _CACHE_CLEANUP_INTERVAL_SECONDS
+ ):
+ return 0
+ _last_cleanup_monotonic = monotonic_now
+
+ try:
+ cache_dir = _cache_dir()
+ entries = os.scandir(cache_dir)
+ except Exception as exc:
+ logger.warning(
+ "failed to scan material search cache: "
+ f"error={type(exc).__name__}, detail={exc}"
+ )
+ return 0
+
+ current_time = time.time() if now is None else now
+ deleted_count = 0
+ failed_count = 0
+ with entries:
+ for entry in entries:
+ if not _CACHE_FILE_PATTERN.fullmatch(entry.name):
+ continue
+ try:
+ if not entry.is_file(follow_symlinks=False):
+ continue
+ cache_age = current_time - entry.stat(follow_symlinks=False).st_mtime
+ if 0 <= cache_age < MATERIAL_SEARCH_CACHE_TTL_SECONDS:
+ continue
+ os.unlink(entry.path)
+ deleted_count += 1
+ except OSError as exc:
+ failed_count += 1
+ logger.warning(
+ "failed to delete material search cache file: "
+ f"file={entry.name}, error={exc}"
+ )
+
+ if deleted_count or failed_count:
+ logger.info(
+ "finished cleaning material search cache: "
+ f"deleted={deleted_count}, failed={failed_count}"
+ )
+ return deleted_count
diff --git a/test/services/test_material.py b/test/services/test_material.py
index d4a3388a..09f3a4f6 100644
--- a/test/services/test_material.py
+++ b/test/services/test_material.py
@@ -313,6 +313,12 @@ class TestMaterialTlsVerification(unittest.TestCase):
patch.dict(config.app, {"material_directory": ""}),
patch.object(material, "search_videos_pexels", side_effect=fake_search),
patch.object(material, "save_video", side_effect=fake_save_video),
+ patch.object(
+ material.material_cache,
+ "load_material_search_cache",
+ return_value=None,
+ ),
+ patch.object(material.material_cache, "save_material_search_cache"),
):
result = material.download_videos(
task_id="ordered-materials",
@@ -559,7 +565,12 @@ class TestCoverrProvider(unittest.TestCase):
) as search, patch(
"app.services.material.save_video",
return_value="/tmp/coverr-saved.mp4",
- ) as save:
+ ) as save, patch(
+ "app.services.material.material_cache.load_material_search_cache",
+ return_value=None,
+ ), patch(
+ "app.services.material.material_cache.save_material_search_cache",
+ ):
result = material.download_videos(
task_id="t-coverr",
search_terms=["nature"],
diff --git a/test/services/test_material_cache.py b/test/services/test_material_cache.py
new file mode 100644
index 00000000..442b986b
--- /dev/null
+++ b/test/services/test_material_cache.py
@@ -0,0 +1,402 @@
+import json
+import os
+import tempfile
+import threading
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from app.models.schema import MaterialInfo, VideoAspect
+from app.services import material, material_cache
+
+
+class TestMaterialSearchCache(unittest.TestCase):
+ def setUp(self):
+ self.temp_dir = tempfile.TemporaryDirectory()
+ self.cache_dir_patch = patch(
+ "app.services.material_cache.utils.storage_dir",
+ return_value=self.temp_dir.name,
+ )
+ self.cache_dir_patch.start()
+
+ def tearDown(self):
+ self.cache_dir_patch.stop()
+ self.temp_dir.cleanup()
+
+ @staticmethod
+ def _item(url: str = "https://example.com/video.mp4") -> MaterialInfo:
+ return MaterialInfo(provider="pixabay", url=url, duration=12)
+
+ def _cache_path(self) -> Path:
+ return material_cache._cache_path(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+
+ def test_cache_round_trip_preserves_material_fields(self):
+ """
+ 磁盘缓存必须能跨进程恢复 MaterialInfo 所需的全部字段,不能只缓存 URL
+ 后丢失 provider 或 duration,导致后续下载与时长计算行为发生变化。
+ """
+ saved = material_cache.save_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ items=[self._item()],
+ )
+ loaded = material_cache.load_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+
+ self.assertTrue(saved)
+ self.assertEqual(len(loaded), 1)
+ self.assertEqual(loaded[0].provider, "pixabay")
+ self.assertEqual(loaded[0].url, "https://example.com/video.mp4")
+ self.assertEqual(loaded[0].duration, 12)
+
+ def test_expired_cache_is_removed_and_treated_as_miss(self):
+ """
+ Pixabay 要求搜索结果最多复用 24 小时。过期文件必须立即失效并删除,
+ 防止旧素材 URL 被无限复用,也避免缓存目录持续积累无效 JSON。
+ """
+ material_cache.save_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ items=[self._item()],
+ )
+ cache_path = self._cache_path()
+ now = 2_000_000_000.0
+ expired_mtime = now - material_cache.MATERIAL_SEARCH_CACHE_TTL_SECONDS - 1
+ os.utime(cache_path, (expired_mtime, expired_mtime))
+
+ loaded = material_cache.load_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ now=now,
+ )
+
+ self.assertIsNone(loaded)
+ self.assertFalse(cache_path.exists())
+
+ def test_future_dated_cache_is_removed_and_treated_as_miss(self):
+ """系统时间异常时不能让未来时间戳绕过 24 小时有效期。"""
+ material_cache.save_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ items=[self._item()],
+ )
+ cache_path = self._cache_path()
+ now = 2_000_000_000.0
+ future_mtime = now + 60
+ os.utime(cache_path, (future_mtime, future_mtime))
+
+ loaded = material_cache.load_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ now=now,
+ )
+
+ self.assertIsNone(loaded)
+ self.assertFalse(cache_path.exists())
+
+ def test_corrupted_cache_is_removed_without_breaking_search(self):
+ """
+ 进程异常退出、磁盘故障或用户手动修改都可能留下损坏文件。读取失败应回退
+ 到远端搜索并清理坏文件,不能让一个缓存永久阻断素材生成。
+ """
+ cache_path = self._cache_path()
+ cache_path.write_text("{invalid-json", encoding="utf-8")
+
+ with patch("app.services.material_cache.logger.warning") as warning:
+ loaded = material_cache.load_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+
+ self.assertIsNone(loaded)
+ self.assertFalse(cache_path.exists())
+ self.assertTrue(warning.called)
+
+ def test_empty_results_are_not_cached(self):
+ """
+ 当前 provider 接口用 [] 同时表示没有结果和请求失败。缓存空列表会把
+ Cloudflare 拦截或短暂网络故障固化 24 小时,因此只能缓存非空成功结果。
+ """
+ saved = material_cache.save_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ items=[],
+ )
+
+ self.assertFalse(saved)
+ self.assertEqual(list(Path(self.temp_dir.name).iterdir()), [])
+
+ def test_cache_file_does_not_contain_search_parameters_or_credentials(self):
+ """
+ 缓存文件名使用摘要,内容只保存素材字段。即使用户共享 storage 目录,
+ 文件中也不应出现关键词、API Key 或其它请求配置。
+ """
+ material_cache.save_material_search_cache(
+ provider="pixabay",
+ search_term="private search term",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ items=[self._item()],
+ )
+ cache_files = list(Path(self.temp_dir.name).glob("*.json"))
+
+ self.assertEqual(len(cache_files), 1)
+ self.assertNotIn("private search term", cache_files[0].name)
+ payload = json.loads(cache_files[0].read_text(encoding="utf-8"))
+ self.assertEqual(set(payload), {"version", "items"})
+
+ def test_cache_key_separates_provider_duration_and_aspect(self):
+ """
+ 素材源、最小时长和画幅都会改变远端搜索结果,任何一个参数变化都必须
+ 使用独立缓存,避免把不符合当前任务要求的素材返回给视频生成流程。
+ """
+ base_path = material_cache._cache_path(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+ paths = {
+ base_path,
+ material_cache._cache_path(
+ provider="pexels",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ ),
+ material_cache._cache_path(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=10,
+ video_aspect=VideoAspect.portrait,
+ ),
+ material_cache._cache_path(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.landscape,
+ ),
+ }
+
+ self.assertEqual(len(paths), 4)
+
+ def test_search_wrapper_reuses_cache_across_calls(self):
+ """
+ 第一次调用远端搜索并写缓存,第二次相同参数必须直接复用磁盘结果。
+ 这是减少 Pixabay API 调用和 Cloudflare 风控触发概率的核心行为。
+ """
+ remote_search = Mock(return_value=[self._item()])
+
+ first = material._search_videos_with_cache(
+ provider="pixabay",
+ search_videos=remote_search,
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+ second = material._search_videos_with_cache(
+ provider="pixabay",
+ search_videos=remote_search,
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+
+ self.assertEqual(remote_search.call_count, 1)
+ self.assertEqual(first, second)
+
+ def test_search_wrapper_retries_after_empty_result(self):
+ """空结果不缓存,下一次调用仍应访问远端,以便临时故障恢复后自动重试。"""
+ remote_search = Mock(return_value=[])
+
+ for _ in range(2):
+ results = material._search_videos_with_cache(
+ provider="pixabay",
+ search_videos=remote_search,
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+ self.assertEqual(results, [])
+
+ self.assertEqual(remote_search.call_count, 2)
+
+ def test_cache_read_failure_falls_back_to_remote_search(self):
+ """缓存读取异常只能降级为未命中,不能阻断远端素材搜索。"""
+ remote_items = [self._item()]
+ remote_search = Mock(return_value=remote_items)
+
+ with patch.object(
+ material_cache,
+ "load_material_search_cache",
+ side_effect=RuntimeError("cache read failed"),
+ ), patch.object(material_cache.logger, "warning") as warning:
+ results = material._search_videos_with_cache(
+ provider="pixabay",
+ search_videos=remote_search,
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+
+ self.assertEqual(results, remote_items)
+ self.assertEqual(remote_search.call_count, 1)
+ self.assertTrue(warning.called)
+
+ def test_cache_write_failure_keeps_remote_results(self):
+ """远端搜索成功后,即使缓存写入失败也必须继续返回可用素材。"""
+ remote_items = [self._item()]
+ remote_search = Mock(return_value=remote_items)
+
+ with patch.object(
+ material_cache,
+ "load_material_search_cache",
+ return_value=None,
+ ), patch.object(
+ material_cache,
+ "save_material_search_cache",
+ side_effect=RuntimeError("cache write failed"),
+ ), patch.object(material_cache.logger, "warning") as warning:
+ results = material._search_videos_with_cache(
+ provider="pixabay",
+ search_videos=remote_search,
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+
+ self.assertEqual(results, remote_items)
+ self.assertEqual(remote_search.call_count, 1)
+ self.assertTrue(warning.called)
+
+ def test_invalid_cache_item_does_not_raise(self):
+ """异常素材对象不能让可选缓存写入破坏调用方主流程。"""
+ with patch.object(material_cache.logger, "warning") as warning:
+ saved = material_cache.save_material_search_cache(
+ provider="pixabay",
+ search_term="nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ items=[None],
+ )
+
+ self.assertFalse(saved)
+ self.assertTrue(warning.called)
+
+ def test_concurrent_identical_searches_share_remote_request(self):
+ """
+ API 服务允许多个任务并发。相同条件首次搜索时,后到线程应等待首个线程
+ 写入缓存,而不是再次消耗第三方接口额度。
+ """
+ remote_started = threading.Event()
+ allow_remote_finish = threading.Event()
+ remote_call_lock = threading.Lock()
+ remote_call_count = 0
+ results = []
+
+ def remote_search(**_kwargs):
+ nonlocal remote_call_count
+ with remote_call_lock:
+ remote_call_count += 1
+ remote_started.set()
+ self.assertTrue(allow_remote_finish.wait(timeout=2))
+ return [self._item()]
+
+ def run_search():
+ results.append(
+ material._search_videos_with_cache(
+ provider="pixabay",
+ search_videos=remote_search,
+ search_term="shared nature",
+ minimum_duration=5,
+ video_aspect=VideoAspect.portrait,
+ )
+ )
+
+ first_thread = threading.Thread(target=run_search)
+ second_thread = threading.Thread(target=run_search)
+ first_thread.start()
+ self.assertTrue(remote_started.wait(timeout=2))
+ second_thread.start()
+ # 给第二个线程时间进入缓存锁等待区,确保测试覆盖真实并发未命中。
+ time.sleep(0.05)
+ allow_remote_finish.set()
+ first_thread.join(timeout=2)
+ second_thread.join(timeout=2)
+
+ self.assertFalse(first_thread.is_alive())
+ self.assertFalse(second_thread.is_alive())
+ self.assertEqual(remote_call_count, 1)
+ self.assertEqual(len(results), 2)
+ self.assertEqual(results[0], results[1])
+
+ def test_cleanup_removes_expired_entries_only(self):
+ """低频清理只删除过期缓存,不应影响有效缓存或用户的其它文件。"""
+ stale_path = self._cache_path()
+ stale_path.write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "items": [
+ {
+ "provider": "pixabay",
+ "url": "https://example.com/stale.mp4",
+ "duration": 12,
+ }
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ fresh_path = material_cache._cache_path(
+ provider="pexels",
+ search_term="fresh",
+ minimum_duration=5,
+ video_aspect=VideoAspect.landscape,
+ )
+ fresh_path.write_text("{}", encoding="utf-8")
+ unrelated_path = Path(self.temp_dir.name) / "notes.json"
+ unrelated_path.write_text("keep", encoding="utf-8")
+
+ now = 2_000_000_000.0
+ stale_mtime = now - material_cache.MATERIAL_SEARCH_CACHE_TTL_SECONDS - 1
+ os.utime(stale_path, (stale_mtime, stale_mtime))
+ os.utime(fresh_path, (now - 60, now - 60))
+
+ deleted = material_cache.cleanup_expired_material_search_cache(
+ now=now,
+ force=True,
+ )
+
+ self.assertEqual(deleted, 1)
+ self.assertFalse(stale_path.exists())
+ self.assertTrue(fresh_path.exists())
+ self.assertTrue(unrelated_path.exists())
+
+
+if __name__ == "__main__":
+ unittest.main()