利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix(material): validate video cache writes (#1416)
Rudy Mizrahi Celeklid00d37fb17f8
diff --git a/app/services/cache_manager.py b/app/services/cache_manager.pyindex 62475893..a1c1280a 100644--- a/app/services/cache_manager.py+++ b/app/services/cache_manager.py@@ -16,6 +16,12 @@ from app.utils import utils# 在线素材使用 URL 的 MD5 作为稳定文件名。缓存管理只接受该命名格式,避免把# 用户误放到目录中的视频、说明文件或其它业务文件当作缓存删除。_VIDEO_CACHE_FILE_PATTERN = re.compile(r"^vid-[0-9a-f]{32}\.mp4$")+# save_video writes to this same-directory name before atomically publishing a+# validated clip. A forced process exit can leave one behind, so include only+# aged temporary files in cache cleanup and never touch an active download.+_VIDEO_CACHE_TEMP_FILE_PATTERN = re.compile(+ r"^\.vid-[0-9a-f]{32}-[a-z0-9_]+\.mp4$"+)_SECONDS_PER_DAY = 24 * 60 * 60@@ -54,7 +60,7 @@ def video_cache_dir() -> str:return os.path.realpath(utils.storage_dir("cache_videos"))-def _iter_video_cache_entries() -> Iterator[_VideoCacheEntry]:+def _iter_video_cache_entries(include_temp: bool = False) -> Iterator[_VideoCacheEntry]:"""顺序扫描默认缓存目录第一层。@@ -76,7 +82,9 @@ def _iter_video_cache_entries() -> Iterator[_VideoCacheEntry]:with entries:for entry in entries:- if not _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name):+ if not _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name) and not (+ include_temp and _VIDEO_CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name)+ ):continuetry:@@ -103,6 +111,11 @@ def _is_cleanup_candidate(max_age_days: int | None,now: float,) -> bool:+ if (+ _VIDEO_CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name)+ and entry.mtime >= now - _SECONDS_PER_DAY+ ):+ return Falseif max_age_days is None:return Truereturn entry.mtime < now - max_age_days * _SECONDS_PER_DAY@@ -135,7 +148,7 @@ def get_video_cache_stats(max_age_days: int | None = None) -> VideoCacheStats:oldest_mtime = Nonenewest_mtime = None- for entry in _iter_video_cache_entries():+ for entry in _iter_video_cache_entries(include_temp=True):if not _is_cleanup_candidate(entry, max_age_days, now):continuefile_count += 1@@ -180,7 +193,7 @@ def clean_video_cache(max_age_days: int | None = None) -> VideoCacheCleanupResul# 边扫描边删除,不在内存中保留完整候选列表。即使目录增长到几十万个文件,# 清理过程的额外内存仍保持常量级;执行时使用统一 now,避免长清理过程中# 截止时间不断移动而产生不可预测的候选范围。- for entry in _iter_video_cache_entries():+ for entry in _iter_video_cache_entries(include_temp=True):if not _is_cleanup_candidate(entry, max_age_days, now):continuecandidate_count += 1@@ -190,7 +203,10 @@ def clean_video_cache(max_age_days: int | None = None) -> VideoCacheCleanupResul# 文件名,防止未来修改扫描逻辑时意外扩大可删除范围。if (os.path.realpath(os.path.dirname(entry.path)) != cache_dir- or not _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name)+ or not (+ _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name)+ or _VIDEO_CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name)+ )or os.path.islink(entry.path)):raise ValueError("cache file is outside the managed directory")diff --git a/app/services/material.py b/app/services/material.pyindex 7797013e..37bad973 100644--- a/app/services/material.py+++ b/app/services/material.py@@ -3,6 +3,7 @@ import ioimport mathimport osimport random+import tempfileimport threadingimport timeimport uuid@@ -1053,43 +1054,66 @@ def save_video(video_url: str, save_dir: str = "") -> str:"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}- # if video does not exist, download it- with open(video_path, "wb") as f:- f.write(- requests.get(- video_url,- headers=headers,- proxies=config.proxy,- verify=_get_tls_verify(),- timeout=(60, 240),- ).content- )+ # A nonempty file is treated as a cache hit above. Keep the final path+ # unpublished until the complete download has passed media validation, so a+ # failed download cannot poison this or another concurrent task's cache.+ temp_path = ""+ try:+ with tempfile.NamedTemporaryFile(+ mode="wb",+ prefix=f".{video_id}-",+ suffix=".mp4",+ dir=save_dir,+ delete=False,+ ) as temp_file:+ temp_path = temp_file.name+ temp_file.write(+ requests.get(+ video_url,+ headers=headers,+ proxies=config.proxy,+ verify=_get_tls_verify(),+ timeout=(60, 240),+ ).content+ )++ if os.path.getsize(temp_path) == 0:+ return ""- if os.path.exists(video_path) and os.path.getsize(video_path) > 0:clip = Nonetry:- clip = VideoFileClip(video_path)+ clip = VideoFileClip(temp_path)duration = clip.durationfps = clip.fps- if duration > 0 and fps > 0:- return video_path+ if not (duration > 0 and fps > 0):+ logger.warning(f"invalid video file: {temp_path} => invalid duration or fps")+ return ""except Exception as e:- logger.warning(f"invalid video file: {video_path} => {str(e)}")- try:- os.remove(video_path)- except Exception as remove_error:- logger.warning(- f"failed to remove invalid video file: {video_path}, error: {str(remove_error)}"- )+ logger.warning(f"invalid video file: {temp_path} => {str(e)}")+ return ""finally:if clip is not None:try:clip.close()except Exception as close_error:logger.warning(- f"failed to close video clip: {video_path}, error: {str(close_error)}"+ f"failed to close video clip: {temp_path}, error: {str(close_error)}")- return ""++ os.replace(temp_path, video_path)+ temp_path = ""+ return video_path+ finally:+ if temp_path:+ try:+ os.remove(temp_path)+ except FileNotFoundError:+ pass+ except OSError as remove_error:+ logger.warning(+ f"failed to remove temporary video file: {temp_path}, "+ f"error: {str(remove_error)}"+ )# OpenAI 兼容文生图(Issue #1274)通过 /images/generations 协议为脚本关键词diff --git a/test/services/test_cache_manager.py b/test/services/test_cache_manager.pyindex 50353c86..552332ba 100644--- a/test/services/test_cache_manager.py+++ b/test/services/test_cache_manager.py@@ -94,6 +94,31 @@ class TestVideoCacheManager(unittest.TestCase):self.assertTrue(first.exists())self.assertFalse(second.exists())+ def test_cleanup_reclaims_only_stale_atomic_downloads(self):+ """An interrupted download must not accumulate without deleting an active one."""+ now = 2_000_000_000.0+ stale = self.cache_dir / f".vid-{'a' * 32}-abcdefgh.mp4"+ active = self.cache_dir / f".vid-{'b' * 32}-abcdefgh.mp4"+ unrelated = self.cache_dir / ".personal-download.mp4"+ for path, mtime in (+ (stale, now - 2 * 86400),+ (active, now - 3600),+ (unrelated, now - 2 * 86400),+ ):+ path.write_bytes(b"unfinished")+ os.utime(path, (mtime, mtime))++ with patch.object(cache_manager.time, "time", return_value=now):+ stats = cache_manager.get_video_cache_stats()+ result = cache_manager.clean_video_cache()++ self.assertEqual(stats.file_count, 1)+ self.assertEqual(stats.total_size, len(b"unfinished"))+ self.assertEqual(result.deleted_count, 1)+ self.assertFalse(stale.exists())+ self.assertTrue(active.exists())+ self.assertTrue(unrelated.exists())+def test_invalid_cleanup_age_is_rejected(self):with self.assertRaises(ValueError):cache_manager.get_video_cache_stats(0)diff --git a/test/services/test_material.py b/test/services/test_material.pyindex 73f19d2d..61013b63 100644--- a/test/services/test_material.py+++ b/test/services/test_material.py@@ -582,6 +582,59 @@ class TestMaterialTlsVerification(unittest.TestCase):self.assertTrue(os.path.exists(video_path))self.assertTrue(get.call_args.kwargs["verify"])+ def test_invalid_download_is_not_reused_as_cached_video(self):+ url = "https://example.com/broken-then-valid.mp4"+ cached_name = f"vid-{material.utils.md5(url)}.mp4"++ class FakeVideoFileClip:+ fps = 24++ def __init__(self, path):+ self.duration = 0 if Path(path).read_bytes() == b"broken" else 1++ def close(self):+ pass++ with tempfile.TemporaryDirectory() as temp_dir:+ with patch(+ "app.services.material.requests.get",+ side_effect=[+ SimpleNamespace(content=b"broken"),+ SimpleNamespace(content=b"valid"),+ ],+ ) as get, patch("app.services.material.VideoFileClip", FakeVideoFileClip):+ self.assertEqual(material.save_video(url, save_dir=temp_dir), "")+ self.assertFalse((Path(temp_dir) / cached_name).exists())+ self.assertEqual(+ material.save_video(url, save_dir=temp_dir),+ str(Path(temp_dir) / cached_name),+ )++ self.assertEqual(get.call_count, 2)+ self.assertEqual((Path(temp_dir) / cached_name).read_bytes(), b"valid")++ def test_nonfinite_video_metadata_is_not_published(self):+ class FakeVideoFileClip:+ duration = float("nan")+ fps = 24++ def __init__(self, path):+ pass++ def close(self):+ pass++ with tempfile.TemporaryDirectory() as temp_dir:+ with patch(+ "app.services.material.requests.get",+ return_value=SimpleNamespace(content=b"bad metadata"),+ ), patch("app.services.material.VideoFileClip", FakeVideoFileClip):+ self.assertEqual(+ material.save_video("https://example.com/nan.mp4", save_dir=temp_dir),+ "",+ )+ self.assertEqual(list(Path(temp_dir).iterdir()), [])+def test_download_videos_accepts_plain_string_concat_mode(self):"""download_videos 可能被服务层或测试直接传入字符串模式,而不是