Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(material): validate video cache writes (#1416)

Rudy Mizrahi Celeklid00d37fb17f8
diff --git a/app/services/cache_manager.py b/app/services/cache_manager.py
index 62475893..a1c1280a 100644
--- a/app/services/cache_manager.py
+++ b/app/services/cache_manager.py
@@ -16,6 +16,12 @@ from app.utils import utils
# 在线素材使用 URL 的 MD5 作为稳定文件名。缓存管理只接受该命名格式,避免把
# 用户误放到目录中的视频、说明文件或其它业务文件当作缓存删除。
_VIDEO_CACHE_FILE_PATTERN = re.compile(r"^vid-[0-9a-f]{32}\.mp4$")
+# save_video writes to this same-directory name before atomically publishing a
+# validated clip. A forced process exit can leave one behind, so include only
+# aged temporary files in cache cleanup and never touch an active download.
+_VIDEO_CACHE_TEMP_FILE_PATTERN = re.compile(
+ r"^\.vid-[0-9a-f]{32}-[a-z0-9_]+\.mp4$"
+)
_SECONDS_PER_DAY = 24 * 60 * 60
@@ -54,7 +60,7 @@ def video_cache_dir() -> str:
return os.path.realpath(utils.storage_dir("cache_videos"))
-def _iter_video_cache_entries() -> Iterator[_VideoCacheEntry]:
+def _iter_video_cache_entries(include_temp: bool = False) -> Iterator[_VideoCacheEntry]:
"""
顺序扫描默认缓存目录第一层。
@@ -76,7 +82,9 @@ def _iter_video_cache_entries() -> Iterator[_VideoCacheEntry]:
with entries:
for entry in entries:
- if not _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name):
+ if not _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name) and not (
+ include_temp and _VIDEO_CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name)
+ ):
continue
try:
@@ -103,6 +111,11 @@ def _is_cleanup_candidate(
max_age_days: int | None,
now: float,
) -> bool:
+ if (
+ _VIDEO_CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name)
+ and entry.mtime >= now - _SECONDS_PER_DAY
+ ):
+ return False
if max_age_days is None:
return True
return entry.mtime < now - max_age_days * _SECONDS_PER_DAY
@@ -135,7 +148,7 @@ def get_video_cache_stats(max_age_days: int | None = None) -> VideoCacheStats:
oldest_mtime = None
newest_mtime = None
- for entry in _iter_video_cache_entries():
+ for entry in _iter_video_cache_entries(include_temp=True):
if not _is_cleanup_candidate(entry, max_age_days, now):
continue
file_count += 1
@@ -180,7 +193,7 @@ def clean_video_cache(max_age_days: int | None = None) -> VideoCacheCleanupResul
# 边扫描边删除,不在内存中保留完整候选列表。即使目录增长到几十万个文件,
# 清理过程的额外内存仍保持常量级;执行时使用统一 now,避免长清理过程中
# 截止时间不断移动而产生不可预测的候选范围。
- for entry in _iter_video_cache_entries():
+ for entry in _iter_video_cache_entries(include_temp=True):
if not _is_cleanup_candidate(entry, max_age_days, now):
continue
candidate_count += 1
@@ -190,7 +203,10 @@ def clean_video_cache(max_age_days: int | None = None) -> VideoCacheCleanupResul
# 文件名,防止未来修改扫描逻辑时意外扩大可删除范围。
if (
os.path.realpath(os.path.dirname(entry.path)) != cache_dir
- or not _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name)
+ or not (
+ _VIDEO_CACHE_FILE_PATTERN.fullmatch(entry.name)
+ or _VIDEO_CACHE_TEMP_FILE_PATTERN.fullmatch(entry.name)
+ )
or os.path.islink(entry.path)
):
raise ValueError("cache file is outside the managed directory")
diff --git a/app/services/material.py b/app/services/material.py
index 7797013e..37bad973 100644
--- a/app/services/material.py
+++ b/app/services/material.py
@@ -3,6 +3,7 @@ import io
import math
import os
import random
+import tempfile
import threading
import time
import uuid
@@ -1053,43 +1054,66 @@ def save_video(video_url: str, save_dir: str = "") -> str:
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
}
- # if video does not exist, download it
- with open(video_path, "wb") as f:
- f.write(
- requests.get(
- video_url,
- headers=headers,
- proxies=config.proxy,
- verify=_get_tls_verify(),
- timeout=(60, 240),
- ).content
- )
+ # A nonempty file is treated as a cache hit above. Keep the final path
+ # unpublished until the complete download has passed media validation, so a
+ # failed download cannot poison this or another concurrent task's cache.
+ temp_path = ""
+ try:
+ with tempfile.NamedTemporaryFile(
+ mode="wb",
+ prefix=f".{video_id}-",
+ suffix=".mp4",
+ dir=save_dir,
+ delete=False,
+ ) as temp_file:
+ temp_path = temp_file.name
+ temp_file.write(
+ requests.get(
+ video_url,
+ headers=headers,
+ proxies=config.proxy,
+ verify=_get_tls_verify(),
+ timeout=(60, 240),
+ ).content
+ )
+
+ if os.path.getsize(temp_path) == 0:
+ return ""
- if os.path.exists(video_path) and os.path.getsize(video_path) > 0:
clip = None
try:
- clip = VideoFileClip(video_path)
+ clip = VideoFileClip(temp_path)
duration = clip.duration
fps = clip.fps
- if duration > 0 and fps > 0:
- return video_path
+ if not (duration > 0 and fps > 0):
+ logger.warning(f"invalid video file: {temp_path} => invalid duration or fps")
+ return ""
except Exception as e:
- logger.warning(f"invalid video file: {video_path} => {str(e)}")
- try:
- os.remove(video_path)
- except Exception as remove_error:
- logger.warning(
- f"failed to remove invalid video file: {video_path}, error: {str(remove_error)}"
- )
+ logger.warning(f"invalid video file: {temp_path} => {str(e)}")
+ return ""
finally:
if clip is not None:
try:
clip.close()
except Exception as close_error:
logger.warning(
- f"failed to close video clip: {video_path}, error: {str(close_error)}"
+ f"failed to close video clip: {temp_path}, error: {str(close_error)}"
)
- return ""
+
+ os.replace(temp_path, video_path)
+ temp_path = ""
+ return video_path
+ finally:
+ if temp_path:
+ try:
+ os.remove(temp_path)
+ except FileNotFoundError:
+ pass
+ except OSError as remove_error:
+ logger.warning(
+ f"failed to remove temporary video file: {temp_path}, "
+ f"error: {str(remove_error)}"
+ )
# OpenAI 兼容文生图(Issue #1274)通过 /images/generations 协议为脚本关键词
diff --git a/test/services/test_cache_manager.py b/test/services/test_cache_manager.py
index 50353c86..552332ba 100644
--- a/test/services/test_cache_manager.py
+++ b/test/services/test_cache_manager.py
@@ -94,6 +94,31 @@ class TestVideoCacheManager(unittest.TestCase):
self.assertTrue(first.exists())
self.assertFalse(second.exists())
+ def test_cleanup_reclaims_only_stale_atomic_downloads(self):
+ """An interrupted download must not accumulate without deleting an active one."""
+ now = 2_000_000_000.0
+ stale = self.cache_dir / f".vid-{'a' * 32}-abcdefgh.mp4"
+ active = self.cache_dir / f".vid-{'b' * 32}-abcdefgh.mp4"
+ unrelated = self.cache_dir / ".personal-download.mp4"
+ for path, mtime in (
+ (stale, now - 2 * 86400),
+ (active, now - 3600),
+ (unrelated, now - 2 * 86400),
+ ):
+ path.write_bytes(b"unfinished")
+ os.utime(path, (mtime, mtime))
+
+ with patch.object(cache_manager.time, "time", return_value=now):
+ stats = cache_manager.get_video_cache_stats()
+ result = cache_manager.clean_video_cache()
+
+ self.assertEqual(stats.file_count, 1)
+ self.assertEqual(stats.total_size, len(b"unfinished"))
+ self.assertEqual(result.deleted_count, 1)
+ self.assertFalse(stale.exists())
+ self.assertTrue(active.exists())
+ self.assertTrue(unrelated.exists())
+
def test_invalid_cleanup_age_is_rejected(self):
with self.assertRaises(ValueError):
cache_manager.get_video_cache_stats(0)
diff --git a/test/services/test_material.py b/test/services/test_material.py
index 73f19d2d..61013b63 100644
--- a/test/services/test_material.py
+++ b/test/services/test_material.py
@@ -582,6 +582,59 @@ class TestMaterialTlsVerification(unittest.TestCase):
self.assertTrue(os.path.exists(video_path))
self.assertTrue(get.call_args.kwargs["verify"])
+ def test_invalid_download_is_not_reused_as_cached_video(self):
+ url = "https://example.com/broken-then-valid.mp4"
+ cached_name = f"vid-{material.utils.md5(url)}.mp4"
+
+ class FakeVideoFileClip:
+ fps = 24
+
+ def __init__(self, path):
+ self.duration = 0 if Path(path).read_bytes() == b"broken" else 1
+
+ def close(self):
+ pass
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ with patch(
+ "app.services.material.requests.get",
+ side_effect=[
+ SimpleNamespace(content=b"broken"),
+ SimpleNamespace(content=b"valid"),
+ ],
+ ) as get, patch("app.services.material.VideoFileClip", FakeVideoFileClip):
+ self.assertEqual(material.save_video(url, save_dir=temp_dir), "")
+ self.assertFalse((Path(temp_dir) / cached_name).exists())
+ self.assertEqual(
+ material.save_video(url, save_dir=temp_dir),
+ str(Path(temp_dir) / cached_name),
+ )
+
+ self.assertEqual(get.call_count, 2)
+ self.assertEqual((Path(temp_dir) / cached_name).read_bytes(), b"valid")
+
+ def test_nonfinite_video_metadata_is_not_published(self):
+ class FakeVideoFileClip:
+ duration = float("nan")
+ fps = 24
+
+ def __init__(self, path):
+ pass
+
+ def close(self):
+ pass
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ with patch(
+ "app.services.material.requests.get",
+ return_value=SimpleNamespace(content=b"bad metadata"),
+ ), patch("app.services.material.VideoFileClip", FakeVideoFileClip):
+ self.assertEqual(
+ material.save_video("https://example.com/nan.mp4", save_dir=temp_dir),
+ "",
+ )
+ self.assertEqual(list(Path(temp_dir).iterdir()), [])
+
def test_download_videos_accepts_plain_string_concat_mode(self):
"""
download_videos 可能被服务层或测试直接传入字符串模式,而不是