Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

feat(material): report stock footage download progress

amitk21088b46fb040d44
diff --git a/app/services/material.py b/app/services/material.py
index adda5f1a..5cb1b049 100644
--- a/app/services/material.py
+++ b/app/services/material.py
@@ -1111,6 +1111,28 @@ def _get_downloaded_video_duration(video_path: str) -> float:
return duration
+# requests 流式下载期间不产生任何日志。慢速网络下单个 4K 素材要下载数分钟,
+# 任务看起来像卡死。超过这个间隔仍未下完时记录已下载大小和速度;间隔内
+# 完成的小文件不产生额外日志,避免刷屏。
+_DOWNLOAD_HEARTBEAT_SECONDS = 15.0
+_BYTES_PER_MEGABYTE = 1024 * 1024
+
+
+def _describe_download_progress(
+ downloaded_bytes: int, declared_size: int, elapsed_seconds: float
+) -> str:
+ """把下载进度整理成一段日志文字;服务器未声明大小时不给百分比。"""
+ downloaded_mb = downloaded_bytes / _BYTES_PER_MEGABYTE
+ speed = downloaded_mb / elapsed_seconds if elapsed_seconds > 0 else 0.0
+ if declared_size > 0:
+ percent = min(100, round(downloaded_bytes * 100 / declared_size))
+ return (
+ f"{downloaded_mb:.1f} of {declared_size / _BYTES_PER_MEGABYTE:.1f} MB "
+ f"({percent}%), {speed:.2f} MB/s"
+ )
+ return f"{downloaded_mb:.1f} MB, {speed:.2f} MB/s"
+
+
def save_video(video_url: str, save_dir: str = "") -> str:
if not save_dir:
save_dir = utils.storage_dir("cache_videos")
@@ -1166,12 +1188,26 @@ def save_video(video_url: str, save_dir: str = "") -> str:
raise ValueError("video download exceeds 512 MB limit")
downloaded_bytes = 0
+ started_at = time.monotonic()
+ last_report_at = started_at
for chunk in response.iter_content(chunk_size=1024 * 1024):
if chunk:
downloaded_bytes += len(chunk)
if downloaded_bytes > MAX_VIDEO_DOWNLOAD_BYTES:
raise ValueError("video download exceeds 512 MB limit")
temp_file.write(chunk)
+ now = time.monotonic()
+ if now - last_report_at >= _DOWNLOAD_HEARTBEAT_SECONDS:
+ last_report_at = now
+ # 只记录缓存文件名:下载地址可能带签名或密钥。
+ logger.info(
+ f"downloading video {video_id}.mp4: "
+ + _describe_download_progress(
+ downloaded_bytes,
+ declared_size,
+ now - started_at,
+ )
+ )
if os.path.getsize(temp_path) == 0:
return ""
@@ -1917,18 +1953,84 @@ def _search_terms_in_parallel(
return results
+def _report_material_downloaded(
+ position: int,
+ total: int,
+ item: MaterialInfo,
+ saved_video_path: str,
+ on_downloaded: Callable[[MaterialInfo], None] | None,
+) -> None:
+ """
+ 记录一个素材已下载完成,并通知调用方更新进度。
+
+ 始终在发起下载的线程里调用,逐文件日志和进度更新不依赖下载并发数。
+ """
+ description = os.path.basename(saved_video_path)
+ try:
+ size_mb = os.path.getsize(saved_video_path) / _BYTES_PER_MEGABYTE
+ description += f" ({size_mb:.1f} MB)"
+ except OSError:
+ # 文件大小只是辅助信息,读不到时不影响下载结果。
+ pass
+ logger.info(f"downloaded material {position}/{total}: {description}")
+ if on_downloaded is None:
+ return
+ try:
+ on_downloaded(item)
+ except Exception as exc:
+ # 进度只是展示信息。回调失败(例如状态后端暂时不可用)不能让已经
+ # 成功下载的素材作废,更不能中断整个任务。
+ logger.warning(
+ "failed to report material download progress: "
+ f"error={type(exc).__name__}, detail={exc}"
+ )
+
+
+def _covered_duration_reporter(
+ progress_callback: Callable[[float], None] | None,
+ required_duration: float,
+ max_clip_duration: int,
+) -> Callable[[MaterialInfo], None] | None:
+ """
+ 生成逐素材回调:把“已覆盖的配音时长”换算成 0~1 的下载进度。
+
+ 口径与下载循环的停止条件一致——每个素材最多贡献一个片段时长,覆盖到
+ 所需时长即完成,因此超出部分封顶为 1.0。没有进度回调时返回 None。
+ """
+ if progress_callback is None:
+ return None
+ covered_duration = 0.0
+
+ def report(item: MaterialInfo) -> None:
+ nonlocal covered_duration
+ covered_duration += min(max_clip_duration, item.duration)
+ fraction = 1.0
+ if required_duration > 0:
+ fraction = min(1.0, covered_duration / required_duration)
+ progress_callback(fraction)
+
+ return report
+
+
def _download_materials_in_parallel(
materials: List[tuple[str, MaterialInfo]],
material_directory: str,
+ on_downloaded: Callable[[MaterialInfo], None] | None = None,
) -> list[tuple[str, MaterialInfo, str]]:
- """并行下载一轮素材,保留调用方传入的候选顺序。"""
+ """
+ 并行下载一轮素材,保留调用方传入的候选顺序。
+
+ 每个素材下载成功后记录一条日志并调用 ``on_downloaded``,调用方据此更新
+ 任务进度。回调和日志都发生在调用本函数的线程里。
+ """
if not materials:
return []
- workers = min(_get_material_concurrency(), len(materials))
+ total = len(materials)
+ workers = min(_get_material_concurrency(), total)
if workers == 1:
downloaded = []
- for search_term, item in materials:
+ for position, (search_term, item) in enumerate(materials, start=1):
try:
saved_video_path = save_video(
video_url=item.url,
@@ -1944,6 +2046,9 @@ def _download_materials_in_parallel(
continue
if saved_video_path:
downloaded.append((search_term, item, saved_video_path))
+ _report_material_downloaded(
+ position, total, item, saved_video_path, on_downloaded
+ )
return downloaded
futures = {}
@@ -1959,7 +2064,7 @@ def _download_materials_in_parallel(
)] = (search_term, item)
downloaded = []
- for future in futures:
+ for position, future in enumerate(futures, start=1):
search_term, item = futures[future]
try:
saved_video_path = future.result()
@@ -1973,6 +2078,9 @@ def _download_materials_in_parallel(
continue
if saved_video_path:
downloaded.append((search_term, item, saved_video_path))
+ _report_material_downloaded(
+ position, total, item, saved_video_path, on_downloaded
+ )
return downloaded
@@ -2008,7 +2116,15 @@ def download_videos(
audio_duration: float = 0.0,
max_clip_duration: int = 5,
match_script_order: bool = False,
+ progress_callback: Callable[[float], None] | None = None,
) -> List[str]:
+ """
+ 搜索并下载覆盖配音时长所需的素材,返回本地文件路径。
+
+ ``progress_callback`` 可选:库存素材(Pexels、Pixabay、Coverr)每下完一个
+ 文件就以 0~1 的完成比例调用一次,供任务层推进进度条。按需生成的付费
+ 素材源不报告下载进度。
+ """
provider = "pexels"
remote_search_videos = search_videos_pexels
if source == "pixabay":
@@ -2118,6 +2234,7 @@ def download_videos(
audio_duration=audio_duration,
max_clip_duration=max_clip_duration,
material_directory=material_directory,
+ progress_callback=progress_callback,
)
valid_video_items = []
@@ -2146,6 +2263,9 @@ def download_videos(
random.shuffle(valid_video_items)
total_duration = 0.0
+ on_downloaded = _covered_duration_reporter(
+ progress_callback, audio_duration, max_clip_duration
+ )
pending_items = list(valid_video_items)
# 默认/随机路径同样受素材并发配置控制:按串行逻辑的停止条件预选每轮
# 候选(顺序累加、首次超过配音时长即停)。全部下载成功时,每轮下载集合
@@ -2167,6 +2287,7 @@ def download_videos(
downloaded_materials = _download_materials_in_parallel(
materials=[("", item) for item in batch],
material_directory=material_directory,
+ on_downloaded=on_downloaded,
)
pending_items = pending_items[len(batch):]
for _, item, saved_video_path in downloaded_materials:
@@ -2769,6 +2890,7 @@ def _download_videos_by_script_order(
audio_duration: float,
max_clip_duration: int,
material_directory: str,
+ progress_callback: Callable[[float], None] | None = None,
) -> List[str]:
"""
按脚本文案顺序下载素材。
@@ -2809,6 +2931,9 @@ def _download_videos_by_script_order(
video_paths = []
material_sources: list[dict[str, Any]] = []
total_duration = 0.0
+ on_downloaded = _covered_duration_reporter(
+ progress_callback, audio_duration, max_clip_duration
+ )
# 每个关键词独立推进候选下标:只有本轮真正被选中下载的候选才推进,
# 未被选中的候选保留到下一轮,避免被整轮统一的下标跳过。
next_candidate_indices = [0] * len(candidate_groups)
@@ -2844,6 +2969,7 @@ def _download_videos_by_script_order(
downloaded_materials = _download_materials_in_parallel(
materials=selected_materials,
material_directory=material_directory,
+ on_downloaded=on_downloaded,
)
for search_term, item, saved_video_path in downloaded_materials:
try:
diff --git a/app/services/task.py b/app/services/task.py
index 183e1765..a30f7f52 100644
--- a/app/services/task.py
+++ b/app/services/task.py
@@ -290,6 +290,36 @@ def _mark_task_failed(
return failure
+def _stage_progress_reporter(task_id: str, stage_start: float, stage_end: float):
+ """
+ 返回一个回调,把阶段内 0~1 的完成比例换算成任务进度并写入状态。
+
+ 流水线原先只在阶段边界更新进度,下载素材和处理片段这类耗时阶段期间
+ 进度条完全不动,用户无法区分仍在运行与已经卡死。进度只是展示信息:
+ 比例无法解析时忽略,状态后端写入失败时只记录日志,不能因此中断下载或
+ 合成。
+ """
+
+ def report(fraction) -> None:
+ try:
+ fraction = max(0.0, min(1.0, float(fraction)))
+ except (TypeError, ValueError):
+ return
+ try:
+ sm.state.update_task(
+ task_id,
+ state=const.TASK_STATE_PROCESSING,
+ progress=stage_start + (stage_end - stage_start) * fraction,
+ )
+ except Exception as exc:
+ logger.warning(
+ f"failed to update task progress: task_id={task_id}, "
+ f"error={type(exc).__name__}, detail={exc}"
+ )
+
+ return report
+
+
def generate_script(task_id, params):
logger.info("\n\n## generating video script")
video_script = params.video_script.strip()
@@ -751,6 +781,9 @@ def get_video_materials(
audio_duration=audio_duration * params.video_count,
max_clip_duration=params.video_clip_duration,
match_script_order=params.match_materials_to_script,
+ # 素材阶段占用 40%~50%:每下完一个文件推进一次,慢速网络下
+ # 进度条不再长时间停在 40%。
+ progress_callback=_stage_progress_reporter(task_id, 40, 50),
)
except volcengine_seedance.VolcEngineSeedanceError as exc:
# 未确认状态和已生成但下载失败都对应一个可在方舟控制台恢复的远端
diff --git a/test/services/test_material.py b/test/services/test_material.py
index 740f4d7b..6d3fb4a8 100644
--- a/test/services/test_material.py
+++ b/test/services/test_material.py
@@ -807,6 +807,81 @@ class TestMaterialTlsVerification(unittest.TestCase):
self.assertLessEqual(response.chunk_size, 1024 * 1024)
self.assertTrue(get.call_args.kwargs["stream"])
+ def _save_video_with_chunks(self, chunks, headers=None):
+ """用假响应跑一次 save_video,返回期间写出的 info 日志。"""
+
+ class ChunkedResponse(_FakeVideoDownloadResponse):
+ def __init__(self):
+ super().__init__(b"")
+ self.headers = headers or {}
+
+ def iter_content(self, chunk_size):
+ yield from chunks
+
+ class FakeVideoFileClip:
+ duration = 1
+ fps = 24
+
+ def __init__(self, path):
+ pass
+
+ def close(self):
+ pass
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ with (
+ patch(
+ "app.services.material.requests.get",
+ return_value=ChunkedResponse(),
+ ),
+ patch("app.services.material.VideoFileClip", FakeVideoFileClip),
+ patch.object(material.logger, "info") as info,
+ ):
+ video_path = material.save_video(
+ "https://example.com/large.mp4?key=secret", save_dir=temp_dir
+ )
+ self.assertTrue(video_path)
+ return [str(call.args[0]) for call in info.call_args_list]
+
+ def test_save_video_reports_progress_during_a_slow_download(self):
+ """
+ 慢速网络下单个 4K 素材要下载数分钟,期间没有任何日志,任务看起来像
+ 卡死。超过心跳间隔后必须记录已下载大小、总大小和速度;日志里只能出现
+ 缓存文件名,下载地址可能带密钥,不能写进日志。
+ """
+ megabyte = b"x" * (1024 * 1024)
+ with patch.object(material, "_DOWNLOAD_HEARTBEAT_SECONDS", 0):
+ messages = self._save_video_with_chunks(
+ [megabyte, megabyte, megabyte],
+ headers={"Content-Length": str(3 * 1024 * 1024)},
+ )
+
+ heartbeats = [m for m in messages if m.startswith("downloading video")]
+ self.assertEqual(len(heartbeats), 3)
+ self.assertRegex(
+ heartbeats[0],
+ r"^downloading video vid-[0-9a-f]{32}\.mp4: "
+ r"1\.0 of 3\.0 MB \(33%\), \d+\.\d{2} MB/s$",
+ )
+ self.assertIn("3.0 of 3.0 MB (100%)", heartbeats[2])
+ self.assertFalse([m for m in messages if "secret" in m or "example.com" in m])
+
+ def test_save_video_reports_progress_without_a_declared_size(self):
+ """没有 Content-Length 时仍要报告已下载大小,只是给不出百分比。"""
+ megabyte = b"x" * (1024 * 1024)
+ with patch.object(material, "_DOWNLOAD_HEARTBEAT_SECONDS", 0):
+ messages = self._save_video_with_chunks([megabyte, megabyte])
+
+ heartbeats = [m for m in messages if m.startswith("downloading video")]
+ self.assertEqual(len(heartbeats), 2)
+ self.assertRegex(heartbeats[1], r": 2\.0 MB, \d+\.\d{2} MB/s$")
+
+ def test_save_video_stays_quiet_for_fast_downloads(self):
+ """在心跳间隔内完成的下载不应产生额外日志,避免小文件刷屏。"""
+ messages = self._save_video_with_chunks([b"small", b"video"])
+
+ self.assertEqual([m for m in messages if m.startswith("downloading video")], [])
+
def test_save_video_distinguishes_assets_in_download_query(self):
"""Different paid assets can share a /download path and differ only by query."""
first_url = "https://cdn.example.com/download?file_id=first"
@@ -1436,6 +1511,151 @@ class TestMaterialTlsVerification(unittest.TestCase):
# 与串行逻辑的下载集合一致:{a, c},d 不会被多下载。
self.assertEqual(result, ["/tmp/a.mp4", "/tmp/c.mp4"])
+ def _download_with_progress(self, *, concurrency, progress_callback, **kwargs):
+ """
+ 四个 6 秒候选、配音 10 秒:首轮 [a, b] 中 b 失败,次轮用 c 补足。
+ 返回下载结果,供串行、并行和按文案顺序三条路径共用。
+ """
+ items = [
+ material.MaterialInfo(
+ provider="pexels",
+ url=f"https://v.example/{name}.mp4",
+ duration=6,
+ source_info={"provider": "pexels", "asset_id": name},
+ )
+ for name in ("a", "b", "c", "d")
+ ]
+
+ def fake_save_video(video_url, save_dir=""):
+ if "/b.mp4" in video_url:
+ raise RuntimeError("network down")
+ return f"/tmp/{video_url.rsplit('/', 1)[-1]}"
+
+ with (
+ patch.dict(
+ config.app,
+ {"material_directory": "", "material_concurrency": concurrency},
+ ),
+ patch.object(material, "search_videos_pexels", return_value=items),
+ patch.object(material, "save_video", side_effect=fake_save_video),
+ patch.object(
+ material.material_cache,
+ "load_material_search_cache",
+ return_value=None,
+ ),
+ patch.object(material.material_cache, "save_material_search_cache"),
+ patch.object(
+ material.task_artifacts,
+ "patch_script_data",
+ return_value=True,
+ ),
+ ):
+ return material.download_videos(
+ task_id="download-progress",
+ search_terms=["city"],
+ source="pexels",
+ video_concat_mode=material.VideoConcatMode.sequential,
+ audio_duration=10,
+ max_clip_duration=6,
+ progress_callback=progress_callback,
+ **kwargs,
+ )
+
+ def test_download_videos_reports_covered_duration_as_progress(self):
+ """
+ 下载阶段此前只在开始前把进度设为 40%,全部素材下完才跳到 50%,慢速
+ 网络下进度条会长时间停在 40%。每下完一个素材都要报告一次已覆盖配音
+ 时长的比例;失败的候选不计入,覆盖超出需要时封顶为 1.0。
+ """
+ for concurrency in (1, 4):
+ with self.subTest(concurrency=concurrency):
+ fractions = []
+ result = self._download_with_progress(
+ concurrency=concurrency,
+ progress_callback=fractions.append,
+ )
+
+ self.assertEqual(result, ["/tmp/a.mp4", "/tmp/c.mp4"])
+ self.assertEqual(fractions, [0.6, 1.0])
+
+ def test_script_order_download_reports_progress(self):
+ """按文案顺序匹配素材的路径同样要报告下载进度。"""
+ fractions = []
+ result = self._download_with_progress(
+ concurrency=1,
+ progress_callback=fractions.append,
+ match_script_order=True,
+ )
+
+ self.assertTrue(result)
+ self.assertEqual(fractions[-1], 1.0)
+ self.assertEqual(fractions, sorted(fractions))
+
+ def test_failing_progress_callback_does_not_break_download(self):
+ """进度只是展示信息,回调出错(例如状态后端不可用)不能让下载失败。"""
+
+ def broken_callback(_fraction):
+ raise RuntimeError("state backend unavailable")
+
+ with patch.object(material.logger, "warning") as warning:
+ result = self._download_with_progress(
+ concurrency=1,
+ progress_callback=broken_callback,
+ )
+
+ self.assertEqual(result, ["/tmp/a.mp4", "/tmp/c.mp4"])
+ self.assertTrue(
+ [
+ call
+ for call in warning.call_args_list
+ if "progress" in str(call.args[0])
+ ]
+ )
+
+ def test_each_finished_material_is_logged_with_its_position(self):
+ """
+ 一轮下载结束前此前没有任何逐文件日志。每个素材下完都要记录它是本轮
+ 第几个,用户才能从日志判断下载在推进,以及还剩多少。
+ """
+ items = [
+ material.MaterialInfo(
+ provider="pexels",
+ url=f"https://v.example/{name}.mp4",
+ duration=6,
+ )
+ for name in ("a", "b", "c")
+ ]
+ downloaded = []
+
+ def fake_save_video(video_url, save_dir=""):
+ if "/b.mp4" in video_url:
+ return ""
+ return f"/tmp/{video_url.rsplit('/', 1)[-1]}"
+
+ with (
+ patch.dict(config.app, {"material_concurrency": 1}),
+ patch.object(material, "save_video", side_effect=fake_save_video),
+ patch.object(material.logger, "info") as info,
+ ):
+ material._download_materials_in_parallel(
+ materials=[("city", item) for item in items],
+ material_directory="",
+ on_downloaded=lambda item: downloaded.append(item.url),
+ )
+
+ messages = [str(call.args[0]) for call in info.call_args_list]
+ self.assertEqual(
+ [m for m in messages if m.startswith("downloaded material")],
+ [
+ "downloaded material 1/3: a.mp4",
+ "downloaded material 3/3: c.mp4",
+ ],
+ )
+ self.assertEqual(
+ downloaded,
+ ["https://v.example/a.mp4", "https://v.example/c.mp4"],
+ )
+
def test_parallel_download_logs_belong_to_the_task_log_scope(self):
"""
并发数大于 1 时素材在 material-download 线程池里下载。这些线程写出的
diff --git a/test/services/test_task.py b/test/services/test_task.py
index 6bfc5f35..c120ccd2 100644
--- a/test/services/test_task.py
+++ b/test/services/test_task.py
@@ -119,6 +119,79 @@ class TestTaskService(unittest.TestCase):
params.video_fit_mode,
)
+ def test_stage_progress_reporter_maps_fraction_into_stage_range(self):
+ """
+ 阶段内 0~1 的完成比例要换算到该阶段占用的进度区间,并保留任务已有的
+ 其它字段。越界或无法解析的比例不能把进度推到区间之外。
+ """
+ state = MemoryState()
+ state.update_task("stage-progress", progress=40, video_subject="咖啡")
+
+ with patch.object(tm.sm, "state", state):
+ report = tm._stage_progress_reporter("stage-progress", 40, 50)
+ for fraction, expected in (
+ (0.0, 40),
+ (0.55, 45),
+ (1.0, 50),
+ (3.0, 50),
+ (-1.0, 40),
+ ):
+ with self.subTest(fraction=fraction):
+ report(fraction)
+ self.assertEqual(
+ state.get_task("stage-progress")["progress"], expected
+ )
+
+ report(0.5)
+ report("not a number")
+ task = state.get_task("stage-progress")
+
+ self.assertEqual(task["progress"], 45)
+ self.assertEqual(task["state"], tm.const.TASK_STATE_PROCESSING)
+ self.assertEqual(task["video_subject"], "咖啡")
+
+ def test_stage_progress_reporter_survives_state_backend_failure(self):
+ """进度只是展示信息,状态后端暂时不可用时不能让下载或合成失败。"""
+ with (
+ patch.object(
+ tm.sm.state, "update_task", side_effect=RuntimeError("redis down")
+ ),
+ patch.object(tm.logger, "warning") as warning,
+ ):
+ tm._stage_progress_reporter("stage-progress", 40, 50)(0.5)
+
+ warning.assert_called_once()
+ self.assertIn("redis down", str(warning.call_args.args[0]))
+
+ def test_material_download_moves_progress_between_40_and_50(self):
+ """
+ 素材下载是慢速网络下最耗时的阶段,此前进度一直停在 40%,全部下完才
+ 跳到 50%。下载过程中报告的完成比例必须反映到任务进度上。
+ """
+ params = VideoParams(video_subject="test", video_source="pexels")
+ state = MemoryState()
+ state.update_task("download-progress", progress=40)
+ observed = []
+
+ def fake_download_videos(**kwargs):
+ for fraction in (0.25, 0.5, 1.0):
+ kwargs["progress_callback"](fraction)
+ observed.append(state.get_task("download-progress")["progress"])
+ return ["a.mp4"]
+
+ with (
+ patch.object(tm.sm, "state", state),
+ patch.object(
+ tm.material, "download_videos", side_effect=fake_download_videos
+ ),
+ ):
+ result = tm.get_video_materials(
+ "download-progress", params, ["scene"], 10
+ )
+
+ self.assertEqual(result, ["a.mp4"])
+ self.assertEqual(observed, [42, 45, 50])
+
def test_generate_final_videos_uses_generated_sonilo_music(self):
"""Sonilo 必须针对每条拼接后的视频生成配乐,并传给最终混音。"""
params = VideoParams(