Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

feat: match video materials to script order

harry0703552f0563127c
diff --git a/app/models/schema.py b/app/models/schema.py
index eee96618..967aabb6 100644
--- a/app/models/schema.py
+++ b/app/models/schema.py
@@ -77,6 +77,7 @@ class VideoParams(BaseModel):
video_concat_mode: Optional[VideoConcatMode] = VideoConcatMode.random.value
video_transition_mode: Optional[VideoTransitionMode] = None
video_clip_duration: Optional[int] = 5
+ match_materials_to_script: bool = False
video_count: Optional[int] = 1
video_source: Optional[str] = "pexels"
diff --git a/app/services/llm.py b/app/services/llm.py
index 911ba041..69674168 100644
--- a/app/services/llm.py
+++ b/app/services/llm.py
@@ -718,12 +718,48 @@ def _strip_code_fence(text: str) -> str:
return t.strip()
-def generate_terms(video_subject: str, video_script: str, amount: int = 5) -> List[str]:
+def generate_terms(
+ video_subject: str,
+ video_script: str,
+ amount: int = 5,
+ match_script_order: bool = False,
+) -> List[str]:
+ if match_script_order:
+ goal = (
+ f"Generate {amount} chronological stock-video search terms that follow "
+ "the order of topics in the video script."
+ )
+ ordering_rule = (
+ "6. keep the terms in the same order as the script narration; "
+ "earlier terms must describe earlier visual moments."
+ )
+ # 有序关键词模式下,示例数量要和 amount 保持一致,避免模型被固定
+ # 的 4 个示例误导,导致长文案只返回少量关键词,影响素材覆盖度。
+ example_terms = [
+ "opening visual topic",
+ *[
+ f"script visual topic {index}"
+ for index in range(2, max(amount, 1))
+ ],
+ "final visual topic",
+ ]
+ output_example = json.dumps(example_terms[:amount], ensure_ascii=False)
+ else:
+ goal = (
+ f"Generate {amount} search terms for stock videos, depending on the "
+ "subject of a video."
+ )
+ ordering_rule = ""
+ output_example = (
+ '["search term 1", "search term 2", "search term 3",'
+ '"search term 4", "search term 5"]'
+ )
+
prompt = f"""
# Role: Video Search Terms Generator
## Goals:
-Generate {amount} search terms for stock videos, depending on the subject of a video.
+{goal}
## Constrains:
1. the search terms are to be returned as a json-array of strings.
@@ -731,9 +767,10 @@ Generate {amount} search terms for stock videos, depending on the subject of a v
3. you must only return the json-array of strings. you must not return anything else. you must not return the script.
4. the search terms must be related to the subject of the video.
5. reply with english search terms only.
+{ordering_rule}
## Output Example:
-["search term 1", "search term 2", "search term 3","search term 4","search term 5"]
+{output_example}
## Context:
### Video Subject
@@ -745,7 +782,9 @@ Generate {amount} search terms for stock videos, depending on the subject of a v
Please note that you must use English for generating video search terms; Chinese is not accepted.
""".strip()
- logger.info(f"subject: {video_subject}")
+ logger.info(
+ f"subject: {video_subject}, match_script_order: {match_script_order}"
+ )
search_terms = []
response = ""
diff --git a/app/services/material.py b/app/services/material.py
index f3cbafe1..caf4a954 100644
--- a/app/services/material.py
+++ b/app/services/material.py
@@ -309,16 +309,34 @@ def download_videos(
video_concat_mode: VideoConcatMode = VideoConcatMode.random,
audio_duration: float = 0.0,
max_clip_duration: int = 5,
+ match_script_order: bool = False,
) -> List[str]:
- valid_video_items = []
- valid_video_urls = []
- found_duration = 0.0
search_videos = search_videos_pexels
if source == "pixabay":
search_videos = search_videos_pixabay
elif source == "coverr":
search_videos = search_videos_coverr
+ material_directory = config.app.get("material_directory", "").strip()
+ if material_directory == "task":
+ material_directory = utils.task_dir(task_id)
+ elif material_directory and not os.path.isdir(material_directory):
+ material_directory = ""
+
+ if match_script_order:
+ return _download_videos_by_script_order(
+ task_id=task_id,
+ search_terms=search_terms,
+ search_videos=search_videos,
+ video_aspect=video_aspect,
+ audio_duration=audio_duration,
+ max_clip_duration=max_clip_duration,
+ material_directory=material_directory,
+ )
+
+ valid_video_items = []
+ valid_video_urls = []
+ found_duration = 0.0
for search_term in search_terms:
video_items = search_videos(
search_term=search_term,
@@ -338,12 +356,6 @@ def download_videos(
)
video_paths = []
- material_directory = config.app.get("material_directory", "").strip()
- if material_directory == "task":
- material_directory = utils.task_dir(task_id)
- elif material_directory and not os.path.isdir(material_directory):
- material_directory = ""
-
concat_mode_value = getattr(video_concat_mode, "value", video_concat_mode)
if concat_mode_value == VideoConcatMode.random.value:
random.shuffle(valid_video_items)
@@ -371,6 +383,93 @@ def download_videos(
return video_paths
+def _download_videos_by_script_order(
+ task_id: str,
+ search_terms: List[str],
+ search_videos,
+ video_aspect: VideoAspect,
+ audio_duration: float,
+ max_clip_duration: int,
+ material_directory: str,
+) -> List[str]:
+ """
+ 按脚本文案顺序下载素材。
+
+ 默认下载逻辑会把所有关键词的候选素材合并成一个大列表;如果第一个
+ 关键词返回很多结果,最终下载时可能一直消耗这个关键词的素材,后续
+ 脚本主题就排不上时间线。这里按关键词分组后轮询下载:
+ 第 1 轮取每个关键词的第 1 个候选,第 2 轮取每个关键词的第 2 个候选。
+ 这样在不重写视频合成引擎的前提下,尽量保证素材顺序贴近文案顺序。
+ """
+ logger.info("downloading videos with script-order material matching")
+ candidate_groups = []
+ valid_video_urls = set()
+ found_duration = 0.0
+
+ for search_term in search_terms:
+ video_items = search_videos(
+ search_term=search_term,
+ minimum_duration=max_clip_duration,
+ video_aspect=video_aspect,
+ )
+ logger.info(f"found {len(video_items)} videos for '{search_term}'")
+
+ term_items = []
+ for item in video_items:
+ if item.url in valid_video_urls:
+ continue
+ term_items.append(item)
+ valid_video_urls.add(item.url)
+ found_duration += item.duration
+
+ if term_items:
+ candidate_groups.append((search_term, term_items))
+
+ logger.info(
+ f"found total ordered video candidates: {sum(len(items) for _, items in candidate_groups)}, "
+ f"required duration: {audio_duration} seconds, found duration: {found_duration} seconds"
+ )
+
+ video_paths = []
+ total_duration = 0.0
+ candidate_index = 0
+ while candidate_groups and total_duration <= audio_duration:
+ has_candidate = False
+ for search_term, term_items in candidate_groups:
+ if candidate_index >= len(term_items):
+ continue
+
+ has_candidate = True
+ item = term_items[candidate_index]
+ try:
+ logger.info(
+ f"downloading ordered video for '{search_term}': {item.url}"
+ )
+ saved_video_path = save_video(
+ video_url=item.url, save_dir=material_directory
+ )
+ if saved_video_path:
+ logger.info(f"video saved: {saved_video_path}")
+ video_paths.append(saved_video_path)
+ total_duration += min(max_clip_duration, item.duration)
+ if total_duration > audio_duration:
+ logger.info(
+ f"total duration of downloaded videos: {total_duration} seconds, skip downloading more"
+ )
+ break
+ except Exception as e:
+ logger.error(
+ f"failed to download ordered video: {utils.to_json(item)} => {str(e)}"
+ )
+
+ if not has_candidate:
+ break
+ candidate_index += 1
+
+ logger.success(f"downloaded {len(video_paths)} ordered videos")
+ return video_paths
+
+
if __name__ == "__main__":
download_videos(
"test123", ["Money Exchange Medium"], audio_duration=100, source="pixabay"
diff --git a/app/services/task.py b/app/services/task.py
index 8a5859db..a87be8f0 100644
--- a/app/services/task.py
+++ b/app/services/task.py
@@ -39,8 +39,14 @@ def generate_terms(task_id, params, video_script):
logger.info("\n\n## generating video terms")
video_terms = params.video_terms
if not video_terms:
+ # 开启素材按文案顺序匹配后,关键词本身也必须按脚本叙事顺序生成;
+ # 否则后续即使顺序下载和顺序拼接,也只能复用一组全局主题词,
+ # 无法改善“后面内容的画面提前出现”的问题。
video_terms = llm.generate_terms(
- video_subject=params.video_subject, video_script=video_script, amount=5
+ video_subject=params.video_subject,
+ video_script=video_script,
+ amount=8 if params.match_materials_to_script else 5,
+ match_script_order=params.match_materials_to_script,
)
else:
if isinstance(video_terms, str):
@@ -178,14 +184,21 @@ def get_video_materials(task_id, params, video_terms, audio_duration):
return [material_info.url for material_info in materials]
else:
logger.info(f"\n\n## downloading videos from {params.video_source}")
+ # 顺序匹配模式只在用户显式开启时生效。这里强制素材下载按关键词顺序
+ # 轮询,避免某个早期关键词下载太多素材,把后续脚本主题挤出最终时间线。
downloaded_videos = material.download_videos(
task_id=task_id,
search_terms=video_terms,
source=params.video_source,
video_aspect=params.video_aspect,
- video_concat_mode=params.video_concat_mode,
+ video_concat_mode=(
+ VideoConcatMode.sequential
+ if params.match_materials_to_script
+ else params.video_concat_mode
+ ),
audio_duration=audio_duration * params.video_count,
max_clip_duration=params.video_clip_duration,
+ match_script_order=params.match_materials_to_script,
)
if not downloaded_videos:
sm.state.update_task(task_id, state=const.TASK_STATE_FAILED)
@@ -201,9 +214,14 @@ def generate_final_videos(
):
final_video_paths = []
combined_video_paths = []
- video_concat_mode = (
- params.video_concat_mode if params.video_count == 1 else VideoConcatMode.random
- )
+ # 多视频生成默认会打散素材以增加差异;但“按文案顺序匹配素材”追求的是
+ # 时间线稳定性和可解释性,所以开启后所有输出都使用顺序拼接。
+ if params.match_materials_to_script:
+ video_concat_mode = VideoConcatMode.sequential
+ elif params.video_count == 1:
+ video_concat_mode = params.video_concat_mode
+ else:
+ video_concat_mode = VideoConcatMode.random
video_transition_mode = params.video_transition_mode
_progress = 50
diff --git a/test/services/test_llm.py b/test/services/test_llm.py
index bec37a6e..b53661ac 100644
--- a/test/services/test_llm.py
+++ b/test/services/test_llm.py
@@ -99,6 +99,30 @@ class TestScriptPromptOptions(unittest.TestCase):
self.assertIn("- number of paragraphs: 2", captured["prompt"])
self.assertIn("开头更有悬念", captured["prompt"])
+ def test_generate_terms_can_request_script_ordered_keywords(self):
+ """
+ 按文案顺序匹配素材依赖 LLM 返回有序关键词。这里不调用真实模型,
+ 只验证服务层会把“按脚本叙事顺序输出”的约束写入 prompt,避免
+ 后续素材下载虽然顺序化,但关键词仍然是全局无序主题词。
+ """
+ captured = {}
+
+ def fake_generate_response(prompt):
+ captured["prompt"] = prompt
+ return '["opening city", "middle office", "final sunset"]'
+
+ with patch.object(llm, "_generate_response", side_effect=fake_generate_response):
+ result = llm.generate_terms(
+ video_subject="startup story",
+ video_script="First city. Then office. Finally sunset.",
+ amount=3,
+ match_script_order=True,
+ )
+
+ self.assertEqual(result, ["opening city", "middle office", "final sunset"])
+ self.assertIn("chronological stock-video search terms", captured["prompt"])
+ self.assertIn("same order as the script narration", captured["prompt"])
+
def test_video_script_request_rejects_invalid_advanced_options(self):
"""
API 请求模型需要限制高级 prompt 参数,避免外部调用绕过 WebUI
diff --git a/test/services/test_material.py b/test/services/test_material.py
index 68cb81cc..d5bd1130 100644
--- a/test/services/test_material.py
+++ b/test/services/test_material.py
@@ -129,6 +129,55 @@ class TestMaterialTlsVerification(unittest.TestCase):
self.assertEqual(result, [])
+ def test_download_videos_can_round_robin_terms_in_script_order(self):
+ """
+ 开启按文案顺序匹配素材后,不能让第一个关键词的多个候选先把
+ 音频时长填满。这里模拟两个关键词各有多个候选,验证下载顺序是
+ term1-第1个、term2-第1个、term1-第2个,贴近脚本叙事顺序。
+ """
+ search_results = {
+ "opening city": [
+ material.MaterialInfo(provider="pexels", url="https://v.example/a1.mp4", duration=3),
+ material.MaterialInfo(provider="pexels", url="https://v.example/a2.mp4", duration=3),
+ ],
+ "middle office": [
+ material.MaterialInfo(provider="pexels", url="https://v.example/b1.mp4", duration=3),
+ material.MaterialInfo(provider="pexels", url="https://v.example/b2.mp4", duration=3),
+ ],
+ }
+ downloaded_urls = []
+
+ def fake_search(search_term, minimum_duration, video_aspect):
+ return search_results[search_term]
+
+ def fake_save_video(video_url, save_dir=""):
+ downloaded_urls.append(video_url)
+ return f"/tmp/{video_url.rsplit('/', 1)[-1]}"
+
+ with (
+ patch.dict(config.app, {"material_directory": ""}),
+ patch.object(material, "search_videos_pexels", side_effect=fake_search),
+ patch.object(material, "save_video", side_effect=fake_save_video),
+ ):
+ result = material.download_videos(
+ task_id="ordered-materials",
+ search_terms=["opening city", "middle office"],
+ source="pexels",
+ audio_duration=7,
+ max_clip_duration=3,
+ match_script_order=True,
+ )
+
+ self.assertEqual(
+ downloaded_urls,
+ [
+ "https://v.example/a1.mp4",
+ "https://v.example/b1.mp4",
+ "https://v.example/a2.mp4",
+ ],
+ )
+ self.assertEqual(result, ["/tmp/a1.mp4", "/tmp/b1.mp4", "/tmp/a2.mp4"])
+
class TestCoverrProvider(unittest.TestCase):
"""
diff --git a/test/services/test_task.py b/test/services/test_task.py
index a645facd..35d49cdc 100644
--- a/test/services/test_task.py
+++ b/test/services/test_task.py
@@ -44,6 +44,28 @@ class TestTaskService(unittest.TestCase):
video_script_prompt="语气轻松",
custom_system_prompt="Only write short narration.",
)
+
+ def test_generate_terms_uses_script_order_mode_when_enabled(self):
+ """
+ 默认模式不受影响;只有用户显式开启素材按文案顺序匹配时,任务层才
+ 要求 LLM 生成有序关键词,并适当增加关键词数量以覆盖更多脚本片段。
+ """
+ params = VideoParams(
+ video_subject="城市通勤",
+ video_script="",
+ match_materials_to_script=True,
+ )
+
+ with patch.object(tm.llm, "generate_terms", return_value=["city", "train"]) as generate:
+ result = tm.generate_terms("task-id", params, "先城市,再地铁")
+
+ self.assertEqual(result, ["city", "train"])
+ generate.assert_called_once_with(
+ video_subject="城市通勤",
+ video_script="先城市,再地铁",
+ amount=8,
+ match_script_order=True,
+ )
def test_task_local_materials(self):
task_id = "00000000-0000-0000-0000-000000000000"
diff --git a/webui/Main.py b/webui/Main.py
index b63ed82a..3ad651b7 100644
--- a/webui/Main.py
+++ b/webui/Main.py
@@ -71,6 +71,10 @@ if "custom_system_prompt" not in st.session_state:
st.session_state["custom_system_prompt"] = llm.DEFAULT_SCRIPT_SYSTEM_PROMPT
if "use_custom_system_prompt" not in st.session_state:
st.session_state["use_custom_system_prompt"] = False
+if "match_materials_to_script" not in st.session_state:
+ st.session_state["match_materials_to_script"] = bool(
+ config.app.get("match_materials_to_script", False)
+ )
if "ui_language" not in st.session_state:
st.session_state["ui_language"] = config.ui.get("language", system_locale)
if "local_video_materials" not in st.session_state:
@@ -682,6 +686,9 @@ middle_panel = panel[1]
right_panel = panel[2]
params = VideoParams(video_subject="")
+params.match_materials_to_script = bool(
+ st.session_state.get("match_materials_to_script", False)
+)
uploaded_files = []
uploaded_audio_file = None
@@ -755,7 +762,12 @@ with left_panel:
video_script_prompt=params.video_script_prompt,
custom_system_prompt=params.custom_system_prompt,
)
- terms = llm.generate_terms(params.video_subject, script)
+ terms = llm.generate_terms(
+ params.video_subject,
+ script,
+ amount=8 if params.match_materials_to_script else 5,
+ match_script_order=params.match_materials_to_script,
+ )
if "Error: " in script:
st.error(tr(script))
elif "Error: " in terms:
@@ -772,7 +784,12 @@ with left_panel:
st.stop()
with st.spinner(tr("Generating Video Keywords")):
- terms = llm.generate_terms(params.video_subject, params.video_script)
+ terms = llm.generate_terms(
+ params.video_subject,
+ params.video_script,
+ amount=8 if params.match_materials_to_script else 5,
+ match_script_order=params.match_materials_to_script,
+ )
if "Error: " in terms:
st.error(tr(terms))
else:
@@ -889,6 +906,15 @@ with middle_panel:
)
with st.expander(tr("Advanced Video Settings"), expanded=False):
+ # 默认关闭,避免影响老用户的随机素材体验。开启后只改变关键词和素材
+ # 下载/拼接顺序,用于改善画面主题早于或晚于旁白的问题。
+ params.match_materials_to_script = st.checkbox(
+ tr("Match Materials to Script Order"),
+ help=tr("Match Materials to Script Order Help"),
+ key="match_materials_to_script",
+ )
+ config.app["match_materials_to_script"] = params.match_materials_to_script
+
video_codec_options = [
("libx264 (CPU)", "libx264"),
("NVIDIA NVENC (h264_nvenc)", "h264_nvenc"),
diff --git a/webui/i18n/en.json b/webui/i18n/en.json
index 48d613ad..25712539 100644
--- a/webui/i18n/en.json
+++ b/webui/i18n/en.json
@@ -43,6 +43,8 @@
"Landscape": "Landscape 16:9",
"Clip Duration": "Maximum Duration of Video Clips (seconds)",
"Number of Videos Generated Simultaneously": "Number of Videos Generated Simultaneously",
+ "Match Materials to Script Order": "Match materials to script order",
+ "Match Materials to Script Order Help": "Disabled by default. Mainly applies to online material sources. When enabled, keywords are generated in script order and materials are downloaded/combined in that order to reduce visual mismatch with the current voiceover.",
"Audio Settings": "**Audio Settings**",
"Speech Synthesis": "Speech Synthesis Voice",
"Speech Region": "Region(:red[Required,[Get Region](https://portal.azure.com/#view/Microsoft_Azure_ProjectOxford/CognitiveServicesHub/~/SpeechServices)])",
diff --git a/webui/i18n/zh.json b/webui/i18n/zh.json
index 481e38ba..2fd96d16 100644
--- a/webui/i18n/zh.json
+++ b/webui/i18n/zh.json
@@ -43,6 +43,8 @@
"Landscape": "横屏 16:9(西瓜视频)",
"Clip Duration": "视频片段最大时长(秒)(**不是视频总长度**,是指每个**合成片段**的长度)",
"Number of Videos Generated Simultaneously": "同时生成视频数量",
+ "Match Materials to Script Order": "按文案顺序匹配素材",
+ "Match Materials to Script Order Help": "默认关闭。主要对在线素材源生效。开启后会按文案叙事顺序生成关键词,并尽量按关键词顺序下载和拼接素材,减少画面主题与当前旁白不匹配的问题。",
"Audio Settings": "**音频设置**",
"Speech Synthesis": "朗读声音(:red[**与文案语言保持一致**。注意:V2版效果更好,但是需要API KEY])",
"Speech Region": "服务区域 (:red[必填,[点击获取](https://portal.azure.com/#view/Microsoft_Azure_ProjectOxford/CognitiveServicesHub/~/SpeechServices)])",