Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(task): preserve completed videos when a batch fails

hdu-r3drum412de0c2dd54
diff --git a/app/services/task.py b/app/services/task.py
index 9bd5d036..6e2545a4 100644
--- a/app/services/task.py
+++ b/app/services/task.py
@@ -60,6 +60,8 @@ _ACTIVE_CROSS_POST_STATES = {
}
_CROSS_POST_STATE_WRITE_ATTEMPTS = 3
_CROSS_POST_STATE_RETRY_DELAY_SECONDS = 0.1
+_TASK_FAILURE_STATE_WRITE_ATTEMPTS = 3
+_TASK_FAILURE_STATE_RETRY_DELAY_SECONDS = 0.1
_LOOMLOOM_STATE_WRITE_ATTEMPTS = 3
_LOOMLOOM_STATE_RETRY_DELAY_SECONDS = 0.1
_INTERRUPTED_CROSS_POST_ERROR = (
@@ -283,18 +285,37 @@ def _mark_task_failed(
}
# 某些外部任务已经创建了可用于恢复或排障的远端 ID。失败状态需要保留
# 这些非敏感字段,但不能允许调用方覆盖统一的状态、进度和错误结构。
+ # Keep completed batch outputs in both the persisted state and the direct
+ # return value used by the CLI. These fields are reset before a new attempt.
failure_details = {
- key: value for key, value in dict(details or {}).items() if key not in failure
+ key: existing_task[key]
+ for key in ("videos", "combined_videos", "warnings", "failed_video_index")
+ if existing_task and key in existing_task
}
+ failure_details.update({
+ key: value for key, value in dict(details or {}).items() if key not in failure
+ })
failure.update(failure_details)
- sm.state.update_task(
- task_id,
- state=failure["state"],
- progress=failure["progress"],
- failed_stage=failure["failed_stage"],
- error=failure["error"],
- **failure_details,
- )
+ fields = {key: value for key, value in failure.items() if key != "task_id"}
+ for attempt in range(_TASK_FAILURE_STATE_WRITE_ATTEMPTS):
+ try:
+ if attempt == 0:
+ sm.state.update_task(task_id, **fields)
+ else:
+ # A lost acknowledgement may mean the first write succeeded.
+ # Patch only an existing record so retries cannot revive a task
+ # that the user has since deleted.
+ sm.state.patch_task(task_id, **fields)
+ break
+ except Exception as exc:
+ if attempt + 1 == _TASK_FAILURE_STATE_WRITE_ATTEMPTS:
+ logger.exception(
+ f"failed to persist task failure, task_id: {task_id}, error: {exc}"
+ )
+ else:
+ time.sleep(_TASK_FAILURE_STATE_RETRY_DELAY_SECONDS)
+ # A state outage must not discard the only completed-file snapshot held by
+ # the caller. Return it even when all bounded persistence attempts fail.
return failure
@@ -948,11 +969,20 @@ def _get_material_source_groups(task_id: str, video_paths: list[str]) -> dict[st
def generate_final_videos(
- task_id, params, downloaded_videos, audio_file, subtitle_path, audio_duration
+ task_id, params, downloaded_videos, audio_file, subtitle_path, audio_duration,
+ *, partial_results: dict | None = None,
):
+ """Record completed outputs even if a later clip raises before returning."""
final_video_paths = []
combined_video_paths = []
warnings = []
+ if partial_results is not None:
+ # The caller owns this snapshot so it can report completed files even
+ # when the state backend fails during a progress/checkpoint write.
+ partial_results.update(
+ videos=final_video_paths, combined_videos=combined_video_paths,
+ warnings=warnings,
+ )
allocate_batch_materials = params.video_count > 1 and params.video_source in {
"pexels", "pixabay", "coverr", "local"
}
@@ -981,6 +1011,8 @@ def generate_final_videos(
_progress = 50
for i in range(params.video_count):
index = i + 1
+ if partial_results is not None:
+ partial_results["failed_video_index"] = index
combined_video_path = path.join(
utils.task_dir(task_id), f"combined-{index}.mp4"
)
@@ -1093,11 +1125,13 @@ def generate_final_videos(
}
)
- _progress += 50 / params.video_count / 2
- sm.state.update_task(task_id, progress=_progress)
-
final_video_paths.append(final_video_path)
combined_video_paths.append(combined_video_path)
+ _progress += 50 / params.video_count / 2
+ sm.state.update_task(
+ task_id, progress=_progress, videos=list(final_video_paths),
+ combined_videos=list(combined_video_paths), warnings=list(warnings) or None,
+ )
return final_video_paths, combined_video_paths, warnings
@@ -1488,6 +1522,7 @@ def _run_pipeline(
sm.state.update_task(
task_id, state=const.TASK_STATE_PROCESSING, progress=5,
failed_stage=None, error=None,
+ videos=[], combined_videos=[], warnings=None, failed_video_index=None,
)
if stop_at == "video" and params.subtitle_enabled:
@@ -1766,16 +1801,28 @@ def _run_pipeline(
params.video_concat_mode = VideoConcatMode(params.video_concat_mode)
# 6. Generate final videos
- final_video_paths, combined_video_paths, generation_warnings = (
- generate_final_videos(
+ partial_results = {}
+ try:
+ final_video_paths, combined_video_paths, generation_warnings = generate_final_videos(
task_id,
params,
downloaded_videos,
audio_file,
subtitle_path,
audio_duration,
+ partial_results=partial_results,
+ )
+ except Exception as exc:
+ logger.exception(f"video batch failed, task_id: {task_id}, error: {exc}")
+ # Warnings for the failing clip do not describe any usable output.
+ completed_count = len(partial_results.get("videos", []))
+ partial_results["warnings"] = [
+ warning for warning in partial_results.get("warnings", [])
+ if warning.get("video_index", 0) <= completed_count
+ ] or None
+ return _mark_task_failed(
+ task_id, "video", f"{type(exc).__name__}: {exc}", details=partial_results,
)
- )
if not final_video_paths:
return _mark_task_failed(
diff --git a/cli.py b/cli.py
index e8ecef62..a021b66e 100644
--- a/cli.py
+++ b/cli.py
@@ -1760,6 +1760,8 @@ def run_cli(argv: Sequence[str] | None = None) -> int:
f"CLI task failed: task_id={task_id}, stop_at={args.stop_at}, "
f"stage={failed_stage}, error={error}"
)
+ if result and result.get("videos"):
+ print(json.dumps({"task_id": task_id, "result": result}, ensure_ascii=False))
return 1
print(json.dumps({"task_id": task_id, "result": result}, ensure_ascii=False))
diff --git a/test/services/test_audio_export_volume.py b/test/services/test_audio_export_volume.py
index 665e13cd..af613b4d 100644
--- a/test/services/test_audio_export_volume.py
+++ b/test/services/test_audio_export_volume.py
@@ -50,7 +50,7 @@ def test_audio_only_export_applies_requested_volume_to_real_pcm(tmp_path, gain):
def test_full_video_keeps_unscaled_narration_for_its_existing_mixer(tmp_path):
params = VideoParams(video_subject='Narration', video_script='A narration.', voice_volume=2.0, subtitle_enabled=False)
observed = []
- def render(*args):
+ def render(*args, **_kwargs):
observed.append(_rms(args[3])[0])
return ['video.mp4'], ['combined.mp4'], []
with (patch.object(tm.sm, 'state', MemoryState()),
diff --git a/test/services/test_cli.py b/test/services/test_cli.py
index ac1e7c33..a46af4fa 100644
--- a/test/services/test_cli.py
+++ b/test/services/test_cli.py
@@ -192,6 +192,31 @@ class TestCli(unittest.TestCase):
self.assertIn("stage=audio", log_error.call_args.args[0])
self.assertIn("TTS request timed out", log_error.call_args.args[0])
+ def test_run_cli_prints_completed_videos_when_a_batch_fails(self):
+ """Partial video results remain machine-readable with a failed exit code."""
+ failure = {
+ "task_id": "partial-cli",
+ "state": -1,
+ "progress": 75,
+ "failed_stage": "video",
+ "failed_video_index": 2,
+ "error": "OSError: encoder failed",
+ "videos": ["final-1.mp4"],
+ "combined_videos": ["combined-1.mp4"],
+ }
+ with (
+ patch("app.services.task.start", return_value=failure),
+ patch("app.utils.utils.get_uuid", return_value="partial-cli"),
+ patch.object(cli, "prepare_cli_files"),
+ patch("builtins.print") as print_mock,
+ ):
+ code = cli.run_cli(["--video-subject", "Coffee", "--video-count", "3"])
+ self.assertEqual(code, 1)
+ print_mock.assert_called_once()
+ self.assertEqual(json.loads(print_mock.call_args.args[0]), {
+ "task_id": "partial-cli", "result": failure,
+ })
+
def test_subtitle_enabled_by_default(self):
args = cli.parse_args(["--video-subject", "test"])
params = cli.build_video_params(args)
diff --git a/test/services/test_partial_batch_results.py b/test/services/test_partial_batch_results.py
new file mode 100644
index 00000000..c128e3fc
--- /dev/null
+++ b/test/services/test_partial_batch_results.py
@@ -0,0 +1,203 @@
+"""Preserve usable results when a later video in the same task fails."""
+
+import os
+import tempfile
+import unittest
+from contextlib import ExitStack
+from pathlib import Path
+from unittest.mock import patch
+
+from app.models.schema import VideoParams
+from app.services import task as tm
+from app.services.state import MemoryState
+
+
+class TestPartialBatchResults(unittest.TestCase):
+ def _run_batch(
+ self, directory, state, fail_at=None, failure_stage="render", stop_at="video",
+ ):
+ task_id = "partial-batch"
+ task_directory = Path(directory) / task_id
+ task_directory.mkdir(exist_ok=True)
+ params = VideoParams(
+ video_subject="Coffee", video_count=3, video_source="local",
+ subtitle_enabled=False, bgm_type="", match_materials_to_script=False,
+ )
+ checkpoints = []
+
+ def combine(**kwargs):
+ index = int(Path(kwargs["combined_video_path"]).stem.split("-")[-1])
+ checkpoints.append(state.get_task(task_id))
+ if index == fail_at and failure_stage == "combine":
+ raise OSError("simulated combine failure")
+ Path(kwargs["combined_video_path"]).write_bytes(b"combined video")
+ kwargs["used_video_paths"].append("material.mp4")
+ kwargs["progress_callback"](1.0)
+
+ def render(**kwargs):
+ output = Path(kwargs["output_file"])
+ index = int(output.stem.split("-")[-1])
+ if index == fail_at and failure_stage == "render":
+ # A leftover file must not be discovered as a completed result.
+ output.write_bytes(b"incomplete encoder output")
+ raise OSError("simulated render failure")
+ output.write_bytes(b"complete encoder output")
+ return True
+
+ with ExitStack() as stack:
+ for owner, name, options in (
+ (tm.sm, "state", {"new": state}),
+ (tm.utils, "task_dir", {"side_effect": lambda task=None: os.path.join(directory, task) if task else directory}),
+ (tm.utils, "check_ffmpeg_ready", {"return_value": True}),
+ (tm, "generate_script", {"return_value": "Coffee script"}),
+ (tm, "save_script_data", {}),
+ (tm, "generate_audio", {"return_value": (
+ ("", 0, None) if failure_stage == "audio" else ("audio.mp3", 5, None)
+ )}),
+ (tm, "generate_subtitle", {"return_value": ""}),
+ (tm, "get_video_materials", {"return_value": ["material.mp4"]}),
+ (tm.task_artifacts, "patch_script_data", {}),
+ (tm.video, "combine_videos", {"side_effect": combine}),
+ (tm.video, "generate_video", {"side_effect": render}),
+ (tm.upload_post.upload_post_service, "is_configured", {"return_value": False}),
+ ):
+ stack.enter_context(patch.object(owner, name, **options))
+ schedule = stack.enter_context(patch.object(tm, "_schedule_cross_post"))
+ result = tm.start(task_id, params, stop_at=stop_at)
+ schedule.assert_not_called()
+ return result, checkpoints
+
+ def test_failure_returns_only_completed_videos_and_the_failed_index(self):
+ for stage in ("combine", "render"):
+ for fail_at in (1, 2, 3):
+ with self.subTest(stage=stage, fail_at=fail_at), tempfile.TemporaryDirectory() as directory:
+ state = MemoryState()
+ result, _ = self._run_batch(directory, state, fail_at, stage)
+ expected_videos = [
+ os.path.join(directory, "partial-batch", f"final-{index}.mp4")
+ for index in range(1, fail_at)
+ ]
+ self.assertEqual(result["videos"], expected_videos)
+ self.assertEqual(result["combined_videos"], [
+ os.path.join(directory, "partial-batch", f"combined-{index}.mp4")
+ for index in range(1, fail_at)
+ ])
+ self.assertEqual(result["failed_video_index"], fail_at)
+ self.assertEqual(result["state"], tm.const.TASK_STATE_FAILED)
+ self.assertEqual(result["failed_stage"], "video")
+ self.assertEqual(result["error"], f"OSError: simulated {stage} failure")
+ self.assertLess(result["progress"], 100)
+ self.assertEqual(result, state.get_task("partial-batch"))
+ self.assertTrue(all(Path(video).is_file() for video in result["videos"]))
+ self.assertTrue(all(
+ warning["video_index"] < fail_at
+ for warning in result.get("warnings") or []
+ ))
+
+ def test_completed_video_is_queryable_before_the_next_video_starts(self):
+ with tempfile.TemporaryDirectory() as directory:
+ state = MemoryState()
+ result, checkpoints = self._run_batch(directory, state)
+ self.assertEqual(len(result["videos"]), 3)
+ self.assertEqual(state.get_task("partial-batch")["state"], tm.const.TASK_STATE_COMPLETE)
+ self.assertEqual(checkpoints[1]["videos"], result["videos"][:1])
+ self.assertEqual(checkpoints[2]["videos"], result["videos"][:2])
+ self.assertEqual(checkpoints[1]["state"], tm.const.TASK_STATE_PROCESSING)
+ self.assertNotIn("failed_video_index", result)
+
+ def test_retry_does_not_return_videos_from_an_earlier_attempt(self):
+ with tempfile.TemporaryDirectory() as directory:
+ state = MemoryState()
+ first, _ = self._run_batch(directory, state, fail_at=3)
+ self.assertEqual(len(first["videos"]), 2)
+ second, _ = self._run_batch(directory, state, fail_at=1)
+ self.assertEqual(second["videos"], [])
+ self.assertEqual(second["combined_videos"], [])
+ self.assertEqual(second["failed_video_index"], 1)
+ self.assertIsNone(second.get("warnings"))
+ self.assertEqual(second, state.get_task("partial-batch"))
+
+ def test_failed_task_query_returns_download_urls_for_completed_videos(self):
+ from app.controllers.v1 import video as controller
+ from app.models.schema import TaskQueryResponse
+
+ with tempfile.TemporaryDirectory() as directory:
+ state = MemoryState()
+ result, _ = self._run_batch(directory, state, fail_at=2)
+ public_result = controller._task_response_data(result, "", directory, "request")
+ response = TaskQueryResponse(data=public_result).model_dump()
+ data = response["data"]
+ self.assertEqual(data["state"], tm.const.TASK_STATE_FAILED)
+ self.assertEqual(data["failed_video_index"], 2)
+ self.assertEqual(data["videos"], ["/tasks/partial-batch/final-1.mp4"])
+ self.assertEqual(data["combined_videos"], ["/tasks/partial-batch/combined-1.mp4"])
+
+ def test_checkpoint_error_still_returns_the_completed_file(self):
+ class FailOnceState(MemoryState):
+ failed_checkpoint = False
+
+ def update_task(self, task_id, *args, **kwargs):
+ if (
+ kwargs.get("videos")
+ and kwargs.get("state", tm.const.TASK_STATE_PROCESSING)
+ == tm.const.TASK_STATE_PROCESSING
+ and not self.failed_checkpoint
+ ):
+ self.failed_checkpoint = True
+ raise OSError("checkpoint unavailable")
+ return super().update_task(task_id, *args, **kwargs)
+
+ with tempfile.TemporaryDirectory() as directory:
+ state = FailOnceState()
+ result, checkpoints = self._run_batch(directory, state)
+ self.assertEqual(len(checkpoints), 1)
+ self.assertEqual(result["videos"], [
+ os.path.join(directory, "partial-batch", "final-1.mp4"),
+ ])
+ self.assertEqual(result["state"], tm.const.TASK_STATE_FAILED)
+ self.assertEqual(result["error"], "OSError: checkpoint unavailable")
+ self.assertEqual(result, state.get_task("partial-batch"))
+
+ def test_audio_retry_does_not_return_an_earlier_batchs_videos(self):
+ with tempfile.TemporaryDirectory() as directory:
+ state = MemoryState()
+ first, _ = self._run_batch(directory, state, fail_at=3)
+ self.assertEqual(len(first["videos"]), 2)
+ second, _ = self._run_batch(
+ directory, state, failure_stage="audio", stop_at="audio",
+ )
+ self.assertEqual(second["failed_stage"], "audio")
+ self.assertEqual(second["videos"], [])
+ self.assertEqual(second["combined_videos"], [])
+ self.assertIsNone(second.get("failed_video_index"))
+
+ def test_repeated_state_errors_do_not_discard_the_completed_snapshot(self):
+ class RecoveringState(MemoryState):
+ failed_writes = 0
+
+ def update_task(self, task_id, *args, **kwargs):
+ if kwargs.get("videos") and self.failed_writes < 2:
+ self.failed_writes += 1
+ raise OSError("state temporarily unavailable")
+ return super().update_task(task_id, *args, **kwargs)
+
+ def patch_task(self, task_id, **kwargs):
+ if kwargs.get("videos") and self.failed_writes < 2:
+ self.failed_writes += 1
+ raise OSError("state temporarily unavailable")
+ return super().patch_task(task_id, **kwargs)
+
+ with tempfile.TemporaryDirectory() as directory:
+ state = RecoveringState()
+ result, checkpoints = self._run_batch(directory, state)
+ self.assertEqual(len(checkpoints), 1)
+ self.assertEqual(result["videos"], [
+ os.path.join(directory, "partial-batch", "final-1.mp4"),
+ ])
+ self.assertEqual(result["state"], tm.const.TASK_STATE_FAILED)
+ self.assertEqual(result["error"], "OSError: state temporarily unavailable")
+ self.assertEqual(result, state.get_task("partial-batch"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/services/test_webui_task.py b/test/services/test_webui_task.py
index a72c20ae..7933d585 100644
--- a/test/services/test_webui_task.py
+++ b/test/services/test_webui_task.py
@@ -244,8 +244,16 @@ def test_task_summary_tolerates_directory_removed_during_scan():
({"open_task_folder_on_completion": False}, 0),
],
)
-def test_completed_task_renders_subject_named_video_download(
- tmp_path, ui_config, expected_open_count
+ ("task_state", "has_video"),
+ [
+ (const.TASK_STATE_COMPLETE, True),
+ (const.TASK_STATE_FAILED, True),
+ (const.TASK_STATE_FAILED, False),
+ ],
+)
+def test_terminal_task_renders_subject_named_video_download(
+ tmp_path, ui_config, expected_open_count, task_state, has_video
):
"""完成任务应提供成片下载,并按 WebUI 配置决定是否自动打开目录。"""
tree = ast.parse(WEBUI_MAIN.read_text(encoding="utf-8"))
@@ -279,6 +287,8 @@ def test_completed_task_renders_subject_named_video_download(
self.downloads = []
self.videos = []
self.warnings = []
+ self.errors = []
+ self.successes = []
def columns(self, count):
return [FakeColumn() for _ in range(count)]
@@ -289,14 +299,14 @@ def test_completed_task_renders_subject_named_video_download(
def download_button(self, label, data, **kwargs):
self.downloads.append((label, data.read(), kwargs))
- def success(self, _message):
- pass
+ def success(self, message):
+ self.successes.append(message)
def warning(self, message):
self.warnings.append(message)
- def error(self, _message):
- pass
+ def error(self, message):
+ self.errors.append(message)
video_path = tmp_path / "final-1.mp4"
video_path.write_bytes(b"video-content")
@@ -324,9 +334,10 @@ def test_completed_task_renders_subject_named_video_download(
namespace["_render_generation_task_snapshot"](
"download-test",
{
- "state": const.TASK_STATE_COMPLETE,
+ "state": task_state,
"progress": 100,
- "videos": [str(video_path)],
+ "videos": [str(video_path)] if has_video else [],
+ "error": "OSError: second encoder failed",
"warnings": [
{"code": "batch_materials_reused", "video_index": 2, "count": 3}
],
@@ -334,6 +345,18 @@ def test_completed_task_renders_subject_named_video_download(
},
)
+ if task_state == const.TASK_STATE_FAILED:
+ expected_open_count = 0
+ assert fake_st.errors == ["Video Generation Failed: OSError: second encoder failed"]
+ assert fake_st.successes == []
+ else:
+ assert fake_st.errors == []
+ assert fake_st.successes == ["Video Generation Completed"]
+ if not has_video:
+ assert fake_st.downloads == []
+ assert fake_st.videos == []
+ open_task_folder.assert_not_called()
+ return
assert fake_st.videos == [str(video_path)]
assert fake_st.warnings == ["Video 2 reused 3 source clips."]
assert fake_st.downloads == [
diff --git a/webui/Main.py b/webui/Main.py
index 02a002d8..1fafa885 100644
--- a/webui/Main.py
+++ b/webui/Main.py
@@ -2032,20 +2032,21 @@ def _render_generation_task_snapshot(task_id, task):
_render_generation_logs(task_id)
return
+ video_files = task.get("videos") or []
if state == const.TASK_STATE_FAILED:
error = str(task.get("error") or "").strip()
message = tr("Video Generation Failed")
st.error(f"{message}: {error}" if error else message)
- _render_generation_logs(task_id)
- return
-
- video_files = task.get("videos") or []
- if state != const.TASK_STATE_COMPLETE or not video_files:
+ if not video_files:
+ _render_generation_logs(task_id)
+ return
+ elif state != const.TASK_STATE_COMPLETE or not video_files:
st.error(tr("Video Generation Failed"))
_render_generation_logs(task_id)
return
- st.success(tr("Video Generation Completed"))
+ if state == const.TASK_STATE_COMPLETE:
+ st.success(tr("Video Generation Completed"))
for warning in task.get("warnings") or []:
if isinstance(warning, Mapping) and warning.get("code") == "batch_materials_reused":
st.warning(
@@ -2118,7 +2119,10 @@ def _render_generation_task_snapshot(task_id, task):
)
_render_generation_logs(task_id)
- if st.session_state.get("handled_generation_task_id") != task_id:
+ if (
+ state == const.TASK_STATE_COMPLETE
+ and st.session_state.get("handled_generation_task_id") != task_id
+ ):
# Fragment 可能重复渲染同一个完成任务。无论是否开启自动打开目录,
# 每个任务都只处理一次完成事件,避免重复弹出资源管理器或重复写入日志。
st.session_state["handled_generation_task_id"] = task_id