Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(ByteDance): add Seedance 2.5 task_type for video extension (#15579)

Alexander Piskun0f1fa67ad8a6
diff --git a/comfy_api_nodes/apis/bytedance.py b/comfy_api_nodes/apis/bytedance.py
index 7ee83e5f3..64f8cde37 100644
--- a/comfy_api_nodes/apis/bytedance.py
+++ b/comfy_api_nodes/apis/bytedance.py
@@ -116,6 +116,7 @@ class Seedance2TaskCreationRequest(BaseModel):
seed: int | None = Field(None, ge=0, le=2147483647)
watermark: bool | None = Field(None)
output_format: str | None = Field(None)
+ omni_reference_task_type: str | None = Field(None, description="One of: auto, reference, edit, extend.")
class TaskCreationResponse(BaseModel):
diff --git a/comfy_api_nodes/nodes_bytedance.py b/comfy_api_nodes/nodes_bytedance.py
index 09fc445d8..265f94d13 100644
--- a/comfy_api_nodes/nodes_bytedance.py
+++ b/comfy_api_nodes/nodes_bytedance.py
@@ -2069,7 +2069,7 @@ def _seedance2_text_inputs(resolutions: list[str], default_ratio: str = "16:9"):
]
-def _seedance25_text_inputs(with_ratio: bool = True, with_video_editing: bool = False):
+def _seedance25_text_inputs(with_ratio: bool = True, with_video_editing: bool = False, with_task_type: bool = False):
return [
IO.String.Input(
"prompt",
@@ -2124,6 +2124,29 @@ def _seedance25_text_inputs(with_ratio: bool = True, with_video_editing: bool =
if with_video_editing
else []
),
+ *(
+ [
+ IO.Combo.Input(
+ "task_type",
+ options=["auto", "reference", "edit", "extend"],
+ default="auto",
+ tooltip="What to do with the reference media. Every value except auto is "
+ "validated when the task is submitted, so mismatched settings fail before "
+ "generation starts. auto: the model infers the task from the prompt and "
+ "inputs, and settings that conflict with its reading fail only after "
+ "generation has started. reference: generate a new video guided by the "
+ "reference images, videos, and audio. edit: change a connected reference "
+ "video (add, remove, replace); the output keeps the source clip's own length "
+ "and aspect ratio, and the duration and ratio widgets are ignored. extend: "
+ "continue a connected reference video forward or backward; the prompt should "
+ "say 'extend forward', 'extend backward', or 'continue', the aspect ratio "
+ "follows the source clip, and the output contains only the newly generated "
+ "segment of the duration you set, not the source clip.",
+ )
+ ]
+ if with_task_type
+ else []
+ ),
IO.Combo.Input(
"output_format",
options=["mp4"],
@@ -2133,9 +2156,9 @@ def _seedance25_text_inputs(with_ratio: bool = True, with_video_editing: bool =
]
-def _seedance25_reference_inputs():
+def _seedance25_reference_inputs(with_video_editing: bool = False, with_task_type: bool = False):
return [
- *_seedance25_text_inputs(with_video_editing=True),
+ *_seedance25_text_inputs(with_video_editing=with_video_editing, with_task_type=with_task_type),
IO.Autogrow.Input(
"reference_images",
template=IO.Autogrow.TemplateNames(
@@ -2196,17 +2219,23 @@ def _seedance2_build_request(
watermark: bool,
ratio: str,
) -> Seedance2TaskCreationRequest:
- video_editing = bool(model.get("video_editing"))
+ task_type = model.get("task_type", "auto")
+ duration = model["duration"]
+ if model.get("video_editing") or task_type == "edit":
+ ratio, duration = "adaptive", -1
+ elif task_type == "extend":
+ ratio = "adaptive"
return Seedance2TaskCreationRequest(
model=model_id,
content=content,
generate_audio=model["generate_audio"],
resolution=model["resolution"],
- ratio="adaptive" if video_editing else ratio,
- duration=-1 if video_editing else model["duration"],
+ ratio=ratio,
+ duration=duration,
seed=seed,
watermark=watermark,
output_format=model.get("output_format"),
+ omni_reference_task_type=None if task_type == "auto" else task_type,
)
@@ -2216,7 +2245,7 @@ _SEEDANCE2_PRICE_EXPR_TEMPLATE = """
$res := $lookup(widgets, "model.resolution");
$ratio := $lookup(widgets, "model.ratio");
$dur := $lookup(widgets, "model.duration");
- $auto := $lookup(widgets, "model.video_editing") = true;
+ $auto := __IS_EDIT__;
$hasVideo := __HAS_VIDEO__;
$ready := $type($m) = "string" and $type($res) = "string" and ($auto or $type($dur) = "number");
$ready ? (
@@ -2261,6 +2290,7 @@ _SEEDANCE2_PRICE_EXPR_TEMPLATE = """
_SEEDANCE_AUDIO_POLICY_CODE = "OutputAudioSensitiveContentDetected.PolicyViolation"
_SEEDANCE_TASK_TYPE_CONSTRAINT_CODE = "InvalidParameter.TaskTypeConstraint"
+_SEEDANCE_TASK_TYPE_MISMATCH_CODE = "InvalidParameter.TaskTypeMismatch"
async def _seedance2_poll_video_task(
@@ -2269,6 +2299,7 @@ async def _seedance2_poll_video_task(
model_id: str,
resolution: str,
has_video_input: bool,
+ task_type: str | None = None,
) -> TaskStatusResponse:
try:
return await poll_op(
@@ -2289,19 +2320,48 @@ async def _seedance2_poll_video_task(
"to get a silent video, or adjust the prompt and try again."
) from exc
if _SEEDANCE_TASK_TYPE_CONSTRAINT_CODE in str(exc):
+ if task_type is None:
+ raise ValueError(
+ "Seedance read this prompt as editing the reference video, and an edit always "
+ "takes its duration and aspect ratio from that video. Enable video_editing on "
+ "this node and run again, or reword the prompt so it describes a new video "
+ "rather than a change to the reference one."
+ ) from exc
+ if task_type == "edit":
+ raise ValueError(
+ "The request does not satisfy the 'edit' constraints: the clip being edited "
+ "must be 4 to 30 seconds long."
+ ) from exc
+ if task_type == "extend":
+ raise ValueError(
+ "The request does not satisfy the 'extend' constraints: the clip being "
+ "extended must be 1.9 to 30 seconds long."
+ ) from exc
+ raise ValueError(
+ "Seedance decided from the prompt that this task's duration or aspect ratio "
+ "must come from the reference video, and the current settings conflict with "
+ "that. Set task_type to the task you mean ('edit' or 'extend') and run again, "
+ "or reword the prompt so it describes a new video rather than a change to the "
+ "reference one."
+ ) from exc
+ if _SEEDANCE_TASK_TYPE_MISMATCH_CODE in str(exc):
raise ValueError(
- "Seedance read this prompt as editing the reference video, and an edit always "
- "takes its duration and aspect ratio from that video. Enable video_editing on "
- "this node and run again, or reword the prompt so it describes a new video "
- "rather than a change to the reference one."
+ f"Seedance read this prompt as a different task than the selected task_type "
+ f"'{task_type}'. Reword the prompt so it matches: an extend prompt should say "
+ "'extend forward', 'extend backward', or 'continue'; an edit prompt should use "
+ "words like add, remove, replace, or change. Or set task_type to auto."
) from exc
raise
-def _seedance2_price_badge(with_reference_videos: bool) -> IO.PriceBadge:
+def _seedance2_price_badge(with_reference_videos: bool, legacy_video_editing: bool = False) -> IO.PriceBadge:
widgets = ["model", "model.resolution", "model.ratio", "model.duration"]
+ if legacy_video_editing:
+ is_edit = '$lookup(widgets, "model.video_editing") = true'
+ else:
+ is_edit = '$lookup(widgets, "model.task_type") = "edit"'
if with_reference_videos:
- widgets.append("model.video_editing")
+ widgets.append("model.video_editing" if legacy_video_editing else "model.task_type")
has_video = (
'$exists(inputGroups) and $lookup(inputGroups, "model.reference_videos") > 0'
if with_reference_videos
@@ -2312,7 +2372,7 @@ def _seedance2_price_badge(with_reference_videos: bool) -> IO.PriceBadge:
widgets=widgets,
input_groups=["model.reference_videos"] if with_reference_videos else [],
),
- expr=_SEEDANCE2_PRICE_EXPR_TEMPLATE.replace("__HAS_VIDEO__", has_video),
+ expr=_SEEDANCE2_PRICE_EXPR_TEMPLATE.replace("__HAS_VIDEO__", has_video).replace("__IS_EDIT__", is_edit),
)
@@ -2662,12 +2722,12 @@ def _seedance2_reference_inputs(resolutions: list[str], default_ratio: str = "16
]
-class ByteDance2ReferenceNode(IO.ComfyNode):
+class ByteDance2ReferenceNodeV2(IO.ComfyNode):
@classmethod
def define_schema(cls):
return IO.Schema(
- node_id="ByteDance2ReferenceNode",
+ node_id="ByteDance2ReferenceNodeV2",
display_name="ByteDance Seedance 2.5 Reference to Video",
category="partner/video/ByteDance",
description="Generate, edit, or extend video using Seedance 2.5 or 2.0 with reference "
@@ -2676,7 +2736,7 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
IO.DynamicCombo.Input(
"model",
options=[
- IO.DynamicCombo.Option("Seedance 2.5", _seedance25_reference_inputs()),
+ IO.DynamicCombo.Option("Seedance 2.5", _seedance25_reference_inputs(with_task_type=True)),
IO.DynamicCombo.Option(
"Seedance 2.0",
_seedance2_reference_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"),
@@ -2761,6 +2821,13 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
f"(videos={len(reference_videos)}, video assets={len(reference_video_assets)}). "
f"Maximum is {limits['max_videos']}."
)
+ task_type = model.get("task_type")
+ if task_type in ("edit", "extend") and total_videos == 0:
+ raise ValueError(
+ f"A '{task_type}' task needs at least one reference video. Connect the video "
+ f"you want to {'change' if task_type == 'edit' else 'continue'}, or set "
+ "task_type to 'reference' to generate a new video from the references you have."
+ )
total_audios = len(reference_audios) + len(reference_audio_assets)
if total_audios > limits["max_audios"]:
raise ValueError(
@@ -2893,11 +2960,78 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
response_model=TaskCreationResponse,
)
response = await _seedance2_poll_video_task(
- cls, initial_response.id, model_id, model["resolution"], has_video_input=has_video_input
+ cls,
+ initial_response.id,
+ model_id,
+ model["resolution"],
+ has_video_input=has_video_input,
+ task_type=task_type,
)
return IO.NodeOutput(await download_url_to_video_output(response.content.video_url))
+class ByteDance2ReferenceNode(ByteDance2ReferenceNodeV2):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="ByteDance2ReferenceNode",
+ display_name="ByteDance Seedance 2.5 Reference to Video (Legacy)",
+ category="partner/video/ByteDance",
+ description="Generate, edit, or extend video using Seedance 2.5 or 2.0 with reference "
+ "images, videos, and audio. Supports multimodal reference, video editing, and video extension.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option("Seedance 2.5", _seedance25_reference_inputs(with_video_editing=True)),
+ IO.DynamicCombo.Option(
+ "Seedance 2.0",
+ _seedance2_reference_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"),
+ ),
+ IO.DynamicCombo.Option(
+ "Seedance 2.0 Fast",
+ _seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"),
+ ),
+ IO.DynamicCombo.Option(
+ "Seedance 2.0 Mini",
+ _seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"),
+ ),
+ ],
+ tooltip=SEEDANCE_MODEL_TOOLTIP,
+ ),
+ IO.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed controls whether the node should re-run; "
+ "results are non-deterministic regardless of seed.",
+ ),
+ IO.Boolean.Input(
+ "watermark",
+ default=False,
+ tooltip="Whether to add a watermark to the video.",
+ advanced=True,
+ ),
+ ],
+ outputs=[
+ IO.Video.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ is_deprecated=True,
+ price_badge=_seedance2_price_badge(with_reference_videos=True, legacy_video_editing=True),
+ )
+
+
async def process_video_task(
cls: type[IO.ComfyNode],
payload: Text2VideoTaskCreationRequest | Image2VideoTaskCreationRequest,
@@ -3413,6 +3547,7 @@ class ByteDanceExtension(ComfyExtension):
ByteDance2TextToVideoNode,
ByteDance2FirstLastFrameNode,
ByteDance2ReferenceNode,
+ ByteDance2ReferenceNodeV2,
ByteDanceCreateImageAsset,
ByteDanceCreateVideoAsset,
ByteDanceSeedAudioNode,