Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(Pika): add Pika H3 Flex Video and Reference to Video nodes (#17509)

Alexander Piskun70bcb4e9fc70
diff --git a/comfy_api_nodes/apis/pika.py b/comfy_api_nodes/apis/pika.py
new file mode 100644
index 000000000..e080b3abf
--- /dev/null
+++ b/comfy_api_nodes/apis/pika.py
@@ -0,0 +1,38 @@
+from pydantic import BaseModel, Field
+
+
+class PikaH3FlexRequest(BaseModel):
+ prompt: str = Field(...)
+ duration: int = Field(...)
+ aspect_ratio: str = Field(...)
+ resolution: str = Field(...)
+ seed: int = Field(...)
+ rewrite: bool = Field(...)
+
+
+class PikaH3FlexImageRequest(PikaH3FlexRequest):
+ image: str | None = Field(None)
+ last_frame_image: str | None = Field(None)
+
+
+class PikaH3FlexReference(BaseModel):
+ type: str = Field(...)
+ url: str = Field(...)
+
+
+class PikaH3FlexReferenceRequest(PikaH3FlexRequest):
+ references: list[PikaH3FlexReference] = Field(...)
+
+
+class PikaMediaFile(BaseModel):
+ url: str = Field(...)
+
+
+class PikaJobOutput(BaseModel):
+ video: PikaMediaFile | None = Field(None)
+
+
+class PikaJob(BaseModel):
+ id: str = Field(...)
+ status: str = Field(...)
+ output: PikaJobOutput | None = Field(None)
diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py
new file mode 100644
index 000000000..87f88fba2
--- /dev/null
+++ b/comfy_api_nodes/nodes_pika.py
@@ -0,0 +1,384 @@
+import re
+import uuid
+from io import BytesIO
+
+import av
+from pydantic import BaseModel
+from typing_extensions import override
+
+from comfy_api.latest import IO, ComfyExtension, Input, Types
+from comfy_api_nodes.apis.pika import (
+ PikaH3FlexImageRequest,
+ PikaH3FlexReference,
+ PikaH3FlexReferenceRequest,
+ PikaH3FlexRequest,
+ PikaJob,
+ PikaJobOutput,
+)
+from comfy_api_nodes.util import (
+ ApiEndpoint,
+ download_url_to_video_output,
+ poll_op,
+ sync_op,
+ upload_audio_to_comfyapi,
+ upload_file_to_comfyapi,
+ upload_image_to_comfyapi,
+ upload_video_to_comfyapi,
+ validate_audio_duration,
+ validate_string,
+ validate_video_duration,
+)
+
+PIKA_H3_FLEX_ASPECT_RATIOS = ["auto", "21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16", "9:21"]
+PIKA_H3_FLEX_REF_TAG_RE = re.compile(r"@(?P<kind>image|video|audio)(?P<idx>\d*)(?!\w)", re.IGNORECASE | re.ASCII)
+PIKA_H3_FLEX_REF_LABELS = {"image": "Picture", "video": "Video", "audio": "Audio"}
+
+
+def _seed_input() -> IO.Int.Input:
+ return IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=2147483647,
+ control_after_generate=True,
+ tooltip="With rewrite_prompt off, the same seed repeats the video exactly. "
+ "A local run with the same seed gives a different result.",
+ )
+
+
+def _h3_flex_settings_inputs(aspect_ratio_tooltip: str) -> list[IO.Input]:
+ return [
+ IO.Combo.Input(
+ "resolution",
+ options=["480p", "768p", "1080p", "2k", "4k"],
+ default="768p",
+ tooltip="1080p and above render at 768p and upscale once. The same seed at 480p gives a different video.",
+ ),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=PIKA_H3_FLEX_ASPECT_RATIOS,
+ default="auto",
+ tooltip=aspect_ratio_tooltip,
+ ),
+ IO.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=15,
+ tooltip="The output length follows a frame grid: 5 seconds gives 124 frames, about 5.17 seconds.",
+ ),
+ IO.Boolean.Input(
+ "rewrite_prompt",
+ default=False,
+ tooltip="Let Pika expand the prompt first. The same seed then gives a different video on each run.",
+ ),
+ _seed_input(),
+ ]
+
+
+def _resolve_ref_tags(prompt: str, counts: dict[str, int]) -> str:
+ parts = []
+ pos = 0
+ prev_end = -1
+ for match in PIKA_H3_FLEX_REF_TAG_RE.finditer(prompt):
+ start = match.start()
+ if (
+ start > 0
+ and start != prev_end
+ and prompt[start - 1].isascii()
+ and (prompt[start - 1].isalnum() or prompt[start - 1] == "_")
+ ):
+ continue
+ kind = match.group("kind").lower()
+ idx = int(match.group("idx") or 1)
+ if not 1 <= idx <= counts[kind]:
+ raise ValueError(
+ f"The prompt references @{kind.capitalize()}{idx}, but {counts[kind]} {kind} input(s) are available."
+ )
+ parts.append(prompt[pos:start])
+ parts.append(f"<{PIKA_H3_FLEX_REF_LABELS[kind]} {idx}>")
+ pos = match.end()
+ prev_end = match.end()
+ parts.append(prompt[pos:])
+ return "".join(parts)
+
+
+async def _upload_reference_video(cls: type[IO.ComfyNode], video: Input.Video, keep_audio: bool, label: str) -> str:
+ if keep_audio:
+ return await upload_video_to_comfyapi(cls, video, wait_label=label)
+ source = BytesIO()
+ video.save_to(source, format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264)
+ source.seek(0)
+ silent = BytesIO()
+ with av.open(source) as src, av.open(silent, mode="w", format="mp4") as dst:
+ src_stream = src.streams.video[0]
+ dst_stream = dst.add_stream_from_template(src_stream)
+ for packet in src.demux(src_stream):
+ if packet.dts is not None:
+ packet.stream = dst_stream
+ dst.mux(packet)
+ silent.seek(0)
+ return await upload_file_to_comfyapi(cls, silent, f"{uuid.uuid4()}.mp4", "video/mp4", label)
+
+
+async def _pika_job(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> PikaJobOutput:
+ job = await sync_op(
+ cls,
+ ApiEndpoint(path=f"/proxy/pika/v1/media/pika/{path}", method="POST"),
+ response_model=PikaJob,
+ data=request,
+ )
+ job = await poll_op(
+ cls,
+ ApiEndpoint(path=f"/proxy/pika/v1/media/jobs/{job.id}"),
+ response_model=PikaJob,
+ status_extractor=lambda r: r.status,
+ completed_statuses=["completed"],
+ failed_statuses=["failed"],
+ queued_statuses=["queued"],
+ )
+ if job.output is None:
+ raise Exception(f"Pika returned no output for job {job.id}.")
+ return job.output
+
+
+async def _pika_video(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> IO.NodeOutput:
+ output = await _pika_job(cls, path, request)
+ return IO.NodeOutput(await download_url_to_video_output(output.video.url, cls=cls))
+
+
+class PikaH3FlexVideoNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="PikaH3FlexVideoNode",
+ display_name="Pika H3 Flex Video",
+ category="partner/video/Pika",
+ description="Pika H3 Flex, Pika's post-trained MiniMax H3. "
+ "Generate video with synchronized audio from text and optional first and last frames.",
+ search_aliases=["minimax h3", "h3 flex", "pika h3", "text to video", "image to video", "first last frame"],
+ inputs=[
+ IO.String.Input("prompt", multiline=True, default=""),
+ IO.Image.Input("first_frame", optional=True),
+ IO.Image.Input("last_frame", optional=True),
+ *_h3_flex_settings_inputs("auto follows the first connected frame, otherwise the prompt."),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[IO.Hidden.auth_token_comfy_org, IO.Hidden.api_key_comfy_org, IO.Hidden.unique_id],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["resolution", "duration"]),
+ expr="""
+ (
+ $res := widgets.resolution;
+ $d := widgets.duration;
+ $rate := $lookup({"480p": 0.0143, "768p": 0.0286, "1080p": 0.05005, "2k": 0.0572, "4k": 0.1144}, $res);
+ $type($d) = "number"
+ ? {"type":"usd","usd": $rate * (($res = "480p" or $res = "768p")
+ ? $d : ($ceil((24 * $d - 5) / 17) * 17 + 5) / 24)}
+ : {"type":"usd","usd": $rate,"format":{"suffix":"/second"}}
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ prompt: str,
+ resolution: str,
+ aspect_ratio: str,
+ duration: int,
+ rewrite_prompt: bool,
+ seed: int,
+ first_frame: Input.Image | None = None,
+ last_frame: Input.Image | None = None,
+ ) -> IO.NodeOutput:
+ validate_string(prompt, min_length=1, max_length=8000)
+ settings = {
+ "prompt": prompt,
+ "duration": duration,
+ "aspect_ratio": aspect_ratio,
+ "resolution": resolution,
+ "seed": seed,
+ "rewrite": rewrite_prompt,
+ }
+ if first_frame is None and last_frame is None:
+ return await _pika_video(cls, "pika-h3-flex/text-to-video", PikaH3FlexRequest(**settings))
+ return await _pika_video(
+ cls,
+ "pika-h3-flex/image-to-video",
+ PikaH3FlexImageRequest(
+ **settings,
+ image=None
+ if first_frame is None
+ else await upload_image_to_comfyapi(cls, first_frame, wait_label="Uploading first frame"),
+ last_frame_image=None
+ if last_frame is None
+ else await upload_image_to_comfyapi(cls, last_frame, wait_label="Uploading last frame"),
+ ),
+ )
+
+
+class PikaH3FlexReferenceVideoNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="PikaH3FlexReferenceVideoNode",
+ display_name="Pika H3 Flex Reference to Video",
+ category="partner/video/Pika",
+ description="Pika H3 Flex, Pika's post-trained MiniMax H3. "
+ "Generate or edit video with synchronized audio guided by reference images, videos and audio.",
+ search_aliases=["minimax h3", "h3 flex", "pika h3", "video editing", "motion transfer", "reference video"],
+ inputs=[
+ IO.Autogrow.Input(
+ "reference_images",
+ template=IO.Autogrow.TemplateNames(
+ IO.Image.Input("reference_image"),
+ names=[f"image_{i}" for i in range(1, 10)],
+ min=0,
+ ),
+ optional=True,
+ tooltip="Up to 9 images, referred to in the prompt as @Image1..@Image9 in connection order. "
+ "At least one image or video is required.",
+ ),
+ IO.Autogrow.Input(
+ "reference_videos",
+ template=IO.Autogrow.TemplateNames(
+ IO.Video.Input("reference_video"),
+ names=[f"video_{i}" for i in range(1, 4)],
+ min=0,
+ ),
+ optional=True,
+ tooltip="Up to 3 videos, referred to in the prompt as @Video1..@Video3 in connection order. "
+ "2-15 seconds each, 15 seconds in total. Billed per second.",
+ ),
+ IO.Autogrow.Input(
+ "reference_audios",
+ template=IO.Autogrow.TemplateNames(
+ IO.Audio.Input("reference_audio"),
+ names=[f"audio_{i}" for i in range(1, 4)],
+ min=0,
+ ),
+ optional=True,
+ tooltip="Up to 3 clips, referred to in the prompt as @Audio1..@Audio3. "
+ "2-15 seconds each, 15 seconds in total. When copied to the output, clips play in connection order, "
+ "ahead of the sound of connected videos, whichever number the prompt names.",
+ ),
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Refer to references as @Image1, @Video1, @Audio1, or as <Picture 1>, <Video 1>, <Audio 1>.",
+ ),
+ IO.Boolean.Input(
+ "reference_video_audio",
+ default=True,
+ tooltip="The sound of connected videos is copied to the output. Turn this off for new ambience or "
+ "motion transfer with new music. A talking head with no dialogue in the prompt then gets invented "
+ "speech.",
+ ),
+ *_h3_flex_settings_inputs(
+ "auto follows a reference video, otherwise the prompt. Reference images do not affect it."
+ ),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[IO.Hidden.auth_token_comfy_org, IO.Hidden.api_key_comfy_org, IO.Hidden.unique_id],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["resolution", "duration"], input_groups=["reference_videos"]),
+ expr="""
+ (
+ $res := widgets.resolution;
+ $d := widgets.duration;
+ $rate := $lookup({"480p": 0.0429, "768p": 0.0858, "1080p": 0.10725, "2k": 0.1144, "4k": 0.143}, $res);
+ $vidsRaw := $lookup(inputGroups, "reference_videos");
+ $vids := $vidsRaw ? $vidsRaw : 0;
+ $type($d) != "number"
+ ? {"type":"usd","usd": $rate,"format":{"suffix":"/second"}}
+ : (
+ $base := $rate * (($res = "480p" or $res = "768p")
+ ? $d : ($ceil((24 * $d - 5) / 17) * 17 + 5) / 24);
+ $vids > 0
+ ? {"type":"range_usd","min_usd": $base + $vids * 2 * $rate,"max_usd": $base + 15 * $rate,
+ "format":{"approximate": true}}
+ : {"type":"usd","usd": $base}
+ )
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ reference_images: dict,
+ reference_videos: dict,
+ reference_audios: dict,
+ prompt: str,
+ reference_video_audio: bool,
+ resolution: str,
+ aspect_ratio: str,
+ duration: int,
+ rewrite_prompt: bool,
+ seed: int,
+ ) -> IO.NodeOutput:
+ if not reference_images and not reference_videos:
+ raise ValueError("At least one reference image or video is required.")
+ if len(reference_images) + len(reference_videos) + len(reference_audios) > 12:
+ raise ValueError("At most 12 references are supported in total.")
+ for video in reference_videos.values():
+ validate_video_duration(video, min_duration=2, max_duration=15)
+ if sum(video.get_duration() for video in reference_videos.values()) > 15:
+ raise ValueError("Reference videos must be at most 15 seconds in total.")
+ for audio in reference_audios.values():
+ validate_audio_duration(audio, min_duration=2, max_duration=15)
+ if sum(audio["waveform"].shape[-1] / audio["sample_rate"] for audio in reference_audios.values()) > 15:
+ raise ValueError("Reference audio must be at most 15 seconds in total.")
+ prompt = _resolve_ref_tags(
+ prompt,
+ {
+ "image": len(reference_images),
+ "video": len(reference_videos),
+ "audio": len(reference_audios) + (len(reference_videos) if reference_video_audio else 0),
+ },
+ )
+ validate_string(prompt, min_length=1, max_length=8000)
+ references = []
+ for i, image in enumerate(reference_images.values(), 1):
+ url = await upload_image_to_comfyapi(cls, image, wait_label=f"Uploading image {i}")
+ references.append(PikaH3FlexReference(type="image", url=url))
+ for audio in reference_audios.values():
+ url = await upload_audio_to_comfyapi(
+ cls, audio, container_format="wav", codec_name="pcm_s16le", mime_type="audio/wav"
+ )
+ references.append(PikaH3FlexReference(type="audio", url=url))
+ for i, video in enumerate(reference_videos.values(), 1):
+ url = await _upload_reference_video(cls, video, reference_video_audio, f"Uploading video {i}")
+ references.append(PikaH3FlexReference(type="video", url=url))
+ return await _pika_video(
+ cls,
+ "pika-h3-flex/reference-to-video",
+ PikaH3FlexReferenceRequest(
+ prompt=prompt,
+ duration=duration,
+ aspect_ratio=aspect_ratio,
+ resolution=resolution,
+ seed=seed,
+ rewrite=rewrite_prompt,
+ references=references,
+ ),
+ )
+
+
+class PikaExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:
+ return [
+ PikaH3FlexVideoNode,
+ PikaH3FlexReferenceVideoNode,
+ ]
+
+
+async def comfy_entrypoint() -> PikaExtension:
+ return PikaExtension()