The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] add ByteDance Seed LLM node (#13919)
Alexander Piskun164a9d4bbbec
diff --git a/comfy_api_nodes/apis/bytedance_llm.py b/comfy_api_nodes/apis/bytedance_llm.pynew file mode 100644index 000000000..654c875fc--- /dev/null+++ b/comfy_api_nodes/apis/bytedance_llm.py@@ -0,0 +1,101 @@+"""Pydantic models for BytePlus ModelArk Responses API.++See: https://docs.byteplus.com/en/docs/ModelArk/1585128 (request)+ https://docs.byteplus.com/en/docs/ModelArk/1783703 (response)+"""++from typing import Literal++from pydantic import BaseModel, Field+++class BytePlusInputText(BaseModel):+ type: Literal["input_text"] = "input_text"+ text: str = Field(...)+++class BytePlusInputImage(BaseModel):+ type: Literal["input_image"] = "input_image"+ image_url: str = Field(..., description="Image URL or `data:image/...;base64,...` payload")+ detail: str = Field("auto", description="One of high, low, auto")+++class BytePlusInputVideo(BaseModel):+ type: Literal["input_video"] = "input_video"+ video_url: str = Field(..., description="Video URL or `data:video/...;base64,...` payload")+ fps: float | None = Field(None, ge=0.2, le=5.0)+++BytePlusMessageContent = BytePlusInputText | BytePlusInputImage | BytePlusInputVideo+++class BytePlusInputMessage(BaseModel):+ type: Literal["message"] = "message"+ role: str = Field(..., description="One of user, system, assistant, developer")+ content: list[BytePlusMessageContent] = Field(...)+++class BytePlusResponseCreateRequest(BaseModel):+ model: str = Field(...)+ input: list[BytePlusInputMessage] = Field(...)+ instructions: str | None = Field(None)+ max_output_tokens: int | None = Field(None, ge=1)+ temperature: float | None = Field(None, ge=0.0, le=2.0)+ store: bool | None = Field(False)+ stream: bool | None = Field(False)+++class BytePlusOutputText(BaseModel):+ type: Literal["output_text"] = "output_text"+ text: str = Field(...)+++class BytePlusOutputRefusal(BaseModel):+ type: Literal["refusal"] = "refusal"+ refusal: str = Field(...)+++class BytePlusOutputContent(BaseModel):+ type: str = Field(...)+ text: str | None = Field(None)+ refusal: str | None = Field(None)+++class BytePlusOutputMessage(BaseModel):+ type: str = Field(...)+ id: str | None = Field(None)+ role: str | None = Field(None)+ status: str | None = Field(None)+ content: list[BytePlusOutputContent] | None = Field(None)+++class BytePlusInputTokensDetails(BaseModel):+ cached_tokens: int | None = Field(None)+++class BytePlusOutputTokensDetails(BaseModel):+ reasoning_tokens: int | None = Field(None)+++class BytePlusResponseUsage(BaseModel):+ input_tokens: int | None = Field(None)+ output_tokens: int | None = Field(None)+ total_tokens: int | None = Field(None)+ input_tokens_details: BytePlusInputTokensDetails | None = Field(None)+ output_tokens_details: BytePlusOutputTokensDetails | None = Field(None)+++class BytePlusResponseError(BaseModel):+ code: str = Field(...)+ message: str = Field(...)+++class BytePlusResponseObject(BaseModel):+ id: str | None = Field(None)+ object: str | None = Field(None)+ created_at: int | None = Field(None)+ model: str | None = Field(None)+ status: str | None = Field(None)+ error: BytePlusResponseError | None = Field(None)+ output: list[BytePlusOutputMessage] | None = Field(None)+ usage: BytePlusResponseUsage | None = Field(None)diff --git a/comfy_api_nodes/nodes_bytedance_llm.py b/comfy_api_nodes/nodes_bytedance_llm.pynew file mode 100644index 000000000..fa7fe370a--- /dev/null+++ b/comfy_api_nodes/nodes_bytedance_llm.py@@ -0,0 +1,271 @@+"""API Nodes for ByteDance Seed LLM via the BytePlus ModelArk Responses API.++See: https://docs.byteplus.com/en/docs/ModelArk/1585128+"""++from typing_extensions import override++from comfy_api.latest import IO, ComfyExtension, Input+from comfy_api_nodes.apis.bytedance_llm import (+ BytePlusInputImage,+ BytePlusInputMessage,+ BytePlusInputText,+ BytePlusInputVideo,+ BytePlusMessageContent,+ BytePlusResponseCreateRequest,+ BytePlusResponseObject,+)+from comfy_api_nodes.util import (+ ApiEndpoint,+ get_number_of_images,+ sync_op,+ upload_images_to_comfyapi,+ upload_video_to_comfyapi,+ validate_string,+)++BYTEPLUS_RESPONSES_ENDPOINT = "/proxy/byteplus/api/v3/responses"+SEED_MAX_IMAGES = 20+SEED_MAX_VIDEOS = 4++SEED_MODELS: dict[str, str] = {+ "Seed 2.0 Pro": "seed-2-0-pro-260328",+ "Seed 2.0 Lite": "seed-2-0-lite-260228",+ "Seed 2.0 Mini": "seed-2-0-mini-260215",+}++# USD per 1M tokens: (input, cache_hit_input, output)+_SEED_PRICES_PER_MILLION: dict[str, tuple[float, float, float]] = {+ "seed-2-0-pro-260328": (0.50, 0.10, 3.00),+ "seed-2-0-lite-260228": (0.25, 0.05, 2.00),+ "seed-2-0-mini-260215": (0.10, 0.02, 0.40),+}+++def _seed_model_inputs(max_images: int = SEED_MAX_IMAGES, max_videos: int = SEED_MAX_VIDEOS):+ return [+ IO.Autogrow.Input(+ "images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("image"),+ names=[f"image_{i}" for i in range(1, max_images + 1)],+ min=0,+ ),+ tooltip=f"Optional image(s) to use as context for the model. Up to {max_images} images.",+ ),+ IO.Autogrow.Input(+ "videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("video"),+ names=[f"video_{i}" for i in range(1, max_videos + 1)],+ min=0,+ ),+ tooltip=f"Optional video(s) to use as context for the model. Up to {max_videos} videos.",+ ),+ IO.Float.Input(+ "temperature",+ default=1.0,+ min=0.0,+ max=2.0,+ step=0.01,+ tooltip="Controls randomness. 0.0 is deterministic, higher values are more random.",+ advanced=True,+ ),+ ]+++def _calculate_price(model_id: str, response: BytePlusResponseObject) -> float | None:+ """Compute approximate USD price from response usage."""+ if not response.usage:+ return None+ rates = _SEED_PRICES_PER_MILLION.get(model_id)+ if rates is None:+ return None+ input_rate, cache_hit_rate, output_rate = rates+ input_tokens = response.usage.input_tokens or 0+ output_tokens = response.usage.output_tokens or 0+ cached = 0+ if response.usage.input_tokens_details:+ cached = response.usage.input_tokens_details.cached_tokens or 0+ fresh_input = max(0, input_tokens - cached)+ total = fresh_input * input_rate + cached * cache_hit_rate + output_tokens * output_rate+ return total / 1_000_000.0+++def _get_text_from_response(response: BytePlusResponseObject) -> str:+ """Extract concatenated text from all assistant message output_text blocks."""+ if not response.output:+ return ""+ chunks: list[str] = []+ for item in response.output:+ if item.type != "message" or not item.content:+ continue+ for block in item.content:+ if block.type == "output_text" and block.text:+ chunks.append(block.text)+ elif block.type == "refusal" and block.refusal:+ raise ValueError(f"Model refused to respond: {block.refusal}")+ return "\n".join(chunks)+++async def _build_image_content_blocks(+ cls: type[IO.ComfyNode],+ image_tensors: list[Input.Image],+) -> list[BytePlusInputImage]:+ urls = await upload_images_to_comfyapi(+ cls,+ image_tensors,+ max_images=SEED_MAX_IMAGES,+ wait_label="Uploading reference images",+ )+ return [BytePlusInputImage(image_url=url) for url in urls]+++async def _build_video_content_blocks(+ cls: type[IO.ComfyNode],+ videos: list[Input.Video],+) -> list[BytePlusInputVideo]:+ blocks: list[BytePlusInputVideo] = []+ total = len(videos)+ for idx, video in enumerate(videos):+ label = "Uploading reference video"+ if total > 1:+ label = f"{label} ({idx + 1}/{total})"+ url = await upload_video_to_comfyapi(cls, video, wait_label=label)+ blocks.append(BytePlusInputVideo(video_url=url))+ return blocks+++class ByteDanceSeedNode(IO.ComfyNode):+ """Generate text responses from a ByteDance Seed 2.0 model."""++ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="ByteDanceSeedNode",+ display_name="ByteDance Seed",+ category="api node/text/ByteDance",+ essentials_category="Text Generation",+ description="Generate text responses with ByteDance's Seed 2.0 models. "+ "Provide a text prompt and optionally one or more images or videos for multimodal context.",+ inputs=[+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Text input to the model.",+ ),+ IO.DynamicCombo.Input(+ "model",+ options=[IO.DynamicCombo.Option(label, _seed_model_inputs()) for label in SEED_MODELS],+ tooltip="The Seed model used to generate the response.",+ ),+ IO.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ control_after_generate=True,+ tooltip="Seed controls whether the node should re-run; "+ "results are non-deterministic regardless of seed.",+ ),+ IO.String.Input(+ "system_prompt",+ multiline=True,+ default="",+ optional=True,+ advanced=True,+ tooltip="Foundational instructions that dictate the model's behavior.",+ ),+ ],+ outputs=[IO.String.Output()],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model"]),+ expr="""+ (+ $m := widgets.model;+ $contains($m, "mini") ? {+ "type": "list_usd",+ "usd": [0.00025, 0.0009],+ "format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }+ }+ : $contains($m, "lite") ? {+ "type": "list_usd",+ "usd": [0.0003, 0.002],+ "format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }+ }+ : $contains($m, "pro") ? {+ "type": "list_usd",+ "usd": [0.0005, 0.003],+ "format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }+ }+ : {"type":"text", "text":"Token-based"}+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ prompt: str,+ model: dict,+ seed: int,+ system_prompt: str = "",+ ) -> IO.NodeOutput:+ validate_string(prompt, strip_whitespace=True, min_length=1)+ model_label = model["model"]+ temperature = model["temperature"]+ model_id = SEED_MODELS[model_label]++ image_tensors: list[Input.Image] = [t for t in (model.get("images") or {}).values() if t is not None]+ if sum(get_number_of_images(t) for t in image_tensors) > SEED_MAX_IMAGES:+ raise ValueError(f"Up to {SEED_MAX_IMAGES} images are supported per request.")++ video_inputs: list[Input.Video] = [v for v in (model.get("videos") or {}).values() if v is not None]+ if len(video_inputs) > SEED_MAX_VIDEOS:+ raise ValueError(f"Up to {SEED_MAX_VIDEOS} videos are supported per request.")++ content: list[BytePlusMessageContent] = []+ if image_tensors:+ content.extend(await _build_image_content_blocks(cls, image_tensors))+ if video_inputs:+ content.extend(await _build_video_content_blocks(cls, video_inputs))+ content.append(BytePlusInputText(text=prompt))++ response = await sync_op(+ cls,+ ApiEndpoint(path=BYTEPLUS_RESPONSES_ENDPOINT, method="POST"),+ response_model=BytePlusResponseObject,+ data=BytePlusResponseCreateRequest(+ model=model_id,+ input=[BytePlusInputMessage(role="user", content=content)],+ instructions=system_prompt or None,+ temperature=temperature,+ store=False,+ stream=False,+ ),+ price_extractor=lambda r: _calculate_price(model_id, r),+ )+ if response.error:+ raise ValueError(f"Seed API error ({response.error.code}): {response.error.message}")+ result = _get_text_from_response(response)+ if not result:+ raise ValueError("Empty response from Seed model.")+ return IO.NodeOutput(result)+++class ByteDanceLLMExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:+ return [ByteDanceSeedNode]+++async def comfy_entrypoint() -> ByteDanceLLMExtension:+ return ByteDanceLLMExtension()