Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Gateway gen img + support Nano Banana (#518)

Louis Choquel43064ffd8f53
diff --git a/.pipelex/inference/backends/blackboxai.toml b/.pipelex/inference/backends/blackboxai.toml
index e6cf9dde0..9ee043309 100644
--- a/.pipelex/inference/backends/blackboxai.toml
+++ b/.pipelex/inference/backends/blackboxai.toml
@@ -193,7 +193,7 @@ costs = { input = 0.00, output = 0.00 }
[flux-pro]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/black-forest-labs/flux-pro"
inputs = ["text"]
outputs = ["image"]
@@ -201,7 +201,7 @@ costs = { input = 0.0, output = 0.04 }
["flux-pro/v1.1"]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/black-forest-labs/flux-1.1-pro"
inputs = ["text"]
outputs = ["image"]
@@ -209,15 +209,15 @@ costs = { input = 0.0, output = 0.04 }
["flux-pro/v1.1-ultra"]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/black-forest-labs/flux-1.1-pro-ultra"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.0, output = 0.06 }
-["fast-lightning-sdxl"]
+[fast-lightning-sdxl]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/bytedance/sdxl-lightning-4step"
inputs = ["text"]
outputs = ["image"]
@@ -225,8 +225,16 @@ costs = { input = 0.0, output = 0.0014 }
[nano-banana]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/google/nano-banana"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.0, output = 0.039 }
+
+[nano-banana-pro]
+model_type = "img_gen"
+sdk = "blackboxai_img_gen"
+model_id = "blackboxai/google/nano-banana-pro"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0, output = 0.039 }
diff --git a/.pipelex/inference/backends/fal.toml b/.pipelex/inference/backends/fal.toml
index d54a25fcc..608bfed89 100644
--- a/.pipelex/inference/backends/fal.toml
+++ b/.pipelex/inference/backends/fal.toml
@@ -36,18 +36,52 @@ inputs = ["text"]
outputs = ["image"]
costs = { input = 0.05, output = 0.0 }
+[flux-pro.rules]
+num_images = "fal"
+aspect_ratio = "flux"
+inference = "flux"
+safety_checker = "available"
+specific = "fal"
+
["flux-pro/v1.1"]
model_id = "fal-ai/flux-pro/v1.1"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.05, output = 0.0 }
+["flux-pro/v1.1".rules]
+num_images = "fal"
+aspect_ratio = "flux"
+inference = "flux"
+safety_checker = "available"
+specific = "fal"
+
["flux-pro/v1.1-ultra"]
model_id = "fal-ai/flux-pro/v1.1-ultra"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.06, output = 0.0 }
+["flux-pro/v1.1-ultra".rules]
+num_images = "fal"
+aspect_ratio = "flux_11_ultra"
+inference = "flux_11_ultra"
+safety_checker = "available"
+specific = "fal"
+
+[flux-2]
+model_id = "fal-ai/flux-2"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+
+[flux-2.rules]
+num_images = "fal"
+aspect_ratio = "flux"
+inference = "flux"
+safety_checker = "available"
+specific = "fal"
+
# --- SDXL models --------------------------------------------------------------
[fast-lightning-sdxl]
model_id = "fal-ai/fast-lightning-sdxl"
@@ -55,3 +89,9 @@ inputs = ["text"]
outputs = ["image"]
costs = { input = 0.0003, output = 0.0 }
+[fast-lightning-sdxl.rules]
+num_images = "fal"
+specific = "fal"
+aspect_ratio = "flux"
+inference = "sdxl_lightning"
+safety_checker = "unavailable"
diff --git a/.pipelex/inference/backends/google.toml b/.pipelex/inference/backends/google.toml
index 0d39dacb1..e0d3f0af8 100644
--- a/.pipelex/inference/backends/google.toml
+++ b/.pipelex/inference/backends/google.toml
@@ -67,3 +67,21 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 3000
costs = { input = 2, output = 12.0 }
+
+################################################################################
+# IMAGE GENERATION MODELS (Nano Banana)
+################################################################################
+
+[nano-banana]
+model_type = "img_gen"
+model_id = "gemini-2.5-flash-image"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0, output = 0.039 }
+
+[nano-banana-pro]
+model_type = "img_gen"
+model_id = "gemini-3-pro-image-preview"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0, output = 0.039 }
diff --git a/.pipelex/inference/deck/base_deck.toml b/.pipelex/inference/deck/base_deck.toml
index 6fb0161d6..b2561f5ab 100644
--- a/.pipelex/inference/deck/base_deck.toml
+++ b/.pipelex/inference/deck/base_deck.toml
@@ -35,7 +35,7 @@ vision-groq = "llama-4-scout-17b-16e-instruct"
# Image generation aliases
base-img-gen = "flux-pro/v1.1"
-best-img-gen = "flux-pro/v1.1-ultra"
+best-img-gen = "flux-2"
fast-img-gen = "fast-lightning-sdxl"
####################################################################################################
@@ -95,8 +95,8 @@ cheap_llm = [
"grok-3-mini",
]
cheap_llm_for_vision = [
- "gpt-4o-mini",
"gemini-2.5-flash-lite",
+ "gpt-4o-mini",
"claude-3-haiku",
]
cheap_llm_for_structured = ["gpt-4o-mini", "mistral-small", "claude-3-haiku"]
@@ -188,6 +188,7 @@ choice_default = "gen_image_basic"
gen_image_basic = { model = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
gen_image_fast = { model = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
gen_image_high_quality = { model = "best-img-gen", quality = "high", guidance_scale = 8.0, is_moderated = true, safety_tolerance = 3 }
+gen_image_openai_low_quality = { model = "gpt-image-1", quality = "low" }
# Specific skills
img_gen_for_art = { model = "best-img-gen", quality = "high", guidance_scale = 9.0, is_moderated = false, safety_tolerance = 5 }
diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml
index b3733ae23..68faae398 100644
--- a/.pipelex/inference/routing_profiles.toml
+++ b/.pipelex/inference/routing_profiles.toml
@@ -34,6 +34,7 @@ fallback_order = [
"pipelex_gateway",
"azure_openai",
"bedrock",
+ "google",
"blackboxai",
"mistral",
"fal",
@@ -41,15 +42,15 @@ fallback_order = [
[profiles.pipelex_gateway_first.routes]
# Pattern matching: "model-pattern" = "backend-name"
+
+[profiles.pipelex_gateway_first.optional_routes] # Each optional route is considered only if its backend is available
"gpt-*" = "pipelex_gateway"
+"gpt-image-1" = "openai"
"claude-*" = "pipelex_gateway"
"grok-*" = "pipelex_gateway"
"gemini-*" = "pipelex_gateway"
-
-[profiles.pipelex_gateway_first.optional_routes] # Each optional route is considered only if its backend is available
"*-sdxl" = "fal"
"flux-*" = "fal"
-"gpt-image-1" = "openai"
"mistral-ocr" = "mistral"
[profiles.all_pipelex_gateway]
diff --git a/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeImgGen.md b/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeImgGen.md
index 6f8c8282d..812349215 100644
--- a/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeImgGen.md
+++ b/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeImgGen.md
@@ -56,7 +56,7 @@ This pipe generates one image of a futuristic car without requiring any input.
type = "PipeImgGen"
description = "Generate a futuristic car image"
output = "Image"
-img_gen_prompt = "A sleek, futuristic sports car driving on a neon-lit highway at night."
+prompt = "A sleek, futuristic sports car driving on a neon-lit highway at night."
model = "gen_image_basic"
aspect_ratio = "16:9"
quality = "hd"
diff --git a/pipelex/builder/pipe/pipe_img_gen_spec.py b/pipelex/builder/pipe/pipe_img_gen_spec.py
index 2f15b656e..b3438495a 100644
--- a/pipelex/builder/pipe/pipe_img_gen_spec.py
+++ b/pipelex/builder/pipe/pipe_img_gen_spec.py
@@ -36,6 +36,7 @@ class PipeImgGenSpec(PipeSpec):
type: Literal["PipeImgGen"] = "PipeImgGen"
pipe_category: Literal["PipeOperator"] = "PipeOperator"
img_gen_skill: ImgGenSkill | str = Field(description="Select the most adequate image generation skill according to the task to be performed.")
+ prompt: str
@field_validator("img_gen_skill", mode="before")
@classmethod
@@ -71,8 +72,7 @@ class PipeImgGenSpec(PipeSpec):
description=base_blueprint.description,
inputs=base_blueprint.inputs,
output=base_blueprint.output,
- img_gen_prompt=None,
- img_gen_prompt_var_name=base_blueprint.input_names[0] if base_blueprint.input_names else None,
+ prompt=self.prompt,
model=img_gen_choice,
aspect_ratio=None,
background=None,
diff --git a/pipelex/builder/runner_code.py b/pipelex/builder/runner_code.py
index 1c114ad34..6a12258c3 100644
--- a/pipelex/builder/runner_code.py
+++ b/pipelex/builder/runner_code.py
@@ -220,7 +220,7 @@ def generate_runner_code(pipe: PipeAbstract, output_multiplicity: bool = False)
)
# Build inputs entries
- if pipe.inputs.nb_inputs > 0:
+ if not pipe.inputs.is_empty:
input_entries: list[str] = []
for var_name, input_req in pipe.inputs.root.items():
is_multiple = _is_multiple(input_req.multiplicity)
@@ -251,7 +251,7 @@ def generate_runner_code(pipe: PipeAbstract, output_multiplicity: bool = False)
f' pipe_code="{pipe.code}",',
]
- if pipe.inputs.nb_inputs > 0:
+ if not pipe.inputs.is_empty:
function_lines.extend(
[
" inputs={",
diff --git a/pipelex/cogt/extract/extract_worker_factory.py b/pipelex/cogt/extract/extract_worker_factory.py
index 3f07df432..9db33e0ad 100644
--- a/pipelex/cogt/extract/extract_worker_factory.py
+++ b/pipelex/cogt/extract/extract_worker_factory.py
@@ -21,7 +21,7 @@ class ExtractWorkerFactory:
extract_worker: ExtractWorkerAbstract
match plugin.sdk:
case "gateway_extract":
- from pipelex.plugins.gateway.gateway_extract_worker import PortkeyExtractWorker # noqa: PLC0415
+ from pipelex.plugins.gateway.gateway_extract_worker import GatewayExtractWorker # noqa: PLC0415
from pipelex.plugins.gateway.gateway_factory import GatewayFactory # noqa: PLC0415
extract_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
@@ -29,7 +29,7 @@ class ExtractWorkerFactory:
sdk_instance=GatewayFactory.make_portkey_client(backend=backend),
)
- extract_worker = PortkeyExtractWorker(
+ extract_worker = GatewayExtractWorker(
sdk_instance=extract_sdk_instance,
extra_config=backend.extra_config,
inference_model=inference_model,
diff --git a/pipelex/cogt/img_gen/img_gen_args_factory.py b/pipelex/cogt/img_gen/img_gen_args_factory.py
new file mode 100644
index 000000000..8913f3489
--- /dev/null
+++ b/pipelex/cogt/img_gen/img_gen_args_factory.py
@@ -0,0 +1,230 @@
+from typing import Any
+
+from pipelex import log
+from pipelex.cogt.exceptions import ImgGenParameterError
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat, Quality
+from pipelex.cogt.img_gen.img_gen_model_rules import (
+ AspectRatioTaxonomy,
+ BackgroundTaxonomy,
+ ImgGenArgTopic,
+ ImgGenModelRules,
+ InferenceTaxonomy,
+ NumImagesTaxonomy,
+ SafetyCheckerTaxonomy,
+ SpecificTaxonomy,
+)
+from pipelex.config import get_config
+from pipelex.plugins.openai.openai_img_gen_factory import OpenAIImgGenFactory
+
+
+class ImgGenArgsFactory:
+ @classmethod
+ def make_args_for_model(
+ cls,
+ model_rules: ImgGenModelRules,
+ img_gen_job: ImgGenJob,
+ nb_images: int,
+ ) -> dict[str, Any]:
+ job_params = img_gen_job.job_params
+
+ # Common args that don't need rules
+ args_dict: dict[str, Any] = {
+ "prompt": img_gen_job.img_gen_prompt.positive_text,
+ "output_format": cls._convert_output_format(job_params.output_format),
+ }
+
+ for topic, taxonomy_value in model_rules.items():
+ match topic:
+ case ImgGenArgTopic.NUM_IMAGES:
+ num_images_taxonomy = NumImagesTaxonomy(taxonomy_value)
+ args_dict.update(
+ cls.make_args_from_num_images(
+ num_images_taxonomy=num_images_taxonomy,
+ nb_images=nb_images,
+ )
+ )
+ case ImgGenArgTopic.SPECIFIC:
+ specific_taxonomy = SpecificTaxonomy(taxonomy_value)
+ args_dict.update(
+ cls.make_args_from_specific(
+ specific_taxonomy=specific_taxonomy,
+ )
+ )
+ case ImgGenArgTopic.ASPECT_RATIO:
+ aspect_ratio_taxonomy = AspectRatioTaxonomy(taxonomy_value)
+ args_dict.update(
+ cls.make_args_from_aspect_ratio(
+ aspect_ratio_taxonomy=aspect_ratio_taxonomy,
+ aspect_ratio=job_params.aspect_ratio,
+ )
+ )
+ case ImgGenArgTopic.INFERENCE:
+ inference_taxonomy = InferenceTaxonomy(taxonomy_value)
+ args_dict.update(
+ cls.make_args_from_inference(
+ inference_taxonomy=inference_taxonomy,
+ num_inference_steps=job_params.nb_steps,
+ quality=job_params.quality,
+ guidance_scale=job_params.guidance_scale,
+ is_raw=job_params.is_raw,
+ )
+ )
+ case ImgGenArgTopic.SAFETY_CHECKER:
+ safety_checker_taxonomy = SafetyCheckerTaxonomy(taxonomy_value)
+ args_dict.update(
+ cls.make_args_from_safety_checker(
+ safety_checker_taxonomy=safety_checker_taxonomy,
+ is_moderated=job_params.is_moderated,
+ safety_tolerance=job_params.safety_tolerance,
+ )
+ )
+ case ImgGenArgTopic.BACKGROUND:
+ background_taxonomy = BackgroundTaxonomy(taxonomy_value)
+ args_dict.update(
+ cls.make_args_from_background(
+ background_taxonomy=background_taxonomy,
+ background=job_params.background,
+ )
+ )
+
+ return args_dict
+
+ @classmethod
+ def _convert_output_format(cls, output_format: OutputFormat) -> str:
+ match output_format:
+ case OutputFormat.PNG:
+ return "png"
+ case OutputFormat.JPG:
+ return "jpeg"
+ case OutputFormat.WEBP:
+ msg = "Output format WebP is not supported"
+ raise ImgGenParameterError(msg)
+
+ @classmethod
+ def make_args_from_num_images(cls, num_images_taxonomy: NumImagesTaxonomy, nb_images: int) -> dict[str, Any]:
+ match num_images_taxonomy:
+ case NumImagesTaxonomy.FAL:
+ return {"num_images": nb_images}
+ case NumImagesTaxonomy.GPT:
+ return {"n": nb_images}
+
+ @classmethod
+ def make_args_from_specific(cls, specific_taxonomy: SpecificTaxonomy) -> dict[str, Any]:
+ match specific_taxonomy:
+ case SpecificTaxonomy.FAL:
+ return {"sync_mode": False}
+
+ @classmethod
+ def make_args_from_background(cls, background_taxonomy: BackgroundTaxonomy, background: Background) -> dict[str, Any]:
+ match background_taxonomy:
+ case BackgroundTaxonomy.GPT:
+ return {"background": background.value}
+
+ @classmethod
+ def make_args_from_aspect_ratio(cls, aspect_ratio_taxonomy: AspectRatioTaxonomy, aspect_ratio: AspectRatio) -> dict[str, Any]:
+ key: str
+ value: Any
+ match aspect_ratio_taxonomy:
+ case AspectRatioTaxonomy.FLUX:
+ key = "image_size"
+ match aspect_ratio:
+ case AspectRatio.SQUARE:
+ value = "square_hd"
+ case AspectRatio.LANDSCAPE_4_3:
+ value = "landscape_4_3"
+ case AspectRatio.LANDSCAPE_16_9:
+ value = "landscape_16_9"
+ case AspectRatio.LANDSCAPE_21_9:
+ value = "landscape_21_9"
+ case AspectRatio.PORTRAIT_3_4:
+ value = "portrait_4_3"
+ case AspectRatio.PORTRAIT_9_16:
+ value = "portrait_16_9"
+ case AspectRatio.PORTRAIT_9_21:
+ value = "portrait_21_9"
+ case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model"
+ raise ImgGenParameterError(msg)
+ case AspectRatioTaxonomy.FLUX_11_ULTRA:
+ key = "aspect_ratio"
+ match aspect_ratio:
+ case AspectRatio.SQUARE:
+ value = "1:1"
+ case AspectRatio.LANDSCAPE_4_3:
+ value = "4:3"
+ case AspectRatio.LANDSCAPE_16_9:
+ value = "16:9"
+ case AspectRatio.LANDSCAPE_21_9:
+ value = "21:9"
+ case AspectRatio.PORTRAIT_3_4:
+ value = "3:4"
+ case AspectRatio.PORTRAIT_9_16:
+ value = "9:16"
+ case AspectRatio.PORTRAIT_9_21:
+ value = "9:21"
+ case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model"
+ raise ImgGenParameterError(msg)
+ case AspectRatioTaxonomy.GPT:
+ key = "size"
+ value = OpenAIImgGenFactory.image_size_for_gpt_image_1(aspect_ratio)
+ return {key: value}
+
+ @classmethod
+ def make_args_from_inference(
+ cls,
+ inference_taxonomy: InferenceTaxonomy,
+ num_inference_steps: int | None,
+ quality: Quality | None,
+ guidance_scale: float | None,
+ is_raw: bool | None,
+ ) -> dict[str, Any]:
+ args_dict: dict[str, Any] = {}
+ match inference_taxonomy:
+ case InferenceTaxonomy.SDXL_LIGHTNING:
+ if num_inference_steps:
+ acceptable_steps = [1, 2, 4, 8]
+ if num_inference_steps not in acceptable_steps:
+ # TODO: prevent this when building presets and params
+ log.warning(f"Number of inference steps {num_inference_steps} for SDXL Lightning must be one of {acceptable_steps}")
+ num_inference_steps = 4
+ args_dict["num_inference_steps"] = num_inference_steps
+ else:
+ sdxl_lightning_map_quality_to_steps = get_config().cogt.img_gen_config.fal_config.sdxl_lightning_map_quality_to_steps
+ num_inference_steps = sdxl_lightning_map_quality_to_steps[quality or Quality.MEDIUM]
+ args_dict["num_inference_steps"] = num_inference_steps
+ case InferenceTaxonomy.FLUX:
+ if num_inference_steps:
+ args_dict["num_inference_steps"] = num_inference_steps
+ else:
+ flux_map_quality_to_steps = get_config().cogt.img_gen_config.fal_config.flux_map_quality_to_steps
+ num_inference_steps = flux_map_quality_to_steps[quality or Quality.MEDIUM]
+ args_dict["num_inference_steps"] = num_inference_steps
+ if guidance_scale:
+ args_dict["guidance_scale"] = guidance_scale
+ case InferenceTaxonomy.FLUX_11_ULTRA:
+ if is_raw:
+ args_dict["raw"] = is_raw
+ case InferenceTaxonomy.GPT:
+ if quality:
+ args_dict["quality"] = quality.value
+ return args_dict
+
+ @classmethod
+ def make_args_from_safety_checker(
+ cls,
+ safety_checker_taxonomy: SafetyCheckerTaxonomy,
+ is_moderated: bool | None,
+ safety_tolerance: int | None,
+ ) -> dict[str, Any]:
+ args_dict: dict[str, Any] = {}
+ match safety_checker_taxonomy:
+ case SafetyCheckerTaxonomy.UNAVAILABLE:
+ pass
+ case SafetyCheckerTaxonomy.AVAILABLE:
+ if is_moderated is not None:
+ args_dict["enable_safety_checker"] = is_moderated
+ if safety_tolerance is not None:
+ args_dict["safety_tolerance"] = safety_tolerance
+ return args_dict
diff --git a/pipelex/cogt/img_gen/img_gen_job_components.py b/pipelex/cogt/img_gen/img_gen_job_components.py
index 451a35d42..909cff63a 100644
--- a/pipelex/cogt/img_gen/img_gen_job_components.py
+++ b/pipelex/cogt/img_gen/img_gen_job_components.py
@@ -42,9 +42,9 @@ class ImgGenJobParams(BaseModel):
quality: Quality | None = Field(default=None, strict=False)
nb_steps: int | None = Field(default=None, gt=0)
guidance_scale: float | None = Field(default=None, gt=0)
- is_moderated: bool = False
+ is_moderated: bool | None = None
safety_tolerance: int | None = Field(default=None, ge=1, le=6)
- is_raw: bool
+ is_raw: bool | None = None
output_format: OutputFormat = Field(strict=False)
seed: int | None = Field(default=None, ge=0)
@@ -69,9 +69,9 @@ class ImgGenJobParamsDefaults(ConfigModel):
quality: Quality | None = Field(default=None, strict=False)
nb_steps: int | None = Field(default=None, gt=0)
guidance_scale: float = Field(..., gt=0)
- is_moderated: bool
+ is_moderated: bool | None = None
safety_tolerance: int = Field(..., ge=1, le=6)
- is_raw: bool
+ is_raw: bool | None = None
output_format: OutputFormat = Field(strict=False)
seed: int | Literal["auto"]
diff --git a/pipelex/cogt/img_gen/img_gen_model_rules.py b/pipelex/cogt/img_gen/img_gen_model_rules.py
new file mode 100644
index 000000000..ae9fb6b8b
--- /dev/null
+++ b/pipelex/cogt/img_gen/img_gen_model_rules.py
@@ -0,0 +1,44 @@
+from pipelex.types import StrEnum
+
+
+class ImgGenArgTopic(StrEnum):
+ NUM_IMAGES = "num_images"
+ ASPECT_RATIO = "aspect_ratio"
+ INFERENCE = "inference"
+ SAFETY_CHECKER = "safety_checker"
+ BACKGROUND = "background"
+ SPECIFIC = "specific"
+
+
+class NumImagesTaxonomy(StrEnum):
+ FAL = "fal"
+ GPT = "gpt"
+
+
+class SpecificTaxonomy(StrEnum):
+ FAL = "fal"
+
+
+class AspectRatioTaxonomy(StrEnum):
+ FLUX = "flux"
+ FLUX_11_ULTRA = "flux_11_ultra"
+ GPT = "gpt"
+
+
+class InferenceTaxonomy(StrEnum):
+ SDXL_LIGHTNING = "sdxl_lightning"
+ FLUX = "flux"
+ FLUX_11_ULTRA = "flux_11_ultra"
+ GPT = "gpt"
+
+
+class SafetyCheckerTaxonomy(StrEnum):
+ AVAILABLE = "available"
+ UNAVAILABLE = "unavailable"
+
+
+class BackgroundTaxonomy(StrEnum):
+ GPT = "gpt"
+
+
+ImgGenModelRules = dict[ImgGenArgTopic, str]
diff --git a/pipelex/cogt/img_gen/img_gen_worker_factory.py b/pipelex/cogt/img_gen/img_gen_worker_factory.py
index ecb7fc25c..bc5105fc6 100644
--- a/pipelex/cogt/img_gen/img_gen_worker_factory.py
+++ b/pipelex/cogt/img_gen/img_gen_worker_factory.py
@@ -4,6 +4,7 @@ from pipelex.cogt.exceptions import MissingDependencyError
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.hub import get_models_manager, get_plugin_manager
+from pipelex.plugins.blackboxai.blackboxai_completions_factory import BlackboxaiCompletionsFactory
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.system.exceptions import CredentialsError
@@ -25,6 +26,20 @@ class ImgGenWorkerFactory:
plugin_sdk_registry = get_plugin_manager().plugin_sdk_registry
img_gen_worker: ImgGenWorkerAbstract
match plugin.sdk:
+ case "gateway_img_gen":
+ from pipelex.plugins.gateway.gateway_factory import GatewayFactory # noqa: PLC0415
+ from pipelex.plugins.gateway.gateway_img_gen_worker import GatewayImgGenWorker # noqa: PLC0415
+
+ img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GatewayFactory.make_portkey_client(backend=backend),
+ )
+
+ img_gen_worker = GatewayImgGenWorker(
+ sdk_instance=img_gen_sdk_instance,
+ inference_model=inference_model,
+ reporting_delegate=reporting_delegate,
+ )
case "fal":
if importlib.util.find_spec("fal_client") is None:
lib_name = "fal-client"
@@ -67,9 +82,9 @@ class ImgGenWorkerFactory:
inference_model=inference_model,
reporting_delegate=reporting_delegate,
)
- case "openai_alt_img_gen":
+ case "blackboxai_img_gen":
from pipelex.plugins.openai.openai_client_factory import OpenAIClientFactory # noqa: PLC0415
- from pipelex.plugins.openai.openai_img_gen_alt_worker import OpenAIImgGenAlternativeWorker # noqa: PLC0415
+ from pipelex.plugins.openai.openai_completions_img_gen_worker import OpenAICompletionsImgGenWorker # noqa: PLC0415
img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -79,7 +94,10 @@ class ImgGenWorkerFactory:
),
)
- img_gen_worker = OpenAIImgGenAlternativeWorker(
+ openai_completions_factory = BlackboxaiCompletionsFactory(is_http_url_enabled=True)
+
+ img_gen_worker = OpenAICompletionsImgGenWorker(
+ openai_completions_factory=openai_completions_factory,
sdk_instance=img_gen_sdk_instance,
inference_model=inference_model,
reporting_delegate=reporting_delegate,
@@ -92,6 +110,53 @@ class ImgGenWorkerFactory:
inference_model=inference_model,
reporting_delegate=reporting_delegate,
)
+ case "google":
+ if importlib.util.find_spec("google.genai") is None:
+ lib_name = "google-genai"
+ lib_extra_name = "google"
+ msg = (
+ "The google-genai SDK is required in order to use Google Gemini Image models. "
+ "You can install it with 'pip install google-genai'."
+ )
+ raise MissingDependencyError(
+ lib_name,
+ lib_extra_name,
+ msg,
+ )
+
+ from pipelex.plugins.google.google_factory import GoogleFactory # noqa: PLC0415
+ from pipelex.plugins.google.google_img_gen_worker import GoogleImgGenWorker # noqa: PLC0415
+
+ img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GoogleFactory.make_google_client(backend=backend),
+ )
+
+ img_gen_worker = GoogleImgGenWorker(
+ sdk_instance=img_gen_sdk_instance,
+ inference_model=inference_model,
+ reporting_delegate=reporting_delegate,
+ )
+ case "gateway_completions":
+ from pipelex.plugins.gateway.gateway_completions_factory import GatewayCompletionsFactory # noqa: PLC0415
+ from pipelex.plugins.openai.openai_completions_img_gen_worker import OpenAICompletionsImgGenWorker # noqa: PLC0415
+
+ img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GatewayCompletionsFactory.make_portkey_openai_client_for_completions(
+ plugin=plugin,
+ backend=backend,
+ ),
+ )
+
+ gateway_completions_factory = GatewayCompletionsFactory(is_http_url_enabled=False)
+
+ img_gen_worker = OpenAICompletionsImgGenWorker(
+ openai_completions_factory=gateway_completions_factory,
+ sdk_instance=img_gen_sdk_instance,
+ inference_model=inference_model,
+ reporting_delegate=reporting_delegate,
+ )
case _:
msg = f"Plugin '{plugin}' is not supported for image generation"
raise NotImplementedError(msg)
diff --git a/pipelex/cogt/llm/llm_constants.py b/pipelex/cogt/inference/inference_constants.py
similarity index 77%
rename from pipelex/cogt/llm/llm_constants.py
rename to pipelex/cogt/inference/inference_constants.py
index 53e9fb5b4..2399b68bb 100644
--- a/pipelex/cogt/llm/llm_constants.py
+++ b/pipelex/cogt/inference/inference_constants.py
@@ -1,9 +1,10 @@
from pipelex.types import StrEnum
-class LLMOutputType(StrEnum):
+class InferenceOutputType(StrEnum):
TEXT = "Text"
OBJECT = "Object"
+ IMAGE = "Image"
@classmethod
def is_text(cls, output_desc: str) -> bool:
@@ -16,3 +17,5 @@ class LLMOutputType(StrEnum):
return True
case cls.OBJECT:
return False
+ case cls.IMAGE:
+ return False
diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.py
index abf64de94..03b3eff55 100644
--- a/pipelex/cogt/llm/llm_worker_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_abstract.py
@@ -8,8 +8,8 @@ from opentelemetry.trace import NonRecordingSpan, Span, SpanContext, SpanKind, S
from typing_extensions import override
from pipelex import log
+from pipelex.cogt.inference.inference_constants import InferenceOutputType
from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract
-from pipelex.cogt.llm.llm_constants import LLMOutputType
from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.pipeline.exceptions import JobMetadataError
from pipelex.pipeline.job_metadata import UnitJobId
@@ -81,7 +81,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
"""Get the response model name. Override in subclass."""
return "unknown"
- def _start_otel_span_llm(self, llm_job: LLMJob, output_type: LLMOutputType, output_class_name: str | None = None) -> Span | None:
+ def _start_otel_span_llm(self, llm_job: LLMJob, output_type: InferenceOutputType, output_class_name: str | None = None) -> Span | None:
"""Start an OTel span for the LLM job and return it.
Always includes full (non-redacted) values in Pipelex attributes for PostHog exporters.
@@ -116,11 +116,14 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
# Build output description for span name (full values - exporter handles redaction)
output_desc: str
match output_type:
- case LLMOutputType.TEXT:
+ case InferenceOutputType.TEXT:
output_desc = output_type
- case LLMOutputType.OBJECT:
+ case InferenceOutputType.OBJECT:
# Always use full class name in span name, exporter will redact if needed
output_desc = output_class_name or output_type
+ case InferenceOutputType.IMAGE:
+ msg = "Image output type is not supported for LLM span"
+ raise NotImplementedError(msg)
model_name = self._get_request_model_name()
# Always use full pipe code and class name - exporter handles redaction
@@ -364,7 +367,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
await self._before_job(llm_job=llm_job)
# Start OTel span after _before_job (which may set model info)
- span = self._start_otel_span_llm(llm_job=llm_job, output_type=LLMOutputType.TEXT)
+ span = self._start_otel_span_llm(llm_job=llm_job, output_type=InferenceOutputType.TEXT)
try:
text_result = await self._gen_text(llm_job=llm_job)
@@ -397,7 +400,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
await self._before_job(llm_job=llm_job)
# Start OTel span after _before_job (which may set model info)
- span = self._start_otel_span_llm(llm_job=llm_job, output_type=LLMOutputType.OBJECT, output_class_name=schema.__name__)
+ span = self._start_otel_span_llm(llm_job=llm_job, output_type=InferenceOutputType.OBJECT, output_class_name=schema.__name__)
try:
object_result = await self._gen_object(llm_job=llm_job, schema=schema)
diff --git a/pipelex/cogt/model_backends/model_spec.py b/pipelex/cogt/model_backends/model_spec.py
index 6ed288f1a..627b57cc3 100644
--- a/pipelex/cogt/model_backends/model_spec.py
+++ b/pipelex/cogt/model_backends/model_spec.py
@@ -3,6 +3,7 @@ from typing import Any
from instructor import Mode as InstructorMode
from pydantic import Field
+from pipelex.cogt.img_gen.img_gen_model_rules import ImgGenModelRules
from pipelex.cogt.llm.structured_output import StructureMethod
from pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedConstraint
from pipelex.cogt.model_backends.model_type import ModelType
@@ -28,6 +29,7 @@ class InferenceModelSpec(ConfigModel):
listed_constraints: list[ListedConstraint] = Field(default_factory=empty_list_factory_of(ListedConstraint))
valued_constraints: dict[ValuedConstraint, Any] = Field(default_factory=empty_dict_factory_of(ValuedConstraint))
extra_headers: dict[str, str] | None = None
+ rules: ImgGenModelRules | None = None
@property
def tag(self) -> str:
diff --git a/pipelex/cogt/model_backends/model_spec_factory.py b/pipelex/cogt/model_backends/model_spec_factory.py
index 23650d3c5..340c7384c 100644
--- a/pipelex/cogt/model_backends/model_spec_factory.py
+++ b/pipelex/cogt/model_backends/model_spec_factory.py
@@ -2,6 +2,7 @@ from typing import Any
from pydantic import BaseModel, Field, field_validator
+from pipelex.cogt.img_gen.img_gen_model_rules import ImgGenArgTopic, ImgGenModelRules
from pipelex.cogt.llm.structured_output import StructureMethod
from pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedConstraint
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -28,6 +29,17 @@ class InferenceModelSpecBlueprint(ConfigModel):
prompting_target: PromptingTarget | None = Field(default=None, strict=False)
listed_constraints: list[ListedConstraint] = Field(default_factory=empty_list_factory_of(ListedConstraint))
valued_constraints: dict[ValuedConstraint, Any] = Field(default_factory=empty_dict_factory_of(ValuedConstraint))
+ rules: ImgGenModelRules | None = None
+
+ @field_validator("rules", mode="before")
+ @staticmethod
+ def validate_rules(value: dict[str, str] | None) -> ImgGenModelRules | None:
+ if value is None:
+ return None
+ return ConfigModel.transform_dict_keys_str_to_enum(
+ input_dict=value,
+ key_enum_cls=ImgGenArgTopic,
+ )
@field_validator("costs", mode="before")
@staticmethod
@@ -87,4 +99,5 @@ class InferenceModelSpecFactory(BaseModel):
listed_constraints=merged_listed_constraints,
valued_constraints=merged_valued_constraints,
extra_headers=extra_headers,
+ rules=blueprint.rules,
)
diff --git a/pipelex/cogt/templating/template_blueprint.py b/pipelex/cogt/templating/template_blueprint.py
index 069ad5a5a..d63bb164d 100644
--- a/pipelex/cogt/templating/template_blueprint.py
+++ b/pipelex/cogt/templating/template_blueprint.py
@@ -3,9 +3,11 @@ from typing import Any
from pydantic import BaseModel, Field, model_validator
from pipelex.cogt.templating.template_category import TemplateCategory
+from pipelex.cogt.templating.template_preprocessor import preprocess_template
from pipelex.cogt.templating.templating_style import TemplatingStyle
from pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxError
from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing
+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables
class TemplateBlueprint(BaseModel):
@@ -24,3 +26,10 @@ class TemplateBlueprint(BaseModel):
msg = f"Could not parse template for TemplateBlueprint: {exc}"
raise ValueError(msg) from exc
return self
+
+ def required_variables(self) -> set[str]:
+ template_source = preprocess_template(self.template)
+ return detect_jinja2_required_variables(
+ template_category=self.category,
+ template_source=template_source,
+ )
diff --git a/pipelex/cogt/templating/template_category.py b/pipelex/cogt/templating/template_category.py
index 2529efa58..91f5cb6fd 100644
--- a/pipelex/cogt/templating/template_category.py
+++ b/pipelex/cogt/templating/template_category.py
@@ -16,6 +16,7 @@ class TemplateCategory(StrEnum):
MARKDOWN = "markdown"
MERMAID = "mermaid"
LLM_PROMPT = "llm_prompt"
+ IMG_GEN_PROMPT = "img_gen_prompt"
@property
def filters(self) -> dict[Jinja2FilterName, Callable[[Context, Any, TextFormat | None], Any]]:
@@ -37,5 +38,10 @@ class TemplateCategory(StrEnum):
Jinja2FilterName.FORMAT: text_format,
Jinja2FilterName.TAG: tag,
}
+ case TemplateCategory.IMG_GEN_PROMPT:
+ return {
+ Jinja2FilterName.FORMAT: text_format,
+ Jinja2FilterName.TAG: tag,
+ }
case TemplateCategory.MERMAID:
return {}
diff --git a/pipelex/core/pipes/inputs/input_requirements.py b/pipelex/core/pipes/inputs/input_requirements.py
index f80950be2..f000e7d62 100644
--- a/pipelex/core/pipes/inputs/input_requirements.py
+++ b/pipelex/core/pipes/inputs/input_requirements.py
@@ -126,8 +126,8 @@ class InputRequirements(RootModel[InputRequirementsRoot]):
return the_requirements
@property
- def nb_inputs(self) -> int:
- return len(self.root)
+ def is_empty(self) -> bool:
+ return not bool(self.root)
def _is_multiple(self, multiplicity: VariableMultiplicity | None) -> bool:
"""Check if multiplicity indicates multiple items."""
diff --git a/pipelex/kit/configs/inference/backends/blackboxai.toml b/pipelex/kit/configs/inference/backends/blackboxai.toml
index e6cf9dde0..9ee043309 100644
--- a/pipelex/kit/configs/inference/backends/blackboxai.toml
+++ b/pipelex/kit/configs/inference/backends/blackboxai.toml
@@ -193,7 +193,7 @@ costs = { input = 0.00, output = 0.00 }
[flux-pro]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/black-forest-labs/flux-pro"
inputs = ["text"]
outputs = ["image"]
@@ -201,7 +201,7 @@ costs = { input = 0.0, output = 0.04 }
["flux-pro/v1.1"]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/black-forest-labs/flux-1.1-pro"
inputs = ["text"]
outputs = ["image"]
@@ -209,15 +209,15 @@ costs = { input = 0.0, output = 0.04 }
["flux-pro/v1.1-ultra"]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/black-forest-labs/flux-1.1-pro-ultra"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.0, output = 0.06 }
-["fast-lightning-sdxl"]
+[fast-lightning-sdxl]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/bytedance/sdxl-lightning-4step"
inputs = ["text"]
outputs = ["image"]
@@ -225,8 +225,16 @@ costs = { input = 0.0, output = 0.0014 }
[nano-banana]
model_type = "img_gen"
-sdk = "openai_alt_img_gen"
+sdk = "blackboxai_img_gen"
model_id = "blackboxai/google/nano-banana"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.0, output = 0.039 }
+
+[nano-banana-pro]
+model_type = "img_gen"
+sdk = "blackboxai_img_gen"
+model_id = "blackboxai/google/nano-banana-pro"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0, output = 0.039 }
diff --git a/pipelex/kit/configs/inference/backends/fal.toml b/pipelex/kit/configs/inference/backends/fal.toml
index d54a25fcc..608bfed89 100644
--- a/pipelex/kit/configs/inference/backends/fal.toml
+++ b/pipelex/kit/configs/inference/backends/fal.toml
@@ -36,18 +36,52 @@ inputs = ["text"]
outputs = ["image"]
costs = { input = 0.05, output = 0.0 }
+[flux-pro.rules]
+num_images = "fal"
+aspect_ratio = "flux"
+inference = "flux"
+safety_checker = "available"
+specific = "fal"
+
["flux-pro/v1.1"]
model_id = "fal-ai/flux-pro/v1.1"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.05, output = 0.0 }
+["flux-pro/v1.1".rules]
+num_images = "fal"
+aspect_ratio = "flux"
+inference = "flux"
+safety_checker = "available"
+specific = "fal"
+
["flux-pro/v1.1-ultra"]
model_id = "fal-ai/flux-pro/v1.1-ultra"
inputs = ["text"]
outputs = ["image"]
costs = { input = 0.06, output = 0.0 }
+["flux-pro/v1.1-ultra".rules]
+num_images = "fal"
+aspect_ratio = "flux_11_ultra"
+inference = "flux_11_ultra"
+safety_checker = "available"
+specific = "fal"
+
+[flux-2]
+model_id = "fal-ai/flux-2"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+
+[flux-2.rules]
+num_images = "fal"
+aspect_ratio = "flux"
+inference = "flux"
+safety_checker = "available"
+specific = "fal"
+
# --- SDXL models --------------------------------------------------------------
[fast-lightning-sdxl]
model_id = "fal-ai/fast-lightning-sdxl"
@@ -55,3 +89,9 @@ inputs = ["text"]
outputs = ["image"]
costs = { input = 0.0003, output = 0.0 }
+[fast-lightning-sdxl.rules]
+num_images = "fal"
+specific = "fal"
+aspect_ratio = "flux"
+inference = "sdxl_lightning"
+safety_checker = "unavailable"
diff --git a/pipelex/kit/configs/inference/backends/google.toml b/pipelex/kit/configs/inference/backends/google.toml
index 0d39dacb1..e0d3f0af8 100644
--- a/pipelex/kit/configs/inference/backends/google.toml
+++ b/pipelex/kit/configs/inference/backends/google.toml
@@ -67,3 +67,21 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 3000
costs = { input = 2, output = 12.0 }
+
+################################################################################
+# IMAGE GENERATION MODELS (Nano Banana)
+################################################################################
+
+[nano-banana]
+model_type = "img_gen"
+model_id = "gemini-2.5-flash-image"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0, output = 0.039 }
+
+[nano-banana-pro]
+model_type = "img_gen"
+model_id = "gemini-3-pro-image-preview"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0, output = 0.039 }
diff --git a/pipelex/kit/configs/inference/deck/base_deck.toml b/pipelex/kit/configs/inference/deck/base_deck.toml
index 6fb0161d6..b2561f5ab 100644
--- a/pipelex/kit/configs/inference/deck/base_deck.toml
+++ b/pipelex/kit/configs/inference/deck/base_deck.toml
@@ -35,7 +35,7 @@ vision-groq = "llama-4-scout-17b-16e-instruct"
# Image generation aliases
base-img-gen = "flux-pro/v1.1"
-best-img-gen = "flux-pro/v1.1-ultra"
+best-img-gen = "flux-2"
fast-img-gen = "fast-lightning-sdxl"
####################################################################################################
@@ -95,8 +95,8 @@ cheap_llm = [
"grok-3-mini",
]
cheap_llm_for_vision = [
- "gpt-4o-mini",
"gemini-2.5-flash-lite",
+ "gpt-4o-mini",
"claude-3-haiku",
]
cheap_llm_for_structured = ["gpt-4o-mini", "mistral-small", "claude-3-haiku"]
@@ -188,6 +188,7 @@ choice_default = "gen_image_basic"
gen_image_basic = { model = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
gen_image_fast = { model = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
gen_image_high_quality = { model = "best-img-gen", quality = "high", guidance_scale = 8.0, is_moderated = true, safety_tolerance = 3 }
+gen_image_openai_low_quality = { model = "gpt-image-1", quality = "low" }
# Specific skills
img_gen_for_art = { model = "best-img-gen", quality = "high", guidance_scale = 9.0, is_moderated = false, safety_tolerance = 5 }
diff --git a/pipelex/kit/configs/inference/routing_profiles.toml b/pipelex/kit/configs/inference/routing_profiles.toml
index b3733ae23..68faae398 100644
--- a/pipelex/kit/configs/inference/routing_profiles.toml
+++ b/pipelex/kit/configs/inference/routing_profiles.toml
@@ -34,6 +34,7 @@ fallback_order = [
"pipelex_gateway",
"azure_openai",
"bedrock",
+ "google",
"blackboxai",
"mistral",
"fal",
@@ -41,15 +42,15 @@ fallback_order = [
[profiles.pipelex_gateway_first.routes]
# Pattern matching: "model-pattern" = "backend-name"
+
+[profiles.pipelex_gateway_first.optional_routes] # Each optional route is considered only if its backend is available
"gpt-*" = "pipelex_gateway"
+"gpt-image-1" = "openai"
"claude-*" = "pipelex_gateway"
"grok-*" = "pipelex_gateway"
"gemini-*" = "pipelex_gateway"
-
-[profiles.pipelex_gateway_first.optional_routes] # Each optional route is considered only if its backend is available
"*-sdxl" = "fal"
"flux-*" = "fal"
-"gpt-image-1" = "openai"
"mistral-ocr" = "mistral"
[profiles.all_pipelex_gateway]
diff --git a/pipelex/libraries/pipe/pipe_library.py b/pipelex/libraries/pipe/pipe_library.py
index 44a9fe071..53b8d953e 100644
--- a/pipelex/libraries/pipe/pipe_library.py
+++ b/pipelex/libraries/pipe/pipe_library.py
@@ -38,8 +38,8 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeLibraryAbstract):
def add_new_pipe(self, pipe: PipeAbstract):
if pipe.code in self.root:
msg = (
- f"Pipe '{pipe.code}' already exists in the library. You might be running the same pipe twice in the same pipeline."
- "We do not yet handle this case, so please avoid running the same pipe twice in the same pipeline"
+ f"Pipe '{pipe.code}' already exists in the library. You might be running the same pipe twice in the same pipeline. "
+ "We do not yet handle this case, so please avoid running the same pipe twice in the same pipeline. "
"Or consider adding for good in the library and call it by its code."
)
raise PipeLibraryError(msg)
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index a50c19345..2e8a454bf 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -1,7 +1,7 @@
-from typing import Literal
+from typing import TYPE_CHECKING, Any, Literal
-from pydantic import Field, model_validator
-from typing_extensions import Self, override
+from pydantic import Field
+from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
@@ -10,20 +10,18 @@ from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background,
from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting
from pipelex.cogt.models.model_deck_check import check_img_gen_choice_with_deck
+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint
from pipelex.config import get_config
-from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundError
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType
from pipelex.core.pipes.inputs.input_requirements import InputRequirements
-from pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
from pipelex.core.stuffs.exceptions import StuffContentTypeError
from pipelex.core.stuffs.image_content import ImageContent
from pipelex.core.stuffs.list_content import ListContent
-from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_model_deck, get_native_concept
from pipelex.pipe_operators.img_gen.exceptions import PipeImgGenRunError
@@ -33,29 +31,20 @@ from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams, output_multiplicity_to_apply
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.tools.misc.base_64_utils import extract_base_64_str_from_base64_url_if_possible
+from pipelex.tools.misc.dict_utils import substitute_nested_in_context
+
+if TYPE_CHECKING:
+ from pipelex.core.stuffs.stuff_content import StuffContent
class PipeImgGenOutput(PipeOutput):
- @property
- def image_urls(self) -> list[str]:
- the_urls: list[str] = []
- content = self.main_stuff.content
- if isinstance(content, ListContent):
- items = self.main_stuff_as_items(item_type=ImageContent)
- the_urls = [item.url for item in items]
- elif isinstance(content, ImageContent):
- the_urls = [content.url]
- else:
- msg = f"PipeImgGen output should be a ListContent or an ImageContent, got {type(content)}"
- raise PipeRunParamsError(msg)
- return the_urls
+ pass
class PipeImgGen(PipeOperator[PipeImgGenOutput]):
type: Literal["PipeImgGen"] = "PipeImgGen"
- img_gen_prompt: str | None = None
- img_gen_prompt_var_name: str | None = None
-
+ prompt_blueprint: TemplateBlueprint
img_gen_choice: ImgGenModelChoice | None = None
# One-time settings (not in ImgGenSetting)
@@ -68,42 +57,13 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
@override
def needed_inputs(self, visited_pipes: set[str] | None = None) -> InputRequirements:
- needed_inputs = InputRequirementsFactory.make_empty()
- if not self.img_gen_prompt:
- input_name, requirement = self.inputs.items[0] # We know there is only one input because of the validation
- needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
- return needed_inputs
+ """Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""
+ return self.inputs
@override
def required_variables(self) -> set[str]:
- if self.img_gen_prompt_var_name:
- return {self.img_gen_prompt_var_name}
- return set()
-
- @model_validator(mode="after")
- def validate_fields(self) -> Self:
- # Either we have img_gen_prompt or img_gen_prompt_var_name, but not both
- if self.img_gen_prompt_var_name is not None and self.img_gen_prompt is not None:
- msg = "Either 'img_gen_prompt' or 'img_gen_prompt_var_name' must be provided, but not both"
- raise ValueError(msg)
-
- # If img_gen_prompt is None, validate that inputs are properly configured
- if self.img_gen_prompt is None:
- # Must have exactly one input
- if not self.inputs or len(self.inputs.items) != 1:
- msg = "When 'img_gen_prompt' is not provided, there must be exactly one input"
- raise ValueError(msg)
-
- # The img_gen_prompt_var_name must match the key of the single input
- input_name, _ = self.inputs.items[0]
- if self.img_gen_prompt_var_name != input_name:
- msg = (
- f"When 'img_gen_prompt' is not provided, 'img_gen_prompt_var_name' must match the input name. "
- f"Expected '{input_name}', got '{self.img_gen_prompt_var_name}'"
- )
- raise ValueError(msg)
-
- return self
+ """Required variables are the variables that are used in the prompt template"""
+ return {variable_name for variable_name in self.prompt_blueprint.required_variables() if not variable_name.startswith("_")}
@override
def validate_inputs_static(self):
@@ -112,28 +72,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
@override
def validate_inputs_with_library(self):
- concept_library = get_concept_library()
- input_name = self.inputs.variables[0]
- input_requirement = self.inputs.get_required_input_requirement(input_name)
-
- if not concept_library.is_compatible(
- tested_concept=input_requirement.concept,
- wanted_concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.TEXT),
- strict=True,
- ):
- msg = (
- f"The input of a PipeImgGen must be compatible with the Text concept (or refine it). "
- f"Input '{input_name}' has concept '{input_requirement.concept.concept_string}'. "
- "Image generation requires a text prompt. Use native.Text or a concept that refines it."
- )
- raise PipeValidationError(
- message=msg,
- error_type=PipeValidationErrorType.IMG_GEN_INPUT_NOT_TEXT_COMPATIBLE,
- pipe_code=self.code,
- variable_names=[input_name],
- required_concept_codes=["native.Text"],
- provided_concept_code=input_requirement.concept.concept_string,
- )
+ pass
@override
def validate_output_static(self):
@@ -176,31 +115,24 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
)
applied_output_multiplicity = multiplicity_resolution.resolved_multiplicity
- # TODO: move this to the factory
- log.verbose("Getting image generation prompt from context")
- if self.img_gen_prompt:
- img_gen_prompt_text = self.img_gen_prompt
- elif stuff_name := self.img_gen_prompt_var_name:
- try:
- img_gen_prompt_text = working_memory.get_stuff_as_str(stuff_name)
- except WorkingMemoryStuffNotFoundError as stuff_not_found_error:
- msg = (
- f"While runnning the PipeImgGen '{self.code}' an error occurred: "
- f"Could not find a valid image generation prompt named '{stuff_name}' in the working_memory: {stuff_not_found_error}"
- )
- raise PipeImgGenRunError(message=msg) from stuff_not_found_error
- except StuffContentTypeError as stuff_content_type_error:
- msg = (
- f"While runnning the PipeImgGen '{self.code}' an error occurred: "
- f"The image generation prompt named '{stuff_name}' is not of the right type: {stuff_content_type_error}"
- )
- raise PipeImgGenRunError(message=msg) from stuff_content_type_error
- else:
- msg = (
- f"While runnning the PipeImgGen '{self.code}' an error occurred: "
- "You must provide an image gen prompt either as attribute of the pipe or as a single text input"
+ try:
+ context: dict[str, Any] = working_memory.generate_context()
+ if extra_params := pipe_run_params.params:
+ context = substitute_nested_in_context(context=context, extra_params=extra_params)
+ if extra_context := self.prompt_blueprint.extra_context:
+ context.update(**extra_context)
+ img_gen_prompt_text = await content_generator.make_templated_text(
+ context=context,
+ template=self.prompt_blueprint.template,
+ templating_style=self.prompt_blueprint.templating_style,
+ template_category=self.prompt_blueprint.category,
)
- raise PipeImgGenRunError(msg)
+ except WorkingMemoryStuffNotFoundError as stuff_not_found_error:
+ msg = f"While runnning the PipeImgGen '{self.code}' some inputs could not be found in the working_memory: {stuff_not_found_error}"
+ raise PipeImgGenRunError(message=msg) from stuff_not_found_error
+ except StuffContentTypeError as stuff_content_type_error:
+ msg = f"While runnning the PipeImgGen '{self.code}' some inputs are not of the right type: {stuff_content_type_error}"
+ raise PipeImgGenRunError(message=msg) from stuff_content_type_error
img_gen_config = get_config().cogt.img_gen_config
img_gen_param_defaults = img_gen_config.img_gen_param_defaults
@@ -241,7 +173,6 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
log.verbose(f"Using img_gen handle: {img_gen_handle}")
the_content: StuffContent
- image_urls: list[str] = []
nb_images: int
if isinstance(applied_output_multiplicity, bool):
if self.output_multiplicity:
@@ -259,9 +190,9 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
# Get the structure class from the registry (might be a subclass of ImageContent)
structure_class = get_class_registry().get_required_subclass(
name=self.output.structure_class_name,
- base_class=StuffContent,
+ base_class=ImageContent,
)
-
+ base_64_str: str | None
if nb_images > 1:
generated_image_list = await content_generator.make_image_list(
job_metadata=job_metadata,
@@ -275,13 +206,15 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
)
image_content_items: list[StuffContent] = []
for generated_image in generated_image_list:
+ generated_image_url = generated_image.url
+ base_64_str = extract_base_64_str_from_base64_url_if_possible(possibly_base64_url=generated_image_url)
image_content_items.append(
structure_class(
- url=generated_image.url,
+ url=generated_image_url,
source_prompt=img_gen_prompt_text,
+ base_64=base_64_str,
),
)
- image_urls.append(generated_image.url)
the_content = ListContent(
items=image_content_items,
)
@@ -298,11 +231,12 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
)
generated_image_url = generated_image.url
- image_urls = [generated_image_url]
+ base_64_str = extract_base_64_str_from_base64_url_if_possible(possibly_base64_url=generated_image_url)
the_content = structure_class(
url=generated_image_url,
source_prompt=img_gen_prompt_text,
+ base_64=base_64_str,
)
log.verbose(the_content, title="Single image content")
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
index 62866ecfe..8a1351e7d 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
@@ -5,14 +5,18 @@ from typing_extensions import override
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat
from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice
+from pipelex.cogt.templating.template_category import TemplateCategory
+from pipelex.cogt.templating.template_preprocessor import preprocess_template
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxError
+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing
+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables
class PipeImgGenBlueprint(PipeBlueprint):
type: Literal["PipeImgGen"] = "PipeImgGen"
pipe_category: Literal["PipeOperator"] = "PipeOperator"
- img_gen_prompt: str | None = None
- img_gen_prompt_var_name: str | None = None
+ prompt: str
model: ImgGenModelChoice | None = None
@@ -25,19 +29,37 @@ class PipeImgGenBlueprint(PipeBlueprint):
@override
def validate_inputs(self):
- # check that we have either an img_gen_prompt passed as attribute or as a single text input
- if not self.inputs:
- if not self.img_gen_prompt:
- msg = "If no inputs are provided, you must provide an 'img_gen_prompt' as attribute."
- raise ValueError(msg)
-
- if self.inputs and self.img_gen_prompt:
- msg = "You must provide either an 'img_gen_prompt' as attribute or as a single text input, but not both"
- raise ValueError(msg)
+ # Get all required variables from prompt
+ template_category = TemplateCategory.IMG_GEN_PROMPT
+ preprocessed_template = preprocess_template(self.prompt)
+ try:
+ check_jinja2_parsing(
+ template_source=preprocessed_template,
+ template_category=template_category,
+ )
+ except Jinja2TemplateSyntaxError as exc:
+ msg = f"Could not parse template for PipeImgGen: {exc}"
+ raise ValueError(msg) from exc
+ # Filter out internal variables that start with underscore
+ required_variables = {
+ variable_name
+ for variable_name in detect_jinja2_required_variables(
+ template_category=template_category,
+ template_source=preprocessed_template,
+ )
+ if not variable_name.startswith("_")
+ }
+
+ # Check that all required variables are in inputs
+ input_names: set[str] = set(self.inputs.keys()) if self.inputs else set()
+ missing_variables: set[str] = required_variables - input_names
- nb_inputs = self.nb_inputs
- if nb_inputs > 1:
- msg = f"Too many inputs provided for PipeImgGen: {self.input_names}. Only one input is allowed."
+ if missing_variables:
+ missing_vars_str = ", ".join(sorted(missing_variables))
+ msg = (
+ f"Missing input variable(s) in prompt template: {missing_vars_str}. "
+ "These variables are used in the prompt but not declared in inputs."
+ )
raise ValueError(msg)
@override
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
index f571c8674..ca856fe02 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
@@ -2,6 +2,8 @@ from typing import Any
from typing_extensions import override
+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint
+from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.core.concepts.concept import Concept
from pipelex.core.pipes.inputs.input_requirements import InputRequirements
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
@@ -30,15 +32,10 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
# Convert bracket notation to output_multiplicity (default to 1 if no brackets)
final_multiplicity = output_parse_result.multiplicity if isinstance(output_parse_result.multiplicity, int) else 1
- img_gen_prompt = blueprint.img_gen_prompt
- img_gen_prompt_var_name = blueprint.img_gen_prompt_var_name
-
- # If we have inputs, that means that the prompt is in the inputs.
- # The blueprint already validated that there is only 1 input.
- if blueprint.inputs:
- img_gen_prompt_var_name = blueprint.input_names[0]
- else:
- img_gen_prompt = blueprint.img_gen_prompt
+ prompt_blueprint = TemplateBlueprint(
+ template=blueprint.prompt,
+ category=TemplateCategory.IMG_GEN_PROMPT,
+ )
return PipeImgGen(
domain=domain_code,
@@ -47,8 +44,7 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
inputs=inputs,
output=output,
output_multiplicity=final_multiplicity,
- img_gen_prompt=img_gen_prompt,
- img_gen_prompt_var_name=img_gen_prompt_var_name,
+ prompt_blueprint=prompt_blueprint,
img_gen_choice=blueprint.model,
aspect_ratio=blueprint.aspect_ratio,
is_raw=blueprint.is_raw,
diff --git a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py b/pipelex/pipe_operators/llm/llm_prompt_blueprint.py
index ba9ee9111..f2f9eea6e 100644
--- a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py
+++ b/pipelex/pipe_operators/llm/llm_prompt_blueprint.py
@@ -6,12 +6,10 @@ from pipelex import log
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.templating.template_blueprint import TemplateBlueprint
-from pipelex.cogt.templating.template_preprocessor import preprocess_template
from pipelex.cogt.templating.templating_style import TemplatingStyle
from pipelex.core.stuffs.image_content import ImageContent
from pipelex.hub import get_content_generator
from pipelex.pipe_operators.llm.exceptions import LLMPromptBlueprintValueError
-from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables
from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract, ContextProviderError
from pipelex.tools.misc.dict_utils import substitute_nested_in_context
@@ -32,22 +30,9 @@ class LLMPromptBlueprint(BaseModel):
required_variables.update(user_images_top_object_name)
if self.prompt_blueprint:
- template_source = preprocess_template(self.prompt_blueprint.template)
- required_variables.update(
- detect_jinja2_required_variables(
- template_category=self.prompt_blueprint.category,
- template_source=template_source,
- )
- )
-
+ required_variables.update(self.prompt_blueprint.required_variables())
if self.system_prompt_blueprint:
- system_prompt_template_source = preprocess_template(self.system_prompt_blueprint.template)
- required_variables.update(
- detect_jinja2_required_variables(
- template_category=self.system_prompt_blueprint.category,
- template_source=system_prompt_template_source,
- )
- )
+ required_variables.update(self.system_prompt_blueprint.required_variables())
return {
variable_name
for variable_name in required_variables
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 0ace7f4ad..3ab133fac 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -170,7 +170,7 @@ If some information is not present for an attribute, output the default value or
is_sync_mode = false
[cogt.img_gen_config.img_gen_param_defaults]
-nb_steps = 1 # 28 is a good default for Flux, possible values are [1,2,4,8] for SDXL Lightning
+nb_steps = 28 # 28 is a good default for Flux, possible values are [1,2,4,8] for SDXL Lightning
quality = "low"
aspect_ratio = "square" # "square", "landscape_4_3", "landscape_16_9", "landscape_21_9", "portrait_3_4", "portrait_9_16", "portrait_9_21"
guidance_scale = 3.5
diff --git a/pipelex/plugins/azure_rest/azure_img_gen_worker.py b/pipelex/plugins/azure_rest/azure_img_gen_worker.py
index b34d6baa7..ee61343ee 100644
--- a/pipelex/plugins/azure_rest/azure_img_gen_worker.py
+++ b/pipelex/plugins/azure_rest/azure_img_gen_worker.py
@@ -44,7 +44,7 @@ class AzureImgGenWorker(ImgGenWorkerAbstract):
if not backend.api_key:
msg = "Azure OpenAI API key (subscription_key) is not configured"
raise AzureCredentialsError(msg)
- self.subscription_key: str = backend.api_key
+ self.api_key: str = backend.api_key
#########################################################
# Instance methods
@@ -101,7 +101,7 @@ class AzureImgGenWorker(ImgGenWorkerAbstract):
response = await client.post(
generation_url,
headers={
- "Api-Key": self.subscription_key,
+ "Api-Key": self.api_key,
"Content-Type": "application/json",
},
json=generation_body,
diff --git a/pipelex/plugins/blackboxai/__init__.py b/pipelex/plugins/blackboxai/__init__.py
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/pipelex/plugins/blackboxai/__init__.py
@@ -0,0 +1 @@
+
diff --git a/pipelex/plugins/blackboxai/blackboxai_completions_factory.py b/pipelex/plugins/blackboxai/blackboxai_completions_factory.py
new file mode 100644
index 000000000..6153e4e06
--- /dev/null
+++ b/pipelex/plugins/blackboxai/blackboxai_completions_factory.py
@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import random
+from typing import TYPE_CHECKING, Any
+
+from typing_extensions import override
+
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.plugins.openai.openai_completions_factory import OpenAICompletionsFactory
+
+if TYPE_CHECKING:
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
+ from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+
+
+class BlackboxaiCompletionsFactory(OpenAICompletionsFactory):
+ @override
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
+ if isinstance(inference_job, ImgGenJob):
+ if isinstance(inference_job.job_params.seed, int):
+ seed = inference_job.job_params.seed
+ else:
+ seed = random.randint(0, 2**32 - 1)
+ extra_body = {"seed": seed}
+ else:
+ extra_body = {}
+ return inference_model.extra_headers or {}, extra_body
diff --git a/pipelex/plugins/fal/fal_factory.py b/pipelex/plugins/fal/fal_factory.py
index 5f728ef02..b853ca0a9 100644
--- a/pipelex/plugins/fal/fal_factory.py
+++ b/pipelex/plugins/fal/fal_factory.py
@@ -2,144 +2,11 @@ from typing import Any, cast
from pydantic import ValidationError
-from pipelex import log
-from pipelex.cogt.exceptions import ImgGenGenerationError, ImgGenParameterError
+from pipelex.cogt.exceptions import ImgGenGenerationError
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
-from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, OutputFormat, Quality
-from pipelex.config import get_config
class FalFactory:
- @classmethod
- def make_nb_steps_from_quality_for_flux_pro(cls, quality: Quality) -> int:
- flux_map_quality_to_steps = get_config().cogt.img_gen_config.fal_config.flux_map_quality_to_steps
- return flux_map_quality_to_steps[quality]
-
- @classmethod
- def make_nb_steps_from_quality_for_sdxl_lightning(cls, quality: Quality) -> int:
- sdxl_lightning_map_quality_to_steps = get_config().cogt.img_gen_config.fal_config.sdxl_lightning_map_quality_to_steps
- return sdxl_lightning_map_quality_to_steps[quality]
-
- @classmethod
- def make_image_size_for_flux_1(cls, aspect_ratio: AspectRatio) -> str:
- match aspect_ratio:
- case AspectRatio.SQUARE:
- return "square_hd"
- case AspectRatio.LANDSCAPE_4_3:
- return "landscape_4_3"
- case AspectRatio.LANDSCAPE_16_9:
- return "landscape_16_9"
- case AspectRatio.LANDSCAPE_21_9:
- return "landscape_21_9"
- case AspectRatio.PORTRAIT_3_4:
- return "portrait_4_3"
- case AspectRatio.PORTRAIT_9_16:
- return "portrait_16_9"
- case AspectRatio.PORTRAIT_9_21:
- return "portrait_21_9"
- case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
- msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model"
- raise ImgGenParameterError(msg)
-
- @classmethod
- def make_aspect_ratio_for_flux_1_1_ultra(cls, aspect_ratio: AspectRatio) -> str:
- match aspect_ratio:
- case AspectRatio.SQUARE:
- return "1:1"
- case AspectRatio.LANDSCAPE_4_3:
- return "4:3"
- case AspectRatio.LANDSCAPE_16_9:
- return "16:9"
- case AspectRatio.LANDSCAPE_21_9:
- return "21:9"
- case AspectRatio.PORTRAIT_3_4:
- return "3:4"
- case AspectRatio.PORTRAIT_9_16:
- return "9:16"
- case AspectRatio.PORTRAIT_9_21:
- return "9:21"
- case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
- msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model"
- raise ImgGenParameterError(msg)
-
- @classmethod
- def make_output_format_for_flux(cls, output_format: OutputFormat) -> str:
- match output_format:
- case OutputFormat.PNG:
- return "png"
- case OutputFormat.JPG:
- return "jpeg"
- case OutputFormat.WEBP:
- msg = "Output format WebP is not supported for Flux"
- raise ImgGenParameterError(msg)
-
- @classmethod
- def make_fal_arguments(
- cls,
- fal_application: str,
- img_gen_job: ImgGenJob,
- nb_images: int,
- ) -> dict[str, Any]:
- params = img_gen_job.job_params
- args_dict: dict[str, Any]
- num_inference_steps: int | None
- match fal_application:
- case "fal-ai/flux-pro" | "fal-ai/flux-pro/v1.1":
- num_inference_steps = params.nb_steps
- if not num_inference_steps:
- if not params.quality:
- msg = f"Either nb_steps or quality must be set for image generation with '{fal_application}'"
- raise ImgGenParameterError(msg)
- num_inference_steps = cls.make_nb_steps_from_quality_for_flux_pro(quality=params.quality)
-
- args_dict = {
- "prompt": img_gen_job.img_gen_prompt.positive_text,
- "image_size": cls.make_image_size_for_flux_1(params.aspect_ratio),
- "num_inference_steps": num_inference_steps,
- "guidance_scale": params.guidance_scale,
- "num_images": nb_images,
- "enable_safety_checker": params.is_moderated,
- "safety_tolerance": params.safety_tolerance,
- "seed": params.seed,
- "output_format": cls.make_output_format_for_flux(params.output_format),
- "sync_mode": img_gen_job.job_config.is_sync_mode,
- }
- case "fal-ai/flux-pro/v1.1-ultra":
- args_dict = {
- "prompt": img_gen_job.img_gen_prompt.positive_text,
- "aspect_ratio": cls.make_aspect_ratio_for_flux_1_1_ultra(params.aspect_ratio),
- "num_images": nb_images,
- "enable_safety_checker": params.is_moderated,
- "safety_tolerance": params.safety_tolerance,
- "raw": params.is_raw,
- "seed": params.seed,
- "output_format": cls.make_output_format_for_flux(params.output_format),
- "sync_mode": img_gen_job.job_config.is_sync_mode,
- }
- case "fal-ai/fast-lightning-sdxl":
- num_inference_steps = params.nb_steps
- if not num_inference_steps and (quality := params.quality):
- num_inference_steps = cls.make_nb_steps_from_quality_for_sdxl_lightning(quality=quality)
- acceptable_steps = [1, 2, 4, 8]
- if num_inference_steps not in acceptable_steps:
- log.warning(f"Number of inference steps {num_inference_steps}' for SDXL Lightning must be one of {acceptable_steps}")
- num_inference_steps = 8
- args_dict = {
- "prompt": img_gen_job.img_gen_prompt.positive_text,
- "image_size": cls.make_image_size_for_flux_1(params.aspect_ratio),
- "num_inference_steps": num_inference_steps,
- "num_images": nb_images,
- "seed": params.seed,
- "output_format": cls.make_output_format_for_flux(params.output_format),
- "sync_mode": img_gen_job.job_config.is_sync_mode,
- }
- case _:
- msg = f"Invalid fal application: '{fal_application}'"
- raise ImgGenParameterError(msg)
-
- return args_dict
-
@classmethod
def make_generated_image(cls, fal_result: dict[str, Any]) -> GeneratedImage:
generated_image_list = cls.make_generated_image_list(fal_result=fal_result)
diff --git a/pipelex/plugins/fal/fal_img_gen_worker.py b/pipelex/plugins/fal/fal_img_gen_worker.py
index 5674dd637..08a21ca6e 100644
--- a/pipelex/plugins/fal/fal_img_gen_worker.py
+++ b/pipelex/plugins/fal/fal_img_gen_worker.py
@@ -4,8 +4,9 @@ from fal_client import AsyncClient, InProgress
from typing_extensions import override
from pipelex import log
-from pipelex.cogt.exceptions import SdkTypeError
+from pipelex.cogt.exceptions import ImgGenParameterError, SdkTypeError
from pipelex.cogt.image.generated_image import GeneratedImage
+from pipelex.cogt.img_gen.img_gen_args_factory import ImgGenArgsFactory
from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -34,16 +35,19 @@ class FalImgGenWorker(ImgGenWorkerAbstract):
self,
img_gen_job: ImgGenJob,
) -> GeneratedImage:
- fal_application = self.inference_model.model_id
- arguments = FalFactory.make_fal_arguments(
- fal_application=fal_application,
+ if self.inference_model.rules is None:
+ msg = f"Model '{self.inference_model.name}' does not have rules configured"
+ raise ImgGenParameterError(msg)
+ args_dict = ImgGenArgsFactory.make_args_for_model(
+ model_rules=self.inference_model.rules,
img_gen_job=img_gen_job,
nb_images=1,
)
- log.verbose(arguments, title=f"Fal arguments, application={fal_application}")
+ fal_application = self.inference_model.model_id
+ log.verbose(args_dict, title=f"Fal arguments, application={fal_application}")
handler = await self.fal_async_client.submit(
application=fal_application,
- arguments=arguments,
+ arguments=args_dict,
)
log_index = 0
@@ -67,15 +71,19 @@ class FalImgGenWorker(ImgGenWorkerAbstract):
img_gen_job: ImgGenJob,
nb_images: int,
) -> list[GeneratedImage]:
- application = self.inference_model.model_id
- arguments = FalFactory.make_fal_arguments(
- fal_application=application,
+ if self.inference_model.rules is None:
+ msg = f"Model '{self.inference_model.name}' does not have rules configured"
+ raise ImgGenParameterError(msg)
+ args_dict = ImgGenArgsFactory.make_args_for_model(
+ model_rules=self.inference_model.rules,
img_gen_job=img_gen_job,
nb_images=nb_images,
)
+ fal_application = self.inference_model.model_id
+ log.verbose(args_dict, title=f"Fal arguments, application={fal_application}")
handler = await self.fal_async_client.submit(
- application=application,
- arguments=arguments,
+ application=fal_application,
+ arguments=args_dict,
)
log_index = 0
diff --git a/pipelex/plugins/fal/fal_poller.py b/pipelex/plugins/fal/fal_poller.py
new file mode 100644
index 000000000..774a0463f
--- /dev/null
+++ b/pipelex/plugins/fal/fal_poller.py
@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Any, cast
+
+import httpx
+from tenacity import (
+ AsyncRetrying,
+ retry_if_exception,
+ retry_if_exception_type,
+ retry_if_result,
+ stop_after_delay,
+ wait_random_exponential,
+)
+
+from pipelex.tools.misc.tenacity_utils import log_retry
+
+
+class FalPoller:
+ def _is_transient_http(self, exc: BaseException) -> bool:
+ if not isinstance(exc, httpx.HTTPStatusError):
+ return False
+ code = exc.response.status_code
+ return code == 429 or 500 <= code <= 599
+
+ async def poll_queue_until_complete(self, response_dict: dict[str, Any]) -> dict[str, Any]:
+ """Poll Queue API until completion and return the final response JSON.
+
+ Expects response_dict to include:
+ - status_url (str)
+ - response_url (str)
+
+ Reads API key from env var FAL_KEY.
+ """
+ status_url = response_dict.get("status_url")
+ response_url = response_dict.get("response_url")
+ if not isinstance(status_url, str) or not isinstance(response_url, str):
+ msg = "response_dict must include 'status_url' and 'response_url' as strings"
+ raise TypeError(msg)
+
+ async with httpx.AsyncClient(timeout=httpx.Timeout(20.0)) as client:
+
+ async def _try_once() -> dict[str, Any] | None:
+ # 1) poll status
+ status = await client.get(status_url)
+ status.raise_for_status() # will be retried on 429/5xx via tenacity
+
+ payload = status.json()
+ status = payload.get("status")
+
+ if status in {"IN_QUEUE", "IN_PROGRESS"}:
+ return None # tells tenacity to retry
+
+ if status == "COMPLETED":
+ # 2) fetch the actual response
+ res = await client.get(response_url)
+ res.raise_for_status()
+ return cast("dict[str, Any]", res.json())
+
+ # Terminal / unexpected states: fail fast (no retry)
+ msg = f"fal request ended with status={status!r}: {payload}"
+ raise RuntimeError(msg)
+
+ retrying = AsyncRetrying(
+ retry=(
+ retry_if_result(lambda r: r is None)
+ | retry_if_exception_type((httpx.TimeoutException, httpx.TransportError))
+ | retry_if_exception(self._is_transient_http)
+ ),
+ before_sleep=log_retry,
+ wait=wait_random_exponential(multiplier=0.5, max=8.0), # jittered backoff
+ stop=stop_after_delay(300.0), # total polling budget (seconds)
+ reraise=True,
+ )
+
+ async for attempt in retrying:
+ with attempt:
+ result = await _try_once()
+ if result is not None:
+ return result
+
+ msg = "Polling ended unexpectedly"
+ raise RuntimeError(msg)
diff --git a/pipelex/plugins/gateway/gateway_completions_factory.py b/pipelex/plugins/gateway/gateway_completions_factory.py
index 83085600f..eeb0a4ef7 100644
--- a/pipelex/plugins/gateway/gateway_completions_factory.py
+++ b/pipelex/plugins/gateway/gateway_completions_factory.py
@@ -17,7 +17,7 @@ from pipelex.plugins.openai.openai_completions_factory import OpenAICompletionsF
if TYPE_CHECKING:
from portkey_ai.api_resources.utils import GenericResponse
- from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.plugins.plugin_sdk_registry import Plugin
@@ -92,5 +92,7 @@ class GatewayCompletionsFactory(OpenAICompletionsFactory):
)
@override
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
- return GatewayFactory.make_extras(inference_model=inference_model, llm_job=llm_job, output_desc=output_desc)
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
+ return GatewayFactory.make_extras(inference_model=inference_model, inference_job=inference_job, output_desc=output_desc)
diff --git a/pipelex/plugins/gateway/gateway_deck.py b/pipelex/plugins/gateway/gateway_deck.py
new file mode 100644
index 000000000..4d7afb85f
--- /dev/null
+++ b/pipelex/plugins/gateway/gateway_deck.py
@@ -0,0 +1,12 @@
+from pipelex.plugins.gateway.gateway_exceptions import GatewayDeckError
+from pipelex.plugins.portkey.portkey_constants import PortkeyHeaderKey
+
+
+class GatewayDeck:
+ @classmethod
+ def get_config_id(cls, headers: dict[str, str]) -> str:
+ config_id = headers.get(PortkeyHeaderKey.CONFIG)
+ if not config_id:
+ msg = f"Could not get '{PortkeyHeaderKey.CONFIG}' field from headers"
+ raise GatewayDeckError(msg)
+ return config_id
diff --git a/pipelex/plugins/gateway/gateway_exceptions.py b/pipelex/plugins/gateway/gateway_exceptions.py
index 402ae8c62..e922d1579 100644
--- a/pipelex/plugins/gateway/gateway_exceptions.py
+++ b/pipelex/plugins/gateway/gateway_exceptions.py
@@ -1,13 +1,17 @@
from pipelex.cogt.exceptions import CogtError
-class PortkeyError(CogtError):
+class GatewayError(CogtError):
pass
-class GatewayFactoryError(PortkeyError):
+class GatewayFactoryError(GatewayError):
pass
-class GatewayCredentialsError(PortkeyError):
+class GatewayCredentialsError(GatewayError):
+ pass
+
+
+class GatewayDeckError(GatewayError):
pass
diff --git a/pipelex/plugins/gateway/gateway_extract_worker.py b/pipelex/plugins/gateway/gateway_extract_worker.py
index 6142f822e..738274e9f 100644
--- a/pipelex/plugins/gateway/gateway_extract_worker.py
+++ b/pipelex/plugins/gateway/gateway_extract_worker.py
@@ -15,7 +15,8 @@ from pipelex.cogt.extract.extract_worker_abstract import ExtractWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.config import get_config
from pipelex.plugins.gateway.gateway_completions_factory import GatewayCompletionsFactory
-from pipelex.plugins.portkey.portkey_constants import PortkeyHeaderKey
+from pipelex.plugins.gateway.gateway_deck import GatewayDeck
+from pipelex.plugins.gateway.gateway_factory import GatewayFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.misc.base_64_utils import make_base_64_url_from_location_async
from pipelex.types import StrEnum
@@ -34,7 +35,7 @@ class DocumentKind(StrEnum):
return "document_url"
-class PortkeyExtractWorker(ExtractWorkerAbstract):
+class GatewayExtractWorker(ExtractWorkerAbstract):
def __init__(
self,
sdk_instance: Any,
@@ -115,44 +116,40 @@ class PortkeyExtractWorker(ExtractWorkerAbstract):
document_type: DocumentKind,
should_include_images: bool = False,
) -> ExtractOutput:
- config_id = self._get_portkey_config_id()
+ config_id = GatewayDeck.get_config_id(headers=self.inference_model.extra_headers or {})
log.dev(f"Extracting using config '{config_id}' with should_include_images: {should_include_images}")
doc_tag = document_type.document_tag
attempt_number = 0
response: GenericResponse | None = None
retryer = self._make_retryer()
- async for attempt in retryer:
- with attempt:
- attempt_number += 1
- response = await self.portkey_client.with_options(config=config_id).post( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- "/",
- model=self.inference_model.model_id,
- document={"type": doc_tag, doc_tag: base64_url},
- include_image_base64=True,
- )
+ try:
+ async for attempt in retryer:
+ with attempt:
+ attempt_number += 1
+ response = await self.portkey_client.with_options(config=config_id).post( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+ "/",
+ model=self.inference_model.model_id,
+ document={"type": doc_tag, doc_tag: base64_url},
+ include_image_base64=True,
+ )
+ except portkey_exceptions.APIError as exc:
+ error_summary = GatewayFactory.make_error_summary_from_portkey_error(exc)
+ msg = f"Extract service error for model '{self.inference_model.tag}' after {attempt_number} attempt(s): {error_summary}"
+ raise ExtractJobFailureError(msg) from exc
if response is None:
- msg = f"Could not get a response for model '{self.inference_model.model_id}' via Portkey after {attempt_number} attempts"
+ msg = f"Could not get a response for model '{self.inference_model.tag}' via Portkey after {attempt_number} attempts"
raise ExtractJobFailureError(msg)
if not isinstance(response, GenericResponse):
msg = "Response is not of type GenericResponse"
raise TypeError(msg)
+
return GatewayCompletionsFactory.make_extract_output_from_portkey_response(
response=response,
)
- def _get_portkey_config_id(self) -> str:
- if not self.inference_model.extra_headers:
- msg = f"{PortkeyHeaderKey.CONFIG} header is required"
- raise ExtractInputError(msg)
- config_id = self.inference_model.extra_headers.get(PortkeyHeaderKey.CONFIG)
- if not config_id:
- msg = f"{PortkeyHeaderKey.CONFIG} header is required"
- raise ExtractInputError(msg)
- return config_id
-
def _is_retryable_portkey_error(self, exc: BaseException) -> bool:
if isinstance(exc, portkey_exceptions.NotFoundError):
msg = str(exc).lower()
diff --git a/pipelex/plugins/gateway/gateway_factory.py b/pipelex/plugins/gateway/gateway_factory.py
index 797d87c97..334196574 100644
--- a/pipelex/plugins/gateway/gateway_factory.py
+++ b/pipelex/plugins/gateway/gateway_factory.py
@@ -4,18 +4,22 @@ from typing import TYPE_CHECKING, Any
from portkey_ai import (
PORTKEY_GATEWAY_URL,
- AsyncPortkey, # type: ignore[reportUnknownVariableType]
+ AsyncPortkey,
)
+from portkey_ai.api_resources import exceptions as portkey_exc
from pipelex import log
-from pipelex.cogt.llm.llm_constants import LLMOutputType
+from pipelex.cogt.inference.inference_constants import InferenceOutputType
from pipelex.hub import get_telemetry_manager
from pipelex.plugins.gateway.gateway_exceptions import GatewayCredentialsError
from pipelex.plugins.portkey.portkey_constants import PortkeyHeaderKey
from pipelex.system.telemetry.otel_constants import OTelConstants
+from pipelex.urls import URLs
if TYPE_CHECKING:
- from pipelex.cogt.llm.llm_job import LLMJob
+ from portkey_ai.api_resources import exceptions as portkey_exceptions
+
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -54,7 +58,9 @@ class GatewayFactory:
)
@classmethod
- def make_extras(cls, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
+ def make_extras(
+ cls, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
extra_headers: dict[str, str] = {}
if inference_model.extra_headers:
extra_headers.update(inference_model.extra_headers)
@@ -62,15 +68,57 @@ class GatewayFactory:
extra_headers[PortkeyHeaderKey.PROVIDER] = inference_model.backend_name
# OTel-correlated Portkey tracing (only when enabled and OTel context available)
- if get_telemetry_manager().is_portkey_tracing_enabled() and (otel_context := llm_job.job_metadata.otel_context):
+ if get_telemetry_manager().is_portkey_tracing_enabled() and (otel_context := inference_job.job_metadata.otel_context):
# Use OTel trace_id and span_id for correlation
extra_headers[PortkeyHeaderKey.TRACE_ID] = f"{otel_context.trace_id:032x}"
extra_headers[PortkeyHeaderKey.SPAN_ID] = f"{otel_context.span_id:016x}"
# Build span name with redacted output class name (consistent with Pipelex telemetry policy)
# Pipelex services always redact sensitive data to protect user privacy
- unit_job_id = llm_job.job_metadata.unit_job_id or "unknown"
- display_output = output_desc if output_desc == LLMOutputType.TEXT else OTelConstants.OUTPUT_CLASS_REDACTED
+ unit_job_id = inference_job.job_metadata.unit_job_id or "unknown"
+ display_output = output_desc if output_desc == InferenceOutputType.TEXT else OTelConstants.OUTPUT_CLASS_REDACTED
extra_headers[PortkeyHeaderKey.SPAN_NAME] = f"{unit_job_id} -> {display_output}"
return extra_headers, {}
+
+ @classmethod
+ def make_error_summary_from_portkey_error(cls, exc: portkey_exceptions.APIError) -> str:
+ """Extract a clean, human-readable error summary from a Portkey API error.
+
+ Args:
+ exc: The Portkey API error
+
+ Returns:
+ A concise error message suitable for logging and user display
+ """
+ error_type = type(exc).__name__
+ support_hint = f"If the problem persists, get support on Discord: {URLs.discord}"
+
+ # Connection errors (no HTTP response received)
+ if isinstance(exc, portkey_exc.APITimeoutError):
+ return f"{error_type}: Request timed out - service may be overloaded. {support_hint}"
+ if isinstance(exc, portkey_exc.APIConnectionError):
+ return f"{error_type}: Cannot connect to Pipelex Gateway - check network or service availability. {support_hint}"
+
+ # HTTP status errors (4xx/5xx)
+ if isinstance(exc, portkey_exc.APIStatusError):
+ status_code = exc.status_code
+ error_body = str(exc)
+
+ # For HTML responses, provide a generic message (gateway/proxy error pages)
+ if error_body.strip().startswith("<!DOCTYPE") or "<html" in error_body.lower():
+ return f"{error_type} (HTTP {status_code}): Pipelex Gateway unavailable. {support_hint}"
+
+ # For other errors, truncate if too long
+ max_length = 200
+ if len(error_body) > max_length:
+ error_body = error_body[:max_length] + "..."
+
+ return f"{error_type} (HTTP {status_code}): {error_body}"
+
+ # Response validation errors
+ if isinstance(exc, portkey_exc.APIResponseValidationError):
+ return f"{error_type}: Invalid response from Pipelex Gateway. {support_hint}"
+
+ # Fallback for any other APIError
+ return f"{error_type}: {exc.message}. {support_hint}"
diff --git a/pipelex/plugins/gateway/gateway_img_gen_factory.py b/pipelex/plugins/gateway/gateway_img_gen_factory.py
new file mode 100644
index 000000000..6cab6c772
--- /dev/null
+++ b/pipelex/plugins/gateway/gateway_img_gen_factory.py
@@ -0,0 +1,63 @@
+from typing import Literal
+
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, OutputFormat
+
+GatewayImageSizeType = Literal[
+ "square",
+ "landscape_4_3",
+ "landscape_3_2",
+ "landscape_16_9",
+ "landscape_21_9",
+ "portrait_3_4",
+ "portrait_2_3",
+ "portrait_9_16",
+ "portrait_9_21",
+]
+
+GatewayMimeSubtypeType = Literal["png", "jpeg", "webp"]
+
+
+class GatewayImgGenFactory:
+ @classmethod
+ def image_size_for_gateway(cls, aspect_ratio: AspectRatio) -> GatewayImageSizeType:
+ # Pipelex AspectRatio values match Gateway expected strings (see sample: "landscape_4_3")
+ match aspect_ratio:
+ case AspectRatio.SQUARE:
+ return "square"
+ case AspectRatio.LANDSCAPE_4_3:
+ return "landscape_4_3"
+ case AspectRatio.LANDSCAPE_3_2:
+ return "landscape_3_2"
+ case AspectRatio.LANDSCAPE_16_9:
+ return "landscape_16_9"
+ case AspectRatio.LANDSCAPE_21_9:
+ return "landscape_21_9"
+ case AspectRatio.PORTRAIT_3_4:
+ return "portrait_3_4"
+ case AspectRatio.PORTRAIT_2_3:
+ return "portrait_2_3"
+ case AspectRatio.PORTRAIT_9_16:
+ return "portrait_9_16"
+ case AspectRatio.PORTRAIT_9_21:
+ return "portrait_9_21"
+
+ @classmethod
+ def output_format_for_gateway(cls, output_format: OutputFormat) -> str:
+ match output_format:
+ case OutputFormat.PNG:
+ return "png"
+ case OutputFormat.JPG:
+ return "jpeg"
+ case OutputFormat.WEBP:
+ return "webp"
+
+ @classmethod
+ def mime_subtype_for_output_format(cls, output_format: OutputFormat) -> GatewayMimeSubtypeType:
+ # For data URIs / mime types, jpg should use jpeg.
+ match output_format:
+ case OutputFormat.PNG:
+ return "png"
+ case OutputFormat.JPG:
+ return "jpeg"
+ case OutputFormat.WEBP:
+ return "webp"
diff --git a/pipelex/plugins/gateway/gateway_img_gen_worker.py b/pipelex/plugins/gateway/gateway_img_gen_worker.py
new file mode 100644
index 000000000..5d3e1861e
--- /dev/null
+++ b/pipelex/plugins/gateway/gateway_img_gen_worker.py
@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from portkey_ai import AsyncPortkey
+from portkey_ai.api_resources import exceptions as portkey_exceptions
+from portkey_ai.api_resources.utils import GenericResponse
+from typing_extensions import override
+
+from pipelex.cogt.exceptions import ImgGenGenerationError, ImgGenParameterError, SdkTypeError
+from pipelex.cogt.image.generated_image import GeneratedImage
+from pipelex.cogt.img_gen.img_gen_args_factory import ImgGenArgsFactory
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.plugins.fal.fal_poller import FalPoller
+from pipelex.plugins.gateway.gateway_deck import GatewayDeck
+from pipelex.plugins.gateway.gateway_factory import GatewayFactory
+from pipelex.tools.misc.base_64_utils import prefixed_base64_str_from_base64_str
+
+if TYPE_CHECKING:
+ from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+ from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+ from pipelex.reporting.reporting_protocol import ReportingProtocol
+
+
+class GatewayImgGenWorker(ImgGenWorkerAbstract):
+ def __init__(
+ self,
+ sdk_instance: Any,
+ inference_model: InferenceModelSpec,
+ reporting_delegate: ReportingProtocol | None = None,
+ ):
+ super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)
+
+ if not isinstance(sdk_instance, AsyncPortkey):
+ msg = f"Provided ImgGen sdk_instance for {self.__class__.__name__} is not of type portkey_ai.AsyncPortkey: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
+
+ self.portkey_client: AsyncPortkey = sdk_instance
+
+ @override
+ async def _gen_image(
+ self,
+ img_gen_job: ImgGenJob,
+ ) -> GeneratedImage:
+ one_image_list = await self._gen_image_list(img_gen_job=img_gen_job, nb_images=1)
+ return one_image_list[0]
+
+ @override
+ async def _gen_image_list(
+ self,
+ img_gen_job: ImgGenJob,
+ nb_images: int,
+ ) -> list[GeneratedImage]:
+ if self.inference_model.rules is None:
+ msg = f"Model '{self.inference_model.name}' does not have rules configured"
+ raise ImgGenParameterError(msg)
+ args_dict = ImgGenArgsFactory.make_args_for_model(
+ model_rules=self.inference_model.rules,
+ img_gen_job=img_gen_job,
+ nb_images=nb_images,
+ )
+
+ endpoint_path = f"/{self.inference_model.model_id}"
+ config_id = GatewayDeck.get_config_id(headers=self.inference_model.extra_headers or {})
+ try:
+ response = await self.portkey_client.with_options(config=config_id).post(url=endpoint_path, **args_dict) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+ except portkey_exceptions.APIError as exc:
+ error_summary = GatewayFactory.make_error_summary_from_portkey_error(exc)
+ msg = f"Image generation service error for model '{self.inference_model.model_id}': {error_summary}"
+ raise ImgGenGenerationError(msg) from exc
+
+ if response is None:
+ msg = f"Could not get a response for model '{self.inference_model.model_id}' via Portkey"
+ raise ImgGenGenerationError(msg)
+
+ if not isinstance(response, GenericResponse):
+ msg = "Response is not of type GenericResponse"
+ raise TypeError(msg)
+
+ response_dict: dict[str, Any] = response.model_dump()
+ generated_images: list[GeneratedImage] = []
+
+ if images := response_dict.get("data"):
+ size = response_dict.get("size")
+ if not isinstance(size, str):
+ msg = f"Size from img gen response is not a string: '{size}'"
+ raise ImgGenGenerationError(msg)
+ size_split = size.split("x")
+ if len(size_split) != 2:
+ msg = f"Size from img gen response is not a valid size: '{size}'"
+ raise ImgGenGenerationError(msg)
+ width_str, height_str = size_split
+ width = int(width_str)
+ height = int(height_str)
+ for image in images:
+ b64_str = image.get("b64_json")
+ if not isinstance(b64_str, str):
+ msg = f"No base64 image data received from model '{self.inference_model.model_id}'"
+ raise ImgGenGenerationError(msg)
+ base64_url = prefixed_base64_str_from_base64_str(b64_str=b64_str)
+ generated_images.append(
+ GeneratedImage(
+ url=base64_url,
+ width=width,
+ height=height,
+ ),
+ )
+
+ elif response_dict.get("status") in {"IN_QUEUE", "IN_PROGRESS"}:
+ # Handle FAL queue responses that require polling
+ fal_poller = FalPoller()
+ response_dict = await fal_poller.poll_queue_until_complete(response_dict=response_dict)
+
+ for item in response_dict.get("images", []):
+ generated_image = GeneratedImage(url=item.get("url"), width=item.get("width"), height=item.get("height"))
+ generated_images.append(generated_image)
+ else:
+ msg = f"Unexpected response from model '{self.inference_model.model_id}' has no 'data' or 'images' key"
+ raise ImgGenGenerationError(msg)
+
+ return generated_images
diff --git a/pipelex/plugins/gateway/gateway_responses_factory.py b/pipelex/plugins/gateway/gateway_responses_factory.py
index a5f37c5c0..ac9430fd4 100644
--- a/pipelex/plugins/gateway/gateway_responses_factory.py
+++ b/pipelex/plugins/gateway/gateway_responses_factory.py
@@ -14,7 +14,7 @@ from pipelex.plugins.gateway.gateway_factory import GatewayFactory
from pipelex.plugins.openai.openai_responses_factory import OpenAIResponsesFactory
if TYPE_CHECKING:
- from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.plugins.plugin_sdk_registry import Plugin
@@ -44,5 +44,7 @@ class GatewayResponsesFactory(OpenAIResponsesFactory):
)
@override
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
- return GatewayFactory.make_extras(inference_model=inference_model, llm_job=llm_job, output_desc=output_desc)
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
+ return GatewayFactory.make_extras(inference_model=inference_model, inference_job=inference_job, output_desc=output_desc)
diff --git a/pipelex/plugins/google/google_img_gen_factory.py b/pipelex/plugins/google/google_img_gen_factory.py
new file mode 100644
index 000000000..48ff220e3
--- /dev/null
+++ b/pipelex/plugins/google/google_img_gen_factory.py
@@ -0,0 +1,55 @@
+from typing import ClassVar, Literal
+
+from pipelex.cogt.exceptions import ImgGenParameterError
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
+
+GoogleAspectRatioType = Literal["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"]
+
+
+class GoogleImgGenFactory:
+ """Factory class for Google image generation parameter mappings."""
+
+ # Resolution mappings for Gemini 2.5 Flash Image (Nano Banana)
+ # Reference: https://ai.google.dev/gemini-api/docs/image-generation
+ ASPECT_RATIO_TO_SIZE: ClassVar[dict[GoogleAspectRatioType, tuple[int, int]]] = {
+ "1:1": (1024, 1024),
+ "2:3": (832, 1248),
+ "3:2": (1248, 832),
+ "3:4": (864, 1184),
+ "4:3": (1184, 864),
+ "4:5": (896, 1152),
+ "5:4": (1152, 896),
+ "9:16": (768, 1344),
+ "16:9": (1344, 768),
+ "21:9": (1536, 672),
+ }
+
+ @classmethod
+ def aspect_ratio_string(cls, aspect_ratio: AspectRatio) -> GoogleAspectRatioType:
+ """Map AspectRatio enum to Google's string format."""
+ match aspect_ratio:
+ case AspectRatio.SQUARE:
+ return "1:1"
+ case AspectRatio.LANDSCAPE_4_3:
+ return "4:3"
+ case AspectRatio.LANDSCAPE_3_2:
+ return "3:2"
+ case AspectRatio.LANDSCAPE_16_9:
+ return "16:9"
+ case AspectRatio.LANDSCAPE_21_9:
+ return "21:9"
+ case AspectRatio.PORTRAIT_3_4:
+ return "3:4"
+ case AspectRatio.PORTRAIT_2_3:
+ return "2:3"
+ case AspectRatio.PORTRAIT_9_16:
+ return "9:16"
+ case AspectRatio.PORTRAIT_9_21:
+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Google Gemini Image models"
+ raise ImgGenParameterError(msg)
+
+ @classmethod
+ def image_size_for_aspect_ratio(cls, aspect_ratio: AspectRatio) -> tuple[int, int]:
+ """Get pixel dimensions (width, height) for the given aspect ratio."""
+ aspect_ratio_str = cls.aspect_ratio_string(aspect_ratio)
+ return cls.ASPECT_RATIO_TO_SIZE[aspect_ratio_str]
diff --git a/pipelex/plugins/google/google_img_gen_worker.py b/pipelex/plugins/google/google_img_gen_worker.py
new file mode 100644
index 000000000..2ea22476e
--- /dev/null
+++ b/pipelex/plugins/google/google_img_gen_worker.py
@@ -0,0 +1,155 @@
+import asyncio
+import base64
+from typing import Any
+
+from google import genai
+from google.genai import types as genai_types
+from typing_extensions import override
+
+from pipelex import log
+from pipelex.base_exceptions import PipelexError
+from pipelex.cogt.exceptions import ImgGenGenerationError, SdkTypeError
+from pipelex.cogt.image.generated_image import GeneratedImage
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.plugins.google.google_img_gen_factory import GoogleImgGenFactory
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+from pipelex.tools.misc.base_64_utils import prefixed_base64_str_from_base64_str
+
+
+class GoogleImgGenWorkerError(PipelexError):
+ """Base exception for Google Image Generation Worker errors."""
+
+
+class GoogleImgGenWorker(ImgGenWorkerAbstract):
+ """Worker for generating images using Google Gemini Image models (Nano Banana)."""
+
+ def __init__(
+ self,
+ sdk_instance: Any,
+ inference_model: InferenceModelSpec,
+ reporting_delegate: ReportingProtocol | None = None,
+ ):
+ super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)
+
+ if not isinstance(sdk_instance, genai.Client):
+ msg = f"Provided ImgGen sdk_instance is not of type genai.Client: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
+
+ self.genai_client: genai.Client = sdk_instance
+ self.genai_async_client = sdk_instance.aio
+
+ # Capture the event loop at creation time if one is running
+ self._event_loop: asyncio.AbstractEventLoop | None
+ try:
+ self._event_loop = asyncio.get_running_loop()
+ except RuntimeError:
+ # No running loop at creation time
+ self._event_loop = None
+
+ @override
+ def teardown(self):
+ """Close the async client to free resources."""
+ try:
+ # First, try to use the loop captured at creation time if it's still running
+ if self._event_loop is not None and self._event_loop.is_running():
+ # Schedule cleanup on the captured loop and store reference to prevent garbage collection
+ task = self._event_loop.create_task(self.genai_async_client.aclose())
+ # Add a callback to log any errors that occur during cleanup
+ task.add_done_callback(lambda t: log.debug(f"Google async client cleanup error: {t.exception()}") if t.exception() else None)
+ log.verbose("Scheduled Google async client cleanup on captured event loop")
+ return
+
+ # Otherwise, try to get the current running loop
+ try:
+ current_loop = asyncio.get_running_loop()
+ # Schedule cleanup on the current running loop and store reference to prevent garbage collection
+ task = current_loop.create_task(self.genai_async_client.aclose())
+ # Add a callback to log any errors that occur during cleanup
+ task.add_done_callback(lambda t: log.debug(f"Google async client cleanup error: {t.exception()}") if t.exception() else None)
+ log.verbose("Scheduled Google async client cleanup on current event loop")
+ except RuntimeError:
+ # No running event loop, we can safely use asyncio.run()
+ try:
+ asyncio.run(self.genai_async_client.aclose())
+ log.verbose("Closed Google async client using asyncio.run()")
+ except Exception as exc:
+ # Log but don't fail teardown if cleanup has issues
+ log.verbose(f"Error closing Google async client during teardown: {exc}")
+ except Exception as exc:
+ # Log but don't fail teardown if cleanup has issues
+ log.debug(f"Error during Google async client teardown: {exc}")
+
+ @override
+ async def _gen_image(
+ self,
+ img_gen_job: ImgGenJob,
+ ) -> GeneratedImage:
+ """Generate a single image using Google Gemini Image API."""
+ prompt_text = img_gen_job.img_gen_prompt.positive_text
+ aspect_ratio_str = GoogleImgGenFactory.aspect_ratio_string(img_gen_job.job_params.aspect_ratio)
+ width, height = GoogleImgGenFactory.image_size_for_aspect_ratio(img_gen_job.job_params.aspect_ratio)
+
+ # Build image config for aspect ratio
+ image_config = genai_types.ImageConfig(
+ aspect_ratio=aspect_ratio_str,
+ )
+
+ # Build generation config with image output
+ generation_config = genai_types.GenerateContentConfig(
+ response_modalities=["Image"],
+ image_config=image_config,
+ )
+
+ # Generate content using async client
+ response = await self.genai_async_client.models.generate_content(
+ model=self.inference_model.model_id,
+ contents=prompt_text,
+ config=generation_config,
+ )
+
+ # Extract image from response
+ if not response.candidates:
+ msg = f"No candidates returned from model: {self.inference_model.desc}"
+ raise ImgGenGenerationError(msg)
+
+ candidate = response.candidates[0]
+ if not candidate.content or not candidate.content.parts:
+ msg = f"No content parts in response from model: {self.inference_model.desc}"
+ raise ImgGenGenerationError(msg)
+
+ # Look for image data in response parts
+ for part in candidate.content.parts:
+ if part.inline_data is not None:
+ image_bytes = part.inline_data.data
+ if image_bytes is None:
+ continue
+ # Convert bytes to base64 data URL
+ base64_str = base64.b64encode(image_bytes).decode("utf-8")
+ base64_url = prefixed_base64_str_from_base64_str(base64_str)
+ return GeneratedImage(
+ url=base64_url,
+ width=width,
+ height=height,
+ )
+
+ msg = f"No image data in response from model: {self.inference_model.desc}"
+ raise ImgGenGenerationError(msg)
+
+ @override
+ async def _gen_image_list(
+ self,
+ img_gen_job: ImgGenJob,
+ nb_images: int,
+ ) -> list[GeneratedImage]:
+ """Generate multiple images by calling _gen_image multiple times.
+
+ Google Gemini Image API does not support generating multiple images in a single call,
+ so we generate them sequentially.
+ """
+ generated_images: list[GeneratedImage] = []
+ for _ in range(nb_images):
+ image = await self._gen_image(img_gen_job=img_gen_job)
+ generated_images.append(image)
+ return generated_images
diff --git a/pipelex/plugins/google/google_llm_worker.py b/pipelex/plugins/google/google_llm_worker.py
index 010a6b328..e12427285 100644
--- a/pipelex/plugins/google/google_llm_worker.py
+++ b/pipelex/plugins/google/google_llm_worker.py
@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, cast
import instructor
from google import genai
-from google.genai import types
+from google.genai import types as genai_types
from typing_extensions import override
from pipelex import log
@@ -103,10 +103,10 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):
"""Generate text using Google Gemini API."""
job_params = llm_job.applied_job_params or llm_job.job_params
- contents = await GoogleFactory.prepare_user_contents(llm_job.llm_prompt)
+ contents = await GoogleFactory.prepare_user_contents(llm_prompt=llm_job.llm_prompt)
# Build generation config
- generation_config = types.GenerateContentConfig(
+ generation_config = genai_types.GenerateContentConfig(
temperature=job_params.temperature,
max_output_tokens=job_params.max_tokens,
candidate_count=1, # Generate one candidate
@@ -156,7 +156,7 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):
contents = await GoogleFactory.prepare_user_contents(llm_job.llm_prompt)
# Build generation config
- generation_config = types.GenerateContentConfig(
+ generation_config = genai_types.GenerateContentConfig(
system_instruction=llm_job.llm_prompt.system_text,
temperature=job_params.temperature,
max_output_tokens=job_params.max_tokens,
diff --git a/pipelex/plugins/openai/openai_client_factory.py b/pipelex/plugins/openai/openai_client_factory.py
index 5377fd23e..f6b512c9e 100644
--- a/pipelex/plugins/openai/openai_client_factory.py
+++ b/pipelex/plugins/openai/openai_client_factory.py
@@ -17,7 +17,7 @@ class OpenAISdkVariant(StrEnum):
OPENAI = "openai"
OPENAI_RESPONSES = "openai_responses"
OPENAI_IMG_GEN = "openai_img_gen"
- OPENAI_ALT_IMG_GEN = "openai_alt_img_gen"
+ BLACKBOXAI_IMG_GEN = "blackboxai_img_gen"
class AzureExtraField(StrEnum):
@@ -39,7 +39,7 @@ class OpenAIClientFactory:
# We have a workaround here:
# OpenAI can be used without any API key (for instance when pointing to local Ollama) but the SDK,
- # as it is, raises if there is not API key (api_key is None and there is not env var).
+ # as it is, raises if there is no API key (api_key is None and there is no env var).
# But it works fine with an empty string.
api_key = backend.api_key or ""
@@ -55,7 +55,7 @@ class OpenAIClientFactory:
api_key=api_key,
api_version=backend.get_extra_config(AzureExtraField.API_VERSION),
)
- case OpenAISdkVariant.OPENAI | OpenAISdkVariant.OPENAI_RESPONSES | OpenAISdkVariant.OPENAI_IMG_GEN | OpenAISdkVariant.OPENAI_ALT_IMG_GEN:
+ case OpenAISdkVariant.OPENAI | OpenAISdkVariant.OPENAI_RESPONSES | OpenAISdkVariant.OPENAI_IMG_GEN | OpenAISdkVariant.BLACKBOXAI_IMG_GEN:
log.verbose(f"Making AsyncOpenAI client with endpoint: {backend.endpoint}")
the_client = openai.AsyncOpenAI(
api_key=api_key,
diff --git a/pipelex/plugins/openai/openai_completions_factory.py b/pipelex/plugins/openai/openai_completions_factory.py
index b4ae2c1b9..33b9e1872 100644
--- a/pipelex/plugins/openai/openai_completions_factory.py
+++ b/pipelex/plugins/openai/openai_completions_factory.py
@@ -15,6 +15,7 @@ from typing_extensions import override
from pipelex.cogt.image.prompt_image import PromptImageDetail, PromptImageTypedBase64
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
from pipelex.cogt.image.prompt_image_utils import prep_prompt_images
+from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
@@ -74,5 +75,7 @@ class OpenAICompletionsFactory(PluginFactoryAbstract):
return nb_tokens_by_category
@override
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
return inference_model.extra_headers or {}, {}
diff --git a/pipelex/plugins/openai/openai_img_gen_alt_worker.py b/pipelex/plugins/openai/openai_completions_img_gen_worker.py
similarity index 63%
rename from pipelex/plugins/openai/openai_img_gen_alt_worker.py
rename to pipelex/plugins/openai/openai_completions_img_gen_worker.py
index f68a66f71..9d955a1e2 100644
--- a/pipelex/plugins/openai/openai_img_gen_alt_worker.py
+++ b/pipelex/plugins/openai/openai_completions_img_gen_worker.py
@@ -1,5 +1,4 @@
-import random
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
import openai
from openai import APIConnectionError, BadRequestError, NotFoundError
@@ -10,16 +9,20 @@ from pipelex.cogt.exceptions import LLMCompletionError, LLMModelNotFoundError, S
from pipelex.cogt.image.generated_image import GeneratedImage
from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.inference.inference_constants import InferenceOutputType
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.plugins.openai.openai_completions_factory import OpenAICompletionsFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
if TYPE_CHECKING:
from openai.types.chat import ChatCompletionMessage
+ from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
-class OpenAIImgGenAlternativeWorker(ImgGenWorkerAbstract):
+class OpenAICompletionsImgGenWorker(ImgGenWorkerAbstract):
def __init__(
self,
+ openai_completions_factory: OpenAICompletionsFactory,
sdk_instance: Any,
inference_model: InferenceModelSpec,
reporting_delegate: ReportingProtocol | None = None,
@@ -31,6 +34,7 @@ class OpenAIImgGenAlternativeWorker(ImgGenWorkerAbstract):
raise SdkTypeError(msg)
self.openai_client = sdk_instance
+ self.openai_completions_factory = openai_completions_factory
@override
async def _gen_image(
@@ -39,13 +43,16 @@ class OpenAIImgGenAlternativeWorker(ImgGenWorkerAbstract):
) -> GeneratedImage:
log.debug(f"Generating image with model: {self.inference_model.tag}")
img_gen_prompt_text = img_gen_job.img_gen_prompt.positive_text
- messages = [{"role": "user", "content": img_gen_prompt_text}]
- seed = img_gen_job.job_params.seed or random.randint(0, 2**32 - 1)
+ messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": img_gen_prompt_text}]
try:
+ extra_headers, extra_body = self.openai_completions_factory.make_extras(
+ inference_model=self.inference_model, inference_job=img_gen_job, output_desc=InferenceOutputType.IMAGE
+ )
response = await self.openai_client.chat.completions.create(
model=self.inference_model.model_id,
- messages=messages, # type: ignore[arg-type]
- seed=seed,
+ messages=messages,
+ extra_headers=extra_headers,
+ extra_body=extra_body,
)
except NotFoundError as not_found_error:
# TODO: record llm config so it can be displayed here
@@ -59,12 +66,22 @@ class OpenAIImgGenAlternativeWorker(ImgGenWorkerAbstract):
raise LLMCompletionError(msg) from bad_request_error
openai_message: ChatCompletionMessage = response.choices[0].message
- response_text = openai_message.content
- if response_text is None:
- msg = f"OpenAI response message content is None: {response}\nmodel: {self.inference_model.desc}"
+ url: str | None = None
+ if (content := openai_message.content) and content.startswith("http"):
+ url = openai_message.content
+ elif hasattr(openai_message, "content_blocks"):
+ content_blocks = cast("list[dict[str, Any]]", openai_message.content_blocks) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ for part in content_blocks:
+ if part.get("type") == "image_url":
+ if image_url := part.get("image_url"):
+ if the_url := image_url.get("url"):
+ url = the_url
+ break
+ if not url:
+ msg = f"OpenAI response has no image. Model: {self.inference_model.desc}"
raise LLMCompletionError(msg)
return GeneratedImage(
- url=response_text,
+ url=url,
width=1024,
height=1024,
)
diff --git a/pipelex/plugins/openai/openai_completions_llm_worker.py b/pipelex/plugins/openai/openai_completions_llm_worker.py
index 9bc06f4f2..2a0c31b75 100644
--- a/pipelex/plugins/openai/openai_completions_llm_worker.py
+++ b/pipelex/plugins/openai/openai_completions_llm_worker.py
@@ -17,7 +17,7 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.config import get_config
from pipelex.plugins.openai.openai_completions_factory import OpenAICompletionsFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
-from pipelex.system.telemetry.otel_constants import LLMOutputType
+from pipelex.system.telemetry.otel_constants import InferenceOutputType
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
@@ -73,7 +73,7 @@ class OpenAICompletionsLLMWorker(LLMWorkerInternalAbstract):
try:
extra_headers, extra_body = self.openai_completions_factory.make_extras(
- inference_model=self.inference_model, llm_job=llm_job, output_desc=LLMOutputType.TEXT
+ inference_model=self.inference_model, inference_job=llm_job, output_desc=InferenceOutputType.TEXT
)
response = await self.openai_client_for_text.chat.completions.create(
model=self.inference_model.model_id,
@@ -118,7 +118,7 @@ class OpenAICompletionsLLMWorker(LLMWorkerInternalAbstract):
try:
try:
extra_headers, extra_body = self.openai_completions_factory.make_extras(
- inference_model=self.inference_model, llm_job=llm_job, output_desc=schema.__name__
+ inference_model=self.inference_model, inference_job=llm_job, output_desc=schema.__name__
)
result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
model=self.inference_model.model_id,
diff --git a/pipelex/plugins/openai/openai_img_gen_factory.py b/pipelex/plugins/openai/openai_img_gen_factory.py
index 00279f979..8215c22fc 100644
--- a/pipelex/plugins/openai/openai_img_gen_factory.py
+++ b/pipelex/plugins/openai/openai_img_gen_factory.py
@@ -1,11 +1,13 @@
from typing import Literal
+from openai import Omit, omit
+
from pipelex.cogt.exceptions import ImgGenParameterError
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat, Quality
GptImage1SizeType = Literal["1024x1024", "1536x1024", "1024x1536"]
GptImage1OutputFormatType = Literal["png", "jpeg", "webp"]
-GptImage1ModerationType = Literal["low", "auto"]
+GptImage1ModerationType = Literal["low", "auto"] | Omit
GptImage1QualityType = Literal["low", "medium", "high"]
GptImage1BackgroundType = Literal["transparent", "opaque", "auto"]
@@ -42,8 +44,13 @@ class OpenAIImgGenFactory:
return "webp"
@classmethod
- def moderation_for_gpt_image_1(cls, is_moderated: bool) -> GptImage1ModerationType:
- return "auto" if is_moderated else "low"
+ def moderation_for_gpt_image_1(cls, is_moderated: bool | None) -> GptImage1ModerationType:
+ if is_moderated is None:
+ return omit
+ elif is_moderated:
+ return "low"
+ else:
+ return "auto"
@classmethod
def quality_for_gpt_image_1(cls, quality: Quality) -> GptImage1QualityType:
diff --git a/pipelex/plugins/openai/openai_img_gen_worker.py b/pipelex/plugins/openai/openai_img_gen_worker.py
index bf9156846..65086d537 100644
--- a/pipelex/plugins/openai/openai_img_gen_worker.py
+++ b/pipelex/plugins/openai/openai_img_gen_worker.py
@@ -1,10 +1,8 @@
from typing import Any
import openai
-import shortuuid
from typing_extensions import override
-from pipelex import log
from pipelex.cogt.exceptions import ImgGenGenerationError, SdkTypeError
from pipelex.cogt.image.generated_image import GeneratedImage
from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
@@ -13,10 +11,7 @@ from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.plugins.openai.openai_img_gen_factory import OpenAIImgGenFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
-from pipelex.tools.misc.base_64_utils import save_base_64_str_to_binary_file
-from pipelex.tools.misc.file_utils import ensure_path
-
-TEMP_OUTPUTS_DIR = "temp/img_gen_by_gpt_image"
+from pipelex.tools.misc.base_64_utils import prefixed_base64_str_from_base64_str
class OpenAIImgGenWorker(ImgGenWorkerAbstract):
@@ -69,24 +64,19 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):
msg = "No result from OpenAI"
raise ImgGenGenerationError(msg)
- generated_image_list: list[GeneratedImage] = []
- image_id = shortuuid.uuid()[:4]
- for image_index, image_data in enumerate(result.data):
+ generated_images: list[GeneratedImage] = []
+ for image_data in result.data:
image_base64 = image_data.b64_json
if not image_base64:
msg = "No base64 image data received from OpenAI"
raise ImgGenGenerationError(msg)
- folder_path = TEMP_OUTPUTS_DIR
- ensure_path(folder_path)
- img_path = f"{folder_path}/{image_id}_{image_index}.png"
- save_base_64_str_to_binary_file(base_64_str=image_base64, file_path=img_path)
- log.verbose(f"Saved image to {img_path}")
- generated_image_list.append(
+ base64_url = prefixed_base64_str_from_base64_str(b64_str=image_base64)
+ generated_images.append(
GeneratedImage(
- url=img_path,
+ url=base64_url,
width=1024,
height=1024,
),
)
- return generated_image_list
+ return generated_images
diff --git a/pipelex/plugins/openai/openai_responses_factory.py b/pipelex/plugins/openai/openai_responses_factory.py
index ae62eb5e0..b50aebacb 100644
--- a/pipelex/plugins/openai/openai_responses_factory.py
+++ b/pipelex/plugins/openai/openai_responses_factory.py
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
ResponseUsage,
)
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -76,5 +77,7 @@ class OpenAIResponsesFactory(PluginFactoryAbstract):
return nb_tokens_by_category
@override
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
return inference_model.extra_headers or {}, {}
diff --git a/pipelex/plugins/openai/openai_responses_llm_worker.py b/pipelex/plugins/openai/openai_responses_llm_worker.py
index e56e0fa02..72445b5ef 100644
--- a/pipelex/plugins/openai/openai_responses_llm_worker.py
+++ b/pipelex/plugins/openai/openai_responses_llm_worker.py
@@ -12,7 +12,7 @@ from pipelex.cogt.exceptions import LLMCompletionError, LLMModelNotFoundError, S
from pipelex.cogt.llm.llm_utils import dump_error, dump_kwargs, dump_response_from_structured_gen
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
from pipelex.config import get_config
-from pipelex.system.telemetry.otel_constants import LLMOutputType
+from pipelex.system.telemetry.otel_constants import InferenceOutputType
if TYPE_CHECKING:
from openai.types.chat import ChatCompletionMessageParam
@@ -73,7 +73,7 @@ class OpenAIResponsesLLMWorker(LLMWorkerInternalAbstract):
input_items = await self.openai_responses_factory.make_input_items(llm_job=llm_job)
try:
extra_headers, extra_body = self.openai_responses_factory.make_extras(
- inference_model=self.inference_model, llm_job=llm_job, output_desc=LLMOutputType.TEXT
+ inference_model=self.inference_model, inference_job=llm_job, output_desc=InferenceOutputType.TEXT
)
response = await self.openai_client_for_responses.responses.create(
model=self.inference_model.model_id,
@@ -119,7 +119,7 @@ class OpenAIResponsesLLMWorker(LLMWorkerInternalAbstract):
msg = "Instructor client is not configured for the Responses API. Set a responses-capable structure_method for this model."
raise LLMCompletionError(msg)
extra_headers, extra_body = self.openai_responses_factory.make_extras(
- inference_model=self.inference_model, llm_job=llm_job, output_desc=schema.__name__
+ inference_model=self.inference_model, inference_job=llm_job, output_desc=schema.__name__
)
input_items = await self.openai_responses_factory.make_input_items(llm_job=llm_job)
result_object, completion = await self.instructor_for_objects.responses.create_with_completion( # pyright: ignore[reportUnknownMemberType]
diff --git a/pipelex/plugins/plugin_factory_abstract.py b/pipelex/plugins/plugin_factory_abstract.py
index 860fee423..c59529dba 100644
--- a/pipelex/plugins/plugin_factory_abstract.py
+++ b/pipelex/plugins/plugin_factory_abstract.py
@@ -1,11 +1,13 @@
from abc import ABC, abstractmethod
from typing import Any
-from pipelex.cogt.llm.llm_job import LLMJob
+from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
class PluginFactoryAbstract(ABC):
@abstractmethod
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
pass
diff --git a/pipelex/plugins/portkey/portkey_completions_factory.py b/pipelex/plugins/portkey/portkey_completions_factory.py
index a22cf3c72..a718fc085 100644
--- a/pipelex/plugins/portkey/portkey_completions_factory.py
+++ b/pipelex/plugins/portkey/portkey_completions_factory.py
@@ -14,7 +14,7 @@ from pipelex.plugins.portkey.portkey_exceptions import PortkeyFactoryError
from pipelex.plugins.portkey.portkey_factory import PortkeyFactory
if TYPE_CHECKING:
- from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.plugins.plugin_sdk_registry import Plugin
@@ -46,5 +46,7 @@ class PortkeyCompletionsFactory(OpenAICompletionsFactory):
)
@override
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
- return PortkeyFactory.make_extras(inference_model=inference_model, llm_job=llm_job, output_desc=output_desc)
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
+ return PortkeyFactory.make_extras(inference_model=inference_model, inference_job=inference_job, output_desc=output_desc)
diff --git a/pipelex/plugins/portkey/portkey_factory.py b/pipelex/plugins/portkey/portkey_factory.py
index 18e5c332c..6efc45a39 100644
--- a/pipelex/plugins/portkey/portkey_factory.py
+++ b/pipelex/plugins/portkey/portkey_factory.py
@@ -4,7 +4,8 @@ from typing import TYPE_CHECKING, Any
from portkey_ai import PORTKEY_GATEWAY_URL
-from pipelex.cogt.llm.llm_constants import LLMOutputType
+from pipelex.cogt.inference.inference_constants import InferenceOutputType
+from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.hub import get_telemetry_manager
from pipelex.plugins.openai.openai_constants import OpenAIBodyKey
from pipelex.plugins.portkey.portkey_constants import PortkeyHeaderKey
@@ -13,7 +14,7 @@ from pipelex.system.telemetry.otel_constants import OTelConstants
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
if TYPE_CHECKING:
- from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -36,34 +37,36 @@ class PortkeyFactory:
return backend.api_key
@classmethod
- def make_extras(cls, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
+ def make_extras(
+ cls, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
extra_headers: dict[str, str] = {}
extra_body: dict[str, Any] = {}
# Model-level extras (unchanged)
if inference_model.extra_headers:
extra_headers.update(inference_model.extra_headers)
- if not llm_job.job_params.max_tokens and inference_model.max_tokens:
+ if isinstance(inference_job, LLMJob) and not inference_job.job_params.max_tokens and inference_model.max_tokens:
extra_body[OpenAIBodyKey.MAX_TOKENS] = inference_model.max_tokens
# OTel-correlated Portkey tracing (only when enabled and OTel context available)
- if get_telemetry_manager().is_portkey_tracing_enabled() and (otel_context := llm_job.job_metadata.otel_context):
+ if get_telemetry_manager().is_portkey_tracing_enabled() and (otel_context := inference_job.job_metadata.otel_context):
# Use OTel trace_id and span_id for correlation
extra_headers[PortkeyHeaderKey.TRACE_ID] = f"{otel_context.trace_id:032x}"
extra_headers[PortkeyHeaderKey.SPAN_ID] = f"{otel_context.span_id:016x}"
# Build span name respecting privacy settings
- pipe_code = llm_job.job_metadata.pipe_code or "main"
+ pipe_code = inference_job.job_metadata.pipe_code or "main"
if not TelemetryManagerAbstract.is_capture_pipe_codes_enabled():
pipe_code = OTelConstants.PIPE_CODE_REDACTED
# Redact output class name if not "text" and capture is disabled
- if output_desc == LLMOutputType.TEXT or TelemetryManagerAbstract.is_capture_output_class_name_enabled():
+ if output_desc == InferenceOutputType.TEXT or TelemetryManagerAbstract.is_capture_output_class_name_enabled():
display_output = output_desc
else:
display_output = OTelConstants.OUTPUT_CLASS_REDACTED
- unit_job_id = llm_job.job_metadata.unit_job_id or "unknown"
+ unit_job_id = inference_job.job_metadata.unit_job_id or "unknown"
extra_headers[PortkeyHeaderKey.SPAN_NAME] = f"{pipe_code}: {unit_job_id} -> {display_output}"
return extra_headers, extra_body
diff --git a/pipelex/plugins/portkey/portkey_responses_factory.py b/pipelex/plugins/portkey/portkey_responses_factory.py
index d571729e1..d15eeed0f 100644
--- a/pipelex/plugins/portkey/portkey_responses_factory.py
+++ b/pipelex/plugins/portkey/portkey_responses_factory.py
@@ -14,7 +14,7 @@ from pipelex.plugins.portkey.portkey_exceptions import PortkeyFactoryError
from pipelex.plugins.portkey.portkey_factory import PortkeyFactory
if TYPE_CHECKING:
- from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.plugins.plugin_sdk_registry import Plugin
@@ -44,5 +44,7 @@ class PortkeyResponsesFactory(OpenAIResponsesFactory):
)
@override
- def make_extras(self, inference_model: InferenceModelSpec, llm_job: LLMJob, output_desc: str) -> tuple[dict[str, str], dict[str, Any]]:
- return PortkeyFactory.make_extras(inference_model=inference_model, llm_job=llm_job, output_desc=output_desc)
+ def make_extras(
+ self, inference_model: InferenceModelSpec, inference_job: InferenceJobAbstract, output_desc: str
+ ) -> tuple[dict[str, str], dict[str, Any]]:
+ return PortkeyFactory.make_extras(inference_model=inference_model, inference_job=inference_job, output_desc=output_desc)
diff --git a/pipelex/system/pipelex_service/pipelex_details.py b/pipelex/system/pipelex_service/pipelex_details.py
index cc54a3f10..c17d83a5f 100644
--- a/pipelex/system/pipelex_service/pipelex_details.py
+++ b/pipelex/system/pipelex_service/pipelex_details.py
@@ -2,7 +2,7 @@ from pipelex.tools.misc.hash_utils import hash_sha256
class PipelexDetails:
- REMOTE_CONFIG_URL = "https://pipelex-config.s3.amazonaws.com/pipelex_remote_config.json"
+ REMOTE_CONFIG_URL = "https://pipelex-config.s3.amazonaws.com/pipelex_remote_config_01.json"
PIPELEX_GATEWAY_API_KEY_VAR = "PIPELEX_GATEWAY_API_KEY"
@classmethod
diff --git a/pipelex/system/telemetry/otel_constants.py b/pipelex/system/telemetry/otel_constants.py
index c0df969f2..c1b887dd0 100644
--- a/pipelex/system/telemetry/otel_constants.py
+++ b/pipelex/system/telemetry/otel_constants.py
@@ -6,7 +6,7 @@ with OpenTelemetry, following the GenAI semantic conventions.
from opentelemetry.semconv._incubating.attributes import gen_ai_attributes as otel_gen_ai_attributes # noqa: PLC2701
-from pipelex.cogt.llm.llm_constants import LLMOutputType
+from pipelex.cogt.inference.inference_constants import InferenceOutputType
from pipelex.types import StrEnum
@@ -92,12 +92,14 @@ class PostHogEvent(StrEnum):
def make_otel_gen_ai_output_type(output_type: str) -> otel_gen_ai_attributes.GenAiOutputTypeValues:
try:
- llm_output_type = LLMOutputType(output_type)
+ llm_output_type = InferenceOutputType(output_type)
match llm_output_type:
- case LLMOutputType.TEXT:
+ case InferenceOutputType.TEXT:
return otel_gen_ai_attributes.GenAiOutputTypeValues.TEXT
- case LLMOutputType.OBJECT:
+ case InferenceOutputType.OBJECT:
return otel_gen_ai_attributes.GenAiOutputTypeValues.JSON
+ case InferenceOutputType.IMAGE:
+ return otel_gen_ai_attributes.GenAiOutputTypeValues.IMAGE
except ValueError as exc:
msg = f"Invalid LLM output type: {output_type}, and we only support LLM output types for now in OpenTelemetry"
raise ValueError(msg) from exc
diff --git a/pipelex/tools/jinja2/jinja2_environment.py b/pipelex/tools/jinja2/jinja2_environment.py
index 391450d08..51bbadf5a 100644
--- a/pipelex/tools/jinja2/jinja2_environment.py
+++ b/pipelex/tools/jinja2/jinja2_environment.py
@@ -35,6 +35,10 @@ def make_jinja2_env_from_loader(
autoescape = False
trim_blocks = False
lstrip_blocks = False
+ case TemplateCategory.IMG_GEN_PROMPT:
+ autoescape = False
+ trim_blocks = False
+ lstrip_blocks = False
return Environment(
loader=loader,
diff --git a/pipelex/tools/misc/base_64_utils.py b/pipelex/tools/misc/base_64_utils.py
index c6f28e38d..eac40529c 100644
--- a/pipelex/tools/misc/base_64_utils.py
+++ b/pipelex/tools/misc/base_64_utils.py
@@ -75,6 +75,14 @@ def strip_base_64_str_if_needed(base64_str: str) -> str:
return base64_str
+def extract_base_64_str_from_base64_url_if_possible(possibly_base64_url: str) -> str | None:
+ if not possibly_base64_url.startswith("data:"):
+ return None
+ if ";base64," not in possibly_base64_url:
+ return None
+ return possibly_base64_url.split(";base64,", 1)[1]
+
+
def prefixed_base64_str_from_base64_bytes(b64_bytes: bytes) -> str:
file_type = detect_file_type_from_base64(b64_bytes)
return f"data:{file_type.mime};base64,{base64.b64encode(b64_bytes).decode('utf-8')}"
diff --git a/pipelex/tools/misc/filetype_utils.py b/pipelex/tools/misc/filetype_utils.py
index 717cd7d83..4a22e3d51 100644
--- a/pipelex/tools/misc/filetype_utils.py
+++ b/pipelex/tools/misc/filetype_utils.py
@@ -28,7 +28,7 @@ def detect_file_type_from_path(path: str | Path) -> FileType:
A FileType object containing the file extension and MIME type of the file.
Raises:
- FileTypeException: If the file type cannot be identified.
+ FileTypeError: If the file type cannot be identified.
"""
kind = filetype.guess(path) # pyright: ignore[reportUnknownMemberType]
@@ -50,7 +50,7 @@ def detect_file_type_from_bytes(buf: bytes) -> FileType:
A FileType object containing the file extension and MIME type of the file.
Raises:
- FileTypeException: If the file type cannot be identified.
+ FileTypeError: If the file type cannot be identified.
"""
kind = filetype.guess(buf) # pyright: ignore[reportUnknownMemberType]
@@ -72,7 +72,7 @@ def detect_file_type_from_base64(b64: str | bytes) -> FileType:
A FileType object containing the file extension and MIME type of the file.
Raises:
- FileTypeException: If the file type cannot be identified.
+ FileTypeError: If the file type cannot be identified.
"""
# Normalise to bytes holding only the Base-64 alphabet
diff --git a/pipelex/tools/misc/tenacity_utils.py b/pipelex/tools/misc/tenacity_utils.py
new file mode 100644
index 000000000..06f7bc38c
--- /dev/null
+++ b/pipelex/tools/misc/tenacity_utils.py
@@ -0,0 +1,15 @@
+from tenacity import RetryCallState
+
+from pipelex import log
+
+
+def log_retry(retry_state: RetryCallState) -> None:
+ """Called before sleeping between retries."""
+ if not retry_state.outcome:
+ log.error("Tenacity retry state outcome is None")
+ return
+ exc = retry_state.outcome.exception()
+ attempt = retry_state.attempt_number
+ wait_duration = retry_state.next_action.sleep if retry_state.next_action else 0.0
+ log.verbose(f"Tenacity retry #{attempt} due to '{type(exc).__name__}'.")
+ log.verbose(f"Wait duration before next attempt: {wait_duration:.4f}s")
diff --git a/tests/helpers/image_helpers.py b/tests/helpers/image_helpers.py
new file mode 100644
index 000000000..7b5c8e0db
--- /dev/null
+++ b/tests/helpers/image_helpers.py
@@ -0,0 +1,72 @@
+"""Image helpers for tests."""
+
+import base64
+import tempfile
+from io import BytesIO
+from pathlib import Path
+
+import shortuuid
+from PIL import Image
+
+from pipelex import log
+from pipelex.cogt.image.generated_image import GeneratedImage
+from pipelex.tools.misc.file_utils import ensure_path
+from pipelex.tools.misc.terminal_utils import print_to_stderr
+
+
+def save_generated_image(generated_image: GeneratedImage, topic: str, output_dir: str | None = None) -> str | None:
+ """Save a generated image to the output directory.
+
+ Handles different URL formats:
+
+ - HTTP URLs: Returns None (not saved locally)
+ - Data URI format: data:image/{format};base64,{base64_data}
+ - File path: Opens existing file
+ - Raw base64: Decodes and opens
+
+ Args:
+ generated_image: The generated image with URL
+ topic: Topic name used for the filename
+ output_dir: Directory to save the image to, defaults to system temp directory
+
+ Returns:
+ The path where the image was saved, or None for HTTP URLs
+ """
+ url = generated_image.url
+
+ # HTTP URLs are not saved locally
+ if url.startswith("http"):
+ return None
+
+ # Decode image from various formats
+ if url.startswith("data:image/"):
+ # Data URI format: data:image/{format};base64,{base64_data}
+ base64_data = url.split(",", 1)[1]
+ image = Image.open(BytesIO(base64.b64decode(base64_data)))
+ elif Path(url).exists():
+ # File path format
+ image = Image.open(url)
+ else:
+ # Assume raw base64 (fallback)
+ image = Image.open(BytesIO(base64.b64decode(url)))
+
+ # Resolve output directory
+ resolved_output_dir = output_dir or tempfile.gettempdir()
+
+ # Save to directory with unique filename
+ ensure_path(resolved_output_dir)
+ image_id = shortuuid.uuid()[:8]
+ safe_topic = topic.replace(" ", "_")
+ image_name = f"{resolved_output_dir}/{safe_topic}_{image_id}"
+
+ if image.format:
+ extension = f".{image.format.lower()}"
+ else:
+ log.warning(f"Image format not found for image '{image_name}'")
+ extension = ""
+
+ output_path = f"{image_name}{extension}"
+ image.save(output_path)
+ print_to_stderr(f"✓ Image saved to: {output_path}\n")
+
+ return output_path
diff --git a/tests/integration/pipelex/cogt/test_extract.py b/tests/integration/pipelex/cogt/test_extract.py
index 547370825..a0096b273 100644
--- a/tests/integration/pipelex/cogt/test_extract.py
+++ b/tests/integration/pipelex/cogt/test_extract.py
@@ -17,8 +17,8 @@ from tests.cases import ImageTestCases, PDFTestCases
@pytest.mark.filterwarnings("ignore:Accessing the 'model_fields' attribute on the instance is deprecated:DeprecationWarning")
class TestExtract:
@pytest.mark.parametrize("file_path", PDFTestCases.DOCUMENT_FILE_PATHS)
- async def test_extract_pdf_path(self, extract_handle: str, file_path: str):
- extract_worker = get_extract_worker(extract_handle=extract_handle)
+ async def test_extract_pdf_path(self, extract_handle_from_pdf: str, file_path: str):
+ extract_worker = get_extract_worker(extract_handle=extract_handle_from_pdf)
if not extract_worker.is_pdf_supported:
msg = f"PDF extraction is not supported for this extract worker: '{extract_worker.desc}'"
pytest.skip(msg)
@@ -31,8 +31,8 @@ class TestExtract:
pretty_print(page.text, title=f"Page {page_index}")
@pytest.mark.parametrize("url", PDFTestCases.DOCUMENT_URLS)
- async def test_extract_pdf_url(self, extract_handle: str, url: str):
- extract_worker = get_extract_worker(extract_handle=extract_handle)
+ async def test_extract_pdf_url(self, extract_handle_from_pdf: str, url: str):
+ extract_worker = get_extract_worker(extract_handle=extract_handle_from_pdf)
if not extract_worker.is_pdf_supported:
msg = f"PDF extraction is not supported for this extract worker: '{extract_worker.desc}'"
pytest.skip(msg)
diff --git a/tests/integration/pipelex/cogt/test_img_gen.py b/tests/integration/pipelex/cogt/test_img_gen.py
index 930beab52..d8d89c5ce 100644
--- a/tests/integration/pipelex/cogt/test_img_gen.py
+++ b/tests/integration/pipelex/cogt/test_img_gen.py
@@ -1,21 +1,12 @@
-import base64
-from io import BytesIO
-from pathlib import Path
-
import pytest
-import shortuuid
-from PIL import Image
-from pipelex import log, pretty_print
+from pipelex import pretty_print
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, ImgGenJobParams, OutputFormat
from pipelex.cogt.img_gen.img_gen_job_factory import ImgGenJobFactory
from pipelex.hub import get_img_gen_worker
-from pipelex.tools.misc.file_utils import ensure_path
-from pipelex.tools.misc.terminal_utils import print_to_stderr
+from tests.helpers.image_helpers import save_generated_image
from tests.integration.pipelex.test_data import ImageGenTestCases
-TEMP_OUTPUTS_DIR = "temp/img_gen_by_gpt_image"
-
@pytest.mark.img_gen
@pytest.mark.inference
@@ -23,51 +14,27 @@ TEMP_OUTPUTS_DIR = "temp/img_gen_by_gpt_image"
@pytest.mark.usefixtures("routing_profile_override")
class TestImageGeneration:
@pytest.mark.parametrize(("topic", "img_gen_prompt_text"), ImageGenTestCases.IMAGE_DESC)
- async def test_img_gen_opaque(self, img_gen_handle: str, topic: str, img_gen_prompt_text: str):
+ async def test_img_gen_opaque(self, img_gen_handle: str, img_gen_job_params: ImgGenJobParams, topic: str, img_gen_prompt_text: str):
img_gen_worker_async = get_img_gen_worker(img_gen_handle=img_gen_handle)
img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt_contents(
positive_text=img_gen_prompt_text,
+ img_gen_job_params=img_gen_job_params,
)
generated_image = await img_gen_worker_async.gen_image(
img_gen_job=img_gen_job,
)
- pretty_print(generated_image, title=f"Image generated by '{img_gen_handle}' for topic '{topic}'")
+ pretty_print(generated_image, title=f"Image generation by '{img_gen_handle}' for topic '{topic}'")
- # Handle different URL formats
- if generated_image.url.startswith("http"):
+ saved_path = save_generated_image(generated_image, topic)
+ if saved_path is None:
pretty_print(generated_image.url, title="Image URL")
- else:
- if generated_image.url.startswith("data:image/"):
- # Data URI format: data:image/{format};base64,{base64_data}
- base64_data = generated_image.url.split(",", 1)[1]
- image = Image.open(BytesIO(base64.b64decode(base64_data)))
- elif Path(generated_image.url).exists():
- # File path format
- image = Image.open(generated_image.url)
- else:
- # Assume raw base64 (fallback)
- image = Image.open(BytesIO(base64.b64decode(generated_image.url)))
-
- # Save to temp directory with unique filename
- ensure_path(TEMP_OUTPUTS_DIR)
- image_id = shortuuid.uuid()[:8]
- safe_topic = topic.replace(" ", "_")
- image_name = f"{TEMP_OUTPUTS_DIR}/{safe_topic}_{image_id}"
- if image.format:
- extension = f".{image.format.lower()}"
- else:
- log.warning(f"Image format not found for image '{image_name}'")
- extension = ""
- output_path = f"{image_name}{extension}"
- image.save(output_path)
- print_to_stderr(f"✓ Image saved to: {output_path}\n")
@pytest.mark.parametrize(("topic", "img_gen_prompt_text"), ImageGenTestCases.IMAGE_DESC)
async def test_img_gen_transparent(self, img_gen_handle: str, topic: str, img_gen_prompt_text: str):
img_gen_worker_async = get_img_gen_worker(img_gen_handle=img_gen_handle)
img_gen_job_params = ImgGenJobParams(
aspect_ratio=AspectRatio.SQUARE,
- is_raw=False,
+ is_raw=None,
background=Background.TRANSPARENT,
output_format=OutputFormat.PNG,
)
@@ -80,37 +47,14 @@ class TestImageGeneration:
)
pretty_print(generated_image, title=f"Image generated by '{img_gen_handle}' for topic '{topic}'")
- # Handle different URL formats
- if generated_image.url.startswith("data:image/"):
- # Data URI format: data:image/{format};base64,{base64_data}
- base64_data = generated_image.url.split(",", 1)[1]
- image = Image.open(BytesIO(base64.b64decode(base64_data)))
- elif Path(generated_image.url).exists():
- # File path format
- image = Image.open(generated_image.url)
- else:
- # Assume raw base64 (fallback)
- image = Image.open(BytesIO(base64.b64decode(generated_image.url)))
-
- # Save to temp directory with unique filename
- ensure_path(TEMP_OUTPUTS_DIR)
- image_id = shortuuid.uuid()[:8]
- safe_topic = topic.replace(" ", "_")
- image_name = f"{TEMP_OUTPUTS_DIR}/{safe_topic}_{image_id}"
- if image.format:
- extension = f".{image.format.lower()}"
- else:
- log.warning(f"Image format not found for image '{image_name}'")
- extension = ""
- output_path = f"{image_name}{extension}"
- image.save(output_path)
- print_to_stderr(f"✓ Image saved to: {output_path}\n")
+ save_generated_image(generated_image, topic)
@pytest.mark.parametrize(("topic", "img_gen_prompt_text"), ImageGenTestCases.IMAGE_DESC)
- async def test_img_gen_multiple(self, img_gen_handle: str, topic: str, img_gen_prompt_text: str):
+ async def test_img_gen_multiple(self, img_gen_handle: str, img_gen_job_params: ImgGenJobParams, topic: str, img_gen_prompt_text: str):
img_gen_worker_async = get_img_gen_worker(img_gen_handle=img_gen_handle)
img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt_contents(
positive_text=img_gen_prompt_text,
+ img_gen_job_params=img_gen_job_params,
)
generated_image_list = await img_gen_worker_async.gen_image_list(
img_gen_job=img_gen_job,
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index 849e651fc..d659024f9 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -15,7 +15,7 @@ from .fixtures.extract_fixtures import (
extract_handle_from_image,
extract_handle_from_pdf,
)
-from .fixtures.img_gen_fixtures import img_gen_handle
+from .fixtures.img_gen_fixtures import img_gen_handle, img_gen_job_params
from .fixtures.llm_fixtures import llm_handle, llm_job_params, llm_preset_id
from .fixtures.plugin_fixtures import plugin_for_anthropic, plugin_for_openai
from .fixtures.routing_fixtures import (
@@ -40,6 +40,7 @@ __all__ = [
"plugin_for_anthropic",
# Image generation fixtures
"img_gen_handle",
+ "img_gen_job_params",
# Extract fixtures
"extract_handle",
"extract_handle_from_image",
diff --git a/tests/integration/pipelex/fixtures/__init__.py b/tests/integration/pipelex/fixtures/__init__.py
index 92c57a2bf..e69de29bb 100644
--- a/tests/integration/pipelex/fixtures/__init__.py
+++ b/tests/integration/pipelex/fixtures/__init__.py
@@ -1 +0,0 @@
-"""Test fixtures for integration tests."""
diff --git a/tests/integration/pipelex/fixtures/extract_fixtures.py b/tests/integration/pipelex/fixtures/extract_fixtures.py
index b3d19efdb..f2a42ddc0 100644
--- a/tests/integration/pipelex/fixtures/extract_fixtures.py
+++ b/tests/integration/pipelex/fixtures/extract_fixtures.py
@@ -26,6 +26,7 @@ EXTRACT_HANDLE_FROM_PDF = [
"pypdfium2-extract-text",
"docling-extract-text",
"mistral-ocr",
+ "azure-document-intelligence",
]
EXTRACT_HANDLE_FROM_IMAGE = [
diff --git a/tests/integration/pipelex/fixtures/img_gen_fixtures.py b/tests/integration/pipelex/fixtures/img_gen_fixtures.py
index 3d36666ea..fee5d807c 100644
--- a/tests/integration/pipelex/fixtures/img_gen_fixtures.py
+++ b/tests/integration/pipelex/fixtures/img_gen_fixtures.py
@@ -2,6 +2,7 @@
import pytest
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, ImgGenJobParams, OutputFormat, Quality
from pipelex.hub import get_model_deck
from tests.integration.pipelex.fixtures.routing_fixtures import ALL_BACKENDS, check_backend_supports_model
@@ -32,6 +33,7 @@ FAL_IMG_GEN_MODELS = [
"flux-pro",
"flux-pro/v1.1",
"flux-pro/v1.1-ultra",
+ "flux-2",
]
# --- OpenAI Models ------------------------------------------------------------------------------
@@ -42,6 +44,7 @@ OPENAI_IMG_GEN_MODELS = [
# --- Google Models --------------------------------------------------------------------------
GOOGLE_IMG_GEN_MODELS = [
"nano-banana",
+ "nano-banana-pro",
]
# --- All Image Generation Handles ---------------------------------------------------------------
@@ -64,3 +67,22 @@ def img_gen_handle(request: pytest.FixtureRequest) -> str:
if not is_img_gen_handle_supported_by_enabled_backends(img_gen_handle_param):
pytest.skip(f"Image generation handle '{img_gen_handle_param}' not supported by any enabled backend")
return img_gen_handle_param
+
+
+ params=[
+ ImgGenJobParams(
+ aspect_ratio=AspectRatio.SQUARE,
+ background=Background.OPAQUE,
+ quality=Quality.LOW,
+ guidance_scale=3.5,
+ is_moderated=None,
+ safety_tolerance=1,
+ is_raw=None,
+ output_format=OutputFormat.JPG,
+ )
+ ],
+)
+def img_gen_job_params(request: pytest.FixtureRequest) -> ImgGenJobParams:
+ assert isinstance(request.param, ImgGenJobParams)
+ return request.param
diff --git a/tests/integration/pipelex/fixtures/routing_fixtures.py b/tests/integration/pipelex/fixtures/routing_fixtures.py
index e2b8777a9..07ad8af9b 100644
--- a/tests/integration/pipelex/fixtures/routing_fixtures.py
+++ b/tests/integration/pipelex/fixtures/routing_fixtures.py
@@ -2,6 +2,7 @@
import shutil
import tempfile
+from functools import cache
from pathlib import Path
import pytest
@@ -9,6 +10,8 @@ from pytest import MonkeyPatch
from pipelex.hub import get_console
from pipelex.system.configuration.configs import ConfigPaths
+from pipelex.system.pipelex_service.exceptions import RemoteConfigFetchError, RemoteConfigValidationError
+from pipelex.system.pipelex_service.remote_config_fetcher import RemoteConfigFetcher
from pipelex.tools.misc.toml_utils import load_toml_from_path, load_toml_with_tomlkit, save_toml_to_path
# ================================================================================================
@@ -62,8 +65,27 @@ def extract_backend_from_profile_name_if_possible(profile_name: str) -> str | No
return profile_name[4:] # Remove 'all_' prefix
+@cache
+def _get_pipelex_gateway_backend_model_specs() -> dict[str, object]:
+ """Fetch Pipelex Gateway model specs from the same remote config Pipelex uses.
+
+ Cached to avoid repeated network calls during test collection/parametrization.
+ """
+ remote_config = RemoteConfigFetcher.fetch_remote_config()
+ # BackendModelSpecs is a dict-like "model_handle -> spec dict" mapping.
+ return dict(remote_config.backend_model_specs)
+
+
def check_backend_supports_model(backend_name: str, model_handle: str) -> bool:
"""Check if a backend TOML file defines a specific model (statically, without initializing Pipelex)."""
+ if backend_name == "pipelex_gateway":
+ try:
+ gateway_specs = _get_pipelex_gateway_backend_model_specs()
+ except (RemoteConfigFetchError, RemoteConfigValidationError):
+ # If remote config cannot be fetched, behave as "unsupported" so tests skip cleanly.
+ return False
+ return model_handle in gateway_specs
+
backend_file = Path(ConfigPaths.BACKENDS_DIR_PATH) / f"{backend_name}.toml"
if not backend_file.exists():
return False
diff --git a/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_run.py b/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_run.py
index ca6fa93a2..e01f44fca 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_run.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_run.py
@@ -1,12 +1,13 @@
-from typing import Callable, cast
+from typing import Callable
import pytest
-from pipelex import pretty_print
from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.pipe_factory import PipeFactory
+from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router
-from pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGen, PipeImgGenOutput
+from pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGen
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
from pipelex.pipe_run.pipe_job_factory import PipeJobFactory
from pipelex.pipe_run.pipe_run_params import PipeRunMode
@@ -20,19 +21,20 @@ from tests.integration.pipelex.test_data import ImageGenTestCases
@pytest.mark.asyncio(loop_scope="class")
class TestPipeImgGenRun:
@pytest.mark.parametrize(("topic", "image_desc"), ImageGenTestCases.IMAGE_DESC)
- async def test_pipe_img_gen_run(
+ async def test_pipe_img_gen_run_no_inputs(
self,
pipe_run_mode: PipeRunMode,
img_gen_handle: str,
- topic: str,
+ topic: str, # noqa: ARG002
image_desc: str,
load_empty_library: Callable[[], None],
):
load_empty_library()
pipe_img_gen_blueprint = PipeImgGenBlueprint(
description="Image generation test",
- img_gen_prompt=image_desc,
+ model=img_gen_handle,
output=NativeConceptCode.IMAGE,
+ prompt=image_desc,
)
pipe_job = PipeJobFactory.make_pipe_job(
@@ -43,11 +45,39 @@ class TestPipeImgGenRun:
),
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
)
- pipe_img_gen_output = cast(
- "PipeImgGenOutput",
- await get_pipe_router().run(
- pipe_job=pipe_job,
+ await get_pipe_router().run(
+ pipe_job=pipe_job,
+ )
+
+ @pytest.mark.parametrize(("topic", "image_desc"), ImageGenTestCases.IMAGE_DESC)
+ async def test_pipe_img_gen_run_input_to_template(
+ self,
+ pipe_run_mode: PipeRunMode,
+ img_gen_handle: str,
+ topic: str, # noqa: ARG002
+ image_desc: str,
+ load_empty_library: Callable[[], None],
+ ):
+ load_empty_library()
+ pipe_img_gen_blueprint = PipeImgGenBlueprint(
+ description="Image generation test",
+ model=img_gen_handle,
+ inputs={"image_desc": "Text"},
+ output=NativeConceptCode.IMAGE,
+ prompt="Sketch black and white funny illustration of: $image_desc",
+ )
+
+ pipe_job = PipeJobFactory.make_pipe_job(
+ pipe=PipeFactory[PipeImgGen].make_from_blueprint(
+ domain_code="generic",
+ pipe_code="adhoc_for_test_pipe_img_gen",
+ blueprint=pipe_img_gen_blueprint,
+ ),
+ working_memory=WorkingMemoryFactory.make_from_single_stuff(
+ stuff=StuffFactory.make_from_str(str_value=image_desc, name="image_desc"),
),
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
+ )
+ await get_pipe_router().run(
+ pipe_job=pipe_job,
)
- image_urls = pipe_img_gen_output.image_urls[0]
- pretty_print(image_urls, title=f"Image generated by '{img_gen_handle}' for topic '{topic}'")
diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/pipe_llm_vision.plx b/tests/integration/pipelex/pipes/operator/pipe_llm/pipe_llm_vision.plx
deleted file mode 100644
index aa5cb6e83..000000000
--- a/tests/integration/pipelex/pipes/operator/pipe_llm/pipe_llm_vision.plx
+++ /dev/null
@@ -1,58 +0,0 @@
-domain = "pipe_llm_vision"
-description = "Test PipeLLM with vision capabilities"
-
-[concept]
-VisionAnalysis = "Some analysis based on the image"
-BasicDescription = "Basic description of the image"
-
-[concept.Photo]
-description = "A photo"
-refines = "Image"
-
-[pipe.describe_image]
-type = "PipeLLM"
-description = "Describe what is in the image"
-inputs = { image = "Image" }
-output = "Text"
-model = "llm_for_testing_vision"
-prompt = """
-Describe what you see in this image in 1-2 sentences, be concise.
-$image
-"""
-
-[pipe.describe_image_number_1_only]
-type = "PipeLLM"
-description = "Describe what is in the image"
-inputs = { image_a = "Image", image_b = "Image" }
-output = "BasicDescription"
-model = "llm_for_diagram_to_text"
-prompt = """
-Describe what you see in $image_a only.
-Completely ignore $image_b.
-"""
-
-[pipe.describe_image_number_2_only]
-type = "PipeLLM"
-description = "Describe what is in the image"
-inputs = { image_a = "Image", image_b = "Image" }
-output = "BasicDescription"
-model = "llm_for_diagram_to_text"
-prompt = """
-Describe what you see in $image_b only.
-Completely ignore $image_a.
-"""
-
-
-[pipe.vision_analysis]
-type = "PipeLLM"
-description = "Provide detailed analysis of the image"
-inputs = { image = "Photo" }
-output = "VisionAnalysis"
-model = "llm_for_diagram_to_text"
-system_prompt = "You are an expert image analyst. Provide detailed, accurate descriptions."
-prompt = """
-Analyze this image and describe what's the main topic etc.
-$image
---------------------------------
-"""
-
diff --git a/tests/integration/pipelex/pipes/pipelines/test_image_out_in.plx b/tests/integration/pipelex/pipes/pipelines/test_image_out_in.plx
new file mode 100644
index 000000000..d7e323304
--- /dev/null
+++ b/tests/integration/pipelex/pipes/pipelines/test_image_out_in.plx
@@ -0,0 +1,31 @@
+domain = "test_image_in_out"
+description = "Test domain for verifying image output / input support"
+
+
+[pipe]
+[pipe.generate_image]
+type = "PipeImgGen"
+description = "Generate an image from a prompt"
+output = "Image"
+prompt = "A beautiful landscape"
+model = "img_gen_for_testing"
+
+[pipe.describe_image]
+type = "PipeLLM"
+description = "Describe what is in the image"
+inputs = { image = "Image" }
+output = "Text"
+model = "llm_for_testing_vision"
+prompt = """
+Describe what you see in this image in 1-2 sentences, be concise.
+$image
+"""
+
+[pipe.image_out_in]
+type = "PipeSequence"
+description = "Generate an image from a prompt and describe it"
+output = "Text"
+steps = [
+ { pipe = "generate_image", result = "image" },
+ { pipe = "describe_image", result = "visual_description" },
+]
\ No newline at end of file
diff --git a/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py b/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py
index 86df66086..70180f5ba 100644
--- a/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py
+++ b/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py
@@ -99,6 +99,7 @@ class TestBracketNotationInOperators:
description="Generate 3 images",
inputs={"prompt": NativeConceptCode.TEXT},
output=f"{NativeConceptCode.IMAGE}[3]",
+ prompt="@prompt",
)
pipe = PipeFactory[PipeImgGen].make_from_blueprint(
diff --git a/tests/integration/pipelex/pipes/test_image_out_in.py b/tests/integration/pipelex/pipes/test_image_out_in.py
new file mode 100644
index 000000000..25eab3805
--- /dev/null
+++ b/tests/integration/pipelex/pipes/test_image_out_in.py
@@ -0,0 +1,30 @@
+from pathlib import Path
+from typing import Callable
+
+import pytest
+
+from pipelex.hub import get_pipe_router, get_required_pipe
+from pipelex.pipe_run.pipe_job_factory import PipeJobFactory
+from pipelex.pipe_run.pipe_run_params import PipeRunMode
+from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.pipeline.job_metadata import JobMetadata
+
+
[email protected](loop_scope="class")
+class TestImageOutIn:
+ async def test_image_out_in(self, pipe_run_mode: PipeRunMode, load_test_library: Callable[[list[Path]], None]) -> None:
+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])
+ pipe_output = await get_pipe_router().run(
+ pipe_job=PipeJobFactory.make_pipe_job(
+ pipe=get_required_pipe(pipe_code="image_out_in"),
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
+ job_metadata=JobMetadata(),
+ ),
+ )
+
+ if pipe_run_mode != PipeRunMode.DRY:
+ description = pipe_output.main_stuff_as_str
+ assert description
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index 2389d8c6b..422b123a5 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -163,17 +163,10 @@ class PipeExtractTestCases:
class ImageGenTestCases:
- IMG_GEN_PROMPT_1 = "woman wearing marino cargo pants"
- IMG_GEN_PROMPT_2 = "wide legged denim pants with hippy addition"
- IMG_GEN_PROMPT_3 = """
-Woman typing on a laptop. On the laptop screen you see python code to generate code to write a prompt for an AI model.
-"""
-
IMAGE_DESC: ClassVar[list[tuple[str, str]]] = [ # topic, img_gen_prompt_text
- # (IMG_GEN_PROMPT_1, IMG_GEN_PROMPT_1),
- # (IMG_GEN_PROMPT_2, IMG_GEN_PROMPT_2),
- # (IMG_GEN_PROMPT_3, IMG_GEN_PROMPT_3),
- ("coding girl with dragon tatoo", "a girl with a dragon tatoo, coding in python"),
+ ("dog wearing sunglasses", "a dog wearing sunglasses"),
+ # ("otter playing poker", "an otter playing poker"),
+ # ("coding woman with dragon tatoo", "a woman with a dragon tatoo, wearing a tank top, coding in python"),
]
diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py
index 6e670b907..5b1674f25 100644
--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py
+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py
@@ -13,12 +13,13 @@ class PipeImgGenTestCases:
inputs={"my_prompt": "Text"},
output="native.Image",
img_gen_skill=ImgGenSkill.GEN_IMAGE_BASIC,
+ prompt="@my_prompt",
),
PipeImgGenBlueprint(
description="Generate an image",
inputs={"my_prompt": "Text"},
output="native.Image",
- img_gen_prompt_var_name="my_prompt",
+ prompt="@my_prompt",
model=ImgGenSkill.GEN_IMAGE_BASIC,
aspect_ratio=None,
background=None,
@@ -35,20 +36,20 @@ class PipeImgGenTestCases:
description="Generate image with options",
inputs={"description": "Text"},
output="Image[3]",
+ prompt="@description",
img_gen_skill=ImgGenSkill.GEN_IMAGE_FAST,
),
PipeImgGenBlueprint(
description="Generate image with options",
inputs={"description": "Text"},
output="Image[3]",
- img_gen_prompt=None,
+ prompt="@description",
model=ImgGenSkill.GEN_IMAGE_FAST,
aspect_ratio=None,
background=None,
output_format=None,
is_raw=None,
seed=None,
- img_gen_prompt_var_name="description",
),
)
diff --git a/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py b/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py
index e217a41f1..69da65b0c 100644
--- a/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py
+++ b/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py
@@ -191,6 +191,7 @@ class PipeSorterTestCases:
description="Generates the opposite image using AI",
inputs={"generation_prompt": "ImagePrompt"},
output="Image",
+ prompt="@generation_prompt",
),
"photo_opposite_pipeline": PipeSequenceBlueprint(
description="Main pipeline that generates the opposite of an input photo",
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
index 49486e1e4..d8e0d30e7 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
@@ -11,7 +11,7 @@ description = "Domain with image generation pipe"
type = "PipeImgGen"
description = "Generate an image from a prompt"
output = "Image"
-img_gen_prompt = "A beautiful landscape"
+prompt = "A beautiful landscape"
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -21,7 +21,7 @@ img_gen_prompt = "A beautiful landscape"
type="PipeImgGen",
description="Generate an image from a prompt",
output=NativeConceptCode.IMAGE,
- img_gen_prompt="A beautiful landscape",
+ prompt="A beautiful landscape",
),
},
),
diff --git a/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py b/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py
index 6b558bb2c..207806bf4 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py
@@ -1,7 +1,6 @@
from typing import ClassVar
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat
-from pipelex.core.pipes.exceptions import PipeValidationErrorType
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
@@ -15,6 +14,7 @@ class PipeImgGenInputTestCases:
description="VALID_TEXT_INPUT: Test case: valid_text_input",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
),
)
@@ -24,7 +24,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_INLINE_PROMPT: Test case: valid_with_inline_prompt",
inputs={},
output="native.Image",
- img_gen_prompt="A beautiful sunset over the ocean",
+ prompt="A beautiful sunset over the ocean",
),
)
@@ -34,6 +34,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_ASPECT_RATIO: Test case: valid_with_aspect_ratio",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
aspect_ratio=AspectRatio.LANDSCAPE_16_9,
),
)
@@ -44,6 +45,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_NB_OUTPUT: Test case: valid_with_nb_output",
inputs={"prompt": "native.Text"},
output="native.Image[3]",
+ prompt="@prompt",
),
)
@@ -53,6 +55,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_SEED: Test case: valid_with_seed",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
seed=42,
),
)
@@ -63,6 +66,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_SEED_AUTO: Test case: valid_with_seed_auto",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
seed="auto",
),
)
@@ -73,6 +77,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_BACKGROUND: Test case: valid_with_background",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
background=Background.TRANSPARENT,
),
)
@@ -83,6 +88,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_OUTPUT_FORMAT: Test case: valid_with_output_format",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
output_format=OutputFormat.PNG,
),
)
@@ -93,6 +99,7 @@ class PipeImgGenInputTestCases:
description="VALID_WITH_IS_RAW: Test case: valid_with_is_raw",
inputs={"prompt": "native.Text"},
output="native.Image",
+ prompt="@prompt",
is_raw=True,
),
)
@@ -108,18 +115,3 @@ class PipeImgGenInputTestCases:
VALID_WITH_OUTPUT_FORMAT,
VALID_WITH_IS_RAW,
]
-
- # Error test cases: (test_id, blueprint, expected_error_type)
- ERROR_WRONG_INPUT_TYPE: ClassVar[tuple[str, PipeImgGenBlueprint, PipeValidationErrorType]] = (
- "wrong_input_type",
- PipeImgGenBlueprint(
- description="ERROR_WRONG_INPUT_TYPE: Test case: wrong_input_type",
- inputs={"image": "native.Image"},
- output="native.Image",
- ),
- PipeValidationErrorType.IMG_GEN_INPUT_NOT_TEXT_COMPATIBLE,
- )
-
- ERROR_CASES: ClassVar[list[tuple[str, PipeImgGenBlueprint, PipeValidationErrorType]]] = [
- ERROR_WRONG_INPUT_TYPE,
- ]
diff --git a/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_blueprint.py b/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_blueprint.py
index 1ed75375f..2aba0e78c 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_blueprint.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_blueprint.py
@@ -8,38 +8,25 @@ class TestPipeImgGenBlueprint:
def test_validate_inputs_correct(self):
blueprint = PipeImgGenBlueprint(
inputs=None,
- img_gen_prompt="A beautiful sunset",
output="Image",
+ prompt="A beautiful sunset",
)
assert blueprint.inputs is None
- assert blueprint.img_gen_prompt == "A beautiful sunset"
+ assert blueprint.prompt == "A beautiful sunset"
blueprint = PipeImgGenBlueprint(
- inputs={"prompt": "ImgGenPrompt"},
+ inputs={"topic": "Text"},
output="Image",
+ prompt="Sketch black and white illustration of: $topic",
)
assert blueprint.nb_inputs == 1
- assert blueprint.input_names == ["prompt"]
+ assert blueprint.input_names == ["topic"]
def test_validate_inputs_incorrect(self):
with pytest.raises(ValidationError) as exc_info:
PipeImgGenBlueprint(
- inputs={},
+ inputs={"topic": "Text"},
output="Image",
+ prompt="Sketch black and white illustration of: $bingo",
)
- assert "If no inputs are provided, you must provide an 'img_gen_prompt' as attribute" in str(exc_info.value)
-
- with pytest.raises(ValidationError) as exc_info:
- PipeImgGenBlueprint(
- inputs={"prompt": "ImgGenPrompt"},
- img_gen_prompt="A beautiful sunset",
- output="Image",
- )
- assert "You must provide either an 'img_gen_prompt' as attribute or as a single text input, but not both" in str(exc_info.value)
-
- with pytest.raises(ValidationError) as exc_info:
- PipeImgGenBlueprint(
- inputs={"prompt1": "ImgGenPrompt", "prompt2": "ImgGenPrompt"},
- output="Image",
- )
- assert "Too many inputs provided for PipeImgGen" in str(exc_info.value)
+ assert "Missing input variable(s) in prompt template: bingo" in str(exc_info.value)
diff --git a/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py b/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py
index d90008098..2b3f979ea 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py
@@ -3,7 +3,6 @@ from typing import Callable
import pytest
from pipelex import log
-from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType
from pipelex.core.pipes.pipe_factory import PipeFactory
from pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGen
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
@@ -33,28 +32,3 @@ class TestPipeImgGenValidateInputs:
# Assert that the pipe was created successfully
assert pipe_img_gen is not None
assert pipe_img_gen.code == f"test_pipe_{test_id}"
-
- @pytest.mark.parametrize(
- ("test_id", "blueprint", "expected_error_type"),
- PipeImgGenInputTestCases.ERROR_CASES,
- )
- def test_validate_inputs_error_cases(
- self,
- test_id: str,
- blueprint: PipeImgGenBlueprint,
- expected_error_type: PipeValidationErrorType,
- load_empty_library: Callable[[], None],
- ):
- load_empty_library()
- log.verbose(f"Testing error case: {test_id}")
-
- pipe_img_gen = PipeFactory[PipeImgGen].make_from_blueprint(
- domain_code="test_domain",
- pipe_code=f"test_pipe_{test_id}",
- blueprint=blueprint,
- )
- with pytest.raises(PipeValidationError) as exc_info:
- pipe_img_gen.validate_with_libraries()
-
- # Verify the error type matches
- assert exc_info.value.error_type == expected_error_type, f"Expected error type {expected_error_type}, but got {exc_info.value.error_type}"
diff --git a/tests/unit/pipelex/tools/misc/test_filetype_utils.py b/tests/unit/pipelex/tools/misc/test_filetype_utils.py
index 95efde43e..f9de8b386 100644
--- a/tests/unit/pipelex/tools/misc/test_filetype_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_filetype_utils.py
@@ -28,7 +28,7 @@ class TestFileType:
assert file_type.mime == "image/png"
-class TestFileTypeException:
+class TestFileTypeError:
def test_file_type_exception_inheritance(self):
error = FileTypeError("test message")
assert isinstance(error, Exception)