Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/groq (#445)
0x09090904e779c84d48
diff --git a/.pipelex/inference/backends.toml b/.pipelex/inference/backends.tomlindex baa5cb563..52fe3cbfc 100644--- a/.pipelex/inference/backends.toml+++ b/.pipelex/inference/backends.toml@@ -16,6 +16,11 @@ enabled = trueendpoint = "https://inference.pipelex.com/v1"api_key = "${PIPELEX_INFERENCE_API_KEY}"+[anthropic]+enabled = true+api_key = "${ANTHROPIC_API_KEY}"+claude_4_tokens_limit = 8192+[azure_openai]display_name = "Azure OpenAI"enabled = true@@ -28,53 +33,54 @@ display_name = "Amazon Bedrock"enabled = trueaws_region = "${AWS_REGION}"-[google]-display_name = "Google AI"+[blackboxai]+display_name = "BlackBox AI"enabled = true-api_key = "${GOOGLE_API_KEY}"+endpoint = "https://api.blackbox.ai/v1"+api_key = "${BLACKBOX_API_KEY}"-[vertexai]-display_name = "Google Vertex AI"+[fal]+display_name = "FAL"enabled = true-gcp_project_id = "${GCP_PROJECT_ID}"-gcp_location = "${GCP_LOCATION}"-gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"+api_key = "${FAL_API_KEY}"-[openai]-display_name = "OpenAI"+[google]+display_name = "Google AI"enabled = true-api_key = "${OPENAI_API_KEY}"+api_key = "${GOOGLE_API_KEY}"-[anthropic]+[groq]+display_name = "Groq"enabled = true-api_key = "${ANTHROPIC_API_KEY}"-claude_4_tokens_limit = 8192+endpoint = "https://api.groq.com/openai/v1"+api_key = "${GROQ_API_KEY}"[mistral]display_name = "Mistral AI"enabled = trueapi_key = "${MISTRAL_API_KEY}"-[xai]-display_name = "xAI"-enabled = true-endpoint = "https://api.x.ai/v1"-api_key = "${XAI_API_KEY}"-[ollama]enabled = trueendpoint = "http://localhost:11434/v1"-[blackboxai]-display_name = "BlackBox AI"+[openai]+display_name = "OpenAI"enabled = true-endpoint = "https://api.blackbox.ai/v1"-api_key = "${BLACKBOX_API_KEY}"+api_key = "${OPENAI_API_KEY}"-[fal]-display_name = "FAL"+[vertexai]+display_name = "Google Vertex AI"enabled = true-api_key = "${FAL_API_KEY}"+gcp_project_id = "${GCP_PROJECT_ID}"+gcp_location = "${GCP_LOCATION}"+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"++[xai]+display_name = "xAI"+enabled = true+endpoint = "https://api.x.ai/v1"+api_key = "${XAI_API_KEY}"[internal] # software-only backend, runs internally, without AIenabled = truediff --git a/.pipelex/inference/backends/groq.toml b/.pipelex/inference/backends/groq.tomlnew file mode 100644index 000000000..f1155461b--- /dev/null+++ b/.pipelex/inference/backends/groq.toml@@ -0,0 +1,129 @@+################################################################################+# Groq Backend Configuration+################################################################################+#+# This file defines the model specifications for Groq models.+# It contains model definitions for various LLM models accessible through+# the Groq API, including text-only and vision-capable models.+#+# Configuration structure:+# - Each model is defined in its own section with the model name as the header+# - Headers with dots or slashes must be quoted (e.g., ["meta-llama/llama-4-scout"])+# - Model costs are in USD per million tokens (input/output)+# - Vision models support max 5 images per request, 33MP max resolution+#+# Documentation: https://docs.pipelex.com+# Support: https://go.pipelex.com/discord+#+################################################################################++################################################################################+# MODEL DEFAULTS+################################################################################++[defaults]+model_type = "llm"+sdk = "groq"+prompting_target = "groq"++################################################################################+# PRODUCTION TEXT MODELS+################################################################################++# --- Meta Llama 3.x Series ----------------------------------------------------+["llama-3.1-8b-instant"]+model_id = "llama-3.1-8b-instant"+max_tokens = 131072+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.05, output = 0.08 }++["llama-3.3-70b-versatile"]+model_id = "llama-3.3-70b-versatile"+max_tokens = 32768+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.59, output = 0.79 }++# --- Meta Llama Guard ---------------------------------------------------------+["meta-llama/llama-guard-4-12b"]+model_id = "meta-llama/llama-guard-4-12b"+max_tokens = 1024+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.20, output = 0.20 }++# --- OpenAI GPT-OSS Models ----------------------------------------------------+["openai/gpt-oss-20b"]+model_id = "openai/gpt-oss-20b"+max_tokens = 65536+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.075, output = 0.30 }++["openai/gpt-oss-120b"]+model_id = "openai/gpt-oss-120b"+max_tokens = 65536+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.15, output = 0.60 }++# --- Groq Compound Systems ----------------------------------------------------+["groq/compound"]+model_id = "groq/compound"+max_tokens = 8192+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.15, output = 0.45 }++["groq/compound-mini"]+model_id = "groq/compound-mini"+max_tokens = 8192+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.10, output = 0.30 }++################################################################################+# PREVIEW MODELS+################################################################################++# --- Meta Llama 4 Vision Models (Preview) -------------------------------------+["meta-llama/llama-4-scout-17b-16e-instruct"]+model_id = "meta-llama/llama-4-scout-17b-16e-instruct"+max_tokens = 8192+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 5+costs = { input = 0.11, output = 0.34 }++["meta-llama/llama-4-maverick-17b-128e-instruct"]+model_id = "meta-llama/llama-4-maverick-17b-128e-instruct"+max_tokens = 8192+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 5+costs = { input = 0.20, output = 0.60 }++# --- Moonshot Kimi K2 ---------------------------------------------------------+["moonshotai/kimi-k2-instruct-0905"]+model_id = "moonshotai/kimi-k2-instruct-0905"+max_tokens = 16384+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 1.00, output = 3.00 }++# --- OpenAI Safety Model ------------------------------------------------------+["openai/gpt-oss-safeguard-20b"]+model_id = "openai/gpt-oss-safeguard-20b"+max_tokens = 65536+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.075, output = 0.30 }++# --- Qwen 3 -------------------------------------------------------------------+["qwen/qwen3-32b"]+model_id = "qwen/qwen3-32b"+max_tokens = 40960+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.29, output = 0.59 }\ No newline at end of filediff --git a/.pipelex/inference/deck/base_deck.toml b/.pipelex/inference/deck/base_deck.tomlindex 07dcb3b6f..be034698e 100644--- a/.pipelex/inference/deck/base_deck.toml+++ b/.pipelex/inference/deck/base_deck.toml@@ -21,6 +21,7 @@ base-claude = "claude-4.5-sonnet"base-gpt = "gpt-4o"base-gemini = "gemini-2.5-flash"base-mistral = "mistral-medium"+base-groq = "llama-3.3-70b-versatile"best-gpt = "gpt-4o"best-claude = "claude-4.1-opus"@@ -28,6 +29,10 @@ best-gemini = "gemini-2.5-pro"best-mistral = "mistral-medium"best-grok = "grok-3"+# Groq-specific aliases+fast-groq = "llama-3.1-8b-instant"+vision-groq = "meta-llama/llama-4-scout-17b-16e-instruct"+# Image generation aliasesbase-img-gen = "flux-pro/v1.1"best-img-gen = "flux-pro/v1.1-ultra"@@ -39,6 +44,7 @@ fast-img-gen = "fast-lightning-sdxl"[waterfalls]smart_llm = [+ "llama-3.3-70b-versatile","claude-4.5-sonnet","claude-4.1-opus","claude-4-sonnet",@@ -121,10 +127,15 @@ llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }+# Groq-specific presets (fast inference, low cost)+llm_groq_fast_text = { model = "fast-groq", temperature = 0.2 }+llm_groq_balanced = { model = "base-groq", temperature = 0.5 }+llm_groq_vision = { model = "vision-groq", temperature = 0.3 }+# LLM Presets — For Testing ---------------------------------------------------------------------llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }-llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.5 }+llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.1 }llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }llm_for_testing_vision_structured = { model = "cheap_llm_for_vision", temperature = 0.5 }diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.tomlindex 47e87ecc0..1b7b78743 100644--- a/.pipelex/inference/routing_profiles.toml+++ b/.pipelex/inference/routing_profiles.toml@@ -149,3 +149,7 @@ description = "Example routing profile using specific models""grok-3" = "pipelex_inference""grok-3-mini" = "xai"++[profiles.all_groq]+description = "Use groq backend for all its supported models"+default = "groq"diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.tomlindex 570dae25e..a1faa165f 100644--- a/.pipelex/pipelex.toml+++ b/.pipelex/pipelex.toml@@ -49,6 +49,13 @@ default_log_level = "INFO"[pipelex.log_config.package_log_levels]pipelex = "INFO"+[cogt]+[cogt.model_deck_config]+# Uncomment to disable model fallback: it will raise errors instead of using secondary model options+# is_model_fallback_enabled = false+# Uncomment to change the reaction to missing presets: "raise" (default), "log" or "none"+# missing_presets_reaction = "raise"+[cogt.llm_config]# Uncomment any of these to enable dumping the inputs or outputs of text-genration with an LLM# is_dump_text_prompts_enabled = truediff --git a/pipelex/cogt/llm/llm_worker_factory.py b/pipelex/cogt/llm/llm_worker_factory.pyindex b723fea74..24c86e17d 100644--- a/pipelex/cogt/llm/llm_worker_factory.py+++ b/pipelex/cogt/llm/llm_worker_factory.py@@ -155,6 +155,22 @@ class LLMWorkerFactory:structure_method=StructureMethod.INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS,reporting_delegate=reporting_delegate,)+ case "groq":+ # Groq uses OpenAI-compatible API, so no separate SDK needed+ from pipelex.plugins.groq.groq_factory import GroqFactory # noqa: PLC0415+ from pipelex.plugins.groq.groq_llm_worker import GroqLLMWorker # noqa: PLC0415++ sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(+ plugin=plugin,+ sdk_instance=GroqFactory.make_groq_client(plugin=plugin, backend=backend),+ )++ llm_worker = GroqLLMWorker(+ sdk_instance=sdk_instance,+ inference_model=inference_model,+ structure_method=StructureMethod.INSTRUCTOR_GROQ_TOOLS,+ reporting_delegate=reporting_delegate,+ )case _:msg = f"Plugin '{plugin}' is not supported"raise NotImplementedError(msg)diff --git a/pipelex/cogt/llm/structured_output.py b/pipelex/cogt/llm/structured_output.pyindex 7283aa9e7..d6f395bd3 100644--- a/pipelex/cogt/llm/structured_output.py+++ b/pipelex/cogt/llm/structured_output.py@@ -9,6 +9,7 @@ class StructureMethod(StrEnum):INSTRUCTOR_MISTRAL_TOOLS = "mistral_tools"INSTRUCTOR_VERTEX_JSON = "vertex_json"INSTRUCTOR_GENAI_TOOLS = "genai_tools"+ INSTRUCTOR_GROQ_TOOLS = "groq_tools"INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS = "genai_structured_outputs"def as_instructor_mode(self) -> InstructorMode:@@ -25,3 +26,5 @@ class StructureMethod(StrEnum):return InstructorMode.GENAI_TOOLScase StructureMethod.INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS:return InstructorMode.GENAI_STRUCTURED_OUTPUTS+ case StructureMethod.INSTRUCTOR_GROQ_TOOLS:+ return InstructorMode.JSONdiff --git a/pipelex/cogt/model_backends/prompting_target.py b/pipelex/cogt/model_backends/prompting_target.pyindex f05783d05..24dc2c8e5 100644--- a/pipelex/cogt/model_backends/prompting_target.py+++ b/pipelex/cogt/model_backends/prompting_target.py@@ -7,3 +7,4 @@ class PromptingTarget(StrEnum):MISTRAL = "mistral"GEMINI = "gemini"FAL = "fal"+ GROQ = "groq"diff --git a/pipelex/kit/configs/inference/backends.toml b/pipelex/kit/configs/inference/backends.tomlindex 536f3675d..c87d1ef86 100644--- a/pipelex/kit/configs/inference/backends.toml+++ b/pipelex/kit/configs/inference/backends.toml@@ -49,6 +49,12 @@ display_name = "Google AI"enabled = falseapi_key = "${GOOGLE_API_KEY}"+[groq]+display_name = "Groq"+enabled = false+endpoint = "https://api.groq.com/openai/v1"+api_key = "${GROQ_API_KEY}"+[mistral]display_name = "Mistral AI"enabled = falsediff --git a/pipelex/kit/configs/inference/backends/groq.toml b/pipelex/kit/configs/inference/backends/groq.tomlnew file mode 100644index 000000000..f1155461b--- /dev/null+++ b/pipelex/kit/configs/inference/backends/groq.toml@@ -0,0 +1,129 @@+################################################################################+# Groq Backend Configuration+################################################################################+#+# This file defines the model specifications for Groq models.+# It contains model definitions for various LLM models accessible through+# the Groq API, including text-only and vision-capable models.+#+# Configuration structure:+# - Each model is defined in its own section with the model name as the header+# - Headers with dots or slashes must be quoted (e.g., ["meta-llama/llama-4-scout"])+# - Model costs are in USD per million tokens (input/output)+# - Vision models support max 5 images per request, 33MP max resolution+#+# Documentation: https://docs.pipelex.com+# Support: https://go.pipelex.com/discord+#+################################################################################++################################################################################+# MODEL DEFAULTS+################################################################################++[defaults]+model_type = "llm"+sdk = "groq"+prompting_target = "groq"++################################################################################+# PRODUCTION TEXT MODELS+################################################################################++# --- Meta Llama 3.x Series ----------------------------------------------------+["llama-3.1-8b-instant"]+model_id = "llama-3.1-8b-instant"+max_tokens = 131072+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.05, output = 0.08 }++["llama-3.3-70b-versatile"]+model_id = "llama-3.3-70b-versatile"+max_tokens = 32768+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.59, output = 0.79 }++# --- Meta Llama Guard ---------------------------------------------------------+["meta-llama/llama-guard-4-12b"]+model_id = "meta-llama/llama-guard-4-12b"+max_tokens = 1024+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.20, output = 0.20 }++# --- OpenAI GPT-OSS Models ----------------------------------------------------+["openai/gpt-oss-20b"]+model_id = "openai/gpt-oss-20b"+max_tokens = 65536+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.075, output = 0.30 }++["openai/gpt-oss-120b"]+model_id = "openai/gpt-oss-120b"+max_tokens = 65536+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.15, output = 0.60 }++# --- Groq Compound Systems ----------------------------------------------------+["groq/compound"]+model_id = "groq/compound"+max_tokens = 8192+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.15, output = 0.45 }++["groq/compound-mini"]+model_id = "groq/compound-mini"+max_tokens = 8192+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.10, output = 0.30 }++################################################################################+# PREVIEW MODELS+################################################################################++# --- Meta Llama 4 Vision Models (Preview) -------------------------------------+["meta-llama/llama-4-scout-17b-16e-instruct"]+model_id = "meta-llama/llama-4-scout-17b-16e-instruct"+max_tokens = 8192+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 5+costs = { input = 0.11, output = 0.34 }++["meta-llama/llama-4-maverick-17b-128e-instruct"]+model_id = "meta-llama/llama-4-maverick-17b-128e-instruct"+max_tokens = 8192+inputs = ["text", "images"]+outputs = ["text", "structured"]+max_prompt_images = 5+costs = { input = 0.20, output = 0.60 }++# --- Moonshot Kimi K2 ---------------------------------------------------------+["moonshotai/kimi-k2-instruct-0905"]+model_id = "moonshotai/kimi-k2-instruct-0905"+max_tokens = 16384+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 1.00, output = 3.00 }++# --- OpenAI Safety Model ------------------------------------------------------+["openai/gpt-oss-safeguard-20b"]+model_id = "openai/gpt-oss-safeguard-20b"+max_tokens = 65536+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.075, output = 0.30 }++# --- Qwen 3 -------------------------------------------------------------------+["qwen/qwen3-32b"]+model_id = "qwen/qwen3-32b"+max_tokens = 40960+inputs = ["text"]+outputs = ["text", "structured"]+costs = { input = 0.29, output = 0.59 }\ No newline at end of filediff --git a/pipelex/kit/configs/inference/deck/base_deck.toml b/pipelex/kit/configs/inference/deck/base_deck.tomlindex 07dcb3b6f..44526c444 100644--- a/pipelex/kit/configs/inference/deck/base_deck.toml+++ b/pipelex/kit/configs/inference/deck/base_deck.toml@@ -21,6 +21,7 @@ base-claude = "claude-4.5-sonnet"base-gpt = "gpt-4o"base-gemini = "gemini-2.5-flash"base-mistral = "mistral-medium"+base-groq = "llama-3.3-70b-versatile"best-gpt = "gpt-4o"best-claude = "claude-4.1-opus"@@ -28,6 +29,10 @@ best-gemini = "gemini-2.5-pro"best-mistral = "mistral-medium"best-grok = "grok-3"+# Groq-specific aliases+fast-groq = "llama-3.1-8b-instant"+vision-groq = "meta-llama/llama-4-scout-17b-16e-instruct"+# Image generation aliasesbase-img-gen = "flux-pro/v1.1"best-img-gen = "flux-pro/v1.1-ultra"@@ -121,6 +126,11 @@ llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }+# Groq-specific presets (fast inference, low cost)+llm_groq_fast_text = { model = "fast-groq", temperature = 0.7 }+llm_groq_balanced = { model = "base-groq", temperature = 0.5 }+llm_groq_vision = { model = "vision-groq", temperature = 0.3 }+# LLM Presets — For Testing ---------------------------------------------------------------------llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }diff --git a/pipelex/plugins/groq/__init__.py b/pipelex/plugins/groq/__init__.pynew file mode 100644index 000000000..8b1378917--- /dev/null+++ b/pipelex/plugins/groq/__init__.py@@ -0,0 +1 @@+diff --git a/pipelex/plugins/groq/groq_exceptions.py b/pipelex/plugins/groq/groq_exceptions.pynew file mode 100644index 000000000..712151486--- /dev/null+++ b/pipelex/plugins/groq/groq_exceptions.py@@ -0,0 +1,17 @@+from pipelex.cogt.exceptions import CogtError+++class GroqModelListingError(CogtError):+ pass+++class GroqSDKUnsupportedError(CogtError):+ pass+++class GroqWorkerConfigurationError(CogtError):+ pass+++class GroqFactoryError(CogtError):+ passdiff --git a/pipelex/plugins/groq/groq_factory.py b/pipelex/plugins/groq/groq_factory.pynew file mode 100644index 000000000..9c5b3447c--- /dev/null+++ b/pipelex/plugins/groq/groq_factory.py@@ -0,0 +1,159 @@+import openai+from openai.types.chat import (+ ChatCompletionContentPartImageParam,+ ChatCompletionContentPartParam,+ ChatCompletionContentPartTextParam,+ ChatCompletionMessageParam,+ ChatCompletionSystemMessageParam,+ ChatCompletionUserMessageParam,+)+from openai.types.chat.chat_completion_content_part_image_param import ImageURL+from openai.types.completion_usage import CompletionUsage++from pipelex import log+from pipelex.cogt.exceptions import LLMPromptParameterError+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImagePath, PromptImageUrl+from pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.model_backends.backend import InferenceBackend+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory+from pipelex.plugins.groq.groq_exceptions import GroqFactoryError+from pipelex.plugins.plugin_sdk_registry import Plugin+from pipelex.tools.misc.base_64_utils import load_binary_as_base64+from pipelex.types import StrEnum+++class GroqSdkVariant(StrEnum):+ GROQ = "groq"+++class GroqFactory:+ @classmethod+ def make_groq_client(+ cls,+ plugin: Plugin,+ backend: InferenceBackend,+ ) -> openai.AsyncOpenAI:+ """Create AsyncOpenAI client configured for Groq API endpoint."""+ try:+ sdk_variant = GroqSdkVariant(plugin.sdk)+ except ValueError as exc:+ msg = f"Plugin '{plugin}' is not supported by GroqFactory"+ raise GroqFactoryError(msg) from exc++ # Groq API key handling (similar to OpenAI - empty string works for local models)+ api_key = backend.api_key or ""++ # Groq's OpenAI-compatible endpoint+ groq_base_url = backend.endpoint if backend.endpoint else "https://api.groq.com/openai/v1"++ match sdk_variant:+ case GroqSdkVariant.GROQ:+ log.verbose(f"Making AsyncOpenAI client for Groq with endpoint: {groq_base_url}")+ the_client = openai.AsyncOpenAI(+ api_key=api_key,+ base_url=groq_base_url,+ timeout=60.0, # Default 1 minute+ max_retries=2,+ )++ return the_client++ @classmethod+ def make_simple_messages(+ cls,+ llm_job: LLMJob,+ ) -> list[ChatCompletionMessageParam]:+ """Makes a list of messages with a system message (if provided) and followed by a user message."""+ llm_prompt = llm_job.llm_prompt+ messages: list[ChatCompletionMessageParam] = []+ user_contents: list[ChatCompletionContentPartParam] = []++ if system_content := llm_prompt.system_text:+ messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))++ if user_prompt_text := llm_prompt.user_text:+ user_part_text = ChatCompletionContentPartTextParam(text=user_prompt_text, type="text")+ user_contents.append(user_part_text)++ if llm_prompt.user_images:+ for prompt_image in llm_prompt.user_images:+ groq_image_url = cls.make_groq_image_url(prompt_image=prompt_image)+ image_param = ChatCompletionContentPartImageParam(image_url=groq_image_url, type="image_url")+ user_contents.append(image_param)++ messages.append(ChatCompletionUserMessageParam(role="user", content=user_contents))+ return messages++ @classmethod+ def make_groq_image_url(cls, prompt_image: PromptImage) -> ImageURL:+ """Convert PromptImage to Groq-compatible ImageURL format.++ Groq supports:+ - Direct URLs (max 20MB)+ - Base64 data URIs (max 4MB)+ - Max resolution: 33 megapixels+ - Max 5 images per request+ """+ if isinstance(prompt_image, PromptImageUrl):+ url = prompt_image.url+ groq_image_url = ImageURL(url=url, detail="high")+ elif isinstance(prompt_image, PromptImageBase64):+ # Detect media type from base64 if possible, default to jpeg+ media_type = cls._detect_image_media_type(prompt_image.base_64) or "jpeg"+ url_with_bytes: str = f"data:image/{media_type};base64,{prompt_image.base_64.decode('utf-8')}"+ groq_image_url = ImageURL(url=url_with_bytes, detail="high")+ elif isinstance(prompt_image, PromptImagePath):+ image_bytes = load_binary_as_base64(path=prompt_image.file_path)+ return cls.make_groq_image_url(PromptImageBase64(base_64=image_bytes))+ else:+ msg = f"prompt_image of type {type(prompt_image)} is not supported"+ raise LLMPromptParameterError(msg)+ return groq_image_url++ @staticmethod+ def _detect_image_media_type(base64_bytes: bytes) -> str | None:+ """Detect image media type from base64 bytes header."""+ try:+ # Check first few bytes for common image signatures+ if base64_bytes.startswith(b"\xff\xd8\xff"):+ return "jpeg"+ elif base64_bytes.startswith(b"\x89PNG"):+ return "png"+ elif base64_bytes.startswith(b"GIF"):+ return "gif"+ elif base64_bytes.startswith(b"RIFF") and b"WEBP" in base64_bytes[:12]:+ return "webp"+ except Exception as e:+ log.warning(e)+ return "jpeg" # Default to jpeg++ @staticmethod+ def make_openai_error_info(exception: Exception) -> str:+ """Map OpenAI exceptions to user-friendly error messages for Groq."""+ error_mapping: dict[type, str] = {+ openai.BadRequestError: "Groq API request was invalid.",+ openai.InternalServerError: "Groq is having trouble. Please try again later.",+ openai.RateLimitError: "Groq API request exceeded rate limit.",+ openai.AuthenticationError: "Groq API request was not authorized.",+ openai.PermissionDeniedError: "Groq API request was not permitted.",+ openai.NotFoundError: "Requested resource not found.",+ openai.APITimeoutError: "Groq API request timed out.",+ openai.APIConnectionError: "Groq API request failed to connect.",+ openai.APIError: "Groq API returned an API Error.",+ }+ return error_mapping.get(type(exception), "An unexpected error occurred with the Groq API.")++ @staticmethod+ def make_nb_tokens_by_category(usage: CompletionUsage) -> NbTokensByCategoryDict:+ """Extract token usage from OpenAI-compatible response."""+ nb_tokens_by_category: NbTokensByCategoryDict = {+ TokenCategory.INPUT: usage.prompt_tokens,+ TokenCategory.OUTPUT: usage.completion_tokens,+ }++ # Groq may support cached tokens in the future+ if prompt_tokens_details := usage.prompt_tokens_details:+ if hasattr(prompt_tokens_details, "cached_tokens") and prompt_tokens_details.cached_tokens:+ nb_tokens_by_category[TokenCategory.INPUT_CACHED] = prompt_tokens_details.cached_tokens++ return nb_tokens_by_categorydiff --git a/pipelex/plugins/groq/groq_llm_worker.py b/pipelex/plugins/groq/groq_llm_worker.pynew file mode 100644index 000000000..2cfff807e--- /dev/null+++ b/pipelex/plugins/groq/groq_llm_worker.py@@ -0,0 +1,140 @@+from typing import TYPE_CHECKING, Any++import instructor+import openai+from instructor.exceptions import InstructorRetryException+from openai import APIConnectionError, BadRequestError, NotFoundError, omit+from typing_extensions import override++from pipelex import log+from pipelex.cogt.exceptions import LLMCompletionError, LLMModelNotFoundError, SdkTypeError+from pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_utils import dump_error, dump_kwargs, dump_response_from_structured_gen+from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract+from pipelex.cogt.llm.structured_output import StructureMethod+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.config import get_config+from pipelex.plugins.groq.groq_factory import GroqFactory+from pipelex.reporting.reporting_protocol import ReportingProtocol+from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar++if TYPE_CHECKING:+ from openai.types.chat import ChatCompletionMessage+++class GroqLLMWorker(LLMWorkerInternalAbstract):+ def __init__(+ self,+ sdk_instance: Any,+ inference_model: InferenceModelSpec,+ structure_method: StructureMethod | None,+ reporting_delegate: ReportingProtocol | None = None,+ ):+ LLMWorkerInternalAbstract.__init__(+ self,+ inference_model=inference_model,+ structure_method=structure_method,+ reporting_delegate=reporting_delegate,+ )++ if not isinstance(sdk_instance, openai.AsyncOpenAI):+ msg = f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)++ self.groq_client_for_text: openai.AsyncOpenAI = sdk_instance+ if structure_method:+ instructor_mode = structure_method.as_instructor_mode()+ log.verbose(f"Groq structure mode: {structure_method} --> {instructor_mode}")+ self.instructor_for_objects = instructor.from_openai(client=sdk_instance, mode=instructor_mode)+ else:+ self.instructor_for_objects = instructor.from_openai(client=sdk_instance)++ instructor_config = get_config().cogt.llm_config.instructor_config+ if instructor_config.is_dump_kwargs_enabled:+ self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)+ if instructor_config.is_dump_response_enabled:+ self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response_from_structured_gen)+ if instructor_config.is_dump_error_enabled:+ self.instructor_for_objects.on(hook_name="completion:error", handler=dump_error)++ #########################################################+ @override+ def setup(self):+ pass++ @override+ def teardown(self):+ pass++ @override+ async def _gen_text(+ self,+ llm_job: LLMJob,+ ) -> str:+ messages = GroqFactory.make_simple_messages(llm_job=llm_job)++ try:+ temperature = llm_job.job_params.temperature+ response = await self.groq_client_for_text.chat.completions.create(+ model=self.inference_model.model_id,+ temperature=temperature,+ max_tokens=llm_job.job_params.max_tokens or omit,+ seed=llm_job.job_params.seed,+ messages=messages,+ )+ except NotFoundError as not_found_error:+ msg = f"Groq model not found:\n{self.inference_model.desc}\nmodel: {self.inference_model.model_id}\n{not_found_error}"+ raise LLMModelNotFoundError(msg) from not_found_error+ except APIConnectionError as api_connection_error:+ msg = f"Groq API connection error: {api_connection_error}"+ raise LLMCompletionError(msg) from api_connection_error+ except BadRequestError as bad_request_error:+ msg = f"Groq bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"+ raise LLMCompletionError(msg) from bad_request_error+ except openai.RateLimitError as rate_limit_error:+ msg = f"Groq rate limit exceeded: {rate_limit_error}"+ raise LLMCompletionError(msg) from rate_limit_error++ groq_message: ChatCompletionMessage = response.choices[0].message+ response_text = groq_message.content+ if response_text is None:+ msg = f"Groq response message content is None: {response}\nmodel: {self.inference_model.desc}"+ raise LLMCompletionError(msg)++ if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):+ llm_tokens_usage.nb_tokens_by_category = GroqFactory.make_nb_tokens_by_category(usage=usage)+ return response_text++ @override+ async def _gen_object(+ self,+ llm_job: LLMJob,+ schema: type[BaseModelTypeVar],+ ) -> BaseModelTypeVar:+ messages = GroqFactory.make_simple_messages(llm_job=llm_job)+ try:+ temperature = llm_job.job_params.temperature+ try:+ result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(+ model=self.inference_model.model_id,+ temperature=temperature,+ max_tokens=llm_job.job_params.max_tokens or omit,+ seed=llm_job.job_params.seed,+ messages=messages,+ response_model=schema,+ max_retries=llm_job.job_config.max_retries,+ )+ except InstructorRetryException as exc:+ msg = f"Groq instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"+ raise LLMCompletionError(msg) from exc+ except NotFoundError as exc:+ msg = f"Groq model '{self.inference_model.model_id}' not found: {exc}"+ raise LLMCompletionError(msg) from exc+ except BadRequestError as bad_request_error:+ msg = f"Groq bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"+ raise LLMCompletionError(msg) from bad_request_error++ if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := completion.usage):+ llm_tokens_usage.nb_tokens_by_category = GroqFactory.make_nb_tokens_by_category(usage=usage)++ return result_objectdiff --git a/pipelex/plugins/groq/groq_llms.py b/pipelex/plugins/groq/groq_llms.pynew file mode 100644index 000000000..672a94f9b--- /dev/null+++ b/pipelex/plugins/groq/groq_llms.py@@ -0,0 +1,20 @@+from openai.types import Model++from pipelex.cogt.model_backends.backend import InferenceBackend+from pipelex.plugins.groq.groq_factory import GroqFactory+from pipelex.plugins.plugin_sdk_registry import Plugin+++async def groq_list_available_models(+ plugin: Plugin,+ backend: InferenceBackend,+) -> list[Model]:+ """List available Groq models using OpenAI-compatible client."""+ groq_client_async = GroqFactory.make_groq_client(+ plugin=plugin,+ backend=backend,+ )++ models = await groq_client_async.models.list()+ data = models.data+ return sorted(data, key=lambda model: model.id)diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.pyindex 38f36c39f..66056164b 100644--- a/pipelex/test_extras/shared_pytest_plugins.py+++ b/pipelex/test_extras/shared_pytest_plugins.py@@ -25,6 +25,7 @@ ENV_VAR_KEYS_WHICH_MAY_NEED_PLACEHOLDERS_IN_CI = ["GCP_LOCATION","GCP_CREDENTIALS_FILE_PATH","ANTHROPIC_API_KEY",+ "GROQ_API_KEY","MISTRAL_API_KEY","PERPLEXITY_API_KEY","PERPLEXITY_API_ENDPOINT",diff --git a/pyproject.toml b/pyproject.tomlindex a16fe41f8..4afec6247 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -22,7 +22,8 @@ dependencies = ["backports.strenum>=1.3.0 ; python_version < '3.11'","filetype>=1.2.0","httpx>=0.23.0,<1.0.0",- "instructor>=1.8.3,<1.11.0", # 1.11.x causes typing errors with mypy+ "instructor>=1.8.3,<1.11.0", # 1.11.x causes typing errors with mypy+ "instructor[groq]==1.8.3","jinja2>=3.1.4","json2html>=1.3.0","kajson==0.3.1",