Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/groq (#445)

0x09090904e779c84d48
diff --git a/.pipelex/inference/backends.toml b/.pipelex/inference/backends.toml
index baa5cb563..52fe3cbfc 100644
--- a/.pipelex/inference/backends.toml
+++ b/.pipelex/inference/backends.toml
@@ -16,6 +16,11 @@ enabled = true
endpoint = "https://inference.pipelex.com/v1"
api_key = "${PIPELEX_INFERENCE_API_KEY}"
+[anthropic]
+enabled = true
+api_key = "${ANTHROPIC_API_KEY}"
+claude_4_tokens_limit = 8192
+
[azure_openai]
display_name = "Azure OpenAI"
enabled = true
@@ -28,53 +33,54 @@ display_name = "Amazon Bedrock"
enabled = true
aws_region = "${AWS_REGION}"
-[google]
-display_name = "Google AI"
+[blackboxai]
+display_name = "BlackBox AI"
enabled = true
-api_key = "${GOOGLE_API_KEY}"
+endpoint = "https://api.blackbox.ai/v1"
+api_key = "${BLACKBOX_API_KEY}"
-[vertexai]
-display_name = "Google Vertex AI"
+[fal]
+display_name = "FAL"
enabled = true
-gcp_project_id = "${GCP_PROJECT_ID}"
-gcp_location = "${GCP_LOCATION}"
-gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+api_key = "${FAL_API_KEY}"
-[openai]
-display_name = "OpenAI"
+[google]
+display_name = "Google AI"
enabled = true
-api_key = "${OPENAI_API_KEY}"
+api_key = "${GOOGLE_API_KEY}"
-[anthropic]
+[groq]
+display_name = "Groq"
enabled = true
-api_key = "${ANTHROPIC_API_KEY}"
-claude_4_tokens_limit = 8192
+endpoint = "https://api.groq.com/openai/v1"
+api_key = "${GROQ_API_KEY}"
[mistral]
display_name = "Mistral AI"
enabled = true
api_key = "${MISTRAL_API_KEY}"
-[xai]
-display_name = "xAI"
-enabled = true
-endpoint = "https://api.x.ai/v1"
-api_key = "${XAI_API_KEY}"
-
[ollama]
enabled = true
endpoint = "http://localhost:11434/v1"
-[blackboxai]
-display_name = "BlackBox AI"
+[openai]
+display_name = "OpenAI"
enabled = true
-endpoint = "https://api.blackbox.ai/v1"
-api_key = "${BLACKBOX_API_KEY}"
+api_key = "${OPENAI_API_KEY}"
-[fal]
-display_name = "FAL"
+[vertexai]
+display_name = "Google Vertex AI"
enabled = true
-api_key = "${FAL_API_KEY}"
+gcp_project_id = "${GCP_PROJECT_ID}"
+gcp_location = "${GCP_LOCATION}"
+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+
+[xai]
+display_name = "xAI"
+enabled = true
+endpoint = "https://api.x.ai/v1"
+api_key = "${XAI_API_KEY}"
[internal] # software-only backend, runs internally, without AI
enabled = true
diff --git a/.pipelex/inference/backends/groq.toml b/.pipelex/inference/backends/groq.toml
new file mode 100644
index 000000000..f1155461b
--- /dev/null
+++ b/.pipelex/inference/backends/groq.toml
@@ -0,0 +1,129 @@
+################################################################################
+# Groq Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Groq models.
+# It contains model definitions for various LLM models accessible through
+# the Groq API, including text-only and vision-capable models.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots or slashes must be quoted (e.g., ["meta-llama/llama-4-scout"])
+# - Model costs are in USD per million tokens (input/output)
+# - Vision models support max 5 images per request, 33MP max resolution
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "groq"
+prompting_target = "groq"
+
+################################################################################
+# PRODUCTION TEXT MODELS
+################################################################################
+
+# --- Meta Llama 3.x Series ----------------------------------------------------
+["llama-3.1-8b-instant"]
+model_id = "llama-3.1-8b-instant"
+max_tokens = 131072
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.05, output = 0.08 }
+
+["llama-3.3-70b-versatile"]
+model_id = "llama-3.3-70b-versatile"
+max_tokens = 32768
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.59, output = 0.79 }
+
+# --- Meta Llama Guard ---------------------------------------------------------
+["meta-llama/llama-guard-4-12b"]
+model_id = "meta-llama/llama-guard-4-12b"
+max_tokens = 1024
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.20, output = 0.20 }
+
+# --- OpenAI GPT-OSS Models ----------------------------------------------------
+["openai/gpt-oss-20b"]
+model_id = "openai/gpt-oss-20b"
+max_tokens = 65536
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.075, output = 0.30 }
+
+["openai/gpt-oss-120b"]
+model_id = "openai/gpt-oss-120b"
+max_tokens = 65536
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.15, output = 0.60 }
+
+# --- Groq Compound Systems ----------------------------------------------------
+["groq/compound"]
+model_id = "groq/compound"
+max_tokens = 8192
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.15, output = 0.45 }
+
+["groq/compound-mini"]
+model_id = "groq/compound-mini"
+max_tokens = 8192
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.10, output = 0.30 }
+
+################################################################################
+# PREVIEW MODELS
+################################################################################
+
+# --- Meta Llama 4 Vision Models (Preview) -------------------------------------
+["meta-llama/llama-4-scout-17b-16e-instruct"]
+model_id = "meta-llama/llama-4-scout-17b-16e-instruct"
+max_tokens = 8192
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 5
+costs = { input = 0.11, output = 0.34 }
+
+["meta-llama/llama-4-maverick-17b-128e-instruct"]
+model_id = "meta-llama/llama-4-maverick-17b-128e-instruct"
+max_tokens = 8192
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 5
+costs = { input = 0.20, output = 0.60 }
+
+# --- Moonshot Kimi K2 ---------------------------------------------------------
+["moonshotai/kimi-k2-instruct-0905"]
+model_id = "moonshotai/kimi-k2-instruct-0905"
+max_tokens = 16384
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 1.00, output = 3.00 }
+
+# --- OpenAI Safety Model ------------------------------------------------------
+["openai/gpt-oss-safeguard-20b"]
+model_id = "openai/gpt-oss-safeguard-20b"
+max_tokens = 65536
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.075, output = 0.30 }
+
+# --- Qwen 3 -------------------------------------------------------------------
+["qwen/qwen3-32b"]
+model_id = "qwen/qwen3-32b"
+max_tokens = 40960
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.29, output = 0.59 }
\ No newline at end of file
diff --git a/.pipelex/inference/deck/base_deck.toml b/.pipelex/inference/deck/base_deck.toml
index 07dcb3b6f..be034698e 100644
--- a/.pipelex/inference/deck/base_deck.toml
+++ b/.pipelex/inference/deck/base_deck.toml
@@ -21,6 +21,7 @@ base-claude = "claude-4.5-sonnet"
base-gpt = "gpt-4o"
base-gemini = "gemini-2.5-flash"
base-mistral = "mistral-medium"
+base-groq = "llama-3.3-70b-versatile"
best-gpt = "gpt-4o"
best-claude = "claude-4.1-opus"
@@ -28,6 +29,10 @@ best-gemini = "gemini-2.5-pro"
best-mistral = "mistral-medium"
best-grok = "grok-3"
+# Groq-specific aliases
+fast-groq = "llama-3.1-8b-instant"
+vision-groq = "meta-llama/llama-4-scout-17b-16e-instruct"
+
# Image generation aliases
base-img-gen = "flux-pro/v1.1"
best-img-gen = "flux-pro/v1.1-ultra"
@@ -39,6 +44,7 @@ fast-img-gen = "fast-lightning-sdxl"
[waterfalls]
smart_llm = [
+ "llama-3.3-70b-versatile",
"claude-4.5-sonnet",
"claude-4.1-opus",
"claude-4-sonnet",
@@ -121,10 +127,15 @@ llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }
llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }
llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }
+# Groq-specific presets (fast inference, low cost)
+llm_groq_fast_text = { model = "fast-groq", temperature = 0.2 }
+llm_groq_balanced = { model = "base-groq", temperature = 0.5 }
+llm_groq_vision = { model = "vision-groq", temperature = 0.3 }
+
# LLM Presets — For Testing ---------------------------------------------------------------------
llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }
-llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.5 }
+llm_for_testing_gen_object = { model = "cheap_llm_for_structured", temperature = 0.1 }
llm_for_testing_vision = { model = "cheap_llm_for_vision", temperature = 0.5 }
llm_for_testing_vision_structured = { model = "cheap_llm_for_vision", temperature = 0.5 }
diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml
index 47e87ecc0..1b7b78743 100644
--- a/.pipelex/inference/routing_profiles.toml
+++ b/.pipelex/inference/routing_profiles.toml
@@ -149,3 +149,7 @@ description = "Example routing profile using specific models"
"grok-3" = "pipelex_inference"
"grok-3-mini" = "xai"
+
+[profiles.all_groq]
+description = "Use groq backend for all its supported models"
+default = "groq"
diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml
index 570dae25e..a1faa165f 100644
--- a/.pipelex/pipelex.toml
+++ b/.pipelex/pipelex.toml
@@ -49,6 +49,13 @@ default_log_level = "INFO"
[pipelex.log_config.package_log_levels]
pipelex = "INFO"
+[cogt]
+[cogt.model_deck_config]
+# Uncomment to disable model fallback: it will raise errors instead of using secondary model options
+# is_model_fallback_enabled = false
+# Uncomment to change the reaction to missing presets: "raise" (default), "log" or "none"
+# missing_presets_reaction = "raise"
+
[cogt.llm_config]
# Uncomment any of these to enable dumping the inputs or outputs of text-genration with an LLM
# is_dump_text_prompts_enabled = true
diff --git a/pipelex/cogt/llm/llm_worker_factory.py b/pipelex/cogt/llm/llm_worker_factory.py
index b723fea74..24c86e17d 100644
--- a/pipelex/cogt/llm/llm_worker_factory.py
+++ b/pipelex/cogt/llm/llm_worker_factory.py
@@ -155,6 +155,22 @@ class LLMWorkerFactory:
structure_method=StructureMethod.INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS,
reporting_delegate=reporting_delegate,
)
+ case "groq":
+ # Groq uses OpenAI-compatible API, so no separate SDK needed
+ from pipelex.plugins.groq.groq_factory import GroqFactory # noqa: PLC0415
+ from pipelex.plugins.groq.groq_llm_worker import GroqLLMWorker # noqa: PLC0415
+
+ sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GroqFactory.make_groq_client(plugin=plugin, backend=backend),
+ )
+
+ llm_worker = GroqLLMWorker(
+ sdk_instance=sdk_instance,
+ inference_model=inference_model,
+ structure_method=StructureMethod.INSTRUCTOR_GROQ_TOOLS,
+ reporting_delegate=reporting_delegate,
+ )
case _:
msg = f"Plugin '{plugin}' is not supported"
raise NotImplementedError(msg)
diff --git a/pipelex/cogt/llm/structured_output.py b/pipelex/cogt/llm/structured_output.py
index 7283aa9e7..d6f395bd3 100644
--- a/pipelex/cogt/llm/structured_output.py
+++ b/pipelex/cogt/llm/structured_output.py
@@ -9,6 +9,7 @@ class StructureMethod(StrEnum):
INSTRUCTOR_MISTRAL_TOOLS = "mistral_tools"
INSTRUCTOR_VERTEX_JSON = "vertex_json"
INSTRUCTOR_GENAI_TOOLS = "genai_tools"
+ INSTRUCTOR_GROQ_TOOLS = "groq_tools"
INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS = "genai_structured_outputs"
def as_instructor_mode(self) -> InstructorMode:
@@ -25,3 +26,5 @@ class StructureMethod(StrEnum):
return InstructorMode.GENAI_TOOLS
case StructureMethod.INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS:
return InstructorMode.GENAI_STRUCTURED_OUTPUTS
+ case StructureMethod.INSTRUCTOR_GROQ_TOOLS:
+ return InstructorMode.JSON
diff --git a/pipelex/cogt/model_backends/prompting_target.py b/pipelex/cogt/model_backends/prompting_target.py
index f05783d05..24dc2c8e5 100644
--- a/pipelex/cogt/model_backends/prompting_target.py
+++ b/pipelex/cogt/model_backends/prompting_target.py
@@ -7,3 +7,4 @@ class PromptingTarget(StrEnum):
MISTRAL = "mistral"
GEMINI = "gemini"
FAL = "fal"
+ GROQ = "groq"
diff --git a/pipelex/kit/configs/inference/backends.toml b/pipelex/kit/configs/inference/backends.toml
index 536f3675d..c87d1ef86 100644
--- a/pipelex/kit/configs/inference/backends.toml
+++ b/pipelex/kit/configs/inference/backends.toml
@@ -49,6 +49,12 @@ display_name = "Google AI"
enabled = false
api_key = "${GOOGLE_API_KEY}"
+[groq]
+display_name = "Groq"
+enabled = false
+endpoint = "https://api.groq.com/openai/v1"
+api_key = "${GROQ_API_KEY}"
+
[mistral]
display_name = "Mistral AI"
enabled = false
diff --git a/pipelex/kit/configs/inference/backends/groq.toml b/pipelex/kit/configs/inference/backends/groq.toml
new file mode 100644
index 000000000..f1155461b
--- /dev/null
+++ b/pipelex/kit/configs/inference/backends/groq.toml
@@ -0,0 +1,129 @@
+################################################################################
+# Groq Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Groq models.
+# It contains model definitions for various LLM models accessible through
+# the Groq API, including text-only and vision-capable models.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots or slashes must be quoted (e.g., ["meta-llama/llama-4-scout"])
+# - Model costs are in USD per million tokens (input/output)
+# - Vision models support max 5 images per request, 33MP max resolution
+#
+# Documentation: https://docs.pipelex.com
+# Support: https://go.pipelex.com/discord
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "groq"
+prompting_target = "groq"
+
+################################################################################
+# PRODUCTION TEXT MODELS
+################################################################################
+
+# --- Meta Llama 3.x Series ----------------------------------------------------
+["llama-3.1-8b-instant"]
+model_id = "llama-3.1-8b-instant"
+max_tokens = 131072
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.05, output = 0.08 }
+
+["llama-3.3-70b-versatile"]
+model_id = "llama-3.3-70b-versatile"
+max_tokens = 32768
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.59, output = 0.79 }
+
+# --- Meta Llama Guard ---------------------------------------------------------
+["meta-llama/llama-guard-4-12b"]
+model_id = "meta-llama/llama-guard-4-12b"
+max_tokens = 1024
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.20, output = 0.20 }
+
+# --- OpenAI GPT-OSS Models ----------------------------------------------------
+["openai/gpt-oss-20b"]
+model_id = "openai/gpt-oss-20b"
+max_tokens = 65536
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.075, output = 0.30 }
+
+["openai/gpt-oss-120b"]
+model_id = "openai/gpt-oss-120b"
+max_tokens = 65536
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.15, output = 0.60 }
+
+# --- Groq Compound Systems ----------------------------------------------------
+["groq/compound"]
+model_id = "groq/compound"
+max_tokens = 8192
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.15, output = 0.45 }
+
+["groq/compound-mini"]
+model_id = "groq/compound-mini"
+max_tokens = 8192
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.10, output = 0.30 }
+
+################################################################################
+# PREVIEW MODELS
+################################################################################
+
+# --- Meta Llama 4 Vision Models (Preview) -------------------------------------
+["meta-llama/llama-4-scout-17b-16e-instruct"]
+model_id = "meta-llama/llama-4-scout-17b-16e-instruct"
+max_tokens = 8192
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 5
+costs = { input = 0.11, output = 0.34 }
+
+["meta-llama/llama-4-maverick-17b-128e-instruct"]
+model_id = "meta-llama/llama-4-maverick-17b-128e-instruct"
+max_tokens = 8192
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 5
+costs = { input = 0.20, output = 0.60 }
+
+# --- Moonshot Kimi K2 ---------------------------------------------------------
+["moonshotai/kimi-k2-instruct-0905"]
+model_id = "moonshotai/kimi-k2-instruct-0905"
+max_tokens = 16384
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 1.00, output = 3.00 }
+
+# --- OpenAI Safety Model ------------------------------------------------------
+["openai/gpt-oss-safeguard-20b"]
+model_id = "openai/gpt-oss-safeguard-20b"
+max_tokens = 65536
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.075, output = 0.30 }
+
+# --- Qwen 3 -------------------------------------------------------------------
+["qwen/qwen3-32b"]
+model_id = "qwen/qwen3-32b"
+max_tokens = 40960
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.29, output = 0.59 }
\ No newline at end of file
diff --git a/pipelex/kit/configs/inference/deck/base_deck.toml b/pipelex/kit/configs/inference/deck/base_deck.toml
index 07dcb3b6f..44526c444 100644
--- a/pipelex/kit/configs/inference/deck/base_deck.toml
+++ b/pipelex/kit/configs/inference/deck/base_deck.toml
@@ -21,6 +21,7 @@ base-claude = "claude-4.5-sonnet"
base-gpt = "gpt-4o"
base-gemini = "gemini-2.5-flash"
base-mistral = "mistral-medium"
+base-groq = "llama-3.3-70b-versatile"
best-gpt = "gpt-4o"
best-claude = "claude-4.1-opus"
@@ -28,6 +29,10 @@ best-gemini = "gemini-2.5-pro"
best-mistral = "mistral-medium"
best-grok = "grok-3"
+# Groq-specific aliases
+fast-groq = "llama-3.1-8b-instant"
+vision-groq = "meta-llama/llama-4-scout-17b-16e-instruct"
+
# Image generation aliases
base-img-gen = "flux-pro/v1.1"
best-img-gen = "flux-pro/v1.1-ultra"
@@ -121,6 +126,11 @@ llm_for_table_to_text = { model = "base-claude", temperature = 0.3 }
llm_to_prompt_img_gen = { model = "base-claude", temperature = 0.2 }
llm_to_prompt_img_gen_cheap = { model = "gpt-4o-mini", temperature = 0.5 }
+# Groq-specific presets (fast inference, low cost)
+llm_groq_fast_text = { model = "fast-groq", temperature = 0.7 }
+llm_groq_balanced = { model = "base-groq", temperature = 0.5 }
+llm_groq_vision = { model = "vision-groq", temperature = 0.3 }
+
# LLM Presets — For Testing ---------------------------------------------------------------------
llm_for_testing_gen_text = { model = "cheap_llm", temperature = 0.5 }
diff --git a/pipelex/plugins/groq/__init__.py b/pipelex/plugins/groq/__init__.py
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/pipelex/plugins/groq/__init__.py
@@ -0,0 +1 @@
+
diff --git a/pipelex/plugins/groq/groq_exceptions.py b/pipelex/plugins/groq/groq_exceptions.py
new file mode 100644
index 000000000..712151486
--- /dev/null
+++ b/pipelex/plugins/groq/groq_exceptions.py
@@ -0,0 +1,17 @@
+from pipelex.cogt.exceptions import CogtError
+
+
+class GroqModelListingError(CogtError):
+ pass
+
+
+class GroqSDKUnsupportedError(CogtError):
+ pass
+
+
+class GroqWorkerConfigurationError(CogtError):
+ pass
+
+
+class GroqFactoryError(CogtError):
+ pass
diff --git a/pipelex/plugins/groq/groq_factory.py b/pipelex/plugins/groq/groq_factory.py
new file mode 100644
index 000000000..9c5b3447c
--- /dev/null
+++ b/pipelex/plugins/groq/groq_factory.py
@@ -0,0 +1,159 @@
+import openai
+from openai.types.chat import (
+ ChatCompletionContentPartImageParam,
+ ChatCompletionContentPartParam,
+ ChatCompletionContentPartTextParam,
+ ChatCompletionMessageParam,
+ ChatCompletionSystemMessageParam,
+ ChatCompletionUserMessageParam,
+)
+from openai.types.chat.chat_completion_content_part_image_param import ImageURL
+from openai.types.completion_usage import CompletionUsage
+
+from pipelex import log
+from pipelex.cogt.exceptions import LLMPromptParameterError
+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImagePath, PromptImageUrl
+from pipelex.cogt.llm.llm_job import LLMJob
+from pipelex.cogt.model_backends.backend import InferenceBackend
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.plugins.groq.groq_exceptions import GroqFactoryError
+from pipelex.plugins.plugin_sdk_registry import Plugin
+from pipelex.tools.misc.base_64_utils import load_binary_as_base64
+from pipelex.types import StrEnum
+
+
+class GroqSdkVariant(StrEnum):
+ GROQ = "groq"
+
+
+class GroqFactory:
+ @classmethod
+ def make_groq_client(
+ cls,
+ plugin: Plugin,
+ backend: InferenceBackend,
+ ) -> openai.AsyncOpenAI:
+ """Create AsyncOpenAI client configured for Groq API endpoint."""
+ try:
+ sdk_variant = GroqSdkVariant(plugin.sdk)
+ except ValueError as exc:
+ msg = f"Plugin '{plugin}' is not supported by GroqFactory"
+ raise GroqFactoryError(msg) from exc
+
+ # Groq API key handling (similar to OpenAI - empty string works for local models)
+ api_key = backend.api_key or ""
+
+ # Groq's OpenAI-compatible endpoint
+ groq_base_url = backend.endpoint if backend.endpoint else "https://api.groq.com/openai/v1"
+
+ match sdk_variant:
+ case GroqSdkVariant.GROQ:
+ log.verbose(f"Making AsyncOpenAI client for Groq with endpoint: {groq_base_url}")
+ the_client = openai.AsyncOpenAI(
+ api_key=api_key,
+ base_url=groq_base_url,
+ timeout=60.0, # Default 1 minute
+ max_retries=2,
+ )
+
+ return the_client
+
+ @classmethod
+ def make_simple_messages(
+ cls,
+ llm_job: LLMJob,
+ ) -> list[ChatCompletionMessageParam]:
+ """Makes a list of messages with a system message (if provided) and followed by a user message."""
+ llm_prompt = llm_job.llm_prompt
+ messages: list[ChatCompletionMessageParam] = []
+ user_contents: list[ChatCompletionContentPartParam] = []
+
+ if system_content := llm_prompt.system_text:
+ messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))
+
+ if user_prompt_text := llm_prompt.user_text:
+ user_part_text = ChatCompletionContentPartTextParam(text=user_prompt_text, type="text")
+ user_contents.append(user_part_text)
+
+ if llm_prompt.user_images:
+ for prompt_image in llm_prompt.user_images:
+ groq_image_url = cls.make_groq_image_url(prompt_image=prompt_image)
+ image_param = ChatCompletionContentPartImageParam(image_url=groq_image_url, type="image_url")
+ user_contents.append(image_param)
+
+ messages.append(ChatCompletionUserMessageParam(role="user", content=user_contents))
+ return messages
+
+ @classmethod
+ def make_groq_image_url(cls, prompt_image: PromptImage) -> ImageURL:
+ """Convert PromptImage to Groq-compatible ImageURL format.
+
+ Groq supports:
+ - Direct URLs (max 20MB)
+ - Base64 data URIs (max 4MB)
+ - Max resolution: 33 megapixels
+ - Max 5 images per request
+ """
+ if isinstance(prompt_image, PromptImageUrl):
+ url = prompt_image.url
+ groq_image_url = ImageURL(url=url, detail="high")
+ elif isinstance(prompt_image, PromptImageBase64):
+ # Detect media type from base64 if possible, default to jpeg
+ media_type = cls._detect_image_media_type(prompt_image.base_64) or "jpeg"
+ url_with_bytes: str = f"data:image/{media_type};base64,{prompt_image.base_64.decode('utf-8')}"
+ groq_image_url = ImageURL(url=url_with_bytes, detail="high")
+ elif isinstance(prompt_image, PromptImagePath):
+ image_bytes = load_binary_as_base64(path=prompt_image.file_path)
+ return cls.make_groq_image_url(PromptImageBase64(base_64=image_bytes))
+ else:
+ msg = f"prompt_image of type {type(prompt_image)} is not supported"
+ raise LLMPromptParameterError(msg)
+ return groq_image_url
+
+ @staticmethod
+ def _detect_image_media_type(base64_bytes: bytes) -> str | None:
+ """Detect image media type from base64 bytes header."""
+ try:
+ # Check first few bytes for common image signatures
+ if base64_bytes.startswith(b"\xff\xd8\xff"):
+ return "jpeg"
+ elif base64_bytes.startswith(b"\x89PNG"):
+ return "png"
+ elif base64_bytes.startswith(b"GIF"):
+ return "gif"
+ elif base64_bytes.startswith(b"RIFF") and b"WEBP" in base64_bytes[:12]:
+ return "webp"
+ except Exception as e:
+ log.warning(e)
+ return "jpeg" # Default to jpeg
+
+ @staticmethod
+ def make_openai_error_info(exception: Exception) -> str:
+ """Map OpenAI exceptions to user-friendly error messages for Groq."""
+ error_mapping: dict[type, str] = {
+ openai.BadRequestError: "Groq API request was invalid.",
+ openai.InternalServerError: "Groq is having trouble. Please try again later.",
+ openai.RateLimitError: "Groq API request exceeded rate limit.",
+ openai.AuthenticationError: "Groq API request was not authorized.",
+ openai.PermissionDeniedError: "Groq API request was not permitted.",
+ openai.NotFoundError: "Requested resource not found.",
+ openai.APITimeoutError: "Groq API request timed out.",
+ openai.APIConnectionError: "Groq API request failed to connect.",
+ openai.APIError: "Groq API returned an API Error.",
+ }
+ return error_mapping.get(type(exception), "An unexpected error occurred with the Groq API.")
+
+ @staticmethod
+ def make_nb_tokens_by_category(usage: CompletionUsage) -> NbTokensByCategoryDict:
+ """Extract token usage from OpenAI-compatible response."""
+ nb_tokens_by_category: NbTokensByCategoryDict = {
+ TokenCategory.INPUT: usage.prompt_tokens,
+ TokenCategory.OUTPUT: usage.completion_tokens,
+ }
+
+ # Groq may support cached tokens in the future
+ if prompt_tokens_details := usage.prompt_tokens_details:
+ if hasattr(prompt_tokens_details, "cached_tokens") and prompt_tokens_details.cached_tokens:
+ nb_tokens_by_category[TokenCategory.INPUT_CACHED] = prompt_tokens_details.cached_tokens
+
+ return nb_tokens_by_category
diff --git a/pipelex/plugins/groq/groq_llm_worker.py b/pipelex/plugins/groq/groq_llm_worker.py
new file mode 100644
index 000000000..2cfff807e
--- /dev/null
+++ b/pipelex/plugins/groq/groq_llm_worker.py
@@ -0,0 +1,140 @@
+from typing import TYPE_CHECKING, Any
+
+import instructor
+import openai
+from instructor.exceptions import InstructorRetryException
+from openai import APIConnectionError, BadRequestError, NotFoundError, omit
+from typing_extensions import override
+
+from pipelex import log
+from pipelex.cogt.exceptions import LLMCompletionError, LLMModelNotFoundError, SdkTypeError
+from pipelex.cogt.llm.llm_job import LLMJob
+from pipelex.cogt.llm.llm_utils import dump_error, dump_kwargs, dump_response_from_structured_gen
+from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
+from pipelex.cogt.llm.structured_output import StructureMethod
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.config import get_config
+from pipelex.plugins.groq.groq_factory import GroqFactory
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
+
+if TYPE_CHECKING:
+ from openai.types.chat import ChatCompletionMessage
+
+
+class GroqLLMWorker(LLMWorkerInternalAbstract):
+ def __init__(
+ self,
+ sdk_instance: Any,
+ inference_model: InferenceModelSpec,
+ structure_method: StructureMethod | None,
+ reporting_delegate: ReportingProtocol | None = None,
+ ):
+ LLMWorkerInternalAbstract.__init__(
+ self,
+ inference_model=inference_model,
+ structure_method=structure_method,
+ reporting_delegate=reporting_delegate,
+ )
+
+ if not isinstance(sdk_instance, openai.AsyncOpenAI):
+ msg = f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
+
+ self.groq_client_for_text: openai.AsyncOpenAI = sdk_instance
+ if structure_method:
+ instructor_mode = structure_method.as_instructor_mode()
+ log.verbose(f"Groq structure mode: {structure_method} --> {instructor_mode}")
+ self.instructor_for_objects = instructor.from_openai(client=sdk_instance, mode=instructor_mode)
+ else:
+ self.instructor_for_objects = instructor.from_openai(client=sdk_instance)
+
+ instructor_config = get_config().cogt.llm_config.instructor_config
+ if instructor_config.is_dump_kwargs_enabled:
+ self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)
+ if instructor_config.is_dump_response_enabled:
+ self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response_from_structured_gen)
+ if instructor_config.is_dump_error_enabled:
+ self.instructor_for_objects.on(hook_name="completion:error", handler=dump_error)
+
+ #########################################################
+ @override
+ def setup(self):
+ pass
+
+ @override
+ def teardown(self):
+ pass
+
+ @override
+ async def _gen_text(
+ self,
+ llm_job: LLMJob,
+ ) -> str:
+ messages = GroqFactory.make_simple_messages(llm_job=llm_job)
+
+ try:
+ temperature = llm_job.job_params.temperature
+ response = await self.groq_client_for_text.chat.completions.create(
+ model=self.inference_model.model_id,
+ temperature=temperature,
+ max_tokens=llm_job.job_params.max_tokens or omit,
+ seed=llm_job.job_params.seed,
+ messages=messages,
+ )
+ except NotFoundError as not_found_error:
+ msg = f"Groq model not found:\n{self.inference_model.desc}\nmodel: {self.inference_model.model_id}\n{not_found_error}"
+ raise LLMModelNotFoundError(msg) from not_found_error
+ except APIConnectionError as api_connection_error:
+ msg = f"Groq API connection error: {api_connection_error}"
+ raise LLMCompletionError(msg) from api_connection_error
+ except BadRequestError as bad_request_error:
+ msg = f"Groq bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"
+ raise LLMCompletionError(msg) from bad_request_error
+ except openai.RateLimitError as rate_limit_error:
+ msg = f"Groq rate limit exceeded: {rate_limit_error}"
+ raise LLMCompletionError(msg) from rate_limit_error
+
+ groq_message: ChatCompletionMessage = response.choices[0].message
+ response_text = groq_message.content
+ if response_text is None:
+ msg = f"Groq response message content is None: {response}\nmodel: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
+
+ if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):
+ llm_tokens_usage.nb_tokens_by_category = GroqFactory.make_nb_tokens_by_category(usage=usage)
+ return response_text
+
+ @override
+ async def _gen_object(
+ self,
+ llm_job: LLMJob,
+ schema: type[BaseModelTypeVar],
+ ) -> BaseModelTypeVar:
+ messages = GroqFactory.make_simple_messages(llm_job=llm_job)
+ try:
+ temperature = llm_job.job_params.temperature
+ try:
+ result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
+ model=self.inference_model.model_id,
+ temperature=temperature,
+ max_tokens=llm_job.job_params.max_tokens or omit,
+ seed=llm_job.job_params.seed,
+ messages=messages,
+ response_model=schema,
+ max_retries=llm_job.job_config.max_retries,
+ )
+ except InstructorRetryException as exc:
+ msg = f"Groq instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"
+ raise LLMCompletionError(msg) from exc
+ except NotFoundError as exc:
+ msg = f"Groq model '{self.inference_model.model_id}' not found: {exc}"
+ raise LLMCompletionError(msg) from exc
+ except BadRequestError as bad_request_error:
+ msg = f"Groq bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"
+ raise LLMCompletionError(msg) from bad_request_error
+
+ if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := completion.usage):
+ llm_tokens_usage.nb_tokens_by_category = GroqFactory.make_nb_tokens_by_category(usage=usage)
+
+ return result_object
diff --git a/pipelex/plugins/groq/groq_llms.py b/pipelex/plugins/groq/groq_llms.py
new file mode 100644
index 000000000..672a94f9b
--- /dev/null
+++ b/pipelex/plugins/groq/groq_llms.py
@@ -0,0 +1,20 @@
+from openai.types import Model
+
+from pipelex.cogt.model_backends.backend import InferenceBackend
+from pipelex.plugins.groq.groq_factory import GroqFactory
+from pipelex.plugins.plugin_sdk_registry import Plugin
+
+
+async def groq_list_available_models(
+ plugin: Plugin,
+ backend: InferenceBackend,
+) -> list[Model]:
+ """List available Groq models using OpenAI-compatible client."""
+ groq_client_async = GroqFactory.make_groq_client(
+ plugin=plugin,
+ backend=backend,
+ )
+
+ models = await groq_client_async.models.list()
+ data = models.data
+ return sorted(data, key=lambda model: model.id)
diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.py
index 38f36c39f..66056164b 100644
--- a/pipelex/test_extras/shared_pytest_plugins.py
+++ b/pipelex/test_extras/shared_pytest_plugins.py
@@ -25,6 +25,7 @@ ENV_VAR_KEYS_WHICH_MAY_NEED_PLACEHOLDERS_IN_CI = [
"GCP_LOCATION",
"GCP_CREDENTIALS_FILE_PATH",
"ANTHROPIC_API_KEY",
+ "GROQ_API_KEY",
"MISTRAL_API_KEY",
"PERPLEXITY_API_KEY",
"PERPLEXITY_API_ENDPOINT",
diff --git a/pyproject.toml b/pyproject.toml
index a16fe41f8..4afec6247 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,7 +22,8 @@ dependencies = [
"backports.strenum>=1.3.0 ; python_version < '3.11'",
"filetype>=1.2.0",
"httpx>=0.23.0,<1.0.0",
- "instructor>=1.8.3,<1.11.0", # 1.11.x causes typing errors with mypy
+ "instructor>=1.8.3,<1.11.0", # 1.11.x causes typing errors with mypy
+ "instructor[groq]==1.8.3",
"jinja2>=3.1.4",
"json2html>=1.3.0",
"kajson==0.3.1",