Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/add new llm models (#208)

Louis Choquel195cb2005e4b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e1643302..abad2c5dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,12 @@
# Changelog
+## Unreleased
+
+### Added
+
+- New LLM families `LLMFamily.GPT_5`, `LLMFamily.GPT_5_CHAT` and `LLMFamily.CLAUDE_4_1`
+- Added support for Claude 4.1 and GPT 5 models (inc. mini, nano, chat)
+
## [v0.6.10] - 2025-08-02
### Added
diff --git a/Makefile b/Makefile
index 19525ecfd..9add6d431 100644
--- a/Makefile
+++ b/Makefile
@@ -352,7 +352,7 @@ lint: env
pyright: env
$(call PRINT_TITLE,"Typechecking with pyright")
- $(VENV_PYRIGHT) --pythonpath $(VIRTUAL_ENV)/bin/python3 && \
+ @$(VENV_PYRIGHT) --pythonpath $(VIRTUAL_ENV)/bin/python3 && \
echo "Done typechecking with pyright — disregard warning about latest version, it's giving us false positives"
mypy: env
diff --git a/pipelex/cogt/llm/llm_models/llm_family.py b/pipelex/cogt/llm/llm_models/llm_family.py
index 9c870f359..43a7d3755 100644
--- a/pipelex/cogt/llm/llm_models/llm_family.py
+++ b/pipelex/cogt/llm/llm_models/llm_family.py
@@ -62,11 +62,14 @@ class LLMFamily(StrEnum):
GPT_4_5 = "gpt-4.5"
GPT_4_1 = "gpt-4.1"
O_SERIES = "o"
+ GPT_5 = "gpt-5"
+ GPT_5_CHAT = "gpt-5-chat"
CLAUDE_3 = "claude-3"
CLAUDE_3_5 = "claude-3.5"
CLAUDE_3_7 = "claude-3.7"
CLAUDE_4 = "claude-4"
+ CLAUDE_4_1 = "claude-4.1"
MISTRAL_7B = "mistral-7b"
MISTRAL_8X7B = "mistral-8x7b"
@@ -102,10 +105,18 @@ class LLMFamily(StrEnum):
def creator(self) -> LLMCreator:
match self:
case (
- LLMFamily.GPT_4 | LLMFamily.GPT_3_5 | LLMFamily.GPT_3 | LLMFamily.O_SERIES | LLMFamily.GPT_4_5 | LLMFamily.GPT_4_1 | LLMFamily.GPT_4O
+ LLMFamily.GPT_4
+ | LLMFamily.GPT_3_5
+ | LLMFamily.GPT_3
+ | LLMFamily.O_SERIES
+ | LLMFamily.GPT_4_5
+ | LLMFamily.GPT_4_1
+ | LLMFamily.GPT_4O
+ | LLMFamily.GPT_5
+ | LLMFamily.GPT_5_CHAT
):
return LLMCreator.OPENAI
- case LLMFamily.CLAUDE_3 | LLMFamily.CLAUDE_3_5 | LLMFamily.CLAUDE_3_7 | LLMFamily.CLAUDE_4:
+ case LLMFamily.CLAUDE_3 | LLMFamily.CLAUDE_3_5 | LLMFamily.CLAUDE_3_7 | LLMFamily.CLAUDE_4 | LLMFamily.CLAUDE_4_1:
return LLMCreator.ANTHROPIC
case LLMFamily.BEDROCK_ANTHROPIC_CLAUDE:
return LLMCreator.ANTHROPIC
diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.py
index 2c4028e38..b28e5e39c 100644
--- a/pipelex/cogt/llm/llm_worker_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_abstract.py
@@ -67,7 +67,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
) -> str:
log.debug("LLM Worker gen_text")
log.verbose(llm_job.params_desc)
- log.verbose(llm_job.llm_prompt.desc, title="llm_prompt")
+ log.verbose(llm_job.llm_prompt.desc(), title="llm_prompt")
# metadata
llm_job.job_metadata.unit_job_id = UnitJobId.LLM_GEN_TEXT
@@ -94,7 +94,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
) -> BaseModelTypeVar:
log.debug("LLM Worker gen_object")
log.verbose(llm_job.params_desc)
- log.verbose(llm_job.llm_prompt.desc, title="llm_prompt")
+ log.verbose(llm_job.llm_prompt.desc(), title="llm_prompt")
# metadata
llm_job.job_metadata.unit_job_id = UnitJobId.LLM_GEN_OBJECT
diff --git a/pipelex/core/concept_library.py b/pipelex/core/concept_library.py
index e18479cd3..99b6f3ded 100644
--- a/pipelex/core/concept_library.py
+++ b/pipelex/core/concept_library.py
@@ -107,7 +107,7 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
@override
def is_compatible_by_concept_code(self, tested_concept_code: str, wanted_concept_code: str) -> bool:
if wanted_concept_code == NativeConcept.ANYTHING.code:
- log.debug(
+ log.verbose(
f"Concept '{tested_concept_code}' is compatible with '{wanted_concept_code}' "
f"because '{wanted_concept_code}' is '{NativeConcept.ANYTHING.code}'"
)
@@ -115,11 +115,11 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
tested_concept = self.get_required_concept(concept_code=tested_concept_code)
wanted_concept = self.get_required_concept(concept_code=wanted_concept_code)
if tested_concept.code == wanted_concept.code:
- log.debug(f"Concept '{tested_concept_code}' is compatible with '{wanted_concept_code}' because they have the same code")
+ log.verbose(f"Concept '{tested_concept_code}' is compatible with '{wanted_concept_code}' because they have the same code")
return True
for inherited_concept_code in tested_concept.refines:
if self.is_compatible_by_concept_code(inherited_concept_code, wanted_concept_code):
- log.debug(
+ log.verbose(
f"Concept '{tested_concept_code}' is compatible with '{wanted_concept_code}' "
f"because '{tested_concept_code}' refines '{inherited_concept_code}' which is compatible with '{wanted_concept_code}'"
)
diff --git a/pipelex/libraries/llm_deck/base_llm_deck.toml b/pipelex/libraries/llm_deck/base_llm_deck.toml
index bdb6d43c9..1c9271aeb 100644
--- a/pipelex/libraries/llm_deck/base_llm_deck.toml
+++ b/pipelex/libraries/llm_deck/base_llm_deck.toml
@@ -11,7 +11,8 @@ llm_external_handles = []
# or a complete blueprint including llm_version (defaulting to "latest") and llm_platform_choice (defaulting to "default").
[llm_handles]
gpt-4o-2024-11-20 = { llm_name = "gpt-4o", llm_version = "2024-11-20" }
-best-claude = "claude-4-opus"
+best-gpt = "gpt-5"
+best-claude = "claude-4-1-opus"
best-gemini = "gemini-2.5-pro"
best-mistral = "mistral-large"
best-grok = "grok-3"
diff --git a/pipelex/libraries/llm_integrations/anthropic.toml b/pipelex/libraries/llm_integrations/anthropic.toml
index 510f2a212..079fbcc13 100644
--- a/pipelex/libraries/llm_integrations/anthropic.toml
+++ b/pipelex/libraries/llm_integrations/anthropic.toml
@@ -44,7 +44,7 @@ platform_llm_id = { anthropic = "claude-3-7-sonnet-20250219", bedrock_anthropic
default_platform = "anthropic"
-["claude-4".claude-4-sonnet.latest]
+[claude-4.claude-4-sonnet.latest]
max_tokens = 64000
is_gen_object_supported = true
is_vision_supported = true
@@ -54,7 +54,7 @@ platform_llm_id = { anthropic = "claude-sonnet-4-20250514", bedrock_anthropic =
default_platform = "anthropic"
-["claude-4".claude-4-opus.latest]
+[claude-4.claude-4-opus.latest]
max_tokens = 32000
is_gen_object_supported = true
is_vision_supported = true
@@ -63,3 +63,13 @@ cost_per_million_tokens_usd = { input = 3.0, output = 15.0 }
platform_llm_id = { anthropic = "claude-opus-4-20250514", bedrock_anthropic = "us.anthropic.claude-opus-4-20250514-v1:0" }
default_platform = "anthropic"
+
+["claude-4.1".claude-4-1-opus.latest]
+max_tokens = 32000
+is_gen_object_supported = true
+is_vision_supported = true
+max_prompt_images = 100
+cost_per_million_tokens_usd = { input = 3.0, output = 15.0 }
+platform_llm_id = { anthropic = "claude-opus-4-1-20250805", bedrock_anthropic = "us.anthropic.claude-opus-4-1-20250805-v1:0" }
+default_platform = "anthropic"
+
diff --git a/pipelex/libraries/llm_integrations/openai.toml b/pipelex/libraries/llm_integrations/openai.toml
index 9225e8665..895e8c3d4 100644
--- a/pipelex/libraries/llm_integrations/openai.toml
+++ b/pipelex/libraries/llm_integrations/openai.toml
@@ -77,7 +77,7 @@ default_platform = "openai"
["gpt-4.1"."gpt-4.1".latest]
is_gen_object_supported = true
is_vision_supported = true
-cost_per_million_tokens_usd = { input = 2, output = 8}
+cost_per_million_tokens_usd = { input = 2, output = 8 }
platform_llm_id = { azure_openai = "gpt-4.1-2025-04-14", openai = "gpt-4.1" }
default_platform = "openai"
@@ -110,7 +110,7 @@ platform_llm_id = { azure_openai = "o1-2024-12-17", openai = "o1" }
[o.o3-mini.latest]
is_gen_object_supported = true
cost_per_million_tokens_usd = { input = 1.1, output = 4.4 }
-platform_llm_id = { azure_openai = "o3-mini-2025-01-31", openai = "o3-mini"}
+platform_llm_id = { azure_openai = "o3-mini-2025-01-31", openai = "o3-mini" }
default_platform = "openai"
[o.o3.latest]
@@ -122,6 +122,32 @@ platform_llm_id = { openai = "o3" }
[o.o4-mini.latest]
is_gen_object_supported = true
cost_per_million_tokens_usd = { input = 1.1, output = 4.4 }
-platform_llm_id = { openai = "o4-mini"}
+platform_llm_id = { openai = "o4-mini" }
+
+[gpt-5.gpt-5.latest]
+is_gen_object_supported = true
+is_vision_supported = true
+cost_per_million_tokens_usd = { input = 1.25, output = 10.0 }
+platform_llm_id = { openai = "gpt-5" }
+
+[gpt-5.gpt-5-mini.latest]
+is_gen_object_supported = true
+is_vision_supported = true
+cost_per_million_tokens_usd = { input = 0.25, output = 2.0 }
+platform_llm_id = { azure_openai = "gpt-5-mini-2025-08-07", openai = "gpt-5-mini" }
+default_platform = "openai"
+
+[gpt-5.gpt-5-nano.latest]
+is_gen_object_supported = true
+is_vision_supported = true
+cost_per_million_tokens_usd = { input = 0.05, output = 0.4 }
+platform_llm_id = { azure_openai = "gpt-5-nano-2025-08-07", openai = "gpt-5-nano" }
+default_platform = "openai"
+
+[gpt-5-chat.gpt-5-chat.latest]
+is_gen_object_supported = false
+is_vision_supported = true
+cost_per_million_tokens_usd = { input = 1.25, output = 10.0 }
+platform_llm_id = { azure_openai = "gpt-5-chat-2025-08-07", openai = "gpt-5-chat-latest" }
default_platform = "openai"
diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.py
index d95da3f47..f9257aa28 100644
--- a/pipelex/plugins/openai/openai_llm_worker.py
+++ b/pipelex/plugins/openai/openai_llm_worker.py
@@ -60,7 +60,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
try:
match self.llm_engine.llm_model.llm_family:
- case LLMFamily.O_SERIES:
+ case LLMFamily.O_SERIES | LLMFamily.GPT_5:
# for o1 models, we must use temperature=1, and tokens limit is named max_completion_tokens
response = await self.openai_client_for_text.chat.completions.create(
model=self.llm_engine.llm_id,
@@ -85,6 +85,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
| LLMFamily.GPT_4_5
| LLMFamily.GPT_4_1
| LLMFamily.GPT_4O
+ | LLMFamily.GPT_5_CHAT
| LLMFamily.CUSTOM_LLAMA_4
| LLMFamily.CUSTOM_GEMMA_3
| LLMFamily.CUSTOM_MISTRAL_SMALL_3_1
@@ -107,6 +108,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
| LLMFamily.CLAUDE_3_5
| LLMFamily.CLAUDE_3_7
| LLMFamily.CLAUDE_4
+ | LLMFamily.CLAUDE_4_1
| LLMFamily.MISTRAL_7B
| LLMFamily.MISTRAL_8X7B
| LLMFamily.MISTRAL_LARGE
@@ -155,7 +157,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
)
try:
match self.llm_engine.llm_model.llm_family:
- case LLMFamily.O_SERIES:
+ case LLMFamily.O_SERIES | LLMFamily.GPT_5:
# for o1 models, we must use temperature=1, and tokens limit is named max_completion_tokens
result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
model=self.llm_engine.llm_id,
@@ -184,6 +186,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
| LLMFamily.GPT_4_5
| LLMFamily.GPT_4_1
| LLMFamily.GPT_4O
+ | LLMFamily.GPT_5_CHAT
| LLMFamily.CUSTOM_LLAMA_4
| LLMFamily.CUSTOM_GEMMA_3
| LLMFamily.CUSTOM_MISTRAL_SMALL_3_1
@@ -208,6 +211,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
| LLMFamily.CLAUDE_3_5
| LLMFamily.CLAUDE_3_7
| LLMFamily.CLAUDE_4
+ | LLMFamily.CLAUDE_4_1
| LLMFamily.MISTRAL_7B
| LLMFamily.MISTRAL_8X7B
| LLMFamily.MISTRAL_LARGE
diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.py
index c83ff3f78..fa109cf77 100644
--- a/tests/integration/pipelex/cogt/test_data.py
+++ b/tests/integration/pipelex/cogt/test_data.py
@@ -41,7 +41,7 @@ class Pet(BaseModel):
class LLMVisionTestCases:
VISION_USER_TEXT_1 = "Describe the provide image."
- VISION_USER_TEXT_2 = "What is this image about ?"
+ VISION_USER_TEXT_2 = "What is this image about?"
VISION_IMAGES_COMPARE_PROMPT = "Compare these two images"
URL_WIKIPEDIA_ALAN_TURING = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/40/Alan_Turing_%281912-1954%29_in_1936_at_Princeton_University_%28cropped%29.jpg/440px-Alan_Turing_%281912-1954%29_in_1936_at_Princeton_University_%28cropped%29.jpg"
diff --git a/tests/integration/pipelex/cogt/test_llm_engines.py b/tests/integration/pipelex/cogt/test_llm_engines.py
index a388ddf58..73e70f708 100644
--- a/tests/integration/pipelex/cogt/test_llm_engines.py
+++ b/tests/integration/pipelex/cogt/test_llm_engines.py
@@ -29,9 +29,11 @@ class TestLLMEngines:
else:
log.info(f"No object generation supported for this worker: '{llm_worker.desc}'")
- async def test_one_llm_engine(self, llm_job_params: LLMJobParams, llm_handle: str):
+ async def test_one_llm_engine_by_llm_handle(self, llm_job_params: LLMJobParams, llm_handle: str):
+ log.info(f"Testing {llm_handle}")
inference_manager = get_inference_manager()
llm_worker = inference_manager.get_llm_worker(llm_handle=llm_handle)
+ log.info(f"LLM Worker: {llm_worker.desc}")
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
system_text=None,
user_text=LLMTestConstants.USER_TEXT_SHORT,
diff --git a/tests/integration/pipelex/cogt/test_llm_vision.py b/tests/integration/pipelex/cogt/test_llm_vision.py
index 924dcc22f..8b4a43c27 100644
--- a/tests/integration/pipelex/cogt/test_llm_vision.py
+++ b/tests/integration/pipelex/cogt/test_llm_vision.py
@@ -2,7 +2,7 @@ from typing import Tuple
import pytest
-from pipelex import pretty_print
+from pipelex import log, pretty_print
from pipelex.cogt.exceptions import LLMCapabilityError, PromptImageFormatError
from pipelex.cogt.image.prompt_image import PromptImageBytes, PromptImagePath
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
@@ -21,10 +21,11 @@ class TestLLMVision:
async def test_gen_text_from_vision_by_url(self, llm_handle_for_vision: str, topic: str, image_uri: str):
prompt_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
+ log.info(f"LLM Worker: {llm_worker.desc}")
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
user_images=[prompt_image],
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=200, seed=None),
+ llm_job_params=LLMJobParams(temperature=0.5, max_tokens=1000, seed=None),
)
try:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
@@ -43,7 +44,7 @@ class TestLLMVision:
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
user_images=[prompt_image],
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=200, seed=None),
+ llm_job_params=LLMJobParams(temperature=0.5, max_tokens=1000, seed=None),
)
try:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
@@ -61,7 +62,7 @@ class TestLLMVision:
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
user_images=[prompt_image],
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=200, seed=None),
+ llm_job_params=LLMJobParams(temperature=0.5, max_tokens=1000, seed=None),
)
try:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
@@ -80,7 +81,7 @@ class TestLLMVision:
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
user_text=LLMVisionTestCases.VISION_IMAGES_COMPARE_PROMPT,
user_images=[prompt_image1, prompt_image2],
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=500, seed=None),
+ llm_job_params=LLMJobParams(temperature=0.5, max_tokens=2000, seed=None),
)
try:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index 248c359fb..ab299e51c 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -23,13 +23,17 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
params=[
# "o1",
# "gpt-4o",
- "gpt-4o-mini",
+ # "gpt-4o-mini",
# "gpt-4-5-preview",
# "o1-mini",
# "o3-mini",
+ # "gpt-5-mini",
+ "gpt-5-nano",
+ # "best-gpt",
# "claude-3-haiku",
# "claude-3-5-sonnet",
# "claude-3-7-sonnet",
+ # "best-claude",
# "mistral-large",
# "ministral-3b",
# "ministral-8b",
@@ -58,11 +62,15 @@ def llm_handle(request: pytest.FixtureRequest) -> str:
# "o1",
# "o3-mini",
# "gpt-4o",
- "gpt-4o-mini",
+ # "gpt-4o-mini",
+ # "gpt-5-mini",
+ "gpt-5-nano",
+ # "gpt-5-chat",
# "gpt-4-5-preview",
# "claude-3-haiku",
# "claude-3-5-sonnet",
# "claude-3-7-sonnet",
+ # "claude-4-1-opus",
# "pixtral-12b",
# "pixtral-large",
# "gemini-1-5-pro",
@@ -83,12 +91,14 @@ def llm_handle_for_vision(request: pytest.FixtureRequest) -> str:
@pytest.fixture(
params=[
# LLMFamily.GPT_4,
- LLMFamily.GPT_4O,
+ # LLMFamily.GPT_4O,
+ # LLMFamily.GPT_5,
# LLMFamily.GPT_4_5,
# LLMFamily.GPT_4_1,
# LLMFamily.O_SERIES,
# LLMFamily.CLAUDE_3_7,
# LLMFamily.CLAUDE_4,
+ LLMFamily.CLAUDE_4_1,
# LLMFamily.PERPLEXITY_SEARCH,
# LLMFamily.PERPLEXITY_REASONING,
# LLMFamily.PERPLEXITY_RESEARCH,
@@ -142,7 +152,14 @@ def llm_platform(request: pytest.FixtureRequest) -> LLMPlatform:
@pytest.fixture(
params=[
- "gpt-4o-mini",
+ "gpt-5-mini-2025-08-07",
+ "gpt-5-nano-2025-08-07",
+ "gpt-5-chat-2025-08-07",
+ # "gpt-5-mini",
+ # "gpt-5-nano",
+ # "gpt-5-chat-latest",
+ # "gpt-5",
+ # "gpt-4o-mini",
# "open-mixtral-8x7b",
# "google/gemini-2.0-flash",
# "google/gemini-2.5-pro-preview-05-06",
@@ -153,11 +170,13 @@ def llm_platform(request: pytest.FixtureRequest) -> LLMPlatform:
# "o4-mini",
# "bedrock-mistral-large",
# "sonar",
- # "claude-3-7-sonnet",
- # "claude-4-sonnet",
- # "claude-4-opus",
+ # "claude-3-7-sonnet-20250219",
+ # "claude-sonnet-4-20250514",
+ # "claude-opus-4-20250514",
+ # "claude-opus-4-1-20250805",
# "us.anthropic.claude-sonnet-4-20250514-v1:0",
# "us.anthropic.claude-opus-4-20250514-v1:0",
+ # "us.anthropic.claude-opus-4-1-20250805-v1:0",
# "sonar",
# "sonar-pro",
# "gemma3:4b",