Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
################################################################################
# Test Profiles Configuration
################################################################################
#
# This file defines test profiles for controlling which models are included
# in parametrized tests. Select a profile using TEST_PROFILE when running make.
#
# Usage:
# make regenerate-test-models # Use dev profile (default)
# make regenerate-test-models TEST_PROFILE=full # Use full profile
# make regenerate-test-models TEST_PROFILE=ci # Use CI profile
#
# Or use the CLI directly:
# pipelex-dev preprocess-test-models --generate-fixtures --profile <name>
#
# Model List Syntax:
# - "model-name" : Specific model (direct reference)
# - "@collection_name": All models from named collection (static)
# - "backend:*" : All models available from that backend (dynamic)
# - "gpt-*", "claude-*": Glob pattern matching model names (dynamic)
# - "*" : All available models (same as include_all=true)
# - "!pattern" : Exclude models matching pattern (negation)
#
################################################################################
################################################################################
# Model Collections - Available models organized by manufacturer
# Reference these in profiles using @collection_name syntax
################################################################################
[collections.backends]
# --- All Available Backends ---
all = [
"anthropic",
"azure_openai",
"bedrock",
"blackboxai",
"fal",
"google",
"groq",
"huggingface",
"minimax",
"mistral",
"ollama",
"openai",
"portkey",
"scaleway",
"vertexai",
"xai",
"linkup",
"typesafe",
"internal",
]
[collections.llm]
# --- Amazon Models (Nova) ---
amazon = ["bedrock-nova-pro", "nova-lite-v1", "nova-micro-v1"]
# --- Anthropic Models (Claude) ---
anthropic = [
"claude-3-haiku",
"claude-3.7-sonnet",
"claude-4-opus",
"claude-4-sonnet",
"claude-4.1-opus",
"claude-4.5-haiku",
"claude-4.5-sonnet",
"claude-4.5-opus",
"claude-4.6-sonnet",
"claude-4.6-opus",
"claude-4.7-opus",
"claude-4.8-opus",
"claude-5-sonnet",
"claude-5-opus",
"claude-5-fable",
"claude-5.1-fable",
"claude-5.5-sonnet",
"claude-5.5-opus",
"claude-5.5-haiku",
]
# --- DeepSeek Models ---
deepseek = [
"deepseek-chat",
"deepseek-r1",
"deepseek-v3.1",
"deepseek-r1-distill-llama-70b",
"deepseek-v3.2",
"deepseek-v3.2-speciale",
]
# --- Google Models (Gemini) ---
google = [
"gemini-2.5-flash",
"gemini-2.5-flash-lite",
"gemini-2.5-pro",
"gemini-3.0-flash",
"gemini-3.1-flash-lite",
"gemini-3.1-pro",
"gemini-3.5-flash",
"gemini-pro-latest",
"gemini-flash-latest",
"gemini-flash-lite-latest",
]
# --- Groq Models ---
groq = ["groq/compound", "groq/compound-mini"]
# --- Meta Models (Llama) ---
meta = [
"bedrock-meta-llama-3-3-70b-instruct",
"llama-3.1-8b-instant",
"llama-3.1-8b-instruct",
"llama-3.2-11b-vision-instruct",
"llama-3.3-70b-instruct",
"llama-3.3-70b-instruct-free",
"meta-llama/llama-4-maverick-17b-128e-instruct",
"meta-llama/llama-4-scout-17b-16e-instruct",
"meta-llama/llama-guard-4-12b",
]
# --- Microsoft Models ---
microsoft = ["phi-4", "phi-4-multimodal"]
# --- MiniMax Models ---
minimax = [
"minimax-m2",
"minimax-m2.1",
"minimax-m2.1-highspeed",
"minimax-m2.5",
"minimax-m2.5-highspeed",
"minimax-m2.7",
"minimax-m2.7-highspeed",
]
# --- Mistral Models ---
mistralai = [
"bedrock-mistral-large",
"ministral-3b",
"ministral-8b",
"mistral-7b-2312",
"mistral-8x7b-2312",
"mistral-codestral-2405",
"pixtral-12b",
"pixtral-large",
"mistral-small-2506",
"mistral-small-3.2",
"mistral-small-2603",
"mistral-small-4",
"mistral-small",
"mistral-medium-2508",
"mistral-medium-3.1",
"mistral-medium",
"mistral-large-2512",
"mistral-large-3",
"mistral-large",
"magistral-small-2509",
"magistral-small",
"magistral-medium-2509",
"magistral-medium",
]
# --- Moonshot AI Models ---
moonshotai = ["kimi-k2-instruct-0905", "kimi-k2-thinking"]
# --- OpenAI Models ---
openai = [
"gpt-4o-mini",
"gpt-4o",
"gpt-4.1-nano",
"gpt-4.1-mini",
"gpt-4.1",
"o1",
"o3-mini",
"o3",
"gpt-5-nano",
"gpt-5-mini",
"gpt-5",
"gpt-5-codex",
"gpt-5.1-codex",
"gpt-5.1",
"gpt-5.2",
"gpt-5.3-codex",
"gpt-5.4",
"gpt-5.4-pro",
"gpt-5.4-mini",
"gpt-5.4-nano",
"gpt-5.5",
"gpt-5.6-sol",
"gpt-5.6-terra",
"gpt-5.6-luna",
"gpt-6-astra",
"gpt-6-luna",
"gpt-6.1-sol",
]
# --- OpenAI OSS Models ---
openai_oss = ["gpt-oss-20b", "gpt-oss-120b", "gpt-oss-safeguard-20b"]
# --- Qwen Models ---
qwen = [
"qwen-2.5-72b-instruct",
"qwen3-32b",
"qwen2.5-vl-72b-instruct",
"qwen3-vl-235b-a22b",
"qwen3-235b-a22b-instruct-2507",
"qwen3-coder-30b-a3b-instruct",
]
# --- XAI Models (Grok) ---
xai = [
"grok-3",
"grok-3-mini",
"grok-3-fast",
"grok-3-mini-fast",
"grok-4",
"grok-4-fast-reasoning",
"grok-4-fast-non-reasoning",
]
[collections.img_gen]
# --- Stable Diffusion Models ---
stable_diffusion = ["fast-lightning-sdxl"]
# --- FAL Models ---
fal = [
"flux-pro",
"flux-pro/v1.1",
"flux-pro/v1.1-ultra",
"flux-2",
"flux-2-pro",
]
# --- OpenAI Models ---
openai = ["gpt-image-1", "gpt-image-1-mini", "gpt-image-1.5", "gpt-image-2"]
# --- Google Models ---
google = ["nano-banana", "nano-banana-pro", "nano-banana-2"]
# --- Qwen Models ---
qwen = ["qwen-image"]
[collections.search]
# --- Search Models ---
linkup = ["linkup-standard", "linkup-deep"]
[collections.judgment]
# --- Judgment Models ---
typesafe = ["jev-1.13.0"]
[collections.extract]
# --- Web Extraction Models ---
from_web = ["linkup-fetch"]
# --- PDF Extraction Models ---
from_pdf = [
"pypdfium2-extract-pdf",
"docling-extract-text",
"mistral-ocr",
"mistral-ocr-2503",
"mistral-ocr-2505",
"mistral-ocr-2512",
"mistral-document-ai-2505",
]
# --- Image Extraction Models ---
from_image = [
"docling-extract-text",
"mistral-ocr",
"mistral-ocr-2503",
"mistral-ocr-2505",
"mistral-ocr-2512",
"deepseek-ocr",
]
################################################################################
# Profiles
################################################################################
################################################################################
# CI Profile - Minimal set for CI (no actual inference)
################################################################################
[profiles.ci]
description = "Minimal set for CI - no actual inference"
backends = ["azure_openai"]
llm_models = ["gpt-4o-mini"]
img_gen_models = []
extract_models = []
search_models = []
judgment_models = []
################################################################################
# Dev Profile
################################################################################
[profiles.dev]
description = "Quick dev testing with fast and cheap models from key providers"
backends = ["anthropic", "openai", "google", "mistral", "linkup", "internal"]
llm_models = [
"claude-4.5-haiku",
"gpt-4o-mini",
"linkup",
"gemini-2.5-flash-lite",
]
img_gen_models = ["gpt-image-1-mini"]
extract_models = ["@from_pdf", "linkup-fetch"]
search_models = ["linkup-standard"]
judgment_models = []
################################################################################
# TypeSafe Profile
################################################################################
# The judgment family's live tests and nothing else. Every other model list is empty on purpose:
# the combo fixtures route the whole deck to one backend for the duration of a run, and TypeSafe
# serves judgments alone — a profile naming an LLM here would send it somewhere that cannot answer.
[profiles.typesafe]
description = "TypeSafe judgment models, for the judgment family's live tests"
backends = ["typesafe"]
llm_models = []
img_gen_models = []
extract_models = []
search_models = []
judgment_models = ["@typesafe"]
################################################################################
# Coverage Profile
################################################################################
[profiles.coverage]
description = "One model per backend for coverage"
backends = ["anthropic", "openai", "google", "mistral", "linkup", "internal"]
llm_models = [
"claude-4.5-haiku",
"gpt-4o-mini",
"gemini-2.5-flash-lite",
"mistral-large",
]
img_gen_models = ["gpt-image-1", "nano-banana"]
extract_models = ["pypdfium2-extract-pdf"]
search_models = ["linkup-standard"]
judgment_models = []
################################################################################
# Claude Latest Profile
################################################################################
[profiles.claude_latest]
description = "All Claude 4.5 models"
backends = ["anthropic"]
llm_models = ["claude-4.5-*"]
img_gen_models = []
extract_models = []
search_models = []
judgment_models = []
################################################################################
# GPT No Codex Profile
################################################################################
[profiles.gpt_no_codex]
description = "All GPT models except codex variants"
backends = ["openai"]
llm_models = ["gpt-*", "!*-codex", "!*-codex-*"]
img_gen_models = ["gpt-image-*"]
extract_models = []
search_models = []
judgment_models = []
################################################################################
# All Backends Profile - Test across all backends with select models
################################################################################
[profiles.all_backends]
description = "Test select models across all backends"
backends = ["@all"]
llm_models = ["claude-4.5-haiku", "gpt-4o-mini", "gemini-2.5-flash-lite"]
img_gen_models = ["gpt-image-1-mini"]
extract_models = ["@from_pdf"]
search_models = ["@linkup"]
judgment_models = []
################################################################################
# Full Profile - All available models (use with caution)
################################################################################
[profiles.full]
description = "All available models"
include_all = true