Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Replace talent fields with direct model preset in PipeSpec subclasses (#783)

Louis Choquela8b42fef4750
diff --git a/pipelex/builder/CLAUDE.md b/pipelex/builder/CLAUDE.md
index 65a463978..363f90ba1 100644
--- a/pipelex/builder/CLAUDE.md
+++ b/pipelex/builder/CLAUDE.md
@@ -100,4 +100,4 @@ After fixing, `_prune_unreachable_specs()` removes pipes unreachable from `main_
## Talent System
-Talents are abstract capability labels mapped to concrete model presets. Each talent enum (in `talents/`) maps to a `$preset` code used in MTHDS files. When modifying talents, update both the enum and its preset mapping dict.
+Talents are abstract capability labels mapped to concrete model presets. They remain in `talents/` and are used by the `pipelex-agent models` CLI to help agents choose appropriate presets. PipeSpec subclasses accept model presets directly via a `model: str | None` field, bypassing talent resolution. When modifying talent enums, update both the enum and its preset mapping dict in `pipelex.toml`.
diff --git a/pipelex/builder/pipe/pipe_extract_spec.py b/pipelex/builder/pipe/pipe_extract_spec.py
index e37317670..154b2b759 100644
--- a/pipelex/builder/pipe/pipe_extract_spec.py
+++ b/pipelex/builder/pipe/pipe_extract_spec.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Literal
+from typing import Literal
from pydantic import Field, field_validator
from pydantic.json_schema import SkipJsonSchema
@@ -7,15 +7,10 @@ from rich.text import Text
from typing_extensions import override
from pipelex.builder.pipe.pipe_spec import PipeSpec
-from pipelex.builder.talents.extract_talent import ExtractTalent
from pipelex.cogt.content_generation.dry_run_factory import MockFormat
-from pipelex.config import get_config
from pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprint
from pipelex.tools.misc.pretty import PrettyPrintable
-if TYPE_CHECKING:
- from pipelex.cogt.extract.extract_setting import ExtractModelChoice
-
class PipeExtractSpec(PipeSpec):
"""Spec for OCR (Optical Character Recognition) pipe operations in the Pipelex framework.
@@ -38,31 +33,29 @@ class PipeExtractSpec(PipeSpec):
),
json_schema_extra={"mock_format": MockFormat.DICT_SINGLE_EXTRACT_INPUT},
)
- extract_talent: ExtractTalent | str = Field(
- description="Select the most adequate extraction model talent according to the task to be performed.",
- examples=list(ExtractTalent),
+ model: str | None = Field(
+ default=None,
+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",
)
max_page_images: int | None = Field(
default=None, description="Max number of images to extract from pages: None=unlimited, 0=no images, N=limit to N images."
)
page_views: bool | None = Field(default=None, description="Whether to include rendered page views in the output.")
+ @field_validator("model", mode="before")
+ @classmethod
+ def reject_empty_model(cls, value: str | None) -> str | None:
+ if isinstance(value, str) and not value.strip():
+ msg = "Model cannot be an empty string; omit the field to use defaults"
+ raise ValueError(msg)
+ return value
+
@override
@field_validator("output", mode="before")
@classmethod
def validate_output(cls, output: str) -> str:
return "Page[]"
- @field_validator("extract_talent", mode="before")
- @classmethod
- def validate_extract_talent(cls, extract_talent_value: str) -> ExtractTalent:
- try:
- return ExtractTalent(extract_talent_value)
- except ValueError:
- valid = [talent.value for talent in ExtractTalent]
- msg = f"'{extract_talent_value}' is not a valid ExtractTalent. Valid values: {valid}"
- raise ValueError(msg) from None
-
@field_validator("inputs", mode="before")
@classmethod
def validate_extract_inputs(cls, inputs_value: dict[str, str] | None) -> dict[str, str] | None:
@@ -85,7 +78,7 @@ class PipeExtractSpec(PipeSpec):
# Add extract specific information
extract_group.renderables.append(Text()) # Blank line
- extract_group.renderables.append(Text.from_markup(f"Extract Talent: [bold yellow]{self.extract_talent}[/bold yellow]"))
+ extract_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))
# Add optional extraction settings if they are set
if self.max_page_images is not None:
@@ -99,16 +92,12 @@ class PipeExtractSpec(PipeSpec):
def to_blueprint(self) -> PipeExtractBlueprint:
base_blueprint = super().to_blueprint()
- # Get extract choice from config-based mapping
- mappings = get_config().pipelex.builder_config.talent_preset_mappings.extract
- extract_model_choice: ExtractModelChoice = mappings[self.extract_talent]
-
return PipeExtractBlueprint(
source=None,
description=base_blueprint.description,
inputs=base_blueprint.inputs,
output=base_blueprint.output,
- model=extract_model_choice,
+ model=self.model,
max_page_images=self.max_page_images,
page_image_captions=None,
page_views=self.page_views,
diff --git a/pipelex/builder/pipe/pipe_img_gen_spec.py b/pipelex/builder/pipe/pipe_img_gen_spec.py
index 604dd6862..ab2558dd1 100644
--- a/pipelex/builder/pipe/pipe_img_gen_spec.py
+++ b/pipelex/builder/pipe/pipe_img_gen_spec.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Literal
+from typing import Literal
from pydantic import Field, field_validator
from rich.console import Group
@@ -6,14 +6,9 @@ from rich.text import Text
from typing_extensions import override
from pipelex.builder.pipe.pipe_spec import PipeSpec
-from pipelex.builder.talents.img_gen_talent import ImgGenTalent
-from pipelex.config import get_config
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
from pipelex.tools.misc.pretty import PrettyPrintable
-if TYPE_CHECKING:
- from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice
-
class PipeImgGenSpec(PipeSpec):
"""Specs for image generation pipe operations in the Pipelex framework.
@@ -30,23 +25,19 @@ class PipeImgGenSpec(PipeSpec):
type: Literal["PipeImgGen"] = "PipeImgGen"
pipe_category: Literal["PipeOperator"] = "PipeOperator"
- img_gen_talent: ImgGenTalent | str = Field(
- description="Select the most adequate image generation talent according to the task to be performed.",
- examples=list(ImgGenTalent),
+ model: str | None = Field(
+ default=None,
+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",
)
prompt: str = Field(description="A finalized image generation prompt or prompt template: use `$` prefix for inline variables (e.g., `$topic`).")
- @field_validator("img_gen_talent", mode="before")
+ @field_validator("model", mode="before")
@classmethod
- def validate_img_gen_talent(cls, img_gen_talent_value: str | None) -> ImgGenTalent | None:
- if img_gen_talent_value is None:
- return None
- try:
- return ImgGenTalent(img_gen_talent_value)
- except ValueError:
- valid = [talent.value for talent in ImgGenTalent]
- msg = f"'{img_gen_talent_value}' is not a valid ImgGenTalent. Valid values: {valid}"
- raise ValueError(msg) from None
+ def reject_empty_model(cls, value: str | None) -> str | None:
+ if isinstance(value, str) and not value.strip():
+ msg = "Model cannot be an empty string; omit the field to use defaults"
+ raise ValueError(msg)
+ return value
@override
def rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:
@@ -59,7 +50,7 @@ class PipeImgGenSpec(PipeSpec):
# Add image generation specific information
img_gen_group.renderables.append(Text()) # Blank line
- img_gen_group.renderables.append(Text.from_markup(f"Image Generation Talent: [bold yellow]{self.img_gen_talent}[/bold yellow]"))
+ img_gen_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))
return img_gen_group
@@ -68,16 +59,12 @@ class PipeImgGenSpec(PipeSpec):
"""Convert this PipeImgGenSpec to the core PipeImgGenBlueprint."""
base_blueprint = super().to_blueprint()
- # Get img_gen choice from config-based mapping
- mappings = get_config().pipelex.builder_config.talent_preset_mappings.img_gen
- img_gen_choice: ImgGenModelChoice = mappings[self.img_gen_talent]
-
return PipeImgGenBlueprint(
description=base_blueprint.description,
inputs=base_blueprint.inputs,
output=base_blueprint.output,
prompt=self.prompt,
- model=img_gen_choice,
+ model=self.model,
aspect_ratio=None,
background=None,
output_format=None,
diff --git a/pipelex/builder/pipe/pipe_llm_spec.py b/pipelex/builder/pipe/pipe_llm_spec.py
index 2eeb53741..cf02e72dd 100644
--- a/pipelex/builder/pipe/pipe_llm_spec.py
+++ b/pipelex/builder/pipe/pipe_llm_spec.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Literal
+from typing import Literal
from pydantic import Field, field_validator
from pydantic.json_schema import SkipJsonSchema
@@ -8,14 +8,9 @@ from rich.text import Text
from typing_extensions import override
from pipelex.builder.pipe.pipe_spec import PipeSpec
-from pipelex.builder.talents.llm_talent import LLMTalent
-from pipelex.config import get_config
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
from pipelex.tools.misc.pretty import PrettyPrintable
-if TYPE_CHECKING:
- from pipelex.cogt.llm.llm_setting import LLMModelChoice
-
class PipeLLMSpec(PipeSpec):
"""Spec for LLM-based pipe operations in the Pipelex framework.
@@ -33,9 +28,9 @@ class PipeLLMSpec(PipeSpec):
type: SkipJsonSchema[Literal["PipeLLM"]] = "PipeLLM"
pipe_category: SkipJsonSchema[Literal["PipeOperator"]] = "PipeOperator"
- llm_talent: LLMTalent | str = Field(
- description="Select the simplest LLM talent corresponding to the task to be performed.",
- examples=list(LLMTalent),
+ model: str | None = Field(
+ default=None,
+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",
)
system_prompt: str | None = Field(default=None, description="A system prompt to guide the LLM's behavior, style and skills. Can be a template.")
prompt: str | None = Field(
@@ -58,15 +53,13 @@ So, don't have to write a bullet-list of all the attributes definitions yourself
""",
)
- @field_validator("llm_talent", mode="before")
+ @field_validator("model", mode="before")
@classmethod
- def validate_llm_talent(cls, llm_value: str) -> LLMTalent:
- try:
- return LLMTalent(llm_value)
- except ValueError:
- valid = [talent.value for talent in LLMTalent]
- msg = f"'{llm_value}' is not a valid LLMTalent. Valid values: {valid}"
- raise ValueError(msg) from None
+ def reject_empty_model(cls, value: str | None) -> str | None:
+ if isinstance(value, str) and not value.strip():
+ msg = "Model cannot be an empty string; omit the field to use defaults"
+ raise ValueError(msg)
+ return value
@override
def rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:
@@ -79,7 +72,7 @@ So, don't have to write a bullet-list of all the attributes definitions yourself
# Add LLM-specific information
llm_group.renderables.append(Text()) # Blank line
- llm_group.renderables.append(Text.from_markup(f"LLM Talent: [bold yellow]{self.llm_talent}[/bold yellow]"))
+ llm_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))
# Add system prompt if present
if self.system_prompt:
@@ -111,15 +104,11 @@ So, don't have to write a bullet-list of all the attributes definitions yourself
def to_blueprint(self) -> PipeLLMBlueprint:
base_blueprint = super().to_blueprint()
- # Get llm choice from config-based mapping
- mappings = get_config().pipelex.builder_config.talent_preset_mappings.llm
- llm_choice: LLMModelChoice = mappings[self.llm_talent]
-
return PipeLLMBlueprint(
description=base_blueprint.description,
inputs=base_blueprint.inputs,
output=base_blueprint.output,
system_prompt=self.system_prompt,
prompt=self.prompt,
- model=llm_choice,
+ model=self.model,
)
diff --git a/pipelex/builder/pipe/pipe_search_spec.py b/pipelex/builder/pipe/pipe_search_spec.py
index 28531571c..2e4a5b274 100644
--- a/pipelex/builder/pipe/pipe_search_spec.py
+++ b/pipelex/builder/pipe/pipe_search_spec.py
@@ -1,5 +1,5 @@
import re
-from typing import TYPE_CHECKING, Any, Literal
+from typing import Any, Literal
from pydantic import ConfigDict, Field, field_validator, model_validator
from rich.console import Group
@@ -8,14 +8,9 @@ from rich.text import Text
from typing_extensions import override
from pipelex.builder.pipe.pipe_spec import PipeSpec
-from pipelex.builder.talents.search_talent import SearchTalent
-from pipelex.config import get_config
from pipelex.pipe_operators.search.pipe_search_blueprint import PipeSearchBlueprint
from pipelex.tools.misc.pretty import PrettyPrintable
-if TYPE_CHECKING:
- from pipelex.cogt.search.search_setting import SearchModelChoice
-
class PipeSearchSpec(PipeSpec):
"""Specs for web search pipe operations in the Pipelex framework.
@@ -28,10 +23,9 @@ class PipeSearchSpec(PipeSpec):
type: Literal["PipeSearch"] = "PipeSearch"
pipe_category: Literal["PipeOperator"] = "PipeOperator"
- search_talent: SearchTalent | str = Field(
- default=SearchTalent.WEB_SEARCH,
- description="Select the most adequate search talent according to the task to be performed.",
- examples=list(SearchTalent),
+ model: str | None = Field(
+ default=None,
+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",
)
prompt: str = Field(
validation_alias="query",
@@ -53,6 +47,14 @@ class PipeSearchSpec(PipeSpec):
default=None, description="Maximum number of search results to return. If not set, the search provider's default is used."
)
+ @field_validator("model", mode="before")
+ @classmethod
+ def reject_empty_model(cls, value: str | None) -> str | None:
+ if isinstance(value, str) and not value.strip():
+ msg = "Model cannot be an empty string; omit the field to use defaults"
+ raise ValueError(msg)
+ return value
+
@model_validator(mode="before")
@classmethod
def normalize_prompt_field(cls, values: dict[str, Any]) -> dict[str, Any]:
@@ -75,16 +77,6 @@ class PipeSearchSpec(PipeSpec):
raise ValueError(msg)
return date_value
- @field_validator("search_talent", mode="before")
- @classmethod
- def validate_search_talent(cls, search_talent_value: str) -> SearchTalent:
- try:
- return SearchTalent(search_talent_value)
- except ValueError:
- valid = list(SearchTalent)
- msg = f"'{search_talent_value}' is not a valid SearchTalent. Valid values: {valid}"
- raise ValueError(msg) from None
-
@override
def rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:
# Get base pipe information from parent
@@ -96,7 +88,7 @@ class PipeSearchSpec(PipeSpec):
# Add search specific information
search_group.renderables.append(Text()) # Blank line
- search_group.renderables.append(Text.from_markup(f"Search Talent: [bold yellow]{self.search_talent}[/bold yellow]"))
+ search_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))
prompt_panel = Panel(
self.prompt,
title="Search Prompt",
@@ -131,16 +123,12 @@ class PipeSearchSpec(PipeSpec):
"""Convert this PipeSearchSpec to the core PipeSearchBlueprint."""
base_blueprint = super().to_blueprint()
- # Get search choice from config-based mapping
- mappings = get_config().pipelex.builder_config.talent_preset_mappings.search
- search_choice: SearchModelChoice = mappings[self.search_talent]
-
return PipeSearchBlueprint(
description=base_blueprint.description,
inputs=base_blueprint.inputs,
output=base_blueprint.output,
prompt=self.prompt,
- model=search_choice,
+ model=self.model,
include_images=None,
max_results=self.max_results,
from_date=self.from_date,
diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.md
index 21d680dae..aca3f015a 100644
--- a/pipelex/cli/agent_cli/CLAUDE.md
+++ b/pipelex/cli/agent_cli/CLAUDE.md
@@ -47,7 +47,8 @@ commands/
plxt_passthrough.py # Shared helper for plxt subprocess delegation
concept_cmd.py # concept — JSON spec → raw TOML to stdout
pipe_cmd.py # pipe — JSON spec → raw TOML to stdout
- models_cmd.py # models — list presets, aliases, talent mappings
+ models_cmd.py # models — list presets, aliases, waterfalls
+ check_model_cmd.py # check-model — validate model reference with fuzzy suggestions
init_cmd.py # init — non-interactive config setup (--global/-g, --config/-c)
doctor_cmd.py # doctor — config health check (--global/-g)
```
@@ -64,7 +65,8 @@ commands/
| `inputs` | Generates example input JSON for a pipe/bundle/method (pipe\|bundle\|method subcommands) |
| `concept` | Converts a JSON concept spec into raw TOML (stdout) |
| `pipe` | Converts a JSON pipe spec (typed) into raw TOML (stdout) |
-| `models` | Lists available model presets, aliases, waterfalls, and talent mappings. `--format markdown\|json` (default: markdown) |
+| `models` | Lists available model presets, aliases, and waterfalls. `--format markdown\|json` (default: markdown) |
+| `check-model` | Validates a model reference and suggests alternatives if invalid. `--type`/`-t` for model category, `--format markdown\|json` (default: markdown) |
| `doctor` | Checks config, credentials, models health. `--global`/`-g` for global dir. `--format markdown\|json` (default: markdown) |
## Key Patterns
diff --git a/pipelex/cli/agent_cli/_agent_cli.py b/pipelex/cli/agent_cli/_agent_cli.py
index d7ed8979d..5813fd118 100644
--- a/pipelex/cli/agent_cli/_agent_cli.py
+++ b/pipelex/cli/agent_cli/_agent_cli.py
@@ -8,6 +8,7 @@ from mthds.runners.types import RunnerType
from typer.core import TyperGroup
from typing_extensions import override
+from pipelex.cli.agent_cli.commands.check_model_cmd import agent_check_model_cmd
from pipelex.cli.agent_cli.commands.concept_cmd import concept_cmd
from pipelex.cli.agent_cli.commands.doctor_cmd import agent_doctor_cmd
from pipelex.cli.agent_cli.commands.fmt_cmd import fmt_cmd
@@ -37,6 +38,7 @@ class PipelexAgentCLI(TyperGroup):
"concept",
"pipe",
"models",
+ "check-model",
"doctor",
]
@@ -126,5 +128,6 @@ app.command(name="lint", help="Lint a .mthds, .toml, or .plx file")(lint_cmd)
app.add_typer(inputs_app, name="inputs", help="Generate example input JSON for a pipe")
app.command(name="concept", help="Structure a concept from JSON spec and output TOML")(concept_cmd)
app.command(name="pipe", help="Structure a pipe from JSON spec and output TOML")(pipe_cmd)
-app.command(name="models", help="List available model presets, aliases, and talent mappings")(agent_models_cmd)
+app.command(name="models", help="List available model presets, aliases, and waterfalls")(agent_models_cmd)
+app.command(name="check-model", help="Check if a model reference is valid and suggest alternatives")(agent_check_model_cmd)
app.command(name="doctor", help="Check Pipelex configuration health and auto-fix issues")(agent_doctor_cmd)
diff --git a/pipelex/cli/agent_cli/commands/agent_output.py b/pipelex/cli/agent_cli/commands/agent_output.py
index 5d00d4798..acd33516b 100644
--- a/pipelex/cli/agent_cli/commands/agent_output.py
+++ b/pipelex/cli/agent_cli/commands/agent_output.py
@@ -18,6 +18,10 @@ class CliOutputFormat(StrEnum):
AGENT_ERROR_HINTS: dict[str, str] = {
# Model/routing errors
+ "ModelChoiceNotFoundError": (
+ "Check model name for typos. Use 'pipelex-agent check-model <name> -t <type>' "
+ "to validate or 'pipelex-agent models -t <type>' to list available models."
+ ),
"PipeOperatorModelChoiceError": "Run 'pipelex-agent doctor' to check available models and routing configuration",
"PipeOperatorModelAvailabilityError": "Run 'pipelex-agent doctor' to check available models and verify API keys",
"ModelDeckPresetValidatonError": (
@@ -70,6 +74,7 @@ RETRYABLE_ERROR_TYPES: set[str] = {
AGENT_ERROR_DOMAINS: dict[str, str] = {
# input = agent can fix (bad .mthds, wrong args, bad JSON)
+ "ModelChoiceNotFoundError": "input",
"ValidateBundleError": "input",
"PipeValidationError": "input",
"FileNotFoundError": "input",
diff --git a/pipelex/cli/agent_cli/commands/check_model_cmd.py b/pipelex/cli/agent_cli/commands/check_model_cmd.py
new file mode 100644
index 000000000..dddcc4363
--- /dev/null
+++ b/pipelex/cli/agent_cli/commands/check_model_cmd.py
@@ -0,0 +1,107 @@
+"""Agent CLI check-model command -- validate a model reference and suggest alternatives."""
+
+from typing import Annotated, Any
+
+import typer
+
+from pipelex.cli.agent_cli.commands.agent_cli_factory import make_pipelex_for_agent_cli
+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_error, agent_success
+from pipelex.cli.agent_cli.commands.models_cmd import CATEGORY_TO_MODEL_TYPE, ModelCategory
+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceParseError
+from pipelex.cogt.models.model_suggestion import (
+ KIND_LABELS,
+ get_collection_keys,
+ suggest_model_alternatives,
+)
+from pipelex.hub import get_model_deck
+from pipelex.pipelex import Pipelex
+
+
+def _format_check_markdown(result: dict[str, Any]) -> str:
+ """Format the check-model result as concise markdown."""
+ name: str = result["name"]
+ kind_label: str = result["kind"]
+ category: str = result["model_type"]
+
+ if result["valid"]:
+ return f"{name} is a valid {category} {kind_label}."
+
+ lines: list[str] = [f"{name} is not a valid {category} {kind_label}."]
+
+ suggestions: list[str] = result.get("suggestions", [])
+ if suggestions:
+ lines.append("\nDid you mean:")
+ for suggestion in suggestions:
+ lines.append(f"- {suggestion}")
+
+ cross_suggestions: list[str] = result.get("cross_collection_suggestions", [])
+ if cross_suggestions:
+ if not suggestions:
+ lines.append("\nDid you mean:")
+ for suggestion in cross_suggestions:
+ lines.append(f"- {suggestion}")
+
+ wrong_sigil_hints: list[str] = result.get("wrong_sigil_hints", [])
+ for hint in wrong_sigil_hints:
+ lines.append(f"\nNote: {hint}")
+
+ return "\n".join(lines)
+
+
+def agent_check_model_cmd(
+ ctx: typer.Context,
+ name: Annotated[
+ str,
+ typer.Argument(help="Model reference to check (e.g. $writing-creative, @best-claude, gpt-4o)"),
+ ],
+ model_type: Annotated[
+ ModelCategory,
+ typer.Option("--type", "-t", help="Model category: llm, extract, img_gen, search"),
+ ],
+ output_format: Annotated[
+ CliOutputFormat,
+ typer.Option("--format", help="Output format: markdown (default) or json (structured)"),
+ ] = CliOutputFormat.MARKDOWN,
+) -> None:
+ """Check if a model reference is valid and suggest alternatives if not.
+
+ Parses the reference (with sigil prefix if present), validates it against the
+ model deck, and on failure provides fuzzy suggestions and wrong-sigil hints.
+ """
+ try:
+ make_pipelex_for_agent_cli(log_level=ctx.obj["log_level"], needs_inference=False, needs_model_specs=True)
+
+ model_deck = get_model_deck()
+ ref = ModelReference.parse(name)
+ resolved_model_type = CATEGORY_TO_MODEL_TYPE[model_type]
+
+ candidates = get_collection_keys(model_deck, resolved_model_type, ref.kind)
+ is_valid = ref.name in candidates
+
+ result: dict[str, Any] = {
+ "success": True,
+ "valid": is_valid,
+ "name": name,
+ "kind": KIND_LABELS[ref.kind],
+ "model_type": str(model_type),
+ }
+
+ if not is_valid:
+ suggestions, wrong_sigil_hints, cross_suggestions = suggest_model_alternatives(model_deck, resolved_model_type, ref.name, ref.kind)
+ result["suggestions"] = suggestions
+ result["wrong_sigil_hints"] = wrong_sigil_hints
+ result["cross_collection_suggestions"] = cross_suggestions
+
+ match output_format:
+ case CliOutputFormat.JSON:
+ agent_success(result)
+ case CliOutputFormat.MARKDOWN:
+ print(_format_check_markdown(result))
+ except SystemExit:
+ raise
+ except ModelReferenceParseError as exc:
+ agent_error(f"Invalid model reference: {exc}", "ArgumentError", cause=exc)
+ except Exception as exc:
+ agent_error(f"Failed to check model: {exc}", type(exc).__name__, cause=exc)
+ finally:
+ Pipelex.teardown_if_needed()
diff --git a/pipelex/cli/agent_cli/commands/models_cmd.py b/pipelex/cli/agent_cli/commands/models_cmd.py
index 2164d3157..542df98d7 100644
--- a/pipelex/cli/agent_cli/commands/models_cmd.py
+++ b/pipelex/cli/agent_cli/commands/models_cmd.py
@@ -1,4 +1,4 @@
-"""Agent CLI models command -- list available model presets, aliases, and talent mappings."""
+"""Agent CLI models command -- list available model presets, aliases, and waterfalls."""
from typing import Annotated, Any
@@ -9,7 +9,6 @@ from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_e
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.models.model_deck import ModelDeck
-from pipelex.config import get_config
from pipelex.hub import get_model_deck
from pipelex.pipelex import Pipelex
from pipelex.types import StrEnum
@@ -100,25 +99,6 @@ def _filter_waterfalls_by_backend(
return filtered
-def _filter_talent_mappings_by_backend(
- mappings: dict[str, str],
- presets_dict: dict[str, Any],
- model_deck: ModelDeck,
- model_type: ModelType,
- backend: str,
-) -> dict[str, str]:
- """Filter talent mappings to only include those whose preset resolves to the given backend."""
- filtered: dict[str, str] = {}
- for talent_name, preset_name in mappings.items():
- setting = presets_dict.get(preset_name)
- if setting is None:
- continue
- spec = _resolve_preset_backend(model_deck, setting.model, model_type)
- if spec is not None and spec.backend_name == backend:
- filtered[talent_name] = preset_name
- return filtered
-
-
def _build_presets_for_category(
model_deck: ModelDeck,
category: ModelCategory,
@@ -198,31 +178,6 @@ def _build_waterfalls_for_category(
return waterfalls
-def _build_talent_mappings_for_category(
- model_deck: ModelDeck,
- category: ModelCategory,
- mappings: dict[str, str],
- backend: str | None,
-) -> dict[str, str]:
- """Build the talent mappings dict for a given category, optionally filtered by backend."""
- if backend is None:
- return mappings
-
- model_type = CATEGORY_TO_MODEL_TYPE[category]
- presets_dict: dict[str, Any]
- match category:
- case ModelCategory.LLM:
- presets_dict = model_deck.llm_presets
- case ModelCategory.EXTRACT:
- presets_dict = model_deck.extract_presets
- case ModelCategory.IMG_GEN:
- presets_dict = model_deck.img_gen_presets
- case ModelCategory.SEARCH:
- presets_dict = model_deck.search_presets
-
- return _filter_talent_mappings_by_backend(mappings, presets_dict, model_deck, model_type, backend)
-
-
CATEGORY_DISPLAY_NAMES: dict[str, str] = {
"llm": "LLM",
"img_gen": "Image Generation",
@@ -238,47 +193,41 @@ def _format_models_markdown(result: dict[str, Any]) -> str:
presets: dict[str, list[dict[str, Any]]] = result["presets"]
aliases: dict[str, dict[str, str]] = result["aliases"]
waterfalls: dict[str, dict[str, list[str]]] = result["waterfalls"]
- talent_mappings: dict[str, dict[str, str]] = result["talent_mappings"]
for category_key, display_name in CATEGORY_DISPLAY_NAMES.items():
cat_presets = presets.get(category_key, [])
cat_aliases = aliases.get(category_key, {})
cat_waterfalls = waterfalls.get(category_key, {})
- cat_talents = talent_mappings.get(category_key, {})
- if not cat_presets and not cat_aliases and not cat_waterfalls and not cat_talents:
+ if not cat_presets and not cat_aliases and not cat_waterfalls:
continue
lines.append(f"\n## {display_name}")
if cat_presets:
- lines.append("\n### Presets\n")
+ lines.append("\n### Presets")
+ lines.append("\nPresets are the preferred way to specify models — each preset bundles a model with relevant settings.\n")
for preset in cat_presets:
description = preset.get("description", "")
if description:
- lines.append(f"- **{preset['name']}**: {description}")
+ lines.append(f"- ${preset['name']}: {description}")
else:
- lines.append(f"- **{preset['name']}**")
+ lines.append(f"- ${preset['name']}")
if cat_aliases:
- lines.append("\n### Aliases\n")
+ lines.append("\n### Aliases")
+ lines.append(
+ "\nAliases provide stable names that can be retargeted when new models come out, so pipelines stay up to date without edits.\n"
+ )
for alias_name, alias_target in cat_aliases.items():
- lines.append(f"- **{alias_name}** \u2192 {alias_target}")
+ lines.append(f"- @{alias_name} \u2192 {alias_target}")
if cat_waterfalls:
- lines.append("\n### Waterfalls\n")
+ lines.append("\n### Waterfalls")
+ lines.append("\nWaterfalls define ordered fallback chains — if the first model is unavailable, the next one is tried automatically.\n")
for waterfall_name, fallback_list in cat_waterfalls.items():
chain = " \u2192 ".join(fallback_list)
- lines.append(f"- **{waterfall_name}**: {chain}")
-
- if cat_talents:
- lines.append("\n### Talent Mappings\n")
- for talent_name, preset_name in cat_talents.items():
- lines.append(f"- **{talent_name}** \u2192 {preset_name}")
-
- hint = result.get("talent_mappings_usage_hint")
- if hint:
- lines.append(f"\n---\n\n{hint}")
+ lines.append(f"- ~{waterfall_name}: {chain}")
return "\n".join(lines)
@@ -298,7 +247,7 @@ def agent_models_cmd(
typer.Option("--format", help="Output format: markdown (default) or json (structured)"),
] = CliOutputFormat.MARKDOWN,
) -> None:
- """List available model presets, aliases, waterfalls, and talent mappings.
+ """List available model presets, aliases, and waterfalls.
Outputs model configuration that an agent needs to reference when building pipelines.
Default output is markdown; use --format json for structured JSON.
@@ -307,48 +256,36 @@ def agent_models_cmd(
make_pipelex_for_agent_cli(log_level=ctx.obj["log_level"], needs_inference=False, needs_model_specs=backend is not None)
model_deck = get_model_deck()
- builder_config = get_config().pipelex.builder_config
- talent_mappings = builder_config.talent_preset_mappings
presets: dict[str, list[dict[str, Any]]] = {}
aliases: dict[str, dict[str, str]] = {}
waterfalls: dict[str, dict[str, list[str]]] = {}
- talent: dict[str, dict[str, str]] = {}
if _should_include(ModelCategory.LLM, model_type):
presets["llm"] = _build_presets_for_category(model_deck, ModelCategory.LLM, backend)
aliases["llm"] = _build_aliases_for_category(model_deck, ModelCategory.LLM, backend)
waterfalls["llm"] = _build_waterfalls_for_category(model_deck, ModelCategory.LLM, backend)
- talent["llm"] = _build_talent_mappings_for_category(model_deck, ModelCategory.LLM, talent_mappings.llm, backend)
if _should_include(ModelCategory.IMG_GEN, model_type):
presets["img_gen"] = _build_presets_for_category(model_deck, ModelCategory.IMG_GEN, backend)
aliases["img_gen"] = _build_aliases_for_category(model_deck, ModelCategory.IMG_GEN, backend)
waterfalls["img_gen"] = _build_waterfalls_for_category(model_deck, ModelCategory.IMG_GEN, backend)
- talent["img_gen"] = _build_talent_mappings_for_category(model_deck, ModelCategory.IMG_GEN, talent_mappings.img_gen, backend)
if _should_include(ModelCategory.EXTRACT, model_type):
presets["extract"] = _build_presets_for_category(model_deck, ModelCategory.EXTRACT, backend)
aliases["extract"] = _build_aliases_for_category(model_deck, ModelCategory.EXTRACT, backend)
waterfalls["extract"] = _build_waterfalls_for_category(model_deck, ModelCategory.EXTRACT, backend)
- talent["extract"] = _build_talent_mappings_for_category(model_deck, ModelCategory.EXTRACT, talent_mappings.extract, backend)
if _should_include(ModelCategory.SEARCH, model_type):
presets["search"] = _build_presets_for_category(model_deck, ModelCategory.SEARCH, backend)
aliases["search"] = _build_aliases_for_category(model_deck, ModelCategory.SEARCH, backend)
waterfalls["search"] = _build_waterfalls_for_category(model_deck, ModelCategory.SEARCH, backend)
- talent["search"] = _build_talent_mappings_for_category(model_deck, ModelCategory.SEARCH, talent_mappings.search, backend)
result: dict[str, Any] = {
"success": True,
"presets": presets,
"aliases": aliases,
"waterfalls": waterfalls,
- "talent_mappings": talent,
- "talent_mappings_usage_hint": (
- "Use the talent name (key) as the value for llm_talent / extract_talent / img_gen_talent / search_talent"
- " in pipe specs passed to 'pipelex-agent pipe --spec'"
- ),
}
match output_format:
diff --git a/pipelex/cli/agent_cli/commands/pipe_cmd.py b/pipelex/cli/agent_cli/commands/pipe_cmd.py
index c89599372..2e72d2d5f 100644
--- a/pipelex/cli/agent_cli/commands/pipe_cmd.py
+++ b/pipelex/cli/agent_cli/commands/pipe_cmd.py
@@ -24,65 +24,11 @@ from pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpec
from pipelex.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
from pipelex.builder.pipe.pipe_spec import PipeSpec
from pipelex.builder.pipe.pipe_spec_map import pipe_type_to_spec_class
-from pipelex.builder.talents.extract_talent import ExtractTalent
-from pipelex.builder.talents.img_gen_talent import ImgGenTalent
-from pipelex.builder.talents.llm_talent import LLMTalent
from pipelex.cli.agent_cli.commands.agent_output import agent_error
from pipelex.core.pipes.pipe_blueprint import PipeType
from pipelex.language.toml_string_utils import format_toml_string
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error_for_agent
-# Static talent-to-model preset mappings (from pipelex.toml defaults)
-LLM_TALENT_TO_MODEL: dict[str, str] = {
- "data-retrieval": "$retrieval",
- "hr-expert": "$writing-factual",
- "accounting-expert": "$writing-factual",
- "creative-writer": "$writing-creative",
- "engineer": "$engineering-structured",
- "coder": "$engineering-code",
- "code-analyzer": "$engineering-codebase-analysis",
- "vision-language-model": "$vision",
- "visual-designer": "$img-gen-prompting",
-}
-
-IMG_GEN_TALENT_TO_MODEL: dict[str, str] = {
- "gen-image": "$gen-image",
- "gen-image-fast": "$gen-image-fast",
- "gen-image-high-quality": "$gen-image-high-quality",
-}
-
-EXTRACT_TALENT_TO_MODEL: dict[str, str] = {
- "pdf-basic-text-extractor": "@default-text-from-pdf",
- "image-text-extractor": "@default-extract-image",
- "full-document-extractor": "@default-extract-document",
- "web-page-extractor": "@default-extract-web-page",
-}
-
-# Reverse mappings: preset name → talent name (for fuzzy resolution)
-_MODEL_TO_LLM_TALENT: dict[str, str] = {}
-for _talent, _preset in LLM_TALENT_TO_MODEL.items():
- _MODEL_TO_LLM_TALENT[_preset] = _talent
- _MODEL_TO_LLM_TALENT[_preset.lstrip("$@")] = _talent
-
-_MODEL_TO_IMG_GEN_TALENT: dict[str, str] = {}
-for _talent, _preset in IMG_GEN_TALENT_TO_MODEL.items():
- _MODEL_TO_IMG_GEN_TALENT[_preset] = _talent
- _MODEL_TO_IMG_GEN_TALENT[_preset.lstrip("$@")] = _talent
-
-_MODEL_TO_EXTRACT_TALENT: dict[str, str] = {}
-for _talent, _preset in EXTRACT_TALENT_TO_MODEL.items():
- _MODEL_TO_EXTRACT_TALENT[_preset] = _talent
- _MODEL_TO_EXTRACT_TALENT[_preset.lstrip("$@")] = _talent
-
-# Maps pipe types to their talent field names and valid values.
-# Used to enrich validation errors with field_hints so the agent knows
-# what values are valid when a talent field is missing or wrong.
-PIPE_TYPE_TALENT_HINTS: dict[str, dict[str, list[str]]] = {
- "PipeLLM": {"llm_talent": [talent.value for talent in LLMTalent]},
- "PipeExtract": {"extract_talent": [talent.value for talent in ExtractTalent]},
- "PipeImgGen": {"img_gen_talent": [talent.value for talent in ImgGenTalent]},
-}
-
def _pipe_spec_to_toml(pipe_spec: PipeSpec) -> str:
"""Convert a PipeSpec to TOML string format.
@@ -130,9 +76,8 @@ def _add_type_specific_fields(pipe_spec: PipeSpec, pipe_table: tomlkit.TOMLDocum
pipe_table: The TOML table to add fields to.
"""
if isinstance(pipe_spec, PipeLLMSpec):
- # Convert llm_talent to model preset using static mappings
- model_preset = LLM_TALENT_TO_MODEL.get(pipe_spec.llm_talent, "$writing-creative")
- pipe_table.add("model", model_preset)
+ if pipe_spec.model:
+ pipe_table.add("model", pipe_spec.model)
if pipe_spec.system_prompt:
pipe_table.add("system_prompt", format_toml_string(pipe_spec.system_prompt))
if pipe_spec.prompt:
@@ -198,40 +143,18 @@ def _add_type_specific_fields(pipe_spec: PipeSpec, pipe_table: tomlkit.TOMLDocum
pipe_table.add("input_item_name", pipe_spec.input_item_name)
elif isinstance(pipe_spec, PipeExtractSpec):
- # Convert extract_talent to model preset using static mappings
- model_preset = EXTRACT_TALENT_TO_MODEL.get(pipe_spec.extract_talent, "@default-extract-document")
- pipe_table.add("model", model_preset)
+ if pipe_spec.model:
+ pipe_table.add("model", pipe_spec.model)
elif isinstance(pipe_spec, PipeImgGenSpec):
- # Convert img_gen_talent to model preset using static mappings
- model_preset = IMG_GEN_TALENT_TO_MODEL.get(pipe_spec.img_gen_talent, "$gen-image")
- pipe_table.add("model", model_preset)
+ if pipe_spec.model:
+ pipe_table.add("model", pipe_spec.model)
pipe_table.add("prompt", format_toml_string(pipe_spec.prompt))
elif isinstance(pipe_spec, PipeFuncSpec):
pipe_table.add("function_name", pipe_spec.function_name)
-def _resolve_talent_value(pipe_type: str, raw_value: Any) -> Any:
- """Attempt to resolve a talent value, accepting preset names as aliases."""
- if not isinstance(raw_value, str):
- return raw_value
- reverse_maps: dict[str, dict[str, str]] = {
- "PipeLLM": _MODEL_TO_LLM_TALENT,
- "PipeImgGen": _MODEL_TO_IMG_GEN_TALENT,
- "PipeExtract": _MODEL_TO_EXTRACT_TALENT,
- }
- reverse_map = reverse_maps.get(pipe_type)
- if not reverse_map:
- return raw_value
- # If it's already a valid talent, return as-is
- # Otherwise try reverse-mapping from preset
- resolved = reverse_map.get(raw_value)
- if resolved:
- return resolved
- return raw_value # Let Pydantic validation handle the error
-
-
def _parse_pipe_spec_from_json(pipe_type: str, spec_data: dict[str, Any]) -> PipeSpec:
"""Parse and validate a PipeSpec from JSON data.
@@ -264,23 +187,6 @@ def _parse_pipe_spec_from_json(pipe_type: str, spec_data: dict[str, Any]) -> Pip
else:
spec_data.pop(alias)
- # Accept generic talent aliases and map to the type-specific field
- talent_aliases = ("talent_name", "talent")
- pipe_type_to_talent_field: dict[str, str] = {
- "PipeLLM": "llm_talent",
- "PipeExtract": "extract_talent",
- "PipeImgGen": "img_gen_talent",
- "PipeSearch": "search_talent",
- }
- talent_field = pipe_type_to_talent_field.get(pipe_type)
- if talent_field:
- for alias in talent_aliases:
- if alias in spec_data:
- if talent_field not in spec_data:
- spec_data[talent_field] = spec_data.pop(alias)
- else:
- spec_data.pop(alias)
-
# Handle steps/branches conversion - need to convert pipe to pipe_code
if "steps" in spec_data:
converted_steps = []
@@ -305,11 +211,6 @@ def _parse_pipe_spec_from_json(pipe_type: str, spec_data: dict[str, Any]) -> Pip
else:
spec_data.pop("expression")
- # Resolve preset names to talent names (tolerance for agent mistakes)
- talent_field = pipe_type_to_talent_field.get(pipe_type)
- if talent_field and talent_field in spec_data:
- spec_data[talent_field] = _resolve_talent_value(pipe_type, spec_data[talent_field])
-
# Accept output as dict with "type" key → extract the type string
if "output" in spec_data and isinstance(spec_data["output"], dict):
if "type" in spec_data["output"]:
@@ -347,7 +248,7 @@ def pipe_cmd(
"description": "What the pipe does",
"inputs": {"input_name": "ConceptName"},
"output": "OutputConcept",
- "llm_talent": "creative-writer",
+ "model": "$writing-creative",
"prompt": "Your prompt with @block and $inline vars"
}
@@ -411,8 +312,7 @@ def pipe_cmd(
except ValidationError as exc:
message, details = format_pydantic_validation_error_for_agent(exc)
- field_hints = PIPE_TYPE_TALENT_HINTS.get(resolved_pipe_type, {})
- agent_error(message, "ValidationError", cause=exc, validation_details=details, field_hints=field_hints)
+ agent_error(message, "ValidationError", cause=exc, validation_details=details)
except ValueError as exc:
agent_error(str(exc), "ValueError", cause=exc)
diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.py
index 3bd0fb795..c5654f102 100644
--- a/pipelex/cogt/exceptions.py
+++ b/pipelex/cogt/exceptions.py
@@ -47,14 +47,28 @@ class ModelChoiceNotFoundError(CogtError):
model_choice: str,
reference_kind: ModelReferenceKind | None = None,
available_options: list[str] | None = None,
+ suggestions: list[str] | None = None,
+ wrong_sigil_hints: list[str] | None = None,
+ cross_collection_suggestions: list[str] | None = None,
):
self.model_type = model_type
self.model_choice = model_choice
self.reference_kind = reference_kind
self.available_options = available_options or []
+ self.suggestions = suggestions or []
+ self.wrong_sigil_hints = wrong_sigil_hints or []
+ self.cross_collection_suggestions = cross_collection_suggestions or []
full_message = message
- if self.available_options:
+
+ all_suggestions = self.suggestions + self.cross_collection_suggestions
+ if all_suggestions:
+ full_message += "\n\nDid you mean: " + ", ".join(all_suggestions)
+
+ for hint in self.wrong_sigil_hints:
+ full_message += f"\nNote: {hint}"
+
+ if not all_suggestions and not self.wrong_sigil_hints and self.available_options:
options_str = ", ".join(sorted(self.available_options)[:10])
if len(self.available_options) > 10:
options_str += f" ... and {len(self.available_options) - 10} more"
diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.py
index e07019e74..05af028bc 100644
--- a/pipelex/cogt/models/model_deck_check.py
+++ b/pipelex/cogt/models/model_deck_check.py
@@ -1,14 +1,43 @@
+"""Model choice validation against the model deck with fuzzy suggestions."""
+
+from typing import NoReturn
+
from pipelex.cogt.exceptions import ModelChoiceNotFoundError
from pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSetting
from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting
from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting
from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.cogt.models.model_deck import ModelDeck
from pipelex.cogt.models.model_reference import ModelReferenceKind, ensure_model_reference
+from pipelex.cogt.models.model_suggestion import suggest_model_alternatives
from pipelex.cogt.search.search_setting import SearchModelChoice, SearchSetting
from pipelex.hub import get_model_deck
-def check_llm_choice_with_deck(llm_choice: LLMModelChoice):
+def _raise_model_choice_not_found(
+ msg: str,
+ model_deck: ModelDeck,
+ model_type: ModelType,
+ raw_choice: str,
+ name: str,
+ reference_kind: ModelReferenceKind,
+ available_options: list[str],
+) -> NoReturn:
+ """Compute fuzzy suggestions and raise ModelChoiceNotFoundError."""
+ suggestions, wrong_sigil_hints, cross_suggestions = suggest_model_alternatives(model_deck, model_type, name, reference_kind)
+ raise ModelChoiceNotFoundError(
+ message=msg,
+ model_type=model_type,
+ model_choice=raw_choice,
+ reference_kind=reference_kind,
+ available_options=available_options,
+ suggestions=suggestions,
+ wrong_sigil_hints=wrong_sigil_hints,
+ cross_collection_suggestions=cross_suggestions,
+ )
+
+
+def check_llm_choice_with_deck(llm_choice: LLMModelChoice) -> None:
if isinstance(llm_choice, LLMSetting):
return
@@ -20,49 +49,45 @@ def check_llm_choice_with_deck(llm_choice: LLMModelChoice):
if ref.name in model_deck.llm_presets:
return
msg = f"LLM preset '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.LLM,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.PRESET,
- available_options=list(model_deck.llm_presets.keys()),
+ _raise_model_choice_not_found(
+ msg, model_deck, ModelType.LLM, ref.raw, ref.name, ModelReferenceKind.PRESET, list(model_deck.llm_presets.keys())
)
case ModelReferenceKind.ALIAS:
if ref.name in model_deck.llm_aliases:
return
msg = f"Alias '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.LLM,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.ALIAS,
- available_options=list(model_deck.llm_aliases.keys()),
+ _raise_model_choice_not_found(
+ msg, model_deck, ModelType.LLM, ref.raw, ref.name, ModelReferenceKind.ALIAS, list(model_deck.llm_aliases.keys())
)
case ModelReferenceKind.WATERFALL:
if ref.name in model_deck.llm_waterfalls:
return
msg = f"Waterfall '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.LLM,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.WATERFALL,
- available_options=list(model_deck.llm_waterfalls.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.LLM,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.WATERFALL,
+ list(model_deck.llm_waterfalls.keys()),
)
case ModelReferenceKind.HANDLE:
if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.LLM):
return
msg = f"Model handle '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.LLM,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.HANDLE,
- available_options=list(model_deck.inference_models.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.LLM,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.HANDLE,
+ list(model_deck.inference_models.keys()),
)
-def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):
+def check_extract_choice_with_deck(extract_choice: ExtractModelChoice) -> None:
if isinstance(extract_choice, ExtractSetting):
return
@@ -74,49 +99,57 @@ def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):
if ref.name in model_deck.extract_presets:
return
msg = f"Extract preset '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.TEXT_EXTRACTOR,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.PRESET,
- available_options=list(model_deck.extract_presets.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.TEXT_EXTRACTOR,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.PRESET,
+ list(model_deck.extract_presets.keys()),
)
case ModelReferenceKind.ALIAS:
if ref.name in model_deck.extract_aliases:
return
msg = f"Alias '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.TEXT_EXTRACTOR,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.ALIAS,
- available_options=list(model_deck.extract_aliases.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.TEXT_EXTRACTOR,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.ALIAS,
+ list(model_deck.extract_aliases.keys()),
)
case ModelReferenceKind.WATERFALL:
if ref.name in model_deck.extract_waterfalls:
return
msg = f"Waterfall '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.TEXT_EXTRACTOR,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.WATERFALL,
- available_options=list(model_deck.extract_waterfalls.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.TEXT_EXTRACTOR,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.WATERFALL,
+ list(model_deck.extract_waterfalls.keys()),
)
case ModelReferenceKind.HANDLE:
if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.TEXT_EXTRACTOR):
return
msg = f"Model handle '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.TEXT_EXTRACTOR,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.HANDLE,
- available_options=list(model_deck.inference_models.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.TEXT_EXTRACTOR,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.HANDLE,
+ list(model_deck.inference_models.keys()),
)
-def check_search_choice_with_deck(search_choice: SearchModelChoice):
+def check_search_choice_with_deck(search_choice: SearchModelChoice) -> None:
if isinstance(search_choice, SearchSetting):
return
@@ -128,49 +161,57 @@ def check_search_choice_with_deck(search_choice: SearchModelChoice):
if ref.name in model_deck.search_presets:
return
msg = f"Search preset '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.SEARCH,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.PRESET,
- available_options=list(model_deck.search_presets.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.SEARCH,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.PRESET,
+ list(model_deck.search_presets.keys()),
)
case ModelReferenceKind.ALIAS:
if ref.name in model_deck.search_aliases:
return
msg = f"Alias '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.SEARCH,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.ALIAS,
- available_options=list(model_deck.search_aliases.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.SEARCH,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.ALIAS,
+ list(model_deck.search_aliases.keys()),
)
case ModelReferenceKind.WATERFALL:
if ref.name in model_deck.search_waterfalls:
return
msg = f"Waterfall '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.SEARCH,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.WATERFALL,
- available_options=list(model_deck.search_waterfalls.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.SEARCH,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.WATERFALL,
+ list(model_deck.search_waterfalls.keys()),
)
case ModelReferenceKind.HANDLE:
if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.SEARCH):
return
msg = f"Model handle '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.SEARCH,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.HANDLE,
- available_options=list(model_deck.inference_models.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.SEARCH,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.HANDLE,
+ list(model_deck.inference_models.keys()),
)
-def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):
+def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice) -> None:
if isinstance(img_gen_choice, ImgGenSetting):
return
@@ -182,43 +223,51 @@ def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):
if ref.name in model_deck.img_gen_presets:
return
msg = f"Image generation preset '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.IMG_GEN,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.PRESET,
- available_options=list(model_deck.img_gen_presets.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.IMG_GEN,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.PRESET,
+ list(model_deck.img_gen_presets.keys()),
)
case ModelReferenceKind.ALIAS:
if ref.name in model_deck.img_gen_aliases:
return
msg = f"Alias '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.IMG_GEN,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.ALIAS,
- available_options=list(model_deck.img_gen_aliases.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.IMG_GEN,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.ALIAS,
+ list(model_deck.img_gen_aliases.keys()),
)
case ModelReferenceKind.WATERFALL:
if ref.name in model_deck.img_gen_waterfalls:
return
msg = f"Waterfall '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.IMG_GEN,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.WATERFALL,
- available_options=list(model_deck.img_gen_waterfalls.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.IMG_GEN,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.WATERFALL,
+ list(model_deck.img_gen_waterfalls.keys()),
)
case ModelReferenceKind.HANDLE:
if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.IMG_GEN):
return
msg = f"Model handle '{ref.name}' was not found in the model deck"
- raise ModelChoiceNotFoundError(
- message=msg,
- model_type=ModelType.IMG_GEN,
- model_choice=ref.raw,
- reference_kind=ModelReferenceKind.HANDLE,
- available_options=list(model_deck.inference_models.keys()),
+ _raise_model_choice_not_found(
+ msg,
+ model_deck,
+ ModelType.IMG_GEN,
+ ref.raw,
+ ref.name,
+ ModelReferenceKind.HANDLE,
+ list(model_deck.inference_models.keys()),
)
diff --git a/pipelex/cogt/models/model_suggestion.py b/pipelex/cogt/models/model_suggestion.py
new file mode 100644
index 000000000..b37fa8247
--- /dev/null
+++ b/pipelex/cogt/models/model_suggestion.py
@@ -0,0 +1,113 @@
+"""Fuzzy model name matching and cross-collection suggestion utilities.
+
+Provides functions to find close matches for model names and detect wrong-sigil usage.
+Used by both the CLI check-model command and the model deck validation layer.
+"""
+
+import difflib
+
+from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.cogt.models.model_deck import ModelDeck
+from pipelex.cogt.models.model_reference import (
+ SIGIL_ALIAS,
+ SIGIL_PRESET,
+ SIGIL_WATERFALL,
+ ModelReferenceKind,
+)
+
+KIND_SIGILS: dict[ModelReferenceKind, str] = {
+ ModelReferenceKind.PRESET: SIGIL_PRESET,
+ ModelReferenceKind.ALIAS: SIGIL_ALIAS,
+ ModelReferenceKind.WATERFALL: SIGIL_WATERFALL,
+ ModelReferenceKind.HANDLE: "",
+}
+
+KIND_LABELS: dict[ModelReferenceKind, str] = {
+ ModelReferenceKind.PRESET: "preset",
+ ModelReferenceKind.ALIAS: "alias",
+ ModelReferenceKind.WATERFALL: "waterfall",
+ ModelReferenceKind.HANDLE: "handle",
+}
+
+
+def get_collection_keys(
+ model_deck: ModelDeck,
+ model_type: ModelType,
+ kind: ModelReferenceKind,
+) -> list[str]:
+ """Return known names for a given reference kind and model type."""
+ match kind:
+ case ModelReferenceKind.PRESET:
+ match model_type:
+ case ModelType.LLM:
+ return list(model_deck.llm_presets.keys())
+ case ModelType.TEXT_EXTRACTOR:
+ return list(model_deck.extract_presets.keys())
+ case ModelType.IMG_GEN:
+ return list(model_deck.img_gen_presets.keys())
+ case ModelType.SEARCH:
+ return list(model_deck.search_presets.keys())
+ case ModelReferenceKind.ALIAS:
+ match model_type:
+ case ModelType.LLM:
+ return list(model_deck.llm_aliases.keys())
+ case ModelType.TEXT_EXTRACTOR:
+ return list(model_deck.extract_aliases.keys())
+ case ModelType.IMG_GEN:
+ return list(model_deck.img_gen_aliases.keys())
+ case ModelType.SEARCH:
+ return list(model_deck.search_aliases.keys())
+ case ModelReferenceKind.WATERFALL:
+ match model_type:
+ case ModelType.LLM:
+ return list(model_deck.llm_waterfalls.keys())
+ case ModelType.TEXT_EXTRACTOR:
+ return list(model_deck.extract_waterfalls.keys())
+ case ModelType.IMG_GEN:
+ return list(model_deck.img_gen_waterfalls.keys())
+ case ModelType.SEARCH:
+ return list(model_deck.search_waterfalls.keys())
+ case ModelReferenceKind.HANDLE:
+ return sorted(handle for handle, spec in model_deck.inference_models.items() if spec.model_type == model_type)
+
+
+def suggest_model_alternatives(
+ model_deck: ModelDeck,
+ model_type: ModelType,
+ name: str,
+ kind: ModelReferenceKind,
+) -> tuple[list[str], list[str], list[str]]:
+ """Find fuzzy matches and detect wrong-sigil usage for a model name.
+
+ Args:
+ model_deck: The model deck to search in.
+ model_type: The model type (LLM, TEXT_EXTRACTOR, IMG_GEN, SEARCH).
+ name: The bare name (without sigil) to match.
+ kind: The reference kind the user specified (PRESET, ALIAS, WATERFALL, HANDLE).
+
+ Returns:
+ A tuple of three lists:
+ - suggestions: fuzzy matches within the same collection, with sigil prefix
+ - wrong_sigil_hints: exact matches found in other collections (e.g. "best-claude exists as @best-claude (alias)")
+ - cross_collection_suggestions: fuzzy matches in other collections, with sigil and label
+ """
+ sigil = KIND_SIGILS[kind]
+ candidates = get_collection_keys(model_deck, model_type, kind)
+ fuzzy_matches = difflib.get_close_matches(name, candidates, n=5, cutoff=0.5)
+ suggestions = [f"{sigil}{match}" for match in fuzzy_matches]
+
+ wrong_sigil_hints: list[str] = []
+ cross_suggestions: list[str] = []
+ other_kinds = [other_kind for other_kind in ModelReferenceKind if other_kind != kind]
+ for other_kind in other_kinds:
+ other_candidates = get_collection_keys(model_deck, model_type, other_kind)
+ other_sigil = KIND_SIGILS[other_kind]
+ label = KIND_LABELS[other_kind]
+ if name in other_candidates:
+ wrong_sigil_hints.append(f"{name} exists as {other_sigil}{name} ({label})")
+ else:
+ fuzzy = difflib.get_close_matches(name, other_candidates, n=3, cutoff=0.7)
+ for match in fuzzy:
+ cross_suggestions.append(f"{other_sigil}{match} ({label})")
+
+ return suggestions, wrong_sigil_hints, cross_suggestions
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index be0f5fcc5..99e07e7cb 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -440,7 +440,7 @@ class AssemblePipelexBundleSpecTestCases:
description="Generate a plan from a user brief.",
inputs={"brief": "UserBrief"},
output="PlanDraft",
- llm_talent="engineer",
+ model="$engineering-structured",
prompt="Generate a plan for: @brief",
),
]
diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py
index 3dfd19c7b..5513a0037 100644
--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py
+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py
@@ -1,7 +1,6 @@
from typing import ClassVar
from pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpec
-from pipelex.builder.talents.extract_talent import ExtractTalent
from pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprint
@@ -13,7 +12,7 @@ class PipeExtractTestCases:
description="Extract text from image",
inputs={"image": "Image"},
output="Page",
- extract_talent=ExtractTalent.PDF_BASIC_TEXT_EXTRACTOR,
+ model="@default-text-from-pdf",
),
PipeExtractBlueprint(
source=None,
@@ -31,7 +30,7 @@ class PipeExtractTestCases:
description="Extract with page options",
inputs={"document": "Document"},
output="Page[]",
- extract_talent=ExtractTalent.PDF_BASIC_TEXT_EXTRACTOR,
+ model="@default-text-from-pdf",
max_page_images=None,
page_views=True,
),
diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py
index e94a35119..118a2d569 100644
--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py
+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py
@@ -1,7 +1,6 @@
from typing import ClassVar
from pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec
-from pipelex.builder.talents.img_gen_talent import ImgGenTalent
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
@@ -13,7 +12,7 @@ class PipeImgGenTestCases:
description="Generate an image",
inputs={"my_prompt": "Text"},
output="native.Image",
- img_gen_talent=ImgGenTalent.GEN_IMAGE,
+ model="$gen-image",
prompt="@my_prompt",
),
PipeImgGenBlueprint(
@@ -38,7 +37,7 @@ class PipeImgGenTestCases:
inputs={"description": "Text"},
output="Image[3]",
prompt="@description",
- img_gen_talent=ImgGenTalent.GEN_IMAGE_FAST,
+ model="$gen-image-fast",
),
PipeImgGenBlueprint(
description="Generate image with options",
diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py
index 83939dcf2..4d643ee4d 100644
--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py
+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py
@@ -1,7 +1,6 @@
from typing import ClassVar
from pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpec
-from pipelex.builder.talents.llm_talent import LLMTalent
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
@@ -13,7 +12,7 @@ class PipeLLMTestCases:
description="Generate text",
inputs={"topic": "Text"},
output="Text",
- llm_talent=LLMTalent.CREATIVE_WRITER,
+ model="$writing-creative",
prompt="Write about $topic",
),
PipeLLMBlueprint(
@@ -34,7 +33,7 @@ class PipeLLMTestCases:
description="Generate without inputs",
inputs={},
output="Text",
- llm_talent=LLMTalent.CREATIVE_WRITER,
+ model="$writing-creative",
prompt="Generate something interesting",
),
PipeLLMBlueprint(
@@ -55,7 +54,7 @@ class PipeLLMTestCases:
inputs={"image": "Image"},
output="Text",
prompt="Analyze the image: $image",
- llm_talent=LLMTalent.VISION_LANGUAGE_MODEL,
+ model="$vision",
),
PipeLLMBlueprint(
source=None,
@@ -76,7 +75,7 @@ class PipeLLMTestCases:
inputs={},
output="Text",
prompt="Generate text",
- llm_talent=LLMTalent.CREATIVE_WRITER,
+ model="$writing-creative",
),
PipeLLMBlueprint(
source=None,
@@ -96,7 +95,7 @@ class PipeLLMTestCases:
inputs={},
output="Text",
prompt="Generate text",
- llm_talent=LLMTalent.DATA_RETRIEVAL,
+ model="$retrieval",
),
PipeLLMBlueprint(
source=None,
@@ -117,7 +116,7 @@ class PipeLLMTestCases:
output="Analysis",
system_prompt="You are a data analyst",
prompt="Analyze: @data",
- llm_talent=LLMTalent.DATA_RETRIEVAL,
+ model="$retrieval",
),
PipeLLMBlueprint(
source=None,
@@ -139,7 +138,7 @@ class PipeLLMTestCases:
inputs={},
output="Item[]",
prompt="Generate items",
- llm_talent=LLMTalent.CREATIVE_WRITER,
+ model="$writing-creative",
),
PipeLLMBlueprint(
source=None,
@@ -159,7 +158,7 @@ class PipeLLMTestCases:
inputs={},
output="Item[5]",
prompt="Generate items",
- llm_talent=LLMTalent.CREATIVE_WRITER,
+ model="$writing-creative",
),
PipeLLMBlueprint(
source=None,
diff --git a/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py b/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py
index ffb8b6f44..cfa203282 100644
--- a/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py
+++ b/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py
@@ -301,7 +301,7 @@ class TestBuilderLoopMultiplicityFix:
description="An LLM pipe",
inputs={"input_text": "Text"},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Do something with @input_text",
),
},
diff --git a/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py b/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py
index 7d50249e2..4fbabbdbc 100644
--- a/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py
+++ b/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py
@@ -59,7 +59,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Do stuff",
),
},
@@ -83,7 +83,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Do stuff",
),
},
@@ -107,7 +107,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Do stuff",
),
},
@@ -132,7 +132,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Do stuff",
),
},
@@ -156,7 +156,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Do stuff",
),
},
@@ -193,7 +193,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={"item": "Text"},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Process @item",
),
},
@@ -232,7 +232,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Step 1",
),
"step_two": PipeLLMSpec.model_construct(
@@ -242,7 +242,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Step 2",
),
},
@@ -284,7 +284,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Branch A",
),
"branch_b": PipeLLMSpec.model_construct(
@@ -294,7 +294,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Branch B",
),
},
@@ -336,7 +336,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Good",
),
"handle_bad": PipeLLMSpec.model_construct(
@@ -346,7 +346,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Bad",
),
"handle_default": PipeLLMSpec.model_construct(
@@ -356,7 +356,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Default",
),
},
@@ -398,7 +398,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Good",
),
},
@@ -440,7 +440,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Local",
),
},
@@ -482,7 +482,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Local",
),
},
@@ -552,7 +552,7 @@ class TestBuilderLoopNamespaceStripping:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Step 1",
),
},
diff --git a/tests/unit/pipelex/builder/test_builder_loop_prune.py b/tests/unit/pipelex/builder/test_builder_loop_prune.py
index 3ff4f8ea3..bd529e7e0 100644
--- a/tests/unit/pipelex/builder/test_builder_loop_prune.py
+++ b/tests/unit/pipelex/builder/test_builder_loop_prune.py
@@ -72,7 +72,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={"doc": "external.Document"},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Process @doc",
),
},
@@ -106,7 +106,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="my_domain.Report",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Generate a report",
),
},
@@ -140,7 +140,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="Summary",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Generate a summary",
),
},
@@ -173,7 +173,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="Result",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Generate a result",
),
},
@@ -214,7 +214,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="Derived",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Generate derived content",
),
},
@@ -253,7 +253,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="Container",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Generate container",
),
},
@@ -315,7 +315,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Generate text",
),
},
@@ -351,7 +351,7 @@ class TestBuilderLoopPruneUnreachableSpecs:
pipe_category="PipeOperator",
inputs={},
output="Text",
- llm_talent="data-retrieval",
+ model="$retrieval",
prompt="Orphan",
),
},
diff --git a/tests/unit/pipelex/cli/test_agent_models_cmd.py b/tests/unit/pipelex/cli/test_agent_models_cmd.py
index 56b9c04c8..a8ae8eeea 100644
--- a/tests/unit/pipelex/cli/test_agent_models_cmd.py
+++ b/tests/unit/pipelex/cli/test_agent_models_cmd.py
@@ -24,31 +24,6 @@ class _FakeSetting:
self.description = description
-class _FakeTalentMappings:
- """Minimal stand-in for TalentPresetMappings."""
-
- def __init__(self, llm: dict[str, str], img_gen: dict[str, str], extract: dict[str, str], search: dict[str, str] | None = None):
- self.llm = llm
- self.img_gen = img_gen
- self.extract = extract
- self.search = search or {}
-
-
-class _FakeBuilderConfig:
- def __init__(self, talent_preset_mappings: _FakeTalentMappings):
- self.talent_preset_mappings = talent_preset_mappings
-
-
-class _FakePipelexConfig:
- def __init__(self, builder_config: _FakeBuilderConfig):
- self.builder_config = builder_config
-
-
-class _FakeConfig:
- def __init__(self, pipelex: _FakePipelexConfig):
- self.pipelex = pipelex
-
-
class _FakeInferenceModelSpec:
"""Minimal stand-in for InferenceModelSpec."""
@@ -79,11 +54,6 @@ class TestData:
EXTRACT_WATERFALLS: ClassVar[dict[str, list[str]]] = {"extract-wf": ["textract-model"]}
IMG_GEN_WATERFALLS: ClassVar[dict[str, list[str]]] = {"img-wf": ["dall-e-3"]}
- TALENT_LLM: ClassVar[dict[str, str]] = {"text_gen": "fast"}
- TALENT_IMG_GEN: ClassVar[dict[str, str]] = {"image_gen": "hd-image"}
- TALENT_EXTRACT: ClassVar[dict[str, str]] = {"extraction": "doc-extract"}
-
- # Backend resolution map: model_handle -> (backend_name, model_type)
INFERENCE_MAP: ClassVar[dict[str, tuple[str, ModelType]]] = {
"gpt-4o-mini": ("openai", ModelType.LLM),
"claude-sonnet": ("anthropic", ModelType.LLM),
@@ -123,25 +93,10 @@ def _make_fake_model_deck() -> Any:
return FakeModelDeck()
-def _make_fake_config() -> _FakeConfig:
- """Create a fake config with talent mappings."""
- talent_mappings = _FakeTalentMappings(
- llm=TestData.TALENT_LLM,
- img_gen=TestData.TALENT_IMG_GEN,
- extract=TestData.TALENT_EXTRACT,
- )
- return _FakeConfig(
- pipelex=_FakePipelexConfig(
- builder_config=_FakeBuilderConfig(talent_preset_mappings=talent_mappings),
- ),
- )
-
-
def _setup_mocks(mocker: MockerFixture) -> None:
"""Patch the common dependencies for agent_models_cmd."""
mocker.patch(f"{MODULE_PATH}.make_pipelex_for_agent_cli")
mocker.patch(f"{MODULE_PATH}.get_model_deck", return_value=_make_fake_model_deck())
- mocker.patch(f"{MODULE_PATH}.get_config", return_value=_make_fake_config())
mocker.patch(f"{MODULE_PATH}.Pipelex")
@@ -149,18 +104,28 @@ class TestAgentModelsCmd:
"""Tests for agent_models_cmd JSON output with --type and --backend filters."""
def test_no_filters_returns_all_categories(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
- """No filters should return all 3 categories in every section."""
+ """No filters should return all categories in every section."""
_setup_mocks(mocker)
agent_models_cmd(ctx=agent_ctx, output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
assert parsed["success"] is True
- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):
+ for section in ("presets", "aliases", "waterfalls"):
assert "llm" in parsed[section], f"llm missing from {section}"
assert "img_gen" in parsed[section], f"img_gen missing from {section}"
assert "extract" in parsed[section], f"extract missing from {section}"
+ def test_no_talent_mappings_in_output(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """Output should not contain talent_mappings or usage hint."""
+ _setup_mocks(mocker)
+
+ agent_models_cmd(ctx=agent_ctx, output_format=CliOutputFormat.JSON)
+
+ parsed = json.loads(capsys.readouterr().out)
+ assert "talent_mappings" not in parsed
+ assert "talent_mappings_usage_hint" not in parsed
+
def test_no_filters_preset_content(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
"""No filters should include all preset entries with correct data."""
_setup_mocks(mocker)
@@ -171,7 +136,6 @@ class TestAgentModelsCmd:
llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]
assert "fast" in llm_preset_names
assert "smart" in llm_preset_names
- # Check description is included
fast_preset = next(preset for preset in parsed["presets"]["llm"] if preset["name"] == "fast")
assert fast_preset["description"] == "Fast LLM"
@@ -182,7 +146,7 @@ class TestAgentModelsCmd:
agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.LLM], output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):
+ for section in ("presets", "aliases", "waterfalls"):
assert "llm" in parsed[section], f"llm missing from {section}"
assert "img_gen" not in parsed[section], f"img_gen should not be in {section}"
assert "extract" not in parsed[section], f"extract should not be in {section}"
@@ -194,7 +158,7 @@ class TestAgentModelsCmd:
agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.EXTRACT], output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):
+ for section in ("presets", "aliases", "waterfalls"):
assert "extract" in parsed[section], f"extract missing from {section}"
assert "llm" not in parsed[section], f"llm should not be in {section}"
assert "img_gen" not in parsed[section], f"img_gen should not be in {section}"
@@ -206,7 +170,7 @@ class TestAgentModelsCmd:
agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.IMG_GEN], output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):
+ for section in ("presets", "aliases", "waterfalls"):
assert "img_gen" in parsed[section], f"img_gen missing from {section}"
assert "llm" not in parsed[section], f"llm should not be in {section}"
assert "extract" not in parsed[section], f"extract should not be in {section}"
@@ -218,7 +182,7 @@ class TestAgentModelsCmd:
agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.LLM, ModelCategory.IMG_GEN], output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):
+ for section in ("presets", "aliases", "waterfalls"):
assert "llm" in parsed[section], f"llm missing from {section}"
assert "img_gen" in parsed[section], f"img_gen missing from {section}"
assert "extract" not in parsed[section], f"extract should not be in {section}"
@@ -230,16 +194,12 @@ class TestAgentModelsCmd:
agent_models_cmd(ctx=agent_ctx, backend="openai", output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
- # LLM: only 'fast' preset (gpt-4o-mini on openai), not 'smart' (claude-sonnet on anthropic)
llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]
assert "fast" in llm_preset_names
assert "smart" not in llm_preset_names
- # LLM aliases: best-llm -> claude-sonnet (anthropic), should be excluded
assert len(parsed["aliases"]["llm"]) == 0
- # IMG_GEN: dall-e-3 is on openai, should be included
img_gen_preset_names = [preset["name"] for preset in parsed["presets"]["img_gen"]]
assert "hd-image" in img_gen_preset_names
- # Extract: textract-model is on aws, should be excluded
assert len(parsed["presets"]["extract"]) == 0
def test_backend_filter_anthropic(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
@@ -252,7 +212,6 @@ class TestAgentModelsCmd:
llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]
assert "smart" in llm_preset_names
assert "fast" not in llm_preset_names
- # Alias best-llm -> claude-sonnet (anthropic) should be included
assert "best-llm" in parsed["aliases"]["llm"]
def test_type_and_backend_combined(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
@@ -262,24 +221,22 @@ class TestAgentModelsCmd:
agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.LLM], backend="openai", output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
- # Only llm category
assert "llm" in parsed["presets"]
assert "img_gen" not in parsed["presets"]
assert "extract" not in parsed["presets"]
- # Only openai-backed LLM presets
llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]
assert "fast" in llm_preset_names
assert "smart" not in llm_preset_names
def test_backend_nonexistent(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
- """--backend nonexistent should produce empty presets/aliases/waterfalls/talent_mappings."""
+ """--backend nonexistent should produce empty presets/aliases/waterfalls."""
_setup_mocks(mocker)
agent_models_cmd(ctx=agent_ctx, backend="nonexistent", output_format=CliOutputFormat.JSON)
parsed = json.loads(capsys.readouterr().out)
assert parsed["success"] is True
- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):
+ for section in ("presets", "aliases", "waterfalls"):
for category in ("llm", "img_gen", "extract"):
assert len(parsed[section][category]) == 0, f"{section}.{category} should be empty"
@@ -293,14 +250,3 @@ class TestAgentModelsCmd:
assert "llm-wf" in parsed["waterfalls"]["llm"]
assert len(parsed["waterfalls"]["img_gen"]) == 0
assert len(parsed["waterfalls"]["extract"]) == 0
-
- def test_talent_mappings_backend_filter(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
- """--backend openai should include LLM talent_mapping (fast->gpt-4o-mini) and img_gen, but not extract."""
- _setup_mocks(mocker)
-
- agent_models_cmd(ctx=agent_ctx, backend="openai", output_format=CliOutputFormat.JSON)
-
- parsed = json.loads(capsys.readouterr().out)
- assert "text_gen" in parsed["talent_mappings"]["llm"]
- assert "image_gen" in parsed["talent_mappings"]["img_gen"]
- assert len(parsed["talent_mappings"]["extract"]) == 0
diff --git a/tests/unit/pipelex/cli/test_check_model_cmd.py b/tests/unit/pipelex/cli/test_check_model_cmd.py
new file mode 100644
index 000000000..11080664a
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_check_model_cmd.py
@@ -0,0 +1,194 @@
+"""Unit tests for the agent CLI check-model command."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any, ClassVar
+
+if TYPE_CHECKING:
+ import pytest
+ from pytest_mock import MockerFixture
+
+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat
+from pipelex.cli.agent_cli.commands.check_model_cmd import agent_check_model_cmd
+from pipelex.cli.agent_cli.commands.models_cmd import ModelCategory
+from pipelex.cogt.model_backends.model_type import ModelType
+
+MODULE_PATH = "pipelex.cli.agent_cli.commands.check_model_cmd"
+
+
+class _FakeSetting:
+ """Minimal stand-in for LLMSetting / ExtractSetting."""
+
+ def __init__(self, model: str, description: str | None = None):
+ self.model = model
+ self.description = description
+
+
+class _FakeInferenceModelSpec:
+ """Minimal stand-in for InferenceModelSpec."""
+
+ def __init__(self, model_type: ModelType):
+ self.model_type = model_type
+
+
+class TestData:
+ """Shared test data constants."""
+
+ LLM_PRESETS: ClassVar[dict[str, _FakeSetting]] = {
+ "writing-creative": _FakeSetting(model="claude-sonnet", description="Creative writing"),
+ "writing-factual": _FakeSetting(model="claude-sonnet", description="Factual writing"),
+ "deep-analysis": _FakeSetting(model="claude-opus", description="Deep analysis"),
+ }
+ LLM_ALIASES: ClassVar[dict[str, str]] = {
+ "best-claude": "claude-sonnet",
+ "best-gpt": "gpt-4o",
+ "default-general": "claude-sonnet",
+ }
+ LLM_WATERFALLS: ClassVar[dict[str, list[str]]] = {
+ "robust-llm": ["claude-sonnet", "gpt-4o"],
+ }
+ INFERENCE_MODELS: ClassVar[dict[str, _FakeInferenceModelSpec]] = {
+ "claude-4.5-sonnet": _FakeInferenceModelSpec(model_type=ModelType.LLM),
+ "claude-4-sonnet": _FakeInferenceModelSpec(model_type=ModelType.LLM),
+ "claude-4.6-opus": _FakeInferenceModelSpec(model_type=ModelType.LLM),
+ "gpt-4o": _FakeInferenceModelSpec(model_type=ModelType.LLM),
+ "gpt-4o-mini": _FakeInferenceModelSpec(model_type=ModelType.LLM),
+ }
+
+ EXTRACT_PRESETS: ClassVar[dict[str, Any]] = {}
+ EXTRACT_ALIASES: ClassVar[dict[str, str]] = {}
+ EXTRACT_WATERFALLS: ClassVar[dict[str, list[str]]] = {}
+ IMG_GEN_PRESETS: ClassVar[dict[str, Any]] = {}
+ IMG_GEN_ALIASES: ClassVar[dict[str, str]] = {}
+ IMG_GEN_WATERFALLS: ClassVar[dict[str, list[str]]] = {}
+ SEARCH_PRESETS: ClassVar[dict[str, Any]] = {}
+ SEARCH_ALIASES: ClassVar[dict[str, str]] = {}
+ SEARCH_WATERFALLS: ClassVar[dict[str, list[str]]] = {}
+
+
+def _make_fake_model_deck() -> Any:
+ """Create a fake ModelDeck with LLM test data."""
+
+ class FakeModelDeck:
+ llm_presets = TestData.LLM_PRESETS
+ llm_aliases = TestData.LLM_ALIASES
+ llm_waterfalls = TestData.LLM_WATERFALLS
+ inference_models = TestData.INFERENCE_MODELS
+
+ extract_presets = TestData.EXTRACT_PRESETS
+ extract_aliases = TestData.EXTRACT_ALIASES
+ extract_waterfalls = TestData.EXTRACT_WATERFALLS
+ img_gen_presets = TestData.IMG_GEN_PRESETS
+ img_gen_aliases = TestData.IMG_GEN_ALIASES
+ img_gen_waterfalls = TestData.IMG_GEN_WATERFALLS
+ search_presets = TestData.SEARCH_PRESETS
+ search_aliases = TestData.SEARCH_ALIASES
+ search_waterfalls = TestData.SEARCH_WATERFALLS
+
+ return FakeModelDeck()
+
+
+def _setup_mocks(mocker: MockerFixture) -> None:
+ """Patch the common dependencies for agent_check_model_cmd."""
+ mocker.patch(f"{MODULE_PATH}.make_pipelex_for_agent_cli")
+ mocker.patch(f"{MODULE_PATH}.get_model_deck", return_value=_make_fake_model_deck())
+ mocker.patch(f"{MODULE_PATH}.Pipelex")
+
+
+def _run_check(
+ agent_ctx: Any,
+ mocker: MockerFixture,
+ capsys: pytest.CaptureFixture[str],
+ name: str,
+ model_type: ModelCategory = ModelCategory.LLM,
+ output_format: CliOutputFormat = CliOutputFormat.JSON,
+) -> dict[str, Any]:
+ """Run check-model and return parsed JSON output."""
+ _setup_mocks(mocker)
+ agent_check_model_cmd(ctx=agent_ctx, name=name, model_type=model_type, output_format=output_format)
+ parsed: dict[str, Any] = json.loads(capsys.readouterr().out)
+ return parsed
+
+
+class TestCheckModelCmd:
+ """Tests for agent_check_model_cmd validation and fuzzy matching."""
+
+ def test_valid_preset(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A known preset name with $ sigil should be valid."""
+ result = _run_check(agent_ctx, mocker, capsys, "$writing-creative")
+ assert result["valid"] is True
+ assert result["kind"] == "preset"
+
+ def test_valid_alias(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A known alias name with @ sigil should be valid."""
+ result = _run_check(agent_ctx, mocker, capsys, "@best-claude")
+ assert result["valid"] is True
+ assert result["kind"] == "alias"
+
+ def test_valid_handle(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A known bare model handle should be valid."""
+ result = _run_check(agent_ctx, mocker, capsys, "claude-4.5-sonnet")
+ assert result["valid"] is True
+ assert result["kind"] == "handle"
+
+ def test_invalid_preset_with_fuzzy_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A typo in a preset name should return fuzzy suggestions with $ prefix."""
+ result = _run_check(agent_ctx, mocker, capsys, "$writting-creative")
+ assert result["valid"] is False
+ assert any("$writing-creative" in suggestion for suggestion in result["suggestions"])
+
+ def test_wrong_sigil_detected(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """Using $ for a name that exists as an alias should produce a wrong-sigil hint."""
+ result = _run_check(agent_ctx, mocker, capsys, "$best-claude")
+ assert result["valid"] is False
+ assert any("@best-claude" in hint for hint in result["wrong_sigil_hints"])
+
+ def test_invalid_handle_cross_collection_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A bare name matching a preset should suggest the prefixed version."""
+ result = _run_check(agent_ctx, mocker, capsys, "writing-creative")
+ assert result["valid"] is False
+ assert any("$writing-creative" in hint for hint in result["wrong_sigil_hints"])
+
+ def test_invalid_handle_fuzzy_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A typo in a handle should return fuzzy suggestions without sigil prefix."""
+ result = _run_check(agent_ctx, mocker, capsys, "claude-4.5-sonet")
+ assert result["valid"] is False
+ assert "claude-4.5-sonnet" in result["suggestions"]
+
+ def test_completely_unknown_name(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A completely unknown name should return no suggestions."""
+ result = _run_check(agent_ctx, mocker, capsys, "$zzz-nonexistent-xyz")
+ assert result["valid"] is False
+ assert len(result["suggestions"]) == 0
+
+ def test_json_output_structure(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """JSON output should contain all expected keys on failure."""
+ result = _run_check(agent_ctx, mocker, capsys, "$nonexistent")
+ assert result["success"] is True
+ assert result["valid"] is False
+ assert "suggestions" in result
+ assert "wrong_sigil_hints" in result
+ assert "cross_collection_suggestions" in result
+ assert result["model_type"] == "llm"
+
+ def test_markdown_valid_output(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """Markdown output for a valid reference should be a single confirmation line."""
+ _setup_mocks(mocker)
+ agent_check_model_cmd(ctx=agent_ctx, name="$writing-creative", model_type=ModelCategory.LLM, output_format=CliOutputFormat.MARKDOWN)
+ output = capsys.readouterr().out.strip()
+ assert output == "$writing-creative is a valid llm preset."
+
+ def test_markdown_invalid_output_has_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """Markdown output for an invalid reference should include 'Did you mean' suggestions."""
+ _setup_mocks(mocker)
+ agent_check_model_cmd(ctx=agent_ctx, name="$writting-creative", model_type=ModelCategory.LLM, output_format=CliOutputFormat.MARKDOWN)
+ output = capsys.readouterr().out
+ assert "is not a valid" in output
+ assert "Did you mean:" in output
+
+ def test_valid_waterfall(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:
+ """A known waterfall name with ~ sigil should be valid."""
+ result = _run_check(agent_ctx, mocker, capsys, "~robust-llm")
+ assert result["valid"] is True
+ assert result["kind"] == "waterfall"
diff --git a/tests/unit/pipelex/cli/test_format_models_markdown.py b/tests/unit/pipelex/cli/test_format_models_markdown.py
index 1fa9365a0..e0bd9b53c 100644
--- a/tests/unit/pipelex/cli/test_format_models_markdown.py
+++ b/tests/unit/pipelex/cli/test_format_models_markdown.py
@@ -35,13 +35,6 @@ class TestFormatModelsMarkdown:
"extract": {},
"search": {},
},
- "talent_mappings": {
- "llm": {"creative-writer": "writing-creative"},
- "img_gen": {},
- "extract": {},
- "search": {},
- },
- "talent_mappings_usage_hint": "Use the talent name as the value for llm_talent",
}
def test_output_starts_with_heading(self) -> None:
@@ -54,35 +47,30 @@ class TestFormatModelsMarkdown:
output = _format_models_markdown(self._FULL_RESULT)
assert "## LLM" in output
- def test_preset_with_description(self) -> None:
- """Preset with description should show name and description."""
- output = _format_models_markdown(self._FULL_RESULT)
- assert "- **writing-creative**: Creative writing" in output
-
- def test_preset_without_description(self) -> None:
- """Preset without description should show only the name."""
+ def test_preset_with_description_uses_sigil(self) -> None:
+ """Preset with description should show $-prefixed name and description."""
output = _format_models_markdown(self._FULL_RESULT)
- assert "- **engineering-code**" in output
+ assert "- $writing-creative: Creative writing" in output
- def test_aliases_use_arrow(self) -> None:
- """Aliases should use arrow notation."""
+ def test_preset_without_description_uses_sigil(self) -> None:
+ """Preset without description should show only the $-prefixed name."""
output = _format_models_markdown(self._FULL_RESULT)
- assert "- **fast** \u2192 gpt4o-mini" in output
+ assert "- $engineering-code" in output
- def test_waterfalls_show_chain(self) -> None:
- """Waterfalls should show the full fallback chain with arrows."""
+ def test_aliases_use_sigil_and_arrow(self) -> None:
+ """Aliases should use @-prefix and arrow notation."""
output = _format_models_markdown(self._FULL_RESULT)
- assert "gpt4o \u2192 claude-sonnet \u2192 gemini" in output
+ assert "- @fast \u2192 gpt4o-mini" in output
- def test_talent_mappings_present(self) -> None:
- """Talent mappings should use arrow notation."""
+ def test_waterfalls_use_sigil_and_chain(self) -> None:
+ """Waterfalls should use ~-prefix and show the full fallback chain."""
output = _format_models_markdown(self._FULL_RESULT)
- assert "- **creative-writer** \u2192 writing-creative" in output
+ assert "- ~robust: gpt4o \u2192 claude-sonnet \u2192 gemini" in output
- def test_hint_present(self) -> None:
- """Usage hint should appear at the bottom."""
+ def test_no_bold_markers(self) -> None:
+ """Output should not contain bold markdown markers."""
output = _format_models_markdown(self._FULL_RESULT)
- assert "Use the talent name as the value for llm_talent" in output
+ assert "**" not in output
def test_empty_categories_omitted(self) -> None:
"""Categories with no data should not appear in output."""
@@ -91,24 +79,21 @@ class TestFormatModelsMarkdown:
assert "## Search" not in output
def test_img_gen_category_present(self) -> None:
- """Image Generation category with presets should appear."""
+ """Image Generation category with presets should appear with sigil."""
output = _format_models_markdown(self._FULL_RESULT)
assert "## Image Generation" in output
- assert "- **gen-image**: Image generation" in output
+ assert "- $gen-image: Image generation" in output
def test_all_empty_produces_heading_only(self) -> None:
- """Result with all empty sections should produce only the heading and hint."""
+ """Result with all empty sections should produce only the heading."""
empty_result: dict[str, Any] = {
"presets": {"llm": [], "img_gen": [], "extract": [], "search": []},
"aliases": {"llm": {}, "img_gen": {}, "extract": {}, "search": {}},
"waterfalls": {"llm": {}, "img_gen": {}, "extract": {}, "search": {}},
- "talent_mappings": {"llm": {}, "img_gen": {}, "extract": {}, "search": {}},
- "talent_mappings_usage_hint": "hint text",
}
output = _format_models_markdown(empty_result)
assert "# Available Models" in output
assert "## LLM" not in output
- assert "hint text" in output
def test_empty_subsections_omitted_within_category(self) -> None:
"""Within a category, subsections with no data should be omitted."""
@@ -116,10 +101,18 @@ class TestFormatModelsMarkdown:
"presets": {"llm": [{"name": "only-preset"}]},
"aliases": {"llm": {}},
"waterfalls": {"llm": {}},
- "talent_mappings": {"llm": {}},
}
output = _format_models_markdown(result)
assert "### Presets" in output
assert "### Aliases" not in output
assert "### Waterfalls" not in output
- assert "### Talent Mappings" not in output
+
+ def test_presets_description_line(self) -> None:
+ """Presets section should include the one-liner description."""
+ output = _format_models_markdown(self._FULL_RESULT)
+ assert "Presets are the preferred way to specify models" in output
+
+ def test_aliases_description_line(self) -> None:
+ """Aliases section should include the one-liner description."""
+ output = _format_models_markdown(self._FULL_RESULT)
+ assert "Aliases provide stable names" in output
diff --git a/tests/unit/pipelex/cli/test_pipe_cmd.py b/tests/unit/pipelex/cli/test_pipe_cmd.py
index 9b2659509..ffa9dae73 100644
--- a/tests/unit/pipelex/cli/test_pipe_cmd.py
+++ b/tests/unit/pipelex/cli/test_pipe_cmd.py
@@ -8,117 +8,10 @@ import pytest
from pydantic import ValidationError
from pipelex.cli.agent_cli.commands.pipe_cmd import (
- _MODEL_TO_EXTRACT_TALENT, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]
- _MODEL_TO_IMG_GEN_TALENT, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]
- _MODEL_TO_LLM_TALENT, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]
- LLM_TALENT_TO_MODEL,
_parse_pipe_spec_from_json, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]
_pipe_spec_to_toml, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]
- _resolve_talent_value, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]
)
-# ---------------------------------------------------------------------------
-# Reverse talent mappings
-# ---------------------------------------------------------------------------
-
-
-class TestReverseTalentMappings:
- """Verify the module-level reverse mappings are built correctly."""
-
- def test_llm_preset_with_prefix(self) -> None:
- assert _MODEL_TO_LLM_TALENT["$writing-creative"] == "creative-writer"
-
- def test_llm_preset_without_prefix(self) -> None:
- assert _MODEL_TO_LLM_TALENT["writing-creative"] == "creative-writer"
-
- def test_llm_all_presets_have_reverse_entries(self) -> None:
- for talent, preset in LLM_TALENT_TO_MODEL.items():
- assert _MODEL_TO_LLM_TALENT[preset] == talent or _MODEL_TO_LLM_TALENT[preset] in LLM_TALENT_TO_MODEL
- assert _MODEL_TO_LLM_TALENT[preset.lstrip("$@")] == talent or _MODEL_TO_LLM_TALENT[preset.lstrip("$@")] in LLM_TALENT_TO_MODEL
-
- def test_img_gen_preset_with_prefix(self) -> None:
- assert _MODEL_TO_IMG_GEN_TALENT["$gen-image"] == "gen-image"
-
- def test_img_gen_preset_without_prefix(self) -> None:
- assert _MODEL_TO_IMG_GEN_TALENT["gen-image"] == "gen-image"
-
- def test_extract_preset_with_prefix(self) -> None:
- assert _MODEL_TO_EXTRACT_TALENT["@default-text-from-pdf"] == "pdf-basic-text-extractor"
-
- def test_extract_preset_without_prefix(self) -> None:
- assert _MODEL_TO_EXTRACT_TALENT["default-text-from-pdf"] == "pdf-basic-text-extractor"
-
-
-# ---------------------------------------------------------------------------
-# _resolve_talent_value
-# ---------------------------------------------------------------------------
-
-
-class TestResolveTalentValue:
- """Tests for the preset-to-talent resolution helper."""
-
- # --- PipeLLM ---
-
- def test_valid_talent_passes_through(self) -> None:
- assert _resolve_talent_value("PipeLLM", "creative-writer") == "creative-writer"
-
- def test_preset_without_prefix_resolves(self) -> None:
- assert _resolve_talent_value("PipeLLM", "writing-creative") == "creative-writer"
-
- def test_preset_with_dollar_prefix_resolves(self) -> None:
- assert _resolve_talent_value("PipeLLM", "$writing-creative") == "creative-writer"
-
- def test_invalid_talent_passes_through(self) -> None:
- """Unknown values pass through so Pydantic can produce a clear error."""
- assert _resolve_talent_value("PipeLLM", "totally-wrong") == "totally-wrong"
-
- def test_engineering_code_resolves(self) -> None:
- assert _resolve_talent_value("PipeLLM", "engineering-code") == "coder"
-
- def test_retrieval_resolves(self) -> None:
- assert _resolve_talent_value("PipeLLM", "retrieval") == "data-retrieval"
-
- # --- PipeImgGen ---
-
- def test_img_gen_preset_resolves(self) -> None:
- assert _resolve_talent_value("PipeImgGen", "$gen-image-fast") == "gen-image-fast"
-
- def test_img_gen_valid_talent_passes_through(self) -> None:
- assert _resolve_talent_value("PipeImgGen", "gen-image-high-quality") == "gen-image-high-quality"
-
- # --- PipeExtract ---
-
- def test_extract_preset_resolves(self) -> None:
- assert _resolve_talent_value("PipeExtract", "default-extract-document") == "full-document-extractor"
-
- def test_extract_preset_with_at_resolves(self) -> None:
- assert _resolve_talent_value("PipeExtract", "@default-extract-image") == "image-text-extractor"
-
- # --- Unknown pipe types ---
-
- def test_unknown_pipe_type_passes_through(self) -> None:
- assert _resolve_talent_value("PipeCompose", "anything") == "anything"
-
- def test_pipe_search_passes_through(self) -> None:
- """PipeSearch has no reverse mapping; values pass through."""
- assert _resolve_talent_value("PipeSearch", "web-search") == "web-search"
-
- def test_dict_value_passes_through(self) -> None:
- """Non-string values (dict) should pass through without crashing."""
- result = _resolve_talent_value("PipeLLM", {"name": "creative-writer"})
- assert result == {"name": "creative-writer"}
-
- def test_list_value_passes_through(self) -> None:
- """Non-string values (list) should pass through without crashing."""
- result = _resolve_talent_value("PipeLLM", ["creative-writer"])
- assert result == ["creative-writer"]
-
- def test_int_value_passes_through(self) -> None:
- """Non-string values (int) should pass through without crashing."""
- result = _resolve_talent_value("PipeLLM", 42)
- assert result == 42
-
-
# ---------------------------------------------------------------------------
# _parse_pipe_spec_from_json — pipe_code aliases
# ---------------------------------------------------------------------------
@@ -129,7 +22,7 @@ class TestPipeCodeAliases:
_BASE_LLM: ClassVar[dict[str, Any]] = {
"description": "Test pipe",
- "llm_talent": "creative-writer",
+ "model": "$writing-creative",
"inputs": {"text": "Text"},
"output": "Text",
"prompt": "Write something about @text",
@@ -174,128 +67,6 @@ class TestPipeCodeAliases:
assert result.pipe_code == "my_pipe"
-# ---------------------------------------------------------------------------
-# _parse_pipe_spec_from_json — talent field aliases
-# ---------------------------------------------------------------------------
-
-
-class TestTalentFieldAliases:
- """The parser should accept 'talent_name' and 'talent' as generic talent aliases."""
-
- _BASE: ClassVar[dict[str, Any]] = {
- "pipe_code": "test_pipe",
- "description": "Test pipe",
- "inputs": {"text": "Text"},
- "output": "Text",
- "prompt": "Write about @text",
- }
-
- def test_talent_name_alias_for_llm(self) -> None:
- spec = {**self._BASE, "talent_name": "creative-writer"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]
-
- def test_talent_alias_for_llm(self) -> None:
- spec = {**self._BASE, "talent": "engineer"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "engineer" # type: ignore[attr-defined]
-
- def test_canonical_talent_field_ignores_alias(self) -> None:
- """When llm_talent is present, generic alias is removed so Pydantic doesn't reject it."""
- spec = {**self._BASE, "llm_talent": "coder", "talent_name": "engineer"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "coder" # type: ignore[attr-defined]
-
- def test_talent_alias_for_extract(self) -> None:
- spec = {
- "pipe_code": "test_extract",
- "description": "Extract pipe",
- "inputs": {"doc": "Document"},
- "output": "Page[]",
- "talent_name": "full-document-extractor",
- }
- result = _parse_pipe_spec_from_json("PipeExtract", spec)
- assert result.extract_talent == "full-document-extractor" # type: ignore[attr-defined]
-
- def test_talent_alias_for_img_gen(self) -> None:
- spec = {
- "pipe_code": "test_img_gen",
- "description": "Image gen pipe",
- "inputs": {"prompt_text": "ImgGenPrompt"},
- "output": "Image",
- "talent": "gen-image",
- "prompt": "Generate: $prompt_text",
- }
- result = _parse_pipe_spec_from_json("PipeImgGen", spec)
- assert result.img_gen_talent == "gen-image" # type: ignore[attr-defined]
-
-
-# ---------------------------------------------------------------------------
-# _parse_pipe_spec_from_json — preset-to-talent resolution
-# ---------------------------------------------------------------------------
-
-
-class TestPresetToTalentResolution:
- """Preset names used where talent names are expected should be auto-resolved."""
-
- _BASE_LLM: ClassVar[dict[str, Any]] = {
- "pipe_code": "test_pipe",
- "description": "Test pipe",
- "inputs": {"text": "Text"},
- "output": "Text",
- "prompt": "Write about @text",
- }
-
- def test_preset_name_resolved_to_talent_for_llm(self) -> None:
- spec = {**self._BASE_LLM, "llm_talent": "writing-creative"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]
-
- def test_preset_with_dollar_resolved_for_llm(self) -> None:
- spec = {**self._BASE_LLM, "llm_talent": "$writing-creative"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]
-
- def test_valid_talent_unchanged_for_llm(self) -> None:
- spec = {**self._BASE_LLM, "llm_talent": "creative-writer"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]
-
- def test_invalid_talent_raises_validation_error(self) -> None:
- spec = {**self._BASE_LLM, "llm_talent": "totally-invalid"}
- with pytest.raises(ValidationError):
- _parse_pipe_spec_from_json("PipeLLM", spec)
-
- def test_preset_via_generic_alias_also_resolved(self) -> None:
- """talent_name alias + preset value should both be handled."""
- spec = {**self._BASE_LLM, "talent_name": "engineering-code"}
- result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "coder" # type: ignore[attr-defined]
-
- def test_extract_preset_resolved(self) -> None:
- spec = {
- "pipe_code": "test_extract",
- "description": "Extract pipe",
- "inputs": {"doc": "Document"},
- "output": "Page[]",
- "extract_talent": "default-text-from-pdf",
- }
- result = _parse_pipe_spec_from_json("PipeExtract", spec)
- assert result.extract_talent == "pdf-basic-text-extractor" # type: ignore[attr-defined]
-
- def test_img_gen_preset_resolved(self) -> None:
- spec = {
- "pipe_code": "test_img_gen",
- "description": "Image gen pipe",
- "inputs": {"prompt_text": "ImgGenPrompt"},
- "output": "Image",
- "img_gen_talent": "$gen-image-high-quality",
- "prompt": "Generate: $prompt_text",
- }
- result = _parse_pipe_spec_from_json("PipeImgGen", spec)
- assert result.img_gen_talent == "gen-image-high-quality" # type: ignore[attr-defined]
-
-
# ---------------------------------------------------------------------------
# _parse_pipe_spec_from_json — output dict tolerance
# ---------------------------------------------------------------------------
@@ -307,7 +78,7 @@ class TestOutputDictTolerance:
_BASE_LLM: ClassVar[dict[str, Any]] = {
"pipe_code": "test_pipe",
"description": "Test pipe",
- "llm_talent": "creative-writer",
+ "model": "$writing-creative",
"inputs": {"text": "Text"},
"prompt": "Write about @text",
}
@@ -431,7 +202,7 @@ class TestPipeTypeAlias:
"type": "PipeLLM",
"pipe_code": "test",
"description": "Test",
- "llm_talent": "creative-writer",
+ "model": "$writing-creative",
"inputs": {"text": "Text"},
"output": "Text",
"prompt": "Write about @text",
@@ -456,34 +227,33 @@ class TestInvalidPipeType:
# ---------------------------------------------------------------------------
-# _pipe_spec_to_toml — talent-to-model in TOML output
+# _pipe_spec_to_toml — model in TOML output
# ---------------------------------------------------------------------------
class TestPipeSpecToToml:
- """Verify TOML output maps talents to model presets correctly."""
+ """Verify TOML output contains model presets correctly."""
_BASE_LLM: ClassVar[dict[str, Any]] = {
"pipe_code": "my_llm_pipe",
"description": "Test LLM pipe",
- "llm_talent": "creative-writer",
+ "model": "$writing-creative",
"inputs": {"text": "Text"},
"output": "Text",
"prompt": "Write about @text",
}
- def test_llm_talent_becomes_model_preset_in_toml(self) -> None:
+ def test_llm_model_appears_in_toml(self) -> None:
spec = _parse_pipe_spec_from_json("PipeLLM", {**self._BASE_LLM})
toml = _pipe_spec_to_toml(spec)
assert 'model = "$writing-creative"' in toml
- def test_llm_preset_input_still_maps_to_correct_model(self) -> None:
- """If the agent provides a preset name, it should resolve and then map to model."""
- spec = _parse_pipe_spec_from_json("PipeLLM", {**self._BASE_LLM, "llm_talent": "engineering-structured"})
+ def test_llm_different_model_in_toml(self) -> None:
+ spec = _parse_pipe_spec_from_json("PipeLLM", {**self._BASE_LLM, "model": "$engineering-structured"})
toml = _pipe_spec_to_toml(spec)
assert 'model = "$engineering-structured"' in toml
- def test_extract_talent_becomes_model_in_toml(self) -> None:
+ def test_extract_model_in_toml(self) -> None:
spec = _parse_pipe_spec_from_json(
"PipeExtract",
{
@@ -491,13 +261,13 @@ class TestPipeSpecToToml:
"description": "Extract pipe",
"inputs": {"doc": "Document"},
"output": "Page[]",
- "extract_talent": "full-document-extractor",
+ "model": "@default-extract-document",
},
)
toml = _pipe_spec_to_toml(spec)
assert 'model = "@default-extract-document"' in toml
- def test_img_gen_talent_becomes_model_in_toml(self) -> None:
+ def test_img_gen_model_in_toml(self) -> None:
spec = _parse_pipe_spec_from_json(
"PipeImgGen",
{
@@ -505,7 +275,7 @@ class TestPipeSpecToToml:
"description": "Image gen pipe",
"inputs": {"prompt_text": "ImgGenPrompt"},
"output": "Image",
- "img_gen_talent": "gen-image",
+ "model": "$gen-image",
"prompt": "Generate: $prompt_text",
},
)
@@ -541,35 +311,50 @@ class TestPipeSpecToToml:
assert "step_one" in toml
assert "step_two" in toml
+ def test_llm_no_model_omits_model_in_toml(self) -> None:
+ """When model is None, no model line should appear in the TOML."""
+ spec = _parse_pipe_spec_from_json(
+ "PipeLLM",
+ {
+ "pipe_code": "my_pipe",
+ "description": "No model specified",
+ "inputs": {"text": "Text"},
+ "output": "Text",
+ "prompt": "Write about @text",
+ },
+ )
+ toml = _pipe_spec_to_toml(spec)
+ assert "model =" not in toml
+
# ---------------------------------------------------------------------------
-# End-to-end: combined tolerance scenarios (the original bug)
+# End-to-end: combined tolerance scenarios
# ---------------------------------------------------------------------------
class TestEndToEndTolerance:
- """Reproduce the exact bug scenario that motivated these changes."""
+ """Test combined tolerance scenarios with direct model field."""
- def test_original_bug_preset_as_talent_with_dict_output(self) -> None:
- """The agent sent 'writing-creative' (preset) and output as dict — both should be tolerated."""
+ def test_model_with_dict_output_both_tolerated(self) -> None:
+ """The agent sends model preset and output as dict — both should be tolerated."""
spec: dict[str, Any] = {
"pipe_code": "generate_prompt",
"description": "Generate an image prompt from an idea",
- "llm_talent": "writing-creative",
+ "model": "$writing-creative",
"inputs": {"idea": "Text"},
"output": {"type": "ImgGenPrompt"},
"prompt": "Generate a creative image prompt based on @idea",
}
result = _parse_pipe_spec_from_json("PipeLLM", spec)
- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]
+ assert result.model == "$writing-creative" # type: ignore[attr-defined]
assert result.output == "ImgGenPrompt"
- def test_original_bug_toml_output_has_correct_model(self) -> None:
- """After resolution, the TOML should contain the correct model preset."""
+ def test_toml_output_has_correct_model(self) -> None:
+ """The TOML should contain the model preset directly."""
spec: dict[str, Any] = {
"pipe_code": "generate_prompt",
"description": "Generate an image prompt from an idea",
- "llm_talent": "writing-creative",
+ "model": "$writing-creative",
"inputs": {"idea": "Text"},
"output": {"type": "ImgGenPrompt"},
"prompt": "Generate a creative image prompt based on @idea",