Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Replace talent fields with direct model preset in PipeSpec subclasses (#783)
Louis Choquela8b42fef4750
diff --git a/pipelex/builder/CLAUDE.md b/pipelex/builder/CLAUDE.mdindex 65a463978..363f90ba1 100644--- a/pipelex/builder/CLAUDE.md+++ b/pipelex/builder/CLAUDE.md@@ -100,4 +100,4 @@ After fixing, `_prune_unreachable_specs()` removes pipes unreachable from `main_## Talent System-Talents are abstract capability labels mapped to concrete model presets. Each talent enum (in `talents/`) maps to a `$preset` code used in MTHDS files. When modifying talents, update both the enum and its preset mapping dict.+Talents are abstract capability labels mapped to concrete model presets. They remain in `talents/` and are used by the `pipelex-agent models` CLI to help agents choose appropriate presets. PipeSpec subclasses accept model presets directly via a `model: str | None` field, bypassing talent resolution. When modifying talent enums, update both the enum and its preset mapping dict in `pipelex.toml`.diff --git a/pipelex/builder/pipe/pipe_extract_spec.py b/pipelex/builder/pipe/pipe_extract_spec.pyindex e37317670..154b2b759 100644--- a/pipelex/builder/pipe/pipe_extract_spec.py+++ b/pipelex/builder/pipe/pipe_extract_spec.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import Literalfrom pydantic import Field, field_validatorfrom pydantic.json_schema import SkipJsonSchema@@ -7,15 +7,10 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.pipe.pipe_spec import PipeSpec-from pipelex.builder.talents.extract_talent import ExtractTalentfrom pipelex.cogt.content_generation.dry_run_factory import MockFormat-from pipelex.config import get_configfrom pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprintfrom pipelex.tools.misc.pretty import PrettyPrintable-if TYPE_CHECKING:- from pipelex.cogt.extract.extract_setting import ExtractModelChoice-class PipeExtractSpec(PipeSpec):"""Spec for OCR (Optical Character Recognition) pipe operations in the Pipelex framework.@@ -38,31 +33,29 @@ class PipeExtractSpec(PipeSpec):),json_schema_extra={"mock_format": MockFormat.DICT_SINGLE_EXTRACT_INPUT},)- extract_talent: ExtractTalent | str = Field(- description="Select the most adequate extraction model talent according to the task to be performed.",- examples=list(ExtractTalent),+ model: str | None = Field(+ default=None,+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",)max_page_images: int | None = Field(default=None, description="Max number of images to extract from pages: None=unlimited, 0=no images, N=limit to N images.")page_views: bool | None = Field(default=None, description="Whether to include rendered page views in the output.")+ @field_validator("model", mode="before")+ @classmethod+ def reject_empty_model(cls, value: str | None) -> str | None:+ if isinstance(value, str) and not value.strip():+ msg = "Model cannot be an empty string; omit the field to use defaults"+ raise ValueError(msg)+ return value+@override@field_validator("output", mode="before")@classmethoddef validate_output(cls, output: str) -> str:return "Page[]"- @field_validator("extract_talent", mode="before")- @classmethod- def validate_extract_talent(cls, extract_talent_value: str) -> ExtractTalent:- try:- return ExtractTalent(extract_talent_value)- except ValueError:- valid = [talent.value for talent in ExtractTalent]- msg = f"'{extract_talent_value}' is not a valid ExtractTalent. Valid values: {valid}"- raise ValueError(msg) from None-@field_validator("inputs", mode="before")@classmethoddef validate_extract_inputs(cls, inputs_value: dict[str, str] | None) -> dict[str, str] | None:@@ -85,7 +78,7 @@ class PipeExtractSpec(PipeSpec):# Add extract specific informationextract_group.renderables.append(Text()) # Blank line- extract_group.renderables.append(Text.from_markup(f"Extract Talent: [bold yellow]{self.extract_talent}[/bold yellow]"))+ extract_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))# Add optional extraction settings if they are setif self.max_page_images is not None:@@ -99,16 +92,12 @@ class PipeExtractSpec(PipeSpec):def to_blueprint(self) -> PipeExtractBlueprint:base_blueprint = super().to_blueprint()- # Get extract choice from config-based mapping- mappings = get_config().pipelex.builder_config.talent_preset_mappings.extract- extract_model_choice: ExtractModelChoice = mappings[self.extract_talent]-return PipeExtractBlueprint(source=None,description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- model=extract_model_choice,+ model=self.model,max_page_images=self.max_page_images,page_image_captions=None,page_views=self.page_views,diff --git a/pipelex/builder/pipe/pipe_img_gen_spec.py b/pipelex/builder/pipe/pipe_img_gen_spec.pyindex 604dd6862..ab2558dd1 100644--- a/pipelex/builder/pipe/pipe_img_gen_spec.py+++ b/pipelex/builder/pipe/pipe_img_gen_spec.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import Literalfrom pydantic import Field, field_validatorfrom rich.console import Group@@ -6,14 +6,9 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.pipe.pipe_spec import PipeSpec-from pipelex.builder.talents.img_gen_talent import ImgGenTalent-from pipelex.config import get_configfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprintfrom pipelex.tools.misc.pretty import PrettyPrintable-if TYPE_CHECKING:- from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice-class PipeImgGenSpec(PipeSpec):"""Specs for image generation pipe operations in the Pipelex framework.@@ -30,23 +25,19 @@ class PipeImgGenSpec(PipeSpec):type: Literal["PipeImgGen"] = "PipeImgGen"pipe_category: Literal["PipeOperator"] = "PipeOperator"- img_gen_talent: ImgGenTalent | str = Field(- description="Select the most adequate image generation talent according to the task to be performed.",- examples=list(ImgGenTalent),+ model: str | None = Field(+ default=None,+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",)prompt: str = Field(description="A finalized image generation prompt or prompt template: use `$` prefix for inline variables (e.g., `$topic`).")- @field_validator("img_gen_talent", mode="before")+ @field_validator("model", mode="before")@classmethod- def validate_img_gen_talent(cls, img_gen_talent_value: str | None) -> ImgGenTalent | None:- if img_gen_talent_value is None:- return None- try:- return ImgGenTalent(img_gen_talent_value)- except ValueError:- valid = [talent.value for talent in ImgGenTalent]- msg = f"'{img_gen_talent_value}' is not a valid ImgGenTalent. Valid values: {valid}"- raise ValueError(msg) from None+ def reject_empty_model(cls, value: str | None) -> str | None:+ if isinstance(value, str) and not value.strip():+ msg = "Model cannot be an empty string; omit the field to use defaults"+ raise ValueError(msg)+ return value@overridedef rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:@@ -59,7 +50,7 @@ class PipeImgGenSpec(PipeSpec):# Add image generation specific informationimg_gen_group.renderables.append(Text()) # Blank line- img_gen_group.renderables.append(Text.from_markup(f"Image Generation Talent: [bold yellow]{self.img_gen_talent}[/bold yellow]"))+ img_gen_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))return img_gen_group@@ -68,16 +59,12 @@ class PipeImgGenSpec(PipeSpec):"""Convert this PipeImgGenSpec to the core PipeImgGenBlueprint."""base_blueprint = super().to_blueprint()- # Get img_gen choice from config-based mapping- mappings = get_config().pipelex.builder_config.talent_preset_mappings.img_gen- img_gen_choice: ImgGenModelChoice = mappings[self.img_gen_talent]-return PipeImgGenBlueprint(description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,prompt=self.prompt,- model=img_gen_choice,+ model=self.model,aspect_ratio=None,background=None,output_format=None,diff --git a/pipelex/builder/pipe/pipe_llm_spec.py b/pipelex/builder/pipe/pipe_llm_spec.pyindex 2eeb53741..cf02e72dd 100644--- a/pipelex/builder/pipe/pipe_llm_spec.py+++ b/pipelex/builder/pipe/pipe_llm_spec.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import Literalfrom pydantic import Field, field_validatorfrom pydantic.json_schema import SkipJsonSchema@@ -8,14 +8,9 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.pipe.pipe_spec import PipeSpec-from pipelex.builder.talents.llm_talent import LLMTalent-from pipelex.config import get_configfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprintfrom pipelex.tools.misc.pretty import PrettyPrintable-if TYPE_CHECKING:- from pipelex.cogt.llm.llm_setting import LLMModelChoice-class PipeLLMSpec(PipeSpec):"""Spec for LLM-based pipe operations in the Pipelex framework.@@ -33,9 +28,9 @@ class PipeLLMSpec(PipeSpec):type: SkipJsonSchema[Literal["PipeLLM"]] = "PipeLLM"pipe_category: SkipJsonSchema[Literal["PipeOperator"]] = "PipeOperator"- llm_talent: LLMTalent | str = Field(- description="Select the simplest LLM talent corresponding to the task to be performed.",- examples=list(LLMTalent),+ model: str | None = Field(+ default=None,+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",)system_prompt: str | None = Field(default=None, description="A system prompt to guide the LLM's behavior, style and skills. Can be a template.")prompt: str | None = Field(@@ -58,15 +53,13 @@ So, don't have to write a bullet-list of all the attributes definitions yourself""",)- @field_validator("llm_talent", mode="before")+ @field_validator("model", mode="before")@classmethod- def validate_llm_talent(cls, llm_value: str) -> LLMTalent:- try:- return LLMTalent(llm_value)- except ValueError:- valid = [talent.value for talent in LLMTalent]- msg = f"'{llm_value}' is not a valid LLMTalent. Valid values: {valid}"- raise ValueError(msg) from None+ def reject_empty_model(cls, value: str | None) -> str | None:+ if isinstance(value, str) and not value.strip():+ msg = "Model cannot be an empty string; omit the field to use defaults"+ raise ValueError(msg)+ return value@overridedef rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:@@ -79,7 +72,7 @@ So, don't have to write a bullet-list of all the attributes definitions yourself# Add LLM-specific informationllm_group.renderables.append(Text()) # Blank line- llm_group.renderables.append(Text.from_markup(f"LLM Talent: [bold yellow]{self.llm_talent}[/bold yellow]"))+ llm_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))# Add system prompt if presentif self.system_prompt:@@ -111,15 +104,11 @@ So, don't have to write a bullet-list of all the attributes definitions yourselfdef to_blueprint(self) -> PipeLLMBlueprint:base_blueprint = super().to_blueprint()- # Get llm choice from config-based mapping- mappings = get_config().pipelex.builder_config.talent_preset_mappings.llm- llm_choice: LLMModelChoice = mappings[self.llm_talent]-return PipeLLMBlueprint(description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,system_prompt=self.system_prompt,prompt=self.prompt,- model=llm_choice,+ model=self.model,)diff --git a/pipelex/builder/pipe/pipe_search_spec.py b/pipelex/builder/pipe/pipe_search_spec.pyindex 28531571c..2e4a5b274 100644--- a/pipelex/builder/pipe/pipe_search_spec.py+++ b/pipelex/builder/pipe/pipe_search_spec.py@@ -1,5 +1,5 @@import re-from typing import TYPE_CHECKING, Any, Literal+from typing import Any, Literalfrom pydantic import ConfigDict, Field, field_validator, model_validatorfrom rich.console import Group@@ -8,14 +8,9 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.pipe.pipe_spec import PipeSpec-from pipelex.builder.talents.search_talent import SearchTalent-from pipelex.config import get_configfrom pipelex.pipe_operators.search.pipe_search_blueprint import PipeSearchBlueprintfrom pipelex.tools.misc.pretty import PrettyPrintable-if TYPE_CHECKING:- from pipelex.cogt.search.search_setting import SearchModelChoice-class PipeSearchSpec(PipeSpec):"""Specs for web search pipe operations in the Pipelex framework.@@ -28,10 +23,9 @@ class PipeSearchSpec(PipeSpec):type: Literal["PipeSearch"] = "PipeSearch"pipe_category: Literal["PipeOperator"] = "PipeOperator"- search_talent: SearchTalent | str = Field(- default=SearchTalent.WEB_SEARCH,- description="Select the most adequate search talent according to the task to be performed.",- examples=list(SearchTalent),+ model: str | None = Field(+ default=None,+ description="Model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'.",)prompt: str = Field(validation_alias="query",@@ -53,6 +47,14 @@ class PipeSearchSpec(PipeSpec):default=None, description="Maximum number of search results to return. If not set, the search provider's default is used.")+ @field_validator("model", mode="before")+ @classmethod+ def reject_empty_model(cls, value: str | None) -> str | None:+ if isinstance(value, str) and not value.strip():+ msg = "Model cannot be an empty string; omit the field to use defaults"+ raise ValueError(msg)+ return value+@model_validator(mode="before")@classmethoddef normalize_prompt_field(cls, values: dict[str, Any]) -> dict[str, Any]:@@ -75,16 +77,6 @@ class PipeSearchSpec(PipeSpec):raise ValueError(msg)return date_value- @field_validator("search_talent", mode="before")- @classmethod- def validate_search_talent(cls, search_talent_value: str) -> SearchTalent:- try:- return SearchTalent(search_talent_value)- except ValueError:- valid = list(SearchTalent)- msg = f"'{search_talent_value}' is not a valid SearchTalent. Valid values: {valid}"- raise ValueError(msg) from None-@overridedef rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:# Get base pipe information from parent@@ -96,7 +88,7 @@ class PipeSearchSpec(PipeSpec):# Add search specific informationsearch_group.renderables.append(Text()) # Blank line- search_group.renderables.append(Text.from_markup(f"Search Talent: [bold yellow]{self.search_talent}[/bold yellow]"))+ search_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{self.model or '(default)'}[/bold yellow]"))prompt_panel = Panel(self.prompt,title="Search Prompt",@@ -131,16 +123,12 @@ class PipeSearchSpec(PipeSpec):"""Convert this PipeSearchSpec to the core PipeSearchBlueprint."""base_blueprint = super().to_blueprint()- # Get search choice from config-based mapping- mappings = get_config().pipelex.builder_config.talent_preset_mappings.search- search_choice: SearchModelChoice = mappings[self.search_talent]-return PipeSearchBlueprint(description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,prompt=self.prompt,- model=search_choice,+ model=self.model,include_images=None,max_results=self.max_results,from_date=self.from_date,diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.mdindex 21d680dae..aca3f015a 100644--- a/pipelex/cli/agent_cli/CLAUDE.md+++ b/pipelex/cli/agent_cli/CLAUDE.md@@ -47,7 +47,8 @@ commands/plxt_passthrough.py # Shared helper for plxt subprocess delegationconcept_cmd.py # concept — JSON spec → raw TOML to stdoutpipe_cmd.py # pipe — JSON spec → raw TOML to stdout- models_cmd.py # models — list presets, aliases, talent mappings+ models_cmd.py # models — list presets, aliases, waterfalls+ check_model_cmd.py # check-model — validate model reference with fuzzy suggestionsinit_cmd.py # init — non-interactive config setup (--global/-g, --config/-c)doctor_cmd.py # doctor — config health check (--global/-g)```@@ -64,7 +65,8 @@ commands/| `inputs` | Generates example input JSON for a pipe/bundle/method (pipe\|bundle\|method subcommands) || `concept` | Converts a JSON concept spec into raw TOML (stdout) || `pipe` | Converts a JSON pipe spec (typed) into raw TOML (stdout) |-| `models` | Lists available model presets, aliases, waterfalls, and talent mappings. `--format markdown\|json` (default: markdown) |+| `models` | Lists available model presets, aliases, and waterfalls. `--format markdown\|json` (default: markdown) |+| `check-model` | Validates a model reference and suggests alternatives if invalid. `--type`/`-t` for model category, `--format markdown\|json` (default: markdown) || `doctor` | Checks config, credentials, models health. `--global`/`-g` for global dir. `--format markdown\|json` (default: markdown) |## Key Patternsdiff --git a/pipelex/cli/agent_cli/_agent_cli.py b/pipelex/cli/agent_cli/_agent_cli.pyindex d7ed8979d..5813fd118 100644--- a/pipelex/cli/agent_cli/_agent_cli.py+++ b/pipelex/cli/agent_cli/_agent_cli.py@@ -8,6 +8,7 @@ from mthds.runners.types import RunnerTypefrom typer.core import TyperGroupfrom typing_extensions import override+from pipelex.cli.agent_cli.commands.check_model_cmd import agent_check_model_cmdfrom pipelex.cli.agent_cli.commands.concept_cmd import concept_cmdfrom pipelex.cli.agent_cli.commands.doctor_cmd import agent_doctor_cmdfrom pipelex.cli.agent_cli.commands.fmt_cmd import fmt_cmd@@ -37,6 +38,7 @@ class PipelexAgentCLI(TyperGroup):"concept","pipe","models",+ "check-model","doctor",]@@ -126,5 +128,6 @@ app.command(name="lint", help="Lint a .mthds, .toml, or .plx file")(lint_cmd)app.add_typer(inputs_app, name="inputs", help="Generate example input JSON for a pipe")app.command(name="concept", help="Structure a concept from JSON spec and output TOML")(concept_cmd)app.command(name="pipe", help="Structure a pipe from JSON spec and output TOML")(pipe_cmd)-app.command(name="models", help="List available model presets, aliases, and talent mappings")(agent_models_cmd)+app.command(name="models", help="List available model presets, aliases, and waterfalls")(agent_models_cmd)+app.command(name="check-model", help="Check if a model reference is valid and suggest alternatives")(agent_check_model_cmd)app.command(name="doctor", help="Check Pipelex configuration health and auto-fix issues")(agent_doctor_cmd)diff --git a/pipelex/cli/agent_cli/commands/agent_output.py b/pipelex/cli/agent_cli/commands/agent_output.pyindex 5d00d4798..acd33516b 100644--- a/pipelex/cli/agent_cli/commands/agent_output.py+++ b/pipelex/cli/agent_cli/commands/agent_output.py@@ -18,6 +18,10 @@ class CliOutputFormat(StrEnum):AGENT_ERROR_HINTS: dict[str, str] = {# Model/routing errors+ "ModelChoiceNotFoundError": (+ "Check model name for typos. Use 'pipelex-agent check-model <name> -t <type>' "+ "to validate or 'pipelex-agent models -t <type>' to list available models."+ ),"PipeOperatorModelChoiceError": "Run 'pipelex-agent doctor' to check available models and routing configuration","PipeOperatorModelAvailabilityError": "Run 'pipelex-agent doctor' to check available models and verify API keys","ModelDeckPresetValidatonError": (@@ -70,6 +74,7 @@ RETRYABLE_ERROR_TYPES: set[str] = {AGENT_ERROR_DOMAINS: dict[str, str] = {# input = agent can fix (bad .mthds, wrong args, bad JSON)+ "ModelChoiceNotFoundError": "input","ValidateBundleError": "input","PipeValidationError": "input","FileNotFoundError": "input",diff --git a/pipelex/cli/agent_cli/commands/check_model_cmd.py b/pipelex/cli/agent_cli/commands/check_model_cmd.pynew file mode 100644index 000000000..dddcc4363--- /dev/null+++ b/pipelex/cli/agent_cli/commands/check_model_cmd.py@@ -0,0 +1,107 @@+"""Agent CLI check-model command -- validate a model reference and suggest alternatives."""++from typing import Annotated, Any++import typer++from pipelex.cli.agent_cli.commands.agent_cli_factory import make_pipelex_for_agent_cli+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_error, agent_success+from pipelex.cli.agent_cli.commands.models_cmd import CATEGORY_TO_MODEL_TYPE, ModelCategory+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceParseError+from pipelex.cogt.models.model_suggestion import (+ KIND_LABELS,+ get_collection_keys,+ suggest_model_alternatives,+)+from pipelex.hub import get_model_deck+from pipelex.pipelex import Pipelex+++def _format_check_markdown(result: dict[str, Any]) -> str:+ """Format the check-model result as concise markdown."""+ name: str = result["name"]+ kind_label: str = result["kind"]+ category: str = result["model_type"]++ if result["valid"]:+ return f"{name} is a valid {category} {kind_label}."++ lines: list[str] = [f"{name} is not a valid {category} {kind_label}."]++ suggestions: list[str] = result.get("suggestions", [])+ if suggestions:+ lines.append("\nDid you mean:")+ for suggestion in suggestions:+ lines.append(f"- {suggestion}")++ cross_suggestions: list[str] = result.get("cross_collection_suggestions", [])+ if cross_suggestions:+ if not suggestions:+ lines.append("\nDid you mean:")+ for suggestion in cross_suggestions:+ lines.append(f"- {suggestion}")++ wrong_sigil_hints: list[str] = result.get("wrong_sigil_hints", [])+ for hint in wrong_sigil_hints:+ lines.append(f"\nNote: {hint}")++ return "\n".join(lines)+++def agent_check_model_cmd(+ ctx: typer.Context,+ name: Annotated[+ str,+ typer.Argument(help="Model reference to check (e.g. $writing-creative, @best-claude, gpt-4o)"),+ ],+ model_type: Annotated[+ ModelCategory,+ typer.Option("--type", "-t", help="Model category: llm, extract, img_gen, search"),+ ],+ output_format: Annotated[+ CliOutputFormat,+ typer.Option("--format", help="Output format: markdown (default) or json (structured)"),+ ] = CliOutputFormat.MARKDOWN,+) -> None:+ """Check if a model reference is valid and suggest alternatives if not.++ Parses the reference (with sigil prefix if present), validates it against the+ model deck, and on failure provides fuzzy suggestions and wrong-sigil hints.+ """+ try:+ make_pipelex_for_agent_cli(log_level=ctx.obj["log_level"], needs_inference=False, needs_model_specs=True)++ model_deck = get_model_deck()+ ref = ModelReference.parse(name)+ resolved_model_type = CATEGORY_TO_MODEL_TYPE[model_type]++ candidates = get_collection_keys(model_deck, resolved_model_type, ref.kind)+ is_valid = ref.name in candidates++ result: dict[str, Any] = {+ "success": True,+ "valid": is_valid,+ "name": name,+ "kind": KIND_LABELS[ref.kind],+ "model_type": str(model_type),+ }++ if not is_valid:+ suggestions, wrong_sigil_hints, cross_suggestions = suggest_model_alternatives(model_deck, resolved_model_type, ref.name, ref.kind)+ result["suggestions"] = suggestions+ result["wrong_sigil_hints"] = wrong_sigil_hints+ result["cross_collection_suggestions"] = cross_suggestions++ match output_format:+ case CliOutputFormat.JSON:+ agent_success(result)+ case CliOutputFormat.MARKDOWN:+ print(_format_check_markdown(result))+ except SystemExit:+ raise+ except ModelReferenceParseError as exc:+ agent_error(f"Invalid model reference: {exc}", "ArgumentError", cause=exc)+ except Exception as exc:+ agent_error(f"Failed to check model: {exc}", type(exc).__name__, cause=exc)+ finally:+ Pipelex.teardown_if_needed()diff --git a/pipelex/cli/agent_cli/commands/models_cmd.py b/pipelex/cli/agent_cli/commands/models_cmd.pyindex 2164d3157..542df98d7 100644--- a/pipelex/cli/agent_cli/commands/models_cmd.py+++ b/pipelex/cli/agent_cli/commands/models_cmd.py@@ -1,4 +1,4 @@-"""Agent CLI models command -- list available model presets, aliases, and talent mappings."""+"""Agent CLI models command -- list available model presets, aliases, and waterfalls."""from typing import Annotated, Any@@ -9,7 +9,6 @@ from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_efrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.model_deck import ModelDeck-from pipelex.config import get_configfrom pipelex.hub import get_model_deckfrom pipelex.pipelex import Pipelexfrom pipelex.types import StrEnum@@ -100,25 +99,6 @@ def _filter_waterfalls_by_backend(return filtered-def _filter_talent_mappings_by_backend(- mappings: dict[str, str],- presets_dict: dict[str, Any],- model_deck: ModelDeck,- model_type: ModelType,- backend: str,-) -> dict[str, str]:- """Filter talent mappings to only include those whose preset resolves to the given backend."""- filtered: dict[str, str] = {}- for talent_name, preset_name in mappings.items():- setting = presets_dict.get(preset_name)- if setting is None:- continue- spec = _resolve_preset_backend(model_deck, setting.model, model_type)- if spec is not None and spec.backend_name == backend:- filtered[talent_name] = preset_name- return filtered--def _build_presets_for_category(model_deck: ModelDeck,category: ModelCategory,@@ -198,31 +178,6 @@ def _build_waterfalls_for_category(return waterfalls-def _build_talent_mappings_for_category(- model_deck: ModelDeck,- category: ModelCategory,- mappings: dict[str, str],- backend: str | None,-) -> dict[str, str]:- """Build the talent mappings dict for a given category, optionally filtered by backend."""- if backend is None:- return mappings-- model_type = CATEGORY_TO_MODEL_TYPE[category]- presets_dict: dict[str, Any]- match category:- case ModelCategory.LLM:- presets_dict = model_deck.llm_presets- case ModelCategory.EXTRACT:- presets_dict = model_deck.extract_presets- case ModelCategory.IMG_GEN:- presets_dict = model_deck.img_gen_presets- case ModelCategory.SEARCH:- presets_dict = model_deck.search_presets-- return _filter_talent_mappings_by_backend(mappings, presets_dict, model_deck, model_type, backend)--CATEGORY_DISPLAY_NAMES: dict[str, str] = {"llm": "LLM","img_gen": "Image Generation",@@ -238,47 +193,41 @@ def _format_models_markdown(result: dict[str, Any]) -> str:presets: dict[str, list[dict[str, Any]]] = result["presets"]aliases: dict[str, dict[str, str]] = result["aliases"]waterfalls: dict[str, dict[str, list[str]]] = result["waterfalls"]- talent_mappings: dict[str, dict[str, str]] = result["talent_mappings"]for category_key, display_name in CATEGORY_DISPLAY_NAMES.items():cat_presets = presets.get(category_key, [])cat_aliases = aliases.get(category_key, {})cat_waterfalls = waterfalls.get(category_key, {})- cat_talents = talent_mappings.get(category_key, {})- if not cat_presets and not cat_aliases and not cat_waterfalls and not cat_talents:+ if not cat_presets and not cat_aliases and not cat_waterfalls:continuelines.append(f"\n## {display_name}")if cat_presets:- lines.append("\n### Presets\n")+ lines.append("\n### Presets")+ lines.append("\nPresets are the preferred way to specify models — each preset bundles a model with relevant settings.\n")for preset in cat_presets:description = preset.get("description", "")if description:- lines.append(f"- **{preset['name']}**: {description}")+ lines.append(f"- ${preset['name']}: {description}")else:- lines.append(f"- **{preset['name']}**")+ lines.append(f"- ${preset['name']}")if cat_aliases:- lines.append("\n### Aliases\n")+ lines.append("\n### Aliases")+ lines.append(+ "\nAliases provide stable names that can be retargeted when new models come out, so pipelines stay up to date without edits.\n"+ )for alias_name, alias_target in cat_aliases.items():- lines.append(f"- **{alias_name}** \u2192 {alias_target}")+ lines.append(f"- @{alias_name} \u2192 {alias_target}")if cat_waterfalls:- lines.append("\n### Waterfalls\n")+ lines.append("\n### Waterfalls")+ lines.append("\nWaterfalls define ordered fallback chains — if the first model is unavailable, the next one is tried automatically.\n")for waterfall_name, fallback_list in cat_waterfalls.items():chain = " \u2192 ".join(fallback_list)- lines.append(f"- **{waterfall_name}**: {chain}")-- if cat_talents:- lines.append("\n### Talent Mappings\n")- for talent_name, preset_name in cat_talents.items():- lines.append(f"- **{talent_name}** \u2192 {preset_name}")-- hint = result.get("talent_mappings_usage_hint")- if hint:- lines.append(f"\n---\n\n{hint}")+ lines.append(f"- ~{waterfall_name}: {chain}")return "\n".join(lines)@@ -298,7 +247,7 @@ def agent_models_cmd(typer.Option("--format", help="Output format: markdown (default) or json (structured)"),] = CliOutputFormat.MARKDOWN,) -> None:- """List available model presets, aliases, waterfalls, and talent mappings.+ """List available model presets, aliases, and waterfalls.Outputs model configuration that an agent needs to reference when building pipelines.Default output is markdown; use --format json for structured JSON.@@ -307,48 +256,36 @@ def agent_models_cmd(make_pipelex_for_agent_cli(log_level=ctx.obj["log_level"], needs_inference=False, needs_model_specs=backend is not None)model_deck = get_model_deck()- builder_config = get_config().pipelex.builder_config- talent_mappings = builder_config.talent_preset_mappingspresets: dict[str, list[dict[str, Any]]] = {}aliases: dict[str, dict[str, str]] = {}waterfalls: dict[str, dict[str, list[str]]] = {}- talent: dict[str, dict[str, str]] = {}if _should_include(ModelCategory.LLM, model_type):presets["llm"] = _build_presets_for_category(model_deck, ModelCategory.LLM, backend)aliases["llm"] = _build_aliases_for_category(model_deck, ModelCategory.LLM, backend)waterfalls["llm"] = _build_waterfalls_for_category(model_deck, ModelCategory.LLM, backend)- talent["llm"] = _build_talent_mappings_for_category(model_deck, ModelCategory.LLM, talent_mappings.llm, backend)if _should_include(ModelCategory.IMG_GEN, model_type):presets["img_gen"] = _build_presets_for_category(model_deck, ModelCategory.IMG_GEN, backend)aliases["img_gen"] = _build_aliases_for_category(model_deck, ModelCategory.IMG_GEN, backend)waterfalls["img_gen"] = _build_waterfalls_for_category(model_deck, ModelCategory.IMG_GEN, backend)- talent["img_gen"] = _build_talent_mappings_for_category(model_deck, ModelCategory.IMG_GEN, talent_mappings.img_gen, backend)if _should_include(ModelCategory.EXTRACT, model_type):presets["extract"] = _build_presets_for_category(model_deck, ModelCategory.EXTRACT, backend)aliases["extract"] = _build_aliases_for_category(model_deck, ModelCategory.EXTRACT, backend)waterfalls["extract"] = _build_waterfalls_for_category(model_deck, ModelCategory.EXTRACT, backend)- talent["extract"] = _build_talent_mappings_for_category(model_deck, ModelCategory.EXTRACT, talent_mappings.extract, backend)if _should_include(ModelCategory.SEARCH, model_type):presets["search"] = _build_presets_for_category(model_deck, ModelCategory.SEARCH, backend)aliases["search"] = _build_aliases_for_category(model_deck, ModelCategory.SEARCH, backend)waterfalls["search"] = _build_waterfalls_for_category(model_deck, ModelCategory.SEARCH, backend)- talent["search"] = _build_talent_mappings_for_category(model_deck, ModelCategory.SEARCH, talent_mappings.search, backend)result: dict[str, Any] = {"success": True,"presets": presets,"aliases": aliases,"waterfalls": waterfalls,- "talent_mappings": talent,- "talent_mappings_usage_hint": (- "Use the talent name (key) as the value for llm_talent / extract_talent / img_gen_talent / search_talent"- " in pipe specs passed to 'pipelex-agent pipe --spec'"- ),}match output_format:diff --git a/pipelex/cli/agent_cli/commands/pipe_cmd.py b/pipelex/cli/agent_cli/commands/pipe_cmd.pyindex c89599372..2e72d2d5f 100644--- a/pipelex/cli/agent_cli/commands/pipe_cmd.py+++ b/pipelex/cli/agent_cli/commands/pipe_cmd.py@@ -24,65 +24,11 @@ from pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_sequence_spec import PipeSequenceSpecfrom pipelex.builder.pipe.pipe_spec import PipeSpecfrom pipelex.builder.pipe.pipe_spec_map import pipe_type_to_spec_class-from pipelex.builder.talents.extract_talent import ExtractTalent-from pipelex.builder.talents.img_gen_talent import ImgGenTalent-from pipelex.builder.talents.llm_talent import LLMTalentfrom pipelex.cli.agent_cli.commands.agent_output import agent_errorfrom pipelex.core.pipes.pipe_blueprint import PipeTypefrom pipelex.language.toml_string_utils import format_toml_stringfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error_for_agent-# Static talent-to-model preset mappings (from pipelex.toml defaults)-LLM_TALENT_TO_MODEL: dict[str, str] = {- "data-retrieval": "$retrieval",- "hr-expert": "$writing-factual",- "accounting-expert": "$writing-factual",- "creative-writer": "$writing-creative",- "engineer": "$engineering-structured",- "coder": "$engineering-code",- "code-analyzer": "$engineering-codebase-analysis",- "vision-language-model": "$vision",- "visual-designer": "$img-gen-prompting",-}--IMG_GEN_TALENT_TO_MODEL: dict[str, str] = {- "gen-image": "$gen-image",- "gen-image-fast": "$gen-image-fast",- "gen-image-high-quality": "$gen-image-high-quality",-}--EXTRACT_TALENT_TO_MODEL: dict[str, str] = {- "pdf-basic-text-extractor": "@default-text-from-pdf",- "image-text-extractor": "@default-extract-image",- "full-document-extractor": "@default-extract-document",- "web-page-extractor": "@default-extract-web-page",-}--# Reverse mappings: preset name → talent name (for fuzzy resolution)-_MODEL_TO_LLM_TALENT: dict[str, str] = {}-for _talent, _preset in LLM_TALENT_TO_MODEL.items():- _MODEL_TO_LLM_TALENT[_preset] = _talent- _MODEL_TO_LLM_TALENT[_preset.lstrip("$@")] = _talent--_MODEL_TO_IMG_GEN_TALENT: dict[str, str] = {}-for _talent, _preset in IMG_GEN_TALENT_TO_MODEL.items():- _MODEL_TO_IMG_GEN_TALENT[_preset] = _talent- _MODEL_TO_IMG_GEN_TALENT[_preset.lstrip("$@")] = _talent--_MODEL_TO_EXTRACT_TALENT: dict[str, str] = {}-for _talent, _preset in EXTRACT_TALENT_TO_MODEL.items():- _MODEL_TO_EXTRACT_TALENT[_preset] = _talent- _MODEL_TO_EXTRACT_TALENT[_preset.lstrip("$@")] = _talent--# Maps pipe types to their talent field names and valid values.-# Used to enrich validation errors with field_hints so the agent knows-# what values are valid when a talent field is missing or wrong.-PIPE_TYPE_TALENT_HINTS: dict[str, dict[str, list[str]]] = {- "PipeLLM": {"llm_talent": [talent.value for talent in LLMTalent]},- "PipeExtract": {"extract_talent": [talent.value for talent in ExtractTalent]},- "PipeImgGen": {"img_gen_talent": [talent.value for talent in ImgGenTalent]},-}-def _pipe_spec_to_toml(pipe_spec: PipeSpec) -> str:"""Convert a PipeSpec to TOML string format.@@ -130,9 +76,8 @@ def _add_type_specific_fields(pipe_spec: PipeSpec, pipe_table: tomlkit.TOMLDocumpipe_table: The TOML table to add fields to."""if isinstance(pipe_spec, PipeLLMSpec):- # Convert llm_talent to model preset using static mappings- model_preset = LLM_TALENT_TO_MODEL.get(pipe_spec.llm_talent, "$writing-creative")- pipe_table.add("model", model_preset)+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)if pipe_spec.system_prompt:pipe_table.add("system_prompt", format_toml_string(pipe_spec.system_prompt))if pipe_spec.prompt:@@ -198,40 +143,18 @@ def _add_type_specific_fields(pipe_spec: PipeSpec, pipe_table: tomlkit.TOMLDocumpipe_table.add("input_item_name", pipe_spec.input_item_name)elif isinstance(pipe_spec, PipeExtractSpec):- # Convert extract_talent to model preset using static mappings- model_preset = EXTRACT_TALENT_TO_MODEL.get(pipe_spec.extract_talent, "@default-extract-document")- pipe_table.add("model", model_preset)+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)elif isinstance(pipe_spec, PipeImgGenSpec):- # Convert img_gen_talent to model preset using static mappings- model_preset = IMG_GEN_TALENT_TO_MODEL.get(pipe_spec.img_gen_talent, "$gen-image")- pipe_table.add("model", model_preset)+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)pipe_table.add("prompt", format_toml_string(pipe_spec.prompt))elif isinstance(pipe_spec, PipeFuncSpec):pipe_table.add("function_name", pipe_spec.function_name)-def _resolve_talent_value(pipe_type: str, raw_value: Any) -> Any:- """Attempt to resolve a talent value, accepting preset names as aliases."""- if not isinstance(raw_value, str):- return raw_value- reverse_maps: dict[str, dict[str, str]] = {- "PipeLLM": _MODEL_TO_LLM_TALENT,- "PipeImgGen": _MODEL_TO_IMG_GEN_TALENT,- "PipeExtract": _MODEL_TO_EXTRACT_TALENT,- }- reverse_map = reverse_maps.get(pipe_type)- if not reverse_map:- return raw_value- # If it's already a valid talent, return as-is- # Otherwise try reverse-mapping from preset- resolved = reverse_map.get(raw_value)- if resolved:- return resolved- return raw_value # Let Pydantic validation handle the error--def _parse_pipe_spec_from_json(pipe_type: str, spec_data: dict[str, Any]) -> PipeSpec:"""Parse and validate a PipeSpec from JSON data.@@ -264,23 +187,6 @@ def _parse_pipe_spec_from_json(pipe_type: str, spec_data: dict[str, Any]) -> Pipelse:spec_data.pop(alias)- # Accept generic talent aliases and map to the type-specific field- talent_aliases = ("talent_name", "talent")- pipe_type_to_talent_field: dict[str, str] = {- "PipeLLM": "llm_talent",- "PipeExtract": "extract_talent",- "PipeImgGen": "img_gen_talent",- "PipeSearch": "search_talent",- }- talent_field = pipe_type_to_talent_field.get(pipe_type)- if talent_field:- for alias in talent_aliases:- if alias in spec_data:- if talent_field not in spec_data:- spec_data[talent_field] = spec_data.pop(alias)- else:- spec_data.pop(alias)-# Handle steps/branches conversion - need to convert pipe to pipe_codeif "steps" in spec_data:converted_steps = []@@ -305,11 +211,6 @@ def _parse_pipe_spec_from_json(pipe_type: str, spec_data: dict[str, Any]) -> Pipelse:spec_data.pop("expression")- # Resolve preset names to talent names (tolerance for agent mistakes)- talent_field = pipe_type_to_talent_field.get(pipe_type)- if talent_field and talent_field in spec_data:- spec_data[talent_field] = _resolve_talent_value(pipe_type, spec_data[talent_field])-# Accept output as dict with "type" key → extract the type stringif "output" in spec_data and isinstance(spec_data["output"], dict):if "type" in spec_data["output"]:@@ -347,7 +248,7 @@ def pipe_cmd("description": "What the pipe does","inputs": {"input_name": "ConceptName"},"output": "OutputConcept",- "llm_talent": "creative-writer",+ "model": "$writing-creative","prompt": "Your prompt with @block and $inline vars"}@@ -411,8 +312,7 @@ def pipe_cmd(except ValidationError as exc:message, details = format_pydantic_validation_error_for_agent(exc)- field_hints = PIPE_TYPE_TALENT_HINTS.get(resolved_pipe_type, {})- agent_error(message, "ValidationError", cause=exc, validation_details=details, field_hints=field_hints)+ agent_error(message, "ValidationError", cause=exc, validation_details=details)except ValueError as exc:agent_error(str(exc), "ValueError", cause=exc)diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 3bd0fb795..c5654f102 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -47,14 +47,28 @@ class ModelChoiceNotFoundError(CogtError):model_choice: str,reference_kind: ModelReferenceKind | None = None,available_options: list[str] | None = None,+ suggestions: list[str] | None = None,+ wrong_sigil_hints: list[str] | None = None,+ cross_collection_suggestions: list[str] | None = None,):self.model_type = model_typeself.model_choice = model_choiceself.reference_kind = reference_kindself.available_options = available_options or []+ self.suggestions = suggestions or []+ self.wrong_sigil_hints = wrong_sigil_hints or []+ self.cross_collection_suggestions = cross_collection_suggestions or []full_message = message- if self.available_options:++ all_suggestions = self.suggestions + self.cross_collection_suggestions+ if all_suggestions:+ full_message += "\n\nDid you mean: " + ", ".join(all_suggestions)++ for hint in self.wrong_sigil_hints:+ full_message += f"\nNote: {hint}"++ if not all_suggestions and not self.wrong_sigil_hints and self.available_options:options_str = ", ".join(sorted(self.available_options)[:10])if len(self.available_options) > 10:options_str += f" ... and {len(self.available_options) - 10} more"diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.pyindex e07019e74..05af028bc 100644--- a/pipelex/cogt/models/model_deck_check.py+++ b/pipelex/cogt/models/model_deck_check.py@@ -1,14 +1,43 @@+"""Model choice validation against the model deck with fuzzy suggestions."""++from typing import NoReturn+from pipelex.cogt.exceptions import ModelChoiceNotFoundErrorfrom pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSettingfrom pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSettingfrom pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeckfrom pipelex.cogt.models.model_reference import ModelReferenceKind, ensure_model_reference+from pipelex.cogt.models.model_suggestion import suggest_model_alternativesfrom pipelex.cogt.search.search_setting import SearchModelChoice, SearchSettingfrom pipelex.hub import get_model_deck-def check_llm_choice_with_deck(llm_choice: LLMModelChoice):+def _raise_model_choice_not_found(+ msg: str,+ model_deck: ModelDeck,+ model_type: ModelType,+ raw_choice: str,+ name: str,+ reference_kind: ModelReferenceKind,+ available_options: list[str],+) -> NoReturn:+ """Compute fuzzy suggestions and raise ModelChoiceNotFoundError."""+ suggestions, wrong_sigil_hints, cross_suggestions = suggest_model_alternatives(model_deck, model_type, name, reference_kind)+ raise ModelChoiceNotFoundError(+ message=msg,+ model_type=model_type,+ model_choice=raw_choice,+ reference_kind=reference_kind,+ available_options=available_options,+ suggestions=suggestions,+ wrong_sigil_hints=wrong_sigil_hints,+ cross_collection_suggestions=cross_suggestions,+ )+++def check_llm_choice_with_deck(llm_choice: LLMModelChoice) -> None:if isinstance(llm_choice, LLMSetting):return@@ -20,49 +49,45 @@ def check_llm_choice_with_deck(llm_choice: LLMModelChoice):if ref.name in model_deck.llm_presets:returnmsg = f"LLM preset '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.LLM,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.PRESET,- available_options=list(model_deck.llm_presets.keys()),+ _raise_model_choice_not_found(+ msg, model_deck, ModelType.LLM, ref.raw, ref.name, ModelReferenceKind.PRESET, list(model_deck.llm_presets.keys()))case ModelReferenceKind.ALIAS:if ref.name in model_deck.llm_aliases:returnmsg = f"Alias '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.LLM,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.ALIAS,- available_options=list(model_deck.llm_aliases.keys()),+ _raise_model_choice_not_found(+ msg, model_deck, ModelType.LLM, ref.raw, ref.name, ModelReferenceKind.ALIAS, list(model_deck.llm_aliases.keys()))case ModelReferenceKind.WATERFALL:if ref.name in model_deck.llm_waterfalls:returnmsg = f"Waterfall '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.LLM,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.WATERFALL,- available_options=list(model_deck.llm_waterfalls.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.LLM,+ ref.raw,+ ref.name,+ ModelReferenceKind.WATERFALL,+ list(model_deck.llm_waterfalls.keys()),)case ModelReferenceKind.HANDLE:if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.LLM):returnmsg = f"Model handle '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.LLM,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.HANDLE,- available_options=list(model_deck.inference_models.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.LLM,+ ref.raw,+ ref.name,+ ModelReferenceKind.HANDLE,+ list(model_deck.inference_models.keys()),)-def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):+def check_extract_choice_with_deck(extract_choice: ExtractModelChoice) -> None:if isinstance(extract_choice, ExtractSetting):return@@ -74,49 +99,57 @@ def check_extract_choice_with_deck(extract_choice: ExtractModelChoice):if ref.name in model_deck.extract_presets:returnmsg = f"Extract preset '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.TEXT_EXTRACTOR,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.PRESET,- available_options=list(model_deck.extract_presets.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.TEXT_EXTRACTOR,+ ref.raw,+ ref.name,+ ModelReferenceKind.PRESET,+ list(model_deck.extract_presets.keys()),)case ModelReferenceKind.ALIAS:if ref.name in model_deck.extract_aliases:returnmsg = f"Alias '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.TEXT_EXTRACTOR,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.ALIAS,- available_options=list(model_deck.extract_aliases.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.TEXT_EXTRACTOR,+ ref.raw,+ ref.name,+ ModelReferenceKind.ALIAS,+ list(model_deck.extract_aliases.keys()),)case ModelReferenceKind.WATERFALL:if ref.name in model_deck.extract_waterfalls:returnmsg = f"Waterfall '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.TEXT_EXTRACTOR,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.WATERFALL,- available_options=list(model_deck.extract_waterfalls.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.TEXT_EXTRACTOR,+ ref.raw,+ ref.name,+ ModelReferenceKind.WATERFALL,+ list(model_deck.extract_waterfalls.keys()),)case ModelReferenceKind.HANDLE:if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.TEXT_EXTRACTOR):returnmsg = f"Model handle '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.TEXT_EXTRACTOR,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.HANDLE,- available_options=list(model_deck.inference_models.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.TEXT_EXTRACTOR,+ ref.raw,+ ref.name,+ ModelReferenceKind.HANDLE,+ list(model_deck.inference_models.keys()),)-def check_search_choice_with_deck(search_choice: SearchModelChoice):+def check_search_choice_with_deck(search_choice: SearchModelChoice) -> None:if isinstance(search_choice, SearchSetting):return@@ -128,49 +161,57 @@ def check_search_choice_with_deck(search_choice: SearchModelChoice):if ref.name in model_deck.search_presets:returnmsg = f"Search preset '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.SEARCH,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.PRESET,- available_options=list(model_deck.search_presets.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.SEARCH,+ ref.raw,+ ref.name,+ ModelReferenceKind.PRESET,+ list(model_deck.search_presets.keys()),)case ModelReferenceKind.ALIAS:if ref.name in model_deck.search_aliases:returnmsg = f"Alias '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.SEARCH,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.ALIAS,- available_options=list(model_deck.search_aliases.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.SEARCH,+ ref.raw,+ ref.name,+ ModelReferenceKind.ALIAS,+ list(model_deck.search_aliases.keys()),)case ModelReferenceKind.WATERFALL:if ref.name in model_deck.search_waterfalls:returnmsg = f"Waterfall '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.SEARCH,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.WATERFALL,- available_options=list(model_deck.search_waterfalls.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.SEARCH,+ ref.raw,+ ref.name,+ ModelReferenceKind.WATERFALL,+ list(model_deck.search_waterfalls.keys()),)case ModelReferenceKind.HANDLE:if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.SEARCH):returnmsg = f"Model handle '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.SEARCH,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.HANDLE,- available_options=list(model_deck.inference_models.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.SEARCH,+ ref.raw,+ ref.name,+ ModelReferenceKind.HANDLE,+ list(model_deck.inference_models.keys()),)-def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):+def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice) -> None:if isinstance(img_gen_choice, ImgGenSetting):return@@ -182,43 +223,51 @@ def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice):if ref.name in model_deck.img_gen_presets:returnmsg = f"Image generation preset '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.IMG_GEN,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.PRESET,- available_options=list(model_deck.img_gen_presets.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.IMG_GEN,+ ref.raw,+ ref.name,+ ModelReferenceKind.PRESET,+ list(model_deck.img_gen_presets.keys()),)case ModelReferenceKind.ALIAS:if ref.name in model_deck.img_gen_aliases:returnmsg = f"Alias '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.IMG_GEN,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.ALIAS,- available_options=list(model_deck.img_gen_aliases.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.IMG_GEN,+ ref.raw,+ ref.name,+ ModelReferenceKind.ALIAS,+ list(model_deck.img_gen_aliases.keys()),)case ModelReferenceKind.WATERFALL:if ref.name in model_deck.img_gen_waterfalls:returnmsg = f"Waterfall '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.IMG_GEN,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.WATERFALL,- available_options=list(model_deck.img_gen_waterfalls.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.IMG_GEN,+ ref.raw,+ ref.name,+ ModelReferenceKind.WATERFALL,+ list(model_deck.img_gen_waterfalls.keys()),)case ModelReferenceKind.HANDLE:if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.IMG_GEN):returnmsg = f"Model handle '{ref.name}' was not found in the model deck"- raise ModelChoiceNotFoundError(- message=msg,- model_type=ModelType.IMG_GEN,- model_choice=ref.raw,- reference_kind=ModelReferenceKind.HANDLE,- available_options=list(model_deck.inference_models.keys()),+ _raise_model_choice_not_found(+ msg,+ model_deck,+ ModelType.IMG_GEN,+ ref.raw,+ ref.name,+ ModelReferenceKind.HANDLE,+ list(model_deck.inference_models.keys()),)diff --git a/pipelex/cogt/models/model_suggestion.py b/pipelex/cogt/models/model_suggestion.pynew file mode 100644index 000000000..b37fa8247--- /dev/null+++ b/pipelex/cogt/models/model_suggestion.py@@ -0,0 +1,113 @@+"""Fuzzy model name matching and cross-collection suggestion utilities.++Provides functions to find close matches for model names and detect wrong-sigil usage.+Used by both the CLI check-model command and the model deck validation layer.+"""++import difflib++from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.models.model_reference import (+ SIGIL_ALIAS,+ SIGIL_PRESET,+ SIGIL_WATERFALL,+ ModelReferenceKind,+)++KIND_SIGILS: dict[ModelReferenceKind, str] = {+ ModelReferenceKind.PRESET: SIGIL_PRESET,+ ModelReferenceKind.ALIAS: SIGIL_ALIAS,+ ModelReferenceKind.WATERFALL: SIGIL_WATERFALL,+ ModelReferenceKind.HANDLE: "",+}++KIND_LABELS: dict[ModelReferenceKind, str] = {+ ModelReferenceKind.PRESET: "preset",+ ModelReferenceKind.ALIAS: "alias",+ ModelReferenceKind.WATERFALL: "waterfall",+ ModelReferenceKind.HANDLE: "handle",+}+++def get_collection_keys(+ model_deck: ModelDeck,+ model_type: ModelType,+ kind: ModelReferenceKind,+) -> list[str]:+ """Return known names for a given reference kind and model type."""+ match kind:+ case ModelReferenceKind.PRESET:+ match model_type:+ case ModelType.LLM:+ return list(model_deck.llm_presets.keys())+ case ModelType.TEXT_EXTRACTOR:+ return list(model_deck.extract_presets.keys())+ case ModelType.IMG_GEN:+ return list(model_deck.img_gen_presets.keys())+ case ModelType.SEARCH:+ return list(model_deck.search_presets.keys())+ case ModelReferenceKind.ALIAS:+ match model_type:+ case ModelType.LLM:+ return list(model_deck.llm_aliases.keys())+ case ModelType.TEXT_EXTRACTOR:+ return list(model_deck.extract_aliases.keys())+ case ModelType.IMG_GEN:+ return list(model_deck.img_gen_aliases.keys())+ case ModelType.SEARCH:+ return list(model_deck.search_aliases.keys())+ case ModelReferenceKind.WATERFALL:+ match model_type:+ case ModelType.LLM:+ return list(model_deck.llm_waterfalls.keys())+ case ModelType.TEXT_EXTRACTOR:+ return list(model_deck.extract_waterfalls.keys())+ case ModelType.IMG_GEN:+ return list(model_deck.img_gen_waterfalls.keys())+ case ModelType.SEARCH:+ return list(model_deck.search_waterfalls.keys())+ case ModelReferenceKind.HANDLE:+ return sorted(handle for handle, spec in model_deck.inference_models.items() if spec.model_type == model_type)+++def suggest_model_alternatives(+ model_deck: ModelDeck,+ model_type: ModelType,+ name: str,+ kind: ModelReferenceKind,+) -> tuple[list[str], list[str], list[str]]:+ """Find fuzzy matches and detect wrong-sigil usage for a model name.++ Args:+ model_deck: The model deck to search in.+ model_type: The model type (LLM, TEXT_EXTRACTOR, IMG_GEN, SEARCH).+ name: The bare name (without sigil) to match.+ kind: The reference kind the user specified (PRESET, ALIAS, WATERFALL, HANDLE).++ Returns:+ A tuple of three lists:+ - suggestions: fuzzy matches within the same collection, with sigil prefix+ - wrong_sigil_hints: exact matches found in other collections (e.g. "best-claude exists as @best-claude (alias)")+ - cross_collection_suggestions: fuzzy matches in other collections, with sigil and label+ """+ sigil = KIND_SIGILS[kind]+ candidates = get_collection_keys(model_deck, model_type, kind)+ fuzzy_matches = difflib.get_close_matches(name, candidates, n=5, cutoff=0.5)+ suggestions = [f"{sigil}{match}" for match in fuzzy_matches]++ wrong_sigil_hints: list[str] = []+ cross_suggestions: list[str] = []+ other_kinds = [other_kind for other_kind in ModelReferenceKind if other_kind != kind]+ for other_kind in other_kinds:+ other_candidates = get_collection_keys(model_deck, model_type, other_kind)+ other_sigil = KIND_SIGILS[other_kind]+ label = KIND_LABELS[other_kind]+ if name in other_candidates:+ wrong_sigil_hints.append(f"{name} exists as {other_sigil}{name} ({label})")+ else:+ fuzzy = difflib.get_close_matches(name, other_candidates, n=3, cutoff=0.7)+ for match in fuzzy:+ cross_suggestions.append(f"{other_sigil}{match} ({label})")++ return suggestions, wrong_sigil_hints, cross_suggestionsdiff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.pyindex be0f5fcc5..99e07e7cb 100644--- a/tests/integration/pipelex/test_data.py+++ b/tests/integration/pipelex/test_data.py@@ -440,7 +440,7 @@ class AssemblePipelexBundleSpecTestCases:description="Generate a plan from a user brief.",inputs={"brief": "UserBrief"},output="PlanDraft",- llm_talent="engineer",+ model="$engineering-structured",prompt="Generate a plan for: @brief",),]diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.pyindex 3dfd19c7b..5513a0037 100644--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_extract/test_data.py@@ -1,7 +1,6 @@from typing import ClassVarfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpec-from pipelex.builder.talents.extract_talent import ExtractTalentfrom pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprint@@ -13,7 +12,7 @@ class PipeExtractTestCases:description="Extract text from image",inputs={"image": "Image"},output="Page",- extract_talent=ExtractTalent.PDF_BASIC_TEXT_EXTRACTOR,+ model="@default-text-from-pdf",),PipeExtractBlueprint(source=None,@@ -31,7 +30,7 @@ class PipeExtractTestCases:description="Extract with page options",inputs={"document": "Document"},output="Page[]",- extract_talent=ExtractTalent.PDF_BASIC_TEXT_EXTRACTOR,+ model="@default-text-from-pdf",max_page_images=None,page_views=True,),diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.pyindex e94a35119..118a2d569 100644--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_img_gen/test_data.py@@ -1,7 +1,6 @@from typing import ClassVarfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec-from pipelex.builder.talents.img_gen_talent import ImgGenTalentfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint@@ -13,7 +12,7 @@ class PipeImgGenTestCases:description="Generate an image",inputs={"my_prompt": "Text"},output="native.Image",- img_gen_talent=ImgGenTalent.GEN_IMAGE,+ model="$gen-image",prompt="@my_prompt",),PipeImgGenBlueprint(@@ -38,7 +37,7 @@ class PipeImgGenTestCases:inputs={"description": "Text"},output="Image[3]",prompt="@description",- img_gen_talent=ImgGenTalent.GEN_IMAGE_FAST,+ model="$gen-image-fast",),PipeImgGenBlueprint(description="Generate image with options",diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.pyindex 83939dcf2..4d643ee4d 100644--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py@@ -1,7 +1,6 @@from typing import ClassVarfrom pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpec-from pipelex.builder.talents.llm_talent import LLMTalentfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint@@ -13,7 +12,7 @@ class PipeLLMTestCases:description="Generate text",inputs={"topic": "Text"},output="Text",- llm_talent=LLMTalent.CREATIVE_WRITER,+ model="$writing-creative",prompt="Write about $topic",),PipeLLMBlueprint(@@ -34,7 +33,7 @@ class PipeLLMTestCases:description="Generate without inputs",inputs={},output="Text",- llm_talent=LLMTalent.CREATIVE_WRITER,+ model="$writing-creative",prompt="Generate something interesting",),PipeLLMBlueprint(@@ -55,7 +54,7 @@ class PipeLLMTestCases:inputs={"image": "Image"},output="Text",prompt="Analyze the image: $image",- llm_talent=LLMTalent.VISION_LANGUAGE_MODEL,+ model="$vision",),PipeLLMBlueprint(source=None,@@ -76,7 +75,7 @@ class PipeLLMTestCases:inputs={},output="Text",prompt="Generate text",- llm_talent=LLMTalent.CREATIVE_WRITER,+ model="$writing-creative",),PipeLLMBlueprint(source=None,@@ -96,7 +95,7 @@ class PipeLLMTestCases:inputs={},output="Text",prompt="Generate text",- llm_talent=LLMTalent.DATA_RETRIEVAL,+ model="$retrieval",),PipeLLMBlueprint(source=None,@@ -117,7 +116,7 @@ class PipeLLMTestCases:output="Analysis",system_prompt="You are a data analyst",prompt="Analyze: @data",- llm_talent=LLMTalent.DATA_RETRIEVAL,+ model="$retrieval",),PipeLLMBlueprint(source=None,@@ -139,7 +138,7 @@ class PipeLLMTestCases:inputs={},output="Item[]",prompt="Generate items",- llm_talent=LLMTalent.CREATIVE_WRITER,+ model="$writing-creative",),PipeLLMBlueprint(source=None,@@ -159,7 +158,7 @@ class PipeLLMTestCases:inputs={},output="Item[5]",prompt="Generate items",- llm_talent=LLMTalent.CREATIVE_WRITER,+ model="$writing-creative",),PipeLLMBlueprint(source=None,diff --git a/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py b/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.pyindex ffb8b6f44..cfa203282 100644--- a/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py+++ b/tests/unit/pipelex/builder/test_builder_loop_multiplicity_fix.py@@ -301,7 +301,7 @@ class TestBuilderLoopMultiplicityFix:description="An LLM pipe",inputs={"input_text": "Text"},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Do something with @input_text",),},diff --git a/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py b/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.pyindex 7d50249e2..4fbabbdbc 100644--- a/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py+++ b/tests/unit/pipelex/builder/test_builder_loop_namespace_stripping.py@@ -59,7 +59,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Do stuff",),},@@ -83,7 +83,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Do stuff",),},@@ -107,7 +107,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Do stuff",),},@@ -132,7 +132,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Do stuff",),},@@ -156,7 +156,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Do stuff",),},@@ -193,7 +193,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={"item": "Text"},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Process @item",),},@@ -232,7 +232,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Step 1",),"step_two": PipeLLMSpec.model_construct(@@ -242,7 +242,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Step 2",),},@@ -284,7 +284,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Branch A",),"branch_b": PipeLLMSpec.model_construct(@@ -294,7 +294,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Branch B",),},@@ -336,7 +336,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Good",),"handle_bad": PipeLLMSpec.model_construct(@@ -346,7 +346,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Bad",),"handle_default": PipeLLMSpec.model_construct(@@ -356,7 +356,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Default",),},@@ -398,7 +398,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Good",),},@@ -440,7 +440,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Local",),},@@ -482,7 +482,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Local",),},@@ -552,7 +552,7 @@ class TestBuilderLoopNamespaceStripping:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Step 1",),},diff --git a/tests/unit/pipelex/builder/test_builder_loop_prune.py b/tests/unit/pipelex/builder/test_builder_loop_prune.pyindex 3ff4f8ea3..bd529e7e0 100644--- a/tests/unit/pipelex/builder/test_builder_loop_prune.py+++ b/tests/unit/pipelex/builder/test_builder_loop_prune.py@@ -72,7 +72,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={"doc": "external.Document"},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Process @doc",),},@@ -106,7 +106,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="my_domain.Report",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Generate a report",),},@@ -140,7 +140,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="Summary",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Generate a summary",),},@@ -173,7 +173,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="Result",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Generate a result",),},@@ -214,7 +214,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="Derived",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Generate derived content",),},@@ -253,7 +253,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="Container",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Generate container",),},@@ -315,7 +315,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Generate text",),},@@ -351,7 +351,7 @@ class TestBuilderLoopPruneUnreachableSpecs:pipe_category="PipeOperator",inputs={},output="Text",- llm_talent="data-retrieval",+ model="$retrieval",prompt="Orphan",),},diff --git a/tests/unit/pipelex/cli/test_agent_models_cmd.py b/tests/unit/pipelex/cli/test_agent_models_cmd.pyindex 56b9c04c8..a8ae8eeea 100644--- a/tests/unit/pipelex/cli/test_agent_models_cmd.py+++ b/tests/unit/pipelex/cli/test_agent_models_cmd.py@@ -24,31 +24,6 @@ class _FakeSetting:self.description = description-class _FakeTalentMappings:- """Minimal stand-in for TalentPresetMappings."""-- def __init__(self, llm: dict[str, str], img_gen: dict[str, str], extract: dict[str, str], search: dict[str, str] | None = None):- self.llm = llm- self.img_gen = img_gen- self.extract = extract- self.search = search or {}---class _FakeBuilderConfig:- def __init__(self, talent_preset_mappings: _FakeTalentMappings):- self.talent_preset_mappings = talent_preset_mappings---class _FakePipelexConfig:- def __init__(self, builder_config: _FakeBuilderConfig):- self.builder_config = builder_config---class _FakeConfig:- def __init__(self, pipelex: _FakePipelexConfig):- self.pipelex = pipelex--class _FakeInferenceModelSpec:"""Minimal stand-in for InferenceModelSpec."""@@ -79,11 +54,6 @@ class TestData:EXTRACT_WATERFALLS: ClassVar[dict[str, list[str]]] = {"extract-wf": ["textract-model"]}IMG_GEN_WATERFALLS: ClassVar[dict[str, list[str]]] = {"img-wf": ["dall-e-3"]}- TALENT_LLM: ClassVar[dict[str, str]] = {"text_gen": "fast"}- TALENT_IMG_GEN: ClassVar[dict[str, str]] = {"image_gen": "hd-image"}- TALENT_EXTRACT: ClassVar[dict[str, str]] = {"extraction": "doc-extract"}-- # Backend resolution map: model_handle -> (backend_name, model_type)INFERENCE_MAP: ClassVar[dict[str, tuple[str, ModelType]]] = {"gpt-4o-mini": ("openai", ModelType.LLM),"claude-sonnet": ("anthropic", ModelType.LLM),@@ -123,25 +93,10 @@ def _make_fake_model_deck() -> Any:return FakeModelDeck()-def _make_fake_config() -> _FakeConfig:- """Create a fake config with talent mappings."""- talent_mappings = _FakeTalentMappings(- llm=TestData.TALENT_LLM,- img_gen=TestData.TALENT_IMG_GEN,- extract=TestData.TALENT_EXTRACT,- )- return _FakeConfig(- pipelex=_FakePipelexConfig(- builder_config=_FakeBuilderConfig(talent_preset_mappings=talent_mappings),- ),- )--def _setup_mocks(mocker: MockerFixture) -> None:"""Patch the common dependencies for agent_models_cmd."""mocker.patch(f"{MODULE_PATH}.make_pipelex_for_agent_cli")mocker.patch(f"{MODULE_PATH}.get_model_deck", return_value=_make_fake_model_deck())- mocker.patch(f"{MODULE_PATH}.get_config", return_value=_make_fake_config())mocker.patch(f"{MODULE_PATH}.Pipelex")@@ -149,18 +104,28 @@ class TestAgentModelsCmd:"""Tests for agent_models_cmd JSON output with --type and --backend filters."""def test_no_filters_returns_all_categories(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:- """No filters should return all 3 categories in every section."""+ """No filters should return all categories in every section."""_setup_mocks(mocker)agent_models_cmd(ctx=agent_ctx, output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)assert parsed["success"] is True- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):+ for section in ("presets", "aliases", "waterfalls"):assert "llm" in parsed[section], f"llm missing from {section}"assert "img_gen" in parsed[section], f"img_gen missing from {section}"assert "extract" in parsed[section], f"extract missing from {section}"+ def test_no_talent_mappings_in_output(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """Output should not contain talent_mappings or usage hint."""+ _setup_mocks(mocker)++ agent_models_cmd(ctx=agent_ctx, output_format=CliOutputFormat.JSON)++ parsed = json.loads(capsys.readouterr().out)+ assert "talent_mappings" not in parsed+ assert "talent_mappings_usage_hint" not in parsed+def test_no_filters_preset_content(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:"""No filters should include all preset entries with correct data."""_setup_mocks(mocker)@@ -171,7 +136,6 @@ class TestAgentModelsCmd:llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]assert "fast" in llm_preset_namesassert "smart" in llm_preset_names- # Check description is includedfast_preset = next(preset for preset in parsed["presets"]["llm"] if preset["name"] == "fast")assert fast_preset["description"] == "Fast LLM"@@ -182,7 +146,7 @@ class TestAgentModelsCmd:agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.LLM], output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):+ for section in ("presets", "aliases", "waterfalls"):assert "llm" in parsed[section], f"llm missing from {section}"assert "img_gen" not in parsed[section], f"img_gen should not be in {section}"assert "extract" not in parsed[section], f"extract should not be in {section}"@@ -194,7 +158,7 @@ class TestAgentModelsCmd:agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.EXTRACT], output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):+ for section in ("presets", "aliases", "waterfalls"):assert "extract" in parsed[section], f"extract missing from {section}"assert "llm" not in parsed[section], f"llm should not be in {section}"assert "img_gen" not in parsed[section], f"img_gen should not be in {section}"@@ -206,7 +170,7 @@ class TestAgentModelsCmd:agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.IMG_GEN], output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):+ for section in ("presets", "aliases", "waterfalls"):assert "img_gen" in parsed[section], f"img_gen missing from {section}"assert "llm" not in parsed[section], f"llm should not be in {section}"assert "extract" not in parsed[section], f"extract should not be in {section}"@@ -218,7 +182,7 @@ class TestAgentModelsCmd:agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.LLM, ModelCategory.IMG_GEN], output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):+ for section in ("presets", "aliases", "waterfalls"):assert "llm" in parsed[section], f"llm missing from {section}"assert "img_gen" in parsed[section], f"img_gen missing from {section}"assert "extract" not in parsed[section], f"extract should not be in {section}"@@ -230,16 +194,12 @@ class TestAgentModelsCmd:agent_models_cmd(ctx=agent_ctx, backend="openai", output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)- # LLM: only 'fast' preset (gpt-4o-mini on openai), not 'smart' (claude-sonnet on anthropic)llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]assert "fast" in llm_preset_namesassert "smart" not in llm_preset_names- # LLM aliases: best-llm -> claude-sonnet (anthropic), should be excludedassert len(parsed["aliases"]["llm"]) == 0- # IMG_GEN: dall-e-3 is on openai, should be includedimg_gen_preset_names = [preset["name"] for preset in parsed["presets"]["img_gen"]]assert "hd-image" in img_gen_preset_names- # Extract: textract-model is on aws, should be excludedassert len(parsed["presets"]["extract"]) == 0def test_backend_filter_anthropic(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:@@ -252,7 +212,6 @@ class TestAgentModelsCmd:llm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]assert "smart" in llm_preset_namesassert "fast" not in llm_preset_names- # Alias best-llm -> claude-sonnet (anthropic) should be includedassert "best-llm" in parsed["aliases"]["llm"]def test_type_and_backend_combined(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:@@ -262,24 +221,22 @@ class TestAgentModelsCmd:agent_models_cmd(ctx=agent_ctx, model_type=[ModelCategory.LLM], backend="openai", output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)- # Only llm categoryassert "llm" in parsed["presets"]assert "img_gen" not in parsed["presets"]assert "extract" not in parsed["presets"]- # Only openai-backed LLM presetsllm_preset_names = [preset["name"] for preset in parsed["presets"]["llm"]]assert "fast" in llm_preset_namesassert "smart" not in llm_preset_namesdef test_backend_nonexistent(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:- """--backend nonexistent should produce empty presets/aliases/waterfalls/talent_mappings."""+ """--backend nonexistent should produce empty presets/aliases/waterfalls."""_setup_mocks(mocker)agent_models_cmd(ctx=agent_ctx, backend="nonexistent", output_format=CliOutputFormat.JSON)parsed = json.loads(capsys.readouterr().out)assert parsed["success"] is True- for section in ("presets", "aliases", "waterfalls", "talent_mappings"):+ for section in ("presets", "aliases", "waterfalls"):for category in ("llm", "img_gen", "extract"):assert len(parsed[section][category]) == 0, f"{section}.{category} should be empty"@@ -293,14 +250,3 @@ class TestAgentModelsCmd:assert "llm-wf" in parsed["waterfalls"]["llm"]assert len(parsed["waterfalls"]["img_gen"]) == 0assert len(parsed["waterfalls"]["extract"]) == 0-- def test_talent_mappings_backend_filter(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:- """--backend openai should include LLM talent_mapping (fast->gpt-4o-mini) and img_gen, but not extract."""- _setup_mocks(mocker)-- agent_models_cmd(ctx=agent_ctx, backend="openai", output_format=CliOutputFormat.JSON)-- parsed = json.loads(capsys.readouterr().out)- assert "text_gen" in parsed["talent_mappings"]["llm"]- assert "image_gen" in parsed["talent_mappings"]["img_gen"]- assert len(parsed["talent_mappings"]["extract"]) == 0diff --git a/tests/unit/pipelex/cli/test_check_model_cmd.py b/tests/unit/pipelex/cli/test_check_model_cmd.pynew file mode 100644index 000000000..11080664a--- /dev/null+++ b/tests/unit/pipelex/cli/test_check_model_cmd.py@@ -0,0 +1,194 @@+"""Unit tests for the agent CLI check-model command."""++from __future__ import annotations++import json+from typing import TYPE_CHECKING, Any, ClassVar++if TYPE_CHECKING:+ import pytest+ from pytest_mock import MockerFixture++from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat+from pipelex.cli.agent_cli.commands.check_model_cmd import agent_check_model_cmd+from pipelex.cli.agent_cli.commands.models_cmd import ModelCategory+from pipelex.cogt.model_backends.model_type import ModelType++MODULE_PATH = "pipelex.cli.agent_cli.commands.check_model_cmd"+++class _FakeSetting:+ """Minimal stand-in for LLMSetting / ExtractSetting."""++ def __init__(self, model: str, description: str | None = None):+ self.model = model+ self.description = description+++class _FakeInferenceModelSpec:+ """Minimal stand-in for InferenceModelSpec."""++ def __init__(self, model_type: ModelType):+ self.model_type = model_type+++class TestData:+ """Shared test data constants."""++ LLM_PRESETS: ClassVar[dict[str, _FakeSetting]] = {+ "writing-creative": _FakeSetting(model="claude-sonnet", description="Creative writing"),+ "writing-factual": _FakeSetting(model="claude-sonnet", description="Factual writing"),+ "deep-analysis": _FakeSetting(model="claude-opus", description="Deep analysis"),+ }+ LLM_ALIASES: ClassVar[dict[str, str]] = {+ "best-claude": "claude-sonnet",+ "best-gpt": "gpt-4o",+ "default-general": "claude-sonnet",+ }+ LLM_WATERFALLS: ClassVar[dict[str, list[str]]] = {+ "robust-llm": ["claude-sonnet", "gpt-4o"],+ }+ INFERENCE_MODELS: ClassVar[dict[str, _FakeInferenceModelSpec]] = {+ "claude-4.5-sonnet": _FakeInferenceModelSpec(model_type=ModelType.LLM),+ "claude-4-sonnet": _FakeInferenceModelSpec(model_type=ModelType.LLM),+ "claude-4.6-opus": _FakeInferenceModelSpec(model_type=ModelType.LLM),+ "gpt-4o": _FakeInferenceModelSpec(model_type=ModelType.LLM),+ "gpt-4o-mini": _FakeInferenceModelSpec(model_type=ModelType.LLM),+ }++ EXTRACT_PRESETS: ClassVar[dict[str, Any]] = {}+ EXTRACT_ALIASES: ClassVar[dict[str, str]] = {}+ EXTRACT_WATERFALLS: ClassVar[dict[str, list[str]]] = {}+ IMG_GEN_PRESETS: ClassVar[dict[str, Any]] = {}+ IMG_GEN_ALIASES: ClassVar[dict[str, str]] = {}+ IMG_GEN_WATERFALLS: ClassVar[dict[str, list[str]]] = {}+ SEARCH_PRESETS: ClassVar[dict[str, Any]] = {}+ SEARCH_ALIASES: ClassVar[dict[str, str]] = {}+ SEARCH_WATERFALLS: ClassVar[dict[str, list[str]]] = {}+++def _make_fake_model_deck() -> Any:+ """Create a fake ModelDeck with LLM test data."""++ class FakeModelDeck:+ llm_presets = TestData.LLM_PRESETS+ llm_aliases = TestData.LLM_ALIASES+ llm_waterfalls = TestData.LLM_WATERFALLS+ inference_models = TestData.INFERENCE_MODELS++ extract_presets = TestData.EXTRACT_PRESETS+ extract_aliases = TestData.EXTRACT_ALIASES+ extract_waterfalls = TestData.EXTRACT_WATERFALLS+ img_gen_presets = TestData.IMG_GEN_PRESETS+ img_gen_aliases = TestData.IMG_GEN_ALIASES+ img_gen_waterfalls = TestData.IMG_GEN_WATERFALLS+ search_presets = TestData.SEARCH_PRESETS+ search_aliases = TestData.SEARCH_ALIASES+ search_waterfalls = TestData.SEARCH_WATERFALLS++ return FakeModelDeck()+++def _setup_mocks(mocker: MockerFixture) -> None:+ """Patch the common dependencies for agent_check_model_cmd."""+ mocker.patch(f"{MODULE_PATH}.make_pipelex_for_agent_cli")+ mocker.patch(f"{MODULE_PATH}.get_model_deck", return_value=_make_fake_model_deck())+ mocker.patch(f"{MODULE_PATH}.Pipelex")+++def _run_check(+ agent_ctx: Any,+ mocker: MockerFixture,+ capsys: pytest.CaptureFixture[str],+ name: str,+ model_type: ModelCategory = ModelCategory.LLM,+ output_format: CliOutputFormat = CliOutputFormat.JSON,+) -> dict[str, Any]:+ """Run check-model and return parsed JSON output."""+ _setup_mocks(mocker)+ agent_check_model_cmd(ctx=agent_ctx, name=name, model_type=model_type, output_format=output_format)+ parsed: dict[str, Any] = json.loads(capsys.readouterr().out)+ return parsed+++class TestCheckModelCmd:+ """Tests for agent_check_model_cmd validation and fuzzy matching."""++ def test_valid_preset(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A known preset name with $ sigil should be valid."""+ result = _run_check(agent_ctx, mocker, capsys, "$writing-creative")+ assert result["valid"] is True+ assert result["kind"] == "preset"++ def test_valid_alias(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A known alias name with @ sigil should be valid."""+ result = _run_check(agent_ctx, mocker, capsys, "@best-claude")+ assert result["valid"] is True+ assert result["kind"] == "alias"++ def test_valid_handle(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A known bare model handle should be valid."""+ result = _run_check(agent_ctx, mocker, capsys, "claude-4.5-sonnet")+ assert result["valid"] is True+ assert result["kind"] == "handle"++ def test_invalid_preset_with_fuzzy_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A typo in a preset name should return fuzzy suggestions with $ prefix."""+ result = _run_check(agent_ctx, mocker, capsys, "$writting-creative")+ assert result["valid"] is False+ assert any("$writing-creative" in suggestion for suggestion in result["suggestions"])++ def test_wrong_sigil_detected(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """Using $ for a name that exists as an alias should produce a wrong-sigil hint."""+ result = _run_check(agent_ctx, mocker, capsys, "$best-claude")+ assert result["valid"] is False+ assert any("@best-claude" in hint for hint in result["wrong_sigil_hints"])++ def test_invalid_handle_cross_collection_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A bare name matching a preset should suggest the prefixed version."""+ result = _run_check(agent_ctx, mocker, capsys, "writing-creative")+ assert result["valid"] is False+ assert any("$writing-creative" in hint for hint in result["wrong_sigil_hints"])++ def test_invalid_handle_fuzzy_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A typo in a handle should return fuzzy suggestions without sigil prefix."""+ result = _run_check(agent_ctx, mocker, capsys, "claude-4.5-sonet")+ assert result["valid"] is False+ assert "claude-4.5-sonnet" in result["suggestions"]++ def test_completely_unknown_name(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A completely unknown name should return no suggestions."""+ result = _run_check(agent_ctx, mocker, capsys, "$zzz-nonexistent-xyz")+ assert result["valid"] is False+ assert len(result["suggestions"]) == 0++ def test_json_output_structure(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """JSON output should contain all expected keys on failure."""+ result = _run_check(agent_ctx, mocker, capsys, "$nonexistent")+ assert result["success"] is True+ assert result["valid"] is False+ assert "suggestions" in result+ assert "wrong_sigil_hints" in result+ assert "cross_collection_suggestions" in result+ assert result["model_type"] == "llm"++ def test_markdown_valid_output(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """Markdown output for a valid reference should be a single confirmation line."""+ _setup_mocks(mocker)+ agent_check_model_cmd(ctx=agent_ctx, name="$writing-creative", model_type=ModelCategory.LLM, output_format=CliOutputFormat.MARKDOWN)+ output = capsys.readouterr().out.strip()+ assert output == "$writing-creative is a valid llm preset."++ def test_markdown_invalid_output_has_suggestions(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """Markdown output for an invalid reference should include 'Did you mean' suggestions."""+ _setup_mocks(mocker)+ agent_check_model_cmd(ctx=agent_ctx, name="$writting-creative", model_type=ModelCategory.LLM, output_format=CliOutputFormat.MARKDOWN)+ output = capsys.readouterr().out+ assert "is not a valid" in output+ assert "Did you mean:" in output++ def test_valid_waterfall(self, agent_ctx: Any, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A known waterfall name with ~ sigil should be valid."""+ result = _run_check(agent_ctx, mocker, capsys, "~robust-llm")+ assert result["valid"] is True+ assert result["kind"] == "waterfall"diff --git a/tests/unit/pipelex/cli/test_format_models_markdown.py b/tests/unit/pipelex/cli/test_format_models_markdown.pyindex 1fa9365a0..e0bd9b53c 100644--- a/tests/unit/pipelex/cli/test_format_models_markdown.py+++ b/tests/unit/pipelex/cli/test_format_models_markdown.py@@ -35,13 +35,6 @@ class TestFormatModelsMarkdown:"extract": {},"search": {},},- "talent_mappings": {- "llm": {"creative-writer": "writing-creative"},- "img_gen": {},- "extract": {},- "search": {},- },- "talent_mappings_usage_hint": "Use the talent name as the value for llm_talent",}def test_output_starts_with_heading(self) -> None:@@ -54,35 +47,30 @@ class TestFormatModelsMarkdown:output = _format_models_markdown(self._FULL_RESULT)assert "## LLM" in output- def test_preset_with_description(self) -> None:- """Preset with description should show name and description."""- output = _format_models_markdown(self._FULL_RESULT)- assert "- **writing-creative**: Creative writing" in output-- def test_preset_without_description(self) -> None:- """Preset without description should show only the name."""+ def test_preset_with_description_uses_sigil(self) -> None:+ """Preset with description should show $-prefixed name and description."""output = _format_models_markdown(self._FULL_RESULT)- assert "- **engineering-code**" in output+ assert "- $writing-creative: Creative writing" in output- def test_aliases_use_arrow(self) -> None:- """Aliases should use arrow notation."""+ def test_preset_without_description_uses_sigil(self) -> None:+ """Preset without description should show only the $-prefixed name."""output = _format_models_markdown(self._FULL_RESULT)- assert "- **fast** \u2192 gpt4o-mini" in output+ assert "- $engineering-code" in output- def test_waterfalls_show_chain(self) -> None:- """Waterfalls should show the full fallback chain with arrows."""+ def test_aliases_use_sigil_and_arrow(self) -> None:+ """Aliases should use @-prefix and arrow notation."""output = _format_models_markdown(self._FULL_RESULT)- assert "gpt4o \u2192 claude-sonnet \u2192 gemini" in output+ assert "- @fast \u2192 gpt4o-mini" in output- def test_talent_mappings_present(self) -> None:- """Talent mappings should use arrow notation."""+ def test_waterfalls_use_sigil_and_chain(self) -> None:+ """Waterfalls should use ~-prefix and show the full fallback chain."""output = _format_models_markdown(self._FULL_RESULT)- assert "- **creative-writer** \u2192 writing-creative" in output+ assert "- ~robust: gpt4o \u2192 claude-sonnet \u2192 gemini" in output- def test_hint_present(self) -> None:- """Usage hint should appear at the bottom."""+ def test_no_bold_markers(self) -> None:+ """Output should not contain bold markdown markers."""output = _format_models_markdown(self._FULL_RESULT)- assert "Use the talent name as the value for llm_talent" in output+ assert "**" not in outputdef test_empty_categories_omitted(self) -> None:"""Categories with no data should not appear in output."""@@ -91,24 +79,21 @@ class TestFormatModelsMarkdown:assert "## Search" not in outputdef test_img_gen_category_present(self) -> None:- """Image Generation category with presets should appear."""+ """Image Generation category with presets should appear with sigil."""output = _format_models_markdown(self._FULL_RESULT)assert "## Image Generation" in output- assert "- **gen-image**: Image generation" in output+ assert "- $gen-image: Image generation" in outputdef test_all_empty_produces_heading_only(self) -> None:- """Result with all empty sections should produce only the heading and hint."""+ """Result with all empty sections should produce only the heading."""empty_result: dict[str, Any] = {"presets": {"llm": [], "img_gen": [], "extract": [], "search": []},"aliases": {"llm": {}, "img_gen": {}, "extract": {}, "search": {}},"waterfalls": {"llm": {}, "img_gen": {}, "extract": {}, "search": {}},- "talent_mappings": {"llm": {}, "img_gen": {}, "extract": {}, "search": {}},- "talent_mappings_usage_hint": "hint text",}output = _format_models_markdown(empty_result)assert "# Available Models" in outputassert "## LLM" not in output- assert "hint text" in outputdef test_empty_subsections_omitted_within_category(self) -> None:"""Within a category, subsections with no data should be omitted."""@@ -116,10 +101,18 @@ class TestFormatModelsMarkdown:"presets": {"llm": [{"name": "only-preset"}]},"aliases": {"llm": {}},"waterfalls": {"llm": {}},- "talent_mappings": {"llm": {}},}output = _format_models_markdown(result)assert "### Presets" in outputassert "### Aliases" not in outputassert "### Waterfalls" not in output- assert "### Talent Mappings" not in output++ def test_presets_description_line(self) -> None:+ """Presets section should include the one-liner description."""+ output = _format_models_markdown(self._FULL_RESULT)+ assert "Presets are the preferred way to specify models" in output++ def test_aliases_description_line(self) -> None:+ """Aliases section should include the one-liner description."""+ output = _format_models_markdown(self._FULL_RESULT)+ assert "Aliases provide stable names" in outputdiff --git a/tests/unit/pipelex/cli/test_pipe_cmd.py b/tests/unit/pipelex/cli/test_pipe_cmd.pyindex 9b2659509..ffa9dae73 100644--- a/tests/unit/pipelex/cli/test_pipe_cmd.py+++ b/tests/unit/pipelex/cli/test_pipe_cmd.py@@ -8,117 +8,10 @@ import pytestfrom pydantic import ValidationErrorfrom pipelex.cli.agent_cli.commands.pipe_cmd import (- _MODEL_TO_EXTRACT_TALENT, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]- _MODEL_TO_IMG_GEN_TALENT, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]- _MODEL_TO_LLM_TALENT, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]- LLM_TALENT_TO_MODEL,_parse_pipe_spec_from_json, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]_pipe_spec_to_toml, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]- _resolve_talent_value, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage])-# ----------------------------------------------------------------------------# Reverse talent mappings-# ------------------------------------------------------------------------------class TestReverseTalentMappings:- """Verify the module-level reverse mappings are built correctly."""-- def test_llm_preset_with_prefix(self) -> None:- assert _MODEL_TO_LLM_TALENT["$writing-creative"] == "creative-writer"-- def test_llm_preset_without_prefix(self) -> None:- assert _MODEL_TO_LLM_TALENT["writing-creative"] == "creative-writer"-- def test_llm_all_presets_have_reverse_entries(self) -> None:- for talent, preset in LLM_TALENT_TO_MODEL.items():- assert _MODEL_TO_LLM_TALENT[preset] == talent or _MODEL_TO_LLM_TALENT[preset] in LLM_TALENT_TO_MODEL- assert _MODEL_TO_LLM_TALENT[preset.lstrip("$@")] == talent or _MODEL_TO_LLM_TALENT[preset.lstrip("$@")] in LLM_TALENT_TO_MODEL-- def test_img_gen_preset_with_prefix(self) -> None:- assert _MODEL_TO_IMG_GEN_TALENT["$gen-image"] == "gen-image"-- def test_img_gen_preset_without_prefix(self) -> None:- assert _MODEL_TO_IMG_GEN_TALENT["gen-image"] == "gen-image"-- def test_extract_preset_with_prefix(self) -> None:- assert _MODEL_TO_EXTRACT_TALENT["@default-text-from-pdf"] == "pdf-basic-text-extractor"-- def test_extract_preset_without_prefix(self) -> None:- assert _MODEL_TO_EXTRACT_TALENT["default-text-from-pdf"] == "pdf-basic-text-extractor"---# ----------------------------------------------------------------------------# _resolve_talent_value-# ------------------------------------------------------------------------------class TestResolveTalentValue:- """Tests for the preset-to-talent resolution helper."""-- # --- PipeLLM ----- def test_valid_talent_passes_through(self) -> None:- assert _resolve_talent_value("PipeLLM", "creative-writer") == "creative-writer"-- def test_preset_without_prefix_resolves(self) -> None:- assert _resolve_talent_value("PipeLLM", "writing-creative") == "creative-writer"-- def test_preset_with_dollar_prefix_resolves(self) -> None:- assert _resolve_talent_value("PipeLLM", "$writing-creative") == "creative-writer"-- def test_invalid_talent_passes_through(self) -> None:- """Unknown values pass through so Pydantic can produce a clear error."""- assert _resolve_talent_value("PipeLLM", "totally-wrong") == "totally-wrong"-- def test_engineering_code_resolves(self) -> None:- assert _resolve_talent_value("PipeLLM", "engineering-code") == "coder"-- def test_retrieval_resolves(self) -> None:- assert _resolve_talent_value("PipeLLM", "retrieval") == "data-retrieval"-- # --- PipeImgGen ----- def test_img_gen_preset_resolves(self) -> None:- assert _resolve_talent_value("PipeImgGen", "$gen-image-fast") == "gen-image-fast"-- def test_img_gen_valid_talent_passes_through(self) -> None:- assert _resolve_talent_value("PipeImgGen", "gen-image-high-quality") == "gen-image-high-quality"-- # --- PipeExtract ----- def test_extract_preset_resolves(self) -> None:- assert _resolve_talent_value("PipeExtract", "default-extract-document") == "full-document-extractor"-- def test_extract_preset_with_at_resolves(self) -> None:- assert _resolve_talent_value("PipeExtract", "@default-extract-image") == "image-text-extractor"-- # --- Unknown pipe types ----- def test_unknown_pipe_type_passes_through(self) -> None:- assert _resolve_talent_value("PipeCompose", "anything") == "anything"-- def test_pipe_search_passes_through(self) -> None:- """PipeSearch has no reverse mapping; values pass through."""- assert _resolve_talent_value("PipeSearch", "web-search") == "web-search"-- def test_dict_value_passes_through(self) -> None:- """Non-string values (dict) should pass through without crashing."""- result = _resolve_talent_value("PipeLLM", {"name": "creative-writer"})- assert result == {"name": "creative-writer"}-- def test_list_value_passes_through(self) -> None:- """Non-string values (list) should pass through without crashing."""- result = _resolve_talent_value("PipeLLM", ["creative-writer"])- assert result == ["creative-writer"]-- def test_int_value_passes_through(self) -> None:- """Non-string values (int) should pass through without crashing."""- result = _resolve_talent_value("PipeLLM", 42)- assert result == 42--# ---------------------------------------------------------------------------# _parse_pipe_spec_from_json — pipe_code aliases# ---------------------------------------------------------------------------@@ -129,7 +22,7 @@ class TestPipeCodeAliases:_BASE_LLM: ClassVar[dict[str, Any]] = {"description": "Test pipe",- "llm_talent": "creative-writer",+ "model": "$writing-creative","inputs": {"text": "Text"},"output": "Text","prompt": "Write something about @text",@@ -174,128 +67,6 @@ class TestPipeCodeAliases:assert result.pipe_code == "my_pipe"-# ----------------------------------------------------------------------------# _parse_pipe_spec_from_json — talent field aliases-# ------------------------------------------------------------------------------class TestTalentFieldAliases:- """The parser should accept 'talent_name' and 'talent' as generic talent aliases."""-- _BASE: ClassVar[dict[str, Any]] = {- "pipe_code": "test_pipe",- "description": "Test pipe",- "inputs": {"text": "Text"},- "output": "Text",- "prompt": "Write about @text",- }-- def test_talent_name_alias_for_llm(self) -> None:- spec = {**self._BASE, "talent_name": "creative-writer"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]-- def test_talent_alias_for_llm(self) -> None:- spec = {**self._BASE, "talent": "engineer"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "engineer" # type: ignore[attr-defined]-- def test_canonical_talent_field_ignores_alias(self) -> None:- """When llm_talent is present, generic alias is removed so Pydantic doesn't reject it."""- spec = {**self._BASE, "llm_talent": "coder", "talent_name": "engineer"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "coder" # type: ignore[attr-defined]-- def test_talent_alias_for_extract(self) -> None:- spec = {- "pipe_code": "test_extract",- "description": "Extract pipe",- "inputs": {"doc": "Document"},- "output": "Page[]",- "talent_name": "full-document-extractor",- }- result = _parse_pipe_spec_from_json("PipeExtract", spec)- assert result.extract_talent == "full-document-extractor" # type: ignore[attr-defined]-- def test_talent_alias_for_img_gen(self) -> None:- spec = {- "pipe_code": "test_img_gen",- "description": "Image gen pipe",- "inputs": {"prompt_text": "ImgGenPrompt"},- "output": "Image",- "talent": "gen-image",- "prompt": "Generate: $prompt_text",- }- result = _parse_pipe_spec_from_json("PipeImgGen", spec)- assert result.img_gen_talent == "gen-image" # type: ignore[attr-defined]---# ----------------------------------------------------------------------------# _parse_pipe_spec_from_json — preset-to-talent resolution-# ------------------------------------------------------------------------------class TestPresetToTalentResolution:- """Preset names used where talent names are expected should be auto-resolved."""-- _BASE_LLM: ClassVar[dict[str, Any]] = {- "pipe_code": "test_pipe",- "description": "Test pipe",- "inputs": {"text": "Text"},- "output": "Text",- "prompt": "Write about @text",- }-- def test_preset_name_resolved_to_talent_for_llm(self) -> None:- spec = {**self._BASE_LLM, "llm_talent": "writing-creative"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]-- def test_preset_with_dollar_resolved_for_llm(self) -> None:- spec = {**self._BASE_LLM, "llm_talent": "$writing-creative"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]-- def test_valid_talent_unchanged_for_llm(self) -> None:- spec = {**self._BASE_LLM, "llm_talent": "creative-writer"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]-- def test_invalid_talent_raises_validation_error(self) -> None:- spec = {**self._BASE_LLM, "llm_talent": "totally-invalid"}- with pytest.raises(ValidationError):- _parse_pipe_spec_from_json("PipeLLM", spec)-- def test_preset_via_generic_alias_also_resolved(self) -> None:- """talent_name alias + preset value should both be handled."""- spec = {**self._BASE_LLM, "talent_name": "engineering-code"}- result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "coder" # type: ignore[attr-defined]-- def test_extract_preset_resolved(self) -> None:- spec = {- "pipe_code": "test_extract",- "description": "Extract pipe",- "inputs": {"doc": "Document"},- "output": "Page[]",- "extract_talent": "default-text-from-pdf",- }- result = _parse_pipe_spec_from_json("PipeExtract", spec)- assert result.extract_talent == "pdf-basic-text-extractor" # type: ignore[attr-defined]-- def test_img_gen_preset_resolved(self) -> None:- spec = {- "pipe_code": "test_img_gen",- "description": "Image gen pipe",- "inputs": {"prompt_text": "ImgGenPrompt"},- "output": "Image",- "img_gen_talent": "$gen-image-high-quality",- "prompt": "Generate: $prompt_text",- }- result = _parse_pipe_spec_from_json("PipeImgGen", spec)- assert result.img_gen_talent == "gen-image-high-quality" # type: ignore[attr-defined]--# ---------------------------------------------------------------------------# _parse_pipe_spec_from_json — output dict tolerance# ---------------------------------------------------------------------------@@ -307,7 +78,7 @@ class TestOutputDictTolerance:_BASE_LLM: ClassVar[dict[str, Any]] = {"pipe_code": "test_pipe","description": "Test pipe",- "llm_talent": "creative-writer",+ "model": "$writing-creative","inputs": {"text": "Text"},"prompt": "Write about @text",}@@ -431,7 +202,7 @@ class TestPipeTypeAlias:"type": "PipeLLM","pipe_code": "test","description": "Test",- "llm_talent": "creative-writer",+ "model": "$writing-creative","inputs": {"text": "Text"},"output": "Text","prompt": "Write about @text",@@ -456,34 +227,33 @@ class TestInvalidPipeType:# ----------------------------------------------------------------------------# _pipe_spec_to_toml — talent-to-model in TOML output+# _pipe_spec_to_toml — model in TOML output# ---------------------------------------------------------------------------class TestPipeSpecToToml:- """Verify TOML output maps talents to model presets correctly."""+ """Verify TOML output contains model presets correctly."""_BASE_LLM: ClassVar[dict[str, Any]] = {"pipe_code": "my_llm_pipe","description": "Test LLM pipe",- "llm_talent": "creative-writer",+ "model": "$writing-creative","inputs": {"text": "Text"},"output": "Text","prompt": "Write about @text",}- def test_llm_talent_becomes_model_preset_in_toml(self) -> None:+ def test_llm_model_appears_in_toml(self) -> None:spec = _parse_pipe_spec_from_json("PipeLLM", {**self._BASE_LLM})toml = _pipe_spec_to_toml(spec)assert 'model = "$writing-creative"' in toml- def test_llm_preset_input_still_maps_to_correct_model(self) -> None:- """If the agent provides a preset name, it should resolve and then map to model."""- spec = _parse_pipe_spec_from_json("PipeLLM", {**self._BASE_LLM, "llm_talent": "engineering-structured"})+ def test_llm_different_model_in_toml(self) -> None:+ spec = _parse_pipe_spec_from_json("PipeLLM", {**self._BASE_LLM, "model": "$engineering-structured"})toml = _pipe_spec_to_toml(spec)assert 'model = "$engineering-structured"' in toml- def test_extract_talent_becomes_model_in_toml(self) -> None:+ def test_extract_model_in_toml(self) -> None:spec = _parse_pipe_spec_from_json("PipeExtract",{@@ -491,13 +261,13 @@ class TestPipeSpecToToml:"description": "Extract pipe","inputs": {"doc": "Document"},"output": "Page[]",- "extract_talent": "full-document-extractor",+ "model": "@default-extract-document",},)toml = _pipe_spec_to_toml(spec)assert 'model = "@default-extract-document"' in toml- def test_img_gen_talent_becomes_model_in_toml(self) -> None:+ def test_img_gen_model_in_toml(self) -> None:spec = _parse_pipe_spec_from_json("PipeImgGen",{@@ -505,7 +275,7 @@ class TestPipeSpecToToml:"description": "Image gen pipe","inputs": {"prompt_text": "ImgGenPrompt"},"output": "Image",- "img_gen_talent": "gen-image",+ "model": "$gen-image","prompt": "Generate: $prompt_text",},)@@ -541,35 +311,50 @@ class TestPipeSpecToToml:assert "step_one" in tomlassert "step_two" in toml+ def test_llm_no_model_omits_model_in_toml(self) -> None:+ """When model is None, no model line should appear in the TOML."""+ spec = _parse_pipe_spec_from_json(+ "PipeLLM",+ {+ "pipe_code": "my_pipe",+ "description": "No model specified",+ "inputs": {"text": "Text"},+ "output": "Text",+ "prompt": "Write about @text",+ },+ )+ toml = _pipe_spec_to_toml(spec)+ assert "model =" not in toml+# ----------------------------------------------------------------------------# End-to-end: combined tolerance scenarios (the original bug)+# End-to-end: combined tolerance scenarios# ---------------------------------------------------------------------------class TestEndToEndTolerance:- """Reproduce the exact bug scenario that motivated these changes."""+ """Test combined tolerance scenarios with direct model field."""- def test_original_bug_preset_as_talent_with_dict_output(self) -> None:- """The agent sent 'writing-creative' (preset) and output as dict — both should be tolerated."""+ def test_model_with_dict_output_both_tolerated(self) -> None:+ """The agent sends model preset and output as dict — both should be tolerated."""spec: dict[str, Any] = {"pipe_code": "generate_prompt","description": "Generate an image prompt from an idea",- "llm_talent": "writing-creative",+ "model": "$writing-creative","inputs": {"idea": "Text"},"output": {"type": "ImgGenPrompt"},"prompt": "Generate a creative image prompt based on @idea",}result = _parse_pipe_spec_from_json("PipeLLM", spec)- assert result.llm_talent == "creative-writer" # type: ignore[attr-defined]+ assert result.model == "$writing-creative" # type: ignore[attr-defined]assert result.output == "ImgGenPrompt"- def test_original_bug_toml_output_has_correct_model(self) -> None:- """After resolution, the TOML should contain the correct model preset."""+ def test_toml_output_has_correct_model(self) -> None:+ """The TOML should contain the model preset directly."""spec: dict[str, Any] = {"pipe_code": "generate_prompt","description": "Generate an image prompt from an idea",- "llm_talent": "writing-creative",+ "model": "$writing-creative","inputs": {"idea": "Text"},"output": {"type": "ImgGenPrompt"},"prompt": "Generate a creative image prompt based on @idea",