Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Model-reference-check · L-261007-a80952 (#1375)
Louis Choquelc2d100c64b1e
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.tomlindex fb05b170f..c498de8df 100644--- a/.drift/acks/cli-docs.toml+++ b/.drift/acks/cli-docs.toml@@ -1,8 +1,8 @@contract = "cli-docs"-digest = "sha256:b48cd9256da89788502648136e4b3a6a5a4be0b853d6790d4178ca8091fd21a9"+digest = "sha256:b453fc65605094195a9b7c5eb7db7f52b75144f20fc9cb9958ec1761e133258e"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-06T10:13:22Z"-rationale = "clean-pass: installed_methods.py now validates the provenance sidecar from its bytes, so a sidecar that is not UTF-8 is read as no provenance like an unparsable one, instead of raising. Re-read docs/tools/cli/ (run-by-address.md prints the provenance of a fetch, which is unchanged) and pipelex/cli/agent_cli/CLAUDE.md; neither describes how discovery treats an unreadable sidecar, so nothing is stale."+reviewed_at = "2026-10-08T01:45:16Z"+rationale = "real-catch: check_model_cmd.py's --type now takes ModelCheckCategory, the categories GET /v1/models/check answers in, so doc_gen is accepted. docs/tools/cli/agent-cli.md's check-model row now lists the categories, doc_gen included, and says a handle: reference names its handle literally; pipelex/cli/agent_cli/CLAUDE.md's check-model row names ModelCheckCategory and why the listing's ModelCategory differs. docs/tools/cli/validate.md's pointer to check-model is still true."[trigger_files]"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"@@ -13,7 +13,7 @@ rationale = "clean-pass: installed_methods.py now validates the provenance sidec"pipelex/cli/agent_cli/commands/agent_cli_factory.py" = "blob:57b6a4d76b918626218633873273adbddfaac97d""pipelex/cli/agent_cli/commands/agent_output.py" = "blob:506eabec37ab3b06644b6ae4c321a91780241161""pipelex/cli/agent_cli/commands/bundle_path_resolver.py" = "blob:a1904517c731afb72af7d49adf6d217dc50b2660"-"pipelex/cli/agent_cli/commands/check_model_cmd.py" = "blob:42a6b8c02a1d103394e12243181af8ea1c16f31d"+"pipelex/cli/agent_cli/commands/check_model_cmd.py" = "blob:3a9c93daecadf266b3eba466904b3d005d448d40""pipelex/cli/agent_cli/commands/codegen/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/codegen/app.py" = "blob:56130b89fe9efec5ad3d014ed6bc16bef1bd1645""pipelex/cli/agent_cli/commands/codegen/check_cmd.py" = "blob:9e8beb084a9f04f39e8acca9e6ae2554d40cb667"diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.tomlindex 600f50a7f..e7d505029 100644--- a/.drift/acks/pipelex-kernel-docs.toml+++ b/.drift/acks/pipelex-kernel-docs.toml@@ -1,8 +1,8 @@contract = "pipelex-kernel-docs"-digest = "sha256:31f0c6b43ae0e9120f1103aa12068a80fdf1e10ec120ca5b289f454f85667449"-reviewed_by = "Claude (Opus 5.5)"-reviewed_at = "2026-10-06T22:38:42Z"-rationale = "real-catch: llm_ops.py now raises LLMSettingRefusedError only for a built-in backend's refusal and re-raises an external plugin's as written; pipelex-kernel.md entry paragraph rewritten to say so, re-read against check_llm_setting_with_served_model; rest of the page unaffected."+digest = "sha256:66bd0bb6f412ad068f472fb10bbde2d67578a831a64f18d8c022d09ee9db585a"+reviewed_by = "Louis Choquel"+reviewed_at = "2026-10-08T03:14:32Z"+rationale = "clean-pass: search_ops.py and judgment_ops.py now pin the resolved setting through write_model_handle, so a served handle spelled like a reference rides as handle:<name>; docs/under-the-hood/pipelex-kernel.md lists resolve_search_setting and resolve_judgment_setting but does not describe how the setting is pinned, so nothing there is stale."[trigger_files]"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"@@ -16,7 +16,7 @@ rationale = "real-catch: llm_ops.py now raises LLMSettingRefusedError only for a"pipelex/kernel/img_gen_ops.py" = "blob:b85360f6c118e93be53bf8d1a04504ccb7552469""pipelex/kernel/img_gen_prompt.py" = "blob:e4f72040561929aa0d96ad5f00b1c4af652d3f77""pipelex/kernel/img_gen_results.py" = "blob:46b4ca11c70d906ae6d58ce66461f9b3414f1bfa"-"pipelex/kernel/judgment_ops.py" = "blob:fe66d7d00433a9924d60710e9bb25838197c52e0"+"pipelex/kernel/judgment_ops.py" = "blob:49ce7414eed9cfdf5595e877a64aacd29ada86ca""pipelex/kernel/judgment_results.py" = "blob:40ba1258cc81d34df3e427d5511ccd40c6edf789""pipelex/kernel/llm_ops.py" = "blob:6555af22cadf5316cfcadf975d96b6792043c3a6""pipelex/kernel/llm_prompt_content.py" = "blob:3fade4a660226e63ad4f916952ee37d34e7d2bbd"@@ -24,7 +24,7 @@ rationale = "real-catch: llm_ops.py now raises LLMSettingRefusedError only for a"pipelex/kernel/memory_ops.py" = "blob:7fbbfaaec99f9712d8e215520cb02e26aad3fb40""pipelex/kernel/pipelex_kernel.py" = "blob:5dec4e68d517d5ab1d7e9cbb91d836f37b10e713""pipelex/kernel/prompt_references.py" = "blob:03ec9ef67ae84f141d072ca2d2c3ef888549ccc0"-"pipelex/kernel/search_ops.py" = "blob:f9c1dc09dd3beaf1a54ac8e3774db1d510cbdaa3"+"pipelex/kernel/search_ops.py" = "blob:b7de7ff62c45dde7f7aa8a9f409aecf3b664cce5""pipelex/kernel/search_results.py" = "blob:101b70dc96cc5ee1721c9f6b1cb725a5ca78ff70""pipelex/kernel/templating_style_ops.py" = "blob:395473806d600dfc4f124c07f2afebb951fc086b""tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:63c06904e9f53f80f98a0a45e00aa8116c9beb31"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bc11ef0f1..30aed99ee 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,12 +6,21 @@- **Claude Haiku 5.5 on the `anthropic` backend**: `claude-5.5-haiku` (model id `claude-haiku-5-5`) takes text, images and PDF, produces text and structured output, and reasons with adaptive thinking, with up to 128K output tokens. It costs $0.10 / $0.50 per million tokens for a prompt up to 100,000 tokens and $0.50 / $2.50 above, of which the backend file records the first. It refuses `temperature` but, unlike Sonnet 5.5 and Opus 5.5, accepts a forced `tool_choice`, so structured output keeps `instructor/anthropic_tools`.- **One handle, one model per model type**: two models of different model types may share a handle, so `gpt-6-luna` can be an LLM and a judgment model at once, and each pipe reaches the one its family asks for. A backend file declares the second model of a handle on a table of its own with the new `handle` key, which may not appear in `[defaults]`, and a file declaring one handle twice as the same type fails to load, naming both tables. A plugin may declare an internal model under a name `internal.toml` already declares as another type.+- **`GET /v1/models/check`, the model reference check**: the server answers whether one model reference (`?reference=`, optionally in one `?type=` category, `doc_gen` included) resolves on this runner, as a `200` verdict discriminated on `resolution` (`resolved` or `not_found`) that carries the reference's `kind` and `name`, one match per category where it resolves with the model a run would call (`resolves_to`, `null` where its binding reaches no model the runner can call) and its `target`, `description`, `fallbacks` or `via`, and, on `not_found`, the `suggestions`, `other_kinds` and `other_categories` a failing validation of the same reference offers, written as references. It answers from pipelex's own parser and the deck lookups a validation runs, so a client no longer keeps its own grammar of model references, and it is a Pipelex API extension, not tagged `x-mthds-protocol`. A missing or repeated `reference` or a repeated `type` is a `422` `ValidationError`, a reference that is blank, a sigil or a namespace alone, or longer than 199 characters once trimmed is a `422` of the new `error_type` `InvalidModelReference`, and an unknown or empty `type` is a `422` `InvalidModelCategory`. The library serves the same check as `check_model_reference` in `pipelex.cogt.models.model_reference_check`, and the suggestion rule as `find_model_alternatives` in `pipelex.cogt.models.model_suggestion`.### Changed- **A model served only as another type is refused when the method loads (Breaking)**: a pipe naming a handle the deck serves only as another model type, such as a `PipeJudge` naming an LLM, is refused by `pipelex validate`, `POST /v1/validate` and a run's load as an `unknown_model` item, whose suggestions list only models of the type the pipe asks for, where it passed validation and failed at its first call. A handle served as another type no longer hides an alias or a waterfall of the same name from a lookup of this type.- **Model specs are keyed by model type and handle (Breaking)**: `InferenceBackend.model_specs` and `ModelDeck.inference_models` are the new `ModelSpecIndex`, read through `get(model_type=…, handle=…)`, `handles_of_type`, `all_specs`, `all_handles` and `types_serving`, and `InferenceBackend.get_model_spec` takes `model_type` and `handle`. `InferenceBackend.list_model_names`, `InferenceBackendLibrary.list_all_model_names` and `InferenceBackendLibrary.get_all_models_and_possible_backends` are removed. `PluginModelDeclarations.internal_models` is a tuple of `PluginInternalModel`, which carries its `name`, and `DuplicateInternalModelError` names the `model_type` two plugins both declared.- **Usage is reported per model type and name (Breaking)**: a run calling both kinds of one handle reports two rows in the cost table and the agent CLI's cost summary, and two entries in the graph's usage attribution, where they merged into one row typed by whichever call came last. `AggregatedCosts.grouped_by_model` is keyed by the new `ModelUsageKey`, and its `model_types` is removed.+- **A model served only as another type is named as such, and `handle:` reads no sigil (Breaking)**: the `unknown_model` item refusing a bare handle the deck serves only as another model type now says the deck serves the handle but not as the type the pipe needs, where it said the handle was not found, and so do a deck preset naming one, reported at boot (a warning, or a refusal under `missing_presets_reaction = "raise"`), and an LLM override naming one, which refuses boot. A bare name resolves as a model of the pipe's type, then an alias of that type, then a waterfall of that type, an order the validation and the run's lookup share, and the validation and the setting a pipe's `model` field is read into accept the waterfall only while model fallback is on. A `handle:` reference writes a bare name without reading a sigil in the validation, the run and the check alike, so `handle:@name` is the bare name `@name`, not the alias `name`, and like any bare name it resolves as a model, then an alias, then a waterfall of that name, a validation refusing it where the deck has none. `ModelDeck` gains `is_bare_handle_resolvable`, `is_reference_defined`, `get_presets_for_type` and `peek_inference_model`, the run's model lookup without its logs.++### Fixed++- **A `~name` model reference runs the waterfall**: a pipe naming `~name` or `waterfall:name` called a model or an alias that shares the waterfall's name, because its setting kept the bare name, and every model type's setting now keeps the sigil, so a run calls the model the validation and `GET /v1/models/check` name; the `resolved_model` every pipe reports, and a `PipeLLM`'s `resolved_model_for_object`, now write such a waterfall `~name`, and so does a refusal of a file input naming one.+- **A waterfall step that leads back to itself or runs out serves no model**: an alias and a waterfall leading to each other, or a waterfall listing its own name, overflowed the stack, and a step naming another waterfall, directly or through an alias, none of whose models is served failed its waterfall with that inner waterfall's error. Such a step, whether it leads back to an alias or a waterfall being resolved or reaches a waterfall that runs out, now serves no model, so the waterfall goes on to its next step, and a run fails with the waterfall's own error only when none of its steps serves a model.+- **`pipelex-agent models --backend` lists past a binding no backend serves**: an alias or a preset bound to a waterfall none of whose models the deck serves failed the listing with `ModelWaterfallError`, and it is now left out like any binding of another backend, the listing reading the deck without a run's logs.+- **`pipelex-agent check-model` applies the validation's rule**: a bare alias name, or a bare waterfall name while model fallback is on, is now `valid`, as `pipelex validate` and `GET /v1/models/check` answer, and `--type` takes `doc_gen` as the check does.## [v0.77.0] - 2026-10-07diff --git a/api/CLAUDE.md b/api/CLAUDE.mdindex 890203d99..19684cde3 100644--- a/api/CLAUDE.md+++ b/api/CLAUDE.md@@ -35,7 +35,7 @@ pipelex_api/codegen.py # POST /v1/codegen (crate → typed artifacts)crate_ops.py # Closure resolution, the pipe selection chain, and the invalid-arm envelope the crate routes sharetools.py # POST /v1/lint, /v1/format (editor tooling)- agent/ # GET /v1/models+ agent/ # GET /v1/models, GET /v1/models/check (the model reference check)tests/unit/ # Unit testsscripts/@@ -48,7 +48,7 @@ hatch_build.py # The metadata hook that writes the dependencies andThis server is the reference implementation of the [MTHDS Protocol](https://mthds.ai): `POST /execute`, `POST /start`, `POST /validate`, `GET /models`, `GET /version` under the `/v1` base path, tagged `x-mthds-protocol: true` in the committed OpenAPI artifact (`docs/api-server/openapi/pipelex-api.openapi.yaml` at the repository root, regenerated via `make openapi-export`, drift-checked via `make openapi-check`). Contract nesting: MTHDS Protocol ⊂ Pipelex API ⊂ Pipelex hosted API.-**Only those five operations carry `x-mthds-protocol`** — the flag is how a conformance suite or a third-party runner extracts the portable subset of the artifact, so tagging a Pipelex route would misrepresent the standard. Everything else this server serves is a Pipelex API extension: `/resolve` + `/codegen`, `/pipe-io`, and `/lint` + `/format`. Watch `/resolve` and `/codegen` in particular: they *look* protocol-shaped (they speak the `/validate` verdict discipline, and the crate `/resolve` emits is genuinely standard-owned — the MTHDS Library Crate Format, so its wire fields stay brand-neutral), but the routes are ours and the standard specifies no type projection at all. `tests/unit/test_openapi_contract.py` pins the tagged set exactly, in both directions.+**Only the protocol's own operations carry `x-mthds-protocol`** — the flag is how a conformance suite or a third-party runner extracts the portable subset of the artifact, so tagging a Pipelex route would misrepresent the standard. Everything else this server serves is a Pipelex API extension: `/resolve` + `/codegen`, `/pipe-io`, `/models/check`, and `/lint` + `/format`. Watch `/resolve` and `/codegen` in particular: they *look* protocol-shaped (they speak the `/validate` verdict discipline, and the crate `/resolve` emits is genuinely standard-owned — the MTHDS Library Crate Format, so its wire fields stay brand-neutral), but the routes are ours and the standard specifies no type projection at all. `tests/unit/test_openapi_contract.py` pins the tagged set exactly, in both directions.pipelex's own models ride this wire (the run result, the bundle blueprint, the validation error vocabulary), so a library change can move the artifact with no change to the server's code. `openapi-check` is part of `make agent-check` for that reason: regenerate with `make openapi-export` and read the diff, which states what the change did to the server's callers.diff --git a/api/pipelex_api/error_types.py b/api/pipelex_api/error_types.pyindex 89afda8e4..a3705efe5 100644--- a/api/pipelex_api/error_types.py+++ b/api/pipelex_api/error_types.py@@ -52,6 +52,10 @@ class ErrorType(StrEnum):# the runtime's charset and entry cap (`pipelex.system.run_extras`).INVALID_ANALYTICS_GROUPS = "InvalidAnalyticsGroups"INVALID_MODEL_CATEGORY = "InvalidModelCategory"+ # `GET /v1/models/check` was given a `reference` it cannot check: blank once trimmed, a sigil or a+ # namespace with nothing after it, or longer than the route's bound. A reference that parses and+ # resolves nowhere is never refused: it is the `not_found` verdict.+ INVALID_MODEL_REFERENCE = "InvalidModelReference"INVALID_BASE64 = "InvalidBase64"PAYLOAD_TOO_LARGE = "PayloadTooLarge"# A run request carried a malformed method bundle (`bundle_b64` / `files`): a corrupt zip, andiff --git a/api/pipelex_api/main.py b/api/pipelex_api/main.pyindex 4de7c0936..41bd75e4e 100644--- a/api/pipelex_api/main.py+++ b/api/pipelex_api/main.py@@ -150,7 +150,8 @@ fastapi_app = PipelexFastAPI(f"This server implements the [MTHDS Protocol](https://mthds.ai) v{PROTOCOL_VERSION} ""(`POST /execute`, `POST /start`, `POST /validate`, `GET /models`, `GET /version` — ""marked `x-mthds-protocol: true`) plus the Pipelex API extensions: resolve and codegen "- "(`/resolve`, `/codegen`), the pipe I/O route (`/pipe-io`), and editor tooling (`/lint`, `/format`). "+ "(`/resolve`, `/codegen`), the pipe I/O route (`/pipe-io`), the model reference check (`/models/check`), "+ "and editor tooling (`/lint`, `/format`). ""Contract layering: MTHDS Protocol ⊂ Pipelex API (this server) ⊂ Pipelex hosted API. ""All endpoints are served under the `/v1` base path; ""every error is an RFC 7807 `application/problem+json` problem document, documented per "diff --git a/api/pipelex_api/routes/pipelex/agent/models.py b/api/pipelex_api/routes/pipelex/agent/models.pyindex 0190710c4..1bc6a7510 100644--- a/api/pipelex_api/routes/pipelex/agent/models.py+++ b/api/pipelex_api/routes/pipelex/agent/models.py@@ -1,10 +1,14 @@-"""Models endpoint — the MTHDS Protocol model deck this runner routes to."""+"""Models endpoints — the MTHDS Protocol model deck this runner routes to, and the model reference check."""from typing import Annotatedfrom fastapi import APIRouter, Query, Requestfrom mthds.protocol.models import ModelCategory+from pipelex.cogt.models.exceptions import ModelReferenceParseError+from pipelex.cogt.models.model_reference import ModelReference+from pipelex.cogt.models.model_reference_check import ModelCheckCategory, ModelReferenceVerdict, check_model_referencefrom pipelex.pipeline.runner import PipelexModelDeck+from pipelex.runtime_hub import get_model_deckfrom pipelex_api.error_types import ErrorTypefrom pipelex_api.errors import raise_validation_error@@ -12,6 +16,9 @@ from pipelex_api.routes.pipelex.pipeline import ApiRunnerrouter = APIRouter(tags=["agent"])+# The longest reference the check accepts once trimmed; one character more is refused with `InvalidModelReference`.+MAX_MODEL_REFERENCE_LENGTH = 199+@router.get("/models", openapi_extra={"x-mthds-protocol": True})async def get_models(@@ -47,3 +54,68 @@ async def get_models(error_type=ErrorType.INVALID_MODEL_CATEGORY,)return await ApiRunner().models(category=category)+++# NOT tagged `x-mthds-protocol`: the check is a Pipelex API extension, and the flag marks the+# standard's own operations alone. Its refusals are the composite router's shared problem+json 422.[email protected]("/models/check")+async def check_model(+ request: Request,+ reference: Annotated[+ str,+ Query(+ description=(+ "The model reference as a method's `model` field writes it: `$preset`, `@alias`, `~waterfall`, a bare model handle, "+ "or one of the namespaces `preset:`, `alias:`, `waterfall:` and `handle:`. Surrounding whitespace is ignored, "+ f"and the trimmed reference holds at most {MAX_MODEL_REFERENCE_LENGTH} characters. Single value."+ ),+ ),+ ],+ model_type: Annotated[+ str | None,+ Query(+ alias="type",+ description=(+ "The category to check in: llm, extract, img_gen, search, judgment or doc_gen. "+ "When absent, the check is made in every one of them. Single value."+ ),+ ),+ ] = None,+) -> ModelReferenceVerdict:+ """Check whether one model reference resolves on this runner, as what, to which model (Pipelex API extension).++ Answers from pipelex's own reference parser and deck lookups, the ones a validation runs, so a+ reference found `resolved` in a category is one a pipe of that category may name, and one found+ `not_found` is one a validation refuses. A verdict is a `200` whatever the resolution, and on+ `not_found` it carries the suggestions a failing validation of the same reference offers. A+ request that cannot produce a verdict is an input `422`: `ValidationError` for a missing or+ repeated `reference` or a repeated `type`, `InvalidModelReference` for a reference that is blank,+ a sigil or a namespace alone, or too long, and `InvalidModelCategory` for an unknown `type`.+ """+ for parameter_name in ("reference", "type"):+ if len(request.query_params.getlist(parameter_name)) > 1:+ raise_validation_error(message=f"The `{parameter_name}` query parameter accepts a single value")++ # The parser trims the reference itself, and its `raw` is the trimmed text the length limit measures.+ try:+ parsed_reference = ModelReference.parse(reference)+ except ModelReferenceParseError as exc:+ raise_validation_error(message=exc.message, error_type=ErrorType.INVALID_MODEL_REFERENCE)+ if len(parsed_reference.raw) > MAX_MODEL_REFERENCE_LENGTH:+ too_long_message = (+ f"The model reference holds {len(parsed_reference.raw)} characters once trimmed, more than the {MAX_MODEL_REFERENCE_LENGTH} accepted"+ )+ raise_validation_error(message=too_long_message, error_type=ErrorType.INVALID_MODEL_REFERENCE)++ category: ModelCheckCategory | None = None+ if model_type is not None:+ try:+ category = ModelCheckCategory(model_type)+ except ValueError:+ valid = ", ".join(sorted(member.value for member in ModelCheckCategory))+ raise_validation_error(+ message=f"Invalid model category. Valid values: {valid}",+ error_type=ErrorType.INVALID_MODEL_CATEGORY,+ )++ return check_model_reference(model_deck=get_model_deck(), reference=parsed_reference, category=category)diff --git a/api/tests/unit/test_models_check_route.py b/api/tests/unit/test_models_check_route.pynew file mode 100644index 000000000..d4d609803--- /dev/null+++ b/api/tests/unit/test_models_check_route.py@@ -0,0 +1,141 @@+"""The model reference check route, `GET /v1/models/check`, on the server's own deck."""++from typing import Any++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient++from pipelex_api.exception_handlers import register_exception_handlers+from pipelex_api.routes import router as api_router+from pipelex_api.routes.pipelex.agent.models import MAX_MODEL_REFERENCE_LENGTH++_CHECK_PATH = "/v1/models/check"++_VERDICT_FIELDS = {"reference", "kind", "name", "category", "resolution", "matches", "suggestions", "other_kinds", "other_categories"}+++def _build_client() -> TestClient:+ app = FastAPI()+ app.include_router(api_router, prefix="/v1")+ register_exception_handlers(app)+ return TestClient(app)+++def _verdict(client: TestClient, params: list[tuple[str, str]]) -> dict[str, Any]:+ response = client.get(_CHECK_PATH, params=tuple(params))+ assert response.status_code == 200, response.text+ assert response.headers["content-type"].startswith("application/json")+ verdict: dict[str, Any] = response.json()+ assert set(verdict) == _VERDICT_FIELDS+ return verdict+++def _img_gen_bundle(*, model: str) -> str:+ return f"""+domain = "harbour_board"+description = "Draw the board announcing the tide times"+main_pipe = "draw_tide_board"++[pipe.draw_tide_board]+type = "PipeImgGen"+description = "Draw the painted board announcing the tide times"+output = "Image"+model = "{model}"+prompt = "A painted wooden harbour board announcing the tide times, morning light"+"""+++class TestModelsCheckRoute:+ def test_a_preset_of_the_deck_resolves_in_its_category(self):+ client = _build_client()+ preset = client.get("/v1/models").json()["models"][0]++ verdict = _verdict(client, [("reference", f"${preset['name']}"), ("type", preset["type"])])++ assert verdict["resolution"] == "resolved"+ assert verdict["kind"] == "preset"+ assert verdict["name"] == preset["name"]+ assert verdict["category"] == preset["type"]+ (match,) = verdict["matches"]+ assert set(match) == {"category", "resolves_to", "target", "description"}+ assert match["category"] == preset["type"]+ assert verdict["suggestions"] == verdict["other_kinds"] == verdict["other_categories"] == []++ def test_an_alias_and_its_target_resolve_with_the_alias_in_via(self):+ client = _build_client()+ alias_name, alias_target = next(iter(client.get("/v1/models", params={"type": "llm"}).json()["aliases"]["llm"].items()))++ alias_verdict = _verdict(client, [("reference", f" alias:{alias_name} "), ("type", "llm")])+ handle_verdict = _verdict(client, [("reference", alias_target), ("type", "llm")])++ assert alias_verdict["reference"] == f"alias:{alias_name}"+ assert alias_verdict["matches"] == [{"category": "llm", "resolves_to": alias_target, "target": alias_target}]+ assert handle_verdict["resolution"] == "resolved"+ assert handle_verdict["kind"] == "handle"+ (match,) = handle_verdict["matches"]+ assert match["resolves_to"] == alias_target+ assert f"@{alias_name}" in match["via"]++ def test_an_llm_asked_as_an_image_generation_model_is_not_found_and_fails_validation_too(self):+ """The check and a validation agree: an LLM the deck serves is neither checked nor validated as an image-generation model."""+ client = _build_client()+ llm_handle = next(iter(client.get("/v1/models", params={"type": "llm"}).json()["aliases"]["llm"].values()))++ verdict = _verdict(client, [("reference", llm_handle), ("type", "img_gen")])+ validation = client.post("/v1/validate", json={"mthds_contents": [_img_gen_bundle(model=llm_handle)]})++ assert verdict["resolution"] == "not_found"+ assert verdict["matches"] == []+ assert verdict["other_categories"] == ["llm"]+ assert validation.status_code == 200, validation.text+ report = validation.json()+ assert report["is_valid"] is False+ (item,) = report["validation_errors"]+ assert item["error_type"] == "unknown_model"+ assert item["model_reference"] == llm_handle+ assert item["model_type"] == "img_gen"+ assert "but not as an image-generation model" in item["message"]++ def test_doc_gen_is_a_category_of_the_check(self):+ client = _build_client()++ verdict = _verdict(client, [("reference", "$no-preset-is-named-this"), ("type", "doc_gen")])++ assert verdict["category"] == "doc_gen"+ assert verdict["resolution"] == "not_found"++ def test_the_longest_accepted_reference_is_a_verdict(self):+ client = _build_client()++ verdict = _verdict(client, [("reference", "@" + "x" * (MAX_MODEL_REFERENCE_LENGTH - 1))])++ assert verdict["resolution"] == "not_found"+ assert verdict["category"] is None++ @pytest.mark.parametrize(+ ("params", "error_type"),+ [+ pytest.param([], "ValidationError", id="reference-missing"),+ pytest.param([("reference", "@a"), ("reference", "@b")], "ValidationError", id="reference-repeated"),+ pytest.param([("reference", "@a"), ("type", "llm"), ("type", "extract")], "ValidationError", id="type-repeated"),+ pytest.param([("reference", "")], "InvalidModelReference", id="empty"),+ pytest.param([("reference", " ")], "InvalidModelReference", id="blank"),+ pytest.param([("reference", "~")], "InvalidModelReference", id="sigil-alone"),+ pytest.param([("reference", "handle:")], "InvalidModelReference", id="namespace-alone"),+ pytest.param([("reference", "@" + "x" * MAX_MODEL_REFERENCE_LENGTH)], "InvalidModelReference", id="past-the-bound"),+ pytest.param([("reference", "@a"), ("type", "bogus_category")], "InvalidModelCategory", id="type-unknown"),+ pytest.param([("reference", "@a"), ("type", "")], "InvalidModelCategory", id="type-empty"),+ ],+ )+ def test_a_request_with_no_verdict_is_an_input_422(self, params: list[tuple[str, str]], error_type: str):+ client = _build_client()++ response = client.get(_CHECK_PATH, params=tuple(params))++ assert response.status_code == 422, response.text+ assert response.headers["content-type"] == "application/problem+json"+ problem = response.json()+ assert problem["error_type"] == error_type+ assert problem["error_domain"] == "input"+ assert "resolution" not in problemdiff --git a/api/tests/unit/test_models_check_route_deck_shapes.py b/api/tests/unit/test_models_check_route_deck_shapes.pynew file mode 100644index 000000000..c921dc3ab--- /dev/null+++ b/api/tests/unit/test_models_check_route_deck_shapes.py@@ -0,0 +1,113 @@+"""The model reference check route, `GET /v1/models/check`, on hand-built decks whose bindings lead back to themselves or share names."""++from typing import Any++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndex+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.models.model_deck_config import ModelDeckConfig+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.runtime import ProblemReaction+from pytest_mock import MockerFixture++from pipelex_api.exception_handlers import register_exception_handlers+from pipelex_api.routes import router as api_router++_CHECK_PATH = "/v1/models/check"+_GET_MODEL_DECK_TARGET = "pipelex_api.routes.pipelex.agent.models.get_model_deck"+++def _model_spec(name: str, model_type: ModelType) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name="test_backend",+ name=name,+ sdk="test_sdk",+ model_type=model_type,+ model_id=f"test_model_{name}",+ costs={CostCategory.INPUT: 0.001, CostCategory.OUTPUT: 0.002},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=1000,+ max_prompt_images=None,+ )+++def _make_deck() -> ModelDeck:+ """A deck where an alias and a waterfall lead to each other, an image-generation waterfall is named like an LLM+ and lists its own name first, and an alias and a waterfall share a name.+ """+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("claude-x", ModelType.LLM),+ ]+ ),+ llm_default_temperature=0.7,+ llm_aliases={"cycle-a": "~cycle-waterfall", "best-gpt": "gpt-4o-mini"},+ llm_waterfalls={"cycle-waterfall": ["@cycle-a"], "best-gpt": ["claude-x"]},+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_waterfalls={"gpt-4o-mini": ["gpt-4o-mini", "img-unserved"]},+ img_gen_choice_default="img-unserved",+ search_choice_default="@default-search",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=True, missing_presets_reaction=ProblemReaction.NONE),+ )+++def _verdict(mocker: MockerFixture, *, reference: str, category: str) -> dict[str, Any]:+ mocker.patch(_GET_MODEL_DECK_TARGET, return_value=_make_deck())+ app = FastAPI()+ app.include_router(api_router, prefix="/v1")+ register_exception_handlers(app)+ response = TestClient(app).get(_CHECK_PATH, params={"reference": reference, "type": category})+ assert response.status_code == 200, response.text+ verdict: dict[str, Any] = response.json()+ return verdict+++class TestModelsCheckRouteDeckShapes:+ @pytest.mark.parametrize(+ ("reference", "category", "expected_match"),+ [+ pytest.param("@cycle-a", "llm", {"category": "llm", "resolves_to": None, "target": "~cycle-waterfall"}, id="alias-into-waterfall"),+ pytest.param("~cycle-waterfall", "llm", {"category": "llm", "resolves_to": None, "fallbacks": ["@cycle-a"]}, id="waterfall-into-alias"),+ pytest.param("gpt-4o-mini", "img_gen", {"category": "img_gen", "resolves_to": None, "via": ["~gpt-4o-mini"]}, id="waterfall-into-itself"),+ ],+ )+ def test_a_binding_leading_back_to_itself_is_resolved_to_no_model(+ self,+ mocker: MockerFixture,+ reference: str,+ category: str,+ expected_match: dict[str, Any],+ ):+ verdict = _verdict(mocker, reference=reference, category=category)++ assert verdict["resolution"] == "resolved"+ assert verdict["matches"] == [expected_match]++ @pytest.mark.parametrize(+ ("reference", "expected_model"),+ [+ pytest.param("~best-gpt", "claude-x", id="waterfall"),+ pytest.param("@best-gpt", "gpt-4o-mini", id="alias"),+ pytest.param("best-gpt", "gpt-4o-mini", id="bare-name"),+ ],+ )+ def test_an_alias_and_a_waterfall_sharing_a_name_each_resolve_by_their_sigil(self, mocker: MockerFixture, reference: str, expected_model: str):+ verdict = _verdict(mocker, reference=reference, category="llm")++ (match,) = verdict["matches"]+ assert match["resolves_to"] == expected_modeldiff --git a/docs/api-server/index.md b/docs/api-server/index.mdindex a3167987e..de99d0b37 100644--- a/docs/api-server/index.md+++ b/docs/api-server/index.md@@ -20,7 +20,7 @@ MTHDS Protocol ⊂ Pipelex API (this server) ⊂ Pipelex hosted API```- **MTHDS Protocol** — five routes: `POST /execute`, `POST /start`, `POST /validate`, `GET /models`, `GET /version`. Tagged `x-mthds-protocol: true` in the [committed OpenAPI artifact](openapi/pipelex-api.openapi.yaml), and **only** those five — the flag is how a conformance suite or a third-party runner extracts the portable subset.-- **Pipelex API (this server)** — the protocol verbatim, plus the Pipelex extensions: resolve and codegen (`/resolve`, `/codegen`), pipe I/O (`/pipe-io`), and editor tooling (`/lint`, `/format`). `/upload` and `/resolve-storage-url` were non-contract convenience routes and have been **removed** — see [Storage Transport](storage-transport.md) for where they went and why.+- **Pipelex API (this server)** — the protocol verbatim, plus the Pipelex extensions: resolve and codegen (`/resolve`, `/codegen`), pipe I/O (`/pipe-io`), the model reference check (`/models/check`), and editor tooling (`/lint`, `/format`). `/upload` and `/resolve-storage-url` were non-contract convenience routes and have been **removed** — see [Storage Transport](storage-transport.md) for where they went and why.- **Pipelex hosted API** (`api.pipelex.com/v1`) — everything here, same shapes, plus durable runs, the method catalog, and account management.All routes are served under the `/v1` base path (clients compose `{base}/v1/{endpoint}`).@@ -34,7 +34,7 @@ The API currently allows you to:3. **Resolve** a library closure into its normalized crate, and **codegen** typed artifacts from it (TypeScript/Zod, Pydantic, Pipelex structures)4. **Describe** a method's inputs and output — its pipe I/O contracts, input form and output form, from which a client projects an inputs template5. **Lint and format** single `.mthds` files for editor workflows-6. **List** available model presets and configurations+6. **List** available model presets and configurations, and **check** whether one model reference resolves, as what, to which model## Deployment@@ -181,9 +181,12 @@ Lint and format single `.mthds` files without loading or executing a pipeline.[Learn more →](mthds-tools.md)### Models-List the models this runner can route to.+List the models this runner can route to, and check one model reference against them.- `GET /v1/models` — The protocol model deck this runner routes to (flat `models` list, plus category-keyed `aliases`/`waterfalls` routing extensions); optional single `?type=` category filter+- `GET /v1/models/check` — Whether one model reference (`?reference=`, optionally in one `?type=` category, `doc_gen` included) resolves on this runner, as what kind of reference and to which model, with suggestions when it does not; answered from pipelex's own parser and the deck lookups a validation runs (Pipelex API extension)++[Learn more →](models.md)### Uploader (removed)diff --git a/docs/api-server/models.md b/docs/api-server/models.mdnew file mode 100644index 000000000..22cecf081--- /dev/null+++ b/docs/api-server/models.md@@ -0,0 +1,129 @@+---+title: "API Server Models"+description: "List the model deck a Pipelex runner routes to, and check whether one model reference resolves on it, as what, to which model, and what the caller most likely meant when it does not."+---++# Models++Two routes tell a client about the models a method may name on this runner: the deck it routes to, and whether one model reference resolves on it. A client asks them rather than keeping its own table of providers, its own parser of model references or its own copy of the deck.++## List the deck++**Endpoint:** `GET /v1/models`++The MTHDS Protocol's `models` operation, tagged `x-mthds-protocol` in the [OpenAPI artifact](openapi/pipelex-api.openapi.yaml). It answers the protocol's model deck: a flat `models` list of the deck's presets, each a `{name, type}` entry whose `type` is one of the protocol's model categories, plus this implementation's routing extensions, `aliases` and `waterfalls`, keyed by category because the same alias name exists in several categories.++The optional `type` query parameter filters the deck to one category: `llm`, `extract`, `img_gen`, `search` or `judgment`. It takes a single value, so a repeated `type` is a `422` `ValidationError`, and an unknown or empty one is a `422` `InvalidModelCategory`.++## Check a model reference++**Endpoint:** `GET /v1/models/check`++Answers whether one model reference resolves on this runner, as what kind of reference, to which model, and, when it does not, what the caller most likely meant. It reads the runner's deck and the models the runner can call, and runs no inference.++It is a **Pipelex API extension**, not an MTHDS Protocol route: it is not tagged `x-mthds-protocol`. The answer comes from pipelex's own reference parser and the deck lookups a validation runs, so a reference the check finds resolved in a category is one a pipe of that category may name in its `model` field, and one it finds not found is one a validation refuses. A client therefore needs no grammar of its own to read a reference.++### Request++| Query parameter | Required | Meaning |+|---|---:|---|+| `reference` | yes, once | The reference as a method's `model` field writes it: `$preset`, `@alias`, `~waterfall`, a bare model handle, or one of the spelled-out namespaces `preset:`, `alias:`, `waterfall:` and `handle:`. Leading and trailing whitespace is ignored, and the trimmed reference holds at most 199 characters. |+| `type` | no, at most once | The category to check in. When it is absent, the check is made in every category it covers. |++The check covers the protocol's model categories, in the protocol's order (`llm`, `extract`, `img_gen`, `search`, `judgment`), then `doc_gen`, the family of `PipeDocGen`. The protocol defines no category for `doc_gen`, so `GET /v1/models` leaves it out, but a method names a `doc_gen` model like any other, so the check answers in it. Every list the check returns that is ordered by category follows this order.++### Resolution++The reference is parsed as a method's `model` field is: a sigil first (`$` preset, `@` alias, `~` waterfall), then a spelled-out namespace, and otherwise a bare handle. `alias:best-gpt` and `@best-gpt` are the same reference. `handle:` is the bare kind's own namespace: it writes a bare name without reading a sigil, so `handle:@best-gpt` is the bare name `@best-gpt`, not the alias `best-gpt`, and `handle:best-gpt` is the bare name `best-gpt`. Like any bare name, it resolves as a model of that name, then an alias, then a waterfall of that name: where the runner serves a model `@best-gpt`, the check resolves to it and a run calls it, and where the deck holds nothing of that name, a validation refuses it as the check finds it `not_found`. A suggestion naming a handle spelled like a reference writes it with the namespace, `handle:@best-gpt`, so it can be written back as it is.++A reference **resolves in a category** when the runner holds its name there:++- a preset, an alias or a waterfall, when the deck defines that name in the category;+- a bare handle, as a run reads it: when the runner can call a model of that name in the category, among every model it can call and not only those the deck names; failing that, when the category defines an alias of that name, or a waterfall of that name while model fallback is on. A model of that name in another category does not make it resolve, so an LLM handle checked as `img_gen` is `not_found`, as a validation refuses it in a `PipeImgGen`.++The reference is `resolved` when it resolves in at least one category in scope, and `not_found` otherwise. Both values are definitive: a client that reads a value it does not know treats the reference as unresolved.++Resolution is a matter of names, like a validation. A preset, an alias or a waterfall the deck defines resolves even when its binding reaches no model the runner can call (a target on a backend the runner has not enabled, a waterfall none of whose usable steps it serves, a binding that leads back to itself through aliases and waterfalls), and its match says so with a `resolves_to` of `null`. A sigiled reference resolves by its sigil, in the check and in a run alike: `~best-gpt` is the waterfall even where an alias or a model is also named `best-gpt`. A waterfall step that leads back to an alias or a waterfall being resolved is a step no model serves, and so is a step reaching another waterfall none of whose models the runner serves, so the waterfall goes on to its next step. The check reads the deck as a run would and leaves no trace: it logs nothing, and the one-time notice a run logs when a waterfall falls back is left to that run. A run through such a reference fails where a validation passes, so a client shows it as a warning rather than as an unknown name.++### The verdict++A verdict is a `200`, whatever the resolution:++| Field | Type | Meaning |+|---|---|---|+| `reference` | string | The caller's reference, trimmed. |+| `kind` | `"preset"`, `"alias"`, `"waterfall"` or `"handle"` | What the reference names, from its parsing. |+| `name` | string | The reference without its sigil or namespace. |+| `category` | string or `null` | The `type` asked, or `null` when none was. |+| `resolution` | `"resolved"` or `"not_found"` | Whether the reference resolves in a category in scope. |+| `matches` | list | One entry per category in scope where the reference resolves, in category order. Empty on `not_found`. |+| `suggestions` | list of strings | On `not_found`, the nearest names the caller may have meant. |+| `other_kinds` | list of strings | On `not_found`, the same name under another kind, in the categories in scope. |+| `other_categories` | list of strings | On `not_found` with a `type`, the categories outside it where the same reference resolves. |++A `matches` entry says what the reference is in one category. A field that does not apply to the reference's kind is absent rather than `null`:++| Field | Present for | Meaning |+|---|---|---|+| `category` | every kind | The category the entry is about. |+| `resolves_to` | every kind | The model handle a run through the reference would call now in this category, or `null` when it would find none: a preset and an alias through their target, a waterfall through the first of its steps the runner serves (its first step alone when model fallback is off), and a handle itself, or, for a bare name that resolves through an alias or a waterfall, what that alias or waterfall resolves to. |+| `target` | preset, alias | The model the deck binds the name to, as the deck writes it, which may itself be a reference (`@default-premium`). |+| `description` | preset | The preset's description, or `null` when the deck gives it none. |+| `fallbacks` | waterfall | The waterfall's steps, in order. |+| `via` | handle | The presets, aliases and waterfalls of the category whose binding names the handle directly, each written as a reference. |++A preset, checked in its category:++```json+{+ "reference": "$writing-factual",+ "kind": "preset",+ "name": "writing-factual",+ "category": "llm",+ "resolution": "resolved",+ "matches": [+ { "category": "llm", "resolves_to": "gpt-5.6-sol", "target": "@default-premium", "description": "Factual writing with high accuracy" }+ ],+ "suggestions": [],+ "other_kinds": [],+ "other_categories": []+}+```++### Suggestions++On `not_found`, `suggestions`, `other_kinds` and `other_categories` say what the caller may have meant. They follow the rule a validation follows when it refuses an unknown model, so the check offers what a failing validation of the same reference offers, applied in each category in scope and joined in category order, a name two categories give appearing once, at its first place:++- `suggestions`: up to five names of the reference's own kind, nearest first, then, for each other kind under which the name does not exist exactly, up to three names of that kind, with a stricter threshold of nearness. For a handle, the candidates are every model the runner can call in the category.+- `other_kinds`: the same name under each other kind the category defines it as (`best-gpt` asked as a preset, found as `@best-gpt`). It is the likeliest fault, so a client shows it first.+- `other_categories`: when `type` was given, each other category where the same reference resolves (`$gen-image` asked as `llm`, found in `img_gen`).++Every name is written as a method would write it, a sigil for a preset, an alias or a waterfall and a bare name for a handle, without the kind labels a validation's report adds.++A preset name asked under the alias sigil:++```json+{+ "reference": "@writing-factual",+ "kind": "alias",+ "name": "writing-factual",+ "category": "llm",+ "resolution": "not_found",+ "matches": [],+ "suggestions": [],+ "other_kinds": ["$writing-factual"],+ "other_categories": []+}+```++### Refusals++A request the route cannot produce a verdict for is a `422` `application/problem+json` in the `input` error domain, carrying no verdict:++| Cause | `error_type` |+|---|---|+| `reference` missing or given more than once, or `type` given more than once | `ValidationError` |+| `reference` blank once trimmed, a sigil or namespace with nothing after it, or longer than 199 characters | `InvalidModelReference` |+| `type` unknown or empty | `InvalidModelCategory` |++A reference that parses and resolves nowhere is never refused: it is the `not_found` verdict, which is the successful product of the call. See [Error Responses](error-responses.md) for the problem document's shape.diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yamlindex 72e6c3133..9b0d9083c 100644--- a/docs/api-server/openapi/pipelex-api.openapi.yaml+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml@@ -5,9 +5,10 @@ info:summary: The source-available Pipelex runner — implements MTHDS Protocol v0.7.0.description: 'This server implements the [MTHDS Protocol](https://mthds.ai) v0.7.0 (`POST /execute`, `POST /start`, `POST/validate`, `GET /models`, `GET /version` — marked `x-mthds-protocol: true`) plus the Pipelex API extensions: resolve- and codegen (`/resolve`, `/codegen`), the pipe I/O route (`/pipe-io`), and editor tooling (`/lint`, `/format`). Contract- layering: MTHDS Protocol ⊂ Pipelex API (this server) ⊂ Pipelex hosted API. All endpoints are served under the `/v1` base- path; every error is an RFC 7807 `application/problem+json` problem document, documented per operation as a `ProblemDocument`.'+ and codegen (`/resolve`, `/codegen`), the pipe I/O route (`/pipe-io`), the model reference check (`/models/check`), and+ editor tooling (`/lint`, `/format`). Contract layering: MTHDS Protocol ⊂ Pipelex API (this server) ⊂ Pipelex hosted API.+ All endpoints are served under the `/v1` base path; every error is an RFC 7807 `application/problem+json` problem document,+ documented per operation as a `ProblemDocument`.'license:name: Elastic License 2.0identifier: Elastic-2.0@@ -1148,6 +1149,93 @@ paths:schema:$ref: '#/components/schemas/ProblemDocument'x-mthds-protocol: true+ /v1/models/check:+ get:+ tags:+ - agent+ summary: Check Model+ description: 'Check whether one model reference resolves on this runner, as what, to which model (Pipelex API extension).+++ Answers from pipelex''s own reference parser and deck lookups, the ones a validation runs, so a++ reference found `resolved` in a category is one a pipe of that category may name, and one found++ `not_found` is one a validation refuses. A verdict is a `200` whatever the resolution, and on++ `not_found` it carries the suggestions a failing validation of the same reference offers. A++ request that cannot produce a verdict is an input `422`: `ValidationError` for a missing or++ repeated `reference` or a repeated `type`, `InvalidModelReference` for a reference that is blank,++ a sigil or a namespace alone, or too long, and `InvalidModelCategory` for an unknown `type`.'+ operationId: check_model_v1_models_check_get+ parameters:+ - name: reference+ in: query+ required: true+ schema:+ type: string+ description: 'The model reference as a method''s `model` field writes it: `$preset`, `@alias`, `~waterfall`, a bare+ model handle, or one of the namespaces `preset:`, `alias:`, `waterfall:` and `handle:`. Surrounding whitespace+ is ignored, and the trimmed reference holds at most 199 characters. Single value.'+ title: Reference+ description: 'The model reference as a method''s `model` field writes it: `$preset`, `@alias`, `~waterfall`, a bare+ model handle, or one of the namespaces `preset:`, `alias:`, `waterfall:` and `handle:`. Surrounding whitespace is+ ignored, and the trimmed reference holds at most 199 characters. Single value.'+ - name: type+ in: query+ required: false+ schema:+ anyOf:+ - type: string+ - type: 'null'+ description: 'The category to check in: llm, extract, img_gen, search, judgment or doc_gen. When absent, the check+ is made in every one of them. Single value.'+ title: Type+ description: 'The category to check in: llm, extract, img_gen, search, judgment or doc_gen. When absent, the check+ is made in every one of them. Single value.'+ responses:+ '200':+ description: Successful Response+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/ModelReferenceVerdict'+ '401':+ description: Missing or invalid bearer token. Only reachable when the deployment enables auth (`AUTH_MODE=api_key`+ or `AUTH_MODE=jwt`).+ headers:+ WWW-Authenticate:+ description: Authentication challenge — always `Bearer`.+ schema:+ type: string+ content:+ application/problem+json:+ schema:+ $ref: '#/components/schemas/ProblemDocument'+ '413':+ description: Request body exceeds the deployment's size limit (`MAX_REQUEST_BODY_MIB`, 100 MiB by default).+ content:+ application/problem+json:+ schema:+ $ref: '#/components/schemas/ProblemDocument'+ '422':+ description: 'The request could not be processed: a malformed body, a field failing validation, or an `input`-domain+ pipelex error (a `.mthds` bundle the caller must fix). Note that on the diagnostic routes an *invalid bundle*+ is a **200** verdict, not a 422 — see each route''s response contract.'+ content:+ application/problem+json:+ schema:+ $ref: '#/components/schemas/ProblemDocument'+ '500':+ description: A `config`-domain or `runtime`-domain failure the caller cannot fix (a missing env var, a bad TOML+ override, a backend fault), or an unclassified error sanitized by the catch-all handler. Report the `request_id`.+ content:+ application/problem+json:+ schema:+ $ref: '#/components/schemas/ProblemDocument'/:get:tags:@@ -1217,6 +1305,31 @@ components:fact: who produced it, why, and which upstream absence it chains to. Provenance is capturedat the moment absence is produced, not reconstructed at failure time.'+ AliasMatch:+ properties:+ category:+ $ref: '#/components/schemas/ModelCheckCategory'+ description: The category this entry is about.+ resolves_to:+ anyOf:+ - type: string+ - type: 'null'+ title: Resolves To+ description: The model handle a run through the reference would call now in this category, or null when it would+ find none (a target on a backend the runner has not enabled, a waterfall none of whose usable steps the runner+ serves, an alias cycle).+ target:+ type: string+ title: Target+ description: The model the deck binds the alias to, as the deck writes it, which may itself be a reference.+ additionalProperties: false+ type: object+ required:+ - category+ - resolves_to+ - target+ title: AliasMatch+ description: What an alias reference is in one category.AspectRatio:type: stringenum:@@ -2609,6 +2722,34 @@ components:Both reuse ``NodeUsageSpec`` — one usage shape in the contract, not two. Neither hasa subtree distinct from itself, so their ``subtree_*`` fields repeat their own.'+ HandleMatch:+ properties:+ category:+ $ref: '#/components/schemas/ModelCheckCategory'+ description: The category this entry is about.+ resolves_to:+ anyOf:+ - type: string+ - type: 'null'+ title: Resolves To+ description: The model handle a run through the reference would call now in this category, or null when it would+ find none (a target on a backend the runner has not enabled, a waterfall none of whose usable steps the runner+ serves, an alias cycle).+ via:+ items:+ type: string+ type: array+ title: Via+ description: The presets, aliases and waterfalls of this category whose binding names the handle directly, each+ written as a reference.+ additionalProperties: false+ type: object+ required:+ - category+ - resolves_to+ - via+ title: HandleMatch+ description: What a bare handle reference is in one category.HealthResponse:properties:status:@@ -3402,6 +3543,24 @@ components:`PipeSearch`) and `judgment` (`JudgmentSetting`, `PipeJudge`). A settingsfamily the standard adds brings a category in the next protocol minor.'+ ModelCheckCategory:+ type: string+ enum:+ - llm+ - extract+ - img_gen+ - search+ - judgment+ - doc_gen+ title: ModelCheckCategory+ description: 'The categories the model reference check covers, in the order every list it returns follows.+++ These are the MTHDS Protocol''s model categories, in the protocol''s order, then `doc_gen`, the family++ of `PipeDocGen`. The protocol defines no category for `doc_gen`, so the model listing leaves it out,++ but a method names a `doc_gen` model in its `model` field like any other, so the check covers it.'ModelInfo:properties:name:@@ -3449,6 +3608,77 @@ components:- handletitle: ModelReferenceKinddescription: The kind of model reference.+ ModelReferenceResolution:+ type: string+ enum:+ - resolved+ - not_found+ title: ModelReferenceResolution+ description: Whether a reference resolves in a category in scope. Both values are definitive.+ ModelReferenceVerdict:+ properties:+ reference:+ type: string+ title: Reference+ description: The caller's reference, trimmed.+ kind:+ $ref: '#/components/schemas/ModelReferenceKind'+ description: What the reference names, from its parsing.+ name:+ type: string+ title: Name+ description: The reference without its sigil or namespace.+ category:+ anyOf:+ - $ref: '#/components/schemas/ModelCheckCategory'+ - type: 'null'+ description: The `type` asked, or null when none was.+ resolution:+ $ref: '#/components/schemas/ModelReferenceResolution'+ description: Whether the reference resolves in a category in scope.+ matches:+ items:+ anyOf:+ - $ref: '#/components/schemas/PresetMatch'+ - $ref: '#/components/schemas/AliasMatch'+ - $ref: '#/components/schemas/WaterfallMatch'+ - $ref: '#/components/schemas/HandleMatch'+ type: array+ title: Matches+ description: One entry per category in scope where the reference resolves, in the order of the categories the check+ covers.+ suggestions:+ items:+ type: string+ type: array+ title: Suggestions+ description: On `not_found`, the nearest names the caller may have meant; empty when it is `resolved`.+ other_kinds:+ items:+ type: string+ type: array+ title: Other Kinds+ description: On `not_found`, the same name under another kind, in the categories in scope; empty when it is `resolved`.+ other_categories:+ items:+ $ref: '#/components/schemas/ModelCheckCategory'+ type: array+ title: Other Categories+ description: On `not_found` with a `type`, the categories outside it where the same reference resolves; empty otherwise.+ additionalProperties: false+ type: object+ required:+ - reference+ - kind+ - name+ - category+ - resolution+ - matches+ - suggestions+ - other_kinds+ - other_categories+ title: ModelReferenceVerdict+ description: The verdict of the model reference check, discriminated on `resolution`.ModelUsageSpec:properties:inference_model_name:@@ -5846,6 +6076,38 @@ components:would destroy it for every consumer at once. A consumer that only needs "may this be absent?"answers it as `is_optional`, in exactly one place.'+ PresetMatch:+ properties:+ category:+ $ref: '#/components/schemas/ModelCheckCategory'+ description: The category this entry is about.+ resolves_to:+ anyOf:+ - type: string+ - type: 'null'+ title: Resolves To+ description: The model handle a run through the reference would call now in this category, or null when it would+ find none (a target on a backend the runner has not enabled, a waterfall none of whose usable steps the runner+ serves, an alias cycle).+ target:+ type: string+ title: Target+ description: The model the deck binds the preset to, as the deck writes it, which may itself be a reference.+ description:+ anyOf:+ - type: string+ - type: 'null'+ title: Description+ description: The preset's description, or null when the deck gives it none.+ additionalProperties: false+ type: object+ required:+ - category+ - resolves_to+ - target+ - description+ title: PresetMatch+ description: What a preset reference is in one category.ProblemDocument:properties:type:@@ -7512,6 +7774,33 @@ components:of them would tell a consumer it knows which fault occurred when it does not. Their message is theauthoritative diagnostic.'+ WaterfallMatch:+ properties:+ category:+ $ref: '#/components/schemas/ModelCheckCategory'+ description: The category this entry is about.+ resolves_to:+ anyOf:+ - type: string+ - type: 'null'+ title: Resolves To+ description: The model handle a run through the reference would call now in this category, or null when it would+ find none (a target on a backend the runner has not enabled, a waterfall none of whose usable steps the runner+ serves, an alias cycle).+ fallbacks:+ items:+ type: string+ type: array+ title: Fallbacks+ description: The waterfall's steps, in order.+ additionalProperties: false+ type: object+ required:+ - category+ - resolves_to+ - fallbacks+ title: WaterfallMatch+ description: What a waterfall reference is in one category.WorkingMemory:properties:root:diff --git a/docs/api-server/pipe-validate.md b/docs/api-server/pipe-validate.mdindex 87e5f7c8b..d9ea508dd 100644--- a/docs/api-server/pipe-validate.md+++ b/docs/api-server/pipe-validate.md@@ -104,7 +104,7 @@ The 200 body is one of two arms, discriminated on the mandatory `is_valid` field**Response Fields (invalid arm):**- `is_valid` (`false`): the discriminant of the invalid arm-- `validation_errors` (list): the structured per-error diagnostics a client maps to per-line problems — built by pipelex's one shared builder, so they are byte-for-byte the same items the agent CLI emits. Each item carries a `category` (the closed set `blueprint_validation` | `pipe_factory` | `pipe_validation` | `dry_run`), a `message`, and the locators the runtime can attribute (`error_type`, `pipe_code` / `concept_code` / `domain_code`, `field_path` / `field_name`, and `source`). Absent locators are omitted, not null. An item may also carry a **`suggested_fix`** — a structured, deterministic repair (`fix_code`, `description`, `safety: safe|unsafe`, and the `ops[]` of semantic TOML patch operations to apply). It is present only when the fix planner derived one from the typed error data; a client that ignores it behaves exactly as before. See [Error Responses → Structured validation errors](error-responses.md#structured-validation-errors) and [→ Suggested fixes](error-responses.md#suggested-fixes) for every field. The array is **never empty on an invalid verdict** (the structured-info invariant is total): a failing dry run becomes one `dry_run` item per failing pipe, located at the innermost pipe that failed (its `pipe_code`, `domain_code` and, when known, `source`), a TOML syntax error carries the `line` and `column` the parser stopped at, and a parse-level failure with no attributable locator (an empty blueprint, an elaborator failure) becomes one `blueprint_validation` residual item carrying the message (no `source`). Every refusal of the bundle while it loads is an item of the verdict, never a no-verdict `5xx`: an unknown model is an `unknown_model` item on the pipe's `model` field, a model setting the model it resolves to refuses (a reasoning setting on a model that does not reason, for instance) is an `llm_setting_refused_by_model` item on the field naming the setting, and a call into a method package this runtime cannot resolve (not installed and not fetchable, or a fetch that failed) is an `unresolved_package_dependency` item on the first pipe naming the package, whose `missing_pipe_code` is the reference as written, `<address>->domain.code`+- `validation_errors` (list): the structured per-error diagnostics a client maps to per-line problems — built by pipelex's one shared builder, so they are byte-for-byte the same items the agent CLI emits. Each item carries a `category` (the closed set `blueprint_validation` | `pipe_factory` | `pipe_validation` | `dry_run`), a `message`, and the locators the runtime can attribute (`error_type`, `pipe_code` / `concept_code` / `domain_code`, `field_path` / `field_name`, and `source`). Absent locators are omitted, not null. An item may also carry a **`suggested_fix`** — a structured, deterministic repair (`fix_code`, `description`, `safety: safe|unsafe`, and the `ops[]` of semantic TOML patch operations to apply). It is present only when the fix planner derived one from the typed error data; a client that ignores it behaves exactly as before. See [Error Responses → Structured validation errors](error-responses.md#structured-validation-errors) and [→ Suggested fixes](error-responses.md#suggested-fixes) for every field. The array is **never empty on an invalid verdict** (the structured-info invariant is total): a failing dry run becomes one `dry_run` item per failing pipe, located at the innermost pipe that failed (its `pipe_code`, `domain_code` and, when known, `source`), a TOML syntax error carries the `line` and `column` the parser stopped at, and a parse-level failure with no attributable locator (an empty blueprint, an elaborator failure) becomes one `blueprint_validation` residual item carrying the message (no `source`). Every refusal of the bundle while it loads is an item of the verdict, never a no-verdict `5xx`: an unknown model is an `unknown_model` item on the pipe's `model` field (a reference the deck does not define, or a bare handle it serves only as another model type, such as an LLM named in a `PipeImgGen`; [`GET /v1/models/check`](models.md#check-a-model-reference) answers the same question for one reference), a model setting the model it resolves to refuses (a reasoning setting on a model that does not reason, for instance) is an `llm_setting_refused_by_model` item on the field naming the setting, and a call into a method package this runtime cannot resolve (not installed and not fetchable, or a fetch that failed) is an `unresolved_package_dependency` item on the first pipe naming the package, whose `missing_pipe_code` is the reference as written, `<address>->domain.code`- `pending_signatures` (list[str]): best-effort outstanding signatures (empty on the invalid arm, since no library was assembled)- `is_runnable` (`false`): an invalid bundle is never runnable- `message` (string): the human-readable verdict summary: the lone item's message when there is one item, `N validation errors (first: <the first item's message>)` when there are several, and the parser's own message for a parse-level failurediff --git a/docs/building-methods/configure-ai-llm-to-optimize-methods.md b/docs/building-methods/configure-ai-llm-to-optimize-methods.mdindex a2aa1fafa..5e6de2f38 100644--- a/docs/building-methods/configure-ai-llm-to-optimize-methods.md+++ b/docs/building-methods/configure-ai-llm-to-optimize-methods.md@@ -32,7 +32,7 @@ default-general = "gpt-5.6-terra"default-small = "gpt-5.6-luna"```-The system first looks for direct model names, then checks aliases if no direct match is found. The system handles model routing through backends automatically.+A bare name in a pipe's `model` field is first looked up among the models the enabled backends serve as the pipe's model type, then among the aliases of that type, then among its waterfalls while model fallback is on. A model served as another type does not count: an LLM named in a `PipeImgGen` is refused when the bundle is validated or loaded, with an `unknown_model` item naming the pipe, the handle and the type the pipe needs. The deck's own presets and LLM overrides are held to the same rule when the runner boots. A sigiled name is looked up by its sigil alone, so `~name` is the waterfall even where a model or an alias shares its name. The system handles model routing through backends automatically.!!! tip "Alias Naming Convention"Defining an alias is always meant to describe what model it is. Never define an alias to describe what it is for or what it's good at. LLM Presets are for that.diff --git a/docs/configuration/config-technical/inference-backend-config.md b/docs/configuration/config-technical/inference-backend-config.mdindex 64db7ed53..5b34b4fd7 100644--- a/docs/configuration/config-technical/inference-backend-config.md+++ b/docs/configuration/config-technical/inference-backend-config.md@@ -235,7 +235,7 @@ The table name `acme-one-judgment` names the table and nothing else: methods, thA routing profile routes a handle's name, so its route applies to every kind of model of that name. A model type the routed backend does not serve is looked for along the profile's `fallback_order`, or in the internal backend when it has none, when the name reached that backend by default, is left out when it reached it through a wildcard pattern, and is not served at all when the profile routes the name to that backend exactly: an exact route pins the name to one backend.-A method whose pipe names a handle the deck serves only as another kind of model is refused when it loads: a `PipeJudge` naming a handle served only as an LLM is told that no judgment model of that name exists, and the suggestions list judgment models only.+A method whose pipe names a handle the deck serves only as another kind of model is refused when it loads: a `PipeJudge` naming a handle served only as an LLM is told that the deck serves the handle, but not as a judgment model, and the suggestions list judgment models only.#### Input formatsdiff --git a/docs/errors/model-choice-not-found-error.md b/docs/errors/model-choice-not-found-error.mdindex 2e5851019..ac0503a59 100644--- a/docs/errors/model-choice-not-found-error.md+++ b/docs/errors/model-choice-not-found-error.md@@ -7,7 +7,7 @@ description: "Reference for the `ModelChoiceNotFoundError` Pipelex error class."# Model choice not found-Raised when a model reference names a handle, alias, preset or waterfall the model deck does not define: by the deck check a pipe runs when it is built, and by the deck when a run resolves a reference. When a pipe is built, the pipe operator raises it again as a ``PipeOperatorModelChoiceError`` located on the pipe and the field, so a bundle naming an unknown model is an invalid validation verdict (error type ``unknown_model``), never a failure of the validator.+Raised when a model reference names a handle, alias, preset or waterfall the model deck does not define, or a bare handle the deck serves only as another model type: by the deck check a pipe runs when it is built, and by the deck when a run resolves a reference. When a pipe is built, the pipe operator raises it again as a ``PipeOperatorModelChoiceError`` located on the pipe and the field, so a bundle naming an unknown model is an invalid validation verdict (error type ``unknown_model``), never a failure of the validator.| Field | Value ||---|---|diff --git a/docs/errors/pipe-operator-model-choice-error.md b/docs/errors/pipe-operator-model-choice-error.mdindex 3dec7c482..12932494e 100644--- a/docs/errors/pipe-operator-model-choice-error.md+++ b/docs/errors/pipe-operator-model-choice-error.md@@ -7,7 +7,7 @@ description: "Reference for the `PipeOperatorModelChoiceError` Pipelex error cla# Pipe operator model choice-Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``, ``PipeJudge``) when it is built from its blueprint and a model field names a model the model deck does not define. Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``, and a run, which loads its bundle before any pipe runs, refuses the bundle with that same verdict.+Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``, ``PipeJudge``) when it is built from its blueprint and a model field names a model the model deck does not define, or a bare handle the deck serves only as another model type (an LLM named in a ``PipeImgGen``, for instance). Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``, and a run, which loads its bundle before any pipe runs, refuses the bundle with that same verdict.| Field | Value ||---|---|diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.mdindex eaf299861..7d35d92c3 100644--- a/docs/tools/cli/agent-cli.md+++ b/docs/tools/cli/agent-cli.md@@ -152,6 +152,7 @@ These commands do not have subcommands:| `fmt` | Format a `.mthds`/`.toml`/`.plx` file in-place (delegates to `plxt`) || `lint` | Lint a `.mthds`/`.toml`/`.plx` file for errors (delegates to `plxt`) || `models` | List available model presets, aliases, and waterfalls (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |+| `check-model` | Check a model reference for a model category (`check-model <reference> --type <category>`, the category being `llm`, `extract`, `img_gen`, `search`, `judgment` or `doc_gen`) and suggest alternatives when it is not valid. It applies the rule a validation applies, as [`GET /v1/models/check`](../../api-server/models.md#check-a-model-reference) does: a sigiled reference names that kind of binding, and a bare name is valid when it is a model of the category, an alias of the category, or a waterfall of the category while model fallback is on; a `handle:` reference writes a bare name without reading a sigil, so `handle:@best-gpt` is the bare name `@best-gpt`, not the alias `best-gpt`. `--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value || `doctor` | Check config, credentials, and model health (`--format markdown\|json` success, default: markdown; `--error-format` for errors, defaults to `--format`'s value) |## Output Contractdiff --git a/mkdocs.yml b/mkdocs.ymlindex 28c37b8ec..a5babdf47 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -383,6 +383,7 @@ plugins:- api-server/pipe-io.md: "API Server Pipe I/O"- api-server/codegen.md: "API Server Resolve & Codegen"- api-server/mthds-tools.md: "API Server MTHDS Tools"+ - api-server/models.md: "API Server Models"- api-server/error-responses.md: "API Server Error Responses"- api-server/logging.md: "API Server Logging"@@ -638,6 +639,7 @@ nav:- Pipe I/O: api-server/pipe-io.md- Resolve & Codegen: api-server/codegen.md- MTHDS Tools: api-server/mthds-tools.md+ - Models: api-server/models.md- Error Responses: api-server/error-responses.md- Logging: api-server/logging.md- Storage Transport (Removed): api-server/storage-transport.mddiff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.mdindex d21c1a25a..d46b8f1dc 100644--- a/pipelex/cli/agent_cli/CLAUDE.md+++ b/pipelex/cli/agent_cli/CLAUDE.md@@ -91,7 +91,7 @@ commands/| `inputs` | Generates an example inputs template for a pipe/bundle/method (pipe\|bundle\|method subcommands). `--format json\|toml` (template serialization, default: json — NOT the markdown\|json pair); `toml` prints raw TOML to stdout || `codegen` | Agent mirror of the bare `pipelex codegen` family (types\|check subcommands). `types --target <flavor>` resolves the closure into the normalized crate and writes stamped typed artifacts + `codegen.lock` (write-if-changed); `check` is the offline drift check (pure hashing, no Pipelex boot — exit 0 current, 1 drift as a structured `CodegenDriftError` with `drifts[]`, 2 no/unreadable lock). Both: `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). || `models` | Lists available model presets, aliases, and waterfalls. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value) |-| `check-model` | Validates a model reference and suggests alternatives if invalid. `--type`/`-t` for model category, `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value) |+| `check-model` | Validates a model reference by `ModelDeck.is_reference_defined`, the rule a validation and `GET /v1/models/check` apply, and suggests alternatives if invalid. `--type`/`-t` for model category, a `ModelCheckCategory` (the check's categories, `doc_gen` included, which the `models` listing's `ModelCategory` leaves out), `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value) || `migrate` | Migrates this machine's configuration files (the home configuration directory, `~/.pipelex/` or `PIPELEX_HOME`, and project `.pipelex/`, non-recursively) to the current schema by replaying each surface's ledger. Does **not** boot — it is the command for a configuration that cannot load. Writes only with `--yes`; `--dry-run` plans and is the default; the two together are refused with exit 2. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json`. The verdict is the structured `needs_attention`, not the exit code. || `doctor` | Checks config, credentials, models health. `--global`/`-g` for global dir. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). The telemetry check carries a `finding` (`healthy`/`not_found`/`unparseable`/`out_of_date`/`invalid`) beside its `healthy` and `message`; branch on that, and note `out_of_date` is `pipelex migrate`'s and never `pipelex init telemetry`'s — a reset would discard the settings the migration keeps — and that it means the migration would actually rewrite that file, so a file it would only report on is `invalid`. `checks.pending_migrations` is the machine-wide half: a `finding` (`up_to_date`/`pending`/`needs_attention`/`unavailable`) plus `migratable_files` and `attention_files`. It is the only channel a machine has for a pending migration — a tolerated boot warns a person and this CLI silences logging — and it is **not** scoped by `--global`, because it answers for `pipelex migrate`, which walks both directories. `checks.plugins` is present once the runtime setup ran, which discovers the plugins the way boot does: `healthy` and `message`, not healthy when the plugin registry did not build, in which case the message names the plugin error. `checks.secrets_provider` is present once the runtime setup ran, which builds the provider `[runtime.secrets] method` selects before the sink, as boot does: `healthy` and `message`, not healthy when the method names no registered provider, when its factory raised or when the registry did not build. It is built even when logging was configured before the doctor ran, since `checks.models` resolves the backends' credentials through it, and when it did not build the models check says it could not run; a provider that raises on a lookup is reported in `checks.models` rather than failing the command. `checks.log_sink` is present once the runtime setup ran: `healthy` and `message`, not healthy when `[runtime.log] sink` names no registered sink, when the configured sink failed to install (a `${…}` placeholder in its settings that does not resolve among the causes), when the secrets provider did not build, in which case the sink is not attempted, or when the registry did not build. The message says why, and says where the report's own lines went: onto the console sink on stderr, which the doctor installs in place of the one that failed, or onto a sink already recorded, which it leaves alone rather than installing over — a sink is recorded before the held records are replayed, so a failed installation does not always leave nothing behind. The fourth case is its own: a sink that was installed and then failed while the records held since logging was configured were replayed through it, where the report runs on that sink, which stays installed, and the message says what it raised. A failure the `secrets_provider` and `log_sink` messages quote is scrubbed with the log redaction patterns, the configured `extra_patterns` included, because the envelope reaches its reader without passing through any sink, and so is a failure of the backends' setup that the `models` message quotes, where a secrets provider's own text can appear; the backend-file findings and the gateway and plugin-registry failures it quotes are Pipelex's own text and are quoted as raised. |diff --git a/pipelex/cli/agent_cli/commands/check_model_cmd.py b/pipelex/cli/agent_cli/commands/check_model_cmd.pyindex 42a6b8c02..3a9c93dae 100644--- a/pipelex/cli/agent_cli/commands/check_model_cmd.py+++ b/pipelex/cli/agent_cli/commands/check_model_cmd.py@@ -7,13 +7,9 @@ import typerfrom pipelex.cli.agent_cli.commands.agent_cli_factory import make_pipelex_for_agent_clifrom pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_error, agent_success, set_agent_cli_error_formatfrom pipelex.cogt.models.exceptions import ModelReferenceParseError-from pipelex.cogt.models.model_listing import CATEGORY_TO_MODEL_TYPE, ModelCategoryfrom pipelex.cogt.models.model_reference import ModelReference-from pipelex.cogt.models.model_suggestion import (- KIND_LABELS,- get_collection_keys,- suggest_model_alternatives,-)+from pipelex.cogt.models.model_reference_check import ModelCheckCategory+from pipelex.cogt.models.model_suggestion import KIND_LABELS, suggest_model_alternativesfrom pipelex.pipelex import Pipelexfrom pipelex.runtime_hub import get_model_deck@@ -57,8 +53,8 @@ def agent_check_model_cmd(typer.Argument(help="Model reference to check (e.g. $writing-creative, @best-gpt, gpt-4o)"),],model_type: Annotated[- ModelCategory,- typer.Option("--type", "-t", help="Model category: llm, extract, img_gen, search, judgment"),+ ModelCheckCategory,+ typer.Option("--type", "-t", help="Model category: llm, extract, img_gen, search, judgment, doc_gen"),],output_format: Annotated[CliOutputFormat,@@ -72,7 +68,8 @@ def agent_check_model_cmd("""Check if a model reference is valid and suggest alternatives if not.Parses the reference (with sigil prefix if present), validates it against the- model deck, and on failure provides fuzzy suggestions and wrong-sigil hints.+ model deck by the rule a validation applies, and on failure provides fuzzy+ suggestions and wrong-sigil hints."""set_agent_cli_error_format(error_format or output_format)try:@@ -80,10 +77,11 @@ def agent_check_model_cmd(model_deck = get_model_deck()ref = ModelReference.parse(name)- resolved_model_type = CATEGORY_TO_MODEL_TYPE[model_type]+ resolved_model_type = model_type.model_type- candidates = get_collection_keys(model_deck, model_type=resolved_model_type, kind=ref.kind)- is_valid = ref.name in candidates+ # The rule a validation and `GET /v1/models/check` apply: a bare name is a model of this type,+ # an alias of this type, or a waterfall of this type while model fallback is on.+ is_valid = model_deck.is_reference_defined(reference=ref, model_type=resolved_model_type)result: dict[str, Any] = {"success": True,diff --git a/pipelex/cogt/doc_gen/doc_gen_engine.py b/pipelex/cogt/doc_gen/doc_gen_engine.pyindex 7627b001b..272be4b3c 100644--- a/pipelex/cogt/doc_gen/doc_gen_engine.py+++ b/pipelex/cogt/doc_gen/doc_gen_engine.py@@ -18,7 +18,7 @@ from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice, DocGenSettinfrom pipelex.cogt.doc_gen.exceptions import KNOWN_DOC_GEN_MODEL_NAMES, DocGenEngineGap, DocGenEngineMissingError, DocGenModelCapabilityErrorfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_type import ModelType-from pipelex.cogt.models.model_reference import ModelReferenceKind, ensure_model_reference+from pipelex.cogt.models.model_reference import ModelReferenceKind, ensure_model_reference, write_model_handlefrom pipelex.plugins.inference_backend_registry import InferenceFamilyfrom pipelex.runtime_hub import get_inference_backend_registry, get_model_deck@@ -156,6 +156,8 @@ def resolve_doc_gen_setting(if not prints_document(inference_model=inference_model, doc_gen_format=doc_gen_format, source=source):raise DocGenModelCapabilityError(doc_gen_format=doc_gen_format, source=source, model=inference_model.name, pipe_code=pipe_code)require_doc_gen_engine_installed(inference_model=inference_model, doc_gen_format=doc_gen_format, source=source, pipe_code=pipe_code)- if inference_model.name != doc_gen_setting.model:- doc_gen_setting = doc_gen_setting.model_copy(update={"model": inference_model.name})+ # Pinned as a reference that parses back to the resolved handle, whatever its spelling, since every lookup reads it again.+ resolved_handle = write_model_handle(name=inference_model.name)+ if resolved_handle != doc_gen_setting.model:+ doc_gen_setting = doc_gen_setting.model_copy(update={"model": resolved_handle})return doc_gen_settingdiff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 326c11f76..1edc0c47d 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -194,11 +194,11 @@ class SdkTypeError(CogtError):class ModelChoiceNotFoundError(CogtError):- """Raised when a model reference names a handle, alias, preset or waterfall the model deck does not define:- by the deck check a pipe runs when it is built, and by the deck when a run resolves a reference. When a- pipe is built, the pipe operator raises it again as a ``PipeOperatorModelChoiceError`` located on the pipe- and the field, so a bundle naming an unknown model is an invalid validation verdict (error type- ``unknown_model``), never a failure of the validator.+ """Raised when a model reference names a handle, alias, preset or waterfall the model deck does not define,+ or a bare handle the deck serves only as another model type: by the deck check a pipe runs when it is+ built, and by the deck when a run resolves a reference. When a pipe is built, the pipe operator raises it+ again as a ``PipeOperatorModelChoiceError`` located on the pipe and the field, so a bundle naming an unknown+ model is an invalid validation verdict (error type ``unknown_model``), never a failure of the validator.Includes available options and migration hints in error message."""diff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.pyindex 7e07b4912..174f3f0e2 100644--- a/pipelex/cogt/models/model_deck.py+++ b/pipelex/cogt/models/model_deck.py@@ -1,3 +1,4 @@+from collections.abc import Mappingfrom typing import NoReturn, Selffrom pydantic import Field, PrivateAttr, field_validator, model_validator@@ -33,7 +34,15 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_spec_index import ModelSpecIndexfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.exceptions import ModelReferenceParseError-from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind, ensure_model_reference+from pipelex.cogt.models.model_reference import (+ NAMESPACE_ALIAS,+ NAMESPACE_WATERFALL,+ SIGIL_WATERFALL,+ ModelReference,+ ModelReferenceKind,+ ensure_model_reference,+ write_model_handle,+)from pipelex.cogt.search.search_setting import SearchModelChoice, SearchSettingfrom pipelex.system.configuration.config_model import ConfigModelfrom pipelex.system.exceptions import ConfigValidationError@@ -190,11 +199,103 @@ class ModelDeck(ConfigModel):case ModelType.JUDGMENT:return self.judgment_aliases, self.judgment_waterfalls+ def get_presets_for_type(+ self, *, model_type: ModelType+ ) -> Mapping[str, LLMSetting | ExtractSetting | ImgGenSetting | SearchSetting | DocGenSetting | JudgmentSetting]:+ """Return the type-specific presets dictionary."""+ match model_type:+ case ModelType.LLM:+ return self.llm_presets+ case ModelType.TEXT_EXTRACTOR:+ return self.extract_presets+ case ModelType.IMG_GEN:+ return self.img_gen_presets+ case ModelType.SEARCH:+ return self.search_presets+ case ModelType.DOC_GEN:+ return self.doc_gen_presets+ case ModelType.JUDGMENT:+ return self.judgment_presets++ def is_bare_handle_resolvable(self, *, name: str, model_type: ModelType) -> bool:+ """Whether a bare model name resolves for this model type, as a pipe's `model` field may name it.++ The name resolves when the runner can call a model of that name of this type, among every+ model it can call and not only those the deck names; failing that, when the deck defines an+ alias of that name for this type, or a waterfall of that name while model fallback is on.+ The order is the one the run's lookup (`get_optional_inference_model`) reads a bare name in.+ The fallback gate is the validation's and the setting builders' (`get_*_setting`), which a+ pipe's field passes through before the run's lookup: that lookup itself also resolves a bare+ waterfall name while fallback is off, where a binding inside the deck names one. A model of+ that name served only as another type does not make it resolve, so a pipe naming it is refused+ when its bundle loads rather than when the run reaches it. The name's sigil is not read: a+ `handle:` reference whose name starts with a sigil is that bare name, `@x` and not the alias `x`.+ """+ if self.inference_models.get(model_type=model_type, handle=name) is not None:+ return True+ aliases, waterfalls = self.get_aliases_and_waterfalls_for_type(model_type)+ if name in aliases:+ return True+ return self.model_deck_config.is_model_fallback_enabled and name in waterfalls++ def is_reference_defined(self, *, reference: ModelReference, model_type: ModelType) -> bool:+ """Whether a pipe of this model type may name this reference in its `model` field.++ This is the test a validation applies to the field: a preset, an alias or a waterfall is+ defined when the deck defines that name for the type, and a bare handle when it resolves+ by `is_bare_handle_resolvable`. It is a matter of names: a preset, an alias or a waterfall+ whose binding reaches no model the runner can call is still defined.+ """+ match reference.kind:+ case ModelReferenceKind.PRESET:+ return reference.name in self.get_presets_for_type(model_type=model_type)+ case ModelReferenceKind.ALIAS:+ aliases, _ = self.get_aliases_and_waterfalls_for_type(model_type)+ return reference.name in aliases+ case ModelReferenceKind.WATERFALL:+ _, waterfalls = self.get_aliases_and_waterfalls_for_type(model_type)+ return reference.name in waterfalls+ case ModelReferenceKind.HANDLE:+ return self.is_bare_handle_resolvable(name=reference.name, model_type=model_type)++ def unresolved_handle_sentence(self, *, name: str, model_type: ModelType) -> str:+ """The sentence refusing a bare handle that does not resolve for this model type.++ It names the handle and the type the field needs. When the deck serves a model of that name+ only as other types, it says so rather than calling the handle missing, without naming them.+ """+ served_types = self.inference_models.types_serving(handle=name)+ if served_types and model_type not in served_types:+ return f"Model handle '{name}' is served by the model deck, but not as {model_type.indefinite_description}"+ return f"Model handle '{name}' was not found in the model deck"++ def _unresolved_preset_model_sentence(self, *, preset_label: str, preset_id: str, model_reference: str, model_type: ModelType) -> str:+ """The sentence refusing a deck preset whose model does not resolve for the preset's type, naming the preset.++ A bare handle gets `unresolved_handle_sentence`, so a model the deck serves only as another type+ is named as such rather than as missing.+ """+ sentence = f"Model reference '{model_reference}' was not found in the model deck"+ ref: ModelReference | None+ try:+ ref = ModelReference.parse(model_reference)+ except ModelReferenceParseError:+ ref = None+ if ref is not None:+ match ref.kind:+ case ModelReferenceKind.HANDLE:+ sentence = self.unresolved_handle_sentence(name=ref.name, model_type=model_type)+ case ModelReferenceKind.PRESET | ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL:+ pass+ return f"{preset_label} preset '{preset_id}': {sentence}"+def is_model_handle_defined(self, model_handle: str, *, model_type: ModelType) -> bool:- """Check if a model handle is defined in the model deck.+ """Check if a model handle is defined in the model deck for this model type.Handles prefixed references (e.g., @alias_name, ~waterfall_name) by parsing- them and looking up the appropriate dictionary.+ them and looking up the appropriate dictionary. A bare handle is defined when it+ resolves for this model type (`is_bare_handle_resolvable`): a model the deck serves+ only as another type does not define it."""ref = ModelReference.parse(model_handle)aliases, waterfalls = self.get_aliases_and_waterfalls_for_type(model_type)@@ -210,12 +311,7 @@ class ModelDeck(ConfigModel):# Presets are handled separately, not as direct handlesreturn Falsecase ModelReferenceKind.HANDLE:- # Direct handle lookup - check the models served as this type, aliases, and waterfalls- if self.inference_models.get(model_type=model_type, handle=ref.name) is not None:- return True- if ref.name in aliases:- return True- return self.model_deck_config.is_model_fallback_enabled and ref.name in waterfalls+ return self.is_bare_handle_resolvable(name=ref.name, model_type=model_type)def _warn_if_ambiguous_llm(self, name: str) -> None:"""Log a warning if a bare string handle matches presets/aliases/waterfalls."""@@ -314,7 +410,7 @@ class ModelDeck(ConfigModel):| dict[str, JudgmentSetting],) -> NoReturn:"""Raise ModelChoiceNotFoundError with migration hints if applicable."""- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = self.unresolved_handle_sentence(name=ref.name, model_type=model_type)# Add migration hints if the name matches a preset, alias, or waterfallaliases, waterfalls = self.get_aliases_and_waterfalls_for_type(model_type)@@ -385,9 +481,9 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.HANDLE:self._warn_if_ambiguous_llm(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.LLM):+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.LLM):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = self.unresolved_handle_sentence(name=ref.name, model_type=ModelType.LLM)raise ModelChoiceNotFoundError(message=msg,model_type=ModelType.LLM,@@ -427,8 +523,8 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.WATERFALL:if ref.name in self.llm_waterfalls:- # Use the waterfall name as the model handle (it will resolve via get_optional_inference_model)- return LLMSetting(model=ref.name, temperature=self.llm_default_temperature)+ # Keep the sigil, so the run resolves the waterfall itself and not a model or an alias sharing its name+ return LLMSetting(model=f"{SIGIL_WATERFALL}{ref.name}", temperature=self.llm_default_temperature)msg = f"Waterfall '{ref.name}' was not found in the model deck"raise ModelChoiceNotFoundError(message=msg,@@ -440,8 +536,8 @@ class ModelDeck(ConfigModel):case ModelReferenceKind.HANDLE:# Strict: treat as direct model handle onlyself._warn_if_ambiguous_llm(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.LLM):- return LLMSetting(model=ref.name, temperature=self.llm_default_temperature)+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.LLM):+ return LLMSetting(model=write_model_handle(name=ref.name), temperature=self.llm_default_temperature)# Error includes migration hint if name matches preset/waterfallself._raise_handle_not_found_error(ref=ref,@@ -479,7 +575,7 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.WATERFALL:if ref.name in self.extract_waterfalls:- return ExtractSetting(model=ref.name)+ return ExtractSetting(model=f"{SIGIL_WATERFALL}{ref.name}")msg = f"Waterfall '{ref.name}' was not found in the model deck"raise ModelChoiceNotFoundError(message=msg,@@ -490,8 +586,8 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.HANDLE:self._warn_if_ambiguous_extract(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.TEXT_EXTRACTOR):- return ExtractSetting(model=ref.name)+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.TEXT_EXTRACTOR):+ return ExtractSetting(model=write_model_handle(name=ref.name))self._raise_handle_not_found_error(ref=ref,model_type=ModelType.TEXT_EXTRACTOR,@@ -528,7 +624,7 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.WATERFALL:if ref.name in self.search_waterfalls:- return SearchSetting(model=ref.name)+ return SearchSetting(model=f"{SIGIL_WATERFALL}{ref.name}")msg = f"Waterfall '{ref.name}' was not found in the model deck"raise ModelChoiceNotFoundError(message=msg,@@ -539,8 +635,8 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.HANDLE:self._warn_if_ambiguous_search(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.SEARCH):- return SearchSetting(model=ref.name)+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.SEARCH):+ return SearchSetting(model=write_model_handle(name=ref.name))self._raise_handle_not_found_error(ref=ref,model_type=ModelType.SEARCH,@@ -581,7 +677,7 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.WATERFALL:if ref.name in self.doc_gen_waterfalls:- return DocGenSetting(model=ref.name)+ return DocGenSetting(model=f"{SIGIL_WATERFALL}{ref.name}")msg = f"Waterfall '{ref.name}' was not found in the model deck"raise ModelChoiceNotFoundError(message=msg,@@ -592,8 +688,8 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.HANDLE:self._warn_if_ambiguous_doc_gen(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.DOC_GEN):- return DocGenSetting(model=ref.name)+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.DOC_GEN):+ return DocGenSetting(model=write_model_handle(name=ref.name))self._raise_handle_not_found_error(ref=ref,model_type=ModelType.DOC_GEN,@@ -630,7 +726,7 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.WATERFALL:if ref.name in self.judgment_waterfalls:- return JudgmentSetting(model=ref.name)+ return JudgmentSetting(model=f"{SIGIL_WATERFALL}{ref.name}")msg = f"Waterfall '{ref.name}' was not found in the model deck"raise ModelChoiceNotFoundError(message=msg,@@ -641,8 +737,8 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.HANDLE:self._warn_if_ambiguous_judgment(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.JUDGMENT):- return JudgmentSetting(model=ref.name)+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.JUDGMENT):+ return JudgmentSetting(model=write_model_handle(name=ref.name))self._raise_handle_not_found_error(ref=ref,model_type=ModelType.JUDGMENT,@@ -679,7 +775,7 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.WATERFALL:if ref.name in self.img_gen_waterfalls:- return ImgGenSetting(model=ref.name, quality=self.img_gen_default_quality)+ return ImgGenSetting(model=f"{SIGIL_WATERFALL}{ref.name}", quality=self.img_gen_default_quality)msg = f"Waterfall '{ref.name}' was not found in the model deck"raise ModelChoiceNotFoundError(message=msg,@@ -690,8 +786,8 @@ class ModelDeck(ConfigModel):)case ModelReferenceKind.HANDLE:self._warn_if_ambiguous_img_gen(ref.name)- if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.IMG_GEN):- return ImgGenSetting(model=ref.name, quality=self.img_gen_default_quality)+ if self.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.IMG_GEN):+ return ImgGenSetting(model=write_model_handle(name=ref.name), quality=self.img_gen_default_quality)self._raise_handle_not_found_error(ref=ref,model_type=ModelType.IMG_GEN,@@ -733,7 +829,9 @@ class ModelDeck(ConfigModel):for llm_preset_id, llm_setting in self.llm_presets.items():if not self.is_model_handle_defined(model_handle=llm_setting.model, model_type=ModelType.LLM):enabled_backends = self._get_enabled_backends()- msg = f"LLM handle '{llm_setting.model}' for llm preset '{llm_preset_id}' was not found in the model deck"+ msg = self._unresolved_preset_model_sentence(+ preset_label="LLM", preset_id=llm_preset_id, model_reference=llm_setting.model, model_type=ModelType.LLM+ )raise LLMHandleNotFoundError(message=msg,preset_id=llm_preset_id,@@ -745,7 +843,9 @@ class ModelDeck(ConfigModel):def validate_img_gen_presets(self) -> Self:for img_gen_preset_id, img_gen_setting in self.img_gen_presets.items():if not self.is_model_handle_defined(model_handle=img_gen_setting.model, model_type=ModelType.IMG_GEN):- msg = f"Image generation handle '{img_gen_setting.model}' for preset '{img_gen_preset_id}' was not found in the model deck"+ msg = self._unresolved_preset_model_sentence(+ preset_label="Image generation", preset_id=img_gen_preset_id, model_reference=img_gen_setting.model, model_type=ModelType.IMG_GEN+ )raise ImgGenHandleNotFoundError(message=msg,preset_id=img_gen_preset_id,@@ -756,7 +856,9 @@ class ModelDeck(ConfigModel):def validate_extract_presets(self) -> Self:for extract_preset_id, extract_setting in self.extract_presets.items():if not self.is_model_handle_defined(model_handle=extract_setting.model, model_type=ModelType.TEXT_EXTRACTOR):- msg = f"Extract handle '{extract_setting.model}' for extract preset '{extract_preset_id}' was not found in the model deck"+ msg = self._unresolved_preset_model_sentence(+ preset_label="Extract", preset_id=extract_preset_id, model_reference=extract_setting.model, model_type=ModelType.TEXT_EXTRACTOR+ )raise ExtractHandleNotFoundError(message=msg,preset_id=extract_preset_id,@@ -767,7 +869,9 @@ class ModelDeck(ConfigModel):def validate_search_presets(self) -> Self:for search_preset_id, search_setting in self.search_presets.items():if not self.is_model_handle_defined(model_handle=search_setting.model, model_type=ModelType.SEARCH):- msg = f"Search handle '{search_setting.model}' for search preset '{search_preset_id}' was not found in the model deck"+ msg = self._unresolved_preset_model_sentence(+ preset_label="Search", preset_id=search_preset_id, model_reference=search_setting.model, model_type=ModelType.SEARCH+ )raise SearchHandleNotFoundError(message=msg,preset_id=search_preset_id,@@ -778,7 +882,9 @@ class ModelDeck(ConfigModel):def validate_doc_gen_presets(self) -> Self:for doc_gen_preset_id, doc_gen_setting in self.doc_gen_presets.items():if not self.is_model_handle_defined(model_handle=doc_gen_setting.model, model_type=ModelType.DOC_GEN):- msg = f"Doc gen handle '{doc_gen_setting.model}' for doc gen preset '{doc_gen_preset_id}' was not found in the model deck"+ msg = self._unresolved_preset_model_sentence(+ preset_label="Doc gen", preset_id=doc_gen_preset_id, model_reference=doc_gen_setting.model, model_type=ModelType.DOC_GEN+ )raise DocGenHandleNotFoundError(message=msg,preset_id=doc_gen_preset_id,@@ -789,7 +895,9 @@ class ModelDeck(ConfigModel):def validate_judgment_presets(self) -> Self:for judgment_preset_id, judgment_setting in self.judgment_presets.items():if not self.is_model_handle_defined(model_handle=judgment_setting.model, model_type=ModelType.JUDGMENT):- msg = f"Judgment handle '{judgment_setting.model}' for judgment preset '{judgment_preset_id}' was not found in the model deck"+ msg = self._unresolved_preset_model_sentence(+ preset_label="Judgment", preset_id=judgment_preset_id, model_reference=judgment_setting.model, model_type=ModelType.JUDGMENT+ )raise JudgmentHandleNotFoundError(message=msg,preset_id=judgment_preset_id,@@ -899,7 +1007,7 @@ class ModelDeck(ConfigModel):def validate_inference_models(self):for model_spec in self.inference_models.all_specs():- self.get_required_inference_model(model_handle=model_spec.name, model_type=model_spec.model_type)+ self.get_required_inference_model(model_handle=write_model_handle(name=model_spec.name), model_type=model_spec.model_type)def _get_enabled_backends(self) -> set[str]:"""Return the set of backend names that have at least one model enabled."""@@ -911,10 +1019,25 @@ class ModelDeck(ConfigModel):*,fallback_list: list[str],model_type: ModelType,+ visited: frozenset[str],+ is_quiet: bool,) -> InferenceModelSpec | None:- """Resolve a waterfall to an inference model spec by trying each fallback in order."""+ """Resolve a waterfall to an inference model spec by trying each fallback in order.++ `visited` holds the aliases and waterfalls the lookup is already inside, so a step leading back+ to one of them ends as no model rather than recursing, and so does a step reaching a waterfall that+ runs out. A quiet lookup logs nothing and leaves the one-time fallback notice to the next lookup that+ falls back.+ """+ waterfall_key = f"{NAMESPACE_WATERFALL}{waterfall_name}"+ if waterfall_key in visited:+ if not is_quiet:+ log.warning(f"Circular model reference detected: waterfall '{waterfall_name}' leads back to itself")+ return None+ step_visited = visited | {waterfall_key}ideal_model_handle = fallback_list[0]- log.verbose(f"Fallback list for '{waterfall_name}': {fallback_list}")+ if not is_quiet:+ log.verbose(f"Fallback list for '{waterfall_name}': {fallback_list}")for fallback_index, fallback in enumerate(fallback_list):if fallback_index > 0 and not self.model_deck_config.is_model_fallback_enabled:# Waterfall disabled, so we raise an error@@ -927,22 +1050,36 @@ class ModelDeck(ConfigModel):f"or enable a backend that supports '{ideal_model_handle}'. ")raise ModelNotFoundError(message=msg, model_handle=waterfall_name)- if inference_model := self.get_optional_inference_model(model_handle=fallback, model_type=model_type):- if fallback_index > 0:- # Only log if we haven't logged for this waterfall_name before- if waterfall_name not in self._logged_fallback_warnings:- # Waterfall success: we explain what happened in the logs- msg = (- f"Inference model fallback: '{ideal_model_handle}' was not found in the model deck, "- f"so it was replaced by '{fallback}'. "- f"As a consequence, the results of the method may not have the expected quality, "- f"and the method might fail due to feature limitations such as context window size, etc. "- f"Consider getting access to '{ideal_model_handle}'."- )- msg += f" Please see our docs for more details about setting up inference backends:\n{URLs.backend_provider_docs}"- log.info(msg)- # Mark this warning as logged for this waterfall_name- self._logged_fallback_warnings.add(waterfall_name)+ try:+ inference_model = self._get_optional_inference_model(+ model_handle=fallback,+ model_type=model_type,+ visited=step_visited,+ is_quiet=is_quiet,+ )+ except ModelWaterfallError:+ # The step is a waterfall of its own, or reaches one, none of whose steps is served: that step+ # serves no model, and this waterfall goes on. Only the waterfall resolved at the top raises its+ # own error. The refusal of a fallback while fallbacks are disabled is not caught: it holds here too.+ if not is_quiet:+ log.verbose(f"Waterfall '{waterfall_name}': step '{fallback}' reaches a waterfall none of whose models is served")+ inference_model = None+ if inference_model is not None:+ # Only log if we haven't logged for this waterfall_name before, and never from a quiet lookup,+ # which would otherwise use up the notice the next real fallback owes its run.+ if fallback_index > 0 and not is_quiet and waterfall_name not in self._logged_fallback_warnings:+ # Waterfall success: we explain what happened in the logs+ msg = (+ f"Inference model fallback: '{ideal_model_handle}' was not found in the model deck, "+ f"so it was replaced by '{fallback}'. "+ f"As a consequence, the results of the method may not have the expected quality, "+ f"and the method might fail due to feature limitations such as context window size, etc. "+ f"Consider getting access to '{ideal_model_handle}'."+ )+ msg += f" Please see our docs for more details about setting up inference backends:\n{URLs.backend_provider_docs}"+ log.info(msg)+ # Mark this warning as logged for this waterfall_name+ self._logged_fallback_warnings.add(waterfall_name)return inference_modelmsg = (f"Model handle '{waterfall_name}' is a waterfall (i.e. a list of models to try in order) "@@ -950,6 +1087,30 @@ class ModelDeck(ConfigModel):)raise ModelWaterfallError(message=msg, model_handle=waterfall_name, fallback_list=fallback_list)+ def _resolve_alias(+ self,+ *,+ alias_name: str,+ alias_target: str,+ model_type: ModelType,+ visited: frozenset[str],+ is_quiet: bool,+ ) -> InferenceModelSpec | None:+ """Resolve an alias through its target, ending as no model when the target leads back to an alias or a waterfall being resolved."""+ alias_key = f"{NAMESPACE_ALIAS}{alias_name}"+ if alias_key in visited:+ if not is_quiet:+ log.warning(f"Circular model reference detected: alias '{alias_name}' leads back to itself")+ return None+ if not is_quiet:+ log.verbose(f"Alias '{alias_name}' -> '{alias_target}'")+ return self._get_optional_inference_model(+ model_handle=alias_target,+ model_type=model_type,+ visited=visited | {alias_key},+ is_quiet=is_quiet,+ )+def get_optional_inference_model(self, model_handle: str, *, model_type: ModelType) -> InferenceModelSpec | None:"""Get an inference model spec, resolving aliases and waterfalls as needed.@@ -959,17 +1120,42 @@ class ModelDeck(ConfigModel):return self._get_optional_inference_model(model_handle=model_handle,model_type=model_type,- _visited=frozenset(),+ visited=frozenset(),+ is_quiet=False,)+ def peek_inference_model(self, *, model_handle: str, model_type: ModelType) -> InferenceModelSpec | None:+ """The model a run through `model_handle` would call now, or None when it would find none, looked up without side effects.++ It follows the lookup `get_optional_inference_model` makes, but logs nothing, leaves the one-time+ fallback notice to the next run that falls back, and answers None where that lookup raises for a+ waterfall none of whose usable steps the deck serves. The model reference check reads a run this way.+ """+ try:+ return self._get_optional_inference_model(+ model_handle=model_handle,+ model_type=model_type,+ visited=frozenset(),+ is_quiet=True,+ )+ except ModelNotFoundError:+ return None+def _get_optional_inference_model(self,model_handle: str,*,model_type: ModelType,- _visited: frozenset[str],+ visited: frozenset[str],+ is_quiet: bool,) -> InferenceModelSpec | None:- """Internal implementation with cycle detection for alias resolution."""+ """Internal implementation, with cycle detection across aliases and waterfalls.++ A reference is resolved by its kind: `@name` as the alias, `~name` as the waterfall, and a bare+ name as a model of this type, failing that as an alias of that name, failing that as a waterfall.+ `visited` holds the aliases and waterfalls the lookup is already inside, so a binding leading back+ to one of them ends as no model.+ """# Parse the model_handle to handle prefixed referencestry:ref = ModelReference.parse(model_handle)@@ -978,53 +1164,53 @@ class ModelDeck(ConfigModel):return Nonealiases, waterfalls = self.get_aliases_and_waterfalls_for_type(model_type)- # Handle prefixed alias reference (e.g., @best-gpt)match ref.kind:case ModelReferenceKind.ALIAS:if alias_target := aliases.get(ref.name):- log.verbose(f"Prefixed alias '{model_handle}' -> '{alias_target}'")- if alias_target in _visited:- log.error(f"Circular alias detected: '{model_handle}' -> '{alias_target}'")- return None- return self._get_optional_inference_model(- model_handle=alias_target,+ return self._resolve_alias(+ alias_name=ref.name,+ alias_target=alias_target,model_type=model_type,- _visited=_visited | {model_handle},+ visited=visited,+ is_quiet=is_quiet,)- log.verbose(f"Prefixed alias '{model_handle}' not found in aliases")+ if not is_quiet:+ log.verbose(f"Prefixed alias '{model_handle}' not found in aliases")return Nonecase ModelReferenceKind.WATERFALL:if fallback_list := waterfalls.get(ref.name):- log.verbose(f"Prefixed waterfall '{model_handle}' -> {fallback_list}")return self._resolve_waterfall(waterfall_name=ref.name,fallback_list=fallback_list,model_type=model_type,+ visited=visited,+ is_quiet=is_quiet,)- log.verbose(f"Prefixed waterfall '{model_handle}' not found in waterfalls")+ if not is_quiet:+ log.verbose(f"Prefixed waterfall '{model_handle}' not found in waterfalls")return Nonecase ModelReferenceKind.PRESET:# Presets should not be resolved here - they should be looked up in presets directly- log.verbose(f"Preset reference '{model_handle}' cannot be resolved as an inference model")+ if not is_quiet:+ log.verbose(f"Preset reference '{model_handle}' cannot be resolved as an inference model")return Nonecase ModelReferenceKind.HANDLE:# Direct handle - proceed with normal lookuppass- # For direct handles (HANDLE kind), try the models served as this type first: a handle served as- # another type only does not stop the lookup, since this type may have an alias or a waterfall of that name+ # For direct handles (HANDLE kind), try the model served as this type first. A model of that name served+ # only as another type does not answer this lookup, which goes on to this type's aliases and waterfalls,+ # as the load-time check (`is_bare_handle_resolvable`) reads the same name.if inference_model := self.inference_models.get(model_type=model_type, handle=ref.name):return inference_model# Then try aliases (without prefix)- if alias := aliases.get(ref.name):- log.verbose(f"Alias for '{model_handle}': {alias}")- if alias in _visited:- log.warning(f"Circular alias detected: '{model_handle}' -> '{alias}'")- return None- return self._get_optional_inference_model(- model_handle=alias,+ if alias_target := aliases.get(ref.name):+ return self._resolve_alias(+ alias_name=ref.name,+ alias_target=alias_target,model_type=model_type,- _visited=_visited | {model_handle},+ visited=visited,+ is_quiet=is_quiet,)# Finally try waterfalls (without prefix)if fallback_list := waterfalls.get(ref.name):@@ -1032,7 +1218,11 @@ class ModelDeck(ConfigModel):waterfall_name=ref.name,fallback_list=fallback_list,model_type=model_type,+ visited=visited,+ is_quiet=is_quiet,)+ if is_quiet:+ return Noneif served_types := self.inference_models.types_serving(handle=ref.name):served_types_description = ", ".join(f"'{served_type}'" for served_type in served_types)log.warning(f"Model handle '{ref.name}' is served as {served_types_description} but was requested as '{model_type}'. Skipping.")diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.pyindex cece1469a..94abaf8d7 100644--- a/pipelex/cogt/models/model_deck_check.py+++ b/pipelex/cogt/models/model_deck_check.py@@ -90,9 +90,9 @@ def check_llm_choice_with_deck(llm_choice: LLMModelChoice) -> None:available_options=list(model_deck.llm_waterfalls.keys()),)case ModelReferenceKind.HANDLE:- if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.LLM):+ if model_deck.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.LLM):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = model_deck.unresolved_handle_sentence(name=ref.name, model_type=ModelType.LLM)_raise_model_choice_not_found(msg,model_deck=model_deck,@@ -152,9 +152,9 @@ def check_extract_choice_with_deck(extract_choice: ExtractModelChoice) -> None:available_options=list(model_deck.extract_waterfalls.keys()),)case ModelReferenceKind.HANDLE:- if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.TEXT_EXTRACTOR):+ if model_deck.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.TEXT_EXTRACTOR):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = model_deck.unresolved_handle_sentence(name=ref.name, model_type=ModelType.TEXT_EXTRACTOR)_raise_model_choice_not_found(msg,model_deck=model_deck,@@ -214,9 +214,9 @@ def check_search_choice_with_deck(search_choice: SearchModelChoice) -> None:available_options=list(model_deck.search_waterfalls.keys()),)case ModelReferenceKind.HANDLE:- if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.SEARCH):+ if model_deck.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.SEARCH):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = model_deck.unresolved_handle_sentence(name=ref.name, model_type=ModelType.SEARCH)_raise_model_choice_not_found(msg,model_deck=model_deck,@@ -276,9 +276,9 @@ def check_doc_gen_choice_with_deck(*, doc_gen_choice: DocGenModelChoice) -> Noneavailable_options=list(model_deck.doc_gen_waterfalls.keys()),)case ModelReferenceKind.HANDLE:- if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.DOC_GEN):+ if model_deck.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.DOC_GEN):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = model_deck.unresolved_handle_sentence(name=ref.name, model_type=ModelType.DOC_GEN)_raise_model_choice_not_found(msg,model_deck=model_deck,@@ -338,9 +338,9 @@ def check_judgment_choice_with_deck(judgment_choice: JudgmentModelChoice) -> Nonavailable_options=list(model_deck.judgment_waterfalls.keys()),)case ModelReferenceKind.HANDLE:- if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.JUDGMENT):+ if model_deck.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.JUDGMENT):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = model_deck.unresolved_handle_sentence(name=ref.name, model_type=ModelType.JUDGMENT)_raise_model_choice_not_found(msg,model_deck=model_deck,@@ -400,9 +400,9 @@ def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice) -> None:available_options=list(model_deck.img_gen_waterfalls.keys()),)case ModelReferenceKind.HANDLE:- if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.IMG_GEN):+ if model_deck.is_bare_handle_resolvable(name=ref.name, model_type=ModelType.IMG_GEN):return- msg = f"Model handle '{ref.name}' was not found in the model deck"+ msg = model_deck.unresolved_handle_sentence(name=ref.name, model_type=ModelType.IMG_GEN)_raise_model_choice_not_found(msg,model_deck=model_deck,diff --git a/pipelex/cogt/models/model_listing.py b/pipelex/cogt/models/model_listing.pyindex 25a01cc92..f0c982422 100644--- a/pipelex/cogt/models/model_listing.py+++ b/pipelex/cogt/models/model_listing.py@@ -1,8 +1,9 @@"""The model listing: the deck's presets, aliases and waterfalls, by category.-Behind `pipelex-agent models`, `pipelex-agent check-model` and the runner's model listing.+Behind `pipelex-agent models` and the runner's model listing."""+from collections.abc import Mappingfrom enum import StrEnumfrom typing import Any@@ -44,14 +45,17 @@ def _resolve_preset_backend(model_handle: str,model_type: ModelType,) -> InferenceModelSpec | None:- """Resolve a preset's model handle to an InferenceModelSpec, returning None if unresolvable."""- return model_deck.get_optional_inference_model(model_handle=model_handle, model_type=model_type)+ """Resolve a preset's model handle to an InferenceModelSpec, returning None if unresolvable.++ A listing reads the deck without a run's side effects, and a waterfall that would raise in a run is unresolvable here.+ """+ return model_deck.peek_inference_model(model_handle=model_handle, model_type=model_type)def _filter_presets_by_backend(presets_list: list[dict[str, Any]],*,- presets_dict: dict[str, Any],+ presets_dict: Mapping[str, Any],model_deck: ModelDeck,model_type: ModelType,backend: str,@@ -79,7 +83,7 @@ def _filter_aliases_by_backend("""Filter aliases to only include those whose target resolves to the given backend."""filtered: dict[str, str] = {}for alias_name, alias_target in aliases.items():- spec = model_deck.get_optional_inference_model(model_handle=alias_target, model_type=model_type)+ spec = model_deck.peek_inference_model(model_handle=alias_target, model_type=model_type)if spec is not None and spec.backend_name == backend:filtered[alias_name] = alias_targetreturn filtered@@ -96,7 +100,7 @@ def _filter_waterfalls_by_backend(filtered: dict[str, list[str]] = {}for waterfall_name, fallback_list in waterfalls.items():for fallback in fallback_list:- spec = model_deck.get_optional_inference_model(model_handle=fallback, model_type=model_type)+ spec = model_deck.peek_inference_model(model_handle=fallback, model_type=model_type)if spec is not None and spec.backend_name == backend:filtered[waterfall_name] = fallback_listbreak@@ -110,19 +114,8 @@ def _build_presets_for_category(backend: str | None,) -> list[dict[str, Any]]:"""Build the presets list for a given category, optionally filtered by backend."""- presets_dict: dict[str, Any]model_type = CATEGORY_TO_MODEL_TYPE[category]- match category:- case ModelCategory.LLM:- presets_dict = model_deck.llm_presets- case ModelCategory.EXTRACT:- presets_dict = model_deck.extract_presets- case ModelCategory.IMG_GEN:- presets_dict = model_deck.img_gen_presets- case ModelCategory.SEARCH:- presets_dict = model_deck.search_presets- case ModelCategory.JUDGMENT:- presets_dict = model_deck.judgment_presets+ presets_dict = model_deck.get_presets_for_type(model_type=model_type)presets_list: list[dict[str, Any]] = []for preset_name, setting in presets_dict.items():@@ -147,18 +140,7 @@ def _build_aliases_for_category() -> dict[str, str]:"""Build the aliases dict for a given category, optionally filtered by backend."""model_type = CATEGORY_TO_MODEL_TYPE[category]- aliases: dict[str, str]- match category:- case ModelCategory.LLM:- aliases = model_deck.llm_aliases- case ModelCategory.EXTRACT:- aliases = model_deck.extract_aliases- case ModelCategory.IMG_GEN:- aliases = model_deck.img_gen_aliases- case ModelCategory.SEARCH:- aliases = model_deck.search_aliases- case ModelCategory.JUDGMENT:- aliases = model_deck.judgment_aliases+ aliases, _ = model_deck.get_aliases_and_waterfalls_for_type(model_type)if backend is not None:aliases = _filter_aliases_by_backend(aliases, model_deck=model_deck, model_type=model_type, backend=backend)@@ -174,18 +156,7 @@ def _build_waterfalls_for_category() -> dict[str, list[str]]:"""Build the waterfalls dict for a given category, optionally filtered by backend."""model_type = CATEGORY_TO_MODEL_TYPE[category]- waterfalls: dict[str, list[str]]- match category:- case ModelCategory.LLM:- waterfalls = model_deck.llm_waterfalls- case ModelCategory.EXTRACT:- waterfalls = model_deck.extract_waterfalls- case ModelCategory.IMG_GEN:- waterfalls = model_deck.img_gen_waterfalls- case ModelCategory.SEARCH:- waterfalls = model_deck.search_waterfalls- case ModelCategory.JUDGMENT:- waterfalls = model_deck.judgment_waterfalls+ _, waterfalls = model_deck.get_aliases_and_waterfalls_for_type(model_type)if backend is not None:waterfalls = _filter_waterfalls_by_backend(waterfalls, model_deck=model_deck, model_type=model_type, backend=backend)diff --git a/pipelex/cogt/models/model_reference.py b/pipelex/cogt/models/model_reference.pyindex ddb8aec27..7bbb4e45d 100644--- a/pipelex/cogt/models/model_reference.py+++ b/pipelex/cogt/models/model_reference.py@@ -162,6 +162,24 @@ class ModelReference(BaseModel):return self.raw+def write_model_handle(*, name: str) -> str:+ """Write a model handle's name as a reference that parses back to that same bare name.++ A name a bare reference reads as the same handle is written bare, as a method writes it, so+ `gpt-4o` stays `gpt-4o`. A name a bare reference would read as something else, such as `@named`+ (an alias) or `alias:x` (a namespaced alias), is written `handle:<name>`, the bare kind's own+ namespace, whose name is a bare name with no sigil read: a lookup that parses the written+ reference reads the bare name `@named`, and resolves it as it resolves any bare name.+ """+ try:+ reference = ModelReference.parse(name)+ except ModelReferenceParseError:+ return f"{NAMESPACE_HANDLE}{name}"+ if reference.kind == ModelReferenceKind.HANDLE and reference.name == name:+ return name+ return f"{NAMESPACE_HANDLE}{name}"++def ensure_model_reference(value: str | ModelReference) -> ModelReference:"""Ensure we have a ModelReference, parsing string if needed.diff --git a/pipelex/cogt/models/model_reference_check.py b/pipelex/cogt/models/model_reference_check.pynew file mode 100644index 000000000..80d495b2f--- /dev/null+++ b/pipelex/cogt/models/model_reference_check.py@@ -0,0 +1,285 @@+"""The model reference check: whether one model reference resolves on this runtime, as what, to which model, and what the caller meant if not.++The check answers from the parser and the deck lookups a validation runs (`ModelReference.parse` and+`ModelDeck.is_reference_defined`), so a reference it finds resolved in a category is one a pipe of that+category may name in its `model` field, and one it finds not found is one a validation refuses. Its+suggestions follow the rule a validation's refusal follows (`find_model_alternatives`). The Pipelex API+serves it as `GET /v1/models/check`.+"""++from enum import StrEnum++from pydantic import BaseModel, ConfigDict, Field++from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.exceptions import ModelReferenceParseError+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind, write_model_handle+from pipelex.cogt.models.model_suggestion import ModelReferenceName, find_model_alternatives+++class ModelCheckCategory(StrEnum):+ """The categories the model reference check covers, in the order every list it returns follows.++ These are the MTHDS Protocol's model categories, in the protocol's order, then `doc_gen`, the family+ of `PipeDocGen`. The protocol defines no category for `doc_gen`, so the model listing leaves it out,+ but a method names a `doc_gen` model in its `model` field like any other, so the check covers it.+ """++ LLM = "llm"+ EXTRACT = "extract"+ IMG_GEN = "img_gen"+ SEARCH = "search"+ JUDGMENT = "judgment"+ DOC_GEN = "doc_gen"++ @property+ def model_type(self) -> ModelType:+ """The model type of the deck family this category names."""+ match self:+ case ModelCheckCategory.LLM:+ return ModelType.LLM+ case ModelCheckCategory.EXTRACT:+ return ModelType.TEXT_EXTRACTOR+ case ModelCheckCategory.IMG_GEN:+ return ModelType.IMG_GEN+ case ModelCheckCategory.SEARCH:+ return ModelType.SEARCH+ case ModelCheckCategory.JUDGMENT:+ return ModelType.JUDGMENT+ case ModelCheckCategory.DOC_GEN:+ return ModelType.DOC_GEN+++class ModelReferenceResolution(StrEnum):+ """Whether a reference resolves in a category in scope. Both values are definitive."""++ RESOLVED = "resolved"+ NOT_FOUND = "not_found"+++_CATEGORY_DESCRIPTION = "The category this entry is about."+_RESOLVES_TO_DESCRIPTION = (+ "The model handle a run through the reference would call now in this category, "+ "or null when it would find none (a target on a backend the runner has not enabled, "+ "a waterfall none of whose usable steps the runner serves, an alias cycle)."+)+++class PresetMatch(BaseModel):+ """What a preset reference is in one category."""++ model_config = ConfigDict(extra="forbid", frozen=True)++ category: ModelCheckCategory = Field(description=_CATEGORY_DESCRIPTION)+ resolves_to: str | None = Field(description=_RESOLVES_TO_DESCRIPTION)+ target: str = Field(description="The model the deck binds the preset to, as the deck writes it, which may itself be a reference.")+ description: str | None = Field(description="The preset's description, or null when the deck gives it none.")+++class AliasMatch(BaseModel):+ """What an alias reference is in one category."""++ model_config = ConfigDict(extra="forbid", frozen=True)++ category: ModelCheckCategory = Field(description=_CATEGORY_DESCRIPTION)+ resolves_to: str | None = Field(description=_RESOLVES_TO_DESCRIPTION)+ target: str = Field(description="The model the deck binds the alias to, as the deck writes it, which may itself be a reference.")+++class WaterfallMatch(BaseModel):+ """What a waterfall reference is in one category."""++ model_config = ConfigDict(extra="forbid", frozen=True)++ category: ModelCheckCategory = Field(description=_CATEGORY_DESCRIPTION)+ resolves_to: str | None = Field(description=_RESOLVES_TO_DESCRIPTION)+ fallbacks: list[str] = Field(description="The waterfall's steps, in order.")+++class HandleMatch(BaseModel):+ """What a bare handle reference is in one category."""++ model_config = ConfigDict(extra="forbid", frozen=True)++ category: ModelCheckCategory = Field(description=_CATEGORY_DESCRIPTION)+ resolves_to: str | None = Field(description=_RESOLVES_TO_DESCRIPTION)+ via: list[str] = Field(+ description="The presets, aliases and waterfalls of this category whose binding names the handle directly, each written as a reference."+ )+++ModelReferenceMatch = PresetMatch | AliasMatch | WaterfallMatch | HandleMatch+++class ModelReferenceVerdict(BaseModel):+ """The verdict of the model reference check, discriminated on `resolution`."""++ model_config = ConfigDict(extra="forbid", frozen=True)++ reference: str = Field(description="The caller's reference, trimmed.")+ kind: ModelReferenceKind = Field(description="What the reference names, from its parsing.")+ name: str = Field(description="The reference without its sigil or namespace.")+ category: ModelCheckCategory | None = Field(description="The `type` asked, or null when none was.")+ resolution: ModelReferenceResolution = Field(description="Whether the reference resolves in a category in scope.")+ matches: list[ModelReferenceMatch] = Field(+ description="One entry per category in scope where the reference resolves, in the order of the categories the check covers."+ )+ suggestions: list[str] = Field(description="On `not_found`, the nearest names the caller may have meant; empty when it is `resolved`.")+ other_kinds: list[str] = Field(+ description="On `not_found`, the same name under another kind, in the categories in scope; empty when it is `resolved`."+ )+ other_categories: list[ModelCheckCategory] = Field(+ description="On `not_found` with a `type`, the categories outside it where the same reference resolves; empty otherwise."+ )+++def check_model_reference(*, model_deck: ModelDeck, reference: ModelReference, category: ModelCheckCategory | None) -> ModelReferenceVerdict:+ """Check whether a parsed model reference resolves on this runtime, in `category` or, when it is None, in every category the check covers.++ Args:+ model_deck: The deck the runtime booted with.+ reference: The reference, parsed as a method's `model` field is.+ category: The category to check in, or None to check in every category the check covers.++ Returns:+ The verdict: `resolved` with one match per category in scope where the reference resolves, or+ `not_found` with what the caller most likely meant.+ """+ categories_in_scope = [category] if category is not None else list(ModelCheckCategory)+ matches: list[ModelReferenceMatch] = []+ for scope_category in categories_in_scope:+ match_in_category = _match_in_category(model_deck=model_deck, reference=reference, category=scope_category)+ if match_in_category is not None:+ matches.append(match_in_category)++ if matches:+ return ModelReferenceVerdict(+ reference=reference.raw,+ kind=reference.kind,+ name=reference.name,+ category=category,+ resolution=ModelReferenceResolution.RESOLVED,+ matches=matches,+ suggestions=[],+ other_kinds=[],+ other_categories=[],+ )++ suggestions: list[str] = []+ other_kinds: list[str] = []+ for scope_category in categories_in_scope:+ alternatives = find_model_alternatives(+ model_deck=model_deck,+ model_type=scope_category.model_type,+ name=reference.name,+ kind=reference.kind,+ )+ suggestions.extend(_written(names=[*alternatives.same_kind, *alternatives.cross_kind]))+ other_kinds.extend(_written(names=alternatives.other_kinds))++ other_categories: list[ModelCheckCategory] = []+ if category is not None:+ other_categories = [+ other_category+ for other_category in ModelCheckCategory+ if other_category != category and model_deck.is_reference_defined(reference=reference, model_type=other_category.model_type)+ ]++ return ModelReferenceVerdict(+ reference=reference.raw,+ kind=reference.kind,+ name=reference.name,+ category=category,+ resolution=ModelReferenceResolution.NOT_FOUND,+ matches=[],+ suggestions=_first_occurrences(items=suggestions),+ other_kinds=_first_occurrences(items=other_kinds),+ other_categories=other_categories,+ )+++def _match_in_category(*, model_deck: ModelDeck, reference: ModelReference, category: ModelCheckCategory) -> ModelReferenceMatch | None:+ """What the reference is in one category, or None when it does not resolve there."""+ model_type = category.model_type+ if not model_deck.is_reference_defined(reference=reference, model_type=model_type):+ return None+ aliases, waterfalls = model_deck.get_aliases_and_waterfalls_for_type(model_type)+ match reference.kind:+ case ModelReferenceKind.PRESET:+ preset = model_deck.get_presets_for_type(model_type=model_type)[reference.name]+ return PresetMatch(+ category=category,+ resolves_to=_model_a_run_calls(model_deck=model_deck, model_handle=preset.model, model_type=model_type),+ target=preset.model,+ description=preset.description,+ )+ case ModelReferenceKind.ALIAS:+ alias_target = aliases[reference.name]+ return AliasMatch(+ category=category,+ resolves_to=_model_a_run_calls(model_deck=model_deck, model_handle=alias_target, model_type=model_type),+ target=alias_target,+ )+ case ModelReferenceKind.WATERFALL:+ waterfall = ModelReferenceName(kind=ModelReferenceKind.WATERFALL, name=reference.name)+ return WaterfallMatch(+ category=category,+ resolves_to=_model_a_run_calls(model_deck=model_deck, model_handle=waterfall.written, model_type=model_type),+ fallbacks=list(waterfalls[reference.name]),+ )+ case ModelReferenceKind.HANDLE:+ return HandleMatch(+ category=category,+ resolves_to=_model_a_run_calls(model_deck=model_deck, model_handle=write_model_handle(name=reference.name), model_type=model_type),+ via=_deck_names_binding_the_handle(model_deck=model_deck, handle=reference.name, model_type=model_type),+ )+++def _model_a_run_calls(*, model_deck: ModelDeck, model_handle: str, model_type: ModelType) -> str | None:+ """The handle of the model a run through `model_handle` would call now, or None when it would find none.++ The deck is read without side effects: a check logs nothing and leaves the fallback notice to the run.+ """+ inference_model = model_deck.peek_inference_model(model_handle=model_handle, model_type=model_type)+ if inference_model is None:+ return None+ return inference_model.name+++def _deck_names_binding_the_handle(*, model_deck: ModelDeck, handle: str, model_type: ModelType) -> list[str]:+ """The presets, aliases and waterfalls of this model type whose binding names `handle` directly, each written as a reference."""+ via: list[ModelReferenceName] = []+ for preset_name, preset in model_deck.get_presets_for_type(model_type=model_type).items():+ if _binding_names_handle(binding=preset.model, handle=handle):+ via.append(ModelReferenceName(kind=ModelReferenceKind.PRESET, name=preset_name))+ aliases, waterfalls = model_deck.get_aliases_and_waterfalls_for_type(model_type)+ for alias_name, alias_target in aliases.items():+ if _binding_names_handle(binding=alias_target, handle=handle):+ via.append(ModelReferenceName(kind=ModelReferenceKind.ALIAS, name=alias_name))+ for waterfall_name, fallbacks in waterfalls.items():+ if any(_binding_names_handle(binding=fallback, handle=handle) for fallback in fallbacks):+ via.append(ModelReferenceName(kind=ModelReferenceKind.WATERFALL, name=waterfall_name))+ return _written(names=via)+++def _binding_names_handle(*, binding: str, handle: str) -> bool:+ """Whether a deck binding (a preset's model, an alias's target, a waterfall's step) names `handle` as a bare handle, however spelled."""+ try:+ bound = ModelReference.parse(binding)+ except ModelReferenceParseError:+ return False+ match bound.kind:+ case ModelReferenceKind.HANDLE:+ return bound.name == handle+ case ModelReferenceKind.PRESET | ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL:+ return False+++def _written(*, names: list[ModelReferenceName]) -> list[str]:+ return [name.written for name in names]+++def _first_occurrences(*, items: list[str]) -> list[str]:+ """The items in order, each kept once, at its first place."""+ return list(dict.fromkeys(items))diff --git a/pipelex/cogt/models/model_suggestion.py b/pipelex/cogt/models/model_suggestion.pyindex fc5a95506..6554e6298 100644--- a/pipelex/cogt/models/model_suggestion.py+++ b/pipelex/cogt/models/model_suggestion.py@@ -1,10 +1,11 @@"""Fuzzy model name matching and cross-collection suggestion utilities.Provides functions to find close matches for model names and detect wrong-sigil usage.-Used by both the CLI check-model command and the model deck validation layer.+Used by the CLI check-model command, the model deck validation layer and the model reference check."""import difflib+from typing import NamedTuplefrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.model_deck import ModelDeck@@ -13,6 +14,7 @@ from pipelex.cogt.models.model_reference import (SIGIL_PRESET,SIGIL_WATERFALL,ModelReferenceKind,+ write_model_handle,)KIND_SIGILS: dict[ModelReferenceKind, str] = {@@ -36,58 +38,107 @@ def get_collection_keys(model_type: ModelType,kind: ModelReferenceKind,) -> list[str]:- """Return known names for a given reference kind and model type."""+ """Return known names for a given reference kind and model type, read through the deck's own per-type accessors."""match kind:case ModelReferenceKind.PRESET:- match model_type:- case ModelType.LLM:- return list(model_deck.llm_presets.keys())- case ModelType.TEXT_EXTRACTOR:- return list(model_deck.extract_presets.keys())- case ModelType.IMG_GEN:- return list(model_deck.img_gen_presets.keys())- case ModelType.SEARCH:- return list(model_deck.search_presets.keys())- case ModelType.DOC_GEN:- return list(model_deck.doc_gen_presets.keys())- case ModelType.JUDGMENT:- return list(model_deck.judgment_presets.keys())+ return list(model_deck.get_presets_for_type(model_type=model_type))case ModelReferenceKind.ALIAS:- match model_type:- case ModelType.LLM:- return list(model_deck.llm_aliases.keys())- case ModelType.TEXT_EXTRACTOR:- return list(model_deck.extract_aliases.keys())- case ModelType.IMG_GEN:- return list(model_deck.img_gen_aliases.keys())- case ModelType.SEARCH:- return list(model_deck.search_aliases.keys())- case ModelType.DOC_GEN:- return list(model_deck.doc_gen_aliases.keys())- case ModelType.JUDGMENT:- return list(model_deck.judgment_aliases.keys())+ aliases, _ = model_deck.get_aliases_and_waterfalls_for_type(model_type)+ return list(aliases)case ModelReferenceKind.WATERFALL:- match model_type:- case ModelType.LLM:- return list(model_deck.llm_waterfalls.keys())- case ModelType.TEXT_EXTRACTOR:- return list(model_deck.extract_waterfalls.keys())- case ModelType.IMG_GEN:- return list(model_deck.img_gen_waterfalls.keys())- case ModelType.SEARCH:- return list(model_deck.search_waterfalls.keys())- case ModelType.DOC_GEN:- return list(model_deck.doc_gen_waterfalls.keys())- case ModelType.JUDGMENT:- return list(model_deck.judgment_waterfalls.keys())+ _, waterfalls = model_deck.get_aliases_and_waterfalls_for_type(model_type)+ return list(waterfalls)case ModelReferenceKind.HANDLE:return model_deck.inference_models.handles_of_type(model_type=model_type)+class ModelReferenceName(NamedTuple):+ """A name the deck holds under one kind of model reference."""++ kind: ModelReferenceKind+ name: str++ @property+ def written(self) -> str:+ """The name as a method writes it: a sigil for a preset, an alias or a waterfall, and the bare name for a handle.++ A handle whose bare name would read as another kind of reference is written `handle:<name>`, which writes that+ bare name with no sigil read (`write_model_handle`).+ """+ match self.kind:+ case ModelReferenceKind.HANDLE:+ return write_model_handle(name=self.name)+ case ModelReferenceKind.PRESET | ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL:+ return f"{KIND_SIGILS[self.kind]}{self.name}"++ @property+ def labelled(self) -> str:+ """The written name followed by its kind in words, as a validation's report shows it (`@best-gpt (alias)`)."""+ return f"{self.written} ({KIND_LABELS[self.kind]})"+++class ModelAlternatives(NamedTuple):+ """What a caller who named a model the deck does not hold most likely meant, in one model type."""++ # The nearest names of the reference's own kind, nearest first, as many as `_SAME_KIND_MAX_MATCHES` allows.+ same_kind: list[ModelReferenceName]+ # The same name under each other kind the deck defines it as, which is the likeliest fault.+ other_kinds: list[ModelReferenceName]+ # For each other kind the name does not exist under exactly, the nearest names of that kind, as many as `_CROSS_KIND_MAX_MATCHES` allows.+ cross_kind: list[ModelReferenceName]+++# How near a name must be to be offered, as difflib's similarity ratio, and how many are offered: the+# threshold is stricter for another kind than for the reference's own, since a slip of the name alone is+# likelier than a slip of both the name and the sigil.+_SAME_KIND_CUTOFF = 0.5+_SAME_KIND_MAX_MATCHES = 5+_CROSS_KIND_CUTOFF = 0.7+_CROSS_KIND_MAX_MATCHES = 3+++def find_model_alternatives(*, model_deck: ModelDeck, model_type: ModelType, name: str, kind: ModelReferenceKind) -> ModelAlternatives:+ """Find the names a caller most likely meant by a model name the deck does not hold as `kind` for `model_type`.++ This is the rule a validation follows when it refuses an unknown model, and the rule the model+ reference check follows when it answers that a reference resolves nowhere.++ Args:+ model_deck: The model deck to search in.+ model_type: The model type the name was asked for.+ name: The bare name (without sigil) to match.+ kind: The reference kind the caller specified.++ Returns:+ The close matches of the same kind, the exact matches under other kinds, and the close matches under other kinds.+ """+ candidates = get_collection_keys(model_deck, model_type=model_type, kind=kind)+ same_kind = [+ ModelReferenceName(kind=kind, name=match)+ for match in difflib.get_close_matches(name, candidates, n=_SAME_KIND_MAX_MATCHES, cutoff=_SAME_KIND_CUTOFF)+ ]++ other_kinds: list[ModelReferenceName] = []+ cross_kind: list[ModelReferenceName] = []+ for other_kind in ModelReferenceKind:+ if other_kind == kind:+ continue+ other_candidates = get_collection_keys(model_deck, model_type=model_type, kind=other_kind)+ if name in other_candidates:+ other_kinds.append(ModelReferenceName(kind=other_kind, name=name))+ else:+ cross_kind.extend(+ ModelReferenceName(kind=other_kind, name=match)+ for match in difflib.get_close_matches(name, other_candidates, n=_CROSS_KIND_MAX_MATCHES, cutoff=_CROSS_KIND_CUTOFF)+ )++ return ModelAlternatives(same_kind=same_kind, other_kinds=other_kinds, cross_kind=cross_kind)++def suggest_model_alternatives(*, model_deck: ModelDeck, model_type: ModelType, name: str, kind: ModelReferenceKind) -> tuple[list[str], list[str], list[str]]:- """Find fuzzy matches and detect wrong-sigil usage for a model name.+ """Find fuzzy matches and detect wrong-sigil usage for a model name, phrased as a validation's report shows them.Args:model_deck: The model deck to search in.@@ -101,23 +152,8 @@ def suggest_model_alternatives(- wrong_sigil_hints: exact matches found in other collections (e.g. "best-gpt exists as @best-gpt (alias)")- cross_collection_suggestions: fuzzy matches in other collections, with sigil and label"""- sigil = KIND_SIGILS[kind]- candidates = get_collection_keys(model_deck, model_type=model_type, kind=kind)- fuzzy_matches = difflib.get_close_matches(name, candidates, n=5, cutoff=0.5)- suggestions = [f"{sigil}{match}" for match in fuzzy_matches]-- wrong_sigil_hints: list[str] = []- cross_suggestions: list[str] = []- other_kinds = [other_kind for other_kind in ModelReferenceKind if other_kind != kind]- for other_kind in other_kinds:- other_candidates = get_collection_keys(model_deck, model_type=model_type, kind=other_kind)- other_sigil = KIND_SIGILS[other_kind]- label = KIND_LABELS[other_kind]- if name in other_candidates:- wrong_sigil_hints.append(f"{name} exists as {other_sigil}{name} ({label})")- else:- fuzzy = difflib.get_close_matches(name, other_candidates, n=3, cutoff=0.7)- for match in fuzzy:- cross_suggestions.append(f"{other_sigil}{match} ({label})")-+ alternatives = find_model_alternatives(model_deck=model_deck, model_type=model_type, name=name, kind=kind)+ suggestions = [match.written for match in alternatives.same_kind]+ wrong_sigil_hints = [f"{name} exists as {match.labelled}" for match in alternatives.other_kinds]+ cross_suggestions = [match.labelled for match in alternatives.cross_kind]return suggestions, wrong_sigil_hints, cross_suggestionsdiff --git a/pipelex/core/pipes/exceptions.py b/pipelex/core/pipes/exceptions.pyindex 40207ca7b..ddb1e0db3 100644--- a/pipelex/core/pipes/exceptions.py+++ b/pipelex/core/pipes/exceptions.py@@ -57,7 +57,8 @@ class PipeRunError(PipelexError):class PipeOperatorModelChoiceError(PipelexError):"""Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``, ``PipeJudge``)- when it is built from its blueprint and a model field names a model the model deck does not define.+ when it is built from its blueprint and a model field names a model the model deck does not define, or a+ bare handle the deck serves only as another model type (an LLM named in a ``PipeImgGen``, for instance).Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``,and a run, which loads its bundle before any pipe runs, refuses the bundle with that same verdict.diff --git a/pipelex/kernel/judgment_ops.py b/pipelex/kernel/judgment_ops.pyindex fe66d7d00..49ce7414e 100644--- a/pipelex/kernel/judgment_ops.py+++ b/pipelex/kernel/judgment_ops.py@@ -32,6 +32,7 @@ from pipelex.cogt.judgment.judgment_models import (from pipelex.cogt.judgment.judgment_setting import JudgmentModelChoice, JudgmentSettingfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_reference import write_model_handlefrom pipelex.cogt.templating.template_rendering import render_templatefrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.memory.working_memory import WorkingMemory@@ -104,8 +105,10 @@ def resolve_judgment_setting(return judgment_settinginference_model = model_deck.get_required_inference_model(model_handle=judgment_setting.model, model_type=ModelType.JUDGMENT)- if inference_model.name != judgment_setting.model:- judgment_setting = judgment_setting.model_copy(update={"model": inference_model.name})+ # Pinned as a reference that parses back to the resolved handle, whatever its spelling, since every lookup reads it again.+ resolved_handle = write_model_handle(name=inference_model.name)+ if resolved_handle != judgment_setting.model:+ judgment_setting = judgment_setting.model_copy(update={"model": resolved_handle})return judgment_settingdiff --git a/pipelex/kernel/search_ops.py b/pipelex/kernel/search_ops.pyindex f9c1dc09d..b7de7ff62 100644--- a/pipelex/kernel/search_ops.py+++ b/pipelex/kernel/search_ops.py@@ -13,6 +13,7 @@ library's business, so the kernel never asks for one.from pipelex.cogt.content_generation.assignment_models import SearchAssignmentfrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParamsfrom pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_reference import write_model_handlefrom pipelex.cogt.search.search_setting import SearchModelChoice, SearchSettingfrom pipelex.cogt.templating.template_rendering import render_templatefrom pipelex.core.concepts.concept import Concept@@ -43,8 +44,10 @@ def resolve_search_setting(search_setting = model_deck.get_search_setting(search_choice=resolved_choice)inference_model = model_deck.get_required_inference_model(model_handle=search_setting.model, model_type=ModelType.SEARCH)- if inference_model.name != search_setting.model:- search_setting = search_setting.model_copy(update={"model": inference_model.name})+ # Pinned as a reference that parses back to the resolved handle, whatever its spelling, since every lookup reads it again.+ resolved_handle = write_model_handle(name=inference_model.name)+ if resolved_handle != search_setting.model:+ search_setting = search_setting.model_copy(update={"model": resolved_handle})if include_images_override is not None:search_setting = search_setting.model_copy(update={"include_images": include_images_override})diff --git a/tests/integration/pipelex/pipeline/test_file_input_consumers.py b/tests/integration/pipelex/pipeline/test_file_input_consumers.pyindex b8ac60bc4..06b92ad89 100644--- a/tests/integration/pipelex/pipeline/test_file_input_consumers.py+++ b/tests/integration/pipelex/pipeline/test_file_input_consumers.py@@ -496,5 +496,6 @@ model = "~fic-mixed-extractors"consumers = _consumers(load_empty_library=load_empty_library, domain="fic_waterfall", pipes=pipes)(consumer,) = consumers["transcript"]- assert consumer.model == "fic-mixed-extractors"+ # Its members are different models, so the consumer names the waterfall as the pipe writes it, sigil included.+ assert consumer.model == "~fic-mixed-extractors"assert consumer.readable_formats == DOCLING_FORMATSdiff --git a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.pyindex e170b1e65..adfc541b2 100644--- a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py+++ b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py@@ -11,6 +11,8 @@ bundles through ``validate_bundle`` and pin the verdict each refusal produces:when the deck offers exactly one suggestion. Every pipe type that names a model gives the same item(``PipeExtract`` and ``PipeSearch`` used to turn it into an ``unknown_validation_error`` whose messagewas a Python repr).+- **A model served as another type.** A pipe naming a handle the deck serves only as another model type, such as an+ LLM in an image-generation pipe, gets the same ``unknown_model`` item, whose message names the type the pipe needs.- **The general arm.** Any other ``input``-domained refusal raised while building a pipe validates to oneitem located on the pipe and its file, keeping its message only when it is caller-facing; a``config``-domained fault raised at the same place still propagates as no verdict.@@ -28,12 +30,14 @@ import pytestfrom pytest_mock import MockerFixturefrom pipelex.base_exceptions import DisclosureMode, ErrorDomain, PipelexError, ValidationErrorCategory, ValidationErrorItem+from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.config import get_configfrom pipelex.core.pipes.exceptions import PipeOperatorModelChoiceErrorfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.fixes.planner import RENAME_MODEL_FIX_CODEfrom pipelex.pipeline.pipeline_run_setup import pipeline_run_setupfrom pipelex.pipeline.validate_bundle import validate_bundle+from pipelex.runtime_hub import get_model_deckfrom pipelex.suggested_fix import RemapValueOpfrom pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.validation_error_types import PipeValidationErrorType@@ -70,6 +74,22 @@ model = "nano-banana-9"prompt = "A painted wooden harbour board announcing the tide times, morning light""""+# An LLM the session deck serves, named in an image-generation pipe.+_LLM_HANDLE_SERVED = "gpt-5.6-luna"++_LLM_IN_IMG_GEN_BUNDLE = f"""+domain = "{_DOMAIN}"+description = "Draw the board announcing the tide times"+main_pipe = "draw_tide_board"++[pipe.draw_tide_board]+type = "PipeImgGen"+description = "Draw the painted board announcing the tide times"+output = "Image"+model = "{_LLM_HANDLE_SERVED}"+prompt = "A painted wooden harbour board announcing the tide times, morning light"+"""+_EXTRACT_BUNDLE = f"""domain = "{_DOMAIN}"description = "Read the pages of a printed tide almanac"@@ -257,6 +277,28 @@ class TestValidateBundleLoadRefusals:for suggestion in item.suggestions:assert suggestion in item.message+ async def test_a_model_served_as_another_type_is_one_unknown_model_item_naming_the_type_needed(self, tmp_path: Path) -> None:+ """An LLM the deck serves, named in an image-generation pipe, is refused when the bundle loads rather than when the run reaches it."""+ served_types = get_model_deck().inference_models.types_serving(handle=_LLM_HANDLE_SERVED)+ assert served_types == [ModelType.LLM], f"the session deck must serve {_LLM_HANDLE_SERVED!r} as an LLM alone: pick an LLM it serves so"+ bundle_path = _write_bundle(directory=tmp_path, content=_LLM_IN_IMG_GEN_BUNDLE)++ item = await _single_item(bundle_path=bundle_path)++ assert item.category == ValidationErrorCategory.PIPE_VALIDATION+ assert item.error_type == PipeValidationErrorType.UNKNOWN_MODEL+ assert item.pipe_code == "draw_tide_board"+ assert item.domain_code == _DOMAIN+ assert item.source == str(bundle_path)+ assert item.field_path == "pipe.draw_tide_board.model"+ assert item.model_reference == _LLM_HANDLE_SERVED+ assert item.model_type == "img_gen"+ assert item.message.startswith(+ f"Pipe 'draw_tide_board' (PipeImgGen), field 'model': "+ f"Model handle '{_LLM_HANDLE_SERVED}' is served by the model deck, but not as an image-generation model"+ )+ assert "was not found" not in item.message+async def test_one_suggestion_carries_a_rename_fix(self, tmp_path: Path) -> None:bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "@best-sonet"'))diff --git a/tests/unit/pipelex/cli/test_agent_models_cmd.py b/tests/unit/pipelex/cli/test_agent_models_cmd.pyindex a777ff49e..34d7b44d1 100644--- a/tests/unit/pipelex/cli/test_agent_models_cmd.py+++ b/tests/unit/pipelex/cli/test_agent_models_cmd.py@@ -3,7 +3,7 @@from __future__ import annotationsimport json-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, ClassVarif TYPE_CHECKING:import pytest@@ -11,41 +11,36 @@ if TYPE_CHECKING:from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormatfrom pipelex.cli.agent_cli.commands.models_cmd import agent_models_cmd+from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.extract.extract_setting import ExtractSetting+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.img_gen.img_gen_setting import ImgGenSetting+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndexfrom pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeckfrom pipelex.cogt.models.model_listing import ModelCategory+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.runtime import ProblemReactionCMD_MODULE_PATH = "pipelex.cli.agent_cli.commands.models_cmd"OPS_MODULE_PATH = "pipelex.cogt.models.model_listing"-class _FakeSetting:- """Minimal stand-in for LLMSetting / ExtractSetting / ImgGenSetting."""-- def __init__(self, model: str, description: str | None = None):- self.model = model- self.description = description---class _FakeInferenceModelSpec:- """Minimal stand-in for InferenceModelSpec."""-- def __init__(self, backend_name: str, model_type: ModelType):- self.backend_name = backend_name- self.model_type = model_type--class TestData:"""Shared test data constants."""- LLM_PRESETS: ClassVar[dict[str, _FakeSetting]] = {- "fast": _FakeSetting(model="gpt-4o-mini", description="Fast LLM"),- "smart": _FakeSetting(model="claude-sonnet", description="Smart LLM"),+ LLM_PRESETS: ClassVar[dict[str, LLMSetting]] = {+ "fast": LLMSetting(model="gpt-4o-mini", temperature=0.5, description="Fast LLM"),+ "smart": LLMSetting(model="claude-sonnet", temperature=0.5, description="Smart LLM"),}- EXTRACT_PRESETS: ClassVar[dict[str, _FakeSetting]] = {- "doc-extract": _FakeSetting(model="textract-model", description="Doc extraction"),+ EXTRACT_PRESETS: ClassVar[dict[str, ExtractSetting]] = {+ "doc-extract": ExtractSetting(model="textract-model", description="Doc extraction"),}- IMG_GEN_PRESETS: ClassVar[dict[str, _FakeSetting]] = {- "hd-image": _FakeSetting(model="dall-e-3", description="HD images"),+ IMG_GEN_PRESETS: ClassVar[dict[str, ImgGenSetting]] = {+ "hd-image": ImgGenSetting(model="dall-e-3", description="HD images"),}LLM_ALIASES: ClassVar[dict[str, str]] = {"best-llm": "claude-sonnet"}@@ -56,6 +51,7 @@ class TestData:EXTRACT_WATERFALLS: ClassVar[dict[str, list[str]]] = {"extract-wf": ["textract-model"]}IMG_GEN_WATERFALLS: ClassVar[dict[str, list[str]]] = {"img-wf": ["dall-e-3"]}+ # The models the deck serves: handle -> (backend, model type).INFERENCE_MAP: ClassVar[dict[str, tuple[str, ModelType]]] = {"gpt-4o-mini": ("openai", ModelType.LLM),"claude-sonnet": ("anthropic", ModelType.LLM),@@ -64,44 +60,60 @@ class TestData:}-def _make_fake_model_deck() -> Any:- """Create a fake ModelDeck with all test data."""-- class FakeModelDeck:- llm_presets = TestData.LLM_PRESETS- extract_presets = TestData.EXTRACT_PRESETS- img_gen_presets = TestData.IMG_GEN_PRESETS- search_presets: ClassVar[dict[str, Any]] = {}- judgment_presets: ClassVar[dict[str, Any]] = {}-- llm_aliases = TestData.LLM_ALIASES- extract_aliases = TestData.EXTRACT_ALIASES- img_gen_aliases = TestData.IMG_GEN_ALIASES- search_aliases: ClassVar[dict[str, str]] = {}- judgment_aliases: ClassVar[dict[str, str]] = {}-- llm_waterfalls = TestData.LLM_WATERFALLS- extract_waterfalls = TestData.EXTRACT_WATERFALLS- img_gen_waterfalls = TestData.IMG_GEN_WATERFALLS- search_waterfalls: ClassVar[dict[str, list[str]]] = {}- judgment_waterfalls: ClassVar[dict[str, list[str]]] = {}-- def get_optional_inference_model(self, model_handle: str, model_type: ModelType) -> _FakeInferenceModelSpec | None:- entry = TestData.INFERENCE_MAP.get(model_handle)- if entry is None:- return None- backend_name, spec_model_type = entry- if spec_model_type != model_type:- return None- return _FakeInferenceModelSpec(backend_name=backend_name, model_type=spec_model_type)-- return FakeModelDeck()---def _setup_mocks(mocker: MockerFixture) -> None:+def _model_spec(*, name: str, backend_name: str, model_type: ModelType) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name=backend_name,+ name=name,+ sdk="test_sdk",+ model_type=model_type,+ model_id=name,+ costs={CostCategory.INPUT: 0.001, CostCategory.OUTPUT: 0.002},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=1000,+ max_prompt_images=None,+ )+++def _make_model_deck(+ *,+ llm_aliases: dict[str, str] | None = None,+ llm_waterfalls: dict[str, list[str]] | None = None,+) -> ModelDeck:+ """A real model deck serving the test data's models, with its presets, aliases and waterfalls."""+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec(name=name, backend_name=backend_name, model_type=model_type)+ for name, (backend_name, model_type) in TestData.INFERENCE_MAP.items()+ ]+ ),+ llm_default_temperature=0.7,+ llm_presets=TestData.LLM_PRESETS,+ llm_aliases=TestData.LLM_ALIASES if llm_aliases is None else llm_aliases,+ llm_waterfalls=TestData.LLM_WATERFALLS if llm_waterfalls is None else llm_waterfalls,+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_presets=TestData.EXTRACT_PRESETS,+ extract_aliases=TestData.EXTRACT_ALIASES,+ extract_waterfalls=TestData.EXTRACT_WATERFALLS,+ extract_choice_default="textract-model",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_presets=TestData.IMG_GEN_PRESETS,+ img_gen_aliases=TestData.IMG_GEN_ALIASES,+ img_gen_waterfalls=TestData.IMG_GEN_WATERFALLS,+ img_gen_choice_default="dall-e-3",+ search_choice_default="@default-search",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=True, missing_presets_reaction=ProblemReaction.NONE),+ )+++def _setup_mocks(mocker: MockerFixture, *, model_deck: ModelDeck | None = None) -> None:"""Patch the common dependencies for agent_models_cmd."""mocker.patch(f"{CMD_MODULE_PATH}.make_pipelex_for_agent_cli")- mocker.patch(f"{OPS_MODULE_PATH}.get_model_deck", return_value=_make_fake_model_deck())+ mocker.patch(f"{OPS_MODULE_PATH}.get_model_deck", return_value=model_deck or _make_model_deck())mocker.patch(f"{CMD_MODULE_PATH}.Pipelex")@@ -255,3 +267,17 @@ class TestAgentModelsCmd:assert "llm-wf" in parsed["waterfalls"]["llm"]assert len(parsed["waterfalls"]["img_gen"]) == 0assert len(parsed["waterfalls"]["extract"]) == 0++ def test_backend_filter_skips_an_alias_whose_waterfall_no_backend_serves(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """An alias bound to a waterfall none of whose models is served resolves on no backend, rather than failing the listing."""+ model_deck = _make_model_deck(+ llm_aliases={"robust": "~unserved", "best-gpt": "gpt-4o-mini"},+ llm_waterfalls={"unserved": ["gpt-9", "gpt-10"]},+ )+ _setup_mocks(mocker, model_deck=model_deck)++ agent_models_cmd(backend="openai", output_format=CliOutputFormat.JSON)++ parsed = json.loads(capsys.readouterr().out)+ assert parsed["success"] is True+ assert parsed["aliases"]["llm"] == {"best-gpt": "gpt-4o-mini"}diff --git a/tests/unit/pipelex/cli/test_check_model_cmd.py b/tests/unit/pipelex/cli/test_check_model_cmd.pyindex 156a65401..39eb92146 100644--- a/tests/unit/pipelex/cli/test_check_model_cmd.py+++ b/tests/unit/pipelex/cli/test_check_model_cmd.py@@ -1,104 +1,91 @@-"""Unit tests for the agent CLI check-model command."""+"""Unit tests for the agent CLI check-model command, against a real model deck."""from __future__ import annotationsimport json-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, Any++import pytestif TYPE_CHECKING:- import pytestfrom pytest_mock import MockerFixturefrom pipelex.cli.agent_cli.commands.agent_output import CliOutputFormatfrom pipelex.cli.agent_cli.commands.check_model_cmd import agent_check_model_cmd+from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaultsfrom pipelex.cogt.llm.thinking_mode import ThinkingModefrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_spec_index import ModelSpecIndexfrom pipelex.cogt.model_backends.model_type import ModelType-from pipelex.cogt.models.model_listing import ModelCategory+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.models.model_reference import ModelReference+from pipelex.cogt.models.model_reference_check import ModelCheckCategory+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.runtime import ProblemReactionMODULE_PATH = "pipelex.cli.agent_cli.commands.check_model_cmd"-class _FakeSetting:- """Minimal stand-in for LLMSetting / ExtractSetting."""-- def __init__(self, model: str, description: str | None = None):- self.model = model- self.description = description---def _llm_spec(name: str) -> InferenceModelSpec:+def _model_spec(name: str, model_type: ModelType) -> InferenceModelSpec:return InferenceModelSpec(backend_name="test_backend",name=name,sdk="test_sdk",- model_type=ModelType.LLM,- model_id=name,- costs={},+ model_type=model_type,+ model_id=f"test_model_{name}",+ costs={CostCategory.INPUT: 0.001, CostCategory.OUTPUT: 0.002},thinking_mode=ThinkingMode.NONE,- max_tokens=None,+ max_tokens=1000,max_prompt_images=None,)-class TestData:- """Shared test data constants."""-- LLM_PRESETS: ClassVar[dict[str, _FakeSetting]] = {- "writing-creative": _FakeSetting(model="claude-sonnet", description="Creative writing"),- "writing-factual": _FakeSetting(model="claude-sonnet", description="Factual writing"),- "deep-analysis": _FakeSetting(model="claude-opus", description="Deep analysis"),- }- LLM_ALIASES: ClassVar[dict[str, str]] = {- "best-claude": "claude-sonnet",- "best-gpt": "gpt-4o",- "default-general": "claude-sonnet",- }- LLM_WATERFALLS: ClassVar[dict[str, list[str]]] = {- "robust-llm": ["claude-sonnet", "gpt-4o"],- }- INFERENCE_MODELS: ClassVar[ModelSpecIndex] = ModelSpecIndex.make_from_specs(- model_specs=[_llm_spec(name) for name in ("claude-4.5-sonnet", "claude-4-sonnet", "claude-4.6-opus", "gpt-4o", "gpt-4o-mini")]+def _make_model_deck(*, is_model_fallback_enabled: bool = True) -> ModelDeck:+ """A deck with LLM presets, aliases and a waterfall, one image-generation model and one document-generation engine."""+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec("claude-4.5-sonnet", ModelType.LLM),+ _model_spec("claude-4-sonnet", ModelType.LLM),+ _model_spec("claude-4.6-opus", ModelType.LLM),+ _model_spec("gpt-4o", ModelType.LLM),+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("img-painter", ModelType.IMG_GEN),+ _model_spec("reportlab-pdf", ModelType.DOC_GEN),+ ]+ ),+ llm_default_temperature=0.7,+ llm_presets={+ "writing-creative": LLMSetting(model="claude-4.5-sonnet", temperature=0.9, description="Creative writing"),+ "writing-factual": LLMSetting(model="claude-4.5-sonnet", temperature=0.1, description="Factual writing"),+ "deep-analysis": LLMSetting(model="claude-4.6-opus", temperature=0.2, description="Deep analysis"),+ },+ llm_aliases={+ "best-claude": "claude-4.5-sonnet",+ "best-gpt": "gpt-4o",+ "default-general": "claude-4.5-sonnet",+ },+ llm_waterfalls={"robust-llm": ["claude-4.5-sonnet", "gpt-4o"]},+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="claude-4.5-sonnet", temperature=0.7),+ for_object=LLMSetting(model="claude-4.5-sonnet", temperature=0.1),+ ),+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_choice_default="img-painter",+ search_choice_default="web-searcher",+ doc_gen_aliases={"default-pdf": "reportlab-pdf"},+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),)- EXTRACT_PRESETS: ClassVar[dict[str, Any]] = {}- EXTRACT_ALIASES: ClassVar[dict[str, str]] = {}- EXTRACT_WATERFALLS: ClassVar[dict[str, list[str]]] = {}- IMG_GEN_PRESETS: ClassVar[dict[str, Any]] = {}- IMG_GEN_ALIASES: ClassVar[dict[str, str]] = {}- IMG_GEN_WATERFALLS: ClassVar[dict[str, list[str]]] = {}- SEARCH_PRESETS: ClassVar[dict[str, Any]] = {}- SEARCH_ALIASES: ClassVar[dict[str, str]] = {}- SEARCH_WATERFALLS: ClassVar[dict[str, list[str]]] = {}---def _make_fake_model_deck() -> Any:- """Create a fake ModelDeck with LLM test data."""-- class FakeModelDeck:- llm_presets = TestData.LLM_PRESETS- llm_aliases = TestData.LLM_ALIASES- llm_waterfalls = TestData.LLM_WATERFALLS- inference_models = TestData.INFERENCE_MODELS-- extract_presets = TestData.EXTRACT_PRESETS- extract_aliases = TestData.EXTRACT_ALIASES- extract_waterfalls = TestData.EXTRACT_WATERFALLS- img_gen_presets = TestData.IMG_GEN_PRESETS- img_gen_aliases = TestData.IMG_GEN_ALIASES- img_gen_waterfalls = TestData.IMG_GEN_WATERFALLS- search_presets = TestData.SEARCH_PRESETS- search_aliases = TestData.SEARCH_ALIASES- search_waterfalls = TestData.SEARCH_WATERFALLS-- return FakeModelDeck()---def _setup_mocks(mocker: MockerFixture) -> None:- """Patch the common dependencies for agent_check_model_cmd."""++def _setup_mocks(mocker: MockerFixture, *, model_deck: ModelDeck) -> None:+ """Patch the boot and the deck accessor of agent_check_model_cmd."""mocker.patch(f"{MODULE_PATH}.make_pipelex_for_agent_cli")- mocker.patch(f"{MODULE_PATH}.get_model_deck", return_value=_make_fake_model_deck())+ mocker.patch(f"{MODULE_PATH}.get_model_deck", return_value=model_deck)mocker.patch(f"{MODULE_PATH}.Pipelex")@@ -106,12 +93,13 @@ def _run_check(mocker: MockerFixture,capsys: pytest.CaptureFixture[str],name: str,- model_type: ModelCategory = ModelCategory.LLM,- output_format: CliOutputFormat = CliOutputFormat.JSON,+ model_type: ModelCheckCategory = ModelCheckCategory.LLM,+ *,+ model_deck: ModelDeck | None = None,) -> dict[str, Any]:- """Run check-model and return parsed JSON output."""- _setup_mocks(mocker)- agent_check_model_cmd(name=name, model_type=model_type, output_format=output_format)+ """Run check-model and return its parsed JSON output."""+ _setup_mocks(mocker, model_deck=model_deck or _make_model_deck())+ agent_check_model_cmd(name=name, model_type=model_type, output_format=CliOutputFormat.JSON)parsed: dict[str, Any] = json.loads(capsys.readouterr().out)return parsed@@ -137,6 +125,98 @@ class TestCheckModelCmd:assert result["valid"] is Trueassert result["kind"] == "handle"+ def test_valid_waterfall(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A known waterfall name with ~ sigil should be valid."""+ result = _run_check(mocker, capsys, "~robust-llm")+ assert result["valid"] is True+ assert result["kind"] == "waterfall"++ def test_bare_alias_name_is_a_valid_handle(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A bare name the type's aliases hold is a handle a pipe resolves, so it is valid, as a validation accepts it."""+ result = _run_check(mocker, capsys, "best-claude")+ assert result["valid"] is True+ assert result["kind"] == "handle"++ @pytest.mark.parametrize(+ ("is_model_fallback_enabled", "expected_valid"),+ [+ (True, True),+ (False, False),+ ],+ )+ def test_bare_waterfall_name_is_valid_while_fallback_is_on(+ self,+ mocker: MockerFixture,+ capsys: pytest.CaptureFixture[str],+ is_model_fallback_enabled: bool,+ expected_valid: bool,+ ) -> None:+ """A bare name the type's waterfalls hold is a handle while model fallback is on."""+ model_deck = _make_model_deck(is_model_fallback_enabled=is_model_fallback_enabled)+ result = _run_check(mocker, capsys, "robust-llm", model_deck=model_deck)+ assert result["valid"] is expected_valid++ @pytest.mark.parametrize(+ ("name", "expected_kind"),+ [+ ("@default-pdf", "alias"),+ ("reportlab-pdf", "handle"),+ ("default-pdf", "handle"),+ ],+ )+ def test_a_doc_gen_reference_is_checked_in_its_category(+ self,+ mocker: MockerFixture,+ capsys: pytest.CaptureFixture[str],+ name: str,+ expected_kind: str,+ ) -> None:+ """`doc_gen` is a category of the check, as `GET /v1/models/check` answers it, though the model listing leaves it out."""+ result = _run_check(mocker, capsys, name, ModelCheckCategory.DOC_GEN)+ assert result["valid"] is True+ assert result["kind"] == expected_kind+ assert result["model_type"] == "doc_gen"++ def test_handle_served_as_another_type_is_not_valid(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A model the deck serves as an LLM is not a valid image-generation handle."""+ result = _run_check(mocker, capsys, "gpt-4o", ModelCheckCategory.IMG_GEN)+ assert result["valid"] is False+ assert result["model_type"] == "img_gen"++ @pytest.mark.parametrize(+ "name",+ [+ "$writing-creative",+ "@best-gpt",+ "~robust-llm",+ "claude-4.5-sonnet",+ "img-painter",+ "best-claude",+ "robust-llm",+ "writing-creative",+ "alias:best-gpt",+ "handle:gpt-4o",+ "handle:@best-gpt",+ "$best-claude",+ "@held-nowhere",+ "@default-pdf",+ "reportlab-pdf",+ ],+ )+ @pytest.mark.parametrize("model_type", list(ModelCheckCategory))+ def test_verdict_agrees_with_the_deck(+ self,+ mocker: MockerFixture,+ capsys: pytest.CaptureFixture[str],+ name: str,+ model_type: ModelCheckCategory,+ ) -> None:+ """The command answers as `ModelDeck.is_reference_defined`, the rule a validation and the reference check apply."""+ model_deck = _make_model_deck()+ result = _run_check(mocker, capsys, name, model_type, model_deck=model_deck)+ expected = model_deck.is_reference_defined(reference=ModelReference.parse(name), model_type=model_type.model_type)+ assert result["valid"] is expected+def test_invalid_preset_with_fuzzy_suggestions(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:"""A typo in a preset name should return fuzzy suggestions with $ prefix."""result = _run_check(mocker, capsys, "$writting-creative")@@ -149,8 +229,8 @@ class TestCheckModelCmd:assert result["valid"] is Falseassert any("@best-claude" in hint for hint in result["wrong_sigil_hints"])- def test_invalid_handle_cross_collection_suggestions(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:- """A bare name matching a preset should suggest the prefixed version."""+ def test_bare_preset_name_is_not_a_handle(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:+ """A bare name matching only a preset is not a handle, and the hint names the prefixed preset."""result = _run_check(mocker, capsys, "writing-creative")assert result["valid"] is Falseassert any("$writing-creative" in hint for hint in result["wrong_sigil_hints"])@@ -179,21 +259,15 @@ class TestCheckModelCmd:def test_markdown_valid_output(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:"""Markdown output for a valid reference should be a single confirmation line."""- _setup_mocks(mocker)- agent_check_model_cmd(name="$writing-creative", model_type=ModelCategory.LLM, output_format=CliOutputFormat.MARKDOWN)+ _setup_mocks(mocker, model_deck=_make_model_deck())+ agent_check_model_cmd(name="$writing-creative", model_type=ModelCheckCategory.LLM, output_format=CliOutputFormat.MARKDOWN)output = capsys.readouterr().out.strip()assert output == "$writing-creative is a valid llm preset."def test_markdown_invalid_output_has_suggestions(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:"""Markdown output for an invalid reference should include 'Did you mean' suggestions."""- _setup_mocks(mocker)- agent_check_model_cmd(name="$writting-creative", model_type=ModelCategory.LLM, output_format=CliOutputFormat.MARKDOWN)+ _setup_mocks(mocker, model_deck=_make_model_deck())+ agent_check_model_cmd(name="$writting-creative", model_type=ModelCheckCategory.LLM, output_format=CliOutputFormat.MARKDOWN)output = capsys.readouterr().outassert "is not a valid" in outputassert "Did you mean:" in output-- def test_valid_waterfall(self, mocker: MockerFixture, capsys: pytest.CaptureFixture[str]) -> None:- """A known waterfall name with ~ sigil should be valid."""- result = _run_check(mocker, capsys, "~robust-llm")- assert result["valid"] is True- assert result["kind"] == "waterfall"diff --git a/tests/unit/pipelex/cogt/models/test_data.py b/tests/unit/pipelex/cogt/models/test_data.pynew file mode 100644index 000000000..64cfb40f8--- /dev/null+++ b/tests/unit/pipelex/cogt/models/test_data.py@@ -0,0 +1,37 @@+from typing import ClassVar+++class ModelDeckCheckTestData:+ # Every name `TestModelDeckCheck`'s deck holds, under every kind and spelling, plus names it holds nowhere,+ # so a check made in any model type meets references it accepts and references it refuses.+ EVERY_REFERENCE: ClassVar[list[str]] = [+ "$cheap-llm",+ "@best-gpt",+ "~small-llm",+ "gpt-4o-mini",+ "$cheap-extract",+ "@best-extract",+ "~fallback-extract",+ "extract-engine",+ "$cheap-img",+ "@best-img",+ "~fallback-img",+ "img-painter",+ "$cheap-search",+ "@best-search",+ "~fallback-search",+ "web-searcher",+ "$cheap-judgment",+ "@best-judgment",+ "~fallback-judgment",+ "verdict-giver",+ "handle:img-painter",+ "alias:best-gpt",+ "preset:cheap-img",+ "waterfall:small-llm",+ "best-gpt",+ "small-llm",+ "cheap-llm",+ "held-nowhere",+ "@held-nowhere",+ ]diff --git a/tests/unit/pipelex/cogt/models/test_model_deck.py b/tests/unit/pipelex/cogt/models/test_model_deck.pyindex 6bc8f3ce0..365faec82 100644--- a/tests/unit/pipelex/cogt/models/test_model_deck.py+++ b/tests/unit/pipelex/cogt/models/test_model_deck.py@@ -218,6 +218,9 @@ class TestModelDeckGetOptionalInferenceModel:assert model_deck.get_optional_inference_model("mistral-extractor", model_type=ModelType.LLM) is Noneassert model_deck.get_optional_inference_model("mistral-extractor", model_type=ModelType.TEXT_EXTRACTOR) is not None+ # The load-time check reads the bare name the same way: a model served only as another type does not define it.+ assert model_deck.is_model_handle_defined("mistral-extractor", model_type=ModelType.LLM) is False+ assert model_deck.is_model_handle_defined("mistral-extractor", model_type=ModelType.TEXT_EXTRACTOR) is Truedef test_a_handle_served_as_another_type_does_not_shadow_this_types_alias(self):"""The LLM half of the deck may name an alias like a handle the deck serves as an extractor: the alias resolves."""@@ -228,6 +231,8 @@ class TestModelDeckGetOptionalInferenceModel:)assert model_deck.get_optional_inference_model("mistral-extractor", model_type=ModelType.LLM) == gpt_spec+ # The load-time check agrees: the bare name is defined for the LLM half through its alias.+ assert model_deck.is_model_handle_defined("mistral-extractor", model_type=ModelType.LLM) is Truedef test_complex_waterfall_scenario(self):# Arrangediff --git a/tests/unit/pipelex/cogt/models/test_model_deck_check.py b/tests/unit/pipelex/cogt/models/test_model_deck_check.pyindex 7c5711ffa..6445c95d2 100644--- a/tests/unit/pipelex/cogt/models/test_model_deck_check.py+++ b/tests/unit/pipelex/cogt/models/test_model_deck_check.py@@ -16,6 +16,7 @@ from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndexfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.model_deck import ModelDeckfrom pipelex.cogt.models.model_deck_check import (+ check_doc_gen_choice_with_deck,check_extract_choice_with_deck,check_img_gen_choice_with_deck,check_judgment_choice_with_deck,@@ -26,12 +27,18 @@ from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKifrom pipelex.cogt.search.search_setting import SearchSettingfrom pipelex.cogt.usage.cost_category import CostCategoryfrom pipelex.system.runtime import ProblemReaction+from tests.unit.pipelex.cogt.models.test_data import ModelDeckCheckTestDataCheckFunction = Callable[[Any], None]GET_MODEL_DECK_TARGET = "pipelex.cogt.models.model_deck_check.get_model_deck"+def _check_doc_gen_choice(doc_gen_choice: Any) -> None:+ """The doc gen deck check, called like the other families' checks, which take the choice positionally."""+ check_doc_gen_choice_with_deck(doc_gen_choice=doc_gen_choice)++class TestModelDeckCheck:def _create_model_spec(self, name: str, model_type: ModelType) -> InferenceModelSpec:return InferenceModelSpec(@@ -289,7 +296,10 @@ class TestModelDeckCheck:expected_model_type: ModelType,options_attr: str,) -> None:- """A missing name raises ModelChoiceNotFoundError carrying the sigil-form choice, kind, type and the right deck collection."""+ """A missing name raises ModelChoiceNotFoundError carrying the sigil-form choice, kind, type and the right deck collection.++ A handle's options are the deck's models of the pipe's type alone, the ones its field may name.+ """model_deck = self._create_test_model_deck()mocker.patch(GET_MODEL_DECK_TARGET, return_value=model_deck)@@ -349,3 +359,61 @@ class TestModelDeckCheck:check_llm_choice_with_deck(model_choice)check_llm_choice_with_deck(ModelReference.parse(model_choice))++ @pytest.mark.parametrize(+ ("check_fn", "model_choice", "needed_model"),+ [+ pytest.param(check_img_gen_choice_with_deck, "gpt-4o-mini", "an image-generation model", id="llm-in-img_gen"),+ pytest.param(check_llm_choice_with_deck, "img-painter", "an LLM", id="img_gen-in-llm"),+ pytest.param(check_extract_choice_with_deck, "handle:web-searcher", "a text-extraction model", id="search-in-extract-spelled-out"),+ pytest.param(check_judgment_choice_with_deck, "gpt-4o-mini", "a judgment model", id="llm-in-judgment"),+ ],+ )+ def test_handle_served_as_another_type_is_refused_naming_the_type_needed(+ self,+ mocker: MockerFixture,+ check_fn: CheckFunction,+ model_choice: str,+ needed_model: str,+ ) -> None:+ """A bare handle the deck serves only as another model type is refused, and the sentence names that mismatch rather than a missing handle."""+ model_deck = self._create_test_model_deck()+ mocker.patch(GET_MODEL_DECK_TARGET, return_value=model_deck)++ with pytest.raises(ModelChoiceNotFoundError) as exc_info:+ check_fn(model_choice)++ error = exc_info.value+ name = ModelReference.parse(model_choice).name+ assert error.model_choice == model_choice+ assert error.reference_kind == ModelReferenceKind.HANDLE+ assert str(error).startswith(f"Model handle '{name}' is served by the model deck, but not as {needed_model}")+ assert "was not found" not in str(error)++ @pytest.mark.parametrize(+ ("check_fn", "model_type"),+ [+ pytest.param(check_llm_choice_with_deck, ModelType.LLM, id="llm"),+ pytest.param(check_extract_choice_with_deck, ModelType.TEXT_EXTRACTOR, id="extract"),+ pytest.param(check_img_gen_choice_with_deck, ModelType.IMG_GEN, id="img_gen"),+ pytest.param(check_search_choice_with_deck, ModelType.SEARCH, id="search"),+ pytest.param(_check_doc_gen_choice, ModelType.DOC_GEN, id="doc_gen"),+ pytest.param(check_judgment_choice_with_deck, ModelType.JUDGMENT, id="judgment"),+ ],+ )+ def test_deck_check_agrees_with_is_reference_defined(self, mocker: MockerFixture, check_fn: CheckFunction, model_type: ModelType) -> None:+ """The load-time check accepts exactly the references `is_reference_defined` holds, which the model reference check answers from."""+ model_deck = self._create_test_model_deck()+ mocker.patch(GET_MODEL_DECK_TARGET, return_value=model_deck)++ for model_choice in ModelDeckCheckTestData.EVERY_REFERENCE:+ is_defined = model_deck.is_reference_defined(reference=ModelReference.parse(model_choice), model_type=model_type)+ try:+ check_fn(model_choice)+ except ModelChoiceNotFoundError:+ is_accepted = False+ else:+ is_accepted = True+ assert is_accepted == is_defined, (+ f"{model_choice!r} as {model_type}: the deck check accepts it: {is_accepted}, is_reference_defined: {is_defined}"+ )diff --git a/tests/unit/pipelex/cogt/models/test_model_deck_type_strictness.py b/tests/unit/pipelex/cogt/models/test_model_deck_type_strictness.pynew file mode 100644index 000000000..1b58f2633--- /dev/null+++ b/tests/unit/pipelex/cogt/models/test_model_deck_type_strictness.py@@ -0,0 +1,152 @@+"""The deck's own bindings and refusals hold a bare handle to the model type it is named for."""++import pytest++from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.exceptions import ImgGenHandleNotFoundError, LLMHandleNotFoundError, ModelChoiceNotFoundError, ModelDeckPresetValidatonError+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.img_gen.img_gen_setting import ImgGenSetting+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndex+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.runtime import ProblemReaction+++def _model_spec(name: str, model_type: ModelType) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name="test_backend",+ name=name,+ sdk="test_sdk",+ model_type=model_type,+ model_id=f"test_model_{name}",+ costs={CostCategory.INPUT: 0.001, CostCategory.OUTPUT: 0.002},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=1000,+ max_prompt_images=None,+ )+++def _make_deck(+ *,+ llm_presets: dict[str, LLMSetting] | None = None,+ img_gen_presets: dict[str, ImgGenSetting] | None = None,+ missing_presets_reaction: ProblemReaction = ProblemReaction.NONE,+) -> ModelDeck:+ """A deck serving LLMs and an image-generation model."""+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("claude-x", ModelType.LLM),+ _model_spec("img-painter", ModelType.IMG_GEN),+ ]+ ),+ llm_default_temperature=0.7,+ llm_presets=llm_presets or {},+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_presets=img_gen_presets or {},+ img_gen_choice_default="img-painter",+ search_choice_default="@default-search",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=True, missing_presets_reaction=missing_presets_reaction),+ )+++def _build_setting(*, model_deck: ModelDeck, model_type: ModelType, model_choice: str) -> None:+ """Build the setting a pipe of this type naming `model_choice` runs with."""+ match model_type:+ case ModelType.LLM:+ model_deck.get_llm_setting(llm_choice=model_choice)+ case ModelType.TEXT_EXTRACTOR:+ model_deck.get_extract_setting(extract_choice=model_choice)+ case ModelType.IMG_GEN:+ model_deck.get_img_gen_setting(img_gen_choice=model_choice)+ case ModelType.SEARCH:+ model_deck.get_search_setting(search_choice=model_choice)+ case ModelType.DOC_GEN:+ model_deck.get_doc_gen_setting(doc_gen_choice=model_choice)+ case ModelType.JUDGMENT:+ model_deck.get_judgment_setting(judgment_choice=model_choice)+++class TestModelDeckTypeStrictness:+ def test_an_llm_preset_naming_an_image_generation_model_refuses_boot_naming_the_type_needed(self) -> None:+ model_deck = _make_deck(llm_presets={"painter": LLMSetting(model="img-painter", temperature=0.5)})++ with pytest.raises(LLMHandleNotFoundError) as exc_info:+ model_deck.validate_llm_presets()++ assert str(exc_info.value) == "LLM preset 'painter': Model handle 'img-painter' is served by the model deck, but not as an LLM"+ assert exc_info.value.preset_id == "painter"+ assert exc_info.value.model_handle == "img-painter"++ def test_an_image_generation_preset_naming_an_llm_refuses_boot_naming_the_type_needed(self) -> None:+ model_deck = _make_deck(img_gen_presets={"writer": ImgGenSetting(model="claude-x")})++ with pytest.raises(ImgGenHandleNotFoundError) as exc_info:+ model_deck.validate_img_gen_presets()++ assert str(exc_info.value) == (+ "Image generation preset 'writer': Model handle 'claude-x' is served by the model deck, but not as an image-generation model"+ )++ def test_a_preset_naming_a_model_served_nowhere_says_it_was_not_found(self) -> None:+ model_deck = _make_deck(llm_presets={"ghost": LLMSetting(model="gpt-9", temperature=0.5)})++ with pytest.raises(LLMHandleNotFoundError) as exc_info:+ model_deck.validate_llm_presets()++ assert str(exc_info.value) == "LLM preset 'ghost': Model handle 'gpt-9' was not found in the model deck"++ def test_a_preset_naming_an_unknown_alias_says_the_reference_was_not_found(self) -> None:+ model_deck = _make_deck(llm_presets={"ghost": LLMSetting(model="@nowhere", temperature=0.5)})++ with pytest.raises(LLMHandleNotFoundError) as exc_info:+ model_deck.validate_llm_presets()++ assert str(exc_info.value) == "LLM preset 'ghost': Model reference '@nowhere' was not found in the model deck"++ def test_the_boot_refusal_carries_the_preset_sentence(self) -> None:+ model_deck = _make_deck(+ llm_presets={"painter": LLMSetting(model="img-painter", temperature=0.5)},+ missing_presets_reaction=ProblemReaction.RAISE,+ )++ with pytest.raises(ModelDeckPresetValidatonError, match=r"Failed to validate all LLM presets: LLM preset 'painter': .* but not as an LLM"):+ model_deck.validate_registered_models()++ def test_an_llm_override_naming_an_image_generation_model_is_refused_with_the_llms_alone(self) -> None:+ model_deck = _make_deck()++ with pytest.raises(ModelChoiceNotFoundError) as exc_info:+ model_deck.check_llm_choice(llm_choice="img-painter")++ assert exc_info.value.message.startswith("Model handle 'img-painter' is served by the model deck, but not as an LLM\n")+ assert "Available handle: claude-x, gpt-4o-mini" in exc_info.value.message+ assert exc_info.value.available_options == ["claude-x", "gpt-4o-mini"]++ @pytest.mark.parametrize(+ ("model_type", "expected_options"),+ [+ (ModelType.LLM, ["claude-x", "gpt-4o-mini"]),+ (ModelType.IMG_GEN, ["img-painter"]),+ (ModelType.SEARCH, []),+ ],+ )+ def test_a_handle_refusal_offers_the_models_of_its_type_alone(self, model_type: ModelType, expected_options: list[str]) -> None:+ model_deck = _make_deck()++ with pytest.raises(ModelChoiceNotFoundError) as exc_info:+ _build_setting(model_deck=model_deck, model_type=model_type, model_choice="held-nowhere")++ assert exc_info.value.available_options == expected_options+ assert model_deck.inference_models.handles_of_type(model_type=model_type) == expected_optionsdiff --git a/tests/unit/pipelex/cogt/models/test_model_reference.py b/tests/unit/pipelex/cogt/models/test_model_reference.pyindex 0730e9db1..d389bacff 100644--- a/tests/unit/pipelex/cogt/models/test_model_reference.py+++ b/tests/unit/pipelex/cogt/models/test_model_reference.py@@ -12,6 +12,7 @@ from pipelex.cogt.models.model_reference import (ModelReferenceKind,ModelReferenceParseError,parse_model_reference,+ write_model_handle,)@@ -204,6 +205,27 @@ class TestModelReferenceHelpers:assert handle_ref.is_handle() is Trueassert preset_ref.is_handle() is False+ @pytest.mark.parametrize(+ ("name", "expected_written"),+ [+ ("gpt-4o-mini", "gpt-4o-mini"),+ ("claude-4.5-sonnet", "claude-4.5-sonnet"),+ ("@named", "handle:@named"),+ ("~named", "handle:~named"),+ ("$named", "handle:$named"),+ ("alias:named", "handle:alias:named"),+ ("handle:named", "handle:handle:named"),+ ],+ )+ def test_write_model_handle_parses_back_to_the_same_handle(self, name: str, expected_written: str):+ """A handle is written bare unless its bare name would read as another reference, then with the `handle:` namespace."""+ written = write_model_handle(name=name)++ assert written == expected_written+ parsed = ModelReference.parse(written)+ assert parsed.kind == ModelReferenceKind.HANDLE+ assert parsed.name == name+class TestParseModelReference:"""Tests for parse_model_reference() BeforeValidator function."""diff --git a/tests/unit/pipelex/cogt/models/test_model_reference_check.py b/tests/unit/pipelex/cogt/models/test_model_reference_check.pynew file mode 100644index 000000000..ee4272604--- /dev/null+++ b/tests/unit/pipelex/cogt/models/test_model_reference_check.py@@ -0,0 +1,617 @@+from typing import Any++import pytest+from mthds.protocol.models import ModelCategory as MthdsModelCategory+from pytest_mock import MockerFixture++from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenSetting+from pipelex.cogt.exceptions import ModelChoiceNotFoundError, ModelNotFoundError, ModelWaterfallError+from pipelex.cogt.extract.extract_setting import ExtractSetting+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.img_gen.img_gen_setting import ImgGenSetting+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndex+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind+from pipelex.cogt.models.model_reference_check import (+ AliasMatch,+ HandleMatch,+ ModelCheckCategory,+ ModelReferenceResolution,+ ModelReferenceVerdict,+ PresetMatch,+ WaterfallMatch,+ check_model_reference,+)+from pipelex.cogt.search.search_setting import SearchSetting+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.runtime import ProblemReaction++_DECK_LOG_TARGET = "pipelex.cogt.models.model_deck.log"+_DECK_CHECK_GET_MODEL_DECK_TARGET = "pipelex.cogt.models.model_deck_check.get_model_deck"+++def _model_spec(name: str, model_type: ModelType) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name="test_backend",+ name=name,+ sdk="test_sdk",+ model_type=model_type,+ model_id=f"test_model_{name}",+ costs={CostCategory.INPUT: 0.001, CostCategory.OUTPUT: 0.002},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=1000,+ max_prompt_images=None,+ )+++def _make_deck(*, is_model_fallback_enabled: bool = True) -> ModelDeck:+ """A deck whose bindings cover every resolution: served, unserved, through a reference, cyclic, and names shared across types."""+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("claude-x", ModelType.LLM),+ _model_spec("img-painter", ModelType.IMG_GEN),+ _model_spec("extract-engine", ModelType.TEXT_EXTRACTOR),+ _model_spec("reportlab-pdf", ModelType.DOC_GEN),+ # An extraction model whose name is also an LLM alias.+ _model_spec("shared-name", ModelType.TEXT_EXTRACTOR),+ ]+ ),+ llm_default_temperature=0.7,+ llm_aliases={+ "best-gpt": "gpt-4o-mini",+ "best-claude": "claude-x",+ "spelled-handle": "handle:claude-x",+ "unserved-alias": "gpt-9",+ "cycle-a": "@cycle-b",+ "cycle-b": "@cycle-a",+ "shared-name": "gpt-4o-mini",+ },+ llm_waterfalls={+ "small-llm": ["gpt-9", "gpt-4o-mini"],+ "none-served": ["gpt-9", "gpt-10"],+ "via-alias": ["@best-claude"],+ },+ llm_presets={+ "cheap-llm": LLMSetting(model="gpt-4o-mini", temperature=0.5, description="Cheap and quick"),+ "premium": LLMSetting(model="@best-claude", temperature=0.5),+ "unserved-preset": LLMSetting(model="gpt-9", temperature=0.5),+ },+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_aliases={"best-gpt": "img-painter"},+ img_gen_presets={"cheap-img": ImgGenSetting(model="img-painter")},+ img_gen_choice_default="img-painter",+ search_choice_default="@default-search",+ doc_gen_aliases={"default-pdf": "reportlab-pdf"},+ doc_gen_presets={"plain-pdf": DocGenSetting(model="@default-pdf", description="A plain PDF")},+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),+ )+++def _check(reference: str, *, category: ModelCheckCategory | None = None, is_model_fallback_enabled: bool = True) -> ModelReferenceVerdict:+ return check_model_reference(+ model_deck=_make_deck(is_model_fallback_enabled=is_model_fallback_enabled),+ reference=ModelReference.parse(reference),+ category=category,+ )+++def _deck_with(+ *,+ inference_models: list[InferenceModelSpec],+ is_model_fallback_enabled: bool = True,+ **bindings: Any,+) -> ModelDeck:+ """A deck serving `inference_models` with the aliases, waterfalls and presets `bindings` names, by their deck field."""+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(model_specs=inference_models),+ llm_default_temperature=0.7,+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_choice_default="img-painter",+ search_choice_default="@default-search",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),+ **bindings,+ )+++# The prefix of each model type's binding fields on the deck (`llm_aliases`, `extract_waterfalls`, ...).+_BINDING_PREFIXES: dict[ModelType, str] = {+ ModelType.LLM: "llm",+ ModelType.TEXT_EXTRACTOR: "extract",+ ModelType.IMG_GEN: "img_gen",+ ModelType.SEARCH: "search",+ ModelType.DOC_GEN: "doc_gen",+ ModelType.JUDGMENT: "judgment",+}+++def _preset(*, model_type: ModelType, model: str) -> LLMSetting | ExtractSetting | ImgGenSetting | SearchSetting | DocGenSetting | JudgmentSetting:+ match model_type:+ case ModelType.LLM:+ return LLMSetting(model=model, temperature=0.5)+ case ModelType.TEXT_EXTRACTOR:+ return ExtractSetting(model=model)+ case ModelType.IMG_GEN:+ return ImgGenSetting(model=model)+ case ModelType.SEARCH:+ return SearchSetting(model=model)+ case ModelType.DOC_GEN:+ return DocGenSetting(model=model)+ case ModelType.JUDGMENT:+ return JudgmentSetting(model=model)+++def _make_collision_deck(*, model_type: ModelType) -> ModelDeck:+ """A deck of one model type where models, aliases and waterfalls share names.++ `shared` is a model, an alias to `other` and a waterfall to `other`; `twin` is no model, an alias to+ `other`, a waterfall to `shared` and a preset bound to the waterfall `~twin`.+ """+ prefix = _BINDING_PREFIXES[model_type]+ bindings: dict[str, Any] = {+ f"{prefix}_aliases": {"shared": "other", "twin": "other"},+ f"{prefix}_waterfalls": {"shared": ["other"], "twin": ["shared"]},+ f"{prefix}_presets": {"twin": _preset(model_type=model_type, model="~twin")},+ }+ return _deck_with(+ inference_models=[+ _model_spec("shared", model_type),+ _model_spec("other", model_type),+ ],+ **bindings,+ )+++def _model_the_run_calls(*, model_deck: ModelDeck, reference: str, model_type: ModelType) -> str:+ """The model a pipe of this type naming `reference` calls: its setting, as a run builds it, then the deck's lookup of the setting's model."""+ setting_model: str+ match model_type:+ case ModelType.LLM:+ setting_model = model_deck.get_llm_setting(llm_choice=reference).model+ case ModelType.TEXT_EXTRACTOR:+ setting_model = model_deck.get_extract_setting(extract_choice=reference).model+ case ModelType.IMG_GEN:+ setting_model = model_deck.get_img_gen_setting(img_gen_choice=reference).model+ case ModelType.SEARCH:+ setting_model = model_deck.get_search_setting(search_choice=reference).model+ case ModelType.DOC_GEN:+ setting_model = model_deck.get_doc_gen_setting(doc_gen_choice=reference).model+ case ModelType.JUDGMENT:+ setting_model = model_deck.get_judgment_setting(judgment_choice=reference).model+ return model_deck.get_required_inference_model(model_handle=setting_model, model_type=model_type).name+++def _category_of(model_type: ModelType) -> ModelCheckCategory:+ return next(category for category in ModelCheckCategory if category.model_type == model_type)+++def _assert_resolved(verdict: ModelReferenceVerdict) -> None:+ assert verdict.resolution == ModelReferenceResolution.RESOLVED, verdict+ assert verdict.suggestions == []+ assert verdict.other_kinds == []+ assert verdict.other_categories == []+++def _assert_not_found(verdict: ModelReferenceVerdict) -> None:+ assert verdict.resolution == ModelReferenceResolution.NOT_FOUND, verdict+ assert verdict.matches == []+++class TestModelReferenceCheck:+ def test_categories_are_the_protocols_in_its_order_then_doc_gen(self) -> None:+ assert [category.value for category in ModelCheckCategory] == [*(category.value for category in MthdsModelCategory), "doc_gen"]++ @pytest.mark.parametrize(+ ("reference", "expected_match"),+ [+ pytest.param(+ "$cheap-llm",+ PresetMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", target="gpt-4o-mini", description="Cheap and quick"),+ id="preset-on-a-handle",+ ),+ pytest.param(+ "preset:premium",+ PresetMatch(category=ModelCheckCategory.LLM, resolves_to="claude-x", target="@best-claude", description=None),+ id="preset-on-an-alias-spelled-out",+ ),+ pytest.param(+ "$unserved-preset",+ PresetMatch(category=ModelCheckCategory.LLM, resolves_to=None, target="gpt-9", description=None),+ id="preset-on-an-unserved-model",+ ),+ pytest.param("@best-gpt", AliasMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", target="gpt-4o-mini"), id="alias"),+ pytest.param("@unserved-alias", AliasMatch(category=ModelCheckCategory.LLM, resolves_to=None, target="gpt-9"), id="alias-unserved"),+ pytest.param("@cycle-a", AliasMatch(category=ModelCheckCategory.LLM, resolves_to=None, target="@cycle-b"), id="alias-cycle"),+ pytest.param(+ "~small-llm",+ WaterfallMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", fallbacks=["gpt-9", "gpt-4o-mini"]),+ id="waterfall-first-served-step",+ ),+ pytest.param(+ "~none-served",+ WaterfallMatch(category=ModelCheckCategory.LLM, resolves_to=None, fallbacks=["gpt-9", "gpt-10"]),+ id="waterfall-none-served",+ ),+ pytest.param(+ "waterfall:via-alias",+ WaterfallMatch(category=ModelCheckCategory.LLM, resolves_to="claude-x", fallbacks=["@best-claude"]),+ id="waterfall-through-an-alias",+ ),+ pytest.param(+ "gpt-4o-mini",+ HandleMatch(+ category=ModelCheckCategory.LLM,+ resolves_to="gpt-4o-mini",+ via=["$cheap-llm", "@best-gpt", "@shared-name", "~small-llm"],+ ),+ id="handle-with-the-deck-names-binding-it",+ ),+ pytest.param(+ "handle:claude-x",+ HandleMatch(category=ModelCheckCategory.LLM, resolves_to="claude-x", via=["@best-claude", "@spelled-handle"]),+ id="handle-spelled-out-bound-directly-only",+ ),+ pytest.param("best-gpt", HandleMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", via=[]), id="bare-name-of-an-alias"),+ pytest.param("small-llm", HandleMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", via=[]), id="bare-name-of-a-waterfall"),+ pytest.param(+ "shared-name",+ HandleMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", via=[]),+ id="bare-name-of-an-alias-and-of-a-model-of-another-type",+ ),+ ],+ )+ def test_a_reference_resolved_in_its_category_says_what_it_is(self, reference: str, expected_match: Any) -> None:+ verdict = _check(reference, category=ModelCheckCategory.LLM)++ _assert_resolved(verdict)+ parsed = ModelReference.parse(reference)+ assert verdict.reference == reference+ assert verdict.kind == parsed.kind+ assert verdict.name == parsed.name+ assert verdict.category == ModelCheckCategory.LLM+ assert verdict.matches == [expected_match]++ def test_with_model_fallback_off_a_waterfall_resolves_to_its_first_step_alone(self) -> None:+ verdict = _check("~small-llm", category=ModelCheckCategory.LLM, is_model_fallback_enabled=False)++ _assert_resolved(verdict)+ assert verdict.matches == [WaterfallMatch(category=ModelCheckCategory.LLM, resolves_to=None, fallbacks=["gpt-9", "gpt-4o-mini"])]++ def test_with_model_fallback_off_the_bare_name_of_a_waterfall_does_not_resolve(self) -> None:+ verdict = _check("small-llm", category=ModelCheckCategory.LLM, is_model_fallback_enabled=False)++ _assert_not_found(verdict)+ assert verdict.other_kinds == ["~small-llm"]++ def test_a_model_of_another_category_does_not_make_a_bare_handle_resolve(self) -> None:+ verdict = _check("gpt-4o-mini", category=ModelCheckCategory.IMG_GEN)++ _assert_not_found(verdict)+ assert verdict.kind == ModelReferenceKind.HANDLE+ assert verdict.category == ModelCheckCategory.IMG_GEN+ assert verdict.other_categories == [ModelCheckCategory.LLM]++ def test_without_a_type_every_category_where_it_resolves_is_a_match_in_order(self) -> None:+ verdict = _check("shared-name")++ _assert_resolved(verdict)+ assert verdict.category is None+ assert verdict.matches == [+ HandleMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", via=[]),+ HandleMatch(category=ModelCheckCategory.EXTRACT, resolves_to="shared-name", via=[]),+ ]++ def test_doc_gen_is_a_category_the_check_covers(self) -> None:+ verdict = _check("$plain-pdf")++ _assert_resolved(verdict)+ assert verdict.matches == [+ PresetMatch(category=ModelCheckCategory.DOC_GEN, resolves_to="reportlab-pdf", target="@default-pdf", description="A plain PDF")+ ]++ def test_a_misspelt_name_suggests_its_neighbours_once_each(self) -> None:+ """`@best-gp` is near `@best-gpt` in both the LLM and the image-generation categories, which the suggestions name once."""+ verdict = _check("@best-gp")++ _assert_not_found(verdict)+ assert verdict.suggestions.count("@best-gpt") == 1+ assert verdict.other_kinds == []+ assert verdict.other_categories == []++ def test_a_name_under_another_kind_is_in_other_kinds_written_with_its_sigil(self) -> None:+ verdict = _check("$best-gpt")++ _assert_not_found(verdict)+ assert verdict.other_kinds == ["@best-gpt"]++ def test_suggestions_of_another_kind_are_names_without_labels(self) -> None:+ verdict = _check("$best-claud", category=ModelCheckCategory.LLM)++ _assert_not_found(verdict)+ assert "@best-claude" in verdict.suggestions+ assert all("(" not in suggestion for suggestion in verdict.suggestions), verdict.suggestions++ def test_a_reference_asked_in_another_category_names_the_categories_where_it_resolves(self) -> None:+ verdict = _check("@best-gpt", category=ModelCheckCategory.SEARCH)++ _assert_not_found(verdict)+ assert verdict.other_categories == [ModelCheckCategory.LLM, ModelCheckCategory.IMG_GEN]++ def test_a_reference_held_nowhere_is_not_found_with_nothing_to_offer(self) -> None:+ verdict = _check("~held-nowhere-at-all", category=ModelCheckCategory.JUDGMENT)++ _assert_not_found(verdict)+ assert verdict.suggestions == []+ assert verdict.other_kinds == []+ assert verdict.other_categories == []++ def test_the_verdict_echoes_the_trimmed_reference_and_its_parsing(self) -> None:+ verdict = _check(" alias:best-gpt ", category=ModelCheckCategory.LLM)++ assert verdict.reference == "alias:best-gpt"+ assert verdict.kind == ModelReferenceKind.ALIAS+ assert verdict.name == "best-gpt"++ @pytest.mark.parametrize(+ ("reference", "match_keys"),+ [+ pytest.param("$premium", {"category", "resolves_to", "target", "description"}, id="preset"),+ pytest.param("@unserved-alias", {"category", "resolves_to", "target"}, id="alias"),+ pytest.param("~none-served", {"category", "resolves_to", "fallbacks"}, id="waterfall"),+ pytest.param("claude-x", {"category", "resolves_to", "via"}, id="handle"),+ ],+ )+ def test_a_match_carries_the_fields_of_its_kind_and_nulls_only_where_they_apply(self, reference: str, match_keys: set[str]) -> None:+ dumped = _check(reference, category=ModelCheckCategory.LLM).model_dump(mode="json")++ assert set(dumped) == {"reference", "kind", "name", "category", "resolution", "matches", "suggestions", "other_kinds", "other_categories"}+ (match,) = dumped["matches"]+ assert set(match) == match_keys++ @pytest.mark.parametrize(+ ("reference", "expected_model"),+ [+ pytest.param("~twin", "shared", id="waterfall-named-like-an-alias"),+ pytest.param("waterfall:twin", "shared", id="waterfall-namespace"),+ pytest.param("@twin", "other", id="alias-named-like-a-waterfall"),+ pytest.param("twin", "other", id="bare-name-of-an-alias-and-a-waterfall"),+ pytest.param("~shared", "other", id="waterfall-named-like-a-model"),+ pytest.param("@shared", "other", id="alias-named-like-a-model"),+ pytest.param("shared", "shared", id="bare-name-of-a-model"),+ pytest.param("$twin", "shared", id="preset-bound-to-a-waterfall"),+ ],+ )+ @pytest.mark.parametrize("model_type", list(ModelType))+ def test_the_verdict_names_the_model_a_run_calls_when_names_collide(self, reference: str, expected_model: str, model_type: ModelType) -> None:+ """A sigiled reference resolves by its sigil in the check and in a run alike, whatever shares its name."""+ model_deck = _make_collision_deck(model_type=model_type)++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse(reference), category=_category_of(model_type))++ _assert_resolved(verdict)+ (match,) = verdict.matches+ assert match.resolves_to == expected_model+ assert _model_the_run_calls(model_deck=model_deck, reference=reference, model_type=model_type) == expected_model++ @pytest.mark.parametrize("reference", ["@cycle-a", "~cycle-waterfall"])+ def test_an_alias_and_a_waterfall_leading_to_each_other_end_as_no_model(self, reference: str) -> None:+ """A cycle through an alias and a waterfall resolves to nothing in the check, and a run refuses it cleanly."""+ model_deck = _deck_with(+ inference_models=[_model_spec("gpt-4o-mini", ModelType.LLM)],+ llm_aliases={"cycle-a": "~cycle-waterfall"},+ llm_waterfalls={"cycle-waterfall": ["@cycle-a"]},+ )++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse(reference), category=ModelCheckCategory.LLM)++ _assert_resolved(verdict)+ (match,) = verdict.matches+ assert match.resolves_to is None+ with pytest.raises(ModelNotFoundError):+ _model_the_run_calls(model_deck=model_deck, reference=reference, model_type=ModelType.LLM)++ @pytest.mark.parametrize(+ ("second_step", "expected_model"),+ [+ pytest.param("img-painter", "img-painter", id="next-step-served"),+ pytest.param("img-unserved", None, id="next-step-unserved"),+ ],+ )+ def test_a_waterfall_step_leading_back_to_its_waterfall_is_a_step_no_model_serves(self, second_step: str, expected_model: str | None) -> None:+ """An image-generation waterfall named like an LLM, whose first step is its own name: that step leads back to the waterfall.++ The step ends as no model, as an unserved one does, so the waterfall goes on to its next step.+ """+ model_deck = _deck_with(+ inference_models=[+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("img-painter", ModelType.IMG_GEN),+ ],+ img_gen_waterfalls={"gpt-4o-mini": ["gpt-4o-mini", second_step]},+ )++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("gpt-4o-mini"), category=ModelCheckCategory.IMG_GEN)++ _assert_resolved(verdict)+ assert verdict.matches == [HandleMatch(category=ModelCheckCategory.IMG_GEN, resolves_to=expected_model, via=["~gpt-4o-mini"])]+ if expected_model is None:+ with pytest.raises(ModelNotFoundError):+ _model_the_run_calls(model_deck=model_deck, reference="gpt-4o-mini", model_type=ModelType.IMG_GEN)+ else:+ assert _model_the_run_calls(model_deck=model_deck, reference="gpt-4o-mini", model_type=ModelType.IMG_GEN) == expected_model++ def test_the_check_logs_nothing_and_leaves_the_fallback_notice_to_the_run(self, mocker: MockerFixture) -> None:+ """A check reads the run's lookup without its side effects; the run that follows still logs what it always logged."""+ model_deck = _make_deck()+ deck_log = mocker.patch(_DECK_LOG_TARGET)++ for reference in ("~small-llm", "small-llm", "@cycle-a", "~none-served", "$premium", "gpt-9", "@unserved-alias"):+ check_model_reference(model_deck=model_deck, reference=ModelReference.parse(reference), category=None)++ assert deck_log.mock_calls == []+ model_deck.get_optional_inference_model(model_handle="~small-llm", model_type=ModelType.LLM)+ deck_log.info.assert_called_once()+ model_deck.get_optional_inference_model(model_handle="@cycle-a", model_type=ModelType.LLM)+ deck_log.warning.assert_called_once()++ def test_with_model_fallback_off_a_bare_waterfall_name_is_refused_by_the_check_the_validation_and_the_run(self, mocker: MockerFixture) -> None:+ """A pipe's bare name reaches the deck's lookup only through its setting, which refuses a waterfall name while fallback is off.++ The deck's lookup alone would serve the waterfall's first step, but a pipe never reaches it with this name.+ """+ model_deck = _make_deck(is_model_fallback_enabled=False)+ mocker.patch(_DECK_CHECK_GET_MODEL_DECK_TARGET, return_value=model_deck)++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("small-llm"), category=ModelCheckCategory.LLM)++ _assert_not_found(verdict)+ with pytest.raises(ModelChoiceNotFoundError):+ check_llm_choice_with_deck("small-llm")+ with pytest.raises(ModelChoiceNotFoundError):+ model_deck.get_llm_setting(llm_choice="small-llm")++ @pytest.mark.parametrize(+ ("waterfalls", "aliases"),+ [+ pytest.param({"outer": ["~inner", "gpt-4o-mini"], "inner": ["~outer"]}, {}, id="nested-waterfall-leading-back"),+ pytest.param({"outer": ["~inner", "gpt-4o-mini"], "inner": ["unserved-model"]}, {}, id="nested-waterfall-unserved"),+ pytest.param(+ {"outer": ["@to-inner", "gpt-4o-mini"], "inner": ["unserved-model"]}, {"to-inner": "~inner"}, id="alias-to-an-unserved-waterfall"+ ),+ ],+ )+ def test_a_step_whose_waterfall_runs_out_lets_its_waterfall_go_on(self, waterfalls: dict[str, list[str]], aliases: dict[str, str]) -> None:+ """A step reaching a waterfall none of whose steps is served serves no model, so the outer waterfall tries its next step."""+ model_deck = _deck_with(+ inference_models=[_model_spec("gpt-4o-mini", ModelType.LLM)],+ llm_waterfalls=waterfalls,+ llm_aliases=aliases,+ )++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("~outer"), category=ModelCheckCategory.LLM)++ _assert_resolved(verdict)+ assert verdict.matches == [WaterfallMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-4o-mini", fallbacks=waterfalls["outer"])]+ assert _model_the_run_calls(model_deck=model_deck, reference="~outer", model_type=ModelType.LLM) == "gpt-4o-mini"++ def test_with_model_fallback_off_a_nested_waterfall_refusing_its_fallback_refuses_the_outer_one(self) -> None:+ """The refusal of a fallback while fallbacks are disabled is not a waterfall running out: it holds through the outer waterfall."""+ model_deck = _deck_with(+ inference_models=[+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("claude-x", ModelType.LLM),+ ],+ is_model_fallback_enabled=False,+ llm_waterfalls={"outer": ["~inner", "claude-x"], "inner": ["unserved-model", "gpt-4o-mini"]},+ )++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("~outer"), category=ModelCheckCategory.LLM)++ assert verdict.matches == [WaterfallMatch(category=ModelCheckCategory.LLM, resolves_to=None, fallbacks=["~inner", "claude-x"])]+ with pytest.raises(ModelNotFoundError) as exc_info:+ _model_the_run_calls(model_deck=model_deck, reference="~outer", model_type=ModelType.LLM)+ assert not isinstance(exc_info.value, ModelWaterfallError)+ assert "model fallbacks are disabled" in exc_info.value.message++ def test_a_handle_reference_reads_no_sigil(self, mocker: MockerFixture) -> None:+ """`handle:@best-gpt` is the bare name `@best-gpt`, not the alias `best-gpt`.++ With nothing of that name, the check, the validation and the run refuse it alike.+ """+ model_deck = _make_deck()+ mocker.patch(_DECK_CHECK_GET_MODEL_DECK_TARGET, return_value=model_deck)++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("handle:@best-gpt"), category=ModelCheckCategory.LLM)++ _assert_not_found(verdict)+ assert model_deck.is_reference_defined(reference=ModelReference.parse("handle:@best-gpt"), model_type=ModelType.LLM) is False+ with pytest.raises(ModelChoiceNotFoundError):+ check_llm_choice_with_deck("handle:@best-gpt")+ with pytest.raises(ModelChoiceNotFoundError):+ model_deck.get_llm_setting(llm_choice="handle:@best-gpt")+ with pytest.raises(ModelChoiceNotFoundError):+ model_deck.check_llm_choice(llm_choice="handle:@best-gpt")++ @pytest.mark.parametrize("literal_name", ["@named", "~named", "$named", "alias:named"])+ @pytest.mark.parametrize(+ "bindings",+ [+ pytest.param({}, id="no-colliding-binding"),+ pytest.param({"llm_aliases": {"named": "other"}, "llm_waterfalls": {"named": ["other"]}}, id="colliding-alias-and-waterfall"),+ ],+ )+ def test_a_served_handle_spelled_like_a_reference_is_called_by_its_bare_name(+ self, mocker: MockerFixture, literal_name: str, bindings: dict[str, Any]+ ) -> None:+ """`handle:@named` names the model served as `@named`: the validation, the run and the check all select it, not the alias."""+ model_deck = _deck_with(+ inference_models=[+ _model_spec(literal_name, ModelType.LLM),+ _model_spec("other", ModelType.LLM),+ ],+ **bindings,+ )+ mocker.patch(_DECK_CHECK_GET_MODEL_DECK_TARGET, return_value=model_deck)+ reference = f"handle:{literal_name}"++ check_llm_choice_with_deck(reference)+ model_deck.validate_inference_models()+ assert _model_the_run_calls(model_deck=model_deck, reference=reference, model_type=ModelType.LLM) == literal_name+ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse(reference), category=ModelCheckCategory.LLM)+ _assert_resolved(verdict)+ (match,) = verdict.matches+ assert match.resolves_to == literal_name++ def test_a_suggested_handle_spelled_like_a_reference_is_written_with_its_namespace(self) -> None:+ """A suggestion is a reference a caller may write back, so a handle spelled like an alias is offered as `handle:@named`."""+ model_deck = _deck_with(inference_models=[_model_spec("@named", ModelType.LLM)])++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("handle:@namedd"), category=ModelCheckCategory.LLM)++ _assert_not_found(verdict)+ assert "handle:@named" in verdict.suggestions++ def test_a_handle_served_as_two_types_resolves_in_each_and_is_named_as_served_elsewhere_in_others(self) -> None:+ """One handle names one model per model type: the check finds it in every category serving it, and a refusal elsewhere says it is served."""+ model_deck = _deck_with(+ inference_models=[+ _model_spec("gpt-6-luna", ModelType.LLM),+ _model_spec("gpt-6-luna", ModelType.JUDGMENT),+ ],+ )++ verdict = check_model_reference(model_deck=model_deck, reference=ModelReference.parse("gpt-6-luna"), category=None)++ _assert_resolved(verdict)+ assert verdict.matches == [+ HandleMatch(category=ModelCheckCategory.LLM, resolves_to="gpt-6-luna", via=[]),+ HandleMatch(category=ModelCheckCategory.JUDGMENT, resolves_to="gpt-6-luna", via=[]),+ ]+ img_gen_verdict = check_model_reference(+ model_deck=model_deck, reference=ModelReference.parse("gpt-6-luna"), category=ModelCheckCategory.IMG_GEN+ )+ _assert_not_found(img_gen_verdict)+ assert img_gen_verdict.other_categories == [ModelCheckCategory.LLM, ModelCheckCategory.JUDGMENT]+ assert model_deck.unresolved_handle_sentence(name="gpt-6-luna", model_type=ModelType.IMG_GEN) == (+ "Model handle 'gpt-6-luna' is served by the model deck, but not as an image-generation model"+ )diff --git a/tests/unit/pipelex/kernel/test_resolved_handle_pinning.py b/tests/unit/pipelex/kernel/test_resolved_handle_pinning.pynew file mode 100644index 000000000..01f56b9a4--- /dev/null+++ b/tests/unit/pipelex/kernel/test_resolved_handle_pinning.py@@ -0,0 +1,128 @@+"""A kernel op pins its setting to the handle it resolved, written so the leaf lookup reads that same handle back.++A served handle may be spelled like a reference (`@named`): pinned bare, the leaf lookup would read it as the alias+`named`, or as nothing at all, rather than as the model the op resolved.+"""++from typing import Any++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.doc_gen.doc_gen_engine import resolve_doc_gen_setting+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndex+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.kernel.judgment_ops import resolve_judgment_setting+from pipelex.kernel.search_ops import resolve_search_setting+from pipelex.system.runtime import ProblemReaction++_LITERAL_HANDLE = "@named"++# The deck field prefix of each model type's bindings (`search_aliases`, ...).+_BINDING_PREFIXES: dict[ModelType, str] = {+ ModelType.SEARCH: "search",+ ModelType.JUDGMENT: "judgment",+ ModelType.DOC_GEN: "doc_gen",+}+++def _model_spec(*, name: str, model_type: ModelType) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name="test_backend",+ name=name,+ sdk="test_sdk",+ model_type=model_type,+ model_id=f"test_model_{name}",+ costs={},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ # A document engine declares the source it prints from and the format it prints.+ inputs=[str(DocGenSource.LAYOUT)],+ outputs=[str(DocGenFormat.PDF)],+ )+++def _make_deck(*, model_type: ModelType, is_alias_colliding: bool) -> ModelDeck:+ """A deck serving `@named` and `other` as `model_type`, with an alias bound to `@named`, and optionally an alias `named` bound to `other`."""+ prefix = _BINDING_PREFIXES[model_type]+ aliases = {"to-literal": f"handle:{_LITERAL_HANDLE}"}+ if is_alias_colliding:+ aliases["named"] = "other"+ bindings: dict[str, Any] = {f"{prefix}_aliases": aliases}+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec(name=_LITERAL_HANDLE, model_type=model_type),+ _model_spec(name="other", model_type=model_type),+ ]+ ),+ llm_default_temperature=0.7,+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_choice_default="img-painter",+ search_choice_default="@default-search",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=True, missing_presets_reaction=ProblemReaction.NONE),+ **bindings,+ )+++def _pinned_model(*, mocker: MockerFixture, model_deck: ModelDeck, model_type: ModelType, choice: str) -> str:+ """The model the op pins its setting to, the deck patched where the op reads it."""+ match model_type:+ case ModelType.SEARCH:+ mocker.patch("pipelex.kernel.search_ops.get_model_deck", return_value=model_deck)+ return resolve_search_setting(search_choice=choice).model+ case ModelType.JUDGMENT:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=model_deck)+ return resolve_judgment_setting(judgment_choice=choice, pipe_code="judge").model+ case ModelType.DOC_GEN:+ mocker.patch("pipelex.cogt.doc_gen.doc_gen_engine.get_model_deck", return_value=model_deck)+ mocker.patch(+ "pipelex.cogt.doc_gen.doc_gen_engine.get_inference_backend_registry", return_value=mocker.Mock(has=mocker.Mock(return_value=True))+ )+ return resolve_doc_gen_setting(+ doc_gen_choice=choice, doc_gen_format=DocGenFormat.PDF, source=DocGenSource.LAYOUT, pipe_code="print"+ ).model+ case ModelType.LLM | ModelType.TEXT_EXTRACTOR | ModelType.IMG_GEN:+ pytest.fail(f"{model_type} has no op pinning its resolved handle")+++class TestResolvedHandlePinning:+ @pytest.mark.parametrize("model_type", [ModelType.SEARCH, ModelType.JUDGMENT, ModelType.DOC_GEN])+ @pytest.mark.parametrize("is_alias_colliding", [False, True], ids=["no-colliding-alias", "colliding-alias"])+ @pytest.mark.parametrize("choice", [f"handle:{_LITERAL_HANDLE}", "@to-literal"], ids=["literal-handle", "alias-to-the-literal-handle"])+ def test_the_leaf_lookup_reads_back_the_handle_the_op_resolved(+ self,+ mocker: MockerFixture,+ model_type: ModelType,+ is_alias_colliding: bool,+ choice: str,+ ) -> None:+ model_deck = _make_deck(model_type=model_type, is_alias_colliding=is_alias_colliding)++ pinned_model = _pinned_model(mocker=mocker, model_deck=model_deck, model_type=model_type, choice=choice)++ assert pinned_model == f"handle:{_LITERAL_HANDLE}"+ assert model_deck.get_required_inference_model(model_handle=pinned_model, model_type=model_type).name == _LITERAL_HANDLE++ @pytest.mark.parametrize("model_type", [ModelType.SEARCH, ModelType.JUDGMENT, ModelType.DOC_GEN])+ def test_an_ordinary_handle_is_pinned_bare(self, mocker: MockerFixture, model_type: ModelType) -> None:+ model_deck = _make_deck(model_type=model_type, is_alias_colliding=False)+ model_deck_with_alias = model_deck.model_copy(update={f"{_BINDING_PREFIXES[model_type]}_aliases": {"best": "other"}})++ pinned_model = _pinned_model(mocker=mocker, model_deck=model_deck_with_alias, model_type=model_type, choice="@best")++ assert pinned_model == "other"diff --git a/tests/unit/pipelex/pipeline/test_resolve_model_specs.py b/tests/unit/pipelex/pipeline/test_resolve_model_specs.pynew file mode 100644index 000000000..ebb21746a--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_resolve_model_specs.py@@ -0,0 +1,84 @@+"""`resolve_model_specs`, the models a file-input consumer's reference can be served by, reads a bare name as the run does."""++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.img_gen.img_gen_job_components import Quality+from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoicesDefaults+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_spec_index import ModelSpecIndex+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.pipeline.file_input_consumers import resolve_model_specs+from pipelex.system.runtime import ProblemReaction++_GET_MODEL_DECK_TARGET = "pipelex.pipeline.file_input_consumers.get_model_deck"+++def _model_spec(name: str, model_type: ModelType) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name="test_backend",+ name=name,+ sdk="test_sdk",+ model_type=model_type,+ model_id=f"test_model_{name}",+ costs={CostCategory.INPUT: 0.001, CostCategory.OUTPUT: 0.002},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=1000,+ max_prompt_images=None,+ )+++def _make_deck() -> ModelDeck:+ """A deck where an LLM's name is also an extraction alias and an image-generation waterfall."""+ return ModelDeck(+ inference_models=ModelSpecIndex.make_from_specs(+ model_specs=[+ _model_spec("gpt-4o-mini", ModelType.LLM),+ _model_spec("extract-engine", ModelType.TEXT_EXTRACTOR),+ _model_spec("img-painter", ModelType.IMG_GEN),+ ]+ ),+ llm_default_temperature=0.7,+ llm_choice_defaults=LLMSettingChoicesDefaults(+ default_temperature=0.7,+ for_text=LLMSetting(model="gpt-4o-mini", temperature=0.7),+ for_object=LLMSetting(model="gpt-4o-mini", temperature=0.1),+ ),+ extract_aliases={"gpt-4o-mini": "extract-engine"},+ extract_choice_default="extract-engine",+ img_gen_default_quality=Quality.MEDIUM,+ img_gen_waterfalls={"gpt-4o-mini": ["gpt-4o-mini", "img-painter"]},+ img_gen_choice_default="img-painter",+ search_choice_default="@default-search",+ model_deck_config=ModelDeckConfig(is_model_fallback_enabled=True, missing_presets_reaction=ProblemReaction.NONE),+ )+++class TestResolveModelSpecs:+ @pytest.mark.parametrize(+ ("model_type", "expected_names"),+ [+ pytest.param(ModelType.LLM, ["gpt-4o-mini"], id="the-model-of-its-type"),+ pytest.param(ModelType.TEXT_EXTRACTOR, ["extract-engine"], id="an-alias-of-that-name"),+ pytest.param(ModelType.IMG_GEN, ["img-painter"], id="a-waterfall-of-that-name"),+ pytest.param(ModelType.SEARCH, [], id="nothing-of-that-name"),+ ],+ )+ def test_a_model_served_as_another_type_gives_way_to_an_alias_or_a_waterfall_of_its_name(+ self,+ mocker: MockerFixture,+ model_type: ModelType,+ expected_names: list[str],+ ) -> None:+ model_deck = _make_deck()+ mocker.patch(_GET_MODEL_DECK_TARGET, return_value=model_deck)++ model_specs = resolve_model_specs(model_reference="gpt-4o-mini", model_type=model_type)++ assert [model_spec.name for model_spec in model_specs] == expected_names+ run_model = model_deck.peek_inference_model(model_handle="gpt-4o-mini", model_type=model_type)+ assert (run_model.name if run_model else None) == (expected_names[0] if expected_names else None)