Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Protocol-judgment-category · L-261003-f71db6 (#1323)

Louis Choquelfb57a15f9aa6
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml
index d7042d04c..62d08bbe0 100644
--- a/.drift/acks/cli-docs.toml
+++ b/.drift/acks/cli-docs.toml
@@ -1,8 +1,8 @@
contract = "cli-docs"
-digest = "sha256:485194a5b58846f1599a2f0d12cdd4822eb920e74e971fdbedd04a49f19b871b"
-reviewed_by = "Claude (Opus 5.5)"
-reviewed_at = "2026-10-03T14:26:33Z"
-rationale = "real-catch: check_models now reports a secrets provider that raises on a lookup in the models row instead of failing the command, and quotes every setup failure scrubbed with the log redaction patterns; pipelex/cli/agent_cli/CLAUDE.md said only the secrets_provider and log_sink messages were scrubbed and said nothing of a lookup failure, and now names checks.models for both. docs/tools/cli/ lists doctor in tables only and describes none of its rows, so nothing there went stale."
+digest = "sha256:ad09974548259c9bd0b110aaabf7a1c052d58dde77601936940d6e5f499d55e9"
+reviewed_by = "Louis Choquel"
+reviewed_at = "2026-10-03T15:51:48Z"
+rationale = "clean-pass: the models --type help now names judgment beside the other categories; docs/tools/cli/ and pipelex/cli/agent_cli/CLAUDE.md never list the --type values, they point at pipelex-agent models -t <type>, so nothing there went stale."
[trigger_files]
"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
@@ -34,7 +34,7 @@ rationale = "real-catch: check_models now reports a secrets provider that raises
"pipelex/cli/agent_cli/commands/inputs/pipe_cmd.py" = "blob:d109d3103b2ad851d281a9ef1a91e2822e92926e"
"pipelex/cli/agent_cli/commands/lint_cmd.py" = "blob:91ad9cb4d618ba4c0b7da366cde2f74f38d350df"
"pipelex/cli/agent_cli/commands/migrate_cmd.py" = "blob:ce733b78df03e65c3c8a10d46b27b57c05ec054c"
-"pipelex/cli/agent_cli/commands/models_cmd.py" = "blob:dde4c246a28eba9f54cd582f0b171c9c26194be8"
+"pipelex/cli/agent_cli/commands/models_cmd.py" = "blob:5678c00ab389b0a433c080be3454c7905e569308"
"pipelex/cli/agent_cli/commands/pipe_cmd.py" = "blob:9bf0480fcd1164ec1120b2514724fc0f5fefd30e"
"pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:a1c530d110482485ce462f199b9e79fac29f8896"
"pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index af00afee4..71b74a297 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,12 +4,13 @@
### Added
-- **A `judgment` inference family**: a new family beside `llm`, `img_gen`, `extract`, `search` and `doc_gen`, for models that answer a closed-form question over some material — yes or no, one option out of a declared set, or a position on a declared scale — and report how sure they are. The worker contract is batch-shaped (one state, a map of questions, a map of answers), every measure of uncertainty is optional so a backend that measures none can still serve the family honestly, and usage is reported as real input and output tokens priced per million. The deck gains a `[judgment]` section in a new managed file, `6_judgment_deck.toml`, which `pipelex update` installs, and a deck installed before it keeps loading with no judgment model; it ships with no `choice_default`, because the family's models come with your own key and a default the Pipelex Gateway does not serve would fail every boot, so a judgment names its own model. `pipelex-agent models` lists the new category, and a `judgment` pytest marker joins the inference markers.
+- **A `judgment` inference family**: a new family beside `llm`, `img_gen`, `extract`, `search` and `doc_gen`, for models that answer a closed-form question over some material — yes or no, one option out of a declared set, or a position on a declared scale — and report how sure they are. The worker contract is batch-shaped (one state, a map of questions, a map of answers), every measure of uncertainty is optional so a backend that measures none can still serve the family honestly, and usage is reported as real input and output tokens priced per million. The deck gains a `[judgment]` section in a new managed file, `6_judgment_deck.toml`, which `pipelex update` installs, and a deck installed before it keeps loading with no judgment model; it ships with no `choice_default`, because the family's models come with your own key and a default the Pipelex Gateway does not serve would fail every boot, so a judgment names its own model. `pipelex-agent models` lists the new category, and so does the MTHDS protocol's model list, `GET /v1/models` on the API server, where a judgment preset is an entry typed `judgment` and `?type=judgment` filters on it; a `judgment` pytest marker joins the inference markers.
- **TypeSafe, the judgment family's first backend**: a built-in `typesafe` plugin, installed with the new `pipelex[typesafe]` extra, serves the family from TypeSafe's System One API and is reachable as `@default-judgment`, pinned to the versioned model `jev-1.13.0`. Set `TYPESAFE_API_KEY` to use it: the default routing profile sends `jev-*` models to the backend whenever it is enabled, because the Pipelex Gateway does not serve them. A rating scale longer than ten levels is refused by this backend before any request is made, and a response answered by a model other than the pinned one is reported as a warning.
- **Log sink settings can name secrets**: the values of `headers` under `[runtime.log.otlp]` and `credentials_file_path` under `[runtime.log.gcp]` accept the `${…}` placeholders the inference backends use, resolved through the configured secrets provider, so a collector's token is written `"Bearer ${OTLP_AUTH_TOKEN}"` and the key path `"${GCP_CREDENTIALS_FILE_PATH}"`. The configuration keeps the placeholders, and one that does not resolve stops the boot with `LogSinkVariableError` naming the section, the key and the variable. A header value that holds a line break once resolved, as a secret read from a file often does, stops the boot with `LogSinkHeaderValueError` rather than installing an exporter that refuses every batch. `pipelex doctor` gains a Secrets Provider row, `checks.secrets_provider` in `pipelex-agent doctor`.
### Changed
+- **`mthds` version, MTHDS protocol 0.7.0**: the pin moves to `mthds==0.18.0`, the release that carries protocol 0.7.0, whose model list adds the `judgment` category beside `llm`, `extract`, `img_gen` and `search`, so the runner and `GET /v1/version` report `protocol_version` `0.7.0`. `PipelexModelDeck` inherits the release's open `ModelInfo.type`, typed `ModelCategory | str | None`, though pipelex itself only ever emits a protocol category; everyone downstream inherits that exact `mthds`.
- **The method-app template and the SDK references link `Pipelex/pipelex-sdk`**: the SDKs and the method-app templates now live in the `Pipelex/pipelex-sdk` repository, so the README and the documentation's **Quick Start** link the method-app template to its `method-apps/` directory, the webapp's by-hand commands clone it and copy `method-apps/webapp-js/`, and the `@pipelex/sdk` and `pipelex-sdk` references point at its `js/docs/` and `python/docs/` directories.
- **Boot builds the secrets provider before the log sink, and a sink factory receives it (Breaking)**: a `LogSinkFactoryFn` is now called as `factory(log_config, secrets_provider=…)`, and the plugin API version moves to 5, so a plugin declaring `targets_api = 4` is refused at discovery and is rebuilt against this release. A secrets provider that fails to build now reports on stderr, redacted, rather than through the configured sink.
diff --git a/api/pipelex_api/routes/pipelex/agent/models.py b/api/pipelex_api/routes/pipelex/agent/models.py
index 37b5ca83f..0190710c4 100644
--- a/api/pipelex_api/routes/pipelex/agent/models.py
+++ b/api/pipelex_api/routes/pipelex/agent/models.py
@@ -18,7 +18,7 @@ async def get_models(
request: Request,
model_type: Annotated[
str | None,
- Query(alias="type", description="Filter by model category: llm, extract, img_gen, search. Single value (protocol arity)."),
+ Query(alias="type", description="Filter by model category: llm, extract, img_gen, search, judgment. Single value (protocol arity)."),
] = None,
) -> PipelexModelDeck:
"""List the model deck this runner can route to (MTHDS Protocol `GET /models`).
diff --git a/api/tests/unit/test_build_and_agent_routes.py b/api/tests/unit/test_build_and_agent_routes.py
index e8fd71843..6a05cccd5 100644
--- a/api/tests/unit/test_build_and_agent_routes.py
+++ b/api/tests/unit/test_build_and_agent_routes.py
@@ -163,8 +163,9 @@ class TestBuildAndAgentRoutes:
# per-category payload caused in the SDK — F2) plus this implementation's routing
# extensions, keyed by category (the same alias name exists in several categories —
# a flat map would silently drop entries on collision). The old raw keys (`presets`
- # by category, `success`) are gone. The flat list speaks the protocol's categories only,
- # while the extensions are keyed by every category this runtime serves, `judgment` included.
+ # by category, `success`) are gone. Every entry of the flat list is typed by one of the
+ # protocol's categories, `judgment` among them, and never by a value the protocol does not
+ # define; the extensions are keyed by the categories this runtime serves.
client = _build_client()
response = client.get("/v1/models")
assert response.status_code == 200, response.text
diff --git a/docs/api-server/logging.md b/docs/api-server/logging.md
index 2d71fe4a1..327e941be 100644
--- a/docs/api-server/logging.md
+++ b/docs/api-server/logging.md
@@ -7,7 +7,7 @@ The server writes structured logs: one JSON object per line, on **stderr**, with
An error response produces exactly one line. A caller mistake:
```json
-{"time": "2026-09-18T13:53:17.919Z", "severity": "WARNING", "logger": "pipelex_api.exception_handlers", "message": "API error 422: InvalidModelCategory", "request_id": "01M2TCP02W04RZG6DTM8AR508C", "event": "api_error", "route": "/v1/models", "error_type": "InvalidModelCategory", "error_domain": "input", "retryable": false, "status": 422, "detail": "Invalid model category. Valid values: extract, img_gen, llm, search"}
+{"time": "2026-09-18T13:53:17.919Z", "severity": "WARNING", "logger": "pipelex_api.exception_handlers", "message": "API error 422: InvalidModelCategory", "request_id": "01M2TCP02W04RZG6DTM8AR508C", "event": "api_error", "route": "/v1/models", "error_type": "InvalidModelCategory", "error_domain": "input", "retryable": false, "status": 422, "detail": "Invalid model category. Valid values: extract, img_gen, judgment, llm, search"}
```
A server fault looks the same at `ERROR`, and carries the traceback under an `exception` key.
diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yaml
index 5cebd2da5..8855447a8 100644
--- a/docs/api-server/openapi/pipelex-api.openapi.yaml
+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml
@@ -2,8 +2,8 @@
openapi: 3.1.0
info:
title: Pipelex API
- summary: The source-available Pipelex runner — implements MTHDS Protocol v0.6.0.
- description: 'This server implements the [MTHDS Protocol](https://mthds.ai) v0.6.0 (`POST /execute`, `POST /start`, `POST
+ summary: The source-available Pipelex runner — implements MTHDS Protocol v0.7.0.
+ description: 'This server implements the [MTHDS Protocol](https://mthds.ai) v0.7.0 (`POST /execute`, `POST /start`, `POST
/validate`, `GET /models`, `GET /version` — marked `x-mthds-protocol: true`) plus the Pipelex API extensions: resolve
and codegen (`/resolve`, `/codegen`), build tooling (`/build/*`), and editor tooling (`/lint`, `/format`). Contract layering:
MTHDS Protocol ⊂ Pipelex API (this server) ⊂ Pipelex hosted API. All endpoints are served under the `/v1` base path; every
@@ -1405,9 +1405,9 @@ paths:
anyOf:
- type: string
- type: 'null'
- description: 'Filter by model category: llm, extract, img_gen, search. Single value (protocol arity).'
+ description: 'Filter by model category: llm, extract, img_gen, search, judgment. Single value (protocol arity).'
title: Type
- description: 'Filter by model category: llm, extract, img_gen, search. Single value (protocol arity).'
+ description: 'Filter by model category: llm, extract, img_gen, search, judgment. Single value (protocol arity).'
responses:
'200':
description: Successful Response
@@ -4154,8 +4154,23 @@ components:
- extract
- img_gen
- search
+ - judgment
title: ModelCategory
- description: Model categories accepted by the protocol's `GET /models?type=` filter.
+ description: 'The protocol''s model categories: what a deck entry''s `type` reports and the `GET /models?type=` filter
+ accepts.
+
+
+ A category is a settings family of the language — the inline settings a model
+
+ serves, and so the pipes that can name it: `llm` (`LLMSetting`, named by
+
+ `PipeLLM` and `PipeStructure`), `extract` (`ExtractSetting`, `PipeExtract`),
+
+ `img_gen` (`ImgGenSetting`, `PipeImgGen`), `search` (`SearchSetting`,
+
+ `PipeSearch`) and `judgment` (`JudgmentSetting`, `PipeJudge`). A settings
+
+ family the standard adds brings a category in the next protocol minor.'
ModelInfo:
properties:
name:
@@ -4164,13 +4179,34 @@ components:
type:
anyOf:
- $ref: '#/components/schemas/ModelCategory'
+ - type: string
- type: 'null'
+ title: Type
additionalProperties: true
type: object
required:
- name
title: ModelInfo
- description: One entry of the model deck (`ModelDeck.models[]`) — base fields + extensions.
+ description: 'One entry of the model deck (`ModelDeck.models[]`) — base fields + extensions.
+
+
+ `type` is an open read of the entry''s category, under the protocol''s reader
+
+ rule: "A client reading a model list MUST NOT fail it because an entry carries
+
+ a category it does not recognize; it keeps that entry with its raw value or
+
+ leaves it out." This package keeps it: a category it knows reads as its
+
+ `ModelCategory` member, and one it does not — a category a runner of a later
+
+ protocol minor reports — keeps its raw string, so a consumer handles the
+
+ `str` case rather than the whole deck failing. The union is validated left to
+
+ right on purpose: pydantic''s default smart mode would keep every string,
+
+ known or not, as a plain `str`.'
ModelReference:
type: string
ModelReferenceKind:
diff --git a/docs/contribute/inference-family-surface.md b/docs/contribute/inference-family-surface.md
index 71394eb7e..52f5325c9 100644
--- a/docs/contribute/inference-family-surface.md
+++ b/docs/contribute/inference-family-surface.md
@@ -30,7 +30,7 @@ Most of the list is enforced by the type checker: the matches over the family en
10. **Silent: fakes that stand in for the deck.** Any test double that impersonates `ModelDeck` by attribute — the fake deck in `tests/unit/pipelex/cli/test_agent_models_cmd.py` is one — needs the new family's presets, aliases and waterfalls, or every case using it fails with an `AttributeError` far from the cause.
-11. **Silent: the MTHDS protocol's model categories.** `PipelexMTHDSProtocol.models()` in `pipelex/pipeline/runner.py` types each preset by the protocol's own `ModelCategory`. A family the protocol cannot name is skipped there rather than raised on; the protocol has to learn the member before the family's presets are visible through it.
+11. **The MTHDS protocol's model categories.** `PipelexMTHDSProtocol.models()` in `pipelex/pipeline/runner.py` types each preset by the protocol's own `ModelCategory`, which carries every family this runtime serves, `judgment` included, through two exhaustive matches beside it: `_protocol_category_of` for the deck's entries and `_builder_category_of` for the `?type=` filter. A new member of the builder's `ModelCategory` (step 2) stops the type checker at the first of them, because a new family needs a protocol category too, which the standard adds in a protocol minor for each settings family it adds. Until the protocol defines one, the family's presets go under an extension property `PipelexModelDeck` adds for them, never under a category the protocol does not define, since a runner must not put such a value in a deck entry's `type`.
12. **Silent: a default the gateway does not serve fails every boot.** The model manager checks, at boot, every handle a deck's presets and choice defaults name against the Pipelex Gateway's specs when the active routing profile sends that handle there. A family whose models only a bring-your-own-key backend serves must therefore ship **no** `choice_default` and **no** preset in the kit deck — an alias is fine, because aliases are not walked. Getting this wrong fails the boot of every installation, whether or not it ever uses the family.
diff --git a/pipelex/cli/agent_cli/commands/models_cmd.py b/pipelex/cli/agent_cli/commands/models_cmd.py
index dde4c246a..5678c00ab 100644
--- a/pipelex/cli/agent_cli/commands/models_cmd.py
+++ b/pipelex/cli/agent_cli/commands/models_cmd.py
@@ -13,7 +13,7 @@ from pipelex.pipelex import Pipelex
def agent_models_cmd(
model_type: Annotated[
list[ModelCategory] | None,
- typer.Option("--type", "-t", help="Filter by model category (repeatable): llm, extract, img_gen, search"),
+ typer.Option("--type", "-t", help="Filter by model category (repeatable): llm, extract, img_gen, search, judgment"),
] = None,
backend: Annotated[
str | None,
diff --git a/pipelex/pipeline/runner.py b/pipelex/pipeline/runner.py
index a013df6ff..702226ffb 100644
--- a/pipelex/pipeline/runner.py
+++ b/pipelex/pipeline/runner.py
@@ -67,6 +67,62 @@ class PipelexModelDeck(MthdsModelDeck):
waterfalls: dict[str, dict[str, list[str]]] = Field(default_factory=dict)
+def _protocol_category_of(*, builder_category: ModelCategory) -> MthdsModelCategory:
+ """The protocol's model category a category this runtime serves is listed under in `models`.
+
+ The match is exhaustive on purpose, with no default arm. A runner must never put a value the
+ protocol does not define in a deck entry's `type`, so a family this runtime adds that the
+ protocol has no category for has no arm to write here, and the type checker stops at this
+ function until someone decides where the family goes: its presets would then be reported under
+ an extension property `PipelexModelDeck` adds for them, the way its aliases and waterfalls
+ already travel under the category-keyed routing extensions, and never under an invented
+ category.
+
+ Args:
+ builder_category: A category of the builder's model listing.
+
+ Returns:
+ The protocol category of the same settings family.
+ """
+ match builder_category:
+ case ModelCategory.LLM:
+ return MthdsModelCategory.LLM
+ case ModelCategory.EXTRACT:
+ return MthdsModelCategory.EXTRACT
+ case ModelCategory.IMG_GEN:
+ return MthdsModelCategory.IMG_GEN
+ case ModelCategory.SEARCH:
+ return MthdsModelCategory.SEARCH
+ case ModelCategory.JUDGMENT:
+ return MthdsModelCategory.JUDGMENT
+
+
+def _builder_category_of(*, protocol_category: MthdsModelCategory) -> ModelCategory:
+ """The builder category a protocol `?type=` filter selects.
+
+ Exhaustive like its inverse: a category a later protocol release adds stops the type checker
+ here when the `mthds` pin moves, rather than surfacing as a `ValueError` on the first request
+ that filters on it.
+
+ Args:
+ protocol_category: A category of the protocol's model list.
+
+ Returns:
+ The builder category of the same settings family.
+ """
+ match protocol_category:
+ case MthdsModelCategory.LLM:
+ return ModelCategory.LLM
+ case MthdsModelCategory.EXTRACT:
+ return ModelCategory.EXTRACT
+ case MthdsModelCategory.IMG_GEN:
+ return ModelCategory.IMG_GEN
+ case MthdsModelCategory.SEARCH:
+ return ModelCategory.SEARCH
+ case MthdsModelCategory.JUDGMENT:
+ return ModelCategory.JUDGMENT
+
+
class PipelexVersionInfo(VersionInfo):
"""Pipelex's version handshake — the protocol base plus this
implementation's identification.
@@ -483,22 +539,16 @@ class PipelexMTHDSProtocol(MTHDSProtocol["PipeOutput"]):
PipelexModelDeck with the flat model list and the category-keyed
aliases and routing waterfalls.
"""
- categories = [ModelCategory(category)] if category is not None else None
+ categories = [_builder_category_of(protocol_category=category)] if category is not None else None
deck_raw = list_models(categories=categories)
models: list[MthdsModelInfo] = []
presets_by_category: dict[str, list[dict[str, Any]]] = deck_raw["presets"]
aliases_by_category: dict[str, dict[str, str]] = deck_raw["aliases"]
waterfalls_by_category: dict[str, dict[str, list[str]]] = deck_raw["waterfalls"]
for category_key, category_presets in presets_by_category.items():
- if category_key not in MthdsModelCategory.__members__.values():
- # A category this runtime serves that the standard's flat list has no word for —
- # `judgment` today. Its presets are left out of `models` rather than raising, because
- # one unrepresentable category must not take the whole deck down; its aliases and
- # waterfalls still travel under the category-keyed routing extensions below. This
- # branch disappears when the protocol's `ModelCategory` learns the member.
- continue
+ protocol_category = _protocol_category_of(builder_category=ModelCategory(category_key))
for preset in category_presets:
- models.append(MthdsModelInfo(name=preset["name"], type=MthdsModelCategory(category_key)))
+ models.append(MthdsModelInfo(name=preset["name"], type=protocol_category))
return PipelexModelDeck(models=models, aliases=aliases_by_category, waterfalls=waterfalls_by_category)
@override
diff --git a/pyproject.toml b/pyproject.toml
index 6979d31ab..f4860d31e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,7 +31,7 @@ dependencies = [
"json2html>=1.3.0",
"kajson==0.7.1",
"markdown-it-py[linkify]>=3.0.0", # the Markdown concept, the `markdown` filter and the built-in PDF engine
- "mthds==0.17.0",
+ "mthds==0.18.0",
"networkx>=3.4.2",
"openai>=2.0.0",
"opentelemetry-api>=1.39.0",
diff --git a/tests/unit/pipelex/pipeline/test_runner_protocol_surfaces.py b/tests/unit/pipelex/pipeline/test_runner_protocol_surfaces.py
index e914afb6c..11c20d125 100644
--- a/tests/unit/pipelex/pipeline/test_runner_protocol_surfaces.py
+++ b/tests/unit/pipelex/pipeline/test_runner_protocol_surfaces.py
@@ -98,13 +98,9 @@ class TestRunnerProtocolSurfaces:
assert deck.waterfalls == {"llm": {"main_chain": ["smart_llm", "cheap_llm"]}, "extract": {}}
list_models_mock.assert_called_once_with(categories=None)
- async def test_models_skips_a_category_the_protocol_has_no_word_for(self, mocker: MockerFixture) -> None:
- """A judgment preset must not take the whole deck down.
-
- `judgment` is a category this runtime serves and the standard's flat model list cannot yet
- name, so its presets are left out of `models` rather than raising on the enum; the routing
- extensions, which are keyed by plain strings, still carry it. Delete this test once the
- protocol's `ModelCategory` learns the member.
+ async def test_models_lists_a_judgment_preset_typed_judgment(self, mocker: MockerFixture) -> None:
+ """A judgment preset is an entry of the flat list typed `judgment`, beside the other categories,
+ and its aliases still travel under the category-keyed routing extension.
"""
payload: dict[str, Any] = {
"presets": {"llm": [{"name": "smart_llm"}], "judgment": [{"name": "strict_verdict"}]},
@@ -116,26 +112,56 @@ class TestRunnerProtocolSurfaces:
deck = await runner.models()
- assert [(model_info.name, model_info.type) for model_info in deck.models] == [("smart_llm", MthdsModelCategory.LLM)]
+ assert [(model_info.name, model_info.type) for model_info in deck.models] == [
+ ("smart_llm", MthdsModelCategory.LLM),
+ ("strict_verdict", MthdsModelCategory.JUDGMENT),
+ ]
+ assert deck.model_dump(mode="json")["models"][1] == {"name": "strict_verdict", "type": "judgment"}
assert deck.aliases["judgment"] == {"default-judgment": "strict_verdict"}
- async def test_models_category_filter_translates_to_builder_enum(self, mocker: MockerFixture) -> None:
- """A protocol-level category filter reaches list_models as the builder's enum."""
- llm_only_payload: dict[str, Any] = {
- "presets": {"llm": [{"name": "smart_llm"}]},
- "aliases": {"llm": {"best": "smart_llm"}},
- "waterfalls": {"llm": {}},
+ @pytest.mark.parametrize("builder_category", list(ModelCategory))
+ async def test_models_types_every_builder_category_by_the_protocol_member_of_its_name(
+ self,
+ mocker: MockerFixture,
+ builder_category: ModelCategory,
+ ) -> None:
+ """Every category the builder serves has a protocol category of the same name, so none of
+ its presets is left out of the flat list or typed by an invented value.
+ """
+ payload: dict[str, Any] = {
+ "presets": {builder_category: [{"name": "some_preset"}]},
+ "aliases": {builder_category: {}},
+ "waterfalls": {builder_category: {}},
+ }
+ mocker.patch("pipelex.pipeline.runner.list_models", return_value=payload)
+ runner = PipelexMTHDSProtocol()
+
+ deck = await runner.models()
+
+ assert [(model_info.name, model_info.type) for model_info in deck.models] == [("some_preset", MthdsModelCategory(builder_category))]
+
+ @pytest.mark.parametrize("protocol_category", list(MthdsModelCategory))
+ async def test_models_category_filter_translates_to_builder_enum(
+ self,
+ mocker: MockerFixture,
+ protocol_category: MthdsModelCategory,
+ ) -> None:
+ """Each protocol-level category filter reaches list_models as the builder's member of the same name."""
+ one_category_payload: dict[str, Any] = {
+ "presets": {protocol_category: [{"name": "some_preset"}]},
+ "aliases": {protocol_category: {"best": "some_preset"}},
+ "waterfalls": {protocol_category: {}},
}
- list_models_mock = mocker.patch("pipelex.pipeline.runner.list_models", return_value=llm_only_payload)
+ list_models_mock = mocker.patch("pipelex.pipeline.runner.list_models", return_value=one_category_payload)
runner = PipelexMTHDSProtocol()
- deck = await runner.models(category=MthdsModelCategory.LLM)
+ deck = await runner.models(category=protocol_category)
- list_models_mock.assert_called_once_with(categories=[ModelCategory.LLM])
+ list_models_mock.assert_called_once_with(categories=[ModelCategory(protocol_category)])
assert isinstance(deck, PipelexModelDeck)
- assert [(model_info.name, model_info.type) for model_info in deck.models] == [("smart_llm", MthdsModelCategory.LLM)]
- assert deck.aliases == {"llm": {"best": "smart_llm"}}
- assert deck.waterfalls == {"llm": {}}
+ assert [(model_info.name, model_info.type) for model_info in deck.models] == [("some_preset", protocol_category)]
+ assert deck.aliases == {protocol_category: {"best": "some_preset"}}
+ assert deck.waterfalls == {protocol_category: {}}
async def test_version_reports_installed_distribution(self, mocker: MockerFixture) -> None:
"""All three implementation version fields carry the installed pipelex version,
diff --git a/uv.lock b/uv.lock
index 69634b3b5..f8c980b0e 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 3
+revision = 5
requires-python = ">=3.11, <3.15"
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'darwin'",
@@ -2738,7 +2738,7 @@ wheels = [
[[package]]
name = "mthds"
-version = "0.17.0"
+version = "0.18.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "httpx" },
@@ -2747,9 +2747,9 @@ dependencies = [
{ name = "tomlkit" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/3a/51/e68684d8bea26098f6a03339f97084038dba54b778ad7d55a07b73763e38/mthds-0.17.0.tar.gz", hash = "sha256:7515dea7a014ed55416e428c29172a7482d674a12bad73d1222bc5090d4c669c", size = 276301, upload-time = "2026-09-27T13:08:43.234Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/48/ba/a9b2a7902ccea184c4589f7c9eedeef66aa46a9bd1c8f099a7240669f833/mthds-0.18.0.tar.gz", hash = "sha256:589dfde53672966030b454f099979f91dcc1bd67958585c46ed6c8f89bf0e08c", size = 275816, upload-time = "2026-10-03T15:44:22.98Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/4e/e2/dc57d0a6370546c2ef1563eacfdb5037c6f5d569c23f407d73acb0473c2e/mthds-0.17.0-py3-none-any.whl", hash = "sha256:cfd4530eb7760490489680edbbbaba448150086ac4b122ee272a64f271785ec4", size = 110748, upload-time = "2026-09-27T13:08:41.722Z" },
+ { url = "https://files.pythonhosted.org/packages/44/b0/9868dba5ca03b23ba54b7636ebd194e8c16b2eb90a5631f8ad2cb19a5947/mthds-0.18.0-py3-none-any.whl", hash = "sha256:a2778e43b943b9aa3d3ab4f0422d57a6dc1245abe4ce84435c821f26a4d3f599", size = 111480, upload-time = "2026-10-03T15:44:21.441Z" },
]
[[package]]
@@ -3707,7 +3707,7 @@ requires-dist = [
{ name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = ">=1.1.0" },
{ name = "mkdocs-redirects", marker = "extra == 'docs'", specifier = "==1.2.2" },
{ name = "moto", extras = ["dynamodb", "s3"], marker = "extra == 'dev'", specifier = ">=5.0.0" },
- { name = "mthds", specifier = "==0.17.0" },
+ { name = "mthds", specifier = "==0.18.0" },
{ name = "mypy", marker = "extra == 'dev'", specifier = "==1.19.1" },
{ name = "networkx", specifier = ">=3.4.2" },
{ name = "openai", specifier = ">=2.0.0" },