Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/PipeJudge · L-260919-406599 (#1325)
Louis Choquel40ec3a10f6fa
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.tomlindex 62d08bbe0..74c132367 100644--- a/.drift/acks/cli-docs.toml+++ b/.drift/acks/cli-docs.toml@@ -1,8 +1,8 @@contract = "cli-docs"-digest = "sha256:ad09974548259c9bd0b110aaabf7a1c052d58dde77601936940d6e5f499d55e9"+digest = "sha256:cadd09e33b7f32258183fed5533b8ca706a3254088eb806016991feef15abb4b"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-03T15:51:48Z"-rationale = "clean-pass: the models --type help now names judgment beside the other categories; docs/tools/cli/ and pipelex/cli/agent_cli/CLAUDE.md never list the --type values, they point at pipelex-agent models -t <type>, so nothing there went stale."+reviewed_at = "2026-10-04T00:24:06Z"+rationale = "clean-pass: pipe_cmd.py gains the PipeJudge emission branch writing question, model, options, levels, criteria and threshold; docs/tools/cli/ and pipelex/cli/agent_cli/CLAUDE.md describe the pipe command without listing pipe types or their fields, so nothing there went stale."[trigger_files]"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"@@ -35,7 +35,7 @@ rationale = "clean-pass: the models --type help now names judgment beside the ot"pipelex/cli/agent_cli/commands/lint_cmd.py" = "blob:91ad9cb4d618ba4c0b7da366cde2f74f38d350df""pipelex/cli/agent_cli/commands/migrate_cmd.py" = "blob:ce733b78df03e65c3c8a10d46b27b57c05ec054c""pipelex/cli/agent_cli/commands/models_cmd.py" = "blob:5678c00ab389b0a433c080be3454c7905e569308"-"pipelex/cli/agent_cli/commands/pipe_cmd.py" = "blob:9bf0480fcd1164ec1120b2514724fc0f5fefd30e"+"pipelex/cli/agent_cli/commands/pipe_cmd.py" = "blob:b040d1f71b1f3e0a1e2d6c03e8fc9929431e01de""pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:a1c530d110482485ce462f199b9e79fac29f8896""pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/run/_output_helpers.py" = "blob:40ed1700e0d5e86d7696eadd7f9887baeb8059f6"diff --git a/.drift/acks/hub-layering-convention.toml b/.drift/acks/hub-layering-convention.tomlindex 97994d9b0..e059e8372 100644--- a/.drift/acks/hub-layering-convention.toml+++ b/.drift/acks/hub-layering-convention.toml@@ -1,10 +1,10 @@contract = "hub-layering-convention"-digest = "sha256:75768fa677bc8550119a8b20ce80c22e5edfa1040eee6b32fe574f7ae2019435"+digest = "sha256:644cc1387406a8e2c330a76f4139ec25318bd56a7d78caa46f9bb54f2bf7b682"reviewed_by = "Louis Choquel"-reviewed_at = "2026-09-30T00:36:45Z"-rationale = "The document renderer registry is removed from the runtime hub again, since document engines register as inference backends of the doc_gen family, so runtime_hub.py and docs/contribute/hub-layering.md are both back to dev's state. No layer placement changed."+reviewed_at = "2026-10-04T00:24:05Z"+rationale = "clean-pass: runtime_hub.py changed only in a docstring, which adds PipeJudge to the list of inference operators resolving the content generator; no layer placement or hub member moved, so docs/contribute/hub-layering.md is unaffected."[trigger_files]"pipelex/cli/dev_cli/commands/hub_layering_guard.py" = "blob:2dbadcbab7d44364a7b5f1d16268b23b26700a00""pipelex/interpreter_hub.py" = "blob:327e75d0fbb5bdf46f6cbf06024f83840f0106d0"-"pipelex/runtime_hub.py" = "blob:48caf3ba024b97e112aca1a8c3db5e5d2f17b4cd"+"pipelex/runtime_hub.py" = "blob:b6dcef79b73d5ce05669c478bff9dc4b367696e5"diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.tomlindex 29f07a347..31da09641 100644--- a/.drift/acks/pipelex-kernel-docs.toml+++ b/.drift/acks/pipelex-kernel-docs.toml@@ -1,8 +1,8 @@contract = "pipelex-kernel-docs"-digest = "sha256:3fde64b7f78e5a662bebb1f86757336c3dc437bee2e48dfa9d25b4d74865df8c"+digest = "sha256:7dc6488e6ce37ae17d2cc40cefa51884730086a4a0d7cc766128d2b108af02ae"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-03T09:31:38Z"-rationale = "clean-pass: assemble_llm_prompt now passes each registry image's mime_type into its prompt image, the way documents already carried theirs; the kernel doc describes the assemble functions' responsibilities and signatures, which are unchanged, so no doc change is needed."+reviewed_at = "2026-10-04T02:27:50Z"+rationale = "real-catch: build_judgment_material now reads a dotted input as the value at its path, keyed by its full name, as MTHDS allows; the page states it."[trigger_files]"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"@@ -16,6 +16,8 @@ rationale = "clean-pass: assemble_llm_prompt now passes each registry image's mi"pipelex/kernel/img_gen_ops.py" = "blob:b85360f6c118e93be53bf8d1a04504ccb7552469""pipelex/kernel/img_gen_prompt.py" = "blob:e4f72040561929aa0d96ad5f00b1c4af652d3f77""pipelex/kernel/img_gen_results.py" = "blob:46b4ca11c70d906ae6d58ce66461f9b3414f1bfa"+"pipelex/kernel/judgment_ops.py" = "blob:b205e52e30b55c50e215d5fa2fd4db94ededf347"+"pipelex/kernel/judgment_results.py" = "blob:40ba1258cc81d34df3e427d5511ccd40c6edf789""pipelex/kernel/llm_ops.py" = "blob:64a6d3d77a165f4f70614ced69876f56d873a8c7""pipelex/kernel/llm_prompt_content.py" = "blob:3fade4a660226e63ad4f916952ee37d34e7d2bbd""pipelex/kernel/llm_results.py" = "blob:6e3b8a2878b5bb284ece5fac10262fb59ac2029d"@@ -25,4 +27,4 @@ rationale = "clean-pass: assemble_llm_prompt now passes each registry image's mi"pipelex/kernel/search_ops.py" = "blob:f9c1dc09dd3beaf1a54ac8e3774db1d510cbdaa3""pipelex/kernel/search_results.py" = "blob:101b70dc96cc5ee1721c9f6b1cb725a5ca78ff70""pipelex/kernel/templating_style_ops.py" = "blob:395473806d600dfc4f124c07f2afebb951fc086b"-"tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:5ba8a8315cd5f8ac3b1f57171be46cc8f0598497"+"tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:21dbb38c9b38a051c00604ff4abd3cf715a29b92"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 03fd88197..491fa4afa 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,8 @@### Added+- **`PipeJudge`, an operator asking a judgment model one closed question about its inputs**: a step declaring neither `options` nor `levels` asks a yes/no question, which may carry `criteria` and a `threshold`, and outputs a `YesNo`; one declaring `options` outputs a `Choice`; one declaring `levels` outputs a `Rating`, each verdict carrying the probabilities the model measured. Every declared input is sent to the model as material whether or not the `question` reads it, and `prompt` is read as a synonym of `question`. A step is refused when the method loads if it names no model while the deck names no default for judgments, if its output disagrees with its kind of question (`Dynamic` included), if an input name reaches into a field with a dot, or if it has an image or a document input its model does not read, and a document a run is given in a format its model does not read is refused before the run starts. The builder and `pipelex-agent pipe` write it with `question`, and the MTHDS Test Corpus gains `operator_judge_urgent_message` and the `operator.judge` tag.+- **The judgment family carries files**: `JudgmentJob` and `JudgmentAssignment` carry `images` and `documents` keyed by input name, a scoped run's read authorization sees them, and a judgment worker refuses files its model does not read with the new `JudgmentCapabilityError` before calling the provider. The per-file format checks a model's files go through moved to `pipelex/cogt/inference/prompt_file_checks.py`, which the LLM family now shares.- **A `judgment` inference family**: a new family beside `llm`, `img_gen`, `extract`, `search` and `doc_gen`, for models that answer a closed-form question over some material — yes or no, one option out of a declared set, or a position on a declared scale — and report how sure they are. The worker contract is batch-shaped (one state, a map of questions, a map of answers), every measure of uncertainty is optional so a backend that measures none can still serve the family honestly, and usage is reported as real input and output tokens priced per million. The deck gains a `[judgment]` section in a new managed file, `6_judgment_deck.toml`, which `pipelex update` installs, and a deck installed before it keeps loading with no judgment model; it ships with no `choice_default`, because the family's models come with your own key and a default the Pipelex Gateway does not serve would fail every boot, so a judgment names its own model. `pipelex-agent models` lists the new category, and so does the MTHDS protocol's model list, `GET /v1/models` on the API server, where a judgment preset is an entry typed `judgment` and `?type=judgment` filters on it; a `judgment` pytest marker joins the inference markers.- **TypeSafe, the judgment family's first backend**: a built-in `typesafe` plugin, installed with the new `pipelex[typesafe]` extra, serves the family from TypeSafe's System One API and is reachable as `@default-judgment`, pinned to the versioned model `jev-1.13.0`. Set `TYPESAFE_API_KEY` to use it: the default routing profile sends `jev-*` models to the backend whenever it is enabled, because the Pipelex Gateway does not serve them. A rating scale longer than ten levels is refused by this backend before any request is made, and a response answered by a model other than the pinned one is reported as a warning.- **Log sink settings can name secrets**: the values of `headers` under `[runtime.log.otlp]` and `credentials_file_path` under `[runtime.log.gcp]` accept the `${…}` placeholders the inference backends use, resolved through the configured secrets provider, so a collector's token is written `"Bearer ${OTLP_AUTH_TOKEN}"` and the key path `"${GCP_CREDENTIALS_FILE_PATH}"`. The configuration keeps the placeholders, and one that does not resolve stops the boot with `LogSinkVariableError` naming the section, the key and the variable. A header value that holds a line break once resolved, as a secret read from a file often does, stops the boot with `LogSinkHeaderValueError` rather than installing an exporter that refuses every batch. `pipelex doctor` gains a Secrets Provider row, `checks.secrets_provider` in `pipelex-agent doctor`.@@ -17,7 +19,7 @@### Fixed-- **A bare value is refused where it reads as a concept the input does not accept (Breaking)**: an input declared as a container native (`Html`, `Page`, `TextAndImages`, `SearchResult`, `Composite`), as `Choice` or `Rating`, or as a concept refining one, built a bare string as a `native.Text` and handed that text on, so the step reading the input met a text where it declared another concept. The value is now refused with `StructureValidationError` naming the concept it read as; send it in its `{concept, content}` envelope. The concept's own content sent without that envelope, such as `{"choice": "billing"}`, is refused with the same advice, where it was told to redeclare the input as `JSON`, and so is a bare level or key at a `Rating` or `Choice` input, such as `2`, where it was told to redeclare the input as `Anything`. A `Dynamic` input still keeps whatever its value reads as.+- **A bare value is refused where it reads as a concept the input does not accept (Breaking)**: an input declared as a container native (`Html`, `Page`, `TextAndImages`, `SearchResult`, `Composite`), as `Choice` or `Rating`, or as a concept refining one, built a bare string as a `native.Text` and handed that text on, so the step reading the input met a text where it declared another concept. The value is now refused with `StructureValidationError` naming the concept it read as; send it in its `{concept, content}` envelope. The concept's own content sent without that envelope, such as `{"choice": "billing"}`, is refused with the same advice, where it was told to redeclare the input as `JSON`, and so is a bare number or boolean at a `Rating` or `Choice` input, such as a level `2` sent alone, where it was told to redeclare the input as `Anything`. A `Dynamic` input still keeps whatever its value reads as.- **A name a template sets is read where Jinja binds it (Breaking)**: the input check of `PipeLLM`, `PipeCompose`, `PipeSearch`, `PipeImgGen` and `PipeDocGen`, `PipeDocGen`'s field check and the optional-input guard now treat a `{% set %}` name as the template's own inside the loop, macro or block that sets it and after an `{% if %}` every branch of which sets it, so a template setting a name in a loop body or in each branch of an `if` and reading it there is no longer refused as reading an undeclared input. A name only some branches set is still read from the input of that name. Newly refused: an input declared only so that such a set would load, which is now never read, a macro called before its `{% macro %}`, an optional input read unguarded in a macro argument's default, and a name a loop or a macro binds read inside a `{% block %}` without `scoped`, which Jinja reads from the input of that name.- **File inputs are checked against the models that read them (Breaking)**: run setup now identifies every image and document input from its first bytes, refuses an `Image` input holding something other than an image with `PipelineInputNotAnImageError`, and refuses a document certain to reach a `PipeExtract` or `PipeLLM` step whose model does not read its format with `PipelineInputFormatUnsupportedError`, both 422 input errors naming the input, the step, the model and the formats it reads; a step reached only through a condition refuses the file itself with `ExtractInputFormatError` or `PromptDocumentFormatError`. A `PipeLLM` document its model does not read now raises `PromptDocumentFormatError` where it raised `LLMCapabilityError`, so a consumer branching on `error_type` must follow. `docling-extract-text` now declares the Word, PowerPoint, Excel, HTML, Markdown, CSV, plain-text, WebVTT and email files it already extracted, which `pipelex update` installs.- **Markdown with unclosed emphasis is converted in memory proportional to its length**: the Markdown parser behind the `markdown` filter, the `Markdown` concept's HTML and the built-in PDF engine held memory growing with the square of the text when emphasis or strikethrough markers were left unmatched, about 1.5 GB for 64,000 characters of `**a `; it now joins the text they leave behind in one pass, and renders the same HTML.diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yamlindex 8855447a8..edb78691c 100644--- a/docs/api-server/openapi/pipelex-api.openapi.yaml+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml@@ -3828,6 +3828,42 @@ components:has no effect on this arm (unlike `rendered_markdown`, which rides both arms because failuretext is exactly what a human surface wants).'+ JudgeYesNoCriteria:+ properties:+ 'yes':+ anyOf:+ - type: string+ - type: 'null'+ title: 'Yes'+ 'no':+ anyOf:+ - type: string+ - type: 'null'+ title: 'No'+ additionalProperties: false+ type: object+ title: JudgeYesNoCriteria+ description: 'What a yes and a no mean, for a yes/no question. Closed: a key other than `yes` or `no` is refused.+++ The judging vendor silently ignores a criterion key it does not know, so this model is the only++ place a typo in one is caught.'+ JudgmentSetting:+ properties:+ model:+ type: string+ title: Model+ description:+ anyOf:+ - type: string+ - type: 'null'+ title: Description+ additionalProperties: false+ type: object+ required:+ - model+ title: JudgmentSettingLLMSetting:properties:model:@@ -5704,6 +5740,96 @@ components:with no inputs maps to `{"fields": []}` — an empty form is a valid form, not an omitted entry.Closed shape (`extra="forbid"`).'+ PipeJudgeBlueprint:+ oneOf:+ - required:+ - question+ - required:+ - prompt+ properties:+ source:+ anyOf:+ - type: string+ - type: 'null'+ title: Source+ pipe_category:+ type: string+ const: PipeOperator+ title: Pipe Category+ default: PipeOperator+ type:+ type: string+ const: PipeJudge+ title: Type+ default: PipeJudge+ description:+ type: string+ title: Description+ inputs:+ anyOf:+ - additionalProperties:+ anyOf:+ - type: string+ - $ref: '#/components/schemas/InputSlotBlueprint'+ type: object+ - type: 'null'+ title: Inputs+ output:+ type: string+ title: Output+ question:+ type: string+ title: Question+ model:+ anyOf:+ - $ref: '#/components/schemas/JudgmentSetting'+ - type: string+ - $ref: '#/components/schemas/ModelReference'+ - type: 'null'+ title: Model+ options:+ anyOf:+ - additionalProperties:+ type: string+ type: object+ - type: 'null'+ title: Options+ levels:+ anyOf:+ - items:+ type: string+ type: array+ - type: 'null'+ title: Levels+ criteria:+ anyOf:+ - $ref: '#/components/schemas/JudgeYesNoCriteria'+ - type: 'null'+ threshold:+ anyOf:+ - type: number+ - type: 'null'+ title: Threshold+ prompt:+ type: string+ title: Prompt+ description: A synonym of `question`, read exactly as it.+ additionalProperties: false+ type: object+ required:+ - description+ - output+ title: PipeJudgeBlueprint+ description: 'Asks a judging model one closed question about its inputs: yes/no, a choice, or a rating.+++ The kind is decided by which of `options` and `levels` the pipe declares, never by a field of its++ own. `prompt` is accepted in place of `question` and read exactly as it, so nothing after parsing++ sees the synonym. Every declared input is material to judge, so an input the question never names++ is still read; only a variable the question names must be declared.'PipeLLMBlueprint:properties:source:@@ -6201,6 +6327,7 @@ components:- PipeSearch- PipeStructure- PipeDocGen+ - PipeJudge- PipeBatch- PipeCondition- PipeParallel@@ -6374,6 +6501,7 @@ components:- $ref: '#/components/schemas/PipeLLMBlueprint'- $ref: '#/components/schemas/PipeExtractBlueprint'- $ref: '#/components/schemas/PipeSearchBlueprint'+ - $ref: '#/components/schemas/PipeJudgeBlueprint'- $ref: '#/components/schemas/PipeStructureBlueprint'- $ref: '#/components/schemas/PipeDocGenBlueprint'- $ref: '#/components/schemas/PipeBatchBlueprint'@@ -6391,6 +6519,7 @@ components:PipeExtract: '#/components/schemas/PipeExtractBlueprint'PipeFunc: '#/components/schemas/PipeFuncBlueprint'PipeImgGen: '#/components/schemas/PipeImgGenBlueprint'+ PipeJudge: '#/components/schemas/PipeJudgeBlueprint'PipeLLM: '#/components/schemas/PipeLLMBlueprint'PipeParallel: '#/components/schemas/PipeParallelBlueprint'PipeSearch: '#/components/schemas/PipeSearchBlueprint'diff --git a/docs/building-methods/pipes/index.md b/docs/building-methods/pipes/index.mdindex 5260d883f..84a4cb4bf 100644--- a/docs/building-methods/pipes/index.md+++ b/docs/building-methods/pipes/index.md@@ -12,7 +12,7 @@ This guide provides an overview of how to design your pipelines.A pipeline is composed of pipes. There are two fundamental types of pipes you will use to build your methods:-* **[Pipe Operators](./pipe-operators/index.md)**: These are the "workers" of your pipeline. They perform concrete actions like calling an LLM (`PipeLLM`), extracting text from a document (`PipeExtract`), searching the web (`PipeSearch`), or running a Python function (`PipeFunc`). Each operator is a specialized tool designed for a specific task.+* **[Pipe Operators](./pipe-operators/index.md)**: These are the "workers" of your pipeline. They perform concrete actions like calling an LLM (`PipeLLM`), extracting text from a document (`PipeExtract`), searching the web (`PipeSearch`), answering a closed question with a verdict (`PipeJudge`), or running a Python function (`PipeFunc`). Each operator is a specialized tool designed for a specific task.* **[Pipe Controllers](./pipe-controllers/index.md)**: These are the "managers" of your pipeline. They don't perform tasks themselves but orchestrate the execution flow of other pipes. They define the logic of your method, such as running pipes in sequence (`PipeSequence`), in parallel (`PipeParallel`), or based on a condition (`PipeCondition`).A third, design-time-only kind of pipe — **[Signature Pipes](./signature-pipes.md)** — lets you stub a pipe's contract (inputs and output) by declaring it with no `type` and no implementation: a typeless, contract-only pipe *is* a signature. Signatures dry-run as mocks under `--allow-signatures`, so you can sketch a pipeline top-down before writing each operator. Strict validation refuses pipelines that still depend on a signature.diff --git a/docs/building-methods/pipes/pipe-operators/PipeJudge.md b/docs/building-methods/pipes/pipe-operators/PipeJudge.mdnew file mode 100644index 000000000..b49117800--- /dev/null+++ b/docs/building-methods/pipes/pipe-operators/PipeJudge.md@@ -0,0 +1,111 @@+---+description: "PipeJudge asks a judgment model one closed question about its inputs and returns a verdict with a measure of how sure it is: yes or no, one option out of a set, or a level on a scale."+---++# PipeJudge++The `PipeJudge` operator asks a judgment model one closed question about its inputs, and stores the verdict it answers with: a `YesNo`, a `Choice` or a `Rating`, each carrying the probabilities the model measured. A judgment model is built for this kind of question: it answers with a verdict and the probabilities it measured, where a language model asked the same question writes its answer out as text.++## How it works++The fields a `PipeJudge` declares decide the kind of question it asks, and the kind decides its output:++| Declares | Asks | Output |+| -------- | ---- | ------ |+| neither `options` nor `levels` | a yes/no question, which may carry `criteria` and a `threshold` | `YesNo` |+| `options` | which option fits | `Choice` |+| `levels` | which level fits, the levels being ordered from lowest to highest | `Rating` |++The output must be that native or a concept refining it, and `Dynamic` is neither: a pipe declaring `options` whose output is `YesNo` is refused when the method loads, with a message naming both sides. A `PipeJudge` produces one verdict, so its output cannot be a list; to judge every item of a list, map the `PipeJudge` over it with a [`PipeBatch`](../pipe-controllers/PipeBatch.md).++**Every input is material to judge.** The model receives each input the pipe declares, by name, whether or not the question mentions it: a `Text` as its text, a `Number` as its number, a `YesNo` as its boolean, a list as a list, and a structured concept as its fields. An optional input that is absent is left out. Each input is sent whole, so an input name cannot reach into a field with a dot, as `invoice.total` would: declare `invoice` instead. So, unlike the other operators, a `PipeJudge` does not have to read its inputs in its `question`; it may still interpolate one, which suits a short parameter such as `"Is the message about $topic?"`, and every variable the question reads must be a declared input.++**Images and documents are sent as files.** An `Image` or a `Document` input, or a list of either, reaches the model as a file rather than as text. Whether a model reads files is stated in its model configuration, and a `PipeJudge` with an image or a document input is refused when the method loads if its model does not read that kind of file, with a message naming the input and what the model reads. An input declared `Dynamic` names no kind of value, so a file it holds is checked when the step runs instead. A document given to the run in a format the model does not read is refused when the run starts, before any step spends anything. TypeSafe's model reads text only, so turn a document into text with a [`PipeExtract`](PipeExtract.md) step first.++**The verdict.** A yes/no question's verdict follows from its probability: yes when the probability of yes is at least the `threshold`, which defaults to `0.5`. A model that reports no probability gives its own verdict, and a declared threshold then cannot apply, which the run logs as a warning. A choice's verdict is the key of the option chosen, with the model's `confidence` and the `probabilities` of every option when it reports them. A rating's verdict is the index of the level chosen, `0` being the first level declared, with the model's `confidence`, the `probabilities` of every level keyed by its index written as text, and a continuous `position` on the scale, when it reports them.++The [native concepts page](../../concepts/native-concepts.md) describes the fields of the three verdicts. A downstream pipe reads them like any other fields, for example a [`PipeCondition`](../pipe-controllers/PipeCondition.md) routing on `routing.choice`, or one gating on `approved.probability`.++## Configuration++### The judgment model++No judgment model is served out of the box: the model deck names no default for judgments, so a `PipeJudge` names its model in its `model` field, and a `PipeJudge` that names none is refused when the method loads. The deck's `@default-judgment` alias points at TypeSafe's model, which works once a `TYPESAFE_API_KEY` is set (see [the inference backend configuration](../../../configuration/config-technical/inference-backend-config.md)). To let your pipes leave `model` out, set a `choice_default` under `[judgment]` in your model deck.++### MTHDS Parameters++| Parameter | Type | Description | Required |+| ------------- | ---------- | ----------- | -------- |+| `type` | string | The type of the pipe: `PipeJudge`. | Yes |+| `description` | string | A description of the judgment. | Yes |+| `inputs` | dictionary | The material to judge, as a dictionary mapping input names to concept codes. Every input is sent to the model. | No |+| `output` | string | `YesNo`, `Choice` or `Rating`, according to the kind of question, or a concept refining it. | Yes |+| `model` | string | The judgment model, alias or preset, such as `"@default-judgment"`. Required unless your model deck names a default for judgments. | No |+| `question` | string | The one closed question to ask. It may interpolate an input with the `$` prefix. `prompt` is read as a synonym, but a pipe may not set both. | Yes |+| `options` | dictionary | For a choice question: each key is an option, each value describes it. An empty string leaves an option undescribed. At least two options. | No |+| `levels` | list of strings | For a rating question: the levels from lowest to highest. At least two. | No |+| `criteria` | dictionary | For a yes/no question: what a `yes` and what a `no` mean, either or both. | No |+| `threshold` | number | For a yes/no question: the probability of yes at or above which the verdict is yes, strictly between 0 and 1. Defaults to `0.5`. | No |++### Example: A yes/no question with a threshold++```toml+[pipe.judge_is_urgent]+type = "PipeJudge"+description = "Judges whether a message needs an answer today"+inputs = { message = "Text" }+output = "YesNo"+model = "@default-judgment"+question = "Does this message need an answer today?"+criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }+threshold = 0.7+```++### Example: A choice that routes a ticket++```toml+[pipe.judge_team]+type = "PipeJudge"+description = "Judges which team should handle a support ticket"+inputs = { ticket = "Text" }+output = "Choice"+model = "@default-judgment"+question = "Which team should handle this support ticket?"+options = { billing = "Charges, invoices, refunds and subscriptions", technical = "Errors, outages and features that do not work", other = "" }++[pipe.dispatch_ticket]+type = "PipeCondition"+description = "Sends the ticket to the team the judgment chose"+inputs = { routing = "Choice", ticket = "Text" }+output = "Text"+expression = "routing.choice"+outcomes = { billing = "acknowledge_billing", technical = "acknowledge_technical" }+default_outcome = "acknowledge_other"+```++### Example: A rating on a scale++```toml+[pipe.rate_severity]+type = "PipeJudge"+description = "Rates how severe a bug report is"+inputs = { report = "Text" }+output = "Rating"+model = "@default-judgment"+question = "How severe is the bug this report describes?"+levels = ["Cosmetic: nothing stops working", "Degraded: a workaround exists", "Blocking: users cannot do their work at all"]+```++## Writing good judgments++- **Ask one narrow question per pipe.** A question that bundles two judgments ("is it urgent and about billing?") gets one verdict for both. Split it into two `PipeJudge` steps, and combine their verdicts downstream.+- **Make levels describe situations, not degrees.** "Blocking: users cannot do their work at all" gives the model something to recognize, where "Very severe" only restates the scale. A long scale of thin levels buys resolution the model cannot supply, and TypeSafe's model reads at most ten levels.+- **Give a choice a catch-all option** when the material may fit none of the others, such as `other = ""`, so the model is not forced to pick a wrong one.+- **Leave counting, arithmetic and dates to code.** Whether an invoice is over a sum, or a date is past a deadline, is a comparison a [`PipeFunc`](PipeFunc.md) or a `PipeCondition` expression makes exactly.+- **State the condition directly** in the question rather than through an indirection, and keep interpolated inputs to short parameters: the material reaches the model anyway.++## Related Documentation++- [Judgment Feature](../../../features/judgment.md) - Overview of judgments in a method+- [Native concepts](../../concepts/native-concepts.md) - The `YesNo`, `Choice` and `Rating` verdicts+- [PipeCondition](../pipe-controllers/PipeCondition.md) - Route on a verdictdiff --git a/docs/building-methods/pipes/pipe-operators/index.md b/docs/building-methods/pipes/pipe-operators/index.mdindex 7a3be34d5..198d663dd 100644--- a/docs/building-methods/pipes/pipe-operators/index.md+++ b/docs/building-methods/pipes/pipe-operators/index.md@@ -20,6 +20,7 @@ Here are the primary pipe operators available in Pipelex:- [**`PipeExtract`**](./PipeExtract.md): Performs Optical Character Recognition (OCR) on images and PDF documents to extract text and embedded images, and fetches and extracts content from web pages.- [**`PipeImgGen`**](./PipeImgGen.md): Generates images from a text prompt using models like GPT Image, Flux, or other image generation models.- [**`PipeSearch`**](./PipeSearch.md): Searches the web using a configurable search provider and returns structured results with an answer and source citations.+- [**`PipeJudge`**](./PipeJudge.md): Asks a judgment model one closed question about its inputs, and returns a `YesNo`, a `Choice` or a `Rating` verdict with the probabilities the model measured.- [**`PipeFunc`**](./PipeFunc.md): An escape hatch that allows you to execute any custom Python function, giving you maximum flexibility.- [**`PipeCompose`**](./PipeCompose.md): Composes outputs deterministically from working memory — renders Jinja2 templates for formatted reports or complex prompts, or constructs structured objects by mapping fields from inputs, without an LLM.- [**`PipeDocGen`**](./PipeDocGen.md): Generates a document file from its inputs, calling no AI model: a PDF laid out from a structure or formatted from a Markdown report, and with the Pipelex document generation plugin a PDF from an HTML template, an Excel workbook, a Word document or a PowerPoint deck.@@ -34,6 +35,7 @@ Pipelex provides the following pipe operators:- `PipeFunc`: For executing custom functions- `PipeImgGen`: For AI-powered image generation- `PipeSearch`: For web search with structured results+- `PipeJudge`: For a closed question answered with a verdict and its probabilities- `PipeStructure`: For turning free-form text into structured data- `PipeDocGen`: For generating a document file, such as a PDF, from structured data@@ -116,3 +118,13 @@ Searches the web and returns structured results with sources.- Structured results with answer and source citations- Dynamic prompt templates with `$variable` syntax- Standard and deep search models++## PipeJudge++Asks a judgment model one closed question about its inputs.++### Key Features++- Yes/no, choice and rating questions, the kind read from the fields the pipe declares+- Verdicts carrying the probabilities the model measured, and a threshold for a yes/no+- Every input sent to the model as material, images and documents as files for a model that reads themdiff --git a/docs/building-methods/pipes/understanding-optionality.md b/docs/building-methods/pipes/understanding-optionality.mdindex f4b93950a..312b4eedd 100644--- a/docs/building-methods/pipes/understanding-optionality.md+++ b/docs/building-methods/pipes/understanding-optionality.md@@ -78,7 +78,7 @@ Write a brief about this topic: $topic- **A presence block** — `{% if enriched %} ... {% endif %}` (any reference inside the block is guarded, including `$enriched` and attribute access).- **An inline presence conditional** — `{{ enriched.text if enriched else "no enrichment" }}`.-The guard-lint covers every pipe that renders authored templates over its inputs: PipeLLM (prompt and system prompt), PipeCompose (template mode), PipeCondition (expression), PipeSearch, and PipeImgGen.+The guard-lint covers every pipe that renders authored templates over its inputs: PipeLLM (prompt and system prompt), PipeCompose (template mode), PipeCondition (expression), PipeSearch, PipeJudge (question), and PipeImgGen.## Controllers Under Absencediff --git a/docs/building-methods/templating-style.md b/docs/building-methods/templating-style.mdindex 800a51324..0c80040c0 100644--- a/docs/building-methods/templating-style.md+++ b/docs/building-methods/templating-style.md@@ -131,7 +131,7 @@ So a pipe with `templating_style = "no_tag"` and one with `$article` instead of## Beyond PipeLLM- **[`PipeCompose`](pipes/pipe-operators/PipeCompose.md)** takes `templating_style` on its `[pipe.name.template]` section, for templates that use the `tag` or `format` filters directly. A compose pipe that declares nothing renders under the same runtime default.-- **`PipeImgGen` and `PipeSearch`** have no authored style of their own — their prompts render under the runtime default. Image and search prompts rarely benefit from tagging, but they do render under a real style rather than an invisible fallback.+- **`PipeImgGen`, `PipeSearch` and `PipeJudge`** have no authored style of their own — their prompts and questions render under the runtime default. Image and search prompts rarely benefit from tagging, but they do render under a real style rather than an invisible fallback.- **Construct-mode template fields** and the built-in structuring prompts likewise resolve a real style, so a template of yours that uses `| tag` behaves the same everywhere.!!! warning "A style is always required, and never invented"diff --git a/docs/contribute/generate-projection-corpus.md b/docs/contribute/generate-projection-corpus.mdindex 738256101..eacca2a37 100644--- a/docs/contribute/generate-projection-corpus.md+++ b/docs/contribute/generate-projection-corpus.md@@ -72,7 +72,7 @@ A key is a whole template and never a slot, and the shaper stops at the first sl## The bundles-`tests/data/input_semantics/` holds the corpus bundles, and the command above names every one of them: a bundle that exists in that directory but is left off the command line does not appear in the capture, and the regeneration silently drops the pipes it contributed. `probe_bundle.mthds` exercises every construct the language accepts and `hinted_bundle.mthds` the intent hints; `output_bundle.mthds` covers the output form specifically, since every bundle before it resolved to `Text`; `scaffold_bundle.mthds` was added for this corpus and covers what the others do not — a text field merely named `url` beside a real file position, an optional nested structure, optional `native.Image` and `native.Document` fields inside a structure, and both a fixed `[N]` and a variable `[]` slot over a structured concept. Its `scaffold_open_natives` pipe holds the natives whose payload shape the descriptor states openly, at slot positions: `native.Dynamic`, `native.Composite` and `native.Anything`, which declare no pinned structure and whose node is therefore `unknown`, beside `native.JSON`, which does declare one — its pinned blueprint carries a required `json_obj`, so its explicit template expands like any other pinned native's, while its compact template is the bare object with `json_obj` unwrapped, and that dict is where `unknown-empty-object` is measured. Its `scaffold_dynamic_prompt` pipe puts a `native.Dynamic` slot at an LLM prompt position under both markers, which the corpus could not carry while the prompt analyzers read a dynamic concept's universal compatibility as identity and demanded an `ImageContent` for it. Its `scaffold_verdict_natives` pipe holds the three verdict natives at slot positions: `native.YesNo`, which a slot reads as a bare boolean, beside `native.Choice` and `native.Rating`, which a slot takes in their envelope. `output_bundle.mthds` carries the other side in `output_native_yes_no`, whose `YesNo` output is the object holding `yes_no` and its optional `probability`, the one native whose output form differs from its input form.+`tests/data/input_semantics/` holds the corpus bundles, and the command above names every one of them: a bundle that exists in that directory but is left off the command line does not appear in the capture, and the regeneration silently drops the pipes it contributed. `probe_bundle.mthds` exercises every construct the language accepts and `hinted_bundle.mthds` the intent hints; `output_bundle.mthds` covers the output form specifically, since every bundle before it resolved to `Text`; `scaffold_bundle.mthds` was added for this corpus and covers what the others do not — a text field merely named `url` beside a real file position, an optional nested structure, optional `native.Image` and `native.Document` fields inside a structure, and both a fixed `[N]` and a variable `[]` slot over a structured concept. Its `scaffold_open_natives` pipe holds the natives whose payload shape the descriptor states openly, at slot positions: `native.Dynamic`, `native.Composite` and `native.Anything`, which declare no pinned structure and whose node is therefore `unknown`, beside `native.JSON`, which does declare one — its pinned blueprint carries a required `json_obj`, so its explicit template expands like any other pinned native's, while its compact template is the bare object with `json_obj` unwrapped, and that dict is where `unknown-empty-object` is measured. Its `scaffold_dynamic_prompt` pipe puts a `native.Dynamic` slot at an LLM prompt position under both markers, which the corpus could not carry while the prompt analyzers read a dynamic concept's universal compatibility as identity and demanded an `ImageContent` for it. Its `scaffold_verdict_natives` pipe holds the verdict natives at slot positions: `native.YesNo`, which a slot reads as a bare boolean, beside `native.Choice` and `native.Rating`, which a slot takes in their envelope. `output_bundle.mthds` carries the other side in `output_native_yes_no`, whose `YesNo` output is the object holding `yes_no` and its optional `probability`, the one native whose output form differs from its input form.`native.Anything`'s compact template keeps its envelope, `{"concept": "native.Anything", "content": {}}`, in the engine and in every projection: its bare placeholder, `{}`, would read as "send an object", while the envelope names the concept that says any value goes. The input shaper takes that envelope back, reading its empty object as a JSON object under the `Anything` concept.diff --git a/docs/contribute/inference-family-surface.md b/docs/contribute/inference-family-surface.mdindex 52f5325c9..30f1b1a25 100644--- a/docs/contribute/inference-family-surface.md+++ b/docs/contribute/inference-family-surface.md@@ -32,7 +32,9 @@ Most of the list is enforced by the type checker: the matches over the family en11. **The MTHDS protocol's model categories.** `PipelexMTHDSProtocol.models()` in `pipelex/pipeline/runner.py` types each preset by the protocol's own `ModelCategory`, which carries every family this runtime serves, `judgment` included, through two exhaustive matches beside it: `_protocol_category_of` for the deck's entries and `_builder_category_of` for the `?type=` filter. A new member of the builder's `ModelCategory` (step 2) stops the type checker at the first of them, because a new family needs a protocol category too, which the standard adds in a protocol minor for each settings family it adds. Until the protocol defines one, the family's presets go under an extension property `PipelexModelDeck` adds for them, never under a category the protocol does not define, since a runner must not put such a value in a deck entry's `type`.-12. **Silent: a default the gateway does not serve fails every boot.** The model manager checks, at boot, every handle a deck's presets and choice defaults name against the Pipelex Gateway's specs when the active routing profile sends that handle there. A family whose models only a bring-your-own-key backend serves must therefore ship **no** `choice_default` and **no** preset in the kit deck — an alias is fine, because aliases are not walked. Getting this wrong fails the boot of every installation, whether or not it ever uses the family.+12. **Files, when the family reads them.** The job and the assignment carry them keyed by input name, as `JudgmentJob` and `JudgmentAssignment` carry `images` and `documents`, and the assignment's `referenced_uris()` returns each one, or a scoped run's read authorization never sees them. The worker's template method checks that the model reads them before its hook runs, raising the family's capability error (`JudgmentWorkerAbstract._check_can_read_files`), and runs the per-file format checks in `pipelex/cogt/inference/prompt_file_checks.py`, which the LLM family shares. What a model reads is the `inputs` of its spec in the backend's model file, so the capability is configuration, and an operator that knows its model at load time refuses a file input there too, as `PipeJudge` does.++13. **Silent: a default the gateway does not serve fails every boot.** The model manager checks, at boot, every handle a deck's presets and choice defaults name against the Pipelex Gateway's specs when the active routing profile sends that handle there. A family whose models only a bring-your-own-key backend serves must therefore ship **no** `choice_default` and **no** preset in the kit deck — an alias is fine, because aliases are not walked. Getting this wrong fails the boot of every installation, whether or not it ever uses the family.## A new backend for a familydiff --git a/docs/errors/authoring-and-language.md b/docs/errors/authoring-and-language.mdindex eedfb7bdb..5acc692d3 100644--- a/docs/errors/authoring-and-language.md+++ b/docs/errors/authoring-and-language.md@@ -82,6 +82,8 @@ own page. Classes are grouped by subsystem.- [`PipeFuncTransportError`](pipe-func-transport-error.md) — Pipe func transport- [`PipeImgGenFactoryError`](pipe-img-gen-factory-error.md) — Pipe img gen factory- [`PipeImgGenRunError`](pipe-img-gen-run-error.md) — Pipe img gen run+- [`PipeJudgeError`](pipe-judge-error.md) — Pipe judge+- [`PipeJudgeInputCapabilityError`](pipe-judge-input-capability-error.md) — Pipe judge input capability- [`PipeLLMFactoryError`](pipe-llm-factory-error.md) — Pipe LLM factory- [`PipeOperatorModelAvailabilityError`](pipe-operator-model-availability-error.md) — Pipe operator model availability- [`PipeSearchError`](pipe-search-error.md) — Pipe searchdiff --git a/docs/errors/inference-and-providers.md b/docs/errors/inference-and-providers.mdindex fdad57de7..2f3f1c4b2 100644--- a/docs/errors/inference-and-providers.md+++ b/docs/errors/inference-and-providers.md@@ -44,8 +44,10 @@ own page. Classes are grouped by subsystem.- [`InferenceBackendLibraryValidationError`](inference-backend-library-validation-error.md) — Inference backend library validation- [`InferenceModelSpecError`](inference-model-spec-error.md) — Inference model spec- [`JudgmentAnswerMismatchError`](judgment-answer-mismatch-error.md) — Judgment answer mismatch+- [`JudgmentCapabilityError`](judgment-capability-error.md) — Judgment capability- [`JudgmentHandleNotFoundError`](judgment-handle-not-found-error.md) — Judgment handle not found- [`JudgmentJobFailureError`](judgment-job-failure-error.md) — Judgment job failure+- [`JudgmentModelMissingError`](judgment-model-missing-error.md) — Judgment model missing- [`JudgmentModelNotFoundError`](judgment-model-not-found-error.md) — Judgment model not found- [`LLMAssignmentError`](llm-assignment-error.md) — LLM assignment- [`LLMCapabilityError`](llm-capability-error.md) — LLM capabilitydiff --git a/docs/errors/judgment-capability-error.md b/docs/errors/judgment-capability-error.mdnew file mode 100644index 000000000..2c05af80a--- /dev/null+++ b/docs/errors/judgment-capability-error.md@@ -0,0 +1,21 @@+---+title: "Judgment capability"+description: "Reference for the `JudgmentCapabilityError` Pipelex error class."+---++<!-- pipelex:generated -->++# Judgment capability++A judgment job carries files its model does not read: images to a model without vision, documents to one that reads none.++| Field | Value |+|---|---|+| `error_type` | `JudgmentCapabilityError` |+| `title` | Judgment capability |+| `type_uri` | `https://docs.pipelex.com/latest/errors/judgment-capability-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`CogtError`](cogt-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/judgment-model-missing-error.md b/docs/errors/judgment-model-missing-error.mdnew file mode 100644index 000000000..6ae365521--- /dev/null+++ b/docs/errors/judgment-model-missing-error.md@@ -0,0 +1,21 @@+---+title: "Judgment model missing"+description: "Reference for the `JudgmentModelMissingError` Pipelex error class."+---++<!-- pipelex:generated -->++# Judgment model missing++A judgment has no model to run on: the step names none, and the model deck names no default.++| Field | Value |+|---|---|+| `error_type` | `JudgmentModelMissingError` |+| `title` | Judgment model missing |+| `type_uri` | `https://docs.pipelex.com/latest/errors/judgment-model-missing-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-judge-error.md b/docs/errors/pipe-judge-error.mdnew file mode 100644index 000000000..b6c17cff4--- /dev/null+++ b/docs/errors/pipe-judge-error.md@@ -0,0 +1,19 @@+---+title: "Pipe judge"+description: "Reference for the `PipeJudgeError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe judge++| Field | Value |+|---|---|+| `error_type` | `PipeJudgeError` |+| `title` | Pipe judge |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-judge-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.pipe_operators.judge.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-judge-input-capability-error.md b/docs/errors/pipe-judge-input-capability-error.mdnew file mode 100644index 000000000..492ef36a4--- /dev/null+++ b/docs/errors/pipe-judge-input-capability-error.md@@ -0,0 +1,21 @@+---+title: "Pipe judge input capability"+description: "Reference for the `PipeJudgeInputCapabilityError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe judge input capability++A `PipeJudge` declares an image or a document input, and the judging model it resolves to does not read one.++| Field | Value |+|---|---|+| `error_type` | `PipeJudgeInputCapabilityError` |+| `title` | Pipe judge input capability |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-judge-input-capability-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.pipe_operators.judge.exceptions` |+| Parent class | [`PipeJudgeError`](pipe-judge-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-operator-model-choice-error.md b/docs/errors/pipe-operator-model-choice-error.mdindex 75e3611a6..3dec7c482 100644--- a/docs/errors/pipe-operator-model-choice-error.md+++ b/docs/errors/pipe-operator-model-choice-error.md@@ -7,7 +7,7 @@ description: "Reference for the `PipeOperatorModelChoiceError` Pipelex error cla# Pipe operator model choice-Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``) when it is built from its blueprint and a model field names a model the model deck does not define. Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``, and a run, which loads its bundle before any pipe runs, refuses the bundle with that same verdict.+Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``, ``PipeJudge``) when it is built from its blueprint and a model field names a model the model deck does not define. Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``, and a run, which loads its bundle before any pipe runs, refuses the bundle with that same verdict.| Field | Value ||---|---|diff --git a/docs/errors/structure-validation-error.md b/docs/errors/structure-validation-error.mdindex 468d17039..c8451b171 100644--- a/docs/errors/structure-validation-error.md+++ b/docs/errors/structure-validation-error.md@@ -11,7 +11,7 @@ A provided input value could not be built as the concept its input declares.Most often the value is the right JSON kind but does not fit the concept's structure: an object missing a required field, a malformed `{"url": ...}` for an image or a document, a string that is not an ISO date for a `Date` input. The message names the input and the declared concept, and ends with the expected shape rendered from the method's signature. The fix is the value.-The other case is a value that an input reading bare values by their own shape has no reading for, or reads as a concept the input does not accept. An input declared `Dynamic`, as one of the container natives (`Html`, `Page`, `TextAndImages`, `SearchResult`, `Composite`), or as a `Choice` or a `Rating`, reads a bare value that way. A bare string reads as a text, which only a `Dynamic` input takes: at a `Choice` input the message is `Input 'team' could not be built as 'native.Choice': you provided a string ("billing"), which reads as 'native.Text', and a 'native.Text' is not a 'native.Choice'.`, and the fix is the value, sent in the expected shape the message ends with. So it is for the input's own content sent without its envelope: `{"choice": "billing"}` at a `Choice` input is refused as `the content of a 'native.Choice' without the envelope that names its concept`, and declaring the input `JSON` instead would discard the concept the method asked for. A bare level or key at a `Rating` or a `Choice` input, such as `2`, is refused the same way, as a value that `takes its value in the envelope that names its concept`, since declaring the input `Anything` would discard the verdict. Some values have no reading at all: a number, or a list of plain objects. The message says so, `Input 'records' could not be built as 'native.Dynamic': you provided a list of 2 item(s), and an input of this concept reads a bare value by its own shape, with no reading for this one.`, and its next step names the declaration that would read the value: `JSON` for an object, `JSON[]` for a list of objects, `Anything` or `Anything[]` for anything else, or a concept with a structure that describes it. The fix is usually the method's declaration rather than the value.+The other case is a value that an input reading bare values by their own shape has no reading for, or reads as a concept the input does not accept. An input declared `Dynamic`, as one of the container natives (`Html`, `Page`, `TextAndImages`, `SearchResult`, `Composite`), or as a `Choice` or a `Rating`, reads a bare value that way. A bare string reads as a text, which only a `Dynamic` input takes: at a `Choice` input the message is `Input 'team' could not be built as 'native.Choice': you provided a string ("billing"), which reads as 'native.Text', and a 'native.Text' is not a 'native.Choice'.`, and the fix is the value, sent in the expected shape the message ends with. So it is for the input's own content sent without its envelope: `{"choice": "billing"}` at a `Choice` input is refused as `the content of a 'native.Choice' without the envelope that names its concept`, and declaring the input `JSON` instead would discard the concept the method asked for. A bare number or boolean at a `Rating` or a `Choice` input, such as a level `2` sent alone, is refused the same way, as a value that `takes its value in the envelope that names its concept`, since declaring the input `Anything` would discard the verdict. Some values have no reading at all: a number, or a list of plain objects. The message says so, `Input 'records' could not be built as 'native.Dynamic': you provided a list of 2 item(s), and an input of this concept reads a bare value by its own shape, with no reading for this one.`, and its next step names the declaration that would read the value: `JSON` for an object, `JSON[]` for a list of objects, `Anything` or `Anything[]` for anything else, or a concept with a structure that describes it. The fix is usually the method's declaration rather than the value.When no declaration reads the value, such as an empty list at a `Html[]` input, a list mixing prebuilt contents of different kinds, or a list holding nulls or nested lists, the message carries the reason itself.diff --git a/docs/features/index.md b/docs/features/index.mdindex 936749e6c..a2fca32a4 100644--- a/docs/features/index.md+++ b/docs/features/index.md@@ -25,7 +25,7 @@ This section covers all Pipelex capabilities, organized by area.- **[Pipe Operators](pipe-operators.md)**- The workers that do the actual processing: PipeLLM, PipeStructure, PipeExtract, PipeImgGen, PipeSearch, PipeCompose, and PipeFunc.+ The workers that do the actual processing: PipeLLM, PipeStructure, PipeExtract, PipeImgGen, PipeSearch, PipeJudge, PipeCompose, PipeDocGen, and PipeFunc.- **[Pipeline Orchestration](pipeline-orchestration.md)**@@ -59,6 +59,10 @@ This section covers all Pipelex capabilities, organized by area.Structured web search results with source citations and advanced filters via PipeSearch.+- **[Judgment](judgment.md)**++ Closed questions answered with a verdict and its probabilities via PipeJudge: yes or no, one option out of a set, or a level on a scale.+</div>---diff --git a/docs/features/judgment.md b/docs/features/judgment.mdnew file mode 100644index 000000000..2d697d9fc--- /dev/null+++ b/docs/features/judgment.md@@ -0,0 +1,32 @@+---+title: "Judgment"+description: "Closed questions answered by a judgment model inside Pipelex methods. PipeJudge returns a yes/no, a choice or a rating with the probabilities the model measured, ready to route or gate the steps after it."+---++# Judgment++Closed questions about your data, answered with a verdict and how sure the model is.++## Overview++Many steps of a method come down to a closed question: is this message urgent, which team owns this ticket, how severe is this report. PipeJudge asks such a question of a judgment model, a model built to answer it with a verdict and the probabilities it measured rather than with free text. The verdict is a typed `YesNo`, `Choice` or `Rating`, so the steps after it read it like any other concept.++## Key Capabilities++- **Three kinds of question** — A yes/no question, a choice among declared options, or a rating on a scale of declared levels, the kind read from the fields the pipe declares+- **Measured probabilities** — Each verdict carries the probabilities the model measured, so a method can act on how sure it is as well as on what it decided+- **Thresholds** — A yes/no question can declare the probability at or above which its verdict is yes+- **Material, not prompts** — Every input of the step is sent to the model as material to judge, images and documents as files for a model that reads them+- **Checked when the method loads** — A step with no model, an output that disagrees with its kind of question, or a file input its model does not read is refused before any run spends anything++## Usage in Pipelines++Put a PipeJudge where a method needs a decision, then route on the verdict: a `PipeCondition` whose expression is `routing.choice` sends each item to the branch its judgment chose, and one comparing `approved.probability` with a bound gates a step on the model's certainty. To judge every item of a list, map the PipeJudge over it with a `PipeBatch`.++No judgment model is served out of the box: a judgment names its model, such as the deck's `@default-judgment` alias for TypeSafe's model, which needs a `TYPESAFE_API_KEY`.++## Related Documentation++- [PipeJudge](../building-methods/pipes/pipe-operators/PipeJudge.md) - Operator reference and authoring guidance+- [Native concepts](../building-methods/concepts/native-concepts.md) - The `YesNo`, `Choice` and `Rating` verdicts+- [Pipeline Orchestration](./pipeline-orchestration.md) - Route and gate steps on a verdictdiff --git a/docs/features/pipe-operators.md b/docs/features/pipe-operators.mdindex 875b3c73c..42379ed84 100644--- a/docs/features/pipe-operators.md+++ b/docs/features/pipe-operators.md@@ -1,6 +1,6 @@---title: "Pipe Operators Overview"-description: "Pipe operators are the building blocks of Pipelex methods: PipeLLM, PipeStructure, PipeExtract, PipeImgGen, PipeSearch, PipeCompose, and PipeFunc — each handling a specific type of AI processing."+description: "Pipe operators are the building blocks of Pipelex methods: PipeLLM, PipeStructure, PipeExtract, PipeImgGen, PipeSearch, PipeJudge, PipeCompose, PipeDocGen, and PipeFunc — each handling a specific type of AI processing."---# Pipe Operators@@ -9,7 +9,7 @@ The workers that perform the actual processing in your pipelines.## Overview-Pipe operators are the building blocks of Pipelex methods. Each operator performs a specific type of work: calling an LLM, structuring text into typed data, extracting text from documents, generating images, searching the web, composing data, or running custom code. Operators declare their input and output concepts, and the runtime validates type compatibility at definition time.+Pipe operators are the building blocks of Pipelex methods. Each operator performs a specific type of work: calling an LLM, structuring text into typed data, extracting text from documents, generating images, searching the web, judging, composing data, or running custom code. Operators declare their input and output concepts, and the runtime validates type compatibility at definition time.## PipeLLM@@ -41,6 +41,12 @@ Web search with structured results and source citations. Powered by Linkup or PiSee [PipeSearch reference](../building-methods/pipes/pipe-operators/PipeSearch.md).+## PipeJudge++One closed question asked of a judgment model about the step's inputs: yes or no, one option out of a set, or a level on a scale. The verdict comes back as a typed `YesNo`, `Choice` or `Rating` carrying the probabilities the model measured, ready to route a `PipeCondition` or gate a step on a threshold.++See [PipeJudge reference](../building-methods/pipes/pipe-operators/PipeJudge.md).+## PipeComposeDeterministic object construction without an LLM. Compose outputs from working memory variables, fixed values, Jinja2 templates, and nested structures. Ideal for aggregating results from previous steps or building inputs for downstream pipes.diff --git a/docs/features/validation-dry-run.md b/docs/features/validation-dry-run.mdindex b61ed5392..2b1bcb086 100644--- a/docs/features/validation-dry-run.md+++ b/docs/features/validation-dry-run.md@@ -18,7 +18,7 @@ Check pipeline syntax, structure, and compatibility without execution:- **Syntax validation** — Catch MTHDS language errors via plxt linting- **Structure validation** — Resolve pipe and concept references in the loaded library, refuse concepts whose structures form a cycle or whose fields name a concept that cannot be found, and verify each pipe's declared inputs against the concepts that operator accepts — plus a controller's declared output against what it actually produces (for a `PipeSequence`, its last step's output concept and multiplicity)- **Input validation** — Ensure required inputs are provided and correctly typed, and that every variable a step reads is bound by the time that step runs-- **Declared inputs are read** — For `PipeLLM`, `PipeCompose`, `PipeSearch` and `PipeImgGen`, every variable a template reads must be declared in `inputs` (`missing_input_variable`), and every declared input must be read by one of the pipe's templates (`extraneous_input_variable`): the prompt and system prompt of a `PipeLLM`, the template or the construct of a `PipeCompose`, the prompt of a `PipeSearch`, the prompt and negative prompt of a `PipeImgGen`. An unread input is refused on the pipe that declares it, before any controller above it is asked to supply it. A name the template sets with `{% set %}` is its own, not an input, from that statement on: after an `{% if %}` it stays set only when every branch sets it, the `{% else %}` included, so a name only some branches set is read from the input of that name on the other paths; and a loop, a macro or a block keeps what it sets to its own body.+- **Declared inputs are read** — For `PipeLLM`, `PipeCompose`, `PipeSearch` and `PipeImgGen`, every variable a template reads must be declared in `inputs` (`missing_input_variable`), and every declared input must be read by one of the pipe's templates (`extraneous_input_variable`): the prompt and system prompt of a `PipeLLM`, the template or the construct of a `PipeCompose`, the prompt of a `PipeSearch`, the prompt and negative prompt of a `PipeImgGen`. An unread input is refused on the pipe that declares it, before any controller above it is asked to supply it. A `PipeJudge` is the exception on that second rule: every input it declares is material for its model, so only the variables its `question` reads must be declared. A name the template sets with `{% set %}` is its own, not an input, from that statement on: after an `{% if %}` it stays set only when every branch sets it, the `{% else %}` included, so a name only some branches set is read from the input of that name on the other paths; and a loop, a macro or a block keeps what it sets to its own body.A pipe that references a sub-pipe from a package that isn't loaded is reported as **skipped** rather than validated, so a passing run is not proof that every cross-package dependency resolves.diff --git a/docs/under-the-hood/architecture-overview.md b/docs/under-the-hood/architecture-overview.mdindex ff3867581..868e7cd90 100644--- a/docs/under-the-hood/architecture-overview.md+++ b/docs/under-the-hood/architecture-overview.md@@ -45,6 +45,7 @@ Operators perform concrete actions:- **PipeFunc** - Execute custom Python functions- **PipeCompose** - Compose content from templates or construct structured objects- **PipeSearch** - Search the web for information+- **PipeJudge** - Answer a closed question about the inputs with a verdictWhat an operator *does* — model-deck resolution, prompt assembly, generation, memory write-back — lives one layer down in [`pipelex/kernel/`](https://github.com/Pipelex/pipelex/tree/main/pipelex/kernel), as plain functions that take everything method-specific as an explicit argument and read only the kernel-layer services a boot stands up, such as the model deck and the content generator. The operator classes above hold what is specific to being a step in a method: blueprint resolution, input validation, error context, and execution-graph tracing. The split exists so the same semantics can be called without a loaded method, and so there is one implementation rather than two that drift. See [The Pipelex Kernel](./pipelex-kernel.md) for the layering contract, what a programmatic caller imports, and how it boots.@@ -143,6 +144,7 @@ flowchart TBO4["PipeImgGen"]O5["PipeCompose"]O6["PipeSearch"]+ O7["PipeJudge"]endsubgraph Core["Core Domain"]CR1["Concepts"]diff --git a/docs/under-the-hood/distributed-content-generation.md b/docs/under-the-hood/distributed-content-generation.mdindex f9cd004db..5d0a6bbf9 100644--- a/docs/under-the-hood/distributed-content-generation.md+++ b/docs/under-the-hood/distributed-content-generation.md@@ -129,7 +129,7 @@ The sourced-answer path (`make_search_sourced_answer`) has no dynamic class at a### Judgments need neither mechanism-A judgment's leaf, `judgment_gen_answers`, has one entry point where search has three, because nothing about it is dynamic. Its `JudgmentAssignment` carries the state as a JSON object, the questions as the family's own discriminated models and the resolved `JudgmentSetting`; its result is a map of answers that are plain models of the same package. No caller class travels down and no schema is shipped, so the in-process arm and the boundary arm would be the same function, and there is nothing to split.+A judgment's leaf, `judgment_gen_answers`, has one entry point where search has three, because nothing about it is dynamic. Its `JudgmentAssignment` carries the state as a JSON object, the image and document inputs as prompt files keyed by input name, which cross the boundary as an LLM prompt's files do, the questions as the family's own discriminated models and the resolved `JudgmentSetting`; its result is a map of answers that are plain models of the same package. No caller class travels down and no schema is shipped, so the in-process arm and the boundary arm would be the same function, and there is nothing to split.---diff --git a/docs/under-the-hood/pipelex-kernel.md b/docs/under-the-hood/pipelex-kernel.mdindex d888b8dd6..8fd0ad795 100644--- a/docs/under-the-hood/pipelex-kernel.md+++ b/docs/under-the-hood/pipelex-kernel.md@@ -76,6 +76,7 @@ Both façade calls take the concept and the output class the caller wants, defau| `pipelex.kernel.extract_ops` | `resolve_extract_setting`, `build_extract_job_params`, `run_extract` || `pipelex.kernel.img_gen_ops` | `resolve_img_gen_setting`, `resolve_default_size`, `build_img_gen_job_params`, `run_img_gen` || `pipelex.kernel.search_ops` | `resolve_search_setting`, `run_search` |+| `pipelex.kernel.judgment_ops` | `judgment_setting_of_choice`, `served_judgment_model`, `resolve_judgment_setting`, `build_judgment_material`, `material_value`, `make_verdict_content`, `run_judgment` || `pipelex.kernel.compose_ops` | `build_compose_context`, `build_composed_content`, `run_compose_template` || `pipelex.kernel.func_ops` | `call_registered_function`, `run_func` || `pipelex.kernel.memory_ops` | `shape_inputs`, `store_result`, `extract_main_content` / `extract_named_content`, `extract_main_content_as_list` / `extract_named_content_as_list` |@@ -86,7 +87,9 @@ Both façade calls take the concept and the output class the caller wants, defauThe two `assemble_*` functions are there because `run_llm_text` and `run_img_gen` both take a *ready* prompt. A caller that could not build one would be holding an operator it cannot reach, which is what image generation was until `assemble_img_gen_prompt` existed: its only builder was an interpreter-layer blueprint. What they own is the part a caller must not re-derive — resolving `ImageReference` and `DocumentReference` out of working memory, and, on the image side, keeping the `[Image N]` tokens numbered from the same registry that orders `input_images`, since a mismatch mislabels which image the prompt is describing and nothing downstream can detect it.-**Every entry point that renders a template takes a required `templating_style`** — `run_llm_text`, `run_llm_object`, `derive_structure_prompt`, `assemble_llm_prompt`, `assemble_img_gen_prompt`, `run_search`, `run_compose_template`. It is not nullable, and that is the contract: the Jinja2 filters that tag and format a value have no default of their own, so a call that omitted the style would render a prompt in a shape nobody chose. `resolve_templating_style(authored=...)` is what produces one — pass what the caller authored, or `None` to take the runtime default.+`run_judgment` asks one question over the step's inputs by name, which it calls the material. `build_judgment_material` reads each input off its content's class, never off its concept, so the kernel stays free of the concept library: text is its string, a number its number, a yes/no its boolean, a date or a time its ISO string, a list the array of its items, any other structure its JSON object without its absent members, and an image or a document, or a list of either, travels beside that JSON state as prompt images and documents keyed by the input's name. An optional input that holds no value is left out, whether its absence was recorded or it was never written. A dotted input, such as `invoice.total`, is the value at its path, keyed by its full name, and is left out when the path holds nothing. `make_verdict_content` turns the answer into the verdict native: a yes/no answer's probability decides the verdict against the step's threshold, inclusive, or against 0.5 when none is declared; an answer with no probability keeps the model's own verdict, and when a threshold was declared a live run logs a warning and its `JudgmentResult` reports `threshold_applied` as false. A rating's distribution is keyed by the level index written as text. `judgment_setting_of_choice` gives the setting of the step's model, else of the deck's default, and raises `JudgmentModelMissingError` when neither names one, since the deck serves none by default. `served_judgment_model` gives the spec of the model a handle resolves to on this boot, or `None` when no backend serves one, a waterfall none of whose models is served included. `resolve_judgment_setting` then pins the setting to the handle the model resolves to; on a dry run, a model no backend serves on this boot keeps the deck's handle, since no judging worker is called.++**Every entry point that renders a template takes a required `templating_style`** — `run_llm_text`, `run_llm_object`, `derive_structure_prompt`, `assemble_llm_prompt`, `assemble_img_gen_prompt`, `run_search`, `run_judgment`, `run_compose_template`. It is not nullable, and that is the contract: the Jinja2 filters that tag and format a value have no default of their own, so a call that omitted the style would render a prompt in a shape nobody chose. `resolve_templating_style(authored=...)` is what produces one — pass what the caller authored, or `None` to take the runtime default.There are **no re-exports**: `pipelex/kernel/__init__.py` holds doctrine and nothing else, and every symbol is imported from the module that defines it. For this package that is a layering property rather than a style one — a module that re-exports across layers is a layer boundary with the sign filed off.@@ -167,7 +170,7 @@ Passing a `trace_context` adopts its `graph_id` as the run's `pipeline_run_id`,A record emitted inside a `log.context` block carries the run's identifiers — `request_id`, `pipeline_run_id` and `pipe_run_id` — as fields a sink can filter on ([Logging](../tools/logging.md#the-run-scoped-context)). The interpreter binds them around a run and around each pipe; a program driving the kernel has no interpreter around it, so the kernel binds them itself, split the same way as the usage lifecycle above.-- **Each step binds itself.** Every kernel function that takes a `job_metadata` — `run_llm_text`, `run_llm_object`, `generate_object_content`, `run_extract`, `run_search` and `run_img_gen` — opens its body with `with job_metadata.log_context():`, so every line it emits, from prompt assembly to the memory write-back and any failure raised on the way, names the run and the step. A unit test sweeps `pipelex/kernel/` and fails on a function that takes a `job_metadata` without opening its body that way, so the rule holds for the next function added as well.+- **Each step binds itself.** Every kernel function that takes a `job_metadata` — `run_llm_text`, `run_llm_object`, `generate_object_content`, `run_extract`, `run_search`, `run_judgment` and `run_img_gen` — opens its body with `with job_metadata.log_context():`, so every line it emits, from prompt assembly to the memory write-back and any failure raised on the way, names the run and the step. A unit test sweeps `pipelex/kernel/` and fails on a function that takes a `job_metadata` without opening its body that way, so the rule holds for the next function added as well.- **The host binds the run.** `PipelexKernel.log_context()` binds `request_id` and `pipeline_run_id` and no step, and a host wraps its run in it. The kernel cannot do this for the host, for the reason it cannot own the usage lifecycle: a kernel call has no run boundary to hang the binding on. Inside the host's block, each step's binding merges over the run's for the length of the call, so a line the host emits between two calls names the run, and a line emitted during one names the step as well.```pythondiff --git a/docs/under-the-hood/template-sandbox.md b/docs/under-the-hood/template-sandbox.mdindex bddd55a1d..6feb6040c 100644--- a/docs/under-the-hood/template-sandbox.md+++ b/docs/under-the-hood/template-sandbox.md@@ -5,7 +5,7 @@ description: "Every Jinja2 template Pipelex renders runs under one sandbox polic# Template Sandbox-A method's templates are code. PipeLLM prompts, PipeCompose templates and construct templates, PipeImgGen and PipeSearch prompts, and PipeCondition expressions are all Jinja2 templates, and they render inside the process that runs the method. When that process is shared, as it is on a hosted runner, a template must not be able to reach anything but the data it was given.+A method's templates are code. PipeLLM prompts, PipeCompose templates and construct templates, PipeImgGen and PipeSearch prompts, PipeJudge questions, and PipeCondition expressions are all Jinja2 templates, and they render inside the process that runs the method. When that process is shared, as it is on a hosted runner, a template must not be able to reach anything but the data it was given.Pipelex therefore renders every template, its own included, under one policy: **a template reads data and calls methods of plain values, and nothing else.** And because a template that only reads data can still ask for a billion-character string, every render also spends from a [budget of work](#the-render-budget), and a render that would overdraw it is refused before it allocates.diff --git a/mkdocs.yml b/mkdocs.ymlindex f82775bc7..217883bc3 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -241,6 +241,7 @@ plugins:- features/document-extraction.md: "Document Extraction"- features/image-generation.md: "Image Generation"- features/web-search.md: "Web Search"+ - features/judgment.md: "Judgment"- features/pipelex-plugin.md: "Pipelex Plugin"- features/cli.md: "CLI"- features/plxt.md: "plxt Formatter & Linter"@@ -277,6 +278,7 @@ plugins:- building-methods/pipes/pipe-operators/PipeExtract.md: "PipeExtract"- building-methods/pipes/pipe-operators/PipeImgGen.md: "PipeImgGen"- building-methods/pipes/pipe-operators/PipeSearch.md: "PipeSearch"+ - building-methods/pipes/pipe-operators/PipeJudge.md: "PipeJudge"- building-methods/pipes/pipe-operators/PipeCompose.md: "PipeCompose"- building-methods/pipes/pipe-operators/PipeDocGen.md: "PipeDocGen"- building-methods/pipes/pipe-operators/PipeFunc.md: "PipeFunc"@@ -464,6 +466,7 @@ nav:- Document Extraction: features/document-extraction.md- Image Generation: features/image-generation.md- Web Search: features/web-search.md+ - Judgment: features/judgment.md- Developer Tools:- Pipelex Plugin: features/pipelex-plugin.md- CLI: features/cli.md@@ -508,6 +511,7 @@ nav:- PipeExtract: building-methods/pipes/pipe-operators/PipeExtract.md- PipeImgGen: building-methods/pipes/pipe-operators/PipeImgGen.md- PipeSearch: building-methods/pipes/pipe-operators/PipeSearch.md+ - PipeJudge: building-methods/pipes/pipe-operators/PipeJudge.md- PipeCompose: building-methods/pipes/pipe-operators/PipeCompose.md- PipeDocGen: building-methods/pipes/pipe-operators/PipeDocGen.md- PipeFunc: building-methods/pipes/pipe-operators/PipeFunc.mddiff --git a/pipelex/builder/CLAUDE.md b/pipelex/builder/CLAUDE.mdindex 5807b6086..235d2e08b 100644--- a/pipelex/builder/CLAUDE.md+++ b/pipelex/builder/CLAUDE.md@@ -29,6 +29,9 @@ pipe/pipe_img_gen_spec.py # PipeImgGen — image generation operatorpipe_extract_spec.py # PipeExtract — OCR/text extraction operatorpipe_search_spec.py # PipeSearch — web search operator+ pipe_judge_spec.py # PipeJudge — closed-question judgment operator+ pipe_structure_spec.py # PipeStructure — text-to-structure operator+ pipe_doc_gen_spec.py # PipeDocGen — document generation operatorpipe_compose_spec.py # PipeCompose — template/construct operatorpipe_sequence_spec.py # PipeSequence — sequential controllerpipe_parallel_spec.py # PipeParallel — concurrent controller@@ -48,7 +51,7 @@ operations/All specs are Pydantic models (`StructuredContent` base). Two categories of pipes:-**Operators** (data transformation): `PipeLLM`, `PipeFunc`, `PipeImgGen`, `PipeExtract`, `PipeSearch`, `PipeCompose`+**Operators** (data transformation): `PipeLLM`, `PipeFunc`, `PipeImgGen`, `PipeExtract`, `PipeSearch`, `PipeJudge`, `PipeStructure`, `PipeDocGen`, `PipeCompose`**Controllers** (execution flow): `PipeSequence`, `PipeParallel`, `PipeCondition`, `PipeBatch`diff --git a/pipelex/builder/operations/pipe_ops.py b/pipelex/builder/operations/pipe_ops.pyindex a0d9307e6..dd5397ec7 100644--- a/pipelex/builder/operations/pipe_ops.py+++ b/pipelex/builder/operations/pipe_ops.py@@ -20,6 +20,7 @@ from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec+from pipelex.builder.pipe.pipe_judge_spec import PipeJudgeSpecfrom pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpecfrom pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_search_spec import PipeSearchSpec@@ -339,6 +340,26 @@ def add_type_specific_fields(*, pipe_spec: PipeSpec, pipe_table: Table) -> None:if pipe_spec.max_results is not None:pipe_table.add("max_results", pipe_spec.max_results)+ elif isinstance(pipe_spec, PipeJudgeSpec):+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ # A tool that writes a PipeJudge writes `question`, never its `prompt` synonym.+ pipe_table.add("question", pipe_spec.question)+ if pipe_spec.options is not None:+ options_table = tomlkit.inline_table()+ for option, description in pipe_spec.options.items():+ options_table.append(option, description)+ pipe_table.add("options", options_table)+ if pipe_spec.levels is not None:+ pipe_table.add("levels", pipe_spec.levels)+ if pipe_spec.criteria is not None:+ criteria_table = tomlkit.inline_table()+ for verdict, criterion in pipe_spec.criteria.model_dump(exclude_none=True).items():+ criteria_table.append(verdict, criterion)+ pipe_table.add("criteria", criteria_table)+ if pipe_spec.threshold is not None:+ pipe_table.add("threshold", pipe_spec.threshold)+elif isinstance(pipe_spec, PipeImgGenSpec):if pipe_spec.model:pipe_table.add("model", pipe_spec.model)diff --git a/pipelex/builder/pipe/pipe_judge_spec.py b/pipelex/builder/pipe/pipe_judge_spec.pynew file mode 100644index 000000000..8bd9c2aa1--- /dev/null+++ b/pipelex/builder/pipe/pipe_judge_spec.py@@ -0,0 +1,108 @@+from typing import TYPE_CHECKING, Any, Literal++from pydantic import Field, field_validator, model_validator+from typing_extensions import override++from pipelex.builder.pipe.pipe_spec import PipeSpec+from pipelex.pipe_operators.judge.pipe_judge_blueprint import JudgeYesNoCriteria, PipeJudgeBlueprint, read_prompt_as_question+from pipelex.tools.misc.pretty import require_rich_for_rendering++if TYPE_CHECKING:+ from pipelex.tools.misc.pretty import PrettyPrintable+++class PipeJudgeSpec(PipeSpec):+ """Specs for judgment pipe operations in the Pipelex framework.++ PipeJudge asks a judging model one closed question about its inputs. Declare `options` for a+ choice, `levels` for a rating, or neither for a yes/no question, which may carry `criteria` and a+ `threshold`. Every input is material to judge, so the question need not name them.+ """++ type: Literal["PipeJudge"] = "PipeJudge"+ pipe_category: Literal["PipeOperator"] = "PipeOperator"+ model: str | None = Field(+ default=None,+ description=(+ "Judgment model preset, alias, waterfall, or direct model handle. Use presets from 'pipelex-agent models'. "+ "Required unless the deck names a default judgment model (`choice_default` under `[judgment]`), which the shipped deck does not."+ ),+ )+ question: str = Field(+ description="The one closed question to ask about the inputs: use `$` prefix for an inline parameter (e.g., `Is it about $topic?`).",+ )+ options: dict[str, str] | None = Field(+ default=None,+ description="For a choice question: each key is an option, each value describes it as a concrete situation (an empty string for none).",+ )+ levels: list[str] | None = Field(+ default=None,+ description="For a rating question: the levels from lowest to highest, each describing a situation rather than a degree.",+ )+ criteria: JudgeYesNoCriteria | None = Field(default=None, description="For a yes/no question: what a yes and what a no mean.")+ threshold: float | None = Field(+ default=None,+ description="For a yes/no question: the probability of yes at or above which the verdict is yes, strictly between 0 and 1.",+ )++ @field_validator("model", mode="before")+ @classmethod+ def reject_empty_model(cls, value: str | None) -> str | None:+ if isinstance(value, str) and not value.strip():+ msg = "Model cannot be an empty string: name a judgment model, or omit the field only when the deck names a default one"+ raise ValueError(msg)+ return value++ @model_validator(mode="before")+ @classmethod+ def read_prompt_synonym(cls, values: dict[str, Any]) -> dict[str, Any]:+ """Accept `prompt` for `question`, the spelling every other inference operator uses, as the blueprint does."""+ return read_prompt_as_question(values=values)++ @override+ def rendered_pretty(self, *, title: str | None = None, depth: int = 0) -> "PrettyPrintable":+ require_rich_for_rendering()+ from rich.console import Group+ from rich.markup import escape+ from rich.panel import Panel+ from rich.text import Text++ judge_group = Group()+ judge_group.renderables.append(super().rendered_pretty(title=title, depth=depth))+ judge_group.renderables.append(Text())+ judge_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{escape(self.model or '(default)')}[/bold yellow]"))+ judge_group.renderables.append(Text())+ judge_group.renderables.append(Panel(Text(self.question), title="Question", title_align="left", border_style="cyan", padding=(0, 1)))+ detail_lines: list[str] = []+ if self.options is not None:+ detail_lines.extend(f"Option {key}: {description}" if description else f"Option {key}" for key, description in self.options.items())+ if self.levels is not None:+ detail_lines.extend(f"Level {index}: {level}" for index, level in enumerate(self.levels))+ if self.criteria is not None:+ if self.criteria.yes is not None:+ detail_lines.append(f"Yes means: {self.criteria.yes}")+ if self.criteria.no is not None:+ detail_lines.append(f"No means: {self.criteria.no}")+ if self.threshold is not None:+ detail_lines.append(f"Threshold: {self.threshold}")+ if detail_lines:+ judge_group.renderables.append(Text())+ for detail_line in detail_lines:+ judge_group.renderables.append(Text(detail_line, style="dim"))+ return judge_group++ @override+ def to_blueprint(self) -> PipeJudgeBlueprint:+ """Convert this PipeJudgeSpec to the core PipeJudgeBlueprint."""+ base_blueprint = super().to_blueprint()+ return PipeJudgeBlueprint(+ description=base_blueprint.description,+ inputs=base_blueprint.inputs_concept_specs,+ output=base_blueprint.output,+ question=self.question,+ model=self.model,+ options=self.options,+ levels=self.levels,+ criteria=self.criteria,+ threshold=self.threshold,+ )diff --git a/pipelex/builder/pipe/pipe_spec_map.py b/pipelex/builder/pipe/pipe_spec_map.pyindex a48807cd9..7ad1ce7c6 100644--- a/pipelex/builder/pipe/pipe_spec_map.py+++ b/pipelex/builder/pipe/pipe_spec_map.py@@ -5,6 +5,7 @@ from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec+from pipelex.builder.pipe.pipe_judge_spec import PipeJudgeSpecfrom pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpecfrom pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_search_spec import PipeSearchSpec@@ -23,6 +24,7 @@ pipe_type_to_spec_class: dict[str, type[PipeSpec]] = {"PipeLLM": PipeLLMSpec,"PipeExtract": PipeExtractSpec,"PipeSearch": PipeSearchSpec,+ "PipeJudge": PipeJudgeSpec,"PipeStructure": PipeStructureSpec,"PipeDocGen": PipeDocGenSpec,"PipeBatch": PipeBatchSpec,diff --git a/pipelex/builder/pipe/pipe_spec_union.py b/pipelex/builder/pipe/pipe_spec_union.pyindex c8164f6d5..2e9cafc20 100644--- a/pipelex/builder/pipe/pipe_spec_union.py+++ b/pipelex/builder/pipe/pipe_spec_union.py@@ -9,6 +9,7 @@ from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec+from pipelex.builder.pipe.pipe_judge_spec import PipeJudgeSpecfrom pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpecfrom pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_search_spec import PipeSearchSpec@@ -23,6 +24,7 @@ PipeSpecUnion = Annotated[| PipeLLMSpec| PipeExtractSpec| PipeSearchSpec+ | PipeJudgeSpec| PipeStructureSpec| PipeDocGenSpec| PipeBatchSpecdiff --git a/pipelex/cli/agent_cli/commands/pipe_cmd.py b/pipelex/cli/agent_cli/commands/pipe_cmd.pyindex 9bf0480fc..b040d1f71 100644--- a/pipelex/cli/agent_cli/commands/pipe_cmd.py+++ b/pipelex/cli/agent_cli/commands/pipe_cmd.py@@ -21,6 +21,7 @@ from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec+from pipelex.builder.pipe.pipe_judge_spec import PipeJudgeSpecfrom pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpecfrom pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_search_spec import PipeSearchSpec@@ -190,6 +191,26 @@ def _add_type_specific_fields(*, pipe_spec: PipeSpec, pipe_table: tomlkit.TOMLDoif pipe_spec.max_results is not None:pipe_table.add("max_results", pipe_spec.max_results)+ elif isinstance(pipe_spec, PipeJudgeSpec):+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ # A tool that writes a PipeJudge writes `question`, never its `prompt` synonym.+ pipe_table.add("question", format_toml_string(pipe_spec.question))+ if pipe_spec.options is not None:+ options_table = tomlkit.inline_table()+ for option, description in pipe_spec.options.items():+ options_table.append(option, description)+ pipe_table.add("options", options_table)+ if pipe_spec.levels is not None:+ pipe_table.add("levels", pipe_spec.levels)+ if pipe_spec.criteria is not None:+ criteria_table = tomlkit.inline_table()+ for verdict, criterion in pipe_spec.criteria.model_dump(exclude_none=True).items():+ criteria_table.append(verdict, criterion)+ pipe_table.add("criteria", criteria_table)+ if pipe_spec.threshold is not None:+ pipe_table.add("threshold", pipe_spec.threshold)+elif isinstance(pipe_spec, PipeSignatureSpec):if pipe_spec.signature_for is not None:pipe_table.add("signature_for", pipe_spec.signature_for)diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.pyindex 10b4714c1..d8c6d43cc 100644--- a/pipelex/cogt/content_generation/assignment_models.py+++ b/pipelex/cogt/content_generation/assignment_models.py@@ -6,8 +6,10 @@ from typing_extensions import overridefrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParamsfrom pipelex.cogt.doc_gen.doc_gen_setting import DocGenSettingfrom pipelex.cogt.doc_gen.document_composition import DocumentComposition+from pipelex.cogt.document.prompt_document import PromptDocument, PromptDocumentUrifrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParams+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageUrifrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.judgment.judgment_models import JudgmentQuestion, JudgmentState@@ -249,11 +251,15 @@ class JudgmentAssignment(BaseModel):Unlike the structured-search leaf, there is no schema to ship and no dynamic class to rebuild:a judgment's answers are plain models of this package's own, so they cross a boundary as they are.+ The files travel beside the state, keyed by the input they came from, as the prompt images and+ documents an LLM assignment carries."""job_metadata: JobMetadatacogt_run_params: CogtRunParamsstate: JudgmentState+ images: dict[str, list[PromptImage]] = Field(default_factory=dict)+ documents: dict[str, list[PromptDocument]] = Field(default_factory=dict)questions: dict[str, JudgmentQuestion] = Field(min_length=1)judgment_setting: JudgmentSetting@@ -262,5 +268,17 @@ class JudgmentAssignment(BaseModel):return self.judgment_setting.modeldef referenced_uris(self) -> list[UriReference]:- """None: a judgment hands its state to the provider as data and reads no URL the state carries."""- return []+ """The URLs the leaf reads for this assignment: its files given by URI, never a URL the state carries as text."""+ uri_references = [+ UriReference(uri=image.uri, position=f"image {index} of input '{input_name}'")+ for input_name, images in self.images.items()+ for index, image in enumerate(images, start=1)+ if isinstance(image, PromptImageUri)+ ]+ uri_references.extend(+ UriReference(uri=document.uri, position=f"document {index} of input '{input_name}'")+ for input_name, documents in self.documents.items()+ for index, document in enumerate(documents, start=1)+ if isinstance(document, PromptDocumentUri)+ )+ return uri_referencesdiff --git a/pipelex/cogt/content_generation/judgment_generate.py b/pipelex/cogt/content_generation/judgment_generate.pyindex 7ef7befd0..77a28252d 100644--- a/pipelex/cogt/content_generation/judgment_generate.py+++ b/pipelex/cogt/content_generation/judgment_generate.py@@ -35,6 +35,8 @@ def _make_judgment_worker(judgment_assignment: JudgmentAssignment) -> JudgmentWodef _make_judgment_job(judgment_assignment: JudgmentAssignment) -> JudgmentJob:return JudgmentJobFactory.make_judgment_job(state=judgment_assignment.state,+ images=judgment_assignment.images,+ documents=judgment_assignment.documents,questions=judgment_assignment.questions,judgment_setting=judgment_assignment.judgment_setting,job_metadata=judgment_assignment.job_metadata,diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex a2905df86..8ba107f60 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -481,10 +481,43 @@ class JudgmentAnswerMismatchError(CogtError):"""A judgment worker answered questions nobody asked, or answered one in the wrong shape."""+class JudgmentCapabilityError(CogtError):+ """A judgment job carries files its model does not read: images to a model without vision, documents to one that reads none.++ The author's choice of model, and so a configuration error, as `LLMCapabilityError` is for an LLM.+ A document of a format the model does not read is the caller's file, which stays a+ `PromptDocumentFormatError`.+ """++ error_category = InferenceErrorCategory.CONFIGURATION++class JudgmentModelNotFoundError(ModelNotFoundError):pass+class JudgmentModelMissingError(PipelexError):+ """A judgment has no model to run on: the step names none, and the model deck names no default.++ The deck serves no judgment model out of the box, since a judgment backend is one the user brings,+ so a step that names no model is refused when its method loads, before a run spends anything, and+ by the kernel's resolver for a programmatic caller. The message names only the step and the two+ remedies, so it is kept verbatim for the caller.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = True++ def __init__(self, *, pipe_code: str | None = None):+ self.pipe_code = pipe_code+ step = f"PipeJudge '{pipe_code}'" if pipe_code else "This judgment"+ message = (+ f"{step} has no judgment model: it names none, and the model deck names no default for judgments. "+ "Name the model in the step's `model` field, or set a `choice_default` under `[judgment]` in the model deck."+ )+ super().__init__(message)++class RoutingProfileLibraryNotFoundError(CogtError):passdiff --git a/pipelex/cogt/inference/prompt_file_checks.py b/pipelex/cogt/inference/prompt_file_checks.pynew file mode 100644index 000000000..1fcfc67be--- /dev/null+++ b/pipelex/cogt/inference/prompt_file_checks.py@@ -0,0 +1,52 @@+"""The per-file checks every inference family runs on the files a prompt carries, before a provider sees them.++Whether a model reads images or documents at all is the family's own question, raised with the family's+own capability error. What these checks answer is the next one: given a model that reads files, is each+file one it reads? The file is the caller's and changes from run to run, so a format the model does not+read is a content error. Only a known format is refused; an unknown one is left to the provider.+"""++from collections.abc import Sequence++from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.exceptions import PromptDocumentFormatError, PromptImageFormatError+from pipelex.cogt.image.prompt_image import PromptImage+from pipelex.tools.misc.filetype_utils import (+ IMAGE_FORMAT_KEY,+ describe_file_format,+ describe_format_keys,+ format_key_from_mime_type,+)+++def check_prompt_images_are_images(*, model_name: str, prompt_images: Sequence[PromptImage]) -> None:+ """Refuse a prompt image whose known format is not an image.++ Run setup refuses a non-image given to an Image input, but an image the run produced itself never+ went through setup: this is the line that catches it before the provider.+ """+ for image_index, prompt_image in enumerate(prompt_images, start=1):+ mime_type = prompt_image.known_mime_type()+ format_key = format_key_from_mime_type(mime_type=mime_type)+ if format_key is None or format_key == IMAGE_FORMAT_KEY:+ continue+ msg = (+ f"Prompt image {image_index} given to model '{model_name}' is "+ f"{describe_file_format(format_key=format_key, mime_type=mime_type)}, not an image."+ )+ raise PromptImageFormatError(msg)+++def check_prompt_documents_are_read(*, model_name: str, supported_document_types: set[str], prompt_documents: Sequence[PromptDocument]) -> None:+ """Refuse a prompt document whose known format is not among the formats the model reads."""+ for document_index, prompt_document in enumerate(prompt_documents, start=1):+ mime_type = prompt_document.known_mime_type()+ format_key = format_key_from_mime_type(mime_type=mime_type)+ if format_key is None or format_key in supported_document_types:+ continue+ msg = (+ f"Prompt document {document_index} given to model '{model_name}' is "+ f"{describe_file_format(format_key=format_key, mime_type=mime_type)}, which it does not read: "+ f"it reads {describe_format_keys(format_keys=supported_document_types)}."+ )+ raise PromptDocumentFormatError(msg)diff --git a/pipelex/cogt/judgment/judgment_job.py b/pipelex/cogt/judgment/judgment_job.pyindex fb3dac50f..17e385460 100644--- a/pipelex/cogt/judgment/judgment_job.py+++ b/pipelex/cogt/judgment/judgment_job.py@@ -3,6 +3,8 @@ from datetime import datetimefrom pydantic import BaseModel, Fieldfrom typing_extensions import override+from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstractfrom pipelex.cogt.judgment.judgment_models import JudgmentQuestion, JudgmentStatefrom pipelex.cogt.judgment.judgment_report import JudgmentTokensUsage@@ -28,9 +30,16 @@ class JudgmentJob(InferenceJobAbstract):The question keys are the caller's own — the model never sees them — and they are what theanswers come back under.++ The files travel beside the state, keyed by the name of the input they came from, because a file+ is not a JSON value: an image or a document reaches a judgment exactly as it reaches an LLM, as a+ prompt image or a prompt document. Whether the model reads them is its own capability, which the+ worker checks before the backend is called."""state: JudgmentState+ images: dict[str, list[PromptImage]] = Field(default_factory=dict)+ documents: dict[str, list[PromptDocument]] = Field(default_factory=dict)questions: dict[str, JudgmentQuestion] = Field(min_length=1)job_params: JudgmentJobParamsjob_report: JudgmentJobReport = JudgmentJobReport()diff --git a/pipelex/cogt/judgment/judgment_job_factory.py b/pipelex/cogt/judgment/judgment_job_factory.pyindex 314580c72..6a131ded2 100644--- a/pipelex/cogt/judgment/judgment_job_factory.py+++ b/pipelex/cogt/judgment/judgment_job_factory.py@@ -1,3 +1,5 @@+from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.judgment.judgment_job import JudgmentJob, JudgmentJobParamsfrom pipelex.cogt.judgment.judgment_models import JudgmentQuestion, JudgmentStatefrom pipelex.cogt.judgment.judgment_setting import JudgmentSetting@@ -13,12 +15,16 @@ class JudgmentJobFactory:questions: dict[str, JudgmentQuestion],judgment_setting: JudgmentSetting,job_metadata: JobMetadata,+ images: dict[str, list[PromptImage]] | None = None,+ documents: dict[str, list[PromptDocument]] | None = None,) -> JudgmentJob:job_metadata.job_category = JobCategory.JUDGMENT_JOBjob_params = JudgmentJobParams(judgment_setting=judgment_setting)return JudgmentJob(job_metadata=job_metadata,state=state,+ images=images or {},+ documents=documents or {},questions=questions,job_params=job_params,)diff --git a/pipelex/cogt/judgment/judgment_worker_abstract.py b/pipelex/cogt/judgment/judgment_worker_abstract.pyindex 0c5ece47e..cacf23bb4 100644--- a/pipelex/cogt/judgment/judgment_worker_abstract.py+++ b/pipelex/cogt/judgment/judgment_worker_abstract.py@@ -1,12 +1,24 @@+import mathfrom abc import abstractmethod+from collections.abc import Mappingfrom typing_extensions import overridefrom pipelex import log-from pipelex.cogt.exceptions import CogtError, JudgmentAnswerMismatchError+from pipelex.cogt.exceptions import CogtError, JudgmentAnswerMismatchError, JudgmentCapabilityErrorfrom pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract+from pipelex.cogt.inference.prompt_file_checks import check_prompt_documents_are_read, check_prompt_images_are_imagesfrom pipelex.cogt.judgment.judgment_job import JudgmentJob-from pipelex.cogt.judgment.judgment_models import ChoiceAnswer, ChoiceQuestion, JudgmentAnswer, JudgmentQuestion, RatingAnswer, RatingQuestion+from pipelex.cogt.judgment.judgment_models import (+ ChoiceAnswer,+ ChoiceQuestion,+ JudgmentAnswer,+ JudgmentQuestion,+ RatingAnswer,+ RatingQuestion,+ YesNoAnswer,+ YesNoQuestion,+)from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.reporting.reporting_protocol import ReportingProtocolfrom pipelex.system.job_metadata import UnitJobId@@ -37,6 +49,7 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):judgment_job.judgment_job_before_start(inference_model=self.inference_model)try:+ self._check_can_read_files(judgment_job=judgment_job)answers = await self._judge(judgment_job=judgment_job)_check_answers_match_questions(judgment_job=judgment_job, answers=answers)except CogtError as exc:@@ -55,6 +68,38 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):return answers+ def _check_can_read_files(self, *, judgment_job: JudgmentJob) -> None:+ """Refuse a job carrying files the model does not read, before the backend is called.++ The model's spec states what it reads in `inputs`, in the LLM vocabulary: `images` for vision,+ a document format key such as `pdf` for documents. A model that reads text alone refuses any+ file here, which is how a text-only backend is protected without a line of its own.+ """+ prompt_images = [image for images in judgment_job.images.values() for image in images]+ if prompt_images:+ if not self.inference_model.is_vision_supported:+ msg = f"Judgment model '{self.inference_model.name}' does not read images, and the judgment was given {len(prompt_images)}."+ raise JudgmentCapabilityError(msg)+ max_prompt_images = self.inference_model.max_prompt_images+ if max_prompt_images is not None and len(prompt_images) > max_prompt_images:+ msg = (+ f"Judgment model '{self.inference_model.name}' reads at most {max_prompt_images} images, "+ f"and the judgment was given {len(prompt_images)}."+ )+ raise JudgmentCapabilityError(msg)+ check_prompt_images_are_images(model_name=self.inference_model.name, prompt_images=prompt_images)++ prompt_documents = [document for documents in judgment_job.documents.values() for document in documents]+ if prompt_documents:+ if not self.inference_model.is_document_supported:+ msg = f"Judgment model '{self.inference_model.name}' does not read documents, and the judgment was given {len(prompt_documents)}."+ raise JudgmentCapabilityError(msg)+ check_prompt_documents_are_read(+ model_name=self.inference_model.name,+ supported_document_types=self.inference_model.supported_document_types,+ prompt_documents=prompt_documents,+ )+@abstractmethodasync def _judge(self,@@ -81,27 +126,33 @@ def _check_answers_match_questions(*, judgment_job: JudgmentJob, answers: dict[smsg = f"Judgment worker answered the wrong set of questions: missing {missing}, unasked {unasked}"raise JudgmentAnswerMismatchError(msg)for question_key, question in judgment_job.questions.items():- answer = answers[question_key]- if answer.kind != question.kind:- msg = f"Judgment worker answered question '{question_key}' with a '{answer.kind}' answer, but it asked for '{question.kind}'"- raise JudgmentAnswerMismatchError(msg)- _check_answer_is_offered(question_key=question_key, question=question, answer=answer)+ _check_answer_fits_question(question_key=question_key, question=question, answer=answers[question_key])-def _check_answer_is_offered(*, question_key: str, question: JudgmentQuestion, answer: JudgmentAnswer) -> None:- """Refuse a verdict the question did not offer, in the answer itself or in its distribution.+def _check_answer_fits_question(*, question_key: str, question: JudgmentQuestion, answer: JudgmentAnswer) -> None:+ """Refuse an answer of another kind than its question, or naming a verdict the question did not offer.The answer models cannot check this on their own, since they never see their question: this is- the one place that holds both.+ the one place that holds both. The match is over the question alone, with no wildcard, so a new+ kind of question fails the type check here until its answer is checked too."""- match question, answer:- case ChoiceQuestion(), ChoiceAnswer():+ match question:+ case YesNoQuestion():+ if not isinstance(answer, YesNoAnswer):+ raise JudgmentAnswerMismatchError(_wrong_kind_message(question_key=question_key, question=question, answer=answer))+ # A yes/no answer names no option and no level, and its probability is bounded by its model.+ case ChoiceQuestion():+ if not isinstance(answer, ChoiceAnswer):+ raise JudgmentAnswerMismatchError(_wrong_kind_message(question_key=question_key, question=question, answer=answer))named_options = {answer.choice, *(answer.probabilities or {})}unoffered = sorted(named_options - set(question.options))if unoffered:msg = f"Judgment worker answered question '{question_key}' with options it does not offer: {unoffered}"raise JudgmentAnswerMismatchError(msg)- case RatingQuestion(), RatingAnswer():+ _check_probabilities_are_probabilities(question_key=question_key, probabilities=answer.probabilities)+ case RatingQuestion():+ if not isinstance(answer, RatingAnswer):+ raise JudgmentAnswerMismatchError(_wrong_kind_message(question_key=question_key, question=question, answer=answer))nb_levels = len(question.levels)named_levels = {answer.level, *(answer.probabilities or {})}out_of_scale = sorted(level for level in named_levels if not 0 <= level < nb_levels)@@ -111,6 +162,29 @@ def _check_answer_is_offered(*, question_key: str, question: JudgmentQuestion, af"outside its scale of {nb_levels} levels (0 to {nb_levels - 1})")raise JudgmentAnswerMismatchError(msg)- case _:- # A yes/no answer names no option and no level, and its probability is bounded by its model.- pass+ _check_probabilities_are_probabilities(question_key=question_key, probabilities=answer.probabilities)+ if answer.position is not None and not (math.isfinite(answer.position) and 0 <= answer.position <= nb_levels - 1):+ msg = (+ f"Judgment worker answered question '{question_key}' with a position of {answer.position}, "+ f"which is not a position on its scale of {nb_levels} levels (0 to {nb_levels - 1})"+ )+ raise JudgmentAnswerMismatchError(msg)+++def _check_probabilities_are_probabilities(*, question_key: str, probabilities: Mapping[str, float] | Mapping[int, float] | None) -> None:+ """Refuse a distribution with a probability outside the unit interval, which no verdict native can hold.++ The answer models leave a distribution's values unbounded, so without this a backend's rounding past+ one would pass the worker and fail in the verdict's own model, after the call was paid for, as an+ error that says nothing about the answer.+ """+ if probabilities is None:+ return+ out_of_bounds = sorted(str(key) for key, probability in probabilities.items() if not 0 <= probability <= 1)+ if out_of_bounds:+ msg = f"Judgment worker answered question '{question_key}' with probabilities outside 0 to 1 for {out_of_bounds}"+ raise JudgmentAnswerMismatchError(msg)+++def _wrong_kind_message(*, question_key: str, question: JudgmentQuestion, answer: JudgmentAnswer) -> str:+ return f"Judgment worker answered question '{question_key}' with a '{answer.kind}' answer, but it asked for '{question.kind}'"diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.pyindex 53aead7cd..a3c3ee9ee 100644--- a/pipelex/cogt/llm/llm_worker_abstract.py+++ b/pipelex/cogt/llm/llm_worker_abstract.py@@ -9,9 +9,10 @@ from opentelemetry.trace import NonRecordingSpan, Span, SpanContext, SpanKind, Sfrom typing_extensions import overridefrom pipelex import log-from pipelex.cogt.exceptions import CogtError, LLMCapabilityError, PromptDocumentFormatError, PromptImageFormatError+from pipelex.cogt.exceptions import CogtError, LLMCapabilityErrorfrom pipelex.cogt.inference.inference_constants import InferenceOutputTypefrom pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract+from pipelex.cogt.inference.prompt_file_checks import check_prompt_documents_are_read, check_prompt_images_are_imagesfrom pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedConstraintfrom pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.system.exceptions import JobMetadataError@@ -28,12 +29,6 @@ from pipelex.system.telemetry.otel_constants import (from pipelex.system.telemetry.otel_factory import OtelFactoryfrom pipelex.system.telemetry.telemetry_identity import make_run_identity_span_attributesfrom pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract-from pipelex.tools.misc.filetype_utils import (- IMAGE_FORMAT_KEY,- describe_file_format,- describe_format_keys,- format_key_from_mime_type,-)from pipelex.tools.misc.package_utils import get_package_versionif TYPE_CHECKING:@@ -439,19 +434,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):msg = f"LLM Engine '{self.inference_model.tag}' does not accept that many images: {nb_images}."raise LLMCapabilityError(msg)- # Run setup refuses a non-image given to an Image input, but an image the run produced- # itself never went through setup: this is the line that catches it before the provider.- # Only a known format is refused; an unknown one is left to the provider.- for image_index, prompt_image in enumerate(llm_job.llm_prompt.user_images, start=1):- mime_type = prompt_image.known_mime_type()- format_key = format_key_from_mime_type(mime_type=mime_type)- if format_key is None or format_key == IMAGE_FORMAT_KEY:- continue- msg = (- f"Prompt image {image_index} given to model '{self.inference_model.name}' is "- f"{describe_file_format(format_key=format_key, mime_type=mime_type)}, not an image."- )- raise PromptImageFormatError(msg)+ check_prompt_images_are_images(model_name=self.inference_model.name, prompt_images=llm_job.llm_prompt.user_images)def _check_document_support(self, llm_job: LLMJob):if not llm_job.llm_prompt.user_documents:@@ -461,21 +444,11 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):msg = f"LLM Engine '{self.inference_model.tag}' does not support documents."raise LLMCapabilityError(msg)- # Each document's known format must be one the model reads. The file is the caller's and- # changes from run to run, so a format the model does not read is an input error; an unknown- # format is left to the provider.- supported = self.inference_model.supported_document_types- for document_index, prompt_document in enumerate(llm_job.llm_prompt.user_documents, start=1):- mime_type = prompt_document.known_mime_type()- format_key = format_key_from_mime_type(mime_type=mime_type)- if format_key is None or format_key in supported:- continue- msg = (- f"Prompt document {document_index} given to model '{self.inference_model.name}' is "- f"{describe_file_format(format_key=format_key, mime_type=mime_type)}, which it does not read: "- f"it reads {describe_format_keys(format_keys=supported)}."- )- raise PromptDocumentFormatError(msg)+ check_prompt_documents_are_read(+ model_name=self.inference_model.name,+ supported_document_types=self.inference_model.supported_document_types,+ prompt_documents=llm_job.llm_prompt.user_documents,+ )async def gen_text(self,diff --git a/pipelex/core/memory/exceptions.py b/pipelex/core/memory/exceptions.pyindex b080381b4..7f3da87ed 100644--- a/pipelex/core/memory/exceptions.py+++ b/pipelex/core/memory/exceptions.py@@ -231,10 +231,11 @@ class StructureValidationError(InputShapingError):"""The refusal for a scalar, or a list of scalars, at an input of a verdict native.A `Choice` or a `Rating` input takes its value in the `{"concept", "content"}` envelope, and a- caller naturally sends its key or its level alone, as a `Number` or a `YesNo` input would take- it. A verdict is more than its key or its level, so the value has no reading, and a declaration- that reads it, such as `Anything`, would discard the verdict the method branches on: the advice- is the expected shape.+ caller naturally sends a rating's level alone, as a `Number` input would take it. A number or a+ boolean has no reading there, since a verdict is more than its level, and a declaration that+ reads it, such as `Anything`, would discard the verdict the method branches on: the advice is the+ expected shape. A string, such as a choice's key, never reaches this refusal: it reads as a+ `Text`, which `make_for_bare_value_of_another_concept` refuses."""message = (f"Input '{variable_name}' could not be built as '{declared_concept_ref}': you provided {provided_description}, "diff --git a/pipelex/core/memory/input_shaper.py b/pipelex/core/memory/input_shaper.pyindex faa43ec11..57592bf7f 100644--- a/pipelex/core/memory/input_shaper.py+++ b/pipelex/core/memory/input_shaper.py@@ -1070,8 +1070,9 @@ class InputShaper:def _is_scalar_at_a_verdict_input(cls, *, value: Any, declared_concept: Concept, concept_provider: ConceptProviderAbstract) -> bool:"""Whether a bare value the fallback has no reading for is a scalar, or a list of scalars, at an input of a verdict native.- A caller sends a `Choice`'s key or a `Rating`'s level alone by mistake, which no container native- invites, so only an input whose concept is or refines a verdict native answers. An object is+ A caller sends a `Rating`'s level alone by mistake, which no container native invites, so only an+ input whose concept is or refines a verdict native answers. A string never reaches here, since the+ factory reads it as a `Text`. An object isnever a scalar: whether it is the verdict's content unwrapped is `_is_content_of_declared_concept`'squestion. `Dynamic` is compatible with every concept, so it never answers."""diff --git a/pipelex/core/pipes/exceptions.py b/pipelex/core/pipes/exceptions.pyindex 4d459cb64..47c86999d 100644--- a/pipelex/core/pipes/exceptions.py+++ b/pipelex/core/pipes/exceptions.py@@ -56,7 +56,7 @@ class PipeRunError(PipelexError):class PipeOperatorModelChoiceError(PipelexError):- """Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``)+ """Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``, ``PipeJudge``)when it is built from its blueprint and a model field names a model the model deck does not define.Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``,and a run, which loads its bundle before any pipe runs, refuses the bundle with that same verdict.diff --git a/pipelex/kernel/judgment_ops.py b/pipelex/kernel/judgment_ops.pynew file mode 100644index 000000000..b205e52e3--- /dev/null+++ b/pipelex/kernel/judgment_ops.py@@ -0,0 +1,295 @@+"""Judgment operator semantics: deck resolution, question rendering, the material, the verdict, memory write-back.++These are the functions the interpreter's `PipeJudge` calls, and the ones a programmatic caller+invokes on a `RuntimeBoot`-only process. They read the runtime hub for the services a runtime boot+stands up (the model deck, the content generator) and take everything else as an explicit argument.++A judgment is asked over **material**: the step's inputs by name. Every input that is not a file is a+member of a JSON state, its value read off the content's class; every image or document travels beside+the state, keyed by the input's name, as the prompt images and documents an LLM prompt carries. The+dispatch is on the content's class rather than on a concept, because the kernel may not import the+concept library.+"""++from typing import Any, cast++from pydantic_core import to_jsonable_python++from pipelex import log+from pipelex.cogt.content_generation.assignment_models import JudgmentAssignment+from pipelex.cogt.content_generation.cogt_run_params import CogtRunParams+from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.document.prompt_document_factory import PromptDocumentFactory+from pipelex.cogt.exceptions import JudgmentAnswerMismatchError, JudgmentModelMissingError, ModelNotFoundError+from pipelex.cogt.image.prompt_image import PromptImage+from pipelex.cogt.image.prompt_image_factory import PromptImageFactory+from pipelex.cogt.judgment.judgment_models import (+ ChoiceAnswer,+ JudgmentAnswer,+ JudgmentQuestion,+ JudgmentState,+ RatingAnswer,+ YesNoAnswer,+)+from pipelex.cogt.judgment.judgment_setting import JudgmentModelChoice, JudgmentSetting+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.templating.template_rendering import render_template+from pipelex.core.concepts.concept import Concept+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.choice_content import ChoiceContent+from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.rating_content import RatingContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.time_content import TimeContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+from pipelex.kernel.judgment_results import JudgmentResult+from pipelex.kernel.memory_ops import store_result+from pipelex.runtime_hub import get_content_generator, get_model_deck+from pipelex.system.job_metadata import JobMetadata+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.misc.string_utils import get_root_from_dotted_path+from pipelex.tools.templating.templating_style import TemplatingStyle++# The one question a step asks travels under this key: the batch-first contract needs a key, and+# nothing reads it but the answer lookup below.+JUDGMENT_QUESTION_KEY = "question"++# A yes/no verdict reads its reported probability against this when the step declares no threshold.+DEFAULT_YES_NO_THRESHOLD = 0.5+++def judgment_setting_of_choice(*, judgment_choice: JudgmentModelChoice | None = None, pipe_code: str | None = None) -> JudgmentSetting:+ """The deck chain for a judgment: the step's own model, else the deck's default, as a setting.++ The deck serves no judgment model by default, so when the step names none and the deck names no+ default either, there is nothing to resolve: that is `JudgmentModelMissingError`, the same refusal+ the operator raises when its method loads. The setting's model is still the handle the deck names,+ which no backend may serve on this boot.+ """+ model_deck = get_model_deck()+ resolved_choice = judgment_choice or model_deck.judgment_choice_default+ if resolved_choice is None:+ raise JudgmentModelMissingError(pipe_code=pipe_code)+ return model_deck.get_judgment_setting(judgment_choice=resolved_choice)+++def served_judgment_model(*, model_handle: str) -> InferenceModelSpec | None:+ """The spec of the judgment model a handle resolves to on this boot, `None` when no backend serves one.++ The deck answers `None` for an alias or a handle it does not serve, but raises for a waterfall none of+ whose models it serves, or whose fallbacks are disabled: here the two mean the same thing.+ """+ try:+ return get_model_deck().get_optional_inference_model(model_handle=model_handle, model_type=ModelType.JUDGMENT)+ except ModelNotFoundError:+ return None+++def resolve_judgment_setting(+ *, judgment_choice: JudgmentModelChoice | None = None, pipe_code: str | None = None, is_dry: bool = False+) -> JudgmentSetting:+ """The deck chain for a judgment, then handle resolution.++ The returned setting is pinned to the *resolved* handle, so it doubles as a distributed run's+ routing key. A dry run calls no judging worker, so a model the deck names that no backend serves on+ this boot, as on a keyless boot that skipped the backend whose key is unset, keeps the deck's handle.+ """+ model_deck = get_model_deck()+ judgment_setting = judgment_setting_of_choice(judgment_choice=judgment_choice, pipe_code=pipe_code)+ if is_dry and served_judgment_model(model_handle=judgment_setting.model) is None:+ return judgment_setting++ inference_model = model_deck.get_required_inference_model(model_handle=judgment_setting.model, model_type=ModelType.JUDGMENT)+ if inference_model.name != judgment_setting.model:+ judgment_setting = judgment_setting.model_copy(update={"model": inference_model.name})+ return judgment_setting+++async def run_judgment(+ *,+ memory: WorkingMemory,+ question: JudgmentQuestion,+ input_names: list[str],+ judgment_setting: JudgmentSetting,+ concept: Concept,+ job_metadata: JobMetadata,+ cogt_run_params: CogtRunParams,+ templating_style: TemplatingStyle,+ threshold: float | None = None,+ result_name: str | None = None,+ result_code: str | None = None,+) -> JudgmentResult:+ """A whole judgment step: render the question, gather the material, judge, read the verdict, store.++ The question arrives as the cogt question with its instructions still a template, rendered here+ against memory as a search renders its query. The judgment itself goes through the same+ content-generation seam as every other leaf — direct inline, an activity when in-workflow, or a dry+ mock — so it is replay-safe under a distributed orchestrator.++ The content stored is the verdict native's own class, never the output concept's: a concept+ refining `Choice` has no structure of its own, so its content is a `ChoiceContent`, stored under the+ refining concept as every operator stores a refinement.+ """+ with job_metadata.log_context():+ rendered_question = await render_template(+ template=question.instructions,+ category=TemplateCategory.BASIC,+ context=memory.generate_context(),+ templating_style=templating_style,+ )+ asked_question = question.model_copy(update={"instructions": rendered_question})+ state, images, documents = build_judgment_material(memory=memory, input_names=input_names)+ judgment_assignment = JudgmentAssignment(+ job_metadata=job_metadata,+ cogt_run_params=cogt_run_params,+ state=state,+ images=images,+ documents=documents,+ questions={JUDGMENT_QUESTION_KEY: asked_question},+ judgment_setting=judgment_setting,+ )+ answers = await get_content_generator().make_judgment_answers(judgment_assignment=judgment_assignment)+ answer = answers[JUDGMENT_QUESTION_KEY]+ content, threshold_applied = make_verdict_content(answer=answer, threshold=threshold, is_dry=cogt_run_params.run_mode.is_dry)+ if threshold_applied is False:+ log.warning(+ f"Judgment '{job_metadata.pipe_code or 'unnamed step'}' declares a threshold of {threshold}, but model "+ f"'{judgment_setting.model}' reported no probability, so its own verdict stands."+ )+ return JudgmentResult(+ memory=store_result(memory=memory, concept=concept, content=content, result_name=result_name, result_code=result_code),+ content=content,+ rendered_question=rendered_question,+ judgment_setting=judgment_setting,+ answer=answer,+ threshold_applied=threshold_applied,+ )+++def build_judgment_material(+ *,+ memory: WorkingMemory,+ input_names: list[str],+) -> tuple[JudgmentState, dict[str, list[PromptImage]], dict[str, list[PromptDocument]]]:+ """The state and the files a judgment is asked over, one entry per input, keyed by its declared name.++ An optional input that holds no value, whether its absence was recorded or it was never written, is+ left out rather than sent as a null: a required input with no value never reaches the step, whose+ presence scan refuses it. A dotted input, such as `invoice.total`, is the value at its path, keyed+ by its full name, and is left out when the path holds nothing. An image or a document, or a list of+ either, goes to the file channel; every other input is a member of the state. An image nested inside+ a structured input is part of that input's JSON, its URL as text, which is the author's concern.+ """+ state: JudgmentState = {}+ images: dict[str, list[PromptImage]] = {}+ documents: dict[str, list[PromptDocument]] = {}+ for input_name in input_names:+ stuff = memory.get_optional_stuff(name=get_root_from_dotted_path(input_name))+ if stuff is None:+ continue+ content: StuffContent = stuff.content+ if "." in input_name:+ field_value = memory.get_typed_object_or_attribute(name=input_name, accept_list=True)+ if field_value is None:+ continue+ if isinstance(field_value, list) and all(isinstance(item, StuffContent) for item in field_value): # pyright: ignore[reportUnknownVariableType]+ field_value = ListContent[StuffContent](items=cast("list[StuffContent]", field_value))+ if not isinstance(field_value, StuffContent):+ state[input_name] = to_jsonable_python(field_value, exclude_none=True, serialize_as_any=True)+ continue+ content = field_value+ if prompt_images := _prompt_images(content=content):+ images[input_name] = prompt_images+ elif prompt_documents := _prompt_documents(content=content):+ documents[input_name] = prompt_documents+ else:+ state[input_name] = material_value(content=content)+ return state, images, documents+++def material_value(*, content: StuffContent) -> Any:+ """The JSON value a content is judged as: the reading a caller would write for it.++ Text (and every refinement of it) is its string, a number its number, a yes/no its boolean, a date+ or a time its ISO string, a JSON object itself, a list the array of its items' values, and every+ other content — a choice, a rating, any structure — its JSON object without its absent members.+ """+ match content:+ case TextContent():+ return content.text+ case NumberContent():+ return content.number+ case YesNoContent():+ return content.yes_no+ case DateContent() | TimeContent():+ return content.rendered_plain()+ case JSONContent():+ return content.json_obj+ case ListContent():+ return [material_value(content=item) for item in content.items]+ case _:+ # Serialized as what each field holds rather than as its declared type, so a field typed as the+ # content base keeps the fields of the subclass a run put there.+ return content.model_dump(mode="json", exclude_none=True, serialize_as_any=True)+++def _prompt_images(*, content: StuffContent) -> list[PromptImage] | None:+ image_contents: list[ImageContent]+ match content:+ case ImageContent():+ image_contents = [content]+ case ListContent() if content.items and all(isinstance(item, ImageContent) for item in content.items):+ image_contents = [item for item in content.items if isinstance(item, ImageContent)]+ case _:+ return None+ return [PromptImageFactory.make_prompt_image(uri=image.url, mime_type=image.mime_type) for image in image_contents]+++def _prompt_documents(*, content: StuffContent) -> list[PromptDocument] | None:+ document_contents: list[DocumentContent]+ match content:+ case DocumentContent():+ document_contents = [content]+ case ListContent() if content.items and all(isinstance(item, DocumentContent) for item in content.items):+ document_contents = [item for item in content.items if isinstance(item, DocumentContent)]+ case _:+ return None+ return [PromptDocumentFactory.make_prompt_document(uri=document.url, mime_type=document.mime_type) for document in document_contents]+++def make_verdict_content(*, answer: JudgmentAnswer, threshold: float | None, is_dry: bool) -> tuple[StuffContent, bool | None]:+ """The verdict native an answer translates to, and what became of a declared threshold.++ A yes/no answer with a probability is decided by it against the threshold, inclusive, the default+ standing in when none is declared; one without a probability keeps its own verdict, and a declared+ threshold then had nothing to work on, which a live run reports and a dry run, whose answers carry+ no probability by construction, does not. A rating's distribution is keyed by the level index+ written as text, the one place that conversion happens. Nothing absent is synthesised.+ """+ match answer:+ case YesNoAnswer():+ if answer.probability is not None:+ yes_no = answer.probability >= (threshold if threshold is not None else DEFAULT_YES_NO_THRESHOLD)+ threshold_applied = True if threshold is not None else None+ return YesNoContent(yes_no=yes_no, probability=answer.probability), threshold_applied+ if answer.yes_no is None:+ # YesNoAnswer's own validator refuses an answer carrying neither, so a validated answer+ # never reaches this; it is stated for the type checker, which cannot see a validator.+ msg = "Judgment worker answered a yes/no question with neither a probability nor a verdict"+ raise JudgmentAnswerMismatchError(msg)+ threshold_applied = False if threshold is not None and not is_dry else None+ return YesNoContent(yes_no=answer.yes_no), threshold_applied+ case ChoiceAnswer():+ return ChoiceContent(choice=answer.choice, confidence=answer.confidence, probabilities=answer.probabilities), None+ case RatingAnswer():+ probabilities = (+ {str(level): probability for level, probability in answer.probabilities.items()} if answer.probabilities is not None else None+ )+ return RatingContent(level=answer.level, confidence=answer.confidence, probabilities=probabilities, position=answer.position), Nonediff --git a/pipelex/kernel/judgment_results.py b/pipelex/kernel/judgment_results.pynew file mode 100644index 000000000..40ba1258c--- /dev/null+++ b/pipelex/kernel/judgment_results.py@@ -0,0 +1,38 @@+"""What a kernel judgment call hands back.++Three intermediates ride along because the kernel is where they are produced and the interpreter's+execution-graph tracer records them: the rendered question, the `JudgmentSetting` *after* handle+resolution, and the judging model's raw answer, which carries a distribution the verdict native's+rendering does not show. The memory contract holds as everywhere else: **the returned `memory` is the+result**.++The serialization note on `pipelex.kernel.llm_results` applies here too — `content` is annotated with+the base `StuffContent`, so use `kajson` rather than a plain `model_dump()`.+"""++from pydantic import BaseModel, ConfigDict++from pipelex.cogt.judgment.judgment_models import JudgmentAnswer+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.stuff_content import StuffContent+++class JudgmentResult(BaseModel):+ """The outcome of a kernel judgment call.++ `content` is the verdict native the answer translates to: a `YesNoContent`, a `ChoiceContent` or+ a `RatingContent`. `threshold_applied` says what became of a declared threshold: `True` when it+ decided a yes/no verdict from the reported probability, `False` when the judging model reported no+ probability on a live run and its own verdict stood, and `None` when no threshold was declared or+ the run was dry.+ """++ model_config = ConfigDict(frozen=True)++ memory: WorkingMemory+ content: StuffContent+ rendered_question: str+ judgment_setting: JudgmentSetting+ answer: JudgmentAnswer+ threshold_applied: bool | None = Nonediff --git a/pipelex/mthds_parsing/pipelex_bundle_blueprint.py b/pipelex/mthds_parsing/pipelex_bundle_blueprint.pyindex 95d4d9d3a..df7a59e5b 100644--- a/pipelex/mthds_parsing/pipelex_bundle_blueprint.py+++ b/pipelex/mthds_parsing/pipelex_bundle_blueprint.py@@ -23,6 +23,7 @@ from pipelex.pipe_operators.doc_gen.pipe_doc_gen_blueprint import PipeDocGenBluefrom pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprintfrom pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprintfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint+from pipelex.pipe_operators.judge.pipe_judge_blueprint import PipeJudgeBlueprintfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprintfrom pipelex.pipe_operators.search.pipe_search_blueprint import PipeSearchBlueprintfrom pipelex.pipe_operators.structure.pipe_structure_blueprint import PipeStructureBlueprint@@ -61,6 +62,7 @@ PipeBlueprintUnion = Annotated[| PipeLLMBlueprint| PipeExtractBlueprint| PipeSearchBlueprint+ | PipeJudgeBlueprint| PipeStructureBlueprint| PipeDocGenBlueprint| PipeBatchBlueprintdiff --git a/pipelex/pipe_machinery/pipe_blueprint.py b/pipelex/pipe_machinery/pipe_blueprint.pyindex 994716c47..7c4f1f160 100644--- a/pipelex/pipe_machinery/pipe_blueprint.py+++ b/pipelex/pipe_machinery/pipe_blueprint.py@@ -133,6 +133,7 @@ class PipeType(StrEnum):PIPE_SEARCH = "PipeSearch"PIPE_STRUCTURE = "PipeStructure"PIPE_DOC_GEN = "PipeDocGen"+ PIPE_JUDGE = "PipeJudge"# Pipe ControllerPIPE_BATCH = "PipeBatch"PIPE_CONDITION = "PipeCondition"@@ -162,6 +163,8 @@ class PipeType(StrEnum):return PipeCategory.PIPE_OPERATORcase PipeType.PIPE_DOC_GEN:return PipeCategory.PIPE_OPERATOR+ case PipeType.PIPE_JUDGE:+ return PipeCategory.PIPE_OPERATORcase PipeType.PIPE_BATCH:return PipeCategory.PIPE_CONTROLLERcase PipeType.PIPE_CONDITION:diff --git a/pipelex/pipe_machinery/registry_models.py b/pipelex/pipe_machinery/registry_models.pyindex a2203bb10..32318ddd2 100644--- a/pipelex/pipe_machinery/registry_models.py+++ b/pipelex/pipe_machinery/registry_models.py@@ -20,6 +20,8 @@ from pipelex.pipe_operators.func.pipe_func import PipeFuncfrom pipelex.pipe_operators.func.pipe_func_factory import PipeFuncFactoryfrom pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGenfrom pipelex.pipe_operators.img_gen.pipe_img_gen_factory import PipeImgGenFactory+from pipelex.pipe_operators.judge.pipe_judge import PipeJudge+from pipelex.pipe_operators.judge.pipe_judge_factory import PipeJudgeFactoryfrom pipelex.pipe_operators.llm.pipe_llm import PipeLLMfrom pipelex.pipe_operators.llm.pipe_llm_factory import PipeLLMFactoryfrom pipelex.pipe_operators.search.pipe_search import PipeSearch@@ -48,6 +50,7 @@ class PipeRegistryModels(RegistryModels):PipeLLM,PipeExtract,PipeSearch,+ PipeJudge,PipeStructure,PipeDocGen,]@@ -59,6 +62,7 @@ class PipeRegistryModels(RegistryModels):PipeLLMFactory,PipeExtractFactory,PipeSearchFactory,+ PipeJudgeFactory,PipeStructureFactory,PipeDocGenFactory,]diff --git a/pipelex/pipe_machinery/rendering/output_renderer.py b/pipelex/pipe_machinery/rendering/output_renderer.pyindex 390eaf4e8..0bd2a525e 100644--- a/pipelex/pipe_machinery/rendering/output_renderer.py+++ b/pipelex/pipe_machinery/rendering/output_renderer.py@@ -117,6 +117,7 @@ def _collect_possible_outputs(| PipeType.PIPE_SEARCH| PipeType.PIPE_STRUCTURE| PipeType.PIPE_DOC_GEN+ | PipeType.PIPE_JUDGE| PipeType.PIPE_BATCH| PipeType.PIPE_PARALLEL):diff --git a/pipelex/pipe_machinery/template_guard_lint.py b/pipelex/pipe_machinery/template_guard_lint.pyindex d0ee2c79b..accfdfbf9 100644--- a/pipelex/pipe_machinery/template_guard_lint.py+++ b/pipelex/pipe_machinery/template_guard_lint.py@@ -1,6 +1,6 @@"""Shared template lints, applied by every pipe that renders authored templates over its inputs(PipeLLM prompts, PipeCompose templates and construct templates, PipeImgGen and PipeSearch prompts,-PipeCondition expressions):+PipeJudge questions, PipeCondition expressions):- the guard-lint (optionals design D7): each reference to a declared-optional (`?`) input must beguarded, otherwise validation fails with `OPTIONAL_INPUT_UNGUARDED` and the precise fix;diff --git a/pipelex/pipe_machinery/validation.py b/pipelex/pipe_machinery/validation.pyindex 1657fbcc6..5ceb7412b 100644--- a/pipelex/pipe_machinery/validation.py+++ b/pipelex/pipe_machinery/validation.py@@ -65,19 +65,43 @@ def check_inputs_match_variables("""Refuse a variable the templates read that no input declares, then a declared input no template reads.The rule shared by every operator that reads its inputs through templates: PipeLLM, PipeCompose,- PipeSearch and PipeImgGen. ``variable_paths`` are the full dotted paths the operator's templates read,- already rid of the operator's special names, and ``reader`` names the fields that read them, for the- message ("prompt or system_prompt", "template").+ PipeSearch and PipeImgGen. It is the composition of `check_variables_are_declared` and+ `check_inputs_are_read`, in that order. PipeJudge calls the first half alone, since every input it+ declares is material to judge whether its question names it or not.++ Raises:+ PipeValidationError: ``MISSING_INPUT_VARIABLE`` with the undeclared root names, or+ ``EXTRANEOUS_INPUT_VARIABLE`` with the unread input names, each sorted.+ """+ check_variables_are_declared(+ declared_inputs=declared_inputs,+ variable_paths=variable_paths,+ reader=reader,+ dotted_input_supplies_its_path=dotted_input_supplies_its_path,+ )+ check_inputs_are_read(declared_inputs=declared_inputs, variable_paths=variable_paths, reader=reader)+++def check_variables_are_declared(+ *,+ declared_inputs: set[str],+ variable_paths: set[str],+ reader: str,+ dotted_input_supplies_its_path: bool,+) -> None:+ """Refuse a variable the templates read that no input declares.++ ``variable_paths`` are the full dotted paths the operator's templates read, already rid of the+ operator's special names, and ``reader`` names the fields that read them, for the message+ ("prompt or system_prompt", "template").``dotted_input_supplies_its_path`` keeps each operator's own rule for a dotted input name such as``page.page_view`` declared without its root. PipeLLM takes it as supplying the path it names, a shape its- tests run; PipeCompose, PipeSearch and PipeImgGen have always required every read's root to be declared.- Whether the four should agree is a language question this check leaves open. Either way, a dotted input- name counts as read by the path it names.+ tests run; PipeCompose, PipeSearch, PipeImgGen and PipeJudge require every read's root to be declared.+ Whether they should agree is a language question this check leaves open.Raises:- PipeValidationError: ``MISSING_INPUT_VARIABLE`` with the undeclared root names, or- ``EXTRANEOUS_INPUT_VARIABLE`` with the unread input names, each sorted.+ PipeValidationError: ``MISSING_INPUT_VARIABLE`` with the undeclared root names, sorted."""if dotted_input_supplies_its_path:unsatisfied_paths = {@@ -94,6 +118,13 @@ def check_inputs_match_variables(msg = f"Variables {quoted_names} are read by the {reader} but not declared in `inputs`. Declare them in `inputs`, or stop reading them."raise PipeValidationError(message=msg, error_type=PipeValidationErrorType.MISSING_INPUT_VARIABLE, variable_names=missing_names)++def check_inputs_are_read(*, declared_inputs: set[str], variable_paths: set[str], reader: str) -> None:+ """Refuse a declared input no template reads. A dotted input name counts as read by the path it names.++ Raises:+ PipeValidationError: ``EXTRANEOUS_INPUT_VARIABLE`` with the unread input names, sorted.+ """unread_names = sorted(input_name for input_name in declared_inputs if not is_input_used_by_variables(input_name, variable_paths=variable_paths))if unread_names:quoted_names = _quoted_names(names=unread_names)diff --git a/pipelex/pipe_operators/judge/__init__.py b/pipelex/pipe_operators/judge/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/judge/exceptions.py b/pipelex/pipe_operators/judge/exceptions.pynew file mode 100644index 000000000..3451eb5c4--- /dev/null+++ b/pipelex/pipe_operators/judge/exceptions.py@@ -0,0 +1,35 @@+from pipelex.base_exceptions import ErrorDomain, PipelexError+++class PipeJudgeError(PipelexError):+ pass+++class PipeJudgeInputCapabilityError(PipeJudgeError):+ """A `PipeJudge` declares an image or a document input, and the judging model it resolves to does not read one.++ Raised when the method loads, before a run spends anything. The operator does not refuse files: the+ model's spec states what it reads, so a model that reads files gets them, and this refusal names the+ input and what the model reads. Its message names only the step, the input, the model and its+ declared inputs, so it is kept verbatim for the caller.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = True++ def __init__(self, *, pipe_code: str, input_name: str, file_kind: str, model_name: str, model_inputs: list[str]):+ self.pipe_code = pipe_code+ self.input_name = input_name+ self.file_kind = file_kind+ self.model_name = model_name+ self.model_inputs = model_inputs+ message = (+ f"PipeJudge '{pipe_code}' judges input '{input_name}', which is {_with_article(noun=file_kind)}, and its judgment model "+ f"'{model_name}' does not read {file_kind}s: it reads {', '.join(model_inputs)}. Name a judgment model that reads "+ f"{file_kind}s, or turn the {file_kind} into text with a step before this one."+ )+ super().__init__(message)+++def _with_article(*, noun: str) -> str:+ return f"an {noun}" if noun[0] in "aeiou" else f"a {noun}"diff --git a/pipelex/pipe_operators/judge/pipe_judge.py b/pipelex/pipe_operators/judge/pipe_judge.pynew file mode 100644index 000000000..b48652eee--- /dev/null+++ b/pipelex/pipe_operators/judge/pipe_judge.py@@ -0,0 +1,253 @@+from typing import Any, Literal++from typing_extensions import override++from pipelex import log+from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentKind, JudgmentQuestion, RatingQuestion, YesNoQuestion+from pipelex.cogt.judgment.judgment_setting import JudgmentModelChoice+from pipelex.cogt.models.model_deck_check import check_judgment_choice_with_deck+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.exceptions import PipeValidationError+from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.interpreter_hub import get_concept_library, get_native_concept+from pipelex.kernel.judgment_ops import judgment_setting_of_choice, resolve_judgment_setting, run_judgment, served_judgment_model+from pipelex.kernel.templating_style_ops import resolve_templating_style+from pipelex.pipe_machinery.template_guard_lint import lint_authored_template+from pipelex.pipe_operators.judge.exceptions import PipeJudgeInputCapabilityError+from pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipe_run.pipe_run_params import PipeRunParams+from pipelex.system.job_metadata import JobMetadata+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.misc.string_utils import get_root_from_dotted_path+from pipelex.validation_error_types import PipeValidationErrorType+++class PipeJudgeOutput(PipeOutput):+ pass+++def verdict_native_for_kind(*, judgment_kind: JudgmentKind) -> NativeConceptCode:+ """The verdict native a question of this kind produces."""+ match judgment_kind:+ case JudgmentKind.YES_NO:+ return NativeConceptCode.YES_NO+ case JudgmentKind.CHOICE:+ return NativeConceptCode.CHOICE+ case JudgmentKind.RATING:+ return NativeConceptCode.RATING+++def _kind_declaration(*, judgment_kind: JudgmentKind) -> str:+ match judgment_kind:+ case JudgmentKind.YES_NO:+ return "declares neither `options` nor `levels`, so it asks a yes/no question"+ case JudgmentKind.CHOICE:+ return "declares `options`, so it asks a choice question"+ case JudgmentKind.RATING:+ return "declares `levels`, so it asks a rating question"+++class PipeJudge(PipeOperator[PipeJudgeOutput]):+ """Asks a judging model one closed question about its inputs, and stores the verdict native it answers with.++ `judgment_question` is the cogt question with its instructions still the authored template, so the+ kernel renders it against memory when the step runs; its kind decides the verdict native.+ """++ type: Literal["PipeJudge"] = "PipeJudge"+ judgment_choice: JudgmentModelChoice | None+ judgment_question: JudgmentQuestion+ threshold: float | None = None++ @property+ def judgment_kind(self) -> JudgmentKind:+ return self.judgment_question.kind++ @property+ def question_blueprint(self) -> TemplateBlueprint:+ return TemplateBlueprint(template=self.judgment_question.instructions, category=TemplateCategory.BASIC)++ @override+ def needed_inputs(self, *, visited_pipes: set[str] | None = None) -> InputStuffSpecs:+ return self.inputs++ @override+ def required_variables(self) -> set[str]:+ full_paths = self.question_blueprint.required_variables()+ return {get_root_from_dotted_path(path) for path in full_paths if not path.startswith("_")}++ @override+ def validate_inputs_static(self):+ if self.judgment_choice is not None:+ with self.locating_model_choice(field_name="model"):+ check_judgment_choice_with_deck(judgment_choice=self.judgment_choice)++ # Template lints: no private names, and every reference to a declared-optional input guarded (D7).+ question_blueprint = self.question_blueprint+ lint_authored_template(+ pipe_code=self.code,+ domain_code=self.domain_code,+ inputs=self.inputs,+ template_source=question_blueprint.template,+ template_category=question_blueprint.category,+ template_label="question",+ )++ @override+ def validate_inputs_with_library(self):+ """Find the judging model, and refuse an image or a document input it does not read.++ The model is found when the method loads, so a step with no model to judge with, naming none+ where the deck names no default, is refused with `JudgmentModelMissingError` before a run spends+ anything: the deck serves no judgment model out of the box. Every input is material to judge, so+ an `Image` or a `Document` input, or a list of either, is sent to the model as a file. Whether the+ model reads one is its own capability, stated in its spec's `inputs`: a model that does not is+ refused here, naming the input and what it reads. That needs the model's spec, which a boot that+ skipped its backend, a keyless one with the backend's key unset, does not hold, and neither does+ one serving none of a waterfall's models: the check is then left to the worker when the step runs,+ as it is for a `Dynamic` input, whose declaration names no kind of value.+ """+ with self.locating_model_choice(field_name="model"):+ judgment_setting = judgment_setting_of_choice(judgment_choice=self.judgment_choice, pipe_code=self.code)+ inference_model = served_judgment_model(model_handle=judgment_setting.model)+ if inference_model is None:+ return+ concept_library = get_concept_library()+ image_concept = get_native_concept(native_concept=NativeConceptCode.IMAGE)+ document_concept = get_native_concept(native_concept=NativeConceptCode.DOCUMENT)+ for input_name, stuff_spec in self.inputs.items:+ if NativeConceptCode.is_dynamic_concept(concept_code=stuff_spec.concept.code):+ # `Dynamic` is compatible with every concept, so its declaration says nothing about files:+ # the worker checks what such an input actually holds when the step runs.+ continue+ if concept_library.is_compatible(tested_concept=stuff_spec.concept, wanted_concept=image_concept, strict=True):+ if not inference_model.is_vision_supported:+ raise PipeJudgeInputCapabilityError(+ pipe_code=self.code,+ input_name=input_name,+ file_kind="image",+ model_name=inference_model.name,+ model_inputs=inference_model.inputs,+ )+ elif concept_library.is_compatible(tested_concept=stuff_spec.concept, wanted_concept=document_concept, strict=True):+ if not inference_model.is_document_supported:+ raise PipeJudgeInputCapabilityError(+ pipe_code=self.code,+ input_name=input_name,+ file_kind="document",+ model_name=inference_model.name,+ model_inputs=inference_model.inputs,+ )++ @override+ def validate_output_static(self):+ pass++ @override+ def validate_output_with_library(self):+ """The output agrees with the question's kind: its verdict native, or a concept refining it.++ `Dynamic` is compatible with every concept, but a judgment's verdict is always its kind's native,+ so a `Dynamic` output is refused like any other that is not that native.+ """+ verdict_native = verdict_native_for_kind(judgment_kind=self.judgment_kind)+ if not NativeConceptCode.is_dynamic_concept(concept_code=self.output.concept.code) and get_concept_library().is_compatible(+ tested_concept=self.output.concept,+ wanted_concept=get_native_concept(native_concept=verdict_native),+ strict=True,+ ):+ return+ msg = (+ f"This PipeJudge {_kind_declaration(judgment_kind=self.judgment_kind)}: its output must be `{verdict_native}` "+ f"or a concept refining it, and it declares `{self.output.concept.concept_ref}`."+ )+ raise PipeValidationError(+ message=msg,+ error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,+ domain_code=self.domain_code,+ pipe_code=self.code,+ provided_concept_code=self.output.concept.concept_ref,+ required_concept_codes=[verdict_native.concept_ref],+ )++ @override+ async def _live_run_operator_pipe(+ self,+ *,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeJudgeOutput:+ log.dev(f"✨ PipeJudge '{self.code}' asking a {self.judgment_kind} question with judgment choice '{self.judgment_choice or 'default'}' ✨")++ # Resolved per run into a local and never cached onto `self`, for the reason `pipe_llm.py`+ # states about its own settings.+ judgment_setting = resolve_judgment_setting(judgment_choice=self.judgment_choice, pipe_code=self.code, is_dry=pipe_run_params.run_mode.is_dry)+ judgment_result = await run_judgment(+ memory=working_memory,+ question=self.judgment_question,+ input_names=self.inputs.variables,+ judgment_setting=judgment_setting,+ concept=self.output.concept,+ job_metadata=job_metadata,+ cogt_run_params=pipe_run_params.cogt_run_params,+ # No authored style on this operator: a question template takes the runtime default, the+ # same one an LLM pipe that declares nothing gets.+ templating_style=resolve_templating_style(authored=None),+ threshold=self.threshold,+ result_name=output_name,+ )++ # Capture execution data for the graph tracer. The raw answer rides along because it carries+ # what the verdict native's rendering does not show, such as a choice's whole distribution.+ execution_data_dict: dict[str, Any] = {+ "rendered_question": judgment_result.rendered_question,+ "resolved_model": judgment_result.judgment_setting.model,+ "judgment_kind": self.judgment_kind.value,+ "answer": judgment_result.answer.model_dump(mode="json", exclude_none=True),+ }+ if self.threshold is not None:+ execution_data_dict["threshold"] = self.threshold+ if judgment_result.threshold_applied is not None:+ execution_data_dict["threshold_applied"] = judgment_result.threshold_applied++ self._register_execution_data(job_metadata=job_metadata, execution_data=execution_data_dict)+ return PipeJudgeOutput(+ working_memory=judgment_result.memory,+ pipeline_run_id=job_metadata.run_metadata.pipeline_run_id,+ )++ @override+ async def _validate_before_run(+ self, *, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, *, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass+++def make_judgment_question(+ *,+ question_template: str,+ options: dict[str, str] | None,+ levels: list[str] | None,+ yes_criterion: str | None,+ no_criterion: str | None,+) -> JudgmentQuestion:+ """The cogt question frame for a step, its instructions the authored template, its kind read off the fields declared.++ An option declared with an empty description is undescribed, which the cogt question reads as `None`.+ """+ if options is not None:+ return ChoiceQuestion(instructions=question_template, options={key: description or None for key, description in options.items()})+ if levels is not None:+ return RatingQuestion(instructions=question_template, levels=levels)+ return YesNoQuestion(instructions=question_template, yes_criterion=yes_criterion, no_criterion=no_criterion)diff --git a/pipelex/pipe_operators/judge/pipe_judge_blueprint.py b/pipelex/pipe_operators/judge/pipe_judge_blueprint.pynew file mode 100644index 000000000..6e6911ea6--- /dev/null+++ b/pipelex/pipe_operators/judge/pipe_judge_blueprint.py@@ -0,0 +1,178 @@+from typing import Any, Literal, Self++from pydantic import BaseModel, ConfigDict, model_validator+from typing_extensions import override++from pipelex.cogt.judgment.judgment_models import JudgmentKind+from pipelex.cogt.judgment.judgment_setting import JudgmentModelChoice+from pipelex.cogt.templating.exceptions import TemplateSigilSyntaxError+from pipelex.cogt.templating.template_preprocessor import preprocess_template+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity+from pipelex.pipe_machinery.pipe_blueprint import PipeBlueprint+from pipelex.pipe_machinery.validation import check_variables_are_declared+from pipelex.tools.jinja2.exceptions import Jinja2TemplateSyntaxError+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.misc.string_utils import get_root_from_dotted_path++# The one synonym the language admits: every other inference operator calls its template `prompt`.+QUESTION_SYNONYM = "prompt"+++def read_prompt_as_question(*, values: dict[str, Any]) -> dict[str, Any]:+ """The fields with `prompt` renamed `question`, refusing both spellings; the caller's dict is never changed."""+ if QUESTION_SYNONYM not in values:+ return values+ if "question" in values:+ msg = f"A PipeJudge sets `question`, or `{QUESTION_SYNONYM}` as its synonym, but not both: remove one of the two."+ raise ValueError(msg)+ fields = dict(values)+ fields["question"] = fields.pop(QUESTION_SYNONYM)+ return fields+++def _admit_the_question_synonym(schema: dict[str, Any]) -> None: # kw-only: ignore — pydantic calls `json_schema_extra` positionally+ """Give the JSON schema the synonym the before-validator reads, requiring exactly one of the two spellings.++ The synonym is not a field, so without this the schema would require `question` and, forbidding+ extra keys, refuse a table that writes `prompt`, which the blueprint loads.+ """+ properties = schema["properties"]+ properties[QUESTION_SYNONYM] = {**properties["question"], "title": "Prompt", "description": "A synonym of `question`, read exactly as it."}+ schema["required"] = [name for name in schema.get("required", []) if name != "question"]+ schema["oneOf"] = [{"required": ["question"]}, {"required": [QUESTION_SYNONYM]}]+++class JudgeYesNoCriteria(BaseModel):+ """What a yes and a no mean, for a yes/no question. Closed: a key other than `yes` or `no` is refused.++ The judging vendor silently ignores a criterion key it does not know, so this model is the only+ place a typo in one is caught.+ """++ model_config = ConfigDict(extra="forbid")++ yes: str | None = None+ no: str | None = None+++class PipeJudgeBlueprint(PipeBlueprint):+ """Asks a judging model one closed question about its inputs: yes/no, a choice, or a rating.++ The kind is decided by which of `options` and `levels` the pipe declares, never by a field of its+ own. `prompt` is accepted in place of `question` and read exactly as it, so nothing after parsing+ sees the synonym. Every declared input is material to judge, so an input the question never names+ is still read; only a variable the question names must be declared.+ """++ model_config = ConfigDict(json_schema_extra=_admit_the_question_synonym)++ type: Literal["PipeJudge"] = "PipeJudge"+ pipe_category: Literal["PipeOperator"] = "PipeOperator"+ question: str+ model: JudgmentModelChoice | None = None+ options: dict[str, str] | None = None+ levels: list[str] | None = None+ criteria: JudgeYesNoCriteria | None = None+ threshold: float | None = None++ @model_validator(mode="before")+ @classmethod+ def read_prompt_synonym(cls, values: dict[str, Any]) -> dict[str, Any]:+ return read_prompt_as_question(values=values)++ @model_validator(mode="after")+ def validate_question_kind(self) -> Self:+ if not self.question.strip():+ msg = "A PipeJudge asks one question, so its `question` cannot be empty."+ raise ValueError(msg)+ if self.options is not None and self.levels is not None:+ msg = "A PipeJudge declares `options` for a choice question or `levels` for a rating question, not both."+ raise ValueError(msg)+ if self.options is not None or self.levels is not None:+ kind_field = "options" if self.options is not None else "levels"+ for yes_no_field in ("criteria", "threshold"):+ if getattr(self, yes_no_field) is not None:+ msg = (+ f"`{yes_no_field}` applies to a yes/no question only, and this PipeJudge declares `{kind_field}`, "+ f"which makes it a {self.judgment_kind} question. Remove `{yes_no_field}`."+ )+ raise ValueError(msg)+ if self.options is not None:+ if len(self.options) < 2:+ msg = f"A choice question needs at least two `options` to choose between, and this one declares {len(self.options)}."+ raise ValueError(msg)+ if any(not option.strip() for option in self.options):+ msg = "Every key of `options` names an option, so none may be empty."+ raise ValueError(msg)+ if self.levels is not None:+ if len(self.levels) < 2:+ msg = f"A rating question needs at least two `levels` on its scale, and this one declares {len(self.levels)}."+ raise ValueError(msg)+ empty_indices = [index for index, level in enumerate(self.levels) if not level.strip()]+ if empty_indices:+ msg = f"Every level of a rating question describes a situation, so none may be empty: level {empty_indices[0]} is."+ raise ValueError(msg)+ if self.threshold is not None and not 0 < self.threshold < 1:+ msg = f"`threshold` is a probability of yes strictly between 0 and 1, and this PipeJudge declares {self.threshold}."+ raise ValueError(msg)+ return self++ @property+ def judgment_kind(self) -> JudgmentKind:+ if self.options is not None:+ return JudgmentKind.CHOICE+ if self.levels is not None:+ return JudgmentKind.RATING+ return JudgmentKind.YES_NO++ @override+ def validate_inputs(self):+ template_category = TemplateCategory.BASIC+ declared_inputs: set[str] = set(self.inputs.keys()) if self.inputs else set()+ # A step's inputs are keyed by their root once loaded, so a dotted input would be judged as its whole+ # root, fields the step never declared included: refused until the runtime keeps the path.+ for input_name in sorted(declared_inputs):+ if "." in input_name:+ root_name = get_root_from_dotted_path(input_name)+ msg = (+ f"A PipeJudge sends each of its inputs whole, keyed by its name, so input '{input_name}' cannot name a field: "+ f"declare '{root_name}' instead."+ )+ raise ValueError(msg)+ try:+ preprocessed_template = preprocess_template(self.question, declared_inputs=declared_inputs)+ except TemplateSigilSyntaxError as exc:+ msg = f"Template sigil error in PipeJudge question: {exc}"+ raise ValueError(msg) from exc+ try:+ check_jinja2_parsing(+ template_source=preprocessed_template,+ template_category=template_category,+ )+ except Jinja2TemplateSyntaxError as exc:+ msg = f"Could not parse the question template for PipeJudge: {exc}"+ raise ValueError(msg) from exc++ full_paths = detect_jinja2_required_variables(+ template_category=template_category,+ template_source=preprocessed_template,+ )+ # Names starting with an underscore are internal and never count as read inputs+ variable_paths = {path for path in full_paths if not get_root_from_dotted_path(path).startswith("_")}+ # Every declared input is material to judge, read whether the question names it or not: only+ # the variables the question does name must be declared.+ check_variables_are_declared(+ declared_inputs=declared_inputs, variable_paths=variable_paths, reader="question", dotted_input_supplies_its_path=False+ )++ @override+ def validate_output(self):+ parsed_output = parse_concept_with_multiplicity(concept_ref_or_code=self.output)+ if parsed_output.multiplicity:+ msg = (+ f"A PipeJudge produces one verdict, so its output carries no multiplicity, and this one declares '{self.output}'. "+ "To judge each item of a list, map the PipeJudge over it with a PipeBatch."+ )+ raise ValueError(msg)diff --git a/pipelex/pipe_operators/judge/pipe_judge_factory.py b/pipelex/pipe_operators/judge/pipe_judge_factory.pynew file mode 100644index 000000000..7ef0616af--- /dev/null+++ b/pipelex/pipe_operators/judge/pipe_judge_factory.py@@ -0,0 +1,44 @@+from typing import Any++from typing_extensions import override++from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs+from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec+from pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocol+from pipelex.pipe_operators.judge.pipe_judge import PipeJudge, make_judgment_question+from pipelex.pipe_operators.judge.pipe_judge_blueprint import PipeJudgeBlueprint+++class PipeJudgeFactory(PipeFactoryProtocol[PipeJudgeBlueprint, PipeJudge]):+ @classmethod+ @override+ def make(+ cls,+ *,+ pipe_category: Any,+ pipe_type: str,+ pipe_code: str,+ domain_code: str,+ description: str,+ inputs: InputStuffSpecs,+ output: StuffSpec,+ blueprint: PipeJudgeBlueprint,+ ) -> PipeJudge:+ criteria = blueprint.criteria+ judgment_question = make_judgment_question(+ question_template=blueprint.question,+ options=blueprint.options,+ levels=blueprint.levels,+ yes_criterion=criteria.yes if criteria is not None else None,+ no_criterion=criteria.no if criteria is not None else None,+ )+ return PipeJudge(+ domain_code=domain_code,+ code=pipe_code,+ description=description,+ output=output,+ inputs=inputs,+ judgment_choice=blueprint.model,+ judgment_question=judgment_question,+ threshold=blueprint.threshold,+ )diff --git a/pipelex/pipeline/file_input_consumers.py b/pipelex/pipeline/file_input_consumers.pyindex fcda2d876..a13f8c0b1 100644--- a/pipelex/pipeline/file_input_consumers.py+++ b/pipelex/pipeline/file_input_consumers.py@@ -19,8 +19,9 @@ resolving sub-pipes through the hub:unresolved cross-package reference consume nothing as far as the walk knows: it never guesses.- **Consumers.** A `PipeExtract` consumes its document input, and reads the formats of the model itsextract choice resolves to. A `PipeLLM` consumes the documents its prompt references by variable- path, and reads its resolved model's document types. A waterfall reads a format when any of its- members reads it.+ path, and reads its resolved model's document types. A `PipeJudge` consumes every document input+ it declares, since every input is material to judge, and reads its resolved judgment model's+ document types. A waterfall reads a format when any of its members reads it.The walk depends only on the library and the model deck, never on the input values."""@@ -30,7 +31,7 @@ from typing import Final, NamedTuplefrom pydantic import BaseModel, ConfigDict-from pipelex.cogt.exceptions import ModelChoiceNotFoundError+from pipelex.cogt.exceptions import JudgmentModelMissingError, ModelChoiceNotFoundErrorfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.exceptions import ModelReferenceParseError@@ -38,6 +39,7 @@ from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKifrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.interpreter_hub import get_concept_library, get_native_concept, get_optional_pipefrom pipelex.kernel.extract_ops import resolve_extract_setting+from pipelex.kernel.judgment_ops import judgment_setting_of_choicefrom pipelex.kernel.llm_ops import resolve_llm_setting_for_object, resolve_llm_setting_for_textfrom pipelex.pipe_controllers.batch.pipe_batch import PipeBatchfrom pipelex.pipe_controllers.condition.pipe_condition import PipeCondition@@ -46,6 +48,7 @@ from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequencefrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipe_machinery.pipe_abstract import PipeAbstractfrom pipelex.pipe_operators.extract.pipe_extract import PipeExtract+from pipelex.pipe_operators.judge.pipe_judge import PipeJudgefrom pipelex.pipe_operators.llm.pipe_llm import PipeLLMfrom pipelex.pipe_run.pipe_run_params import BatchParamsfrom pipelex.pipeline.exceptions import PipelineInputFormatUnsupportedError@@ -75,6 +78,7 @@ class FileConsumerKind(StrEnum):EXTRACT = "extract"LLM_DOCUMENT = "llm_document"+ JUDGMENT_DOCUMENT = "judgment_document"@propertydef model_type(self) -> ModelType:@@ -83,6 +87,8 @@ class FileConsumerKind(StrEnum):return ModelType.TEXT_EXTRACTORcase FileConsumerKind.LLM_DOCUMENT:return ModelType.LLM+ case FileConsumerKind.JUDGMENT_DOCUMENT:+ return ModelType.JUDGMENTclass FileInputConsumer(BaseModel):@@ -143,7 +149,7 @@ class FileInputConsumer(BaseModel):match self.kind:case FileConsumerKind.EXTRACT:return "extracts it with"- case FileConsumerKind.LLM_DOCUMENT:+ case FileConsumerKind.LLM_DOCUMENT | FileConsumerKind.JUDGMENT_DOCUMENT:return "gives it to"def covers(self, *, file_input_path: tuple[str | int, ...]) -> bool:@@ -309,6 +315,8 @@ def _visit(*, pipe: PipeAbstract, frame: _Frame, is_conditional: bool, visiting:_record_extract_consumer(pipe_extract=pipe, frame=frame, is_conditional=is_conditional, consumers=consumers)elif isinstance(pipe, PipeLLM):_record_llm_document_consumers(pipe_llm=pipe, frame=frame, is_conditional=is_conditional, consumers=consumers)+ elif isinstance(pipe, PipeJudge):+ _record_judgment_document_consumers(pipe_judge=pipe, frame=frame, is_conditional=is_conditional, consumers=consumers)def _visit_sequence(@@ -496,6 +504,55 @@ def _record_llm_document_consumers(*, pipe_llm: PipeLLM, frame: _Frame, is_condi)+def _record_judgment_document_consumers(*, pipe_judge: PipeJudge, frame: _Frame, is_conditional: bool, consumers: list[FileInputConsumer]) -> None:+ # A `Dynamic` input declares no kind of value, and an image input is an image by the setup check,+ # so only the inputs declared as documents are followed, as the operator's load-time check does.+ concept_library = get_concept_library()+ document_concept = get_native_concept(native_concept=NativeConceptCode.DOCUMENT)+ document_input_names = [+ input_name+ for input_name, stuff_spec in pipe_judge.inputs.items+ if not NativeConceptCode.is_dynamic_concept(concept_code=stuff_spec.concept.code)+ and concept_library.is_compatible(tested_concept=stuff_spec.concept, wanted_concept=document_concept, strict=True)+ ]+ if not document_input_names:+ return+ resolved_model = _resolve_judgment_model(pipe_judge=pipe_judge)+ readable_formats: frozenset[str] | None = None+ if resolved_model is not None:+ readable_formats = frozenset[str]().union(*(model_spec.supported_document_types for model_spec in resolved_model.specs))+ # A model that reads no documents at all is the method author's choice of model, which the+ # method's load refuses: there is no caller's format to refuse.+ if not readable_formats:+ return+ for input_name in document_input_names:+ consumed_path = _tracked_path(frame=frame, variable_path=input_name)+ if consumed_path is None:+ continue+ consumers.append(+ FileInputConsumer(+ slot_name=consumed_path[0],+ consumed_path=consumed_path,+ pipe_ref=pipe_judge.pipe_ref,+ pipe_code=pipe_judge.code,+ kind=FileConsumerKind.JUDGMENT_DOCUMENT,+ model=resolved_model.model if resolved_model else None,+ readable_formats=readable_formats,+ reads_web_pages=False,+ is_conditional=is_conditional,+ )+ )+++def _resolve_judgment_model(*, pipe_judge: PipeJudge) -> _ResolvedModel | None:+ """The model a PipeJudge's choice resolves to, through the deck chain and any preset, `None` when unresolvable."""+ try:+ model_reference = judgment_setting_of_choice(judgment_choice=pipe_judge.judgment_choice, pipe_code=pipe_judge.code).model+ except (JudgmentModelMissingError, ModelChoiceNotFoundError, ModelReferenceParseError):+ return None+ return _resolved_model(model_reference=model_reference, model_type=ModelType.JUDGMENT)++def _resolve_extract_model(*, pipe_extract: PipeExtract) -> _ResolvedModel | None:"""The model a PipeExtract's choice resolves to, through the deck chain and any preset, `None` when unresolvable."""model_deck = get_model_deck()diff --git a/pipelex/providers/typesafe/typesafe_judgment_worker.py b/pipelex/providers/typesafe/typesafe_judgment_worker.pyindex b4f9dc57c..3ce0dc4a1 100644--- a/pipelex/providers/typesafe/typesafe_judgment_worker.py+++ b/pipelex/providers/typesafe/typesafe_judgment_worker.py@@ -2,6 +2,7 @@ from typesafe_sdk import AsyncTypeSafeClient, SystemOneResponse, TypeSafeErrorfrom typing_extensions import overridefrom pipelex import log+from pipelex.cogt.exceptions import JudgmentCapabilityErrorfrom pipelex.cogt.inference.error_classification import extract_typesafe_metadatafrom pipelex.cogt.inference.error_render import InferenceErrorFamily, render_inference_errorfrom pipelex.cogt.judgment.judgment_job import JudgmentJob@@ -38,6 +39,16 @@ class TypesafeJudgmentWorker(JudgmentWorkerAbstract):self,judgment_job: JudgmentJob,) -> dict[str, JudgmentAnswer]:+ if judgment_job.images or judgment_job.documents:+ # System One judges a JSON state and takes no file, so a request built from this job would+ # drop the files and the verdict would be given over less than was asked. The kit's spec+ # says the model reads text alone, which refuses files before this; a spec that claims more+ # is refused here rather than believed.+ msg = (+ f"TypeSafe judges a JSON state alone, and this judgment carries files for model '{self.inference_model.name}': "+ "its spec must not declare that it reads images or documents."+ )+ raise JudgmentCapabilityError(msg)typesafe_questions = to_typesafe_questions(questions=judgment_job.questions)try:# The SDK types its state as a recursive JSON alias that pyright cannot resolve to the end.diff --git a/pipelex/runtime_hub.py b/pipelex/runtime_hub.pyindex 48caf3ba0..b6dcef79b 100644--- a/pipelex/runtime_hub.py+++ b/pipelex/runtime_hub.py@@ -597,7 +597,7 @@ _content_generator_override: ContextVar[ContentGeneratorProtocol | None] = Contedef scoped_content_generator(content_generator: ContentGeneratorProtocol) -> Generator[None, None, None]:"""Set ``content_generator`` as the active generator for the scope, then restore the prior value on exit.- Inference operators (PipeLLM / PipeImgGen / PipeExtract / PipeSearch / PipeStructure) resolve+ Inference operators (PipeLLM / PipeImgGen / PipeExtract / PipeSearch / PipeJudge / PipeStructure) resolve:func:`get_content_generator`; under a Temporal-enabled hub that default is``ContentGeneratorInWorkflow``, which dispatches activities. An in-process run (e.g. thedry-run/validation activity body) wraps itself in this scope with an inline generator so itsdiff --git a/pipelex/test_extras/error_report_parity.py b/pipelex/test_extras/error_report_parity.pyindex 82e36cfe9..ef7bb5b9e 100644--- a/pipelex/test_extras/error_report_parity.py+++ b/pipelex/test_extras/error_report_parity.py@@ -16,7 +16,7 @@ The only genuinely per-repo bit — the ``.mthds`` bundle each repo ships to drifrom typing import ClassVar-from pipelex.cogt.exceptions import InferenceErrorCategory, LLMCompletionError, SearchJobFailureError+from pipelex.cogt.exceptions import InferenceErrorCategory, JudgmentJobFailureError, LLMCompletionError, SearchJobFailureErrorfrom pipelex.cogt.inference.error_classification import UserAction, UserActionKind@@ -104,3 +104,46 @@ class SearchErrorReportParityTestData:error.model_handle = cls.FAILURE_MODELerror.backend_name = cls.FAILURE_PROVIDERreturn error+++class JudgeErrorReportParityTestData:+ """Constants + factory driving both arms of the local / Temporal ``ErrorReport`` parity test for judgment.++ The judgment counterpart of :class:`SearchErrorReportParityTestData`. A single deliberately-failing+ ``PipeJudge`` leaf is the source of truth for both arms: the local arm runs the ``native_judge`` pipe+ through the direct ``ContentGenerator`` with ``make_judgment_answers`` mocked to fail; a distributed+ arm runs the same pipe with its activity-side leaf mocked to fail. Both assert the same ``EXPECTED_*``+ classification.+ """++ PIPE_CODE: ClassVar[str] = "native_judge"++ # The worker-side failure injected into the judgment call. CONFIGURATION is non-retryable, so a+ # distributed activity is not retried and the workflow fails on the first attempt — fast and deterministic.+ FAILURE_MESSAGE: ClassVar[str] = "Judgment provider rejected the request: invalid API key"+ FAILURE_MODEL: ClassVar[str] = "jev-1.13.0"+ FAILURE_PROVIDER: ClassVar[str] = "typesafe"+ FAILURE_CATEGORY: ClassVar[InferenceErrorCategory] = InferenceErrorCategory.CONFIGURATION+ FAILURE_USER_ACTION_DETAIL: ClassVar[str] = "Verify the API key configured for this judgment provider."++ # The classification both arms must surface on the recovered ErrorReport.+ EXPECTED_RETRYABLE: ClassVar[bool] = False+ EXPECTED_USER_ACTION_KIND: ClassVar[UserActionKind] = UserActionKind.CHECK_CREDENTIALS++ @classmethod+ def make_failing_judgment_error(cls) -> JudgmentJobFailureError:+ """Build a fresh classified ``JudgmentJobFailureError``.++ Each arm builds its own instance — an exception carries traceback state, so a single shared+ instance must not be reused across runs.+ """+ error = JudgmentJobFailureError(+ cls.FAILURE_MESSAGE,+ error_category=cls.FAILURE_CATEGORY,+ user_action=UserAction(kind=cls.EXPECTED_USER_ACTION_KIND, detail=cls.FAILURE_USER_ACTION_DETAIL),+ )+ # model_handle / backend_name are declared on CogtError; a real judgment worker fills them at its+ # public-method chokepoint. This test mocks above the worker, so set them directly.+ error.model_handle = cls.FAILURE_MODEL+ error.backend_name = cls.FAILURE_PROVIDER+ return errordiff --git a/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthdsnew file mode 100644index 000000000..0779dbea2--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthds@@ -0,0 +1,13 @@+domain = "inbox_triage"+description = "Judging whether an inbox message needs an answer today"+main_pipe = "judge_is_urgent"++[pipe.judge_is_urgent]+type = "PipeJudge"+description = "Judges whether a message needs an answer today"+inputs = { message = "Text" }+output = "YesNo"+model = "@default-judgment"+question = "Does this message need an answer today?"+criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }+threshold = 0.7diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/entry.toml b/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/entry.tomlnew file mode 100644index 000000000..85e726d00--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/entry.toml@@ -0,0 +1,10 @@+name = "operator_judge_urgent_message"+description = "A judgment on whether an inbox message needs an answer today"++validity = "valid"+tier = "inference"+granularity = "focused"++covers = [+ "operator.judge",+]diff --git a/pipelex/test_extras/mthds_corpus/vocabulary.toml b/pipelex/test_extras/mthds_corpus/vocabulary.tomlindex d27a00f5a..4f4ff983b 100644--- a/pipelex/test_extras/mthds_corpus/vocabulary.toml+++ b/pipelex/test_extras/mthds_corpus/vocabulary.toml@@ -93,6 +93,9 @@ code = "PipeStructure"[operator.doc_gen]code = "PipeDocGen"+[operator.judge]+code = "PipeJudge"+[controller.batch]code = "PipeBatch"diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex e0e6a5959..006482da5 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -151,8 +151,10 @@ Jinja2TemplateSyntaxError | Jinja 2 template syntax | https://docs.pipelex.com/lJobMetadataError | Job metadata | https://docs.pipelex.com/latest/errors/job-metadata-error/JsonTypeError | Json type | https://docs.pipelex.com/latest/errors/json-type-error/JudgmentAnswerMismatchError | Judgment answer mismatch | https://docs.pipelex.com/latest/errors/judgment-answer-mismatch-error/+JudgmentCapabilityError | Judgment capability | https://docs.pipelex.com/latest/errors/judgment-capability-error/JudgmentHandleNotFoundError | Judgment handle not found | https://docs.pipelex.com/latest/errors/judgment-handle-not-found-error/JudgmentJobFailureError | Judgment job failure | https://docs.pipelex.com/latest/errors/judgment-job-failure-error/+JudgmentModelMissingError | Judgment model missing | https://docs.pipelex.com/latest/errors/judgment-model-missing-error/JudgmentModelNotFoundError | Judgment model not found | https://docs.pipelex.com/latest/errors/judgment-model-not-found-error/KitError | Kit | https://docs.pipelex.com/latest/errors/kit-error/KitIndexLoadingError | Kit index loading | https://docs.pipelex.com/latest/errors/kit-index-loading-error/@@ -245,6 +247,8 @@ PipeImgGenRunError | Pipe img gen run | https://docs.pipelex.com/latest/errors/pPipeInputError | Pipe input | https://docs.pipelex.com/latest/errors/pipe-input-error/PipeInputsFactoryError | Pipe inputs factory | https://docs.pipelex.com/latest/errors/pipe-inputs-factory-error/PipeJobError | Pipe job | https://docs.pipelex.com/latest/errors/pipe-job-error/+PipeJudgeError | Pipe judge | https://docs.pipelex.com/latest/errors/pipe-judge-error/+PipeJudgeInputCapabilityError | Pipe judge input capability | https://docs.pipelex.com/latest/errors/pipe-judge-input-capability-error/PipeLLMFactoryError | Pipe LLM factory | https://docs.pipelex.com/latest/errors/pipe-llm-factory-error/PipeLibraryError | Pipe library | https://docs.pipelex.com/latest/errors/pipe-library-error/PipeLoadRefusalError | Pipe load refusal | https://docs.pipelex.com/latest/errors/pipe-load-refusal-error/diff --git a/tests/data/input_semantics/scaffold_bundle.mthds b/tests/data/input_semantics/scaffold_bundle.mthdsindex 445a04f61..5bf590fab 100644--- a/tests/data/input_semantics/scaffold_bundle.mthds+++ b/tests/data/input_semantics/scaffold_bundle.mthds@@ -94,7 +94,7 @@ And the bookmark beside it:# keeps their `{concept, content}` envelope, which is the only form the input shaper takes back.[pipe.scaffold_verdict_natives]type = "PipeCompose"-description = "SCAFFOLD_pipe_scaffold_verdict_natives: the three verdict natives, each at a slot position"+description = "SCAFFOLD_pipe_scaffold_verdict_natives: the verdict natives, each at a slot position"inputs = { approved_in = "YesNo", team_in = "Choice", grade_in = "Rating" }output = "Text"template = "SCAFFOLD_template_verdict_natives: $approved_in $team_in $grade_in"diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthdsnew file mode 100644index 000000000..16d8c385d--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthds@@ -0,0 +1,74 @@+domain = "pipe_judge_e2e"+description = "Test PipeJudge with one question of each kind"++[pipe.judge_urgent_e2e]+type = "PipeJudge"+description = "Judge whether a message needs an answer today"+inputs = { message = "Text" }+output = "YesNo"+model = "@default-judgment"+question = "Does this message need an answer today?"+criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }+threshold = 0.7++[pipe.route_ticket_e2e]+type = "PipeSequence"+description = "Judge which team handles a ticket, then route it there"+inputs = { ticket = "Text" }+output = "Text"+steps = [+ { pipe = "judge_team_e2e", result = "routing" },+ { pipe = "dispatch_ticket_e2e", result = "acknowledgement" },+]++[pipe.dispatch_ticket_e2e]+type = "PipeCondition"+description = "Send the ticket to the team the judgment chose"+inputs = { routing = "Choice", ticket = "Text" }+output = "Text"+expression = "routing.choice"+outcomes = { billing = "acknowledge_billing_e2e", technical = "acknowledge_technical_e2e" }+default_outcome = "acknowledge_other_e2e"++[pipe.acknowledge_billing_e2e]+type = "PipeCompose"+description = "Acknowledge a ticket sent to billing"+inputs = { ticket = "Text" }+output = "Text"+template = "Sent to billing: $ticket"++[pipe.acknowledge_technical_e2e]+type = "PipeCompose"+description = "Acknowledge a ticket sent to the technical team"+inputs = { ticket = "Text" }+output = "Text"+template = "Sent to the technical team: $ticket"++[pipe.acknowledge_other_e2e]+type = "PipeCompose"+description = "Acknowledge a ticket no team owns"+inputs = { ticket = "Text" }+output = "Text"+template = "Sent to the front desk: $ticket"++[pipe.judge_team_e2e]+type = "PipeJudge"+description = "Judge which team should handle a support ticket"+inputs = { ticket = "Text" }+output = "Choice"+model = "@default-judgment"+question = "Which team should handle this support ticket?"+options = { billing = "Charges, invoices, refunds and subscriptions", technical = "Errors, outages and features that do not work", other = "" }++[pipe.judge_severity_e2e]+type = "PipeJudge"+description = "Rate how severe a bug report is"+inputs = { report = "Text" }+output = "Rating"+model = "@default-judgment"+question = "How severe is the bug this report describes?"+levels = [+ "Cosmetic: nothing stops working",+ "Degraded: a workaround exists",+ "Blocking: users cannot do their work at all",+]diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.pynew file mode 100644index 000000000..52a7bef31--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.py@@ -0,0 +1,11 @@+"""Test data for PipeJudge E2E tests."""++from typing import ClassVar+++class PipeJudgeTestCases:+ """Inputs pinned with the verdicts a live judgment returns for them."""++ URGENT_MESSAGE: ClassVar[str] = "Our production database has been down for an hour and no customer can log in. Please help."+ BILLING_TICKET: ClassVar[str] = "I was charged twice for my subscription this month and I would like a refund for the second charge."+ BLOCKING_REPORT: ClassVar[str] = "Since this morning's release, clicking Save on any invoice crashes the app, so nobody can issue an invoice."diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.pynew file mode 100644index 000000000..db91d33be--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.py@@ -0,0 +1,85 @@+"""E2E tests for the PipeJudge operator: one question of each kind, run on the deck's default judgment model."""++import pytest++from pipelex import pretty_print+from pipelex.core.stuffs.text_content import TextContent+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.system.pipe_run_mode import PipeRunMode+from tests.e2e.pipelex.pipes.pipe_operators.pipe_judge.test_data import PipeJudgeTestCases++LIBRARY_DIRS = ["tests/e2e/pipelex/pipes/pipe_operators/pipe_judge"]++# A judgment model's probabilities drift a little between versions, never across a verdict.+PROBABILITY_TOLERANCE = 0.05++[email protected]_runnable[email protected](loop_scope="class")+class TestPipeJudge:+ async def test_yes_no_question(self, pipe_run_mode: PipeRunMode) -> None:+ runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)+ pipeline_response = await runner.execute(+ pipe_code="judge_urgent_e2e",+ inputs={"message": TextContent(text=PipeJudgeTestCases.URGENT_MESSAGE)},+ )++ pipe_output = pipeline_response.pipe_output+ assert pipe_output is not None+ verdict = pipe_output.main_stuff_as_yes_no+ if pipe_run_mode.is_live:+ pretty_print(verdict, title="Yes/no judgment")+ # The declared threshold of 0.7 decides the verdict from the probability.+ assert verdict.yes_no is True+ assert verdict.probability is not None+ assert abs(verdict.probability - 0.98) <= PROBABILITY_TOLERANCE++ async def test_choice_question(self, pipe_run_mode: PipeRunMode) -> None:+ runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)+ pipeline_response = await runner.execute(+ pipe_code="judge_team_e2e",+ inputs={"ticket": TextContent(text=PipeJudgeTestCases.BILLING_TICKET)},+ )++ pipe_output = pipeline_response.pipe_output+ assert pipe_output is not None+ verdict = pipe_output.main_stuff_as_choice+ if pipe_run_mode.is_live:+ pretty_print(verdict, title="Choice judgment")+ assert verdict.choice == "billing"+ assert verdict.probabilities is not None+ assert set(verdict.probabilities) == {"billing", "technical", "other"}+ assert abs(verdict.probabilities["billing"] - 1.0) <= PROBABILITY_TOLERANCE++ async def test_choice_routes_a_condition(self, pipe_run_mode: PipeRunMode) -> None:+ runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)+ pipeline_response = await runner.execute(+ pipe_code="route_ticket_e2e",+ inputs={"ticket": TextContent(text=PipeJudgeTestCases.BILLING_TICKET)},+ )++ pipe_output = pipeline_response.pipe_output+ assert pipe_output is not None+ acknowledgement = pipe_output.main_stuff_as_text+ if pipe_run_mode.is_live:+ assert acknowledgement.text.startswith("Sent to billing:")++ async def test_rating_question(self, pipe_run_mode: PipeRunMode) -> None:+ runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)+ pipeline_response = await runner.execute(+ pipe_code="judge_severity_e2e",+ inputs={"report": TextContent(text=PipeJudgeTestCases.BLOCKING_REPORT)},+ )++ pipe_output = pipeline_response.pipe_output+ assert pipe_output is not None+ verdict = pipe_output.main_stuff_as_rating+ if pipe_run_mode.is_live:+ pretty_print(verdict, title="Rating judgment")+ assert verdict.level == 2+ assert verdict.probabilities is not None+ assert set(verdict.probabilities) == {"0", "1", "2"}+ assert verdict.position is not None+ assert abs(verdict.position - 2.0) <= PROBABILITY_TOLERANCE * 2diff --git a/tests/integration/pipelex/error_handling/bundles/native_judge.mthds b/tests/integration/pipelex/error_handling/bundles/native_judge.mthdsnew file mode 100644index 000000000..3191a2874--- /dev/null+++ b/tests/integration/pipelex/error_handling/bundles/native_judge.mthds@@ -0,0 +1,9 @@+domain = "native_judge_test"+description = "Single PipeJudge step over a fixed question (no dynamic classes)"++[pipe.native_judge]+type = "PipeJudge"+description = "Judge whether a fixed claim holds"+output = "YesNo"+model = "jev-1.13.0"+question = "Is the sky blue on a clear day?"diff --git a/tests/integration/pipelex/error_handling/test_data.py b/tests/integration/pipelex/error_handling/test_data.pyindex a08e42e72..3954d3ed6 100644--- a/tests/integration/pipelex/error_handling/test_data.py+++ b/tests/integration/pipelex/error_handling/test_data.py@@ -10,6 +10,7 @@ ships is genuinely per-repo, so that is all these subclasses add.from typing import ClassVarfrom pipelex.test_extras.error_report_parity import ErrorReportParityTestData as SharedErrorReportParityTestData+from pipelex.test_extras.error_report_parity import JudgeErrorReportParityTestData as SharedJudgeErrorReportParityTestDatafrom pipelex.test_extras.error_report_parity import SearchErrorReportParityTestData as SharedSearchErrorReportParityTestData@@ -23,3 +24,9 @@ class SearchErrorReportParityTestData(SharedSearchErrorReportParityTestData):"""Local-arm view of the search parity fixtures, pointing at this repo's ``native_search`` bundle."""BUNDLE_FILE: ClassVar[str] = "tests/integration/pipelex/error_handling/bundles/native_search.mthds"+++class JudgeErrorReportParityTestData(SharedJudgeErrorReportParityTestData):+ """Local-arm view of the judgment parity fixtures, pointing at this repo's ``native_judge`` bundle."""++ BUNDLE_FILE: ClassVar[str] = "tests/integration/pipelex/error_handling/bundles/native_judge.mthds"diff --git a/tests/integration/pipelex/error_handling/test_judge_error_report_local_full_chain.py b/tests/integration/pipelex/error_handling/test_judge_error_report_local_full_chain.pynew file mode 100644index 000000000..76f1a4c8f--- /dev/null+++ b/tests/integration/pipelex/error_handling/test_judge_error_report_local_full_chain.py@@ -0,0 +1,65 @@+"""Local arm of the local / Temporal ``ErrorReport`` parity pair for judgment.++Runs the ``native_judge`` pipe through the local (non-Temporal) ``PipeRouter`` with the judgment call+mocked to fail, and asserts the resulting ``ErrorReport`` carries the full classification —+``error_category`` / ``retryable`` / ``model`` / ``provider`` / ``user_action``.++This is the baseline a distributed arm must match, asserting the same ``JudgeErrorReportParityTestData``+constants so local / distributed parity holds by construction. The judgment operator does not wrap the+leaf error (unlike PipeLLM), so the ``PipeRouter`` locates the raw ``JudgmentJobFailureError`` (a+``CogtError``) itself, and the report carries its identity.+"""++from collections.abc import Generator++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.content_generation.content_generator import ContentGenerator+from pipelex.cogt.exceptions import JudgmentJobFailureError+from pipelex.interpreter_hub import get_pipe_router+from pipelex.pipe_run.exceptions import PipeRouterError+from pipelex.pipe_run.pipe_job import PipeJob+from pipelex.system.pipe_run_mode import PipeRunMode+from tests.integration.pipelex.error_handling.test_data import JudgeErrorReportParityTestData+from tests.integration.pipelex.fixtures.pipe_job_helpers import pipe_job_from_bundle++[email protected](loop_scope="class")+class TestJudgeErrorReportLocalFullChain:+ """A failing judgment pipe run locally yields a fully classified ``ErrorReport``."""++ @pytest.fixture+ def failing_judge_pipe_job_local(self) -> Generator[PipeJob, None, None]:+ """A PipeJob for the judgment pipe, in LIVE mode so the judgment call actually fires."""+ yield from pipe_job_from_bundle(+ bundle_file=JudgeErrorReportParityTestData.BUNDLE_FILE,+ pipe_code=JudgeErrorReportParityTestData.PIPE_CODE,+ pipe_run_mode=PipeRunMode.LIVE,+ )++ async def test_error_report_from_local_execution(+ self,+ mocker: MockerFixture,+ failing_judge_pipe_job_local: PipeJob,+ ) -> None:+ """Local execution surfaces the worker failure, located, with the ``JudgmentJobFailureError``'s classification."""+ mocker.patch.object(+ ContentGenerator,+ "make_judgment_answers",+ side_effect=JudgeErrorReportParityTestData.make_failing_judgment_error(),+ )++ with pytest.raises(PipeRouterError) as exc_info:+ await get_pipe_router().run(pipe_job=failing_judge_pipe_job_local)+ assert isinstance(exc_info.value.__cause__, JudgmentJobFailureError)++ # The classification fields — the parity target.+ report = exc_info.value.to_error_report()+ assert report.error_type == "JudgmentJobFailureError"+ assert report.error_category == JudgeErrorReportParityTestData.FAILURE_CATEGORY+ assert report.retryable == JudgeErrorReportParityTestData.EXPECTED_RETRYABLE+ assert report.model == JudgeErrorReportParityTestData.FAILURE_MODEL+ assert report.provider == JudgeErrorReportParityTestData.FAILURE_PROVIDER+ assert report.user_action is not None+ assert report.user_action.kind == JudgeErrorReportParityTestData.EXPECTED_USER_ACTION_KINDdiff --git a/tests/integration/pipelex/pipeline/test_file_input_consumers.py b/tests/integration/pipelex/pipeline/test_file_input_consumers.pyindex b085ba1a0..55aa94ebd 100644--- a/tests/integration/pipelex/pipeline/test_file_input_consumers.py+++ b/tests/integration/pipelex/pipeline/test_file_input_consumers.py@@ -374,6 +374,30 @@ prompt = "Compare these transcripts: $transcripts, given their summaries: $summaassert consumer.covers(file_input_path=("transcripts", 1))assert not consumer.covers(file_input_path=("notes", 1))+ def test_a_pipe_judge_consumes_its_document_inputs(self, load_empty_library: Callable[[], str], mocker: MockerFixture):+ """Every input is material to judge, so a document input is sent as a file whether or not the question names it."""+ model_deck = get_model_deck()+ judgment_model = model_deck.judgment_aliases["default-judgment"]+ booted_spec = model_deck.inference_models[judgment_model]+ mocker.patch.dict(model_deck.inference_models, {judgment_model: booted_spec.model_copy(update={"inputs": ["text", "images", "pdf"]})})+ pipes = """+[pipe.main]+type = "PipeJudge"+description = "Judge whether the claims are complete"+inputs = { claims = "Document[]", note = "Text", photo = "Image" }+output = "YesNo"+model = "@default-judgment"+question = "Are these claims complete, given $note?"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_judge", pipes=pipes)++ assert set(consumers) == {"claims"}+ (consumer,) = consumers["claims"]+ assert consumer.kind == FileConsumerKind.JUDGMENT_DOCUMENT+ assert consumer.model == judgment_model+ assert consumer.readable_formats == frozenset({"pdf"})+ assert consumer.covers(file_input_path=("claims", 0))+def test_a_waterfall_reads_a_format_when_any_member_reads_it(self, load_empty_library: Callable[[], str], mocker: MockerFixture):mocker.patch.dict(get_model_deck().extract_waterfalls, {"fic-mixed-extractors": [PDF_ONLY_MODEL, DOCLING_MODEL]})pipes = """diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/conftest.py b/tests/integration/pipelex/pipes/operator/pipe_judge/conftest.pynew file mode 100644index 000000000..224759c57--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/conftest.py@@ -0,0 +1,41 @@+from collections.abc import Iterator++import pytest++from pipelex.runtime_hub import get_model_deck+from tests.integration.pipelex.pipes.operator.pipe_judge.test_data import PipeJudgeLoadTestData+++def judgment_model_reading_files() -> Iterator[None]:+ """The judgment model, for one test, as a backend that reads images and PDFs would declare it."""+ model_deck = get_model_deck()+ booted_spec = model_deck.inference_models[PipeJudgeLoadTestData.JUDGMENT_MODEL]+ model_deck.inference_models[PipeJudgeLoadTestData.JUDGMENT_MODEL] = booted_spec.model_copy(update={"inputs": ["text", "images", "pdf"]})+ try:+ yield+ finally:+ model_deck.inference_models[PipeJudgeLoadTestData.JUDGMENT_MODEL] = booted_spec+++def no_judgment_default() -> Iterator[None]:+ """A deck that names no default judgment model, which is the kit's own state, made explicit for one test."""+ model_deck = get_model_deck()+ booted_default = model_deck.judgment_choice_default+ model_deck.judgment_choice_default = None+ try:+ yield+ finally:+ model_deck.judgment_choice_default = booted_default+++def unserved_judgment_waterfall() -> Iterator[str]:+ """A judgment waterfall, for one test, whose only model no backend serves: its reference."""+ model_deck = get_model_deck()+ model_deck.judgment_waterfalls["unserved_judges"] = ["jev-0.0.0-unserved"]+ try:+ yield "~unserved_judges"+ finally:+ del model_deck.judgment_waterfalls["unserved_judges"]diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.pynew file mode 100644index 000000000..1047a1514--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.py@@ -0,0 +1,23 @@+class PipeJudgeLoadTestData:+ """A bundle with one PipeJudge step, built from the step's own fields."""++ JUDGMENT_MODEL = "jev-1.13.0"++ @classmethod+ def bundle(cls, *, inputs: str = '{ message = "Text" }', output: str = "YesNo", step_fields: str = "") -> str:+ return f"""+domain = "judge_load"+description = "Loading a PipeJudge step"++[concept]+Team = {{ description = "The team a ticket goes to", refines = "Choice" }}+Ticket = "A support ticket"++[pipe.judge_it]+type = "PipeJudge"+description = "Judge the material"+inputs = {inputs}+output = "{output}"+question = "Is it urgent?"+{step_fields}+"""diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.pynew file mode 100644index 000000000..cb7d25eeb--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.py@@ -0,0 +1,34 @@+"""What a PipeJudge step may declare and still load."""++import pytest++from pipelex.pipeline.validate_bundle import validate_bundle+from tests.integration.pipelex.pipes.operator.pipe_judge.test_data import PipeJudgeLoadTestData++_MODEL = f'model = "{PipeJudgeLoadTestData.JUDGMENT_MODEL}"'++[email protected](loop_scope="class")+class TestPipeJudgeLoadAcceptances:+ async def test_an_output_refining_the_verdict_native_is_admitted(self) -> None:+ bundle = PipeJudgeLoadTestData.bundle(output="Team", step_fields=f'{_MODEL}\noptions = {{ billing = "", technical = "" }}')+ result = await validate_bundle(mthds_contents=[bundle])+ assert "judge_load.judge_it" in {pipe.pipe_ref for pipe in result.pipes}++ async def test_an_input_the_question_never_names_is_material(self) -> None:+ bundle = PipeJudgeLoadTestData.bundle(inputs='{ message = "Text", ticket = "Ticket" }', step_fields=_MODEL)+ await validate_bundle(mthds_contents=[bundle])++ @pytest.mark.usefixtures("judgment_model_reading_files")+ @pytest.mark.parametrize("inputs", ['{ photo = "Image" }', '{ photos = "Image[]", claim = "Document" }'])+ async def test_a_file_input_is_admitted_for_a_model_that_reads_files(self, inputs: str) -> None:+ await validate_bundle(mthds_contents=[PipeJudgeLoadTestData.bundle(inputs=inputs, step_fields=_MODEL)])++ async def test_a_dynamic_input_is_admitted_for_a_text_only_model(self) -> None:+ # `Dynamic` is compatible with every concept, `Image` included, so its value is checked when it runs, never at load.+ await validate_bundle(mthds_contents=[PipeJudgeLoadTestData.bundle(inputs='{ payload = "Dynamic" }', step_fields=_MODEL)])++ async def test_a_waterfall_no_backend_serves_is_left_to_the_run(self, unserved_judgment_waterfall: str) -> None:+ # Its model's spec is not on this boot, so whether it reads files is checked when the step runs, as on a keyless boot.+ bundle = PipeJudgeLoadTestData.bundle(inputs='{ photo = "Image" }', step_fields=f'model = "{unserved_judgment_waterfall}"')+ await validate_bundle(mthds_contents=[bundle])diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.pynew file mode 100644index 000000000..46ac03ac4--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.py@@ -0,0 +1,62 @@+"""A PipeJudge step is refused when its method loads, before a run spends anything, for each thing it cannot do."""++import pytest++from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.validate_bundle import validate_bundle+from tests.integration.pipelex.pipes.operator.pipe_judge.test_data import PipeJudgeLoadTestData++_MODEL = f'model = "{PipeJudgeLoadTestData.JUDGMENT_MODEL}"'+++async def _refusal_report(bundle: str) -> str:+ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_contents=[bundle])+ return str(exc_info.value.to_error_report().model_dump())++[email protected](loop_scope="class")+class TestPipeJudgeLoadRefusals:+ @pytest.mark.usefixtures("no_judgment_default")+ async def test_a_step_with_no_model_and_no_default_is_refused(self) -> None:+ report = await _refusal_report(PipeJudgeLoadTestData.bundle())+ assert "PipeJudge 'judge_it' has no judgment model" in report+ assert "`choice_default` under `[judgment]`" in report++ async def test_an_unknown_model_is_refused_on_its_field(self) -> None:+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(step_fields='model = "jev-9.99.9"'))+ assert "jev-9.99.9" in report+ assert "model" in report++ @pytest.mark.parametrize(+ ("inputs", "file_kind"),+ [+ pytest.param('{ photo = "Image" }', "images", id="image"),+ pytest.param('{ photo = "Image[]" }', "images", id="list_of_images"),+ pytest.param('{ photo = "Document" }', "documents", id="document"),+ ],+ )+ async def test_a_file_input_is_refused_for_a_text_only_model(self, inputs: str, file_kind: str) -> None:+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(inputs=inputs, step_fields=_MODEL))+ assert f"does not read {file_kind}" in report+ assert "'photo'" in report+ assert PipeJudgeLoadTestData.JUDGMENT_MODEL in report++ @pytest.mark.parametrize(+ ("output", "step_fields", "asks"),+ [+ pytest.param(+ "YesNo",+ 'options = { billing = "", technical = "" }',+ "asks a choice question: its output must be `Choice`",+ id="choice_into_yes_no",+ ),+ pytest.param("Choice", 'levels = ["low", "high"]', "asks a rating question: its output must be `Rating`", id="rating_into_choice"),+ pytest.param("Text", "", "asks a yes/no question: its output must be `YesNo`", id="yes_no_into_text"),+ pytest.param("Dynamic", "", "asks a yes/no question: its output must be `YesNo`", id="yes_no_into_dynamic"),+ ],+ )+ async def test_an_output_disagreeing_with_the_kind_is_refused_naming_both_sides(self, output: str, step_fields: str, asks: str) -> None:+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(output=output, step_fields=f"{_MODEL}\n{step_fields}"))+ assert asks in report+ assert f"declares `native.{output}`" in reportdiff --git a/tests/integration/pipelex/system/test_keyless_boot_forced_dry.py b/tests/integration/pipelex/system/test_keyless_boot_forced_dry.pyindex 3fae78ef8..c619aa5cf 100644--- a/tests/integration/pipelex/system/test_keyless_boot_forced_dry.py+++ b/tests/integration/pipelex/system/test_keyless_boot_forced_dry.py@@ -16,6 +16,7 @@ a keyless boot is exactly the boot ``pipelex/kernel/`` documents as its target."""from collections.abc import Generator+from pathlib import Pathimport pytest@@ -29,6 +30,7 @@ from pipelex.pipe_operators.search.pipe_search import PipeSearchfrom pipelex.pipe_operators.search.pipe_search_blueprint import PipeSearchBlueprintfrom pipelex.pipelex import Pipelexfrom pipelex.pipeline.execution_seams import load_libraries_and_activate, prepare_pipe_job+from pipelex.pipeline.validate_bundle import validate_bundlefrom pipelex.runtime_hub import get_content_generator, is_dry_run_forcedfrom pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.system.runtime import IntegrationMode, runtime_manager@@ -141,6 +143,23 @@ class TestKeylessBootForcedDry:finally:Pipelex.teardown_if_needed()+ @pytest.mark.asyncio+ async def test_keyless_boot_validates_a_judge_whose_model_backend_it_skipped(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A PipeJudge naming the TypeSafe model still loads and dry-runs when the keyless boot skipped TypeSafe.++ The deck still names the alias, so the pipe names a model the deck knows; what the keyless boot+ cannot read is the spec of a model whose backend it skipped, so it leaves what that model reads+ to the keyed boot that runs the step.+ """+ monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)+ bundle_path = Path("pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthds")+ try:+ self._boot_keyless()+ result = await validate_bundle(mthds_file_path=bundle_path)+ assert [pipe.code for pipe in result.pipes]+ finally:+ Pipelex.teardown_if_needed()+def test_keyed_boot_does_not_force_dry(self) -> None:"""Control arm: a normal boot leaves the flag unset."""try:diff --git a/tests/unit/pipelex/builder/operations/test_parse_pipe_spec.py b/tests/unit/pipelex/builder/operations/test_parse_pipe_spec.pyindex 305193198..c30fbadc0 100644--- a/tests/unit/pipelex/builder/operations/test_parse_pipe_spec.py+++ b/tests/unit/pipelex/builder/operations/test_parse_pipe_spec.py@@ -15,6 +15,7 @@ from pipelex.builder.pipe.pipe_condition_spec import PipeConditionSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec+from pipelex.builder.pipe.pipe_judge_spec import PipeJudgeSpecfrom pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpecfrom pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_search_spec import PipeSearchSpec@@ -450,6 +451,17 @@ class TestParsePipeSpec:},PipeSearchSpec,),+ (+ "PipeJudge",+ {+ "pipe_code": "is_urgent",+ "description": "Judge urgency",+ "inputs": {"message": "Text"},+ "output": "YesNo",+ "prompt": "Is the message urgent?",+ },+ PipeJudgeSpec,+ ),("PipeBatch",{@@ -468,3 +480,43 @@ class TestParsePipeSpec:def test_correct_subclass(self, pipe_type: str, spec_data: dict[str, Any], expected_class: type) -> None:result = parse_pipe_spec(spec_data, pipe_type=pipe_type)assert isinstance(result, expected_class)++ @pytest.mark.parametrize("question_key", ["question", "prompt"])+ def test_judge_reads_prompt_as_question(self, question_key: str) -> None:+ spec = parse_pipe_spec(+ {"pipe_code": "is_urgent", "description": "Judge", "inputs": {"message": "Text"}, "output": "YesNo", question_key: "Is it urgent?"},+ pipe_type="PipeJudge",+ )+ assert isinstance(spec, PipeJudgeSpec)+ assert spec.question == "Is it urgent?"++ def test_judge_refuses_both_spellings_and_leaves_the_callers_dict_alone(self) -> None:+ spec_data: dict[str, Any] = {+ "pipe_code": "is_urgent",+ "description": "Judge",+ "inputs": {"message": "Text"},+ "output": "YesNo",+ "question": "Is it urgent?",+ "prompt": "Is it urgent now?",+ }+ with pytest.raises(ValidationError, match="but not both"):+ PipeJudgeSpec.model_validate(spec_data)+ assert spec_data["prompt"] == "Is it urgent now?"++ def test_judge_reading_the_synonym_leaves_the_callers_dict_alone(self) -> None:+ spec_data: dict[str, Any] = {+ "pipe_code": "is_urgent",+ "description": "Judge",+ "inputs": {"message": "Text"},+ "output": "YesNo",+ "prompt": "Is it urgent?",+ }+ spec = PipeJudgeSpec.model_validate(spec_data)+ assert spec.question == "Is it urgent?"+ assert spec_data == {+ "pipe_code": "is_urgent",+ "description": "Judge",+ "inputs": {"message": "Text"},+ "output": "YesNo",+ "prompt": "Is it urgent?",+ }diff --git a/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.py b/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.pyindex 54c1b7ec0..ab1e92465 100644--- a/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.py+++ b/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.py@@ -528,3 +528,60 @@ class TestPipeSpecToToml:toml = pipe_spec_to_toml(spec)assert "target_format" not in tomlassert "template" not in toml++ # -- PipeJudge --------------------------------------------------------++ def test_judge_choice_writes_question_and_options(self) -> None:+ spec = parse_pipe_spec(+ {+ "pipe_code": "route_ticket",+ "description": "Route a ticket",+ "inputs": {"ticket": "Text"},+ "output": "Choice",+ "model": "@default-judgment",+ "prompt": "Which team should handle the ticket?",+ "options": {"billing": "Charges and invoices", "other": ""},+ },+ pipe_type="PipeJudge",+ )+ toml = pipe_spec_to_toml(spec)+ assert 'question = "Which team should handle the ticket?"' in toml+ assert "prompt" not in toml, "A tool that writes a PipeJudge writes `question`"+ assert 'model = "@default-judgment"' in toml+ assert 'billing = "Charges and invoices"' in toml+ assert 'other = ""' in toml++ def test_judge_yes_no_writes_criteria_and_threshold(self) -> None:+ spec = parse_pipe_spec(+ {+ "pipe_code": "is_urgent",+ "description": "Judge urgency",+ "inputs": {"message": "Text"},+ "output": "YesNo",+ "question": "Is the message urgent?",+ "criteria": {"yes": "Needs an answer today"},+ "threshold": 0.8,+ },+ pipe_type="PipeJudge",+ )+ toml = pipe_spec_to_toml(spec)+ assert 'yes = "Needs an answer today"' in toml+ assert "no =" not in toml+ assert "threshold = 0.8" in toml++ def test_judge_rating_writes_levels(self) -> None:+ spec = parse_pipe_spec(+ {+ "pipe_code": "rate_severity",+ "description": "Rate severity",+ "inputs": {"report": "Text"},+ "output": "Rating",+ "question": "How severe is it?",+ "levels": ["Cosmetic", "A workaround exists", "Blocking"],+ },+ pipe_type="PipeJudge",+ )+ toml = pipe_spec_to_toml(spec)+ assert "Cosmetic" in toml+ assert "Blocking" in toml+ assert "options" not in tomldiff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_judge/test_pipe_judge_spec_rendering.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_judge/test_pipe_judge_spec_rendering.pynew file mode 100644index 000000000..bced6e47a--- /dev/null+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_judge/test_pipe_judge_spec_rendering.py@@ -0,0 +1,21 @@+"""PipeJudgeSpec.rendered_pretty shows what the author wrote, markup characters included."""++from rich.console import Console++from pipelex.builder.pipe.pipe_judge_spec import PipeJudgeSpec+++class TestPipeJudgeSpecRendering:+ def test_a_bracketed_word_in_the_question_is_shown_as_written(self) -> None:+ spec = PipeJudgeSpec(+ pipe_code="is_flagged",+ description="Judge the flag",+ inputs={"message": "Text"},+ output="YesNo",+ question="Is the [urgent] flag set, or the [/x] one?",+ )+ console = Console(record=True, width=120)++ console.print(spec.rendered_pretty())++ assert "Is the [urgent] flag set, or the [/x] one?" in console.export_text()diff --git a/tests/unit/pipelex/cli/test_pipe_cmd.py b/tests/unit/pipelex/cli/test_pipe_cmd.pyindex 1547e87ee..181e545a8 100644--- a/tests/unit/pipelex/cli/test_pipe_cmd.py+++ b/tests/unit/pipelex/cli/test_pipe_cmd.py@@ -270,3 +270,24 @@ class TestCliPipeCmd:toml = _pipe_spec_to_toml(result)assert 'model = "$writing-creative"' in tomlassert 'output = "Text"' in toml++ def test_judge_fields_appear_in_toml(self) -> None:+ """PipeJudge type-specific fields round-trip to TOML, its template written as `question` whatever spelling came in."""+ spec = parse_pipe_spec(+ {+ "pipe_code": "route_ticket",+ "description": "Route a ticket",+ "inputs": {"ticket": "Text"},+ "output": "Choice",+ "model": "@default-judgment",+ "prompt": "Which team should handle the ticket?",+ "options": {"billing": "Charges and invoices", "technical": ""},+ },+ pipe_type="PipeJudge",+ )+ toml = _pipe_spec_to_toml(spec)+ assert 'type = "PipeJudge"' in toml+ assert 'model = "@default-judgment"' in toml+ assert 'question = "Which team should handle the ticket?"' in toml+ assert "prompt" not in toml+ assert 'billing = "Charges and invoices"' in tomldiff --git a/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py b/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.pyindex 9e1a186f0..56fef8812 100644--- a/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py+++ b/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py@@ -40,6 +40,8 @@ from pipelex.cogt.extract.extract_job_components import ExtractJobConfig, Extracfrom pipelex.cogt.image.prompt_image import PromptImageUrifrom pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt+from pipelex.cogt.judgment.judgment_models import YesNoQuestion+from pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.system.job_metadata import JobMetadata, RunMetadata@@ -63,7 +65,7 @@ URL_FIELD_INVENTORY: dict[type[BaseModel], set[str]] = {RenderDocumentAssignment: {"composition.layout.blocks.url", "composition.layout.blocks.blocks.url"},SearchAssignment: set(),SearchObjectAssignment: set(),- JudgmentAssignment: set(),+ JudgmentAssignment: {"images.uri", "documents.uri"},}@@ -196,6 +198,18 @@ URL_BEARING_SAMPLES: dict[type[BaseModel], tuple[_DeclaresReads, set[str]]] = {),{"pipelex-storage://s/cover.png", "pipelex-storage://s/figure.png"},),+ JudgmentAssignment: (+ JudgmentAssignment(+ job_metadata=_job_metadata(),+ cogt_run_params=CogtRunParams(run_mode=PipeRunMode.DRY),+ state={"message": "the url https://example.com/in-the-state is data, never read"},+ images={"before": [PromptImageUri(uri="pipelex-storage://s/before.png")], "after": [PromptImageUri(uri="pipelex-storage://s/after.png")]},+ documents={"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf")]},+ questions={"is_urgent": YesNoQuestion(instructions="Is it urgent?")},+ judgment_setting=JudgmentSetting(model="h"),+ ),+ {"pipelex-storage://s/before.png", "pipelex-storage://s/after.png", "pipelex-storage://s/claim.pdf"},+ ),}diff --git a/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.py b/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.pynew file mode 100644index 000000000..cced838d7--- /dev/null+++ b/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.py@@ -0,0 +1,75 @@+"""A judgment worker that answers whatever it is told to, and the job and model spec it is tested with."""++from typing_extensions import override++from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.exceptions import CogtError+from pipelex.cogt.image.prompt_image import PromptImage+from pipelex.cogt.judgment.judgment_job import JudgmentJob+from pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory+from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, JudgmentQuestion+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.cogt.judgment.judgment_worker_abstract import JudgmentWorkerAbstract+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.reporting.reporting_protocol import ReportingProtocol+from pipelex.system.job_metadata import JobMetadata, RunMetadata+++class FakeJudgmentWorker(JudgmentWorkerAbstract):+ """A worker that answers whatever it was told to, including the wrong thing, and records whether it was reached."""++ def __init__(+ self,+ inference_model: InferenceModelSpec,+ *,+ answers: dict[str, JudgmentAnswer] | None = None,+ raises: CogtError | None = None,+ reporting_delegate: ReportingProtocol | None = None,+ ) -> None:+ JudgmentWorkerAbstract.__init__(self, inference_model, reporting_delegate=reporting_delegate)+ self._answers = answers or {}+ self._raises = raises+ self.was_called = False++ @override+ async def _judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentAnswer]:+ self.was_called = True+ if self._raises is not None:+ raise self._raises+ return self._answers+++def make_fake_judgment_job(+ questions: dict[str, JudgmentQuestion],+ *,+ images: dict[str, list[PromptImage]] | None = None,+ documents: dict[str, list[PromptDocument]] | None = None,+) -> JudgmentJob:+ return JudgmentJobFactory.make_judgment_job(+ state={"message": "the roof is on fire"},+ images=images,+ documents=documents,+ questions=questions,+ judgment_setting=JudgmentSetting(model="fake-judgment-handle"),+ job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_judgment")),+ )+++def make_fake_judgment_model(*, inputs: list[str] | None = None, max_prompt_images: int | None = None) -> InferenceModelSpec:+ """A judgment model spec reading `inputs`, text alone when none are given."""+ return InferenceModelSpec(+ backend_name="fake_backend",+ name="fake-judgment-handle",+ sdk="fake_judgment_sdk",+ model_type=ModelType.JUDGMENT,+ model_id="fake-judgment-1.0",+ inputs=inputs or ["text"],+ outputs=["judgments"],+ costs={CostCategory.INPUT: 0.042, CostCategory.OUTPUT: 0},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=max_prompt_images,+ )diff --git a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.py b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.pyindex bfd39b3f2..bb2773c21 100644--- a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.py+++ b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.py@@ -8,10 +8,11 @@ exactly one of these goes red.import pytestfrom typing_extensions import override-from pipelex.cogt.exceptions import CogtError, JudgmentAnswerMismatchError+from pipelex.cogt.exceptions import (+ CogtError,+ JudgmentAnswerMismatchError,+)from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract-from pipelex.cogt.judgment.judgment_job import JudgmentJob-from pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactoryfrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,ChoiceQuestion,@@ -22,39 +23,12 @@ from pipelex.cogt.judgment.judgment_models import (YesNoAnswer,YesNoQuestion,)-from pipelex.cogt.judgment.judgment_setting import JudgmentSetting-from pipelex.cogt.judgment.judgment_worker_abstract import JudgmentWorkerAbstract-from pipelex.cogt.llm.thinking_mode import ThinkingMode-from pipelex.cogt.model_backends.model_spec import InferenceModelSpec-from pipelex.cogt.model_backends.model_type import ModelType-from pipelex.cogt.usage.cost_category import CostCategory-from pipelex.reporting.reporting_protocol import ReportingNoOp, ReportingProtocol-from pipelex.system.job_metadata import JobCategory, JobMetadata, RunMetadata, UnitJobId+from pipelex.reporting.reporting_protocol import ReportingNoOp+from pipelex.system.job_metadata import JobCategory, UnitJobId+from tests.unit.pipelex.cogt.judgment.fake_judgment_worker import FakeJudgmentWorker, make_fake_judgment_job, make_fake_judgment_modelfrom tests.unit.pipelex.cogt.judgment.test_data import JudgmentTestCases-class _FakeJudgmentWorker(JudgmentWorkerAbstract):- """A worker that answers whatever it was told to, including the wrong thing."""-- def __init__(- self,- inference_model: InferenceModelSpec,- *,- answers: dict[str, JudgmentAnswer] | None = None,- raises: CogtError | None = None,- reporting_delegate: ReportingProtocol | None = None,- ) -> None:- JudgmentWorkerAbstract.__init__(self, inference_model, reporting_delegate=reporting_delegate)- self._answers = answers or {}- self._raises = raises-- @override- async def _judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentAnswer]:- if self._raises is not None:- raise self._raises- return self._answers--class _RecordingDelegate(ReportingNoOp):"""A reporting delegate that keeps what it was handed, so the `finally` can be asserted on."""@@ -68,34 +42,9 @@ class _RecordingDelegate(ReportingNoOp):@pytest.mark.asyncioclass TestJudgmentWorkerContract:- def _job(self, questions: dict[str, JudgmentQuestion]) -> JudgmentJob:- return JudgmentJobFactory.make_judgment_job(- state={"message": "the roof is on fire"},- questions=questions,- judgment_setting=JudgmentSetting(model="fake-judgment-handle"),- job_metadata=JobMetadata(- run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_judgment")- ),- )-- def _model(self) -> InferenceModelSpec:- return InferenceModelSpec(- backend_name="fake_backend",- name="fake-judgment-handle",- sdk="fake_judgment_sdk",- model_type=ModelType.JUDGMENT,- model_id="fake-judgment-1.0",- inputs=["text"],- outputs=["judgments"],- costs={CostCategory.INPUT: 0.042, CostCategory.OUTPUT: 0},- thinking_mode=ThinkingMode.NONE,- max_tokens=None,- max_prompt_images=None,- )-async def test_it_answers_every_question_and_stamps_the_job(self) -> None:- job = self._job(JudgmentTestCases.THREE_QUESTIONS)- worker = _FakeJudgmentWorker(self._model(), answers=JudgmentTestCases.THREE_ANSWERS)+ job = make_fake_judgment_job(JudgmentTestCases.THREE_QUESTIONS)+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers=JudgmentTestCases.THREE_ANSWERS)answers = await worker.judge(job)@@ -106,9 +55,9 @@ class TestJudgmentWorkerContract:assert job.job_metadata.completed_at is not Noneasync def test_it_refuses_an_answer_to_a_question_nobody_asked(self) -> None:- job = self._job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})- worker = _FakeJudgmentWorker(- self._model(),+ job = make_fake_judgment_job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})+ worker = FakeJudgmentWorker(+ make_fake_judgment_model(),answers={"is_urgent": YesNoAnswer(probability=0.9), "is_spam": YesNoAnswer(probability=0.1)},)@@ -118,23 +67,41 @@ class TestJudgmentWorkerContract:assert "is_spam" in str(exc_info.value)async def test_it_refuses_a_dropped_answer(self) -> None:- job = self._job(JudgmentTestCases.THREE_QUESTIONS)- worker = _FakeJudgmentWorker(self._model(), answers={"is_urgent": YesNoAnswer(probability=0.9)})+ job = make_fake_judgment_job(JudgmentTestCases.THREE_QUESTIONS)+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"is_urgent": YesNoAnswer(probability=0.9)})with pytest.raises(JudgmentAnswerMismatchError) as exc_info:await worker.judge(job)assert "severity" in str(exc_info.value)- async def test_it_refuses_an_answer_of_the_wrong_kind(self) -> None:- job = self._job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})- worker = _FakeJudgmentWorker(self._model(), answers={"severity": ChoiceAnswer(choice="mild")})+ @pytest.mark.parametrize(+ ("question", "answer", "asked_kind"),+ [+ pytest.param(YesNoQuestion(instructions="Is it severe?"), ChoiceAnswer(choice="mild"), "yes_no", id="yes_no_answered_by_a_choice"),+ pytest.param(+ ChoiceQuestion(instructions="Which severity?", options={"mild": None, "bad": None}),+ RatingAnswer(level=0),+ "choice",+ id="choice_answered_by_a_rating",+ ),+ pytest.param(+ RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ ChoiceAnswer(choice="mild"),+ "rating",+ id="rating_answered_by_a_choice",+ ),+ ],+ )+ async def test_it_refuses_an_answer_of_the_wrong_kind(self, question: JudgmentQuestion, answer: JudgmentAnswer, asked_kind: str) -> None:+ job = make_fake_judgment_job({"severity": question})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"severity": answer})with pytest.raises(JudgmentAnswerMismatchError) as exc_info:await worker.judge(job)assert "severity" in str(exc_info.value)- assert "rating" in str(exc_info.value)+ assert f"it asked for '{asked_kind}'" in str(exc_info.value)@pytest.mark.parametrize(("answer", "named"),@@ -144,8 +111,8 @@ class TestJudgmentWorkerContract:],)async def test_it_refuses_an_option_the_question_does_not_offer(self, answer: ChoiceAnswer, named: str) -> None:- job = self._job({"topic": ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": "water damage"})})- worker = _FakeJudgmentWorker(self._model(), answers={"topic": answer})+ job = make_fake_judgment_job({"topic": ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": "water damage"})})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"topic": answer})with pytest.raises(JudgmentAnswerMismatchError) as exc_info:await worker.judge(job)@@ -162,8 +129,8 @@ class TestJudgmentWorkerContract:],)async def test_it_refuses_a_level_beyond_the_scale(self, answer: RatingAnswer, named: str) -> None:- job = self._job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})- worker = _FakeJudgmentWorker(self._model(), answers={"severity": answer})+ job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"severity": answer})with pytest.raises(JudgmentAnswerMismatchError) as exc_info:await worker.judge(job)@@ -171,15 +138,55 @@ class TestJudgmentWorkerContract:assert "severity" in str(exc_info.value)assert f"[{named}]" in str(exc_info.value)+ @pytest.mark.parametrize(+ ("question", "answer", "named"),+ [+ pytest.param(+ ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": None}),+ ChoiceAnswer(choice="fire", probabilities={"fire": 1.0000001, "flood": 0.0}),+ "fire",+ id="choice_probability_above_one",+ ),+ pytest.param(+ RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingAnswer(level=1, probabilities={0: -0.1, 1: 1.1}),+ "0",+ id="rating_probability_below_zero",+ ),+ pytest.param(+ RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingAnswer(level=2, position=float("inf")),+ "position",+ id="rating_infinite_position",+ ),+ pytest.param(+ RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingAnswer(level=2, position=2.5),+ "position of 2.5",+ id="rating_position_past_the_last_level",+ ),+ ],+ )+ async def test_it_refuses_a_measure_its_verdict_cannot_hold(self, question: JudgmentQuestion, answer: JudgmentAnswer, named: str) -> None:+ """A probability outside the unit interval or a position off the scale is refused as a mismatch, not left to the verdict's own model."""+ job = make_fake_judgment_job({"verdict": question})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"verdict": answer})++ with pytest.raises(JudgmentAnswerMismatchError) as exc_info:+ await worker.judge(job)++ assert "verdict" in str(exc_info.value)+ assert named in str(exc_info.value)+async def test_it_accepts_the_top_level_and_a_full_distribution(self) -> None:- job = self._job(+ job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),"topic": ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": "water damage"}),})- worker = _FakeJudgmentWorker(- self._model(),+ worker = FakeJudgmentWorker(+ make_fake_judgment_model(),answers={"severity": RatingAnswer(level=2, probabilities={0: 0.1, 1: 0.2, 2: 0.7}),"topic": ChoiceAnswer(choice="flood", probabilities={"fire": 0.4, "flood": 0.6}),@@ -192,10 +199,10 @@ class TestJudgmentWorkerContract:async def test_it_reports_on_the_way_out_even_when_the_answers_are_rejected(self) -> None:"""A rejected answer set was still paid for: the provider answered before the guard ran."""- job = self._job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})+ job = make_fake_judgment_job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})delegate = _RecordingDelegate()- worker = _FakeJudgmentWorker(- self._model(),+ worker = FakeJudgmentWorker(+ make_fake_judgment_model(),answers={"something_else": YesNoAnswer(probability=0.9)},reporting_delegate=delegate,)@@ -207,17 +214,17 @@ class TestJudgmentWorkerContract:assert job.job_metadata.completed_at is not Noneasync def test_it_reports_on_the_happy_path_too(self) -> None:- job = self._job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})+ job = make_fake_judgment_job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})delegate = _RecordingDelegate()- worker = _FakeJudgmentWorker(self._model(), answers={"is_urgent": YesNoAnswer(probability=0.9)}, reporting_delegate=delegate)+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"is_urgent": YesNoAnswer(probability=0.9)}, reporting_delegate=delegate)await worker.judge(job)assert delegate.reported == [job]async def test_it_names_the_model_and_the_backend_on_a_provider_failure(self) -> None:- job = self._job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})- worker = _FakeJudgmentWorker(self._model(), raises=CogtError("the provider said no"))+ job = make_fake_judgment_job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), raises=CogtError("the provider said no"))with pytest.raises(CogtError) as exc_info:await worker.judge(job)@@ -226,8 +233,8 @@ class TestJudgmentWorkerContract:assert exc_info.value.backend_name == "fake_backend"async def test_a_choice_answer_carries_the_option_it_names(self) -> None:- job = self._job({"topic": ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": "water damage"})})- worker = _FakeJudgmentWorker(self._model(), answers={"topic": ChoiceAnswer(choice="fire", confidence=0.8)})+ job = make_fake_judgment_job({"topic": ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": "water damage"})})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"topic": ChoiceAnswer(choice="fire", confidence=0.8)})answers = await worker.judge(job)@@ -236,8 +243,8 @@ class TestJudgmentWorkerContract:assert answer.choice == "fire"async def test_a_rating_answer_carries_its_level(self) -> None:- job = self._job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})- worker = _FakeJudgmentWorker(self._model(), answers={"severity": RatingAnswer(level=2, position=1.8)})+ job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"severity": RatingAnswer(level=2, position=1.8)})answers = await worker.judge(job)diff --git a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.py b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.pynew file mode 100644index 000000000..6c3a6a81f--- /dev/null+++ b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.py@@ -0,0 +1,93 @@+"""The files a judgment carries are refused by a model that does not read them, before its backend is called.++Each guard in `JudgmentWorkerAbstract._check_can_read_files` is mutation-tested: break it and exactly+one of these goes red.+"""++from typing import ClassVar++import pytest++from pipelex.cogt.document.prompt_document import PromptDocument, PromptDocumentUri+from pipelex.cogt.exceptions import JudgmentCapabilityError, PromptDocumentFormatError, PromptImageFormatError+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageUri+from pipelex.cogt.judgment.judgment_job import JudgmentJob+from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, JudgmentQuestion, YesNoAnswer, YesNoQuestion+from tests.unit.pipelex.cogt.judgment.fake_judgment_worker import FakeJudgmentWorker, make_fake_judgment_job, make_fake_judgment_model+++class TestJudgmentWorkerFileCapability:+ """The files a judgment carries are refused by a model that does not read them, before the backend is called."""++ _IMAGE = PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")+ _PDF = PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")+ _DOCX = PromptDocumentUri(+ uri="pipelex-storage://s/claim.docx",+ mime_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ )+ _QUESTION: ClassVar[dict[str, JudgmentQuestion]] = {"is_urgent": YesNoQuestion(instructions="Is it urgent?")}+ _ANSWER: ClassVar[dict[str, JudgmentAnswer]] = {"is_urgent": YesNoAnswer(probability=0.9)}++ def _worker(self, *, inputs: list[str], max_prompt_images: int | None = None) -> FakeJudgmentWorker:+ return FakeJudgmentWorker(make_fake_judgment_model(inputs=inputs, max_prompt_images=max_prompt_images), answers=self._ANSWER)++ def _job(self, *, images: dict[str, list[PromptImage]] | None = None, documents: dict[str, list[PromptDocument]] | None = None) -> JudgmentJob:+ return make_fake_judgment_job(self._QUESTION, images=images, documents=documents)++ async def test_a_text_only_model_refuses_an_image(self) -> None:+ worker = self._worker(inputs=["text"])++ with pytest.raises(JudgmentCapabilityError, match="does not read images") as exc_info:+ await worker.judge(self._job(images={"photo": [self._IMAGE]}))++ assert not worker.was_called+ # The message names the model as the deck does, with none of the console's markup escapes.+ assert "Judgment model 'fake-judgment-handle' does not read images" in str(exc_info.value)++ async def test_a_text_only_model_refuses_a_document(self) -> None:+ worker = self._worker(inputs=["text"])++ with pytest.raises(JudgmentCapabilityError, match="does not read documents"):+ await worker.judge(self._job(documents={"claim": [self._PDF]}))++ assert not worker.was_called++ async def test_a_model_reading_images_and_pdf_judges_both(self) -> None:+ worker = self._worker(inputs=["text", "images", "pdf"])++ answers = await worker.judge(self._job(images={"photo": [self._IMAGE]}, documents={"claim": [self._PDF]}))++ assert set(answers) == {"is_urgent"}+ assert worker.was_called++ async def test_a_document_of_a_format_the_model_does_not_read_is_refused(self) -> None:+ worker = self._worker(inputs=["text", "pdf"])++ with pytest.raises(PromptDocumentFormatError, match="does not read"):+ await worker.judge(self._job(documents={"claim": [self._PDF, self._DOCX]}))++ assert not worker.was_called++ async def test_a_document_of_unknown_format_is_left_to_the_provider(self) -> None:+ worker = self._worker(inputs=["text", "pdf"])++ await worker.judge(self._job(documents={"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim")]}))++ assert worker.was_called++ async def test_an_image_input_carrying_a_document_is_refused(self) -> None:+ worker = self._worker(inputs=["text", "images"])++ with pytest.raises(PromptImageFormatError, match="not an image"):+ await worker.judge(self._job(images={"photo": [PromptImageUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]}))++ assert not worker.was_called++ async def test_more_images_than_the_model_reads_are_refused(self) -> None:+ worker = self._worker(inputs=["text", "images"], max_prompt_images=1)++ with pytest.raises(JudgmentCapabilityError, match="at most 1 images"):+ await worker.judge(self._job(images={"before": [self._IMAGE], "after": [self._IMAGE]}))++ assert not worker.was_calleddiff --git a/tests/unit/pipelex/kernel/test_judgment_material.py b/tests/unit/pipelex/kernel/test_judgment_material.pynew file mode 100644index 000000000..92ed34f79--- /dev/null+++ b/tests/unit/pipelex/kernel/test_judgment_material.py@@ -0,0 +1,181 @@+"""The material a judgment is asked over: one entry per input, read off its content's class, files beside the state."""++import datetime+from typing import Any++import pytest+from pydantic import Field++from pipelex.cogt.document.prompt_document import PromptDocumentUri+from pipelex.cogt.image.prompt_image import PromptImageUri+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.absence import AbsenceKind, AbsenceRecord+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.choice_content import ChoiceContent+from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.rating_content import RatingContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.time_content import TimeContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+from pipelex.kernel.judgment_ops import build_judgment_material+++class _ReportWithNestedContents(StructuredContent):+ """A structure whose fields are typed as the content base, so they hold whichever content subclass a run put there."""++ payload: StuffContent+ extras: list[StuffContent]+++class _Invoice(StructuredContent):+ """A structure a dotted input reaches into."""++ total: float+ note: str | None = None+ scan: ImageContent | None = None+ lines: list[TextContent] = Field(default_factory=list[TextContent])+++def _memory(contents: dict[str, StuffContent]) -> WorkingMemory:+ """A memory holding each content under its name.++ The concept is `Anything` for every one, because the material is read off the content's class and+ never off its concept: that is what keeps the kernel free of the concept library.+ """+ anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)+ stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in contents.items()]+ return WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)+++class TestJudgmentMaterial:+ @pytest.mark.parametrize(+ ("content", "expected"),+ [+ pytest.param(TextContent(text="the roof leaks"), "the roof leaks", id="text"),+ pytest.param(MarkdownContent(text="# Leak"), "# Leak", id="text_refinement"),+ pytest.param(NumberContent(number=42), 42, id="number"),+ pytest.param(YesNoContent(yes_no=False, probability=0.2), False, id="yes_no_as_its_boolean"),+ pytest.param(DateContent(date=datetime.date(2026, 10, 4)), "2026-10-04", id="date"),+ pytest.param(TimeContent(time=datetime.time(15, 40)), "15:40:00", id="time"),+ pytest.param(JSONContent(json_obj={"a": 1}), {"a": 1}, id="json"),+ pytest.param(+ ChoiceContent(choice="billing", confidence=0.7), {"choice": "billing", "confidence": 0.7}, id="choice_without_absent_members"+ ),+ pytest.param(RatingContent(level=2), {"level": 2}, id="rating"),+ pytest.param(HtmlContent(inner_html="<p>hi</p>"), {"inner_html": "<p>hi</p>"}, id="other_structure"),+ pytest.param(ListContent(items=[TextContent(text="a"), TextContent(text="b")]), ["a", "b"], id="list"),+ ],+ )+ def test_a_non_file_input_is_a_member_of_the_state(self, content: StuffContent, expected: Any) -> None:+ state, images, documents = build_judgment_material(memory=_memory({"material": content}), input_names=["material"])++ assert state == {"material": expected}+ assert images == {}+ assert documents == {}++ def test_the_state_is_keyed_by_the_declared_names_in_their_order(self) -> None:+ memory = _memory({"message": TextContent(text="help"), "count": NumberContent(number=3), "unasked": TextContent(text="ignored")})++ state, _, _ = build_judgment_material(memory=memory, input_names=["count", "message"])++ assert list(state) == ["count", "message"]++ def test_an_absent_optional_input_is_left_out(self) -> None:+ memory = _memory({"message": TextContent(text="help")})+ memory.record_resolved_absence(AbsenceRecord(variable_name="context", kind=AbsenceKind.NOT_PROVIDED, reason="not provided"))++ state, images, documents = build_judgment_material(memory=memory, input_names=["message", "context"])++ assert state == {"message": "help"}+ assert images == {}+ assert documents == {}++ def test_an_optional_input_never_written_is_left_out(self) -> None:+ """An optional input with neither a value nor a recorded absence reaches the step: the presence scan lets it through."""+ memory = _memory({"message": TextContent(text="help")})++ state, images, documents = build_judgment_material(memory=memory, input_names=["message", "context"])++ assert state == {"message": "help"}+ assert images == {}+ assert documents == {}++ def test_an_image_and_a_document_go_to_the_file_channel(self) -> None:+ memory = _memory(+ {+ "photo": ImageContent(url="pipelex-storage://s/photo.png", mime_type="image/png"),+ "claim": DocumentContent(url="pipelex-storage://s/claim.pdf", mime_type="application/pdf"),+ "message": TextContent(text="see attached"),+ }+ )++ state, images, documents = build_judgment_material(memory=memory, input_names=["photo", "claim", "message"])++ assert state == {"message": "see attached"}+ assert images == {"photo": [PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")]}+ assert documents == {"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]}++ def test_a_list_of_images_and_a_list_of_documents_go_to_the_file_channel_as_lists(self) -> None:+ memory = _memory(+ {+ "photos": ListContent(items=[ImageContent(url="pipelex-storage://s/a.png"), ImageContent(url="pipelex-storage://s/b.png")]),+ "claims": ListContent(items=[DocumentContent(url="pipelex-storage://s/a.pdf")]),+ }+ )++ state, images, documents = build_judgment_material(memory=memory, input_names=["photos", "claims"])++ assert state == {}+ assert [image.uri for image in images["photos"] if isinstance(image, PromptImageUri)] == [+ "pipelex-storage://s/a.png",+ "pipelex-storage://s/b.png",+ ]+ assert [document.uri for document in documents["claims"] if isinstance(document, PromptDocumentUri)] == ["pipelex-storage://s/a.pdf"]++ def test_an_empty_list_is_an_empty_array_in_the_state(self) -> None:+ state, images, _ = build_judgment_material(memory=_memory({"photos": ListContent[ImageContent](items=[])}), input_names=["photos"])++ assert state == {"photos": []}+ assert images == {}++ def test_a_structure_keeps_the_fields_of_the_contents_nested_in_it(self) -> None:+ report = _ReportWithNestedContents(payload=TextContent(text="The roof is on fire"), extras=[NumberContent(number=3)])+ state, _, _ = build_judgment_material(memory=_memory({"report": report}), input_names=["report"])+ assert state == {"report": {"payload": {"text": "The roof is on fire"}, "extras": [{"number": 3}]}}++ def test_a_dotted_input_is_the_value_at_its_path_keyed_by_its_full_name(self) -> None:+ invoice = _Invoice(total=1250.0, note="rush", lines=[TextContent(text="roof"), TextContent(text="gutter")])+ memory = _memory({"invoice": invoice})++ state, images, documents = build_judgment_material(memory=memory, input_names=["invoice.total", "invoice.note", "invoice.lines"])++ assert state == {"invoice.total": 1250.0, "invoice.note": "rush", "invoice.lines": ["roof", "gutter"]}+ assert images == {}+ assert documents == {}++ def test_a_dotted_input_reaching_a_file_goes_to_the_file_channel(self) -> None:+ invoice = _Invoice(total=1.0, scan=ImageContent(url="pipelex-storage://s/scan.png", mime_type="image/png"))++ state, images, _ = build_judgment_material(memory=_memory({"invoice": invoice}), input_names=["invoice.scan"])++ assert state == {}+ assert images == {"invoice.scan": [PromptImageUri(uri="pipelex-storage://s/scan.png", mime_type="image/png")]}++ def test_a_dotted_input_whose_field_or_root_holds_nothing_is_left_out(self) -> None:+ memory = _memory({"message": TextContent(text="help"), "invoice": _Invoice(total=1.0)})++ state, _, _ = build_judgment_material(memory=memory, input_names=["message", "invoice.note", "estimate.total"])++ assert state == {"message": "help"}diff --git a/tests/unit/pipelex/kernel/test_judgment_verdict_content.py b/tests/unit/pipelex/kernel/test_judgment_verdict_content.pynew file mode 100644index 000000000..449f03716--- /dev/null+++ b/tests/unit/pipelex/kernel/test_judgment_verdict_content.py@@ -0,0 +1,74 @@+"""The verdict native an answer becomes, and what became of a declared threshold."""++import pytest++from pipelex.cogt.judgment.judgment_models import (+ ChoiceAnswer,+ RatingAnswer,+ YesNoAnswer,+)+from pipelex.core.stuffs.choice_content import ChoiceContent+from pipelex.core.stuffs.rating_content import RatingContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+from pipelex.kernel.judgment_ops import make_verdict_content+++class TestVerdictContent:+ @pytest.mark.parametrize(+ ("probability", "threshold", "expected_yes_no"),+ [+ pytest.param(0.7, 0.7, True, id="at_the_threshold"),+ pytest.param(0.6999, 0.7, False, id="just_under"),+ pytest.param(0.7001, 0.7, True, id="just_over"),+ pytest.param(0.5, None, True, id="at_the_default"),+ pytest.param(0.4999, None, False, id="just_under_the_default"),+ ],+ )+ def test_a_probability_decides_the_verdict_against_the_threshold(+ self, probability: float, threshold: float | None, expected_yes_no: bool+ ) -> None:+ content, threshold_applied = make_verdict_content(answer=YesNoAnswer(probability=probability), threshold=threshold, is_dry=False)++ assert content == YesNoContent(yes_no=expected_yes_no, probability=probability)+ assert threshold_applied is (True if threshold is not None else None)++ def test_a_probability_outranks_the_models_own_verdict(self) -> None:+ content, _ = make_verdict_content(answer=YesNoAnswer(probability=0.3, yes_no=True), threshold=0.5, is_dry=False)++ assert content == YesNoContent(yes_no=False, probability=0.3)++ @pytest.mark.parametrize(+ ("threshold", "is_dry", "expected_threshold_applied"),+ [+ pytest.param(0.8, False, False, id="declared_threshold_live"),+ pytest.param(0.8, True, None, id="declared_threshold_dry"),+ pytest.param(None, False, None, id="no_threshold_live"),+ ],+ )+ def test_without_a_probability_the_models_verdict_stands(+ self, threshold: float | None, is_dry: bool, expected_threshold_applied: bool | None+ ) -> None:+ content, threshold_applied = make_verdict_content(answer=YesNoAnswer(yes_no=False), threshold=threshold, is_dry=is_dry)++ assert content == YesNoContent(yes_no=False)+ assert threshold_applied is expected_threshold_applied++ def test_a_choice_is_carried_as_reported(self) -> None:+ answer = ChoiceAnswer(choice="billing", confidence=0.8, probabilities={"billing": 0.8, "technical": 0.2})++ content, threshold_applied = make_verdict_content(answer=answer, threshold=None, is_dry=False)++ assert content == ChoiceContent(choice="billing", confidence=0.8, probabilities={"billing": 0.8, "technical": 0.2})+ assert threshold_applied is None++ def test_a_ratings_distribution_is_keyed_by_the_level_index_as_text(self) -> None:+ answer = RatingAnswer(level=2, position=1.8, confidence=0.7, probabilities={0: 0.1, 1: 0.2, 2: 0.7})++ content, _ = make_verdict_content(answer=answer, threshold=None, is_dry=False)++ assert content == RatingContent(level=2, position=1.8, confidence=0.7, probabilities={"0": 0.1, "1": 0.2, "2": 0.7})++ def test_nothing_absent_is_synthesised(self) -> None:+ content, _ = make_verdict_content(answer=RatingAnswer(level=0), threshold=None, is_dry=False)++ assert content == RatingContent(level=0)diff --git a/tests/unit/pipelex/kernel/test_kernel_boot_contract.py b/tests/unit/pipelex/kernel/test_kernel_boot_contract.pyindex 5ba8a8315..21dbb38c9 100644--- a/tests/unit/pipelex/kernel/test_kernel_boot_contract.py+++ b/tests/unit/pipelex/kernel/test_kernel_boot_contract.py@@ -65,7 +65,7 @@ SUBPROCESS_TIMEOUT_SECONDS = 300#: Both LLM arms run because they share almost nothing below the façade — `resolve_llm_setting_for_text`#: vs `_for_object`, and `dry_llm_gen_text` vs `dry_llm_gen_object` — so covering one leaves the#: other's closure unproven, and the contract is stated over *a kernel call*, not over one of them.-#: The five operator ops below are called as module-level functions rather than through `PipelexKernel`+#: The operator ops below are called as module-level functions rather than through `PipelexKernel`#: for the plain reason that the façade does not expose them._KERNEL_CALL_SCRIPT = textwrap.dedent("""@@ -77,6 +77,8 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(from pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_setting import ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenSetting+ from pipelex.cogt.judgment.judgment_models import ChoiceQuestion+ from pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.cogt.search.search_setting import SearchSettingfrom pipelex.cogt.templating.template_blueprint import TemplateBlueprint@@ -88,6 +90,7 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec+ from pipelex.core.stuffs.choice_content import ChoiceContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.number_content import NumberContent@@ -100,6 +103,7 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(from pipelex.kernel.func_ops import run_funcfrom pipelex.kernel.img_gen_ops import build_img_gen_job_params, run_img_genfrom pipelex.kernel.img_gen_prompt import assemble_img_gen_prompt+ from pipelex.kernel.judgment_ops import run_judgmentfrom pipelex.kernel.llm_results import LlmObjectResult, LlmTextResultfrom pipelex.kernel.memory_ops import (extract_main_content,@@ -365,6 +369,28 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(fail(f"run_search rendered {search_result.rendered_query!r}, not the query it was given")check_stored(search_result, "run_search", SearchResultContent)+ judgment_memory = WorkingMemoryFactory.make_from_single_stuff(+ stuff=StuffFactory.make_stuff(concept=text_concept, content=TextContent(text="My invoice is wrong."), name="message")+ )+ judgment_result = asyncio.run(+ run_judgment(+ memory=judgment_memory,+ question=ChoiceQuestion(instructions="Which team handles {{ message }}?", options={"billing": None, "technical": None}),+ input_names=["message"],+ judgment_setting=JudgmentSetting(model="kernel-boot-contract-judgment-model"),+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.CHOICE),+ job_metadata=kernel.make_step_metadata(),+ cogt_run_params=kernel.cogt_run_params,+ templating_style=templating_style,+ result_name="team",+ )+ )++ # The rendered question proves the templating ran against memory rather than being skipped.+ if judgment_result.rendered_question != "Which team handles My invoice is wrong.?":+ fail(f"run_judgment rendered {judgment_result.rendered_question!r}, not the question over its memory")+ check_stored(judgment_result, "run_judgment", ChoiceContent)+composed_text = "Composed on a kernel-only boot."compose_result = asyncio.run(run_compose_template(diff --git a/tests/unit/pipelex/kernel/test_kernel_log_context.py b/tests/unit/pipelex/kernel/test_kernel_log_context.pyindex 0fd71d81c..7ef362673 100644--- a/tests/unit/pipelex/kernel/test_kernel_log_context.py+++ b/tests/unit/pipelex/kernel/test_kernel_log_context.py@@ -21,6 +21,8 @@ from pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_setting import ExtractSettingfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenSetting+from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, YesNoAnswer, YesNoQuestion+from pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.cogt.search.search_setting import SearchSetting@@ -33,6 +35,7 @@ from pipelex.core.stuffs.search_result_content import SearchResultContentfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.kernel.extract_ops import build_extract_job_params, run_extractfrom pipelex.kernel.img_gen_ops import build_img_gen_job_params, run_img_gen+from pipelex.kernel.judgment_ops import run_judgmentfrom pipelex.kernel.llm_ops import generate_object_content, run_llm_object, run_llm_textfrom pipelex.kernel.llm_prompt_content import LlmPromptContentfrom pipelex.kernel.pipelex_kernel import PipelexKernel@@ -86,6 +89,10 @@ class ContentGeneratorProbe:self._record()return ImageContent(url="https://example.com/kernel-log-context.png")+ async def make_judgment_answers(self, **kwargs: Any) -> dict[str, JudgmentAnswer]: # ruff: ignore[unused-method-argument]+ self._record()+ return {"question": YesNoAnswer(probability=0.8)}+def _install_probe(mocker: MockerFixture, *, ops_module: str) -> ContentGeneratorProbe:"""Replace the content generator where the kernel ops module under test reads it."""@@ -200,6 +207,24 @@ class TestKernelLogContext:_assert_the_step_was_bound(probe=probe, step=step)+ async def test_run_judgment_binds_its_step(self, mocker: MockerFixture) -> None:+ probe = _install_probe(mocker, ops_module="judgment_ops")+ kernel, step = _kernel_and_step()++ await run_judgment(+ memory=WorkingMemoryFactory.make_empty(),+ question=YesNoQuestion(instructions="Does the log context bind?"),+ input_names=[],+ judgment_setting=JudgmentSetting(model="kernel-log-context-judgment-model"),+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.YES_NO),+ job_metadata=step,+ cogt_run_params=kernel.cogt_run_params,+ templating_style=resolve_templating_style(authored=None),+ result_name="verdict",+ )++ _assert_the_step_was_bound(probe=probe, step=step)+async def test_run_img_gen_binds_its_step(self, mocker: MockerFixture) -> None:probe = _install_probe(mocker, ops_module="img_gen_ops")kernel, step = _kernel_and_step()diff --git a/tests/unit/pipelex/kernel/test_resolve_judgment_setting.py b/tests/unit/pipelex/kernel/test_resolve_judgment_setting.pynew file mode 100644index 000000000..d55c17715--- /dev/null+++ b/tests/unit/pipelex/kernel/test_resolve_judgment_setting.py@@ -0,0 +1,79 @@+"""The judgment model a step runs on, and the refusal when there is none."""++from typing import Any++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.exceptions import JudgmentModelMissingError, ModelNotFoundError, ModelWaterfallError+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.kernel.judgment_ops import resolve_judgment_setting+++class _DeckWithoutJudgmentDefault:+ judgment_choice_default = None+++def _deck_serving_no_judgment_model(mocker: MockerFixture) -> Any:+ """A deck whose judgment alias names a model no backend serves, as on a keyless boot that skipped TypeSafe."""+ model_deck = mocker.Mock(judgment_choice_default=None)+ model_deck.get_judgment_setting.return_value = JudgmentSetting(model="jev-1.13.0")+ model_deck.get_optional_inference_model.return_value = None+ model_deck.get_required_inference_model.side_effect = ModelNotFoundError(+ message="Model handle 'jev-1.13.0' was not found in the model deck.", model_handle="jev-1.13.0"+ )+ return model_deck+++def _deck_with_an_unserved_judgment_waterfall(mocker: MockerFixture) -> Any:+ """A deck whose judgment waterfall names only models no backend serves, which the deck answers by raising."""+ model_deck = mocker.Mock(judgment_choice_default=None)+ model_deck.get_judgment_setting.return_value = JudgmentSetting(model="~judges")+ waterfall_error = ModelWaterfallError(message="Model handle 'judges' is a waterfall", model_handle="judges", fallback_list=["jev-1.13.0"])+ model_deck.get_optional_inference_model.side_effect = waterfall_error+ model_deck.get_required_inference_model.side_effect = waterfall_error+ return model_deck+++class TestResolveJudgmentSetting:+ def test_no_choice_and_no_default_is_the_missing_model_error(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=_DeckWithoutJudgmentDefault())++ with pytest.raises(JudgmentModelMissingError) as exc_info:+ resolve_judgment_setting(pipe_code="triage")++ assert "PipeJudge 'triage' has no judgment model" in str(exc_info.value)+ assert "`model`" in str(exc_info.value)+ assert "`choice_default` under `[judgment]`" in str(exc_info.value)++ def test_no_choice_and_no_default_is_the_missing_model_error_on_a_dry_run_too(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=_DeckWithoutJudgmentDefault())++ with pytest.raises(JudgmentModelMissingError):+ resolve_judgment_setting(pipe_code="triage", is_dry=True)++ def test_a_dry_run_keeps_the_handle_of_a_model_no_backend_serves(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=_deck_serving_no_judgment_model(mocker))++ judgment_setting = resolve_judgment_setting(judgment_choice="@default-judgment", pipe_code="triage", is_dry=True)++ assert judgment_setting.model == "jev-1.13.0"++ def test_a_live_run_refuses_a_model_no_backend_serves(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=_deck_serving_no_judgment_model(mocker))++ with pytest.raises(ModelNotFoundError, match=r"jev-1\.13\.0"):+ resolve_judgment_setting(judgment_choice="@default-judgment", pipe_code="triage")++ def test_a_dry_run_keeps_the_handle_of_a_waterfall_no_backend_serves(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=_deck_with_an_unserved_judgment_waterfall(mocker))++ judgment_setting = resolve_judgment_setting(judgment_choice="~judges", pipe_code="triage", is_dry=True)++ assert judgment_setting.model == "~judges"++ def test_a_live_run_refuses_a_waterfall_no_backend_serves(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.kernel.judgment_ops.get_model_deck", return_value=_deck_with_an_unserved_judgment_waterfall(mocker))++ with pytest.raises(ModelWaterfallError):+ resolve_judgment_setting(judgment_choice="~judges", pipe_code="triage")diff --git a/tests/unit/pipelex/kernel/test_run_judgment.py b/tests/unit/pipelex/kernel/test_run_judgment.pynew file mode 100644index 000000000..e52d6bdb0--- /dev/null+++ b/tests/unit/pipelex/kernel/test_run_judgment.py@@ -0,0 +1,111 @@+"""A whole judgment step: the rendered question over the material, the stored verdict, and the threshold warning."""++from typing import Any++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.content_generation.assignment_models import JudgmentAssignment+from pipelex.cogt.judgment.judgment_models import (+ JudgmentAnswer,+ YesNoAnswer,+ YesNoQuestion,+)+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+from pipelex.kernel.judgment_ops import run_judgment+from pipelex.kernel.pipelex_kernel import PipelexKernel+from pipelex.kernel.templating_style_ops import resolve_templating_style+from pipelex.system.pipe_run_mode import PipeRunMode+++def _memory(contents: dict[str, StuffContent]) -> WorkingMemory:+ """A memory holding each content under its name.++ The concept is `Anything` for every one, because the material is read off the content's class and+ never off its concept: that is what keeps the kernel free of the concept library.+ """+ anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)+ stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in contents.items()]+ return WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)+++class _JudgmentProbe:+ """Stands in for the content generator: records the assignment and answers what it was told to."""++ def __init__(self, answer: JudgmentAnswer) -> None:+ self.answer = answer+ self.assignments: list[JudgmentAssignment] = []++ async def make_judgment_answers(self, *, judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentAnswer]:+ self.assignments.append(judgment_assignment)+ return {"question": self.answer}++[email protected](loop_scope="class")+class TestRunJudgment:+ async def _run(+ self, mocker: MockerFixture, *, answer: JudgmentAnswer, threshold: float | None, run_mode: PipeRunMode+ ) -> tuple[_JudgmentProbe, Any]:+ probe = _JudgmentProbe(answer)+ mocker.patch("pipelex.kernel.judgment_ops.get_content_generator", return_value=probe)+ kernel = PipelexKernel.make(storage_scope="test/scope", read_scope=None, run_mode=run_mode, user_id="judgment-ops")+ result = await run_judgment(+ memory=_memory({"message": TextContent(text="The roof is on fire."), "photo": ImageContent(url="pipelex-storage://s/roof.png")}),+ question=YesNoQuestion(instructions="Is this urgent: {{ message }}"),+ input_names=["message", "photo"],+ judgment_setting=JudgmentSetting(model="judgment-ops-model"),+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.YES_NO),+ job_metadata=kernel.make_step_metadata(pipe_code="is_urgent"),+ cogt_run_params=kernel.cogt_run_params,+ templating_style=resolve_templating_style(authored=None),+ threshold=threshold,+ result_name="verdict",+ )+ return probe, result++ async def test_it_sends_the_rendered_question_over_the_material_and_stores_the_verdict(self, mocker: MockerFixture) -> None:+ probe, result = await self._run(mocker, answer=YesNoAnswer(probability=0.9), threshold=0.8, run_mode=PipeRunMode.LIVE)++ (assignment,) = probe.assignments+ assert assignment.questions["question"].instructions == "Is this urgent: The roof is on fire."+ assert assignment.state == {"message": "The roof is on fire."}+ assert list(assignment.images) == ["photo"]+ assert result.rendered_question == "Is this urgent: The roof is on fire."+ assert result.content == YesNoContent(yes_no=True, probability=0.9)+ assert result.threshold_applied is True+ assert result.memory.get_stuff("verdict").content == YesNoContent(yes_no=True, probability=0.9)++ async def test_a_live_threshold_without_a_probability_warns(self, mocker: MockerFixture) -> None:+ warning = mocker.patch("pipelex.kernel.judgment_ops.log.warning")++ _, result = await self._run(mocker, answer=YesNoAnswer(yes_no=True), threshold=0.8, run_mode=PipeRunMode.LIVE)++ assert result.threshold_applied is False+ warning.assert_called_once()+ assert "is_urgent" in warning.call_args.args[0]+ assert "judgment-ops-model" in warning.call_args.args[0]++ async def test_a_dry_threshold_without_a_probability_neither_warns_nor_records(self, mocker: MockerFixture) -> None:+ warning = mocker.patch("pipelex.kernel.judgment_ops.log.warning")++ _, result = await self._run(mocker, answer=YesNoAnswer(yes_no=True), threshold=0.8, run_mode=PipeRunMode.DRY)++ assert result.threshold_applied is None+ warning.assert_not_called()++ async def test_no_threshold_without_a_probability_does_not_warn(self, mocker: MockerFixture) -> None:+ warning = mocker.patch("pipelex.kernel.judgment_ops.log.warning")++ _, result = await self._run(mocker, answer=YesNoAnswer(yes_no=True), threshold=None, run_mode=PipeRunMode.LIVE)++ assert result.threshold_applied is None+ warning.assert_not_called()diff --git a/tests/unit/pipelex/language/test_mthds_schema.py b/tests/unit/pipelex/language/test_mthds_schema.pyindex 4f40d8648..1989b3154 100644--- a/tests/unit/pipelex/language/test_mthds_schema.py+++ b/tests/unit/pipelex/language/test_mthds_schema.py@@ -26,6 +26,7 @@ _PIPE_KIND_EXTRA_FIELDS: dict[str, dict[str, Any]] = {"PipeSearch": {"prompt": "find it"},"PipeStructure": {},"PipeDocGen": {"format": "pdf"},+ "PipeJudge": {"question": "is it?"},"PipeBatch": {"branch_pipe_code": "sub_pipe", "input_list_name": "items", "input_item_name": "item"},"PipeCondition": {"default_outcome": "fallback_pipe", "outcomes": {"yes": "yes_pipe"}},"PipeParallel": {"branches": [{"pipe": "sub_pipe"}]},@@ -284,6 +285,23 @@ class TestMthdsSchemaGeneration:errors = sorted(validator.iter_errors(table), key=str)assert not errors, f"{pipe_type} table should match exactly one oneOf arm, got errors: {[e.message for e in errors]}"+ @pytest.mark.parametrize(+ ("question_fields", "should_validate"),+ [+ pytest.param({"question": "is it?"}, True, id="question"),+ pytest.param({"prompt": "is it?"}, True, id="prompt-synonym"),+ pytest.param({"question": "is it?", "prompt": "is it?"}, False, id="both"),+ pytest.param({}, False, id="neither"),+ ],+ )+ def test_pipe_judge_takes_its_question_or_the_prompt_synonym(+ self, schema: dict[str, Any], question_fields: dict[str, Any], should_validate: bool+ ) -> None:+ """A PipeJudge writes its question as `question` or as `prompt`, exactly one, as its blueprint reads it."""+ validator = _pipe_union_oneof_validator(schema)+ table = {"type": "PipeJudge", "description": "A judge", "output": "YesNo", **question_fields}+ assert validator.is_valid(table) is should_validate, f"{sorted(question_fields)} should {'' if should_validate else 'not '}validate"+@pytest.mark.parametrize(("size_value", "should_validate"),[diff --git a/tests/unit/pipelex/pipe_operators/pipe_judge/data.py b/tests/unit/pipelex/pipe_operators/pipe_judge/data.pynew file mode 100644index 000000000..94e4993f0--- /dev/null+++ b/tests/unit/pipelex/pipe_operators/pipe_judge/data.py@@ -0,0 +1,45 @@+from typing import Any, ClassVar++from pipelex.cogt.judgment.judgment_models import JudgmentKind+++def _judge(**fields: Any) -> dict[str, Any]:+ """A PipeJudge blueprint's fields: a yes/no question over one message, with the given fields on top."""+ blueprint: dict[str, Any] = {"description": "d", "inputs": {"message": "Text"}, "output": "YesNo", "question": "Is it urgent?"}+ blueprint.update(fields)+ return {key: value for key, value in blueprint.items() if value is not None}+++class PipeJudgeBlueprintTestCases:+ """Each refusal is (test_id, blueprint fields, a fragment of its message); each acceptance is (test_id, fields, the kind it resolves to)."""++ REFUSED: ClassVar[list[tuple[str, dict[str, Any], str]]] = [+ ("both_spellings", _judge(prompt="Is it urgent?"), "sets `question`, or `prompt` as its synonym, but not both"),+ ("empty_question", _judge(question=""), "`question` cannot be empty"),+ ("blank_question", _judge(question=" \n "), "`question` cannot be empty"),+ ("dotted_input", _judge(inputs={"invoice.total": "Number"}), "input 'invoice.total' cannot name a field"),+ ("options_and_levels", _judge(output="Choice", options={"a": "", "b": ""}, levels=["low", "high"]), "not both"),+ (+ "criteria_beside_options",+ _judge(output="Choice", options={"a": "", "b": ""}, criteria={"yes": "y"}),+ "`criteria` applies to a yes/no question only",+ ),+ ("threshold_beside_levels", _judge(output="Rating", levels=["low", "high"], threshold=0.7), "`threshold` applies to a yes/no question only"),+ ("one_option", _judge(output="Choice", options={"only": "the one"}), "at least two `options`"),+ ("empty_option_key", _judge(output="Choice", options={"": "nothing", "b": ""}), "none may be empty"),+ ("one_level", _judge(output="Rating", levels=["only"]), "at least two `levels`"),+ ("empty_level", _judge(output="Rating", levels=["low", " "]), "level 1 is"),+ ("threshold_zero", _judge(threshold=0), "strictly between 0 and 1"),+ ("threshold_one", _judge(threshold=1), "strictly between 0 and 1"),+ ("unknown_criteria_key", _judge(criteria={"yes": "y", "maybe": "m"}), "maybe"),+ ("output_with_brackets", _judge(output="YesNo[]"), "produces one verdict"),+ ("undeclared_variable", _judge(question="Is $message about $topic?"), "Variable 'topic' is read by the question but not declared"),+ ]++ ACCEPTED: ClassVar[list[tuple[str, dict[str, Any], JudgmentKind]]] = [+ ("question_yes_no", _judge(threshold=0.8, criteria={"yes": "needs an answer today", "no": "can wait"}), JudgmentKind.YES_NO),+ ("prompt_alone", {**_judge(question=None), "prompt": "Is it urgent?"}, JudgmentKind.YES_NO),+ ("unreferenced_input", _judge(inputs={"message": "Text", "history": "Text"}, question="Is $message urgent?"), JudgmentKind.YES_NO),+ ("empty_string_option_description", _judge(output="Choice", options={"billing": "", "technical": "Bugs"}), JudgmentKind.CHOICE),+ ("rating", _judge(output="Rating", levels=["cosmetic", "a workaround exists", "blocking"]), JudgmentKind.RATING),+ ]diff --git a/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.py b/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.pynew file mode 100644index 000000000..ca369136e--- /dev/null+++ b/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.py@@ -0,0 +1,26 @@+from typing import Any++import pytest+from pydantic import ValidationError++from pipelex.cogt.judgment.judgment_models import JudgmentKind+from pipelex.pipe_operators.judge.pipe_judge_blueprint import PipeJudgeBlueprint+from tests.unit.pipelex.pipe_operators.pipe_judge.data import PipeJudgeBlueprintTestCases+++class TestPipeJudgeBlueprint:+ @pytest.mark.parametrize(("test_id", "blueprint_fields", "message_fragment"), PipeJudgeBlueprintTestCases.REFUSED)+ def test_refused(self, test_id: str, blueprint_fields: dict[str, Any], message_fragment: str):+ with pytest.raises(ValidationError) as exc_info:+ PipeJudgeBlueprint.model_validate(blueprint_fields)+ assert message_fragment in str(exc_info.value), test_id++ @pytest.mark.parametrize(("test_id", "blueprint_fields", "expected_kind"), PipeJudgeBlueprintTestCases.ACCEPTED)+ def test_accepted(self, test_id: str, blueprint_fields: dict[str, Any], expected_kind: JudgmentKind):+ blueprint = PipeJudgeBlueprint.model_validate(blueprint_fields)+ assert blueprint.judgment_kind == expected_kind, test_id++ def test_prompt_is_read_as_question(self):+ blueprint = PipeJudgeBlueprint.model_validate({"description": "d", "output": "YesNo", "prompt": "Is it urgent?"})+ assert blueprint.question == "Is it urgent?"+ assert "prompt" not in blueprint.model_dump()diff --git a/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.py b/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.pyindex e17d7698a..8556758fc 100644--- a/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.py+++ b/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.py@@ -6,7 +6,9 @@ from typesafe_sdk import AsyncTypeSafeClient, SystemOneResponsefrom typesafe_sdk import Noul as TypesafeNoulfrom typing_extensions import override-from pipelex.cogt.exceptions import JudgmentModelNotFoundError+from pipelex.cogt.document.prompt_document import PromptDocument, PromptDocumentUri+from pipelex.cogt.exceptions import JudgmentCapabilityError, JudgmentModelNotFoundError+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageUrifrom pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstractfrom pipelex.cogt.judgment.judgment_job import JudgmentJobfrom pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory@@ -44,14 +46,14 @@ class _RecordingDelegate(ReportingNoOp):self.reported.append(inference_job)-def _model() -> InferenceModelSpec:+def _model(*, inputs: list[str] | None = None) -> InferenceModelSpec:return InferenceModelSpec(backend_name="typesafe",name="jev-1.13.0",sdk="typesafe",model_type=ModelType.JUDGMENT,model_id="jev-1.13.0",- inputs=["text"],+ inputs=inputs or ["text"],outputs=["judgments"],costs={CostCategory.INPUT: 0.042, CostCategory.OUTPUT: 0},thinking_mode=ThinkingMode.NONE,@@ -71,6 +73,44 @@ def _job(questions: dict[str, JudgmentQuestion]) -> JudgmentJob:@pytest.mark.asyncioclass TestTypesafeJudgmentWorker:+ @pytest.mark.parametrize(+ ("inputs", "images", "documents"),+ [+ pytest.param(+ ["text", "images"], {"photo": [PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")]}, None, id="image"+ ),+ pytest.param(+ ["text", "pdf"], None, {"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]}, id="document"+ ),+ ],+ )+ async def test_a_job_carrying_files_is_refused_whatever_the_spec_claims(+ self,+ mocker: MockerFixture,+ inputs: list[str],+ images: dict[str, list[PromptImage]] | None,+ documents: dict[str, list[PromptDocument]] | None,+ ) -> None:+ """TypeSafe judges a JSON state alone, so a spec claiming it reads files would have its files dropped from the request."""+ client = mocker.Mock(spec=AsyncTypeSafeClient)+ client.system_one = mocker.AsyncMock(return_value=load_recorded_response(TestData.THREE_SHAPES))+ worker = TypesafeJudgmentWorker(sdk_instance=cast("AsyncTypeSafeClient", client), inference_model=_model(inputs=inputs))+ job = JudgmentJobFactory.make_judgment_job(+ state={"message": "See the attached file."},+ images=images,+ documents=documents,+ questions={"is_urgent": YesNoQuestion(instructions=TestData.THREE_SHAPES_INSTRUCTIONS["is_urgent"])},+ judgment_setting=JudgmentSetting(model="jev-1.13.0"),+ job_metadata=JobMetadata(+ run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_typesafe")+ ),+ )++ with pytest.raises(JudgmentCapabilityError, match="TypeSafe"):+ await worker.judge(job)++ client.system_one.assert_not_called()+async def test_one_request_carries_the_whole_job_and_its_usage_is_recorded(self, mocker: MockerFixture) -> None:"""The job goes out as a single request under the pinned model id, and the billed tokens land on the report."""client = mocker.Mock(spec=AsyncTypeSafeClient)