Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Judge-evidence-prompt · L-261009-860e46 (#1411)
Louis Choquelf9d6719c5806
diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.tomlindex d87ba7c08..38e142c76 100644--- a/.drift/acks/pipelex-kernel-docs.toml+++ b/.drift/acks/pipelex-kernel-docs.toml@@ -1,8 +1,8 @@contract = "pipelex-kernel-docs"-digest = "sha256:e6e3f137fdaf249212cfe9710089203db64a6cf09219861edc1438ee8bf4d3d7"-reviewed_by = "Claude (Opus 5.5)"-reviewed_at = "2026-10-10T11:27:10Z"-rationale = "clean-pass: judgment_ops.py rewords the warning of run_judgment when a declared threshold meets an answer with no probability, same level and the same pipe_code, threshold and model_handle fields; no kernel function, parameter or behaviour moved. Re-read pipelex-kernel.md and checked its run_judgment paragraph against make_verdict_content: threshold_applied is false only when a threshold is declared, the answer has no probability and the run is not dry, and run_judgment warns exactly then, so the claim that a live run logs a warning and reports threshold_applied as false still holds; the page quotes no message. Nothing was stale."+digest = "sha256:2432a4877f71dbe3676b5a174163343ddb2804f4a7987e9a52f8cac4522fb1f4"+reviewed_by = "Louis Choquel"+reviewed_at = "2026-10-10T18:04:24Z"+rationale = "real-catch: the shared assembly now skips a file reference whose root input is declared optional and was not given, so it takes no number and hands over no file, while ImageReference and DocumentReference gained is_optional and the judgment_ops docstring stopped calling absence the template's business alone. pipelex-kernel.md still said nothing was left out on the kernel's side; the judgment paragraph now says the assembly skips an absent optional input's files, and the prompt-assembly paragraph describes the skip, the is_optional mark the shared factory analysis sets and ContextProviderAbstract.is_variable_present."[trigger_files]"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"@@ -14,17 +14,18 @@ rationale = "clean-pass: judgment_ops.py rewords the warning of run_judgment whe"pipelex/kernel/func_ops.py" = "blob:97c68c977ddc260c9b36b7eda5f4c7964e956a27""pipelex/kernel/func_results.py" = "blob:a7425a0ddc21571f23c53b92f5d7559ffdd08b71""pipelex/kernel/img_gen_ops.py" = "blob:b85360f6c118e93be53bf8d1a04504ccb7552469"-"pipelex/kernel/img_gen_prompt.py" = "blob:1f6347e10eaab2eb229fa1cee080c6bbc7878bd2"+"pipelex/kernel/img_gen_prompt.py" = "blob:c653190d40e4323f763b9da511bd55a4a1646888""pipelex/kernel/img_gen_results.py" = "blob:46b4ca11c70d906ae6d58ce66461f9b3414f1bfa"-"pipelex/kernel/judgment_ops.py" = "blob:4b01f5618b8484ae29041151c8cd3f8ad523ce70"-"pipelex/kernel/judgment_results.py" = "blob:40ba1258cc81d34df3e427d5511ccd40c6edf789"+"pipelex/kernel/judgment_ops.py" = "blob:46baec716f5d3a6f2a969afd68c9ba7b2533af2c"+"pipelex/kernel/judgment_results.py" = "blob:ad759849054a74aaf1fedcbf7bec31d2f753a13c""pipelex/kernel/llm_ops.py" = "blob:d37a9c3718a37f5e7d1550d1512dddbf79032cae"-"pipelex/kernel/llm_prompt_content.py" = "blob:eabf0ee760c3a26b8967973d511e712e9275e99b"+"pipelex/kernel/llm_prompt_content.py" = "blob:f996cead6e7789494f01f69c128ae71d6dcf8466""pipelex/kernel/llm_results.py" = "blob:6e3b8a2878b5bb284ece5fac10262fb59ac2029d""pipelex/kernel/memory_ops.py" = "blob:7fbbfaaec99f9712d8e215520cb02e26aad3fb40""pipelex/kernel/pipelex_kernel.py" = "blob:5dec4e68d517d5ab1d7e9cbb91d836f37b10e713"-"pipelex/kernel/prompt_references.py" = "blob:03ec9ef67ae84f141d072ca2d2c3ef888549ccc0"+"pipelex/kernel/prompt_assembly.py" = "blob:7559725bc0d813cd136bf258b8178d96a8f49194"+"pipelex/kernel/prompt_references.py" = "blob:a4a8c5535c7e51a16ac6c8d1296439b331192a7f""pipelex/kernel/search_ops.py" = "blob:b7de7ff62c45dde7f7aa8a9f409aecf3b664cce5""pipelex/kernel/search_results.py" = "blob:101b70dc96cc5ee1721c9f6b1cb725a5ca78ff70""pipelex/kernel/templating_style_ops.py" = "blob:395473806d600dfc4f124c07f2afebb951fc086b"-"tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:63c06904e9f53f80f98a0a45e00aa8116c9beb31"+"tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:2b60152e6fa07fb0a1eb242cf065d0106d47bc44"diff --git a/.test_durations b/.test_durationsindex a7f4f9a91..b9501eb2f 100644--- a/.test_durations+++ b/.test_durations@@ -1149,7 +1149,7 @@"tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py::TestPipeParallelValidation::test_pipe_parallel_with_real_pipe_structure": 1.2679,"tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_graph.py::TestBindingGraph::test_a_binding_failing_on_any_error_closes_its_node_as_failed": 0.0238,"tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_graph.py::TestBindingGraph::test_a_binding_is_a_node_producing_what_it_binds": 0.0878,- "tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_material.py::TestBindingJudgeMaterial::test_v28_a_judge_reading_a_bound_total_sees_it_alone": 0.1341,+ "tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_evidence.py::TestBindingJudgeEvidence::test_v28_a_judge_reading_a_bound_total_sees_it_alone": 0.1341,"tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_nested_writes.py::TestBindingNestedWrites::test_a_document_swapped_for_another_concept_is_refused": 0.0104,"tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_nested_writes.py::TestBindingNestedWrites::test_a_lifted_nested_sequence_resolves_what_it_always_stores": 0.0693,"tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_nested_writes.py::TestBindingNestedWrites::test_a_name_a_nested_sequence_always_stores_is_no_input_of_the_caller[read-by-a-binding]": 0.038,@@ -8976,29 +8976,6 @@"tests/unit/pipelex/kernel/test_img_gen_size_default.py::TestResolveDefaultSize::test_resolve_default_size[no-default-stays-none]": 0.121,"tests/unit/pipelex/kernel/test_img_gen_size_default.py::TestResolveDefaultSize::test_resolve_default_size[tier-default-applies-without-ratio]": 0.0009,"tests/unit/pipelex/kernel/test_img_gen_size_default.py::TestResolveDefaultSize::test_resolve_default_size[tier-default-composes-with-explicit-ratio]": 0.0014,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_dotted_input_is_the_value_at_its_path_keyed_by_its_full_name": 0.0007,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_dotted_input_reaching_a_file_goes_to_the_file_channel": 0.002,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_dotted_input_whose_field_or_root_holds_nothing_is_left_out": 0.09,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_dotted_name_is_never_walked_as_a_path": 0.0005,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_list_of_images_and_a_list_of_documents_go_to_the_file_channel_as_lists": 0.097,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[choice_without_absent_members]": 0.0004,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[date]": 0.0009,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[json]": 0.0005,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[list]": 0.1293,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[number]": 0.0004,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[other_structure]": 0.0008,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[rating]": 0.0019,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[text]": 0.1017,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[text_refinement]": 0.0009,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[time]": 0.0848,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_non_file_input_is_a_member_of_the_state[yes_no_as_its_boolean]": 0.0007,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_structure_keeps_the_fields_of_the_contents_nested_in_it": 0.0009,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_a_structured_input_is_its_whole_value_keyed_by_its_plain_name": 0.0783,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_an_absent_optional_input_is_left_out": 0.0004,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_an_empty_list_is_an_empty_array_in_the_state": 0.001,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_an_image_and_a_document_go_to_the_file_channel": 0.0008,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_an_optional_input_never_written_is_left_out": 0.0004,- "tests/unit/pipelex/kernel/test_judgment_material.py::TestJudgmentMaterial::test_the_state_is_keyed_by_the_declared_names_in_their_order": 0.0006,"tests/unit/pipelex/kernel/test_judgment_verdict_content.py::TestVerdictContent::test_a_choice_is_carried_as_reported": 0.088,"tests/unit/pipelex/kernel/test_judgment_verdict_content.py::TestVerdictContent::test_a_probability_decides_the_verdict_against_the_threshold[at_the_default]": 0.0072,"tests/unit/pipelex/kernel/test_judgment_verdict_content.py::TestVerdictContent::test_a_probability_decides_the_verdict_against_the_threshold[at_the_threshold]": 0.1012,diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9a2ad1e88..a4cbe77ed 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,13 +4,15 @@### Added-- **`Rating` carries the label of its level**: the `Rating` native has an optional `label`, which the MTHDS standard's native set pinned at `5.0.0` defines as the name of the selected level when the scale declares labels, held in `RatingContent.label` and stated by the input-form descriptor, the I/O contracts and the generated Python and TypeScript types. A `PipeJudge` declares no labelled levels yet, so a `Rating` it produces carries no label, while a `PipeLLM` whose output is `Rating`, or any other producer, may fill it like any optional member; either way a `Rating` renders in a prompt as its label when it has one and as its level otherwise, so `$severity` reads `Workaround available` for a rating carrying that label and `1` for one carrying none, and its JSON rendering carries the label only when present. Since a crate materializes the pinned definition, a crate whose methods use `Rating` gets a new normalized fingerprint, and types generated before this release gain the field when `pipelex codegen types` regenerates them.+- **`Rating` carries the label of its level**: the `Rating` native has an optional `label`, which the MTHDS standard's native set pinned at `5.0.0` defines as the name of the selected level when the scale declares labels, held in `RatingContent.label` and stated by the input-form descriptor, the I/O contracts and the generated Python and TypeScript types. A `PipeJudge` whose levels carry labels fills it with the label of the level it chose, while a `PipeLLM` whose output is `Rating`, or any other producer, may fill it like any optional member; either way a `Rating` renders in a prompt as its label when it has one and as its level otherwise, so `$severity` reads `Workaround available` for a rating carrying that label and `1` for one carrying none, and its JSON rendering carries the label only when present. Since a crate materializes the pinned definition, a crate whose methods use `Rating` gets a new normalized fingerprint, and types generated before this release gain the field when `pipelex codegen types` regenerates them.- **A log line's advice in `user_action`**: a log call puts what its reader should do in a `user_action` field, one imperative sentence, and the `console` sink prints it whole on a line of its own under the record, after `→ `, in cyan, while every other sink writes it as a field like any other. Pipelex's own warnings carry their advice there, such as `Run pipelex migrate to update it` on a stale configuration file, so their message says only what happened.- **`package_prefix` chooses which console lines name their package**: a setting under `[runtime.log.rich_log]` decides which lines the `console` sink starts with their logger's top-level package name, dimmed. `libraries`, the default, keeps the rule that names another library's package and leaves a line from `pipelex` or a `pipelex_` package bare; `all` names every line's package, so `pipelex: `, `pipelex_api: ` and a plugin's name tell apart which component spoke; `none` names none. A line logged on the root logger never carries one.### Changed-- **MTHDS standard 5.0.0 (Breaking)**: pipelex now requires `mthds==0.21.0` and `pipelex-sdk==0.38.0`, the releases that report MTHDS standard `5.0.0`, so everyone downstream inherits those exact versions and a project that pins either package itself has to move with them. `pipelex --version` prints `mthds-standard 5.0.0`, every crate pipelex normalizes is stamped with `mthds_version` `5.0.0` and materializes the native set the standard pinned at `5.0.0`, and a package whose `METHODS.toml` constrains `mthds_version` to `^4.0.0`, `~4.0.0`, `4.*` or exactly `4.0.0` now logs a warning when it loads, which `>=4.0.0` or `^5.0.0` does not. `PipeJudge` keeps the fields its page documents, without the evidence `prompt`, the `questions` and the two-sided criteria the standard's `5.0.0` PipeJudge defines.+- **MTHDS standard 5.0.0 (Breaking)**: pipelex now requires `mthds==0.21.0` and `pipelex-sdk==0.38.0`, the releases that report MTHDS standard `5.0.0`, so everyone downstream inherits those exact versions and a project that pins either package itself has to move with them. `pipelex --version` prints `mthds-standard 5.0.0`, every crate pipelex normalizes is stamped with `mthds_version` `5.0.0` and materializes the native set the standard pinned at `5.0.0`, and a package whose `METHODS.toml` constrains `mthds_version` to `^4.0.0`, `~4.0.0`, `4.*` or exactly `4.0.0` now logs a warning when it loads, which `>=4.0.0` or `^5.0.0` does not. `PipeJudge` judges the evidence of its `prompt` with two-sided criteria and labelled levels, as the standard's `5.0.0` PipeJudge defines, but does not yet take the several `questions` it defines.+- **`PipeJudge` judges the evidence of its `prompt` (Breaking)**: `prompt` is now required and holds the evidence, a template rendered exactly as a `PipeLLM`'s prompt is with its images and documents presented to the judging model as files, beside a required `question`: `prompt` is no longer a synonym of `question`, a pipe writing its question there is refused with a message saying where each goes, and a pipe is rewritten as `prompt = "@message"` beside its question. The model sees an input only as the two templates render it: every declared input must be read by the prompt or the question (`extraneous_input_variable`), an image or a document read by the question is refused (`input_stuff_spec_mismatch`), the model's file capability is checked against what the prompt reads, an absent optional input renders as its guard says, such as `@?note`, and a `Dynamic` input renders as text. `criteria` takes both `yes` and `no`, a one-sided or empty table being refused; a level of `levels` may be a `{ label, description }` table, labels all or none and distinct, and the `Rating` verdict then carries the label of the level chosen; a model that declines the question fails the step with the new `JudgmentRefusedError`, a content error naming the pipe and the model, where no verdict is ever read off a refusal.+- **The judgment worker contract carries a rendered prompt (Breaking)**: `JudgmentJob` and `JudgmentAssignment` carry a `JudgmentPrompt`, the rendered text with its images and documents, in place of the `state`, `images` and `documents` keyed by input name, and `JudgmentState` is gone; `YesNoQuestion` carries one optional `criteria` of type `YesNoCriteria` with both sides in place of `yes_criterion` and `no_criterion`; `RatingQuestion.levels` is a list of `RatingLevel`, each with a `label`, a `description` or both; and `JudgmentWorkerAbstract.judge` returns a `JudgmentOutcome` per question, an answer or a `JudgmentRefusal`. The judgment contract a backend plugin implements, with the family's errors a worker raises, is published in the Inference SPI table, and the kernel's `run_judgment` takes the evidence as a `UserPromptContent`, `build_judgment_material` and `material_value` being gone.- **Console lines without the emoji prefix or the path column (Breaking)**: a line logged under the `pipelex` package or a `pipelex_` one, the runtime's, the API server's or a plugin's named that way, and a line logged on the root logger start with their message, and a line another library logged starts with that library's top-level package name, dimmed, such as `httpx: `, where every line used to start with an emoji or a bracketed logger name. The column naming the source file and line of each record is off by default; set `is_show_path = true` under `[runtime.log.rich_log]` to show it, `is_link_path_enabled` still making it clickable. The `pipelex.tools.log.log_formatter` module is gone with its `EmojiLogFormatter`, so code that formatted records with it uses the stdlib's `logging.Formatter`.- **Log messages are one short sentence (Breaking)**: Pipelex's log messages, and its API server's, say what happened in one sentence that starts with a capital and ends with no period, and at `INFO` and above hold no identifier and no backtick in at most 80 characters, the advice they used to append now riding `user_action`. Warnings about one event became one: whichever kind of inference job carried no usage, the line is `An inference job reported no usage` with `inference_job_type`; a stale configuration file is `A configuration file is out of date and was migrated in memory only` whatever its remedy; and a `PipeBatch` over a large list is `A PipeBatch fans out over a large list without durable execution`, bounded or not, with `max_concurrency` absent when the fan-out is unbounded. What breaks is a log query or an alert written against an old message.- **The API server's refusals of a caller's credentials log at `DEBUG` (Breaking)**: a request with no bearer token, an expired or invalid token or a wrong API key logs `A caller's credentials were refused` at `DEBUG`, with `auth_mode` and `auth_refusal_reason` (`missing_bearer_token`, `expired_token`, `invalid_token` or `api_key_mismatch`), since the error line the server writes at `WARNING` for every `401` already records the refusal. A token that verifies but whose `user_id` claim is missing, not a path-safe segment or the reserved single-tenant id still logs at `WARNING`, its `user_action` telling the token issuer what to fix, and an unset `JWT_SECRET_KEY` or `API_KEY` logs at `ERROR` with the variable's name in `env_var`. An alert on the old authentication warnings counts the `401` error lines instead.@@ -21,6 +23,7 @@### Fixed+- **An absent optional image or document is left out of its prompt**: a `PipeLLM` or `PipeImgGen` whose prompt read an optional image or document behind its guard, such as `@?photo` or `{% if photo %}`, failed with "Could not find image 'photo' in context" when the input was not given, dry runs included. The file is now left out, taking no number and reaching no model, while an absent required file is still refused. A template reading a name aliased to a slot later resolved absent no longer fails with a `KeyError` either: the alias holds nothing, as its target does.- **A Google client's close at teardown is no longer garbage-collected**: when a Google LLM or image-generation worker was torn down inside a running event loop, nothing but the loop referenced the background task closing its async client, and a loop holds its tasks weakly, so the task could be collected before the client was closed, with the `asyncio` error `Task was destroyed but it is pending!`. Each close task is now held until it is done.- **A condition whose outcome may be skipped declares it**: a PipeCondition routing to an outcome pipe that takes plainly an input the condition declares optional must declare its own output optional, and is refused with `optional_not_handled` otherwise, since a run without that input skips the outcome and resolves the condition's output absent; an outcome the condition's expression chooses only with that input present declares it forced (`!`) instead, since validation does not read the expression. Every name such an outcome always stores in its caller's memory counts as maybe-absent too, so a sequence binding one into a plain output is refused where it used to validate and then resolve absent; the outcome is listed in `liftable_pipes` under its condition, and a `!` input on an outcome counts in the `optional_force_redundant` lint.- **A sequence step that may be skipped keeps what its run may store**: a step an absent input may lift, whose pipe stores a name on some runs only, now leaves that name maybe-absent when a run may store an absence there, so a sequence binding it into a plain output is refused with `optional_not_handled` rather than validating and then resolving absent.diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yamlindex ac75d940c..e1dfa9a8c 100644--- a/docs/api-server/openapi/pipelex-api.openapi.yaml+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml@@ -3164,27 +3164,51 @@ components:has no effect on this arm (unlike `rendered_markdown`, which rides both arms because failuretext is exactly what a human surface wants).'- JudgeYesNoCriteria:+ JudgeRatingLevel:properties:- 'yes':+ label:anyOf:- type: string- type: 'null'- title: 'Yes'- 'no':+ title: Label+ description:anyOf:- type: string- type: 'null'+ title: Description+ additionalProperties: false+ type: object+ title: JudgeRatingLevel+ description: 'A rating level written as a table: a `label` naming it in a few words, a `description` of the situation+ it stands for, or both.+++ Closed, as the standard requires. A level written as a plain string is its description.'+ JudgeYesNoCriteria:+ properties:+ 'yes':+ type: string+ title: 'Yes'+ 'no':+ type: stringtitle: 'No'additionalProperties: falsetype: object+ required:+ - 'yes'+ - 'no'title: JudgeYesNoCriteria- description: 'What a yes and a no mean, for a yes/no question. Closed: a key other than `yes` or `no` is refused.+ description: 'What a yes and a no mean, for a yes/no question: both sides, each non-empty. Closed: a key other than+ `yes` or `no` is refused.- The judging vendor silently ignores a criterion key it does not know, so this model is the only+ The judging vendors silently ignore a criterion key they do not know, so this model is the only- place a typo in one is caught.'+ place a typo in one is caught. Both sides are required because a lone side cannot be carried with++ its meaning on every backend: one sends a single criterion as one of a pair of options, where the++ side left out would be a blank.'JudgmentSetting:properties:model:@@ -5167,11 +5191,6 @@ components:Closed shape (`extra="forbid"`).'PipeJudgeBlueprint:- oneOf:- - required:- - question- - required:- - promptproperties:source:anyOf:@@ -5203,6 +5222,9 @@ components:output:type: stringtitle: Output+ prompt:+ type: string+ title: Promptquestion:type: stringtitle: Question@@ -5223,7 +5245,9 @@ components:levels:anyOf:- items:- type: string+ anyOf:+ - type: string+ - $ref: '#/components/schemas/JudgeRatingLevel'type: array- type: 'null'title: Levels@@ -5236,26 +5260,25 @@ components:- type: number- type: 'null'title: Threshold- prompt:- type: string- title: Prompt- description: A synonym of `question`, read exactly as it.additionalProperties: falsetype: objectrequired:- description- output+ - prompt+ - questiontitle: PipeJudgeBlueprint- description: 'Asks a judging model one closed question about its inputs: yes/no, a choice, or a rating.+ description: 'Asks a judging model one closed question about the evidence its prompt presents: yes/no, a choice, or+ a rating.- The kind is decided by which of `options` and `levels` the pipe declares, never by a field of its+ `prompt` is the evidence template, rendered and checked as a PipeLLM''s prompt is, the images and- own. `prompt` is accepted in place of `question` and read exactly as it, so nothing after parsing+ documents it reads presented to the model as files. `question` is the instruction, a plain text- sees the synonym. Every declared input is material to judge, so an input the question never names+ template. The kind is decided by which of `options` and `levels` the pipe declares, never by a- is still read; only a variable the question names must be declared.'+ field of its own. Every declared input is read by the prompt or by the question.'PipeLLMBlueprint:properties:source:diff --git a/docs/building-methods/concepts/native-concepts.md b/docs/building-methods/concepts/native-concepts.mdindex 8f7f95ac2..e94a67622 100644--- a/docs/building-methods/concepts/native-concepts.md+++ b/docs/building-methods/concepts/native-concepts.md@@ -264,7 +264,7 @@ class RatingContent(StuffContent):position: float | None = None # a continuous position, from 0 to the last level's index```-A `Rating`'s `label` is not a measure. The MTHDS standard defines it as the name of the selected level, when the scale declares labels. A `PipeJudge` declares no labelled levels yet, so a `Rating` it produces carries no `label`; a `PipeLLM` whose output is `Rating`, or any other producer, may fill it like any optional member. The `level` stays the verdict either way.+A `Rating`'s `label` is not a measure. The MTHDS standard defines it as the name of the selected level, when the scale declares labels. A `PipeJudge` whose levels carry labels fills it with the label of the level it chose, copied from its declaration, and one whose levels carry none leaves it absent; a `PipeLLM` whose output is `Rating`, or any other producer, may fill it like any optional member. The `level` stays the verdict either way.A `Choice` renders as its key, and a `Rating` as its label when it has one and as its level otherwise, so `$team` in a later prompt reads `billing`, and `$severity` reads `Workaround available` for a rating carrying that label and `1` for one carrying none. Branch on them with a `PipeCondition`: `expression = "team.choice"` routes by the key, and `expression = "'severe' if severity.level >= 2 else 'mild'"` by the level, which every `Rating` has, labelled or not.diff --git a/docs/building-methods/pipes/pipe-operators/PipeJudge.md b/docs/building-methods/pipes/pipe-operators/PipeJudge.mdindex dbd8cf31e..dc41aa401 100644--- a/docs/building-methods/pipes/pipe-operators/PipeJudge.md+++ b/docs/building-methods/pipes/pipe-operators/PipeJudge.md@@ -1,13 +1,23 @@----description: "PipeJudge asks a judgment model one closed question about its inputs and returns a verdict with a measure of how sure it is: yes or no, one option out of a set, or a level on a scale."+description: "PipeJudge asks a judgment model one closed question about the evidence its prompt presents, and returns a verdict with a measure of how sure it is: yes or no, one option out of a set, or a level on a scale."---# PipeJudge-The `PipeJudge` operator asks a judgment model one closed question about its inputs, and stores the verdict it answers with: a `YesNo`, a `Choice` or a `Rating`, each carrying the probabilities the model measured. A judgment model is built for this kind of question: it answers with a verdict and the probabilities it measured, where a language model asked the same question writes its answer out as text.+The `PipeJudge` operator asks a judgment model one closed question about the evidence its prompt presents, and stores the verdict it answers with: a `YesNo`, a `Choice` or a `Rating`, each carrying the probabilities the model measured. A judgment model is built for this kind of question: it answers with a verdict and the probabilities it measured, where a language model asked the same question writes its answer out as text.## How it works+A `PipeJudge` keeps what is judged apart from what is asked. Its `prompt` renders the evidence, the material the model judges, and its `question` asks about that evidence:++```toml+prompt = """+A message from someone writing to the inbox:+@message+"""+question = "Does this message need an answer today?"+```+The fields a `PipeJudge` declares decide the kind of question it asks, and the kind decides its output:| Declares | Asks | Output |@@ -18,11 +28,15 @@ The fields a `PipeJudge` declares decide the kind of question it asks, and the kThe output must be that native or a concept refining it, and `Dynamic` is neither: a pipe declaring `options` whose output is `YesNo` is refused when the method loads, with a message naming both sides. A `PipeJudge` produces one verdict, so its output cannot be a list; to judge every item of a list, map the `PipeJudge` over it with a [`PipeBatch`](../pipe-controllers/PipeBatch.md).-**Every input is material to judge.** The model receives each input the pipe declares, by name, whether or not the question mentions it: a `Text` as its text, a `Number` as its number, a `YesNo` as its boolean, a list as a list, and a structured concept as its fields. An optional input that is absent is left out. Each input is sent whole, under its plain name: an input name never reaches into a field with a dot, on any pipe, and `"invoice.total" = "Number"` is refused as `invalid_input_name`. To judge one field of a larger value without sending the rest of it, have the calling sequence bind the field under a plain name with a binding step, `{ from = "invoice.total", result = "total_amount" }`, and declare `total_amount = "Number"`. So, unlike the other operators, a `PipeJudge` does not have to read its inputs in its `question`; it may still interpolate one, which suits a short parameter such as `"Is the message about $topic?"`, and every variable the question reads must be a declared input.+**The evidence is the rendered prompt.** `prompt` is a template rendered exactly as a [`PipeLLM`](PipeLLM.md)'s prompt is, and the model sees the pipe's inputs only as the prompt and the question render them. `@message` sets the input out as a block tagged with its name, `$total_amount` writes a short value inline, and a dotted reference such as `$invoice.total` reads one field of a larger value, so the model sees that field and nothing else of the invoice. Every declared input must be read by the prompt or by the question, and an input neither of them reads is refused when the method loads, since it would never reach the model. Absence is the template's business too: an optional input is read through a guarded reference such as `@?note`, which renders nothing when the input is absent, and an optional image or document left out that way is not presented to the model at all (see [Understanding optionality](../understanding-optionality.md)).-**Images and documents are sent as files.** An `Image` or a `Document` input, or a list of either, reaches the model as a file rather than as text. Whether a model reads files is stated in its model configuration, and a `PipeJudge` with an image or a document input is refused when the method loads if its model does not read that kind of file, with a message naming the input and what the model reads. An input declared `Dynamic` names no kind of value, so a file it holds is checked when the step runs instead. A document given to the run in a format the model does not read is refused when the run starts, before any step spends anything. TypeSafe's model reads text only, so turn a document into text with a [`PipeExtract`](PipeExtract.md) step first.+**The question is plain text.** `question` is a template as well, with the same shorthand, but it presents no file: it suits the instruction itself and a short parameter such as `"Is the message about $topic?"`. The evidence belongs in the prompt, and a question that reads an `Image` or a `Document` is refused when the method loads, since only the prompt presents files.-**The verdict.** A yes/no question's verdict follows from its probability: yes when the probability of yes is at least the `threshold`, which defaults to `0.5`. A model that reports no probability gives its own verdict, and a declared threshold then cannot apply, which the run logs as a warning. A choice's verdict is the key of the option chosen, with the model's `confidence` and the `probabilities` of every option when it reports them. A rating's verdict is the index of the level chosen, `0` being the first level declared, with the model's `confidence`, the `probabilities` of every level keyed by its index written as text, and a continuous `position` on the scale, when it reports them.+**Images and documents are sent as files.** An `Image` or a `Document` the prompt reads, a single value or a list, reaches the model as a file rather than as text, and a numbered token such as `[Image 1]` marks the place in the rendered prompt where the prompt reads it. Whether a model reads files is stated in its model configuration, and a `PipeJudge` whose prompt reads an image or a document is refused when the method loads if its model does not read that kind of file, with a message naming the variable and what the model reads. An input declared `Dynamic` names no kind of value, so the prompt renders it as text, as a `PipeLLM`'s does, and an image it holds reaches the model as its address rather than as a file. A document given to the run in a format the model does not read is refused when the run starts, before any step spends anything. TypeSafe's model reads text only, so turn a document into text with a [`PipeExtract`](PipeExtract.md) step first.++**The verdict.** A yes/no question's verdict follows from its probability: yes when the probability of yes is at least the `threshold`, which defaults to `0.5`. A model that reports no probability gives its own verdict, and a declared threshold then cannot apply, which the run logs as a warning. A choice's verdict is the key of the option chosen, with the model's `confidence` and the `probabilities` of every option when it reports them. A rating's verdict is the index of the level chosen, `0` being the first level declared, with the model's `confidence`, the `probabilities` of every level keyed by its index written as text, and a continuous `position` on the scale, when it reports them. When the scale's levels carry labels, the verdict's `label` is the label of the level chosen, copied from the pipe's declaration.++**A refusal fails the step.** A judgment model may decline to answer a question, and a refusal carries no verdict: it is never read as a no, as the first option or as the lowest level. The step fails with a [`JudgmentRefusedError`](../../../errors/judgment-refused-error.md) naming the pipe and the model; reword the question so it can be answered from the evidence, or give the step different evidence.The [native concepts page](../../concepts/native-concepts.md) describes the fields of the three verdicts. A downstream pipe reads them like any other fields, for example a [`PipeCondition`](../pipe-controllers/PipeCondition.md) routing on `routing.choice`, or one gating on `approved.probability`.@@ -38,13 +52,14 @@ No judgment model is served out of the box: the model deck names no default for| ------------- | ---------- | ----------- | -------- || `type` | string | The type of the pipe: `PipeJudge`. | Yes || `description` | string | A description of the judgment. | Yes |-| `inputs` | dictionary | The material to judge, as a dictionary mapping input names to concept codes. Every input is sent to the model. | No |+| `inputs` | dictionary | The inputs, as a dictionary mapping input names to concept codes. Every input must be read by `prompt` or by `question`. | No || `output` | string | `YesNo`, `Choice` or `Rating`, according to the kind of question, or a concept refining it. | Yes || `model` | string | The judgment model, alias or preset, such as `"@default-judgment"`. Required unless your model deck names a default for judgments. | No |-| `question` | string | The one closed question to ask. It may interpolate an input with the `$` prefix. `prompt` is read as a synonym, but a pipe may not set both. | Yes |+| `prompt` | string | The evidence template: what the model judges. Rendered as a `PipeLLM` prompt, with the `@variable` and `$variable` shorthand and Jinja2, and presents the images and documents it reads as files. Must not be empty. | Yes |+| `question` | string | The one closed question to ask about the evidence. A plain text template, which may interpolate a short parameter with the `$` prefix and reads no image or document. Must not be empty. | Yes || `options` | dictionary | For a choice question: each key is an option, each value describes it. An empty string leaves an option undescribed. At least two options. | No |-| `levels` | list of strings | For a rating question: the levels from lowest to highest. At least two. | No |-| `criteria` | dictionary | For a yes/no question: what a `yes` and what a `no` mean, either or both. | No |+| `levels` | list | For a rating question: the levels from lowest to highest, at least two. Each is a string describing the level, or a table carrying a `label`, a `description` or both. Either every level carries a label or none does, and labels are distinct. | No |+| `criteria` | dictionary | For a yes/no question: what a `yes` and what a `no` mean, both required. | No || `threshold` | number | For a yes/no question: the probability of yes at or above which the verdict is yes, strictly between 0 and 1. Defaults to `0.5`. | No |### Example: A yes/no question with a threshold@@ -56,6 +71,10 @@ description = "Judges whether a message needs an answer today"inputs = { message = "Text" }output = "YesNo"model = "@default-judgment"+prompt = """+A message from someone writing to the inbox:+@message+"""question = "Does this message need an answer today?"criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }threshold = 0.7@@ -70,6 +89,10 @@ description = "Judges which team should handle a support ticket"inputs = { ticket = "Text" }output = "Choice"model = "@default-judgment"+prompt = """+A support ticket, as the customer wrote it:+@ticket+"""question = "Which team should handle this support ticket?"options = { billing = "Charges, invoices, refunds and subscriptions", technical = "Errors, outages and features that do not work", other = "" }@@ -83,7 +106,7 @@ outcomes = { billing = "acknowledge_billing", technical = "acknowledge_tedefault_outcome = "acknowledge_other"```-### Example: A rating on a scale+### Example: A rating on a labelled scale```toml[pipe.rate_severity]@@ -92,20 +115,49 @@ description = "Rates how severe a bug report is"inputs = { report = "Text" }output = "Rating"model = "@default-judgment"+prompt = """+A bug report filed by a user:+@report+"""question = "How severe is the bug this report describes?"-levels = ["Cosmetic: nothing stops working", "Degraded: a workaround exists", "Blocking: users cannot do their work at all"]+levels = [+ { label = "Cosmetic", description = "Nothing stops working" },+ { label = "Degraded", description = "A workaround exists" },+ { label = "Blocking", description = "Users cannot do their work at all" },+]```+A verdict of the last level reads `level = 2` and `label = "Blocking"`. A level may also be written as a plain string, which is its description: `levels = ["Nothing stops working", "A workaround exists", "Users cannot do their work at all"]`.++### Example: One field of a larger value++```toml+[pipe.judge_needs_second_signature]+type = "PipeJudge"+description = "Judges whether an invoice total needs a second signature"+inputs = { invoice = "Invoice" }+output = "YesNo"+model = "@default-judgment"+prompt = "The total of an invoice, in euros: $invoice.total"+question = "Is this amount high enough to need a second signature?"+criteria = { yes = "The amount is above five thousand euros", no = "The amount is five thousand euros or less" }+```++The model sees the rendered prompt, `The total of an invoice, in euros: 7420.0`, and nothing else of the invoice.+## Writing good judgments- **Ask one narrow question per pipe.** A question that bundles two judgments ("is it urgent and about billing?") gets one verdict for both. Split it into two `PipeJudge` steps, and combine their verdicts downstream.-- **Make levels describe situations, not degrees.** "Blocking: users cannot do their work at all" gives the model something to recognize, where "Very severe" only restates the scale. A long scale of thin levels buys resolution the model cannot supply, and TypeSafe's model reads at most ten levels.-- **Give a choice a catch-all option** when the material may fit none of the others, such as `other = ""`, so the model is not forced to pick a wrong one.+- **Keep the evidence focused.** The judgment vendors advise that a large evidence the question does not need degrades the answers, so render in the prompt what the question is about and leave the rest out: a dotted reference such as `$invoice.total` reads one field, where `@invoice` sets out the whole value.+- **Write criteria for both answers.** `criteria` declares what a `yes` and what a `no` mean, and both are required, so a table with one side or none is refused: when one side is the plain opposite of the other, write it as its complement, "The amount is five thousand euros or less" beside "The amount is above five thousand euros", and leave `criteria` out when the question needs none.+- **Make levels describe situations, not degrees.** "Users cannot do their work at all" gives the model something to recognize, where "Very severe" only restates the scale; a label names the level in a few words, and the description says what it stands for. A long scale of thin levels buys resolution the model cannot supply, and a scale holds at most ten levels: TypeSafe's model refuses more, and a longer scale fails the step before the model is called.+- **Give a choice a catch-all option** when the evidence may fit none of the others, such as `other = ""`, so the model is not forced to pick a wrong one.- **Leave counting, arithmetic and dates to code.** Whether an invoice is over a sum, or a date is past a deadline, is a comparison a [`PipeFunc`](PipeFunc.md) or a `PipeCondition` expression makes exactly.-- **State the condition directly** in the question rather than through an indirection, and keep interpolated inputs to short parameters: the material reaches the model anyway.+- **State the condition directly** in the question rather than through an indirection, and keep what the question interpolates to short parameters: the evidence belongs in the prompt.## Related Documentation- [Judgment Feature](../../../features/judgment.md) - Overview of judgments in a method- [Native concepts](../../concepts/native-concepts.md) - The `YesNo`, `Choice` and `Rating` verdicts+- [PipeLLM](PipeLLM.md) - The prompt templating the evidence shares- [PipeCondition](../pipe-controllers/PipeCondition.md) - Route on a verdictdiff --git a/docs/building-methods/pipes/pipe-operators/index.md b/docs/building-methods/pipes/pipe-operators/index.mdindex 198d663dd..fdda61c25 100644--- a/docs/building-methods/pipes/pipe-operators/index.md+++ b/docs/building-methods/pipes/pipe-operators/index.md@@ -20,7 +20,7 @@ Here are the primary pipe operators available in Pipelex:- [**`PipeExtract`**](./PipeExtract.md): Performs Optical Character Recognition (OCR) on images and PDF documents to extract text and embedded images, and fetches and extracts content from web pages.- [**`PipeImgGen`**](./PipeImgGen.md): Generates images from a text prompt using models like GPT Image, Flux, or other image generation models.- [**`PipeSearch`**](./PipeSearch.md): Searches the web using a configurable search provider and returns structured results with an answer and source citations.-- [**`PipeJudge`**](./PipeJudge.md): Asks a judgment model one closed question about its inputs, and returns a `YesNo`, a `Choice` or a `Rating` verdict with the probabilities the model measured.+- [**`PipeJudge`**](./PipeJudge.md): Asks a judgment model one closed question about the evidence its prompt presents, and returns a `YesNo`, a `Choice` or a `Rating` verdict with the probabilities the model measured.- [**`PipeFunc`**](./PipeFunc.md): An escape hatch that allows you to execute any custom Python function, giving you maximum flexibility.- [**`PipeCompose`**](./PipeCompose.md): Composes outputs deterministically from working memory — renders Jinja2 templates for formatted reports or complex prompts, or constructs structured objects by mapping fields from inputs, without an LLM.- [**`PipeDocGen`**](./PipeDocGen.md): Generates a document file from its inputs, calling no AI model: a PDF laid out from a structure or formatted from a Markdown report, and with the Pipelex document generation plugin a PDF from an HTML template, an Excel workbook, a Word document or a PowerPoint deck.@@ -121,10 +121,11 @@ Searches the web and returns structured results with sources.## PipeJudge-Asks a judgment model one closed question about its inputs.+Asks a judgment model one closed question about the evidence its prompt presents.### Key Features- Yes/no, choice and rating questions, the kind read from the fields the pipe declares- Verdicts carrying the probabilities the model measured, and a threshold for a yes/no-- Every input sent to the model as material, images and documents as files for a model that reads them+- The evidence written as a prompt template, rendered as a `PipeLLM`'s prompt is, its images and documents sent as files to a model that reads them+- Two-sided criteria for a yes/no, and rating levels that may carry labels, which the verdict reportsdiff --git a/docs/building-methods/pipes/understanding-optionality.md b/docs/building-methods/pipes/understanding-optionality.mdindex e3c53860e..44c9fe38a 100644--- a/docs/building-methods/pipes/understanding-optionality.md+++ b/docs/building-methods/pipes/understanding-optionality.md@@ -78,7 +78,7 @@ Write a brief about this topic: $topic- **A presence block** — `{% if enriched %} ... {% endif %}` (any reference inside the block is guarded, including `$enriched` and attribute access).- **An inline presence conditional** — `{{ enriched.text if enriched else "no enrichment" }}`.-The guard-lint covers every pipe that renders authored templates over its inputs: PipeLLM (prompt and system prompt), PipeCompose (template mode), PipeCondition (expression), PipeSearch, PipeJudge (question), and PipeImgGen.+The guard-lint covers every pipe that renders authored templates over its inputs: PipeLLM (prompt and system prompt), PipeCompose (template mode), PipeCondition (expression), PipeSearch, PipeJudge (prompt and question), and PipeImgGen. A PipeJudge's model sees an input only as its templates render it, so an optional piece of evidence is written `@?note` in its prompt, and renders nothing when it is absent. An optional image or document guarded this way in a PipeLLM, PipeImgGen or PipeJudge prompt is not handed to the model when it is absent, and the files that are present keep consecutive numbers.## Controllers Under Absencediff --git a/docs/building-methods/templating-style.md b/docs/building-methods/templating-style.mdindex 0c80040c0..f0aa34957 100644--- a/docs/building-methods/templating-style.md+++ b/docs/building-methods/templating-style.md@@ -131,7 +131,7 @@ So a pipe with `templating_style = "no_tag"` and one with `$article` instead of## Beyond PipeLLM- **[`PipeCompose`](pipes/pipe-operators/PipeCompose.md)** takes `templating_style` on its `[pipe.name.template]` section, for templates that use the `tag` or `format` filters directly. A compose pipe that declares nothing renders under the same runtime default.-- **`PipeImgGen`, `PipeSearch` and `PipeJudge`** have no authored style of their own — their prompts and questions render under the runtime default. Image and search prompts rarely benefit from tagging, but they do render under a real style rather than an invisible fallback.+- **`PipeImgGen`, `PipeSearch` and `PipeJudge`** have no authored style of their own — their prompts and questions render under the runtime default. Image and search prompts rarely benefit from tagging, but they do render under a real style rather than an invisible fallback. A `PipeJudge`'s evidence prompt is where tagging earns its keep: `@message` sets the input out under its name, which is how the judging model learns what each block of evidence is.- **Construct-mode template fields** and the built-in structuring prompts likewise resolve a real style, so a template of yours that uses `| tag` behaves the same everywhere.!!! warning "A style is always required, and never invented"diff --git a/docs/contribute/inference-family-surface.md b/docs/contribute/inference-family-surface.mdindex b61827798..44257b317 100644--- a/docs/contribute/inference-family-surface.md+++ b/docs/contribute/inference-family-surface.md@@ -32,9 +32,11 @@ Most of the list is enforced by the type checker: the matches over the family en11. **The MTHDS protocol's model categories.** `PipelexMTHDSProtocol.models()` in `pipelex/pipeline/runner.py` types each preset by the protocol's own `ModelCategory`, which carries every family this runtime serves, `judgment` included, through two exhaustive matches beside it: `_protocol_category_of` for the deck's entries and `_listing_category_of` for the `?type=` filter. A new member of the model listing's `ModelCategory` (step 2) stops the type checker at the first of them, because a new family needs a protocol category too, which the standard adds in a protocol minor for each settings family it adds. Until the protocol defines one, the family's presets go under an extension property `PipelexModelDeck` adds for them, never under a category the protocol does not define, since a runner must not put such a value in a deck entry's `type`.-12. **Files, when the family reads them.** The job and the assignment carry them keyed by input name, as `JudgmentJob` and `JudgmentAssignment` carry `images` and `documents`, and the assignment's `referenced_uris()` returns each one, or a scoped run's read authorization never sees them. The worker's template method checks that the model reads them before its hook runs, raising the family's capability error (`JudgmentWorkerAbstract._check_can_read_files`), and runs the per-file format checks in `pipelex/cogt/inference/prompt_file_checks.py`, which the LLM family shares. What a model reads is the `inputs` of its spec in the backend's model file, so the capability is configuration, and an operator that knows its model at load time refuses a file input there too, as `PipeJudge` does.+12. **Files, when the family reads them.** The job and the assignment carry them inside the prompt that numbers them, as `JudgmentJob` and `JudgmentAssignment` carry a `JudgmentPrompt` whose `images` and `documents` are the files its `text` marks with `[Image N]` and `[Document N]` tokens, and the assignment's `referenced_uris()` returns each one, or a scoped run's read authorization never sees them. A family whose operator renders a template shapes its contract around the rendered prompt rather than the step's inputs: the operator assembles it through `pipelex/kernel/prompt_assembly.py`, which PipeLLM, PipeImgGen and PipeJudge share, so the worker receives one text and its ordered files and never reads an input by name. The worker's template method checks that the model reads them before its hook runs, raising the family's capability error (`JudgmentWorkerAbstract._check_can_read_files`), and runs the per-file format checks in `pipelex/cogt/inference/prompt_file_checks.py`, which the LLM family shares. What a model reads is the `inputs` of its spec in the backend's model file, so the capability is configuration, and an operator that knows its model at load time refuses a file input there too, as `PipeJudge` does.-13. **Silent: a default nothing serves boots cleanly and fails at run time.** A boot checks only that each alias a deck names exists, not that an enabled backend serves the model it resolves to, so a `default-*` alias pointing at a model no backend declares loads without complaint and fails the first pipe that relies on it with a `ModelNotFoundError`, for everyone running on their own keys. `tests/unit/pipelex/kit/test_shipped_deck_defaults.py` is the guard: every `default-*` alias of the shipped deck must reach, through aliases, presets and every entry of a waterfall, a model some backend file of the kit declares. Give a new family's defaults such a model, or ship no default, as `judgment` ships no `choice_default`.+13. **Outcomes the worker may return besides an answer.** A model may decline a question, and the worker reports that as an outcome rather than as an error or a made-up answer: `judge` returns a `JudgmentOutcome` per question, an answer of the question's kind or a `JudgmentRefusal`, and the base checks an answer's key, kind and offered option while accepting a refusal for any question it was asked. What a refusal means is the caller's policy, never the worker's: the kernel's single-question judgment raises `JudgmentRefusedError`, a content error naming the step and the model, and never reads a refusal as a no, a first option or a lowest level.++14. **Silent: a default nothing serves boots cleanly and fails at run time.** A boot checks only that each alias a deck names exists, not that an enabled backend serves the model it resolves to, so a `default-*` alias pointing at a model no backend declares loads without complaint and fails the first pipe that relies on it with a `ModelNotFoundError`, for everyone running on their own keys. `tests/unit/pipelex/kit/test_shipped_deck_defaults.py` is the guard: every `default-*` alias of the shipped deck must reach, through aliases, presets and every entry of a waterfall, a model some backend file of the kit declares. Give a new family's defaults such a model, or ship no default, as `judgment` ships no `choice_default`.## A new backend for a familydiff --git a/docs/errors/inference-and-providers.md b/docs/errors/inference-and-providers.mdindex 46a8176ec..073383659 100644--- a/docs/errors/inference-and-providers.md+++ b/docs/errors/inference-and-providers.md@@ -48,6 +48,7 @@ own page. Classes are grouped by subsystem.- [`JudgmentJobFailureError`](judgment-job-failure-error.md) — Judgment job failure- [`JudgmentModelMissingError`](judgment-model-missing-error.md) — Judgment model missing- [`JudgmentModelNotFoundError`](judgment-model-not-found-error.md) — Judgment model not found+- [`JudgmentRefusedError`](judgment-refused-error.md) — Judgment refused- [`LLMAssignmentError`](llm-assignment-error.md) — LLM assignment- [`LLMCapabilityError`](llm-capability-error.md) — LLM capability- [`LLMCompletionError`](llm-completion-error.md) — LLM completiondiff --git a/docs/errors/judgment-refused-error.md b/docs/errors/judgment-refused-error.mdnew file mode 100644index 000000000..35a3a16ac--- /dev/null+++ b/docs/errors/judgment-refused-error.md@@ -0,0 +1,21 @@+---+title: "Judgment refused"+description: "Reference for the `JudgmentRefusedError` Pipelex error class."+---++<!-- pipelex:generated -->++# Judgment refused++The judging model declined to answer a step's question.++| Field | Value |+|---|---|+| `error_type` | `JudgmentRefusedError` |+| `title` | Judgment refused |+| `type_uri` | `https://docs.pipelex.com/latest/errors/judgment-refused-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`CogtError`](cogt-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-judge-input-capability-error.md b/docs/errors/pipe-judge-input-capability-error.mdindex 492ef36a4..0f974e1ac 100644--- a/docs/errors/pipe-judge-input-capability-error.md+++ b/docs/errors/pipe-judge-input-capability-error.md@@ -7,7 +7,7 @@ description: "Reference for the `PipeJudgeInputCapabilityError` Pipelex error cl# Pipe judge input capability-A `PipeJudge` declares an image or a document input, and the judging model it resolves to does not read one.+A `PipeJudge`'s prompt presents an image or a document, and the judging model it resolves to does not read one.| Field | Value ||---|---|diff --git a/docs/features/judgment.md b/docs/features/judgment.mdindex 2d697d9fc..b84d4b7c8 100644--- a/docs/features/judgment.md+++ b/docs/features/judgment.md@@ -1,23 +1,25 @@---title: "Judgment"-description: "Closed questions answered by a judgment model inside Pipelex methods. PipeJudge returns a yes/no, a choice or a rating with the probabilities the model measured, ready to route or gate the steps after it."+description: "Closed questions answered by a judgment model inside Pipelex methods. PipeJudge asks about the evidence its prompt presents and returns a yes/no, a choice or a rating with the probabilities the model measured, ready to route or gate the steps after it."---# Judgment-Closed questions about your data, answered with a verdict and how sure the model is.+Closed questions about the evidence a prompt presents, answered with a verdict and how sure the model is.## Overview-Many steps of a method come down to a closed question: is this message urgent, which team owns this ticket, how severe is this report. PipeJudge asks such a question of a judgment model, a model built to answer it with a verdict and the probabilities it measured rather than with free text. The verdict is a typed `YesNo`, `Choice` or `Rating`, so the steps after it read it like any other concept.+Many steps of a method come down to a closed question: is this message urgent, which team owns this ticket, how severe is this report. PipeJudge renders the evidence in its prompt, as a PipeLLM renders its prompt, and asks such a question about it of a judgment model, a model built to answer with a verdict and the probabilities it measured rather than with free text. The verdict is a typed `YesNo`, `Choice` or `Rating`, so the steps after it read it like any other concept.## Key Capabilities-- **Three kinds of question** — A yes/no question, a choice among declared options, or a rating on a scale of declared levels, the kind read from the fields the pipe declares+- **Three kinds of question** — A yes/no question with criteria for both answers, a choice among declared options, or a rating on a scale of declared levels, each with a label, a description or both, the kind read from the fields the pipe declares- **Measured probabilities** — Each verdict carries the probabilities the model measured, so a method can act on how sure it is as well as on what it decided- **Thresholds** — A yes/no question can declare the probability at or above which its verdict is yes-- **Material, not prompts** — Every input of the step is sent to the model as material to judge, images and documents as files for a model that reads them-- **Checked when the method loads** — A step with no model, an output that disagrees with its kind of question, or a file input its model does not read is refused before any run spends anything+- **The evidence is a prompt** — The model judges what the step's prompt renders and nothing else, so the author decides what it sees, down to one field of a larger value; the images and documents the prompt reads are sent as files to a model that reads them+- **Labels on a rating** — When the scale's levels carry labels, the verdict carries the label of the level chosen, copied from the declaration+- **No verdict from a refusal** — A model that declines to answer fails the step with an error naming the pipe and the model, rather than a verdict read off its silence+- **Checked when the method loads** — A step with no model, an input neither its prompt nor its question reads, an output that disagrees with its kind of question, or a file its prompt reads that its model does not is refused before any run spends anything## Usage in Pipelinesdiff --git a/docs/features/pipe-operators.md b/docs/features/pipe-operators.mdindex 922f36748..ff5f0ab1f 100644--- a/docs/features/pipe-operators.md+++ b/docs/features/pipe-operators.md@@ -43,7 +43,7 @@ See [PipeSearch reference](../building-methods/pipes/pipe-operators/PipeSearch.m## PipeJudge-One closed question asked of a judgment model about the step's inputs: yes or no, one option out of a set, or a level on a scale. The verdict comes back as a typed `YesNo`, `Choice` or `Rating` carrying the probabilities the model measured, ready to route a `PipeCondition` or gate a step on a threshold.+One closed question asked of a judgment model about the evidence the step's prompt renders: yes or no, one option out of a set, or a level on a scale. The verdict comes back as a typed `YesNo`, `Choice` or `Rating` carrying the probabilities the model measured, ready to route a `PipeCondition` or gate a step on a threshold.See [PipeJudge reference](../building-methods/pipes/pipe-operators/PipeJudge.md).diff --git a/docs/features/validation-dry-run.md b/docs/features/validation-dry-run.mdindex 1c96ebdde..2605e099c 100644--- a/docs/features/validation-dry-run.md+++ b/docs/features/validation-dry-run.md@@ -19,7 +19,7 @@ Check pipeline syntax, structure, and compatibility without execution:- **Structure validation** — Resolve pipe and concept references in the loaded library, refuse concepts whose structures form a cycle or whose fields name a concept that cannot be found, and verify each pipe's declared inputs against the concepts that operator accepts — plus a controller's declared output against what it actually produces (for a `PipeSequence`, its last step's output concept and multiplicity)- **Input validation** — Ensure required inputs are provided and correctly typed, and that every variable a step reads is bound by the time that step runs- **What each sequence step reads** — Follow each `PipeSequence` step by step and check every step against the concept and multiplicity of each name it reads, whichever declared input or earlier step stored it (`input_stuff_spec_mismatch`): a step reading a single `Page` where an extraction stored `Page[]` is refused before anything runs-- **Declared inputs are read** — For `PipeLLM`, `PipeCompose`, `PipeSearch` and `PipeImgGen`, every variable a template reads must be declared in `inputs` (`missing_input_variable`), and every declared input must be read by one of the pipe's templates (`extraneous_input_variable`): the prompt and system prompt of a `PipeLLM`, the template or the construct of a `PipeCompose`, the prompt of a `PipeSearch`, the prompt and negative prompt of a `PipeImgGen`. An unread input is refused on the pipe that declares it, before any controller above it is asked to supply it. A `PipeJudge` is the exception on that second rule: every input it declares is material for its model, so only the variables its `question` reads must be declared. A name the template sets with `{% set %}` is its own, not an input, from that statement on: after an `{% if %}` it stays set only when every branch sets it, the `{% else %}` included, so a name only some branches set is read from the input of that name on the other paths; and a loop, a macro or a block keeps what it sets to its own body.+- **Declared inputs are read** — For `PipeLLM`, `PipeCompose`, `PipeSearch`, `PipeImgGen` and `PipeJudge`, every variable a template reads must be declared in `inputs` (`missing_input_variable`), and every declared input must be read by one of the pipe's templates (`extraneous_input_variable`): the prompt and system prompt of a `PipeLLM`, the template or the construct of a `PipeCompose`, the prompt of a `PipeSearch`, the prompt and negative prompt of a `PipeImgGen`, the prompt and question of a `PipeJudge`. An unread input is refused on the pipe that declares it, before any controller above it is asked to supply it. A `PipeJudge`'s question presents no file, so an `Image` or a `Document` it reads is refused as `input_stuff_spec_mismatch`: only its prompt presents files to the judging model. A name the template sets with `{% set %}` is its own, not an input, from that statement on: after an `{% if %}` it stays set only when every branch sets it, the `{% else %}` included, so a name only some branches set is read from the input of that name on the other paths; and a loop, a macro or a block keeps what it sets to its own body.A pipe that references a sub-pipe from a package that isn't loaded is reported as **skipped** rather than validated, so a passing run is not proof that every cross-package dependency resolves.@@ -31,6 +31,7 @@ A pipe that references a sub-pipe from a package that isn't loaded is reported aExecute pipelines with mocked LLM responses to test pipeline logic, data flow, and orchestration without making API calls.- **Mock generation** — Format-compliant mock values for constrained fields, including structured outputs+- **Judgments rendered, not answered** — A dry `PipeJudge` renders its evidence prompt and its question as a live run does, then answers with a mock verdict of the right kind carrying no probability; on a scale whose levels carry labels, the mock level carries its declared label- **Configurable mock behavior** — Control mock list sizes, template handling, and response formats- **Full pipeline execution** — Working memory, controllers, and data flow all work as in production- **No credentials needed** — A dry run boots without inference credentials, on both `pipelex run --dry-run` and `pipelex-agent run --dry-run`, and so does `pipelex validate`. Such a boot loads every enabled backend with all its models but resolves none of their keys, so validation and dry runs give the same verdict on a machine that holds no key at all as on one that holds them all: presets, the default models and a bare handle such as `model = "gpt-4o-mini"` resolve alikediff --git a/docs/under-the-hood/architecture-overview.md b/docs/under-the-hood/architecture-overview.mdindex 868e7cd90..143a9b98d 100644--- a/docs/under-the-hood/architecture-overview.md+++ b/docs/under-the-hood/architecture-overview.md@@ -45,7 +45,7 @@ Operators perform concrete actions:- **PipeFunc** - Execute custom Python functions- **PipeCompose** - Compose content from templates or construct structured objects- **PipeSearch** - Search the web for information-- **PipeJudge** - Answer a closed question about the inputs with a verdict+- **PipeJudge** - Answer a closed question about the evidence its prompt renders with a verdictWhat an operator *does* — model-deck resolution, prompt assembly, generation, memory write-back — lives one layer down in [`pipelex/kernel/`](https://github.com/Pipelex/pipelex/tree/main/pipelex/kernel), as plain functions that take everything method-specific as an explicit argument and read only the kernel-layer services a boot stands up, such as the model deck and the content generator. The operator classes above hold what is specific to being a step in a method: blueprint resolution, input validation, error context, and execution-graph tracing. The split exists so the same semantics can be called without a loaded method, and so there is one implementation rather than two that drift. See [The Pipelex Kernel](./pipelex-kernel.md) for the layering contract, what a programmatic caller imports, and how it boots.diff --git a/docs/under-the-hood/image-handling-in-llm-prompts.md b/docs/under-the-hood/image-handling-in-llm-prompts.mdindex 96eeb4e96..2f0c4e825 100644--- a/docs/under-the-hood/image-handling-in-llm-prompts.md+++ b/docs/under-the-hood/image-handling-in-llm-prompts.md@@ -194,7 +194,7 @@ flowchart TB### Factory Time: Prompt Template Analysis-When a PipeLLM is created from a blueprint, the `TemplateImageAnalyzer` examines both `prompt` and `system_prompt` templates:+When a PipeLLM is created from a blueprint, the `TemplateImageAnalyzer` examines both `prompt` and `system_prompt` templates. The factory reaches it through `analyze_template_file_references` (`pipelex.pipe_operators.shared.template_file_references`), the one analysis PipeLLM, PipeImgGen and PipeJudge share, which also finds the documents a template reads and marks each reference with whether its root input is declared optional:1. **Parse prompt template AST** - Extract all variable references with their filters2. **Resolve types** - Look up each variable's type from input specifications@@ -213,7 +213,7 @@ system_image_references = [ImageReference(variable_path="context_image", kind=ImWhen the prompt is built:-1. **Create registry** - Fresh `ImageRegistry` for this prompt+1. **Create registry** - Fresh `ImageRegistry` for this prompt; a reference whose root input is optional and was not given is skipped here, taking no number, so the template's guard renders it out2. **Extract system prompt images first** - Direct and list references from `system_prompt` are processed first, getting lower numbers3. **Extract user prompt images second** - Direct and list references from `prompt` continue the numbering sequence4. **Inject registry into context** - Registry available to Jinja2 filters for nested image extractiondiff --git a/docs/under-the-hood/inference-backend-plugins.md b/docs/under-the-hood/inference-backend-plugins.mdindex e7096ac85..f7b2ce243 100644--- a/docs/under-the-hood/inference-backend-plugins.md+++ b/docs/under-the-hood/inference-backend-plugins.md@@ -268,6 +268,12 @@ What an out-of-tree backend plugin imports *is* the contract. The published surf| `SearchResultContent` | `pipelex.core.stuffs.search_result_content` | a search result || `DocumentContent` | `pipelex.core.stuffs.document_content` | a source document a search result cites || `JudgmentWorkerAbstract` | `pipelex.cogt.judgment.judgment_worker_abstract` | the judgment worker contract |+| `JudgmentJob` | `pipelex.cogt.judgment.judgment_job` | the judgment job: one prompt, and the questions asked about it by key |+| `JudgmentPrompt` | `pipelex.cogt.judgment.judgment_models` | the evidence a judgment presents: the rendered prompt text, with the images and documents it numbers |+| `JudgmentKind`, `JudgmentQuestion`, `YesNoQuestion`, `YesNoCriteria`, `ChoiceQuestion`, `RatingQuestion`, `RatingLevel` | `pipelex.cogt.judgment.judgment_models` | the closed questions a judgment asks: yes/no with optional criteria for both sides, a choice among described options, a rating on a scale of levels each with a label, a description or both |+| `JudgmentAnswer`, `YesNoAnswer`, `ChoiceAnswer`, `RatingAnswer`, `JudgmentRefusal`, `JudgmentOutcome` | `pipelex.cogt.judgment.judgment_models` | what a judgment worker returns per question: an answer of the question's kind, or a refusal when the model declines it |+| `PromptImage` | `pipelex.cogt.image.prompt_image` | an image a prompt presents, by URI, inline or as bytes |+| `PromptDocument` | `pipelex.cogt.document.prompt_document` | a document a prompt presents, by URI, inline or as bytes || `DocGenWorkerAbstract` | `pipelex.cogt.doc_gen.doc_gen_worker_abstract` | the document-generation worker contract || `RenderJob`, `RenderedDocument`, `RenderResources`, `LoadedResource` | `pipelex.cogt.doc_gen.render_job` | the job a document engine prints, the file it returns, and how it reads a file the document names, with that file's media type || `NbTokensByCategoryDict`, `TokenCategory` | `pipelex.cogt.usage.token_category` | the token usage a worker reports |@@ -276,6 +282,7 @@ What an out-of-tree backend plugin imports *is* the contract. The published surf| `BaseModelTypeVar` | `pipelex.tools.typing.pydantic_utils` | the schema type variable of a structured-output signature || `CogtError`, `SdkTypeError`, `ImgGenGenerationError`, `ImgGenParameterError`, `InferenceErrorCategory` | `pipelex.cogt.exceptions` | the inference error bases and categories a worker raises with || `LLMCapabilityError` | `pipelex.cogt.exceptions` | a request the model refuses, raised by an LLM request check |+| `JudgmentJobFailureError`, `JudgmentModelNotFoundError`, `JudgmentCapabilityError`, `JudgmentAnswerMismatchError` | `pipelex.cogt.exceptions` | the judgment family's errors a worker raises: a failed call, a model the service does not serve, a file the model cannot read, answers that do not match the questions |A worker implements the provider half of a call, `_gen_text` or `_extract_pages` for instance, and records the usage the provider answered with on its job's report. The base around it logs the event every call ends with, `Inference call ends`, with that usage, its cost, its duration and its outcome, so a worker logs nothing for the call itself (see [Summary events](../tools/logging.md#summary-events)). A provider that bills by the request or by the page rather than by the token is recorded with `record_unit_priced_usage(tokens_usage=…, pricing_unit=PricingUnit.REQUEST, nb_units=1)`, which writes each unit as a million tokens in and out so the per-million rates price one unit, and marks the usage so the event keeps the call's cost and writes no token counts; a worker never writes those counts itself.diff --git a/docs/under-the-hood/pipelex-kernel.md b/docs/under-the-hood/pipelex-kernel.mdindex 1ef980766..e3d3de6b7 100644--- a/docs/under-the-hood/pipelex-kernel.md+++ b/docs/under-the-hood/pipelex-kernel.md@@ -76,22 +76,23 @@ Both façade calls take the concept and the output class the caller wants, defau| `pipelex.kernel.extract_ops` | `resolve_extract_setting`, `build_extract_job_params`, `run_extract` || `pipelex.kernel.img_gen_ops` | `resolve_img_gen_setting`, `resolve_default_size`, `build_img_gen_job_params`, `run_img_gen` || `pipelex.kernel.search_ops` | `resolve_search_setting`, `run_search` |-| `pipelex.kernel.judgment_ops` | `judgment_setting_of_choice`, `served_judgment_model`, `resolve_judgment_setting`, `build_judgment_material`, `material_value`, `make_verdict_content`, `run_judgment` |+| `pipelex.kernel.judgment_ops` | `judgment_setting_of_choice`, `served_judgment_model`, `resolve_judgment_setting`, `make_verdict_content`, `run_judgment` || `pipelex.kernel.compose_ops` | `build_compose_context`, `build_composed_content`, `run_compose_template` || `pipelex.kernel.func_ops` | `call_registered_function`, `run_func` || `pipelex.kernel.memory_ops` | `shape_inputs`, `store_result`, `extract_main_content` / `extract_named_content`, `extract_main_content_as_list` / `extract_named_content_as_list` |+| `pipelex.kernel.prompt_assembly` | `UserPromptContent`, `AssembledUserPrompt`, `assemble_user_prompt`, `PromptFiles` || `pipelex.kernel.llm_prompt_content` | `LlmPromptContent`, `assemble_llm_prompt` || `pipelex.kernel.img_gen_prompt` | `assemble_img_gen_prompt` || `pipelex.kernel.prompt_references` | `ImageReference` / `ImageReferenceKind`, `DocumentReference` / `DocumentReferenceKind` || `pipelex.kernel.*_results` | The typed result envelopes |-The two `assemble_*` functions are there because `run_llm_text` and `run_img_gen` both take a *ready* prompt. A caller that could not build one would be holding an operator it cannot reach, which is what image generation was until `assemble_img_gen_prompt` existed: its only builder was an interpreter-layer blueprint. What they own is the part a caller must not re-derive — resolving `ImageReference` and `DocumentReference` out of working memory, and, on the image side, keeping the `[Image N]` tokens numbered from the same registry that orders `input_images`, since a mismatch mislabels which image the prompt is describing and nothing downstream can detect it.+The two operator-specific `assemble_*` functions are there because `run_llm_text` and `run_img_gen` both take a *ready* prompt. A caller that could not build one would be holding an operator it cannot reach, which is what image generation was until `assemble_img_gen_prompt` existed: its only builder was an interpreter-layer blueprint. Both are thin over `pipelex.kernel.prompt_assembly`, the user-prompt assembly PipeLLM, PipeImgGen and PipeJudge share. It owns the part a caller must not re-derive: resolving `ImageReference` and `DocumentReference` out of working memory, numbering the files they name, substituting the `[Image N]` and `[Document N]` tokens for direct, list and dotted references, and handing the files over in the order their tokens number them, since a mismatch mislabels which file the prompt is describing and nothing downstream can detect it. `assemble_user_prompt` assembles one template into an `AssembledUserPrompt`, its text with its ordered images and documents; `PromptFiles` is the same numbering shared by several templates of one prompt, a PipeLLM's system and user prompts or a PipeImgGen's positive and negative ones, whose references are all registered before the first renders so their tokens number one sequence. A reference whose root input is declared optional and was not given is skipped before numbering, taking no number and handing over no file, so the template's guard renders it out, while an absent required input is refused; the reference carries `is_optional`, which the factories set through the one analysis they share, `analyze_template_file_references` in `pipelex.pipe_operators.shared`, and the context answers presence through `ContextProviderAbstract.is_variable_present`. The caller's `extra_params` are copied, never written to.`served_llm_model` gives the spec of the LLM a handle resolves to on this boot, an alias to its target and a waterfall to its first served member as a run resolves them, or `None` when no backend serves one. `check_llm_setting_with_served_model` refuses an LLM setting the model it resolves to refuses, before anything is spent: it runs the request check the model's backend registered beside its worker factory, which is the check the worker itself runs before every call, against the job params the setting gives once the model's constraints apply, for a text or a structured output. For a built-in backend it raises an `LLMSettingRefusedError`, an `LLMCapabilityError` carrying the worker's reason with the model named by its deck handle rather than by the SDK, backend and provider model id the worker's own refusal names, so its message is shown to a caller even under strict error disclosure. An external plugin's refusal is raised as the plugin wrote it, and its message reaches a caller only when the plugin raised it as caller-facing copy. A backend that registered no check is held to the rule every worker shares, that a model whose spec declares `thinking_mode = "none"` takes no reasoning setting, and a model no backend serves, or whose backend's SDK is not installed, is left to the run. The interpreter's `PipeLLM` and `PipeStructure` call it when a method loads; a programmatic caller calls it before `run_llm_text` or `run_llm_object` to get the same refusal without a call.-`run_judgment` asks one question over the step's inputs by name, which it calls the material. `build_judgment_material` reads each input off its content's class, never off its concept, so the kernel stays free of the concept library: text is its string, a number its number, a yes/no its boolean, a date or a time its ISO string, a list the array of its items, any other structure its JSON object without its absent members, and an image or a document, or a list of either, travels beside that JSON state as prompt images and documents keyed by the input's name. An optional input that holds no value is left out, whether its absence was recorded or it was never written. An input name is a plain name, so each entry is one input's whole value, and the material holds exactly what the step declares: a field reaches a judgment only when the calling sequence binds it under a name of its own. `make_verdict_content` turns the answer into the verdict native: a yes/no answer's probability decides the verdict against the step's threshold, inclusive, or against 0.5 when none is declared; an answer with no probability keeps the model's own verdict, and when a threshold was declared a live run logs a warning and its `JudgmentResult` reports `threshold_applied` as false. A rating's distribution is keyed by the level index written as text. `judgment_setting_of_choice` gives the setting of the step's model, else of the deck's default, and raises `JudgmentModelMissingError` when neither names one, since the deck serves none by default. `served_judgment_model` gives the spec of the model a handle resolves to on this boot, or `None` when no backend serves one, a waterfall none of whose models is served included. `resolve_judgment_setting` then pins the setting to the handle the model resolves to; on a dry run, a model no backend serves on this boot keeps the deck's handle, since no judging worker is called.+`run_judgment` asks one question about the evidence of a prompt. It takes the step's evidence template as a `UserPromptContent`, the same carrier a PipeLLM's user prompt travels in, and assembles it through `assemble_user_prompt`, so the evidence is rendered, and its images and documents numbered into `[Image N]` and `[Document N]` tokens, exactly as a PipeLLM's prompt is; the text and the ordered files become the `JudgmentPrompt` the worker receives. The question arrives as the cogt question with its instructions still a template, rendered against memory as a `BASIC` template, and is sent as the job's only question, under one fixed key. An optional input that holds no value renders as the template's guard says, through `@?name` or a Jinja2 test, and the shared assembly skips its files, as it does for every operator. A refusal is a worker outcome like an answer, and `run_judgment`'s policy for it is to fail: one question has nowhere to leave its verdict absent, so it raises `JudgmentRefusedError`, a content error naming the step and the model. The `JudgmentResult` carries the prompt sent and the rendered question beside the verdict, which is what the operator records as its execution data. `make_verdict_content` turns the answer into the verdict native: a yes/no answer's probability decides the verdict against the step's threshold, inclusive, or against 0.5 when none is declared; an answer with no probability keeps the model's own verdict, and when a threshold was declared a live run logs a warning and its `JudgmentResult` reports `threshold_applied` as false. A rating takes the label of the level the answer names off the scale the question declared, since no backend reports one, a dry run included, and its distribution is keyed by the level index written as text. `judgment_setting_of_choice` gives the setting of the step's model, else of the deck's default, and raises `JudgmentModelMissingError` when neither names one, since the deck serves none by default. `served_judgment_model` gives the spec of the model a handle resolves to on this boot, or `None` when no backend serves one, a waterfall none of whose models is served included. `resolve_judgment_setting` then pins the setting to the handle the model resolves to; on a dry run, a model no backend serves on this boot keeps the deck's handle, since no judging worker is called.-**Every entry point that renders a template takes a required `templating_style`** — `run_llm_text`, `run_llm_object`, `derive_structure_prompt`, `assemble_llm_prompt`, `assemble_img_gen_prompt`, `run_search`, `run_judgment`, `run_compose_template`. It is not nullable, and that is the contract: the Jinja2 filters that tag and format a value have no default of their own, so a call that omitted the style would render a prompt in a shape nobody chose. `resolve_templating_style(authored=...)` is what produces one — pass what the caller authored, or `None` to take the runtime default.+**Every entry point that renders a template takes a required `templating_style`** — `run_llm_text`, `run_llm_object`, `derive_structure_prompt`, `assemble_user_prompt`, `assemble_llm_prompt`, `assemble_img_gen_prompt`, `run_search`, `run_judgment`, `run_compose_template`. It is not nullable, and that is the contract: the Jinja2 filters that tag and format a value have no default of their own, so a call that omitted the style would render a prompt in a shape nobody chose. `resolve_templating_style(authored=...)` is what produces one — pass what the caller authored, or `None` to take the runtime default.There are **no re-exports**: `pipelex/kernel/__init__.py` holds doctrine and nothing else, and every symbol is imported from the module that defines it. For this package that is a layering property rather than a style one — a module that re-exports across layers is a layer boundary with the sign filed off.diff --git a/docs/under-the-hood/template-sandbox.md b/docs/under-the-hood/template-sandbox.mdindex e38aced55..5b89ef48f 100644--- a/docs/under-the-hood/template-sandbox.md+++ b/docs/under-the-hood/template-sandbox.md@@ -5,7 +5,7 @@ description: "Every Jinja2 template Pipelex renders runs under one sandbox polic# Template Sandbox-A method's templates are code. PipeLLM prompts, PipeCompose templates and construct templates, PipeImgGen and PipeSearch prompts, PipeJudge questions, and PipeCondition expressions are all Jinja2 templates, and they render inside the process that runs the method. When that process is shared, as it is on a hosted runner, a template must not be able to reach anything but the data it was given.+A method's templates are code. PipeLLM prompts, PipeCompose templates and construct templates, PipeImgGen and PipeSearch prompts, PipeJudge prompts and questions, and PipeCondition expressions are all Jinja2 templates, and they render inside the process that runs the method. When that process is shared, as it is on a hosted runner, a template must not be able to reach anything but the data it was given.Pipelex therefore renders every template, its own included, under one policy: **a template reads data and calls methods of plain values, and nothing else.** And because a template that only reads data can still ask for a billion-character string, every render also spends from a [budget of work](#the-render-budget), and a render that would overdraw it is refused before it allocates.diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.pyindex d8c6d43cc..1a67a25c5 100644--- a/pipelex/cogt/content_generation/assignment_models.py+++ b/pipelex/cogt/content_generation/assignment_models.py@@ -6,13 +6,13 @@ from typing_extensions import overridefrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParamsfrom pipelex.cogt.doc_gen.doc_gen_setting import DocGenSettingfrom pipelex.cogt.doc_gen.document_composition import DocumentComposition-from pipelex.cogt.document.prompt_document import PromptDocument, PromptDocumentUri+from pipelex.cogt.document.prompt_document import PromptDocumentUrifrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParams-from pipelex.cogt.image.prompt_image import PromptImage, PromptImageUri+from pipelex.cogt.image.prompt_image import PromptImageUrifrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt-from pipelex.cogt.judgment.judgment_models import JudgmentQuestion, JudgmentState+from pipelex.cogt.judgment.judgment_models import JudgmentPrompt, JudgmentQuestionfrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.llm_job_components import LLMJobParamsfrom pipelex.cogt.llm.llm_prompt import LLMPrompt@@ -245,21 +245,18 @@ class JudgmentAssignment(BaseModel):"""Serializable unit for a single judgment leaf call.Carries everything the framework-agnostic ``judgment_generate`` core needs to rebuild the- ``JudgmentJob`` on the other side of a distributed boundary: the state the questions are asked- over, the questions themselves, and the fully resolved ``judgment_setting`` (its ``model`` is the- resolved provider handle, also the routing key). Mirrors ``SearchAssignment`` for the search leaf.+ ``JudgmentJob`` on the other side of a distributed boundary: the prompt the questions are asked+ over, with the images and documents it presents, the questions themselves, and the fully resolved+ ``judgment_setting`` (its ``model`` is the resolved provider handle, also the routing key). Mirrors+ ``SearchAssignment`` for the search leaf.Unlike the structured-search leaf, there is no schema to ship and no dynamic class to rebuild:- a judgment's answers are plain models of this package's own, so they cross a boundary as they are.- The files travel beside the state, keyed by the input they came from, as the prompt images and- documents an LLM assignment carries.+ a judgment's outcomes are plain models of this package's own, so they cross a boundary as they are."""job_metadata: JobMetadatacogt_run_params: CogtRunParams- state: JudgmentState- images: dict[str, list[PromptImage]] = Field(default_factory=dict)- documents: dict[str, list[PromptDocument]] = Field(default_factory=dict)+ prompt: JudgmentPromptquestions: dict[str, JudgmentQuestion] = Field(min_length=1)judgment_setting: JudgmentSetting@@ -268,17 +265,15 @@ class JudgmentAssignment(BaseModel):return self.judgment_setting.modeldef referenced_uris(self) -> list[UriReference]:- """The URLs the leaf reads for this assignment: its files given by URI, never a URL the state carries as text."""+ """The URLs the leaf reads for this assignment: the prompt's files given by URI, never a URL its text carries."""uri_references = [- UriReference(uri=image.uri, position=f"image {index} of input '{input_name}'")- for input_name, images in self.images.items()- for index, image in enumerate(images, start=1)+ UriReference(uri=image.uri, position=f"image {index} of the prompt")+ for index, image in enumerate(self.prompt.images, start=1)if isinstance(image, PromptImageUri)]uri_references.extend(- UriReference(uri=document.uri, position=f"document {index} of input '{input_name}'")- for input_name, documents in self.documents.items()- for index, document in enumerate(documents, start=1)+ UriReference(uri=document.uri, position=f"document {index} of the prompt")+ for index, document in enumerate(self.prompt.documents, start=1)if isinstance(document, PromptDocumentUri))return uri_referencesdiff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex 283dbff79..84e69efcf 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -31,7 +31,7 @@ from pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer+from pipelex.cogt.judgment.judgment_models import JudgmentOutcomefrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.config import get_config@@ -324,5 +324,5 @@ class ContentGenerator(ContentGeneratorProtocol):async def make_judgment_answers(self,judgment_assignment: JudgmentAssignment,- ) -> dict[str, JudgmentAnswer]:+ ) -> dict[str, JudgmentOutcome]:return await judgment_gen_answers(judgment_assignment)diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.pyindex 1e901c592..5df7b3e13 100644--- a/pipelex/cogt/content_generation/content_generator_protocol.py+++ b/pipelex/cogt/content_generation/content_generator_protocol.py@@ -10,7 +10,7 @@ from pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer+from pipelex.cogt.judgment.judgment_models import JudgmentOutcomefrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.core.stuffs.document_content import DocumentContent@@ -169,4 +169,4 @@ class ContentGeneratorProtocol(Protocol):def make_judgment_answers(self,judgment_assignment: JudgmentAssignment,- ) -> Coroutine[Any, Any, dict[str, JudgmentAnswer]]: ...+ ) -> Coroutine[Any, Any, dict[str, JudgmentOutcome]]: ...diff --git a/pipelex/cogt/content_generation/dry_mock.py b/pipelex/cogt/content_generation/dry_mock.pyindex 4bd41c69b..0bcd6ff87 100644--- a/pipelex/cogt/content_generation/dry_mock.py+++ b/pipelex/cogt/content_generation/dry_mock.py@@ -58,7 +58,7 @@ from pipelex.cogt.content_generation.schema_to_model_factory import SchemaToModefrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,ChoiceQuestion,- JudgmentAnswer,+ JudgmentOutcome,RatingAnswer,RatingQuestion,YesNoAnswer,@@ -389,16 +389,17 @@ def dry_search_gen_sourced_answer() -> SearchResultContent:return build_mock_object(SearchResultContent, sources=mock_sources)-def dry_judgment_gen_answers(judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentAnswer]:- """Dry leaf for a judgment: a deterministic verdict per question, with no uncertainty at all.+def dry_judgment_gen_answers(judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentOutcome]:+ """Dry leaf for a judgment: a deterministic verdict per question, with no uncertainty at all, and never a refusal.Every answer is the first thing its question allows — yes, the first option, the lowest level —because a dry run must be reproducible and nothing here measured anything. The uncertaintymembers are left absent for the same reason: the contract makes them optional precisely so aproducer that measured nothing can say so, and a mock probability would be the one number in the- whole family that nobody could tell apart from a real one.+ whole family that nobody could tell apart from a real one. A level's label is not an answer's: the+ operator copies it from the declared scale, dry or live."""- answers: dict[str, JudgmentAnswer] = {}+ answers: dict[str, JudgmentOutcome] = {}for question_key, question in judgment_assignment.questions.items():match question:case YesNoQuestion():diff --git a/pipelex/cogt/content_generation/judgment_generate.py b/pipelex/cogt/content_generation/judgment_generate.pyindex 77a28252d..cfcc4031e 100644--- a/pipelex/cogt/content_generation/judgment_generate.py+++ b/pipelex/cogt/content_generation/judgment_generate.py@@ -6,8 +6,8 @@ runs it. The direct ``ContentGenerator`` calls it inline; a distributed orchestran activity so the result is recorded in the run's history and any failure is converted to aterminal error.-There is only one entry point here, where the search leaf has three. A judgment's answers are plain-models of this package's own — no caller class travels down and no schema has to be shipped — so the+There is only one entry point here, where the search leaf has three. A judgment's outcomes, answers+and refusals, are plain models of this package's own — no caller class travels down and no schema has to be shipped — so thein-process arm and the boundary arm would be the same function, and there is nothing to split."""@@ -16,7 +16,7 @@ from pipelex.cogt.content_generation.dry_mock import dry_judgment_gen_answersfrom pipelex.cogt.content_generation.read_authorization import authorize_assignment_readsfrom pipelex.cogt.judgment.judgment_job import JudgmentJobfrom pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer+from pipelex.cogt.judgment.judgment_models import JudgmentOutcomefrom pipelex.cogt.judgment.judgment_worker_abstract import JudgmentWorkerAbstractfrom pipelex.cogt.judgment.judgment_worker_factory import JudgmentWorkerFactoryfrom pipelex.cogt.model_backends.model_type import ModelType@@ -34,16 +34,14 @@ def _make_judgment_worker(judgment_assignment: JudgmentAssignment) -> JudgmentWodef _make_judgment_job(judgment_assignment: JudgmentAssignment) -> JudgmentJob:return JudgmentJobFactory.make_judgment_job(- state=judgment_assignment.state,- images=judgment_assignment.images,- documents=judgment_assignment.documents,+ prompt=judgment_assignment.prompt,questions=judgment_assignment.questions,judgment_setting=judgment_assignment.judgment_setting,job_metadata=judgment_assignment.job_metadata,)-async def judgment_gen_answers(judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentAnswer]:+async def judgment_gen_answers(judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentOutcome]:authorize_assignment_reads(job_metadata=judgment_assignment.job_metadata, uri_references=judgment_assignment.referenced_uris())if judgment_assignment.cogt_run_params.run_mode.is_dry:return dry_judgment_gen_answers(judgment_assignment)diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 1edc0c47d..200b3be7b 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -505,6 +505,33 @@ class JudgmentModelNotFoundError(ModelNotFoundError):pass+class JudgmentRefusedError(CogtError):+ """The judging model declined to answer a step's question.++ A refusal is a worker's outcome, never its error: this is the operator's policy for a question+ whose verdict has nowhere to be left absent. It is a content error, in the input domain, because+ what a model declines to judge is the question asked over the evidence given, and the remedy is+ the author's or the caller's: a reworded question or different evidence. The message names only+ the step and the model's deck handle, so it is kept verbatim for the caller.+ """++ error_category = InferenceErrorCategory.CONTENT+ _authors_caller_facing_message = True++ def __init__(self, *, pipe_code: str | None, model_handle: str):+ step = f"PipeJudge '{pipe_code}'" if pipe_code else "This judgment"+ message = (+ f"{step}: the judgment model '{model_handle}' declined to answer its question. "+ "Try to reword the question so it can be answered from the evidence, or give the step different evidence."+ )+ super().__init__(+ message,+ user_action=UserAction(kind=UserActionKind.CHANGE_INPUT, detail="Reword the question, or give the step different evidence."),+ )+ self.pipe_code = pipe_code+ self.model_handle = model_handle++class JudgmentModelMissingError(PipelexError):"""A judgment has no model to run on: the step names none, and the model deck names no default.diff --git a/pipelex/cogt/judgment/judgment_job.py b/pipelex/cogt/judgment/judgment_job.pyindex 17e385460..60087938d 100644--- a/pipelex/cogt/judgment/judgment_job.py+++ b/pipelex/cogt/judgment/judgment_job.py@@ -3,10 +3,8 @@ from datetime import datetimefrom pydantic import BaseModel, Fieldfrom typing_extensions import override-from pipelex.cogt.document.prompt_document import PromptDocument-from pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract-from pipelex.cogt.judgment.judgment_models import JudgmentQuestion, JudgmentState+from pipelex.cogt.judgment.judgment_models import JudgmentPrompt, JudgmentQuestionfrom pipelex.cogt.judgment.judgment_report import JudgmentTokensUsagefrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec@@ -21,25 +19,22 @@ class JudgmentJobReport(BaseModel):class JudgmentJob(InferenceJobAbstract):- """One state, a map of questions over it, and the answers they get.+ """One prompt, a map of questions over it, and the outcomes they get.- The shape is batch-first because batching is the backend's whole economy: the spike measured- three questions over one state at 506 input tokens against 1138 for the same three sent+ The shape is batch-first because batching is the backends' whole economy: the spike measured+ three questions over one evidence at 506 input tokens against 1138 for the same three sentseparately. A caller with one question sends a batch of one; a contract that could not expressthe batch would have to be broken to gain it later.- The question keys are the caller's own — the model never sees them — and they are what the- answers come back under.+ The question keys are the caller's own, and they are what the outcomes come back under. A backend+ may show them to its model or not: the one that sends a question's name echoes it on its answer.- The files travel beside the state, keyed by the name of the input they came from, because a file- is not a JSON value: an image or a document reaches a judgment exactly as it reaches an LLM, as a- prompt image or a prompt document. Whether the model reads them is its own capability, which the- worker checks before the backend is called.+ The evidence is the rendered prompt with the images and documents it presents, ordered as its+ tokens number them, exactly as an image or a document reaches an LLM. Whether the model reads them+ is its own capability, which the worker checks before the backend is called."""- state: JudgmentState- images: dict[str, list[PromptImage]] = Field(default_factory=dict)- documents: dict[str, list[PromptDocument]] = Field(default_factory=dict)+ prompt: JudgmentPromptquestions: dict[str, JudgmentQuestion] = Field(min_length=1)job_params: JudgmentJobParamsjob_report: JudgmentJobReport = JudgmentJobReport()diff --git a/pipelex/cogt/judgment/judgment_job_factory.py b/pipelex/cogt/judgment/judgment_job_factory.pyindex 6a131ded2..888143eb8 100644--- a/pipelex/cogt/judgment/judgment_job_factory.py+++ b/pipelex/cogt/judgment/judgment_job_factory.py@@ -1,7 +1,5 @@-from pipelex.cogt.document.prompt_document import PromptDocument-from pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.judgment.judgment_job import JudgmentJob, JudgmentJobParams-from pipelex.cogt.judgment.judgment_models import JudgmentQuestion, JudgmentState+from pipelex.cogt.judgment.judgment_models import JudgmentPrompt, JudgmentQuestionfrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.system.job_metadata import JobCategory, JobMetadata@@ -11,20 +9,16 @@ class JudgmentJobFactory:def make_judgment_job(cls,*,- state: JudgmentState,+ prompt: JudgmentPrompt,questions: dict[str, JudgmentQuestion],judgment_setting: JudgmentSetting,job_metadata: JobMetadata,- images: dict[str, list[PromptImage]] | None = None,- documents: dict[str, list[PromptDocument]] | None = None,) -> JudgmentJob:job_metadata.job_category = JobCategory.JUDGMENT_JOBjob_params = JudgmentJobParams(judgment_setting=judgment_setting)return JudgmentJob(job_metadata=job_metadata,- state=state,- images=images or {},- documents=documents or {},+ prompt=prompt,questions=questions,job_params=job_params,)diff --git a/pipelex/cogt/judgment/judgment_models.py b/pipelex/cogt/judgment/judgment_models.pyindex a4b9e2d26..275711322 100644--- a/pipelex/cogt/judgment/judgment_models.py+++ b/pipelex/cogt/judgment/judgment_models.py@@ -1,25 +1,44 @@-"""The questions a judgment asks and the answers it gets back.+"""The evidence a judgment is asked over, the questions it asks, and the outcomes it gets back.-Both are discriminated unions over :class:`JudgmentKind`, and the three kinds are the language's-own words — yes/no, choice, rating — never a vendor's. A backend translates into its own vocabulary-inside its worker and nowhere else.+This is the judgment family's worker contract, and it speaks no vendor's vocabulary. The evidence is a+prompt: the rendered text and the files it presents, ordered as its `[Image N]` and `[Document N]`+tokens number them. The questions and the answers are discriminated unions over+:class:`JudgmentKind`, whose three kinds are the language's own words — yes/no, choice, rating —+never a vendor's. A backend translates into its own vocabulary inside its worker and nowhere else.**Every uncertainty member is optional, and that is a contract rather than an oversight.** A backendthat measures a probability reports it; a backend that cannot — a generative model emulating ajudgment, say — leaves it absent rather than inventing a number. Only the verdict itself is required,and :class:`YesNoAnswer` requires at least one of its two forms of verdict.++**A refusal is an outcome, not an error.** A model may decline to answer a question, and the worker+returns a :class:`JudgmentRefusal` for it, which discloses nothing. What a refusal means is the+operator's policy, decided above the worker, so a worker never turns one into an exception."""from enum import StrEnum-from typing import Annotated, Any, Literal, Self, TypeAlias+from typing import Annotated, Literal, Self, TypeAlias++from pydantic import BaseModel, ConfigDict, Field, model_validator++from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.image.prompt_image import PromptImage-from pydantic import BaseModel, Field, model_validator-# The material a judgment is passed, as a JSON object with one member per name the questions refer-# to. An object rather than free text or an array on purpose: the vendor reads named fields as-# relationships, and the spike measured an array state — which drops the names — answering-# measurably worse. `Any` is the JSON value space, and the kernel is what fills it.-JudgmentState: TypeAlias = dict[str, Any]+class JudgmentPrompt(BaseModel):+ """The evidence a judgment is asked over: a rendered prompt and the files it presents.++ The images and the documents are ordered as the prompt's registry numbered them, so the n-th image+ is the one the text's `[Image n]` token names. A backend that lays images out beside the text reads+ that correspondence; a backend that reads text alone refuses any file.+ """++ model_config = ConfigDict(extra="forbid")++ text: str+ # Subscripted `list` factories: `empty_list_factory_of` takes a class, and both item types are `Annotated` unions.+ images: list[PromptImage] = Field(default_factory=list[PromptImage])+ documents: list[PromptDocument] = Field(default_factory=list[PromptDocument])class JudgmentKind(StrEnum):@@ -28,34 +47,77 @@ class JudgmentKind(StrEnum):RATING = "rating"+class YesNoCriteria(BaseModel):+ """What a yes and a no mean. Both sides or none: a lone side cannot be carried with its meaning on every backend."""++ model_config = ConfigDict(extra="forbid")++ yes: str+ no: str++class YesNoQuestion(BaseModel):"""Does this condition hold? The answer's probability is its confidence."""+ model_config = ConfigDict(extra="forbid")+kind: Literal[JudgmentKind.YES_NO] = JudgmentKind.YES_NOinstructions: str- yes_criterion: str | None = None- no_criterion: str | None = None+ criteria: YesNoCriteria | None = Noneclass ChoiceQuestion(BaseModel):"""Which one of these options? An option may be declared without a description."""+ model_config = ConfigDict(extra="forbid")+kind: Literal[JudgmentKind.CHOICE] = JudgmentKind.CHOICEinstructions: stroptions: dict[str, str | None] = Field(min_length=1)+class RatingLevel(BaseModel):+ """One level of a rating scale: a label naming it in a few words, a description of the situation it stands for, or both."""++ model_config = ConfigDict(extra="forbid")++ label: str | None = None+ description: str | None = None++ @model_validator(mode="after")+ def validate_has_a_label_or_a_description(self) -> Self:+ if self.label is None and self.description is None:+ msg = "A rating level carries a label, a description or both, and this one carries neither"+ raise ValueError(msg)+ return self++class RatingQuestion(BaseModel):- """Where on this scale? The levels are ordered from low to high, each describing a situation.+ """Where on this scale? The levels are ordered from low to high.One level is the floor here because one is what a judgment needs to be answerable at all. Thelanguage's own minimum is two, and it belongs to the blueprint that authors write against; a- backend's maximum is that backend's, and belongs to its worker.+ backend's maximum is that backend's, and belongs to its worker. The labels are all or none, and+ distinct, because a label is what a rating verdict reports: two equal labels would make it ambiguous."""+ model_config = ConfigDict(extra="forbid")+kind: Literal[JudgmentKind.RATING] = JudgmentKind.RATINGinstructions: str- levels: list[str] = Field(min_length=1)+ levels: list[RatingLevel] = Field(min_length=1)++ @model_validator(mode="after")+ def validate_labels_are_all_or_none_and_distinct(self) -> Self:+ labels = [level.label for level in self.levels if level.label is not None]+ if labels and len(labels) != len(self.levels):+ msg = "On a rating scale every level carries a label or none does, and this one mixes the two"+ raise ValueError(msg)+ duplicated_labels = sorted({label for label in labels if labels.count(label) > 1})+ if duplicated_labels:+ msg = f"The labels of a rating scale are distinct, and this one repeats {', '.join(repr(label) for label in duplicated_labels)}"+ raise ValueError(msg)+ return selfJudgmentQuestion: TypeAlias = Annotated[@@ -114,3 +176,19 @@ JudgmentAnswer: TypeAlias = Annotated[YesNoAnswer | ChoiceAnswer | RatingAnswer,Field(discriminator="kind"),]+++class JudgmentRefusal(BaseModel):+ """The model declined to answer a question. It carries no score, because the one backend that refuses discloses none."""++ model_config = ConfigDict(extra="forbid")++ # A discriminator of its own beside the three kinds an answer carries: a refusal is no kind of answer.+ kind: Literal["refusal"] = "refusal"+++# What a worker returns for each question: an answer of the question's kind, or a refusal.+JudgmentOutcome: TypeAlias = Annotated[+ YesNoAnswer | ChoiceAnswer | RatingAnswer | JudgmentRefusal,+ Field(discriminator="kind"),+]diff --git a/pipelex/cogt/judgment/judgment_worker_abstract.py b/pipelex/cogt/judgment/judgment_worker_abstract.pyindex 29e4dff18..4b6543996 100644--- a/pipelex/cogt/judgment/judgment_worker_abstract.py+++ b/pipelex/cogt/judgment/judgment_worker_abstract.py@@ -13,7 +13,9 @@ from pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,ChoiceQuestion,JudgmentAnswer,+ JudgmentOutcome,JudgmentQuestion,+ JudgmentRefusal,RatingAnswer,RatingQuestion,YesNoAnswer,@@ -50,8 +52,12 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):async def judge(self,judgment_job: JudgmentJob,- ) -> dict[str, JudgmentAnswer]:- """Answer every question in the job over its state, keyed as the questions were."""+ ) -> dict[str, JudgmentOutcome]:+ """Answer every question in the job over its prompt, keyed as the questions were.++ A question the model refused comes back as a `JudgmentRefusal`, never as an exception: what a+ refusal means is the operator's policy, decided by whoever called this.+ """# The call ends with its summary event whichever way it ends, a refusal by the checks includedwith self._call_summary(judgment_job=judgment_job):judgment_job.validate_before_execution()@@ -60,8 +66,8 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):try:self._check_can_read_files(judgment_job=judgment_job)- answers = await self._judge(judgment_job=judgment_job)- _check_answers_match_questions(judgment_job=judgment_job, answers=answers)+ outcomes = await self._judge(judgment_job=judgment_job)+ _check_outcomes_match_questions(judgment_job=judgment_job, outcomes=outcomes)except CogtError as exc:exc.fill_model_and_provider(model_handle=self.inference_model.name, backend_name=self.inference_model.backend_name)raise@@ -77,16 +83,16 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):if self.reporting_delegate:self.reporting_delegate.report_inference_job(inference_job=judgment_job)- return answers+ return outcomesdef _check_can_read_files(self, *, judgment_job: JudgmentJob) -> None:- """Refuse a job carrying files the model does not read, before the backend is called.+ """Refuse a prompt presenting files the model does not read, before the backend is called.The model's spec states what it reads in `inputs`, in the LLM vocabulary: `images` for vision,a document format key such as `pdf` for documents. A model that reads text alone refuses anyfile here, which is how a text-only backend is protected without a line of its own."""- prompt_images = [image for images in judgment_job.images.values() for image in images]+ prompt_images = judgment_job.prompt.imagesif prompt_images:if not self.inference_model.is_vision_supported:msg = f"Judgment model '{self.inference_model.name}' does not read images, and the judgment was given {len(prompt_images)}."@@ -100,7 +106,7 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):raise JudgmentCapabilityError(msg)check_prompt_images_are_images(model_name=self.inference_model.name, prompt_images=prompt_images)- prompt_documents = [document for documents in judgment_job.documents.values() for document in documents]+ prompt_documents = judgment_job.prompt.documentsif prompt_documents:if not self.inference_model.is_document_supported:msg = f"Judgment model '{self.inference_model.name}' does not read documents, and the judgment was given {len(prompt_documents)}."@@ -115,29 +121,33 @@ class JudgmentWorkerAbstract(InferenceWorkerAbstract):async def _judge(self,judgment_job: JudgmentJob,- ) -> dict[str, JudgmentAnswer]:+ ) -> dict[str, JudgmentOutcome]:pass-def _check_answers_match_questions(*, judgment_job: JudgmentJob, answers: dict[str, JudgmentAnswer]) -> None:- """Refuse an answer set that does not correspond, question for question, to what was asked.+def _check_outcomes_match_questions(*, judgment_job: JudgmentJob, outcomes: dict[str, JudgmentOutcome]) -> None:+ """Refuse an outcome set that does not correspond, question for question, to what was asked.Three ways a backend can get this wrong, and all are silent without this check: answering aquestion nobody asked (or dropping one), answering the right question in the wrong shape — achoice where a rating was asked for — and naming a verdict the question never offered, such asan option it does not list or a level beyond its scale. The caller reads the answers by key, bykind and by the option or level they name, so any of them would surface much later, as a- missing key, a verdict of the wrong type or a verdict routed nowhere.+ missing key, a verdict of the wrong type or a verdict routed nowhere. A refusal is accepted for+ any question asked, and only under a key that was asked."""asked = set(judgment_job.questions)- answered = set(answers)+ answered = set(outcomes)if asked != answered:missing = sorted(asked - answered)unasked = sorted(answered - asked)msg = f"Judgment worker answered the wrong set of questions: missing {missing}, unasked {unasked}"raise JudgmentAnswerMismatchError(msg)for question_key, question in judgment_job.questions.items():- _check_answer_fits_question(question_key=question_key, question=question, answer=answers[question_key])+ outcome = outcomes[question_key]+ if isinstance(outcome, JudgmentRefusal):+ continue+ _check_answer_fits_question(question_key=question_key, question=question, answer=outcome)def _check_answer_fits_question(*, question_key: str, question: JudgmentQuestion, answer: JudgmentAnswer) -> None:diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.pyindex 5e31ddd8c..f7b86a4c1 100644--- a/pipelex/core/memory/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -295,9 +295,16 @@ class WorkingMemory(WorkingMemoryAbstract[Stuff], ContextProviderAbstract):for name, stuff in self.root.items():artefact_dict[name] = stuff.make_artefact()for alias, target in self.aliases.items():- artefact_dict[alias] = artefact_dict[target]+ # An alias outlives its target's resolution as an absence (`record_resolved_absence`), and then names nothing.+ if target in artefact_dict:+ artefact_dict[alias] = artefact_dict[target]return artefact_dict+ @override+ def is_variable_present(self, *, name: str) -> bool:+ # Read as `get_optional_stuff` reads: an alias whose target was later resolved absent outlives it and holds nothing.+ return self.get_optional_stuff(name=name) is not None+@overridedef get_typed_object_or_attribute(self, name: str, *, wanted_type: type[Any] | None = None, accept_list: bool = False) -> Any:"""Retrieve a typed object or nested attribute from working memory.diff --git a/pipelex/kernel/img_gen_prompt.py b/pipelex/kernel/img_gen_prompt.pyindex 1f6347e10..c653190d4 100644--- a/pipelex/kernel/img_gen_prompt.py+++ b/pipelex/kernel/img_gen_prompt.py@@ -8,35 +8,25 @@ the sole builder lived in `pipe_operators/` — an interpreter-layer module theresolves them; this is where that becomes true.What lives here is the part a caller must not have to re-derive: the image registry, the `[Image N]`-placeholder tokens, and the correspondence between them. The registry is the single source of truth-for ordering — tokens are numbered from registry indices and `input_images` is read back from the same-registry — because a token/`input_images` order mismatch silently mislabels which image the prompt is-talking about, and nothing downstream can detect it.+placeholder tokens, and the correspondence between them, through the shared user-prompt assembly+(`pipelex.kernel.prompt_assembly`) that PipeLLM and PipeJudge use too. The registry is the single+source of truth for ordering, because a token/`input_images` order mismatch silently mislabels which+image the prompt is talking about, and nothing downstream can detect it.What deliberately stays with the caller is `max_prompt_images`. That limit is a property of the modela caller has chosen, its breach raises an interpreter-layer error, and checking `len(input_images)` isa line of code — not the kind of subtlety this module exists to centralise."""-from typing import TYPE_CHECKING, Any, cast+from typing import Any-from pipelex.cogt.image.prompt_image_factory import PromptImageFactoryfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.templating.template_blueprint import TemplateBlueprint-from pipelex.cogt.templating.template_rendering import render_template-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.kernel.exceptions import PromptContentError-from pipelex.kernel.prompt_references import ImageReference, ImageReferenceKind-from pipelex.tools.jinja2.image_registry import ImageRegistry-from pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey+from pipelex.kernel.prompt_assembly import PromptFiles+from pipelex.kernel.prompt_references import ImageReferencefrom pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract-from pipelex.tools.misc.dict_utils import substitute_nested_in_context-from pipelex.tools.misc.exceptions import ContextProviderErrorfrom pipelex.tools.templating.templating_style import TemplatingStyle-if TYPE_CHECKING:- from pipelex.cogt.image.prompt_image import PromptImage-async def assemble_img_gen_prompt(*,@@ -52,207 +42,38 @@ async def assemble_img_gen_prompt(`context_provider` is typically the `WorkingMemory` the step runs against. `templating_style` isrequired for the reason `assemble_llm_prompt` states: an image prompt is a prompt, so it rendersunder a style the caller resolved rather than under whatever a filter would have defaulted to.+ The positive and negative prompts share one numbering, the positive rendering first.Raises `PromptContentError` when an image reference names something the context does not hold, orholds as the wrong type."""- image_registry = ImageRegistry()- # Maps image variable name to its 0-based registry index (for placeholder generation)- image_registry_indices: dict[str, int] = {}- # Track which variable paths are lists, so we can substitute the whole list- list_image_refs: list[ImageReference] = []-- if image_references:- for image_ref in image_references:- match image_ref.kind:- case ImageReferenceKind.DIRECT:- _extract_direct_image(- image_ref=image_ref,- context_provider=context_provider,- image_registry=image_registry,- image_registry_indices=image_registry_indices,- )- case ImageReferenceKind.DIRECT_LIST:- _extract_direct_list_images(- image_ref=image_ref,- context_provider=context_provider,- image_registry=image_registry,- image_registry_indices=image_registry_indices,- )- list_image_refs.append(image_ref)- case ImageReferenceKind.NESTED:- # Nested images will be extracted by the | with_images filter- # during template rendering - the registry is passed in context- pass-- extra_params = _with_image_placeholders(- extra_params=extra_params,- image_registry_indices=image_registry_indices,- list_image_refs=list_image_refs,+ prompt_files = PromptFiles.make_from_references(+ context_provider=context_provider,+ image_references=image_references or [],+ document_references=[],)+ template_params = prompt_files.template_params(extra_params=extra_params)positive_text: str = ""if prompt_blueprint:- positive_text = await _render_text(+ positive_text = await prompt_files.render(+ template_blueprint=prompt_blueprint,context_provider=context_provider,+ extra_params=template_params,templating_style=templating_style,- template_blueprint=prompt_blueprint,- extra_params=extra_params,- image_registry=image_registry,)negative_text: str | None = Noneif negative_prompt_blueprint:- negative_text = await _render_text(+ negative_text = await prompt_files.render(+ template_blueprint=negative_prompt_blueprint,context_provider=context_provider,+ extra_params=template_params,templating_style=templating_style,- template_blueprint=negative_prompt_blueprint,- extra_params=extra_params,- image_registry=image_registry,)- # The registry contains all images (direct + nested) in the correct order, already deduplicated- # by URL. This ensures [Image N] tokens match positions.- input_images: list[PromptImage] | None = None- if image_registry.images:- input_images = [PromptImageFactory.make_prompt_image(uri=registry_image.url) for registry_image in image_registry.images]-return ImgGenPrompt(positive_text=positive_text,negative_text=negative_text,- input_images=input_images,- )---def _with_image_placeholders(- *,- extra_params: dict[str, Any] | None,- image_registry_indices: dict[str, int],- list_image_refs: list[ImageReference],-) -> dict[str, Any]:- """Add the `[Image N]` tokens the templates substitute, numbered from the registry."""- params = dict(extra_params) if extra_params else {}- if not image_registry_indices:- return params-- # Collect list variable paths for exclusion from direct substitution- list_variable_paths = {list_ref.variable_path for list_ref in list_image_refs}-- for image_name, registry_index in image_registry_indices.items():- # Skip list item references (e.g., "images[1]")- if "[" in image_name:- continue- # Skip dotted paths (e.g., "page.page_view") - handled by finalize/filter at render time- if "." in image_name:- continue- # Skip list variable references (handled separately below)- if image_name in list_variable_paths:- continue- params[image_name] = f"[Image {registry_index + 1}]"-- # For list image references, substitute the list variable itself with a string containing all- # the [Image N] tokens for items in that list- for list_ref in list_image_refs:- list_tokens: list[str] = []- for image_name, registry_index in image_registry_indices.items():- # Check if this image belongs to this list (e.g., "collection_a[1]" belongs to "collection_a")- if image_name.startswith(f"{list_ref.variable_path}["):- list_tokens.append(f"[Image {registry_index + 1}]")- if list_tokens:- params[list_ref.variable_path] = ", ".join(list_tokens)-- return params---def _extract_direct_image(- *,- image_ref: ImageReference,- context_provider: ContextProviderAbstract,- image_registry: ImageRegistry,- image_registry_indices: dict[str, int],-) -> None:- """Extract a single ImageContent from context and register it."""- try:- prompt_image_content = context_provider.get_typed_object_or_attribute(- name=image_ref.variable_path,- wanted_type=ImageContent,- accept_list=False,- )- if isinstance(prompt_image_content, ImageContent):- registry_index = image_registry.register_image(prompt_image_content)- image_registry_indices[image_ref.variable_path] = registry_index- else:- msg = f"Image reference '{image_ref.variable_path}' is of type '{type(prompt_image_content).__name__}', expected ImageContent"- raise PromptContentError(msg)- except ContextProviderError as exc:- msg = f"Could not find image '{image_ref.variable_path}' in context: {exc}"- raise PromptContentError(msg) from exc---def _extract_direct_list_images(- *,- image_ref: ImageReference,- context_provider: ContextProviderAbstract,- image_registry: ImageRegistry,- image_registry_indices: dict[str, int],-) -> None:- """Extract a list of ImageContent from context and register them."""- try:- prompt_image_content = context_provider.get_typed_object_or_attribute(- name=image_ref.variable_path,- wanted_type=ImageContent,- accept_list=True,- )- if isinstance(prompt_image_content, (list, tuple)):- items = cast("list[Any] | tuple[Any, ...]", prompt_image_content)- for list_position, image_item in enumerate(items, start=1):- if not isinstance(image_item, ImageContent):- msg = f"Item of '{image_ref.variable_path}' is of type '{type(image_item).__name__}', expected ImageContent"- raise PromptContentError(msg)- registry_index = image_registry.register_image(image_item)- # Use list position (1-based) for variable name, registry index for image number- image_item_name = f"{image_ref.variable_path}[{list_position}]"- image_registry_indices[image_item_name] = registry_index- else:- msg = (- f"Image list reference '{image_ref.variable_path}' is of type '{type(prompt_image_content).__name__}', "- "expected list or tuple of ImageContent"- )- raise PromptContentError(msg)- except ContextProviderError as exc:- msg = f"Could not find image list '{image_ref.variable_path}' in context: {exc}"- raise PromptContentError(msg) from exc---async def _render_text(- *,- context_provider: ContextProviderAbstract,- template_blueprint: TemplateBlueprint,- templating_style: TemplatingStyle,- extra_params: dict[str, Any] | None = None,- image_registry: ImageRegistry | None = None,-) -> str:- """Render a template with context and optional image registry."""- # A style declared on the blueprint wins over the resolved one — the same precedence- # `_unravel_text` applies on the LLM path, and kept as a local for the same reason.- effective_style = template_blueprint.templating_style or templating_style-- context: dict[str, Any] = context_provider.generate_context()- if extra_params:- context = substitute_nested_in_context(context=context, extra_params=extra_params)- if template_blueprint.extra_context:- context.update(**template_blueprint.extra_context)-- # Add image registry to context for | with_images and | format filters- if image_registry is not None:- context[Jinja2ContextKey.IMAGE_REGISTRY] = image_registry-- finalize = image_registry.make_finalize() if image_registry and image_registry.images else None-- return await render_template(- template=template_blueprint.template,- category=template_blueprint.category,- context=context,- templating_style=effective_style,- finalize=finalize,+ input_images=prompt_files.prompt_images() or None,)diff --git a/pipelex/kernel/judgment_ops.py b/pipelex/kernel/judgment_ops.pyindex 4b01f5618..46baec716 100644--- a/pipelex/kernel/judgment_ops.py+++ b/pipelex/kernel/judgment_ops.py@@ -1,32 +1,29 @@-"""Judgment operator semantics: deck resolution, question rendering, the material, the verdict, memory write-back.+"""Judgment operator semantics: deck resolution, the evidence prompt, question rendering, the verdict, memory write-back.These are the functions the interpreter's `PipeJudge` calls, and the ones a programmatic callerinvokes on a `RuntimeBoot`-only process. They read the runtime hub for the services a runtime bootstands up (the model deck, the content generator) and take everything else as an explicit argument.-A judgment is asked over **material**: the step's inputs by name. Every input that is not a file is a-member of a JSON state, its value read off the content's class; every image or document travels beside-the state, keyed by the input's name, as the prompt images and documents an LLM prompt carries. The-dispatch is on the content's class rather than on a concept, because the kernel may not import the-concept library.+A judgment is asked over **evidence**: a prompt template, rendered against memory and assembled with+the images and documents it references exactly as a PipeLLM's user prompt is, through the shared+user-prompt assembly (`pipelex.kernel.prompt_assembly`). What the evidence holds is the template's+business, so an input reaches the judging model only when the prompt reads it. An optional input the run+was not given renders as the template's guard says, and the assembly skips its files, numbering and+handing over none of them."""-from typing import Any-from pipelex import logfrom pipelex.cogt.content_generation.assignment_models import JudgmentAssignmentfrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParams-from pipelex.cogt.document.prompt_document import PromptDocument-from pipelex.cogt.document.prompt_document_factory import PromptDocumentFactory-from pipelex.cogt.exceptions import JudgmentAnswerMismatchError, JudgmentModelMissingError, ModelNotFoundError-from pipelex.cogt.image.prompt_image import PromptImage-from pipelex.cogt.image.prompt_image_factory import PromptImageFactory+from pipelex.cogt.exceptions import JudgmentAnswerMismatchError, JudgmentModelMissingError, JudgmentRefusedError, ModelNotFoundErrorfrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,JudgmentAnswer,+ JudgmentPrompt,JudgmentQuestion,- JudgmentState,+ JudgmentRefusal,RatingAnswer,+ RatingQuestion,YesNoAnswer,)from pipelex.cogt.judgment.judgment_setting import JudgmentModelChoice, JudgmentSetting@@ -37,19 +34,12 @@ from pipelex.cogt.templating.template_rendering import render_templatefrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.stuffs.choice_content import ChoiceContent-from pipelex.core.stuffs.date_content import DateContent-from pipelex.core.stuffs.document_content import DocumentContent-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.core.stuffs.json_content import JSONContent-from pipelex.core.stuffs.list_content import ListContent-from pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.rating_content import RatingContentfrom pipelex.core.stuffs.stuff_content import StuffContent-from pipelex.core.stuffs.text_content import TextContent-from pipelex.core.stuffs.time_content import TimeContentfrom pipelex.core.stuffs.yes_no_content import YesNoContentfrom pipelex.kernel.judgment_results import JudgmentResultfrom pipelex.kernel.memory_ops import store_result+from pipelex.kernel.prompt_assembly import UserPromptContent, assemble_user_promptfrom pipelex.runtime_hub import get_content_generator, get_model_deckfrom pipelex.system.job_metadata import JobMetadatafrom pipelex.tools.jinja2.template_category import TemplateCategory@@ -115,8 +105,8 @@ def resolve_judgment_setting(async def run_judgment(*,memory: WorkingMemory,+ prompt_content: UserPromptContent,question: JudgmentQuestion,- input_names: list[str],judgment_setting: JudgmentSetting,concept: Concept,job_metadata: JobMetadata,@@ -126,18 +116,25 @@ async def run_judgment(result_name: str | None = None,result_code: str | None = None,) -> JudgmentResult:- """A whole judgment step: render the question, gather the material, judge, read the verdict, store.+ """A whole judgment step: assemble the evidence, render the question, judge, read the verdict, store.++ The evidence is assembled as a PipeLLM's user prompt is, its images and documents numbered into+ `[Image N]` and `[Document N]` tokens. The question arrives as the cogt question with its+ instructions still a template, rendered here against memory as a `BASIC` template. The judgment+ itself goes through the same content-generation seam as every other leaf — direct inline, an+ activity when in-workflow, or a dry mock — so it is replay-safe under a distributed orchestrator.- The question arrives as the cogt question with its instructions still a template, rendered here- against memory as a search renders its query. The judgment itself goes through the same- content-generation seam as every other leaf — direct inline, an activity when in-workflow, or a dry- mock — so it is replay-safe under a distributed orchestrator.+ A refusal is the worker's outcome, and this step's policy for it is to fail: one question has+ nowhere to leave its verdict absent, so it raises `JudgmentRefusedError`, a content error naming+ the step and the model.The content stored is the verdict native's own class, never the output concept's: a conceptrefining `Choice` has no structure of its own, so its content is a `ChoiceContent`, stored under therefining concept as every operator stores a refinement."""with job_metadata.log_context():+ assembled_prompt = await assemble_user_prompt(prompt_content=prompt_content, context_provider=memory, templating_style=templating_style)+ judgment_prompt = JudgmentPrompt(text=assembled_prompt.text, images=assembled_prompt.images, documents=assembled_prompt.documents)rendered_question = await render_template(template=question.instructions,category=TemplateCategory.BASIC,@@ -145,19 +142,20 @@ async def run_judgment(templating_style=templating_style,)asked_question = question.model_copy(update={"instructions": rendered_question})- state, images, documents = build_judgment_material(memory=memory, input_names=input_names)judgment_assignment = JudgmentAssignment(job_metadata=job_metadata,cogt_run_params=cogt_run_params,- state=state,- images=images,- documents=documents,+ prompt=judgment_prompt,questions={JUDGMENT_QUESTION_KEY: asked_question},judgment_setting=judgment_setting,)- answers = await get_content_generator().make_judgment_answers(judgment_assignment=judgment_assignment)- answer = answers[JUDGMENT_QUESTION_KEY]- content, threshold_applied = make_verdict_content(answer=answer, threshold=threshold, is_dry=cogt_run_params.run_mode.is_dry)+ outcomes = await get_content_generator().make_judgment_answers(judgment_assignment=judgment_assignment)+ outcome = outcomes[JUDGMENT_QUESTION_KEY]+ if isinstance(outcome, JudgmentRefusal):+ raise JudgmentRefusedError(pipe_code=job_metadata.pipe_code, model_handle=judgment_setting.model)+ content, threshold_applied = make_verdict_content(+ question=asked_question, answer=outcome, threshold=threshold, is_dry=cogt_run_params.run_mode.is_dry+ )if threshold_applied is False:log.warning("A judgment's model reported no probability, and its own verdict stands",@@ -166,103 +164,25 @@ async def run_judgment(return JudgmentResult(memory=store_result(memory=memory, concept=concept, content=content, result_name=result_name, result_code=result_code),content=content,+ prompt=judgment_prompt,rendered_question=rendered_question,judgment_setting=judgment_setting,- answer=answer,+ answer=outcome,threshold_applied=threshold_applied,)-def build_judgment_material(- *,- memory: WorkingMemory,- input_names: list[str],-) -> tuple[JudgmentState, dict[str, list[PromptImage]], dict[str, list[PromptDocument]]]:- """The state and the files a judgment is asked over, one entry per input, keyed by its declared name.-- An input name is a plain name, so each entry is one whole value: the step's material holds exactly- what it declares, and a field reaches it only when the calling sequence binds it under a name of its- own. An optional input that holds no value, whether its absence was recorded or it was never written, is- left out rather than sent as a null: a required input with no value never reaches the step, whose- presence scan refuses it. An image or a document, or a list of either, goes to the file channel; every- other input is a member of the state. An image nested inside a structured input is part of that input's- JSON, its URL as text, which is the author's concern.- """- state: JudgmentState = {}- images: dict[str, list[PromptImage]] = {}- documents: dict[str, list[PromptDocument]] = {}- for input_name in input_names:- stuff = memory.get_optional_stuff(name=input_name)- if stuff is None:- continue- content: StuffContent = stuff.content- if prompt_images := _prompt_images(content=content):- images[input_name] = prompt_images- elif prompt_documents := _prompt_documents(content=content):- documents[input_name] = prompt_documents- else:- state[input_name] = material_value(content=content)- return state, images, documents---def material_value(*, content: StuffContent) -> Any:- """The JSON value a content is judged as: the reading a caller would write for it.-- Text (and every refinement of it) is its string, a number its number, a yes/no its boolean, a date- or a time its ISO string, a JSON object itself, a list the array of its items' values, and every- other content — a choice, a rating, any structure — its JSON object without its absent members.- """- match content:- case TextContent():- return content.text- case NumberContent():- return content.number- case YesNoContent():- return content.yes_no- case DateContent() | TimeContent():- return content.rendered_plain()- case JSONContent():- return content.json_obj- case ListContent():- return [material_value(content=item) for item in content.items]- case _:- # Serialized as what each field holds rather than as its declared type, so a field typed as the- # content base keeps the fields of the subclass a run put there.- return content.model_dump(mode="json", exclude_none=True, serialize_as_any=True)---def _prompt_images(*, content: StuffContent) -> list[PromptImage] | None:- image_contents: list[ImageContent]- match content:- case ImageContent():- image_contents = [content]- case ListContent() if content.items and all(isinstance(item, ImageContent) for item in content.items):- image_contents = [item for item in content.items if isinstance(item, ImageContent)]- case _:- return None- return [PromptImageFactory.make_prompt_image(uri=image.url, mime_type=image.mime_type) for image in image_contents]---def _prompt_documents(*, content: StuffContent) -> list[PromptDocument] | None:- document_contents: list[DocumentContent]- match content:- case DocumentContent():- document_contents = [content]- case ListContent() if content.items and all(isinstance(item, DocumentContent) for item in content.items):- document_contents = [item for item in content.items if isinstance(item, DocumentContent)]- case _:- return None- return [PromptDocumentFactory.make_prompt_document(uri=document.url, mime_type=document.mime_type) for document in document_contents]---def make_verdict_content(*, answer: JudgmentAnswer, threshold: float | None, is_dry: bool) -> tuple[StuffContent, bool | None]:+def make_verdict_content(+ *, question: JudgmentQuestion, answer: JudgmentAnswer, threshold: float | None, is_dry: bool+) -> tuple[StuffContent, bool | None]:"""The verdict native an answer translates to, and what became of a declared threshold.A yes/no answer with a probability is decided by it against the threshold, inclusive, the defaultstanding in when none is declared; one without a probability keeps its own verdict, and a declaredthreshold then had nothing to work on, which a live run reports and a dry run, whose answers carry- no probability by construction, does not. A rating's distribution is keyed by the level index- written as text, the one place that conversion happens. Nothing absent is synthesised.+ no probability by construction, does not. A rating takes the label of the level it names off the+ scale the question declared, since no backend reports one, and its distribution is keyed by the+ level index written as text, the one place that conversion happens. Nothing absent is synthesised."""match answer:case YesNoAnswer():@@ -283,4 +203,25 @@ def make_verdict_content(*, answer: JudgmentAnswer, threshold: float | None, is_probabilities = ({str(level): probability for level, probability in answer.probabilities.items()} if answer.probabilities is not None else None)- return RatingContent(level=answer.level, confidence=answer.confidence, probabilities=probabilities, position=answer.position), None+ return RatingContent(+ level=answer.level,+ label=_declared_label(question=question, level=answer.level),+ confidence=answer.confidence,+ probabilities=probabilities,+ position=answer.position,+ ), None+++def _declared_label(*, question: JudgmentQuestion, level: int) -> str | None:+ """The label the question's scale declares for a level, `None` when the scale declares none.++ The worker checked that the answer's kind is its question's and that its level is on the scale,+ so the two refusals here are stated for the type checker, which cannot see that check.+ """+ if not isinstance(question, RatingQuestion):+ msg = f"A rating answer came back for a {question.kind} question"+ raise JudgmentAnswerMismatchError(msg)+ if not 0 <= level < len(question.levels):+ msg = f"A rating answer names level {level}, which a scale of {len(question.levels)} levels does not hold"+ raise JudgmentAnswerMismatchError(msg)+ return question.levels[level].labeldiff --git a/pipelex/kernel/judgment_results.py b/pipelex/kernel/judgment_results.pyindex 40ba1258c..ad7598490 100644--- a/pipelex/kernel/judgment_results.py+++ b/pipelex/kernel/judgment_results.py@@ -1,10 +1,10 @@"""What a kernel judgment call hands back.-Three intermediates ride along because the kernel is where they are produced and the interpreter's-execution-graph tracer records them: the rendered question, the `JudgmentSetting` *after* handle-resolution, and the judging model's raw answer, which carries a distribution the verdict native's-rendering does not show. The memory contract holds as everywhere else: **the returned `memory` is the-result**.+Four intermediates ride along because the kernel is where they are produced and the interpreter's+execution-graph tracer records them: the assembled evidence prompt, the rendered question, the+`JudgmentSetting` *after* handle resolution, and the judging model's raw answer, which carries a+distribution the verdict native's rendering does not show. The memory contract holds as everywhere+else: **the returned `memory` is the result**.The serialization note on `pipelex.kernel.llm_results` applies here too — `content` is annotated withthe base `StuffContent`, so use `kajson` rather than a plain `model_dump()`.@@ -12,7 +12,7 @@ the base `StuffContent`, so use `kajson` rather than a plain `model_dump()`.from pydantic import BaseModel, ConfigDict-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer+from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, JudgmentPromptfrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.stuffs.stuff_content import StuffContent@@ -22,7 +22,8 @@ class JudgmentResult(BaseModel):"""The outcome of a kernel judgment call.`content` is the verdict native the answer translates to: a `YesNoContent`, a `ChoiceContent` or- a `RatingContent`. `threshold_applied` says what became of a declared threshold: `True` when it+ a `RatingContent`. `prompt` is the evidence as the model was given it: the rendered text and the+ files its tokens number. `threshold_applied` says what became of a declared threshold: `True` when itdecided a yes/no verdict from the reported probability, `False` when the judging model reported noprobability on a live run and its own verdict stood, and `None` when no threshold was declared orthe run was dry.@@ -32,6 +33,7 @@ class JudgmentResult(BaseModel):memory: WorkingMemorycontent: StuffContent+ prompt: JudgmentPromptrendered_question: strjudgment_setting: JudgmentSettinganswer: JudgmentAnswerdiff --git a/pipelex/kernel/llm_prompt_content.py b/pipelex/kernel/llm_prompt_content.pyindex eabf0ee76..f996cead6 100644--- a/pipelex/kernel/llm_prompt_content.py+++ b/pipelex/kernel/llm_prompt_content.py@@ -6,36 +6,25 @@ keeps its parse-and-validate role — but it now maps down onto :func:`assemble_holding the semantics, so the interpreter and a programmatic caller assemble prompts through thesame code.-The ordering here is load-bearing and is preserved exactly: system-prompt images are registered-before user-prompt ones, and the system text renders before the user text, so the `[Image N]` tokens-a template interpolates match the positions of the images handed to the model.+The numbering of the images and documents a prompt references is the shared user-prompt assembly's+(`pipelex.kernel.prompt_assembly`), which PipeImgGen and PipeJudge use too. The ordering is+load-bearing: system-prompt references are registered before user-prompt ones, and the system text+renders before the user text, so the `[Image N]` tokens a template interpolates match the positions+of the images handed to the model."""-from typing import TYPE_CHECKING, Any, Self, cast+from typing import Any, Selffrom pydantic import BaseModel-from pipelex.cogt.document.prompt_document import PromptDocument-from pipelex.cogt.document.prompt_document_factory import PromptDocumentFactory-from pipelex.cogt.image.prompt_image_factory import PromptImageFactoryfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.templating.template_blueprint import TemplateBlueprint-from pipelex.cogt.templating.template_rendering import render_template-from pipelex.core.stuffs.document_content import DocumentContent-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.kernel.exceptions import PromptContentError-from pipelex.kernel.prompt_references import DocumentReference, DocumentReferenceKind, ImageReference, ImageReferenceKind-from pipelex.tools.jinja2.image_registry import ImageRegistry-from pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey+from pipelex.kernel.prompt_assembly import PromptFiles+from pipelex.kernel.prompt_references import DocumentReference, ImageReferencefrom pipelex.tools.jinja2.template_category import TemplateCategoryfrom pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract-from pipelex.tools.misc.dict_utils import substitute_nested_in_context-from pipelex.tools.misc.exceptions import ContextProviderErrorfrom pipelex.tools.templating.templating_style import TemplatingStyle-if TYPE_CHECKING:- from pipelex.cogt.image.prompt_image import PromptImage-class LlmPromptContent(BaseModel):"""What a prompt is made of, before anything is rendered: two templates and their references.@@ -73,334 +62,45 @@ async def assemble_llm_prompt(extra_params: dict[str, Any] | None = None,templating_style: TemplatingStyle,) -> LLMPrompt:- """Render both templates against the context and collect the images and documents they reference."""- ############################################################- # Image Registry and Direct Image Extraction- # Extract system prompt images FIRST, then user prompt images- ############################################################- image_registry = ImageRegistry()- # Maps image variable name to its 0-based registry index (for placeholder generation)- image_registry_indices: dict[str, int] = {}- # Track which variable paths are lists, so we can substitute the whole list- list_image_refs: list[ImageReference] = []-- # Process system prompt image references first (so they get lower numbers), then the user ones.- for image_references in (prompt_content.system_image_references, prompt_content.user_image_references):- if not image_references:- continue- for image_ref in image_references:- match image_ref.kind:- case ImageReferenceKind.DIRECT:- # Single ImageContent reference- _extract_direct_image(- image_ref=image_ref,- context_provider=context_provider,- image_registry=image_registry,- image_registry_indices=image_registry_indices,- )- case ImageReferenceKind.DIRECT_LIST:- # List of ImageContent reference- _extract_direct_list_images(- image_ref=image_ref,- context_provider=context_provider,- image_registry=image_registry,- image_registry_indices=image_registry_indices,- )- list_image_refs.append(image_ref)- case ImageReferenceKind.NESTED:- # Nested images will be extracted by the | with_images filter- # during template rendering - the registry is passed in context- pass-- ############################################################- # Direct Document Extraction- # Extract system prompt documents FIRST, then user prompt documents- ############################################################- prompt_user_documents: dict[str, PromptDocument] = {}- list_document_refs: list[DocumentReference] = []-- for document_references in (prompt_content.system_document_references, prompt_content.user_document_references):- if not document_references:- continue- for doc_ref in document_references:- match doc_ref.kind:- case DocumentReferenceKind.DIRECT:- _extract_direct_document(- doc_ref=doc_ref,- context_provider=context_provider,- prompt_user_documents=prompt_user_documents,- )- case DocumentReferenceKind.DIRECT_LIST:- _extract_direct_list_documents(- doc_ref=doc_ref,- context_provider=context_provider,- prompt_user_documents=prompt_user_documents,- )- list_document_refs.append(doc_ref)-- ############################################################- # User text- ############################################################- # Add image placeholders to extra_params for substitution in template- # - Direct images (non-dotted paths like "image"): add placeholder directly- # - List images: add list variable with all tokens joined- # - Dotted paths (e.g., "page.page_view"): handled by tag filter via ImageRegistry.get_image_placeholder()- # because dotted paths cannot be substituted in immutable StuffArtefacts- extra_params = extra_params or {}- if image_registry_indices:- # Collect list variable paths for exclusion from direct substitution- list_variable_paths = {list_ref.variable_path for list_ref in list_image_refs}-- # Add placeholders for direct (non-dotted, non-list) images- for image_name, registry_index in image_registry_indices.items():- # Skip list item references (e.g., "images[1]")- if "[" in image_name:- continue- # Skip dotted paths (e.g., "page.page_view") - handled by finalize/filter at render time- if "." in image_name:- continue- # Skip list variable references (handled separately below)- if image_name in list_variable_paths:- continue- # Add direct image placeholder- extra_params[image_name] = f"[Image {registry_index + 1}]"+ """Render both templates against the context and collect the images and documents they reference.- # For list image references, substitute the list variable itself- # with a string containing all the [Image N] tokens for items in that list- for list_ref in list_image_refs:- list_tokens: list[str] = []- for image_name, registry_index in image_registry_indices.items():- # Check if this image belongs to this list (e.g., "collection_a[1]" belongs to "collection_a")- if image_name.startswith(f"{list_ref.variable_path}["):- list_tokens.append(f"[Image {registry_index + 1}]")- if list_tokens:- extra_params[list_ref.variable_path] = ", ".join(list_tokens)-- # Replace direct document variables with numbered tags- if prompt_user_documents:- document_names = list(prompt_user_documents.keys())- for document_index, document_name in enumerate(document_names):- extra_params[document_name] = f"[Document {document_index + 1}]"-- # For list document references, also substitute the list variable itself- for doc_list_ref in list_document_refs:- doc_list_tokens: list[str] = []- for document_name in document_names:- if document_name.startswith(f"{doc_list_ref.variable_path}["):- doc_list_tokens.append(extra_params[document_name])- if doc_list_tokens:- extra_params[doc_list_ref.variable_path] = "\n".join(doc_list_tokens)+ The numbering is the shared user-prompt assembly's (`pipelex.kernel.prompt_assembly`): the system+ prompt's references are registered before the user prompt's, and the system text renders first, so+ its files take the lower numbers in both texts.+ """+ prompt_files = PromptFiles.make_from_references(+ context_provider=context_provider,+ image_references=[*(prompt_content.system_image_references or []), *(prompt_content.user_image_references or [])],+ document_references=[*(prompt_content.system_document_references or []), *(prompt_content.user_document_references or [])],+ )+ template_params = prompt_files.template_params(extra_params=extra_params)- ############################################################- # System text (rendered FIRST so nested images get lower numbers)- ############################################################system_text: str | None = Noneif prompt_content.system_template:- system_text = await _unravel_text(+ system_text = await prompt_files.render(+ template_blueprint=prompt_content.system_template,context_provider=context_provider,- jinja2_blueprint=prompt_content.system_template,- extra_params=extra_params,- image_registry=image_registry,+ extra_params=template_params,templating_style=templating_style,)- ############################################################- # User text (rendered AFTER system text for consistent image ordering)- ############################################################- user_text: str | None = None+ user_text: str | Noneif prompt_content.user_template:- user_text = await _unravel_text(+ user_text = await prompt_files.render(+ template_blueprint=prompt_content.user_template,context_provider=context_provider,- jinja2_blueprint=prompt_content.user_template,- extra_params=extra_params,- image_registry=image_registry,+ extra_params=template_params,templating_style=templating_style,)if output_structure_prompt:user_text += output_structure_promptelse:+ # Can be None: a prompt with no user template and no structure to describe has no user text.user_text = output_structure_prompt- # Note that output_structure_prompt can be None- # it's OK to have a null user_text-- ############################################################- # Collect all images from registry (single source of truth)- ############################################################- # The registry contains all images (direct + nested) in the correct order,- # already deduplicated by URL. This ensures [Image N] tokens match positions.- # Each keeps the format its preparation established, the way a document does, so the worker checks- # the format instead of guessing it again.- all_images: list[PromptImage] = [- PromptImageFactory.make_prompt_image(uri=registry_image.url, mime_type=registry_image.mime_type) for registry_image in image_registry.images- ]- ############################################################- # Collect all documents- ############################################################- all_documents: list[PromptDocument] = list(prompt_user_documents.values())-- ############################################################- # Full LLMPrompt- ############################################################return LLMPrompt(system_text=system_text,user_text=user_text,- user_images=all_images,- user_documents=all_documents,- )---def _extract_direct_image(- *,- image_ref: ImageReference,- context_provider: ContextProviderAbstract,- image_registry: ImageRegistry,- image_registry_indices: dict[str, int],-) -> None:- """Extract a single ImageContent from context and register it."""- try:- prompt_image_content = context_provider.get_typed_object_or_attribute(- name=image_ref.variable_path,- wanted_type=ImageContent,- accept_list=False,- )- if isinstance(prompt_image_content, ImageContent):- registry_index = image_registry.register_image(prompt_image_content)- image_registry_indices[image_ref.variable_path] = registry_index- else:- msg = f"Image reference '{image_ref.variable_path}' is of type '{type(prompt_image_content).__name__}', expected ImageContent"- raise PromptContentError(msg)- except ContextProviderError as exc:- msg = f"Could not find image '{image_ref.variable_path}' in context: {exc}"- raise PromptContentError(msg) from exc---def _extract_direct_list_images(- *,- image_ref: ImageReference,- context_provider: ContextProviderAbstract,- image_registry: ImageRegistry,- image_registry_indices: dict[str, int],-) -> None:- """Extract a list of ImageContent from context and register them."""- try:- prompt_image_content = context_provider.get_typed_object_or_attribute(- name=image_ref.variable_path,- wanted_type=ImageContent,- accept_list=True,- )- if isinstance(prompt_image_content, (list, tuple)):- image_items = cast("list[Any] | tuple[Any, ...]", prompt_image_content)- for list_position, image_item in enumerate(image_items, start=1):- if not isinstance(image_item, ImageContent):- msg = f"Item of '{image_ref.variable_path}' is of type '{type(image_item).__name__}', expected ImageContent"- raise PromptContentError(msg)- registry_index = image_registry.register_image(image_item)- # Use list position (1-based) for variable name, registry index for image number- image_item_name = f"{image_ref.variable_path}[{list_position}]"- image_registry_indices[image_item_name] = registry_index- else:- msg = (- f"Image list reference '{image_ref.variable_path}' is of type '{type(prompt_image_content).__name__}', "- "expected list or tuple of ImageContent"- )- raise PromptContentError(msg)- except ContextProviderError as exc:- msg = f"Could not find image list '{image_ref.variable_path}' in context: {exc}"- raise PromptContentError(msg) from exc---def _extract_direct_document(- *,- doc_ref: DocumentReference,- context_provider: ContextProviderAbstract,- prompt_user_documents: dict[str, PromptDocument],-) -> None:- """Extract a single DocumentContent from context."""- try:- prompt_document_content = context_provider.get_typed_object_or_attribute(- name=doc_ref.variable_path,- wanted_type=DocumentContent,- accept_list=False,- )- if isinstance(prompt_document_content, DocumentContent):- user_document = PromptDocumentFactory.make_prompt_document(- uri=prompt_document_content.url,- mime_type=prompt_document_content.mime_type,- )- prompt_user_documents[doc_ref.variable_path] = user_document- else:- msg = f"Document reference '{doc_ref.variable_path}' is of type '{type(prompt_document_content).__name__}', expected DocumentContent"- raise PromptContentError(msg)- except ContextProviderError as exc:- msg = f"Could not find document '{doc_ref.variable_path}' in context: {exc}"- raise PromptContentError(msg) from exc---def _extract_direct_list_documents(- *,- doc_ref: DocumentReference,- context_provider: ContextProviderAbstract,- prompt_user_documents: dict[str, PromptDocument],-) -> None:- """Extract a list of DocumentContent from context."""- try:- prompt_document_content = context_provider.get_typed_object_or_attribute(- name=doc_ref.variable_path,- wanted_type=DocumentContent,- accept_list=True,- )- if isinstance(prompt_document_content, (list, tuple)):- document_items = cast("list[Any] | tuple[Any, ...]", prompt_document_content)- for doc_index, doc_item in enumerate(document_items, start=1):- if not isinstance(doc_item, DocumentContent):- msg = f"Item of '{doc_ref.variable_path}' is of type '{type(doc_item).__name__}', expected DocumentContent"- raise PromptContentError(msg)- user_document = PromptDocumentFactory.make_prompt_document(- uri=doc_item.url,- mime_type=doc_item.mime_type,- )- user_document_item_name = f"{doc_ref.variable_path}[{doc_index}]"- prompt_user_documents[user_document_item_name] = user_document- else:- msg = (- f"Document list reference '{doc_ref.variable_path}' is of type '{type(prompt_document_content).__name__}', "- "expected list or tuple of DocumentContent"- )- raise PromptContentError(msg)- except ContextProviderError as exc:- msg = f"Could not find document list '{doc_ref.variable_path}' in context: {exc}"- raise PromptContentError(msg) from exc---async def _unravel_text(- *,- context_provider: ContextProviderAbstract,- jinja2_blueprint: TemplateBlueprint,- extra_params: dict[str, Any] | None = None,- image_registry: ImageRegistry | None = None,- templating_style: TemplatingStyle,-) -> str:- # A style declared on the blueprint wins over the run-derived one. Kept as a local: writing- # it onto `jinja2_blueprint` would mutate an object the pipe library holds and hands out.- effective_style = jinja2_blueprint.templating_style or templating_style-- context: dict[str, Any] = context_provider.generate_context()- if extra_params:- context = substitute_nested_in_context(context=context, extra_params=extra_params)- if jinja2_blueprint.extra_context:- context.update(**jinja2_blueprint.extra_context)-- # Add image registry to context for | with_images and | format filters- if image_registry is not None:- context[Jinja2ContextKey.IMAGE_REGISTRY] = image_registry-- finalize = image_registry.make_finalize() if image_registry and image_registry.images else None-- return await render_template(- template=jinja2_blueprint.template,- category=jinja2_blueprint.category,- context=context,- templating_style=effective_style,- finalize=finalize,+ user_images=prompt_files.prompt_images(),+ user_documents=prompt_files.prompt_documents(),)diff --git a/pipelex/kernel/prompt_assembly.py b/pipelex/kernel/prompt_assembly.pynew file mode 100644index 000000000..7559725bc--- /dev/null+++ b/pipelex/kernel/prompt_assembly.py@@ -0,0 +1,329 @@+"""The user-prompt assembly every prompt-shaped operator shares: PipeLLM, PipeImgGen and PipeJudge.++A prompt template reads images and documents out of working memory through the references its+operator's factory resolved when the method loaded. Assembling the prompt means fetching each file a+reference names, numbering it, putting its numbered token where the template reads it, rendering the+template, and handing the model the files in the order their tokens number them. Three operators do+exactly this, so it is done here once: a second copy of the numbering would be free to drift from the+first, and a token that names a different file than the one handed over in its position mislabels the+prompt with nothing downstream able to notice.++**The registry is the single source of truth for the order.** Image tokens are numbered from+`ImageRegistry` indices and the image list is read back from the same registry, which also+deduplicates by URL. Documents are numbered in the order their references are registered.++**Several templates of one prompt share one `PromptFiles`**, so their tokens number one sequence: a+PipeLLM's system and user prompts, a PipeImgGen's positive and negative prompts. The ordering is+load-bearing. Every template's references are registered before any template renders, in the order+the caller lists them, and the templates render in that same order, so the nested images a+`with_images` filter registers while rendering come after every direct one.++**References come in three shapes.** A direct reference names one file (`photo`), a list reference+names a list of them (`album`, whose token is every item's token joined), and a dotted reference+reaches a file through a structure (`page.page_view`). A direct or list image is substituted as a+template parameter; a dotted image cannot be, since it lives inside an immutable artefact, so the+registry's `finalize` hook turns it into its token as Jinja2 prints it. A document is always+substituted as a parameter, a dotted path included.+"""++from collections.abc import Sequence+from typing import Any, Self, cast++from pydantic import BaseModel, ConfigDict++from pipelex.cogt.document.prompt_document import PromptDocument+from pipelex.cogt.document.prompt_document_factory import PromptDocumentFactory+from pipelex.cogt.image.prompt_image import PromptImage+from pipelex.cogt.image.prompt_image_factory import PromptImageFactory+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_rendering import render_template+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.kernel.exceptions import PromptContentError+from pipelex.kernel.prompt_references import DocumentReference, DocumentReferenceKind, ImageReference, ImageReferenceKind+from pipelex.tools.jinja2.image_registry import ImageRegistry+from pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey+from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract+from pipelex.tools.misc.dict_utils import substitute_nested_in_context+from pipelex.tools.misc.exceptions import ContextProviderError+from pipelex.tools.misc.string_utils import get_root_from_dotted_path+from pipelex.tools.templating.templating_style import TemplatingStyle+++class UserPromptContent(BaseModel):+ """One prompt template and the image and document references its operator's factory resolved for it."""++ template: TemplateBlueprint+ image_references: list[ImageReference] | None = None+ document_references: list[DocumentReference] | None = None+++class AssembledUserPrompt(BaseModel):+ """A rendered prompt and the files it presents, each list in the order its tokens number it."""++ model_config = ConfigDict(frozen=True)++ text: str+ images: list[PromptImage]+ documents: list[PromptDocument]+++async def assemble_user_prompt(+ *,+ prompt_content: UserPromptContent,+ context_provider: ContextProviderAbstract,+ templating_style: TemplatingStyle,+ extra_params: dict[str, Any] | None = None,+) -> AssembledUserPrompt:+ """Resolve one template's references, number them, render it, and collect its files in token order.++ Raises:+ PromptContentError: when a reference names something the context does not hold, or holds as+ the wrong type.+ """+ prompt_files = PromptFiles.make_from_references(+ context_provider=context_provider,+ image_references=prompt_content.image_references or [],+ document_references=prompt_content.document_references or [],+ )+ text = await prompt_files.render(+ template_blueprint=prompt_content.template,+ context_provider=context_provider,+ extra_params=prompt_files.template_params(extra_params=extra_params),+ templating_style=templating_style,+ )+ return AssembledUserPrompt(text=text, images=prompt_files.prompt_images(), documents=prompt_files.prompt_documents())+++class PromptFiles:+ """The images and documents one prompt's templates reference, numbered once for all of them.++ Built from every template's references at once, then asked for the parameters the templates+ substitute and used to render each template in turn. Read the files back only after the last+ template rendered, since rendering registers the images a `with_images` filter reaches.+ """++ def __init__(self) -> None:+ self._image_registry = ImageRegistry()+ # Each directly referenced image's 0-based registry index, keyed by its variable path, a list's+ # items keyed `album[1]`, `album[2]`, so the placeholders are numbered from the registry.+ self._image_registry_indices: dict[str, int] = {}+ self._list_image_paths: list[str] = []+ # Each referenced document keyed by its variable path, a list's items keyed `annexes[1]`, in the+ # order their references were registered, which is the order their tokens number them.+ self._documents: dict[str, PromptDocument] = {}+ self._list_document_paths: list[str] = []++ @classmethod+ def make_from_references(+ cls,+ *,+ context_provider: ContextProviderAbstract,+ image_references: Sequence[ImageReference],+ document_references: Sequence[DocumentReference],+ ) -> Self:+ """Fetch every referenced file out of the context and register it, in the order given.++ A reference whose root input is declared optional and absent is skipped before registration: it+ takes no number and hands over no file, and the template's guard (`@?photo`, `{% if photo %}`)+ renders it out. An absent required input is refused here.++ Raises:+ PromptContentError: when a reference names something the context does not hold, unless its+ root input is optional and absent, or holds it as the wrong type.+ """+ prompt_files = cls()+ for image_reference in image_references:+ if _is_absent_optional(+ variable_path=image_reference.variable_path, is_optional=image_reference.is_optional, context_provider=context_provider+ ):+ continue+ match image_reference.kind:+ case ImageReferenceKind.DIRECT:+ prompt_files._register_direct_image(image_reference=image_reference, context_provider=context_provider)+ case ImageReferenceKind.DIRECT_LIST:+ prompt_files._register_direct_list_images(image_reference=image_reference, context_provider=context_provider)+ prompt_files._list_image_paths.append(image_reference.variable_path)+ case ImageReferenceKind.NESTED:+ # Registered while rendering, by the `with_images` filter, which reads the registry+ # from the context.+ pass+ for document_reference in document_references:+ if _is_absent_optional(+ variable_path=document_reference.variable_path, is_optional=document_reference.is_optional, context_provider=context_provider+ ):+ continue+ match document_reference.kind:+ case DocumentReferenceKind.DIRECT:+ prompt_files._register_direct_document(document_reference=document_reference, context_provider=context_provider)+ case DocumentReferenceKind.DIRECT_LIST:+ prompt_files._register_direct_list_documents(document_reference=document_reference, context_provider=context_provider)+ prompt_files._list_document_paths.append(document_reference.variable_path)+ return prompt_files++ def template_params(self, *, extra_params: dict[str, Any] | None) -> dict[str, Any]:+ """The caller's parameters, plus the token each direct, list and dotted-document reference renders as.++ The caller's dict is copied, never written to. A dotted image path gets no parameter: its+ token comes from the registry's `finalize` hook as the template prints it.+ """+ params = dict(extra_params) if extra_params else {}+ for image_path, registry_index in self._image_registry_indices.items():+ # A list's items are rendered through the list's own parameter below, a dotted path through finalize.+ if "[" in image_path or "." in image_path or image_path in self._list_image_paths:+ continue+ params[image_path] = _image_token(registry_index=registry_index)+ for list_path in self._list_image_paths:+ list_tokens = [+ _image_token(registry_index=registry_index)+ for image_path, registry_index in self._image_registry_indices.items()+ if image_path.startswith(f"{list_path}[")+ ]+ if list_tokens:+ params[list_path] = ", ".join(list_tokens)++ document_paths = list(self._documents)+ for document_index, document_path in enumerate(document_paths, start=1):+ params[document_path] = f"[Document {document_index}]"+ for list_path in self._list_document_paths:+ list_tokens = [params[document_path] for document_path in document_paths if document_path.startswith(f"{list_path}[")]+ if list_tokens:+ params[list_path] = "\n".join(list_tokens)+ return params++ async def render(+ self,+ *,+ template_blueprint: TemplateBlueprint,+ context_provider: ContextProviderAbstract,+ extra_params: dict[str, Any],+ templating_style: TemplatingStyle,+ ) -> str:+ """Render one template against the context, its references substituted by `extra_params`.++ A style declared on the blueprint wins over the resolved one, and is kept as a local: writing it+ onto the blueprint would mutate an object the pipe library holds and hands out.+ """+ effective_style = template_blueprint.templating_style or templating_style++ context: dict[str, Any] = context_provider.generate_context()+ if extra_params:+ context = substitute_nested_in_context(context=context, extra_params=extra_params)+ if template_blueprint.extra_context:+ context.update(**template_blueprint.extra_context)+ # The registry rides in the context for the `with_images` and `format` filters.+ context[Jinja2ContextKey.IMAGE_REGISTRY] = self._image_registry++ finalize = self._image_registry.make_finalize() if self._image_registry.images else None++ return await render_template(+ template=template_blueprint.template,+ category=template_blueprint.category,+ context=context,+ templating_style=effective_style,+ finalize=finalize,+ )++ def prompt_images(self) -> list[PromptImage]:+ """Every registered image in registry order, each keeping the format its preparation established.++ Carrying the MIME type lets the worker check the format instead of guessing it again.+ """+ return [PromptImageFactory.make_prompt_image(uri=image.url, mime_type=image.mime_type) for image in self._image_registry.images]++ def prompt_documents(self) -> list[PromptDocument]:+ """Every referenced document in the order its token numbers it."""+ return list(self._documents.values())++ def _register_direct_image(self, *, image_reference: ImageReference, context_provider: ContextProviderAbstract) -> None:+ try:+ image_content = context_provider.get_typed_object_or_attribute(+ name=image_reference.variable_path,+ wanted_type=ImageContent,+ accept_list=False,+ )+ except ContextProviderError as exc:+ msg = f"Could not find image '{image_reference.variable_path}' in context: {exc}"+ raise PromptContentError(msg) from exc+ if not isinstance(image_content, ImageContent):+ msg = f"Image reference '{image_reference.variable_path}' is of type '{type(image_content).__name__}', expected ImageContent"+ raise PromptContentError(msg)+ self._image_registry_indices[image_reference.variable_path] = self._image_registry.register_image(image_content)++ def _register_direct_list_images(self, *, image_reference: ImageReference, context_provider: ContextProviderAbstract) -> None:+ try:+ image_items = context_provider.get_typed_object_or_attribute(+ name=image_reference.variable_path,+ wanted_type=ImageContent,+ accept_list=True,+ )+ except ContextProviderError as exc:+ msg = f"Could not find image list '{image_reference.variable_path}' in context: {exc}"+ raise PromptContentError(msg) from exc+ if not isinstance(image_items, (list, tuple)):+ msg = (+ f"Image list reference '{image_reference.variable_path}' is of type '{type(image_items).__name__}', "+ "expected list or tuple of ImageContent"+ )+ raise PromptContentError(msg)+ for list_position, image_item in enumerate(cast("list[Any] | tuple[Any, ...]", image_items), start=1):+ if not isinstance(image_item, ImageContent):+ msg = f"Item of '{image_reference.variable_path}' is of type '{type(image_item).__name__}', expected ImageContent"+ raise PromptContentError(msg)+ # Keyed by the 1-based list position, numbered by the registry index.+ self._image_registry_indices[f"{image_reference.variable_path}[{list_position}]"] = self._image_registry.register_image(image_item)++ def _register_direct_document(self, *, document_reference: DocumentReference, context_provider: ContextProviderAbstract) -> None:+ try:+ document_content = context_provider.get_typed_object_or_attribute(+ name=document_reference.variable_path,+ wanted_type=DocumentContent,+ accept_list=False,+ )+ except ContextProviderError as exc:+ msg = f"Could not find document '{document_reference.variable_path}' in context: {exc}"+ raise PromptContentError(msg) from exc+ if not isinstance(document_content, DocumentContent):+ msg = f"Document reference '{document_reference.variable_path}' is of type '{type(document_content).__name__}', expected DocumentContent"+ raise PromptContentError(msg)+ self._documents[document_reference.variable_path] = PromptDocumentFactory.make_prompt_document(+ uri=document_content.url,+ mime_type=document_content.mime_type,+ )++ def _register_direct_list_documents(self, *, document_reference: DocumentReference, context_provider: ContextProviderAbstract) -> None:+ try:+ document_items = context_provider.get_typed_object_or_attribute(+ name=document_reference.variable_path,+ wanted_type=DocumentContent,+ accept_list=True,+ )+ except ContextProviderError as exc:+ msg = f"Could not find document list '{document_reference.variable_path}' in context: {exc}"+ raise PromptContentError(msg) from exc+ if not isinstance(document_items, (list, tuple)):+ msg = (+ f"Document list reference '{document_reference.variable_path}' is of type '{type(document_items).__name__}', "+ "expected list or tuple of DocumentContent"+ )+ raise PromptContentError(msg)+ for list_position, document_item in enumerate(cast("list[Any] | tuple[Any, ...]", document_items), start=1):+ if not isinstance(document_item, DocumentContent):+ msg = f"Item of '{document_reference.variable_path}' is of type '{type(document_item).__name__}', expected DocumentContent"+ raise PromptContentError(msg)+ self._documents[f"{document_reference.variable_path}[{list_position}]"] = PromptDocumentFactory.make_prompt_document(+ uri=document_item.url,+ mime_type=document_item.mime_type,+ )+++def _image_token(*, registry_index: int) -> str:+ return f"[Image {registry_index + 1}]"+++def _is_absent_optional(*, variable_path: str, is_optional: bool, context_provider: ContextProviderAbstract) -> bool:+ """Whether a reference reads an optional input the run was not given, which the prompt then leaves out.++ Only the root input's absence counts: a dotted path into a structure that is present resolves as usual.+ """+ return is_optional and not context_provider.is_variable_present(name=get_root_from_dotted_path(variable_path))diff --git a/pipelex/kernel/prompt_references.py b/pipelex/kernel/prompt_references.pyindex 03ec9ef67..a4a8c5535 100644--- a/pipelex/kernel/prompt_references.py+++ b/pipelex/kernel/prompt_references.py@@ -46,6 +46,14 @@ class ImageReference(BaseModel):description="For NESTED kind: relative paths to images within the structure, e.g., ['text_and_images.images', 'page_view']",)+ is_optional: bool = Field(+ default=False,+ description=(+ "Whether the input the path starts from is declared optional (`Image?`, `Album?`). An optional input that is absent is "+ "skipped when the prompt is assembled, so the template's guard renders it out; a required one that is absent is refused."+ ),+ )+@overridedef __str__(self) -> str:match self.kind:@@ -80,6 +88,14 @@ class DocumentReference(BaseModel):kind: DocumentReferenceKind = Field(description="The kind of document reference")+ is_optional: bool = Field(+ default=False,+ description=(+ "Whether the input the path starts from is declared optional (`Image?`, `Album?`). An optional input that is absent is "+ "skipped when the prompt is assembled, so the template's guard renders it out; a required one that is absent is refused."+ ),+ )+@overridedef __str__(self) -> str:match self.kind:diff --git a/pipelex/language/mthds_schema_generator.py b/pipelex/language/mthds_schema_generator.pyindex 5aba25154..c02df715f 100644--- a/pipelex/language/mthds_schema_generator.py+++ b/pipelex/language/mthds_schema_generator.py@@ -21,6 +21,7 @@ from pipelex.pipe_machinery.validation import (SEQUENCE_STEP_BATCH_OVER_PATTERN,STORED_NAME_PATTERN,)+from pipelex.pipe_operators.judge.pipe_judge_blueprint import JudgeRatingLevel, PipeJudgeBlueprintfrom pipelex.pipe_signature.pipe_signature_blueprint import PipeSignatureBlueprintfrom pipelex.tools.misc.package_utils import get_package_version@@ -78,6 +79,7 @@ def generate_mthds_schema() -> dict[str, Any]:# Before `_constrain_batch_over`, so the branch definition it copies from the pipe step's carries these constraints too.schema = _constrain_stored_names(schema)schema = _constrain_batch_over(schema)+ schema = _constrain_rating_levels(schema)return _add_taplo_metadata(schema)@@ -393,6 +395,37 @@ def _set_batch_over_grammar(*, step_schema: dict[str, Any], pattern: str, descriarm["pattern"] = pattern+def _constrain_rating_levels(schema: dict[str, Any]) -> dict[str, Any]:+ """Make a PipeJudge's `levels` label every level or none, as its load does.++ A level is a string, which is its description, or a closed `{label, description}` table, and the load+ refuses a scale labelling some of its levels only, since a `Rating` verdict reports the label of its+ level (`_validate_levels`). Draft 4 says so with two array shapes: strings and unlabelled tables, or+ labelled tables only. A table whose `label` is null counts as unlabelled, as it does at load.+ """+ schema = copy.deepcopy(schema)+ definitions = schema.get("definitions", {})+ levels_schema = definitions.get(PipeJudgeBlueprint.__name__, {}).get("properties", {}).get("levels")+ if levels_schema is None or JudgeRatingLevel.__name__ not in definitions:+ return schema+ level_ref = {"$ref": f"#/definitions/{JudgeRatingLevel.__name__}"}+ unlabelled_level = {"allOf": [level_ref, {"properties": {"label": {"type": "null"}}}]}+ labelled_level = {"allOf": [copy.deepcopy(level_ref), {"required": ["label"], "properties": {"label": {"type": "string"}}}]}+ # `levels` is optional, so its schema is an `anyOf` of the array and null: the array arm splits in two.+ arms: list[dict[str, Any]] = levels_schema.get("anyOf", [])+ array_arms = [arm for arm in arms if arm.get("type") == "array"]+ if len(array_arms) != 1:+ msg = f"Expected one array arm on {PipeJudgeBlueprint.__name__}.levels, found {len(array_arms)}"+ raise RuntimeError(msg)+ array_arm = array_arms[0]+ scale_arms = [+ {**array_arm, "items": {"anyOf": [{"type": "string"}, unlabelled_level]}},+ {**copy.deepcopy(array_arm), "items": labelled_level},+ ]+ levels_schema["anyOf"] = [scale_arm for arm in arms for scale_arm in (scale_arms if arm is array_arm else [arm])]+ return schema++def _build_construct_field_schema() -> dict[str, Any]:"""Build a JSON Schema for a construct field value as written in MTHDS files.diff --git a/pipelex/pipe_machinery/validation.py b/pipelex/pipe_machinery/validation.pyindex 8ff069793..c9ff365c2 100644--- a/pipelex/pipe_machinery/validation.py+++ b/pipelex/pipe_machinery/validation.py@@ -103,9 +103,8 @@ def check_inputs_match_variables("""Refuse a variable the templates read that no input declares, then a declared input no template reads.The rule shared by every operator that reads its inputs through templates: PipeLLM, PipeCompose,- PipeSearch and PipeImgGen. It is the composition of `check_variables_are_declared` and- `check_inputs_are_read`, in that order. PipeJudge calls the first half alone, since every input it- declares is material to judge whether its question names it or not.+ PipeSearch, PipeImgGen and PipeJudge, whose prompt and question are read together. It is the+ composition of `check_variables_are_declared` and `check_inputs_are_read`, in that order.Raises:PipeValidationError: ``MISSING_INPUT_VARIABLE`` with the undeclared root names, ordiff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.pyindex cf1c9217b..f9a97f9ef 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py@@ -10,7 +10,7 @@ from pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocolfrom pipelex.pipe_operators.img_gen.img_gen_prompt_blueprint import ImgGenPromptBlueprintfrom pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGenfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint-from pipelex.pipe_operators.shared.template_image_analyzer import TemplateImageAnalyzer+from pipelex.pipe_operators.shared.template_file_references import analyze_template_file_referencesfrom pipelex.tools.jinja2.template_category import TemplateCategoryif TYPE_CHECKING:@@ -51,37 +51,24 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):else None)- # Analyze both prompts for image references- # Images may be referenced in positive prompt, negative prompt, or both+ # Images may be referenced in the positive prompt, the negative prompt or both. A generation prompt+ # presents images only, so the analysis's documents are not read here.image_references: list[ImageReference] | None = Noneif blueprint.inputs:# Template analyzers read the slot grammar only, so they get the concept-spec projection.blueprint_input_specs = blueprint.inputs_concept_specs or {}all_image_refs: list[ImageReference] = []seen_paths: set[str] = set()-- # Analyze positive prompt- if blueprint.prompt:- prompt_refs = TemplateImageAnalyzer.analyze_template_for_images(- template_source=blueprint.prompt,- input_specs=blueprint_input_specs,- domain_code=domain_code,- template_category=TemplateCategory.IMG_GEN_PROMPT,- )- for ref in prompt_refs:- if ref.variable_path not in seen_paths:- all_image_refs.append(ref)- seen_paths.add(ref.variable_path)-- # Analyze negative prompt- if blueprint.negative_prompt:- negative_refs = TemplateImageAnalyzer.analyze_template_for_images(- template_source=blueprint.negative_prompt,+ for template_source in (blueprint.prompt, blueprint.negative_prompt):+ if not template_source:+ continue+ prompt_files = analyze_template_file_references(+ template_source=template_source,input_specs=blueprint_input_specs,domain_code=domain_code,template_category=TemplateCategory.IMG_GEN_PROMPT,)- for ref in negative_refs:+ for ref in prompt_files.image_references:if ref.variable_path not in seen_paths:all_image_refs.append(ref)seen_paths.add(ref.variable_path)diff --git a/pipelex/pipe_operators/judge/exceptions.py b/pipelex/pipe_operators/judge/exceptions.pyindex 3451eb5c4..6b1f7d227 100644--- a/pipelex/pipe_operators/judge/exceptions.py+++ b/pipelex/pipe_operators/judge/exceptions.py@@ -6,25 +6,25 @@ class PipeJudgeError(PipelexError):class PipeJudgeInputCapabilityError(PipeJudgeError):- """A `PipeJudge` declares an image or a document input, and the judging model it resolves to does not read one.+ """A `PipeJudge`'s prompt presents an image or a document, and the judging model it resolves to does not read one.Raised when the method loads, before a run spends anything. The operator does not refuse files: themodel's spec states what it reads, so a model that reads files gets them, and this refusal names the- input and what the model reads. Its message names only the step, the input, the model and its- declared inputs, so it is kept verbatim for the caller.+ variable the prompt reads and what the model reads. Its message names only the step, the variable,+ the model and its declared inputs, so it is kept verbatim for the caller."""error_domain = ErrorDomain.INPUT_authors_caller_facing_message = True- def __init__(self, *, pipe_code: str, input_name: str, file_kind: str, model_name: str, model_inputs: list[str]):+ def __init__(self, *, pipe_code: str, variable_path: str, file_kind: str, model_name: str, model_inputs: list[str]):self.pipe_code = pipe_code- self.input_name = input_name+ self.variable_path = variable_pathself.file_kind = file_kindself.model_name = model_nameself.model_inputs = model_inputsmessage = (- f"PipeJudge '{pipe_code}' judges input '{input_name}', which is {_with_article(noun=file_kind)}, and its judgment model "+ f"PipeJudge '{pipe_code}' presents '{variable_path}' in its prompt, which is {_with_article(noun=file_kind)}, and its judgment model "f"'{model_name}' does not read {file_kind}s: it reads {', '.join(model_inputs)}. Name a judgment model that reads "f"{file_kind}s, or turn the {file_kind} into text with a step before this one.")diff --git a/pipelex/pipe_operators/judge/pipe_judge.py b/pipelex/pipe_operators/judge/pipe_judge.pyindex 71030dad8..b2d48a032 100644--- a/pipelex/pipe_operators/judge/pipe_judge.py+++ b/pipelex/pipe_operators/judge/pipe_judge.py@@ -2,7 +2,15 @@ from typing import Any, Literalfrom typing_extensions import override-from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentKind, JudgmentQuestion, RatingQuestion, YesNoQuestion+from pipelex.cogt.judgment.judgment_models import (+ ChoiceQuestion,+ JudgmentKind,+ JudgmentQuestion,+ RatingLevel,+ RatingQuestion,+ YesNoCriteria,+ YesNoQuestion,+)from pipelex.cogt.judgment.judgment_setting import JudgmentModelChoicefrom pipelex.cogt.models.model_deck_check import check_judgment_choice_with_deckfrom pipelex.cogt.templating.template_blueprint import TemplateBlueprint@@ -13,6 +21,8 @@ from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.interpreter_hub import get_concept_library, get_native_conceptfrom pipelex.kernel.judgment_ops import judgment_setting_of_choice, resolve_judgment_setting, run_judgment, served_judgment_model+from pipelex.kernel.prompt_assembly import UserPromptContent+from pipelex.kernel.prompt_references import DocumentReference, ImageReferencefrom pipelex.kernel.templating_style_ops import resolve_templating_stylefrom pipelex.pipe_machinery.template_guard_lint import lint_authored_templatefrom pipelex.pipe_operators.judge.exceptions import PipeJudgeInputCapabilityError@@ -50,15 +60,21 @@ def _kind_declaration(*, judgment_kind: JudgmentKind) -> str:class PipeJudge(PipeOperator[PipeJudgeOutput]):- """Asks a judging model one closed question about its inputs, and stores the verdict native it answers with.+ """Asks a judging model one closed question about the evidence its prompt presents, and stores the verdict native it answers with.- `judgment_question` is the cogt question with its instructions still the authored template, so the- kernel renders it against memory when the step runs; its kind decides the verdict native.+ `prompt_content` is the evidence: the prompt's template and the images and documents it references,+ assembled when the step runs exactly as a PipeLLM's user prompt is. `judgment_question` is the cogt+ question with its instructions still the authored template, so the kernel renders it against memory+ when the step runs; its kind decides the verdict native. The question presents no file, so the+ image and document references the factory found in it are held only for the load to refuse them."""type: Literal["PipeJudge"] = "PipeJudge"judgment_choice: JudgmentModelChoice | None+ prompt_content: UserPromptContentjudgment_question: JudgmentQuestion+ question_image_references: list[ImageReference] | None = None+ question_document_references: list[DocumentReference] | None = Nonethreshold: float | None = None@property@@ -75,7 +91,7 @@ class PipeJudge(PipeOperator[PipeJudgeOutput]):@overridedef required_variables(self) -> set[str]:- full_paths = self.question_blueprint.required_variables()+ full_paths = self.prompt_content.template.required_variables() | self.question_blueprint.required_variables()return {get_root_from_dotted_path(path) for path in full_paths if not path.startswith("_")}@override@@ -84,63 +100,75 @@ class PipeJudge(PipeOperator[PipeJudgeOutput]):with self.locating_model_choice(field_name="model"):check_judgment_choice_with_deck(judgment_choice=self.judgment_choice)+ # Only the prompt presents files: the question is rendered against the raw context with no token+ # parameters, so an image or a document it read would be written into the question sent to the+ # vendor as the content's string form, such as its storage URL, rather than presented as a file.+ question_file_paths = [+ *(image_reference.variable_path for image_reference in self.question_image_references or []),+ *(document_reference.variable_path for document_reference in self.question_document_references or []),+ ]+ if question_file_paths:+ quoted_paths = ", ".join(f"'{variable_path}'" for variable_path in question_file_paths)+ msg = (+ f"PipeJudge '{self.code}' reads {quoted_paths} in its question, which is an image or a document, and only the prompt "+ "presents files to the judging model: read it in `prompt`, and ask about it in `question`."+ )+ raise PipeValidationError(+ message=msg,+ error_type=PipeValidationErrorType.INPUT_STUFF_SPEC_MISMATCH,+ domain_code=self.domain_code,+ pipe_code=self.code,+ variable_names=question_file_paths,+ )+# Template lints: no private names, and every reference to a declared-optional input guarded (D7).question_blueprint = self.question_blueprint- lint_authored_template(- pipe_code=self.code,- domain_code=self.domain_code,- inputs=self.inputs,- template_source=question_blueprint.template,- template_category=question_blueprint.category,- template_label="question",- )+ for template_blueprint, template_label in [(self.prompt_content.template, "prompt"), (question_blueprint, "question")]:+ lint_authored_template(+ pipe_code=self.code,+ domain_code=self.domain_code,+ inputs=self.inputs,+ template_source=template_blueprint.template,+ template_category=template_blueprint.category,+ template_label=template_label,+ )@overridedef validate_inputs_with_library(self):- """Find the judging model, and refuse an image or a document input it does not read.+ """Find the judging model, and refuse an image or a document the prompt presents that it does not read.The model is found when the method loads, so a step with no model to judge with, naming nonewhere the deck names no default, is refused with `JudgmentModelMissingError` before a run spends- anything: the deck serves no judgment model out of the box. Every input is material to judge, so- an `Image` or a `Document` input, or a list of either, is sent to the model as a file. Whether the- model reads one is its own capability, stated in its spec's `inputs`: a model that does not is- refused here, naming the input and what it reads. That needs the model's spec, which a boot that- does not enable the model's backend does not hold, and neither does one serving none of a- waterfall's models: the check is then left to the worker when the step runs, as it is for a- `Dynamic` input, whose declaration names no kind of value. A keyless boot holds the spec: it keeps- every enabled backend, whether or not its key is set.+ anything: the deck serves no judgment model out of the box. The prompt presents the images and+ documents it reads to the model as files, and whether the model reads one is its own capability,+ stated in its spec's `inputs`: a model that does not is refused here, naming the variable and what+ it reads. Only what the prompt references is checked, never every declared input, since an input+ reaches the model only through a template. That needs the model's spec, which a boot that does not+ enable the model's backend does not hold, and neither does one serving none of a waterfall's+ models: the check is then left to the worker when the step runs. A keyless boot holds the spec: it+ keeps every enabled backend, whether or not its key is set."""with self.locating_model_choice(field_name="model"):judgment_setting = judgment_setting_of_choice(judgment_choice=self.judgment_choice, pipe_code=self.code)inference_model = served_judgment_model(model_handle=judgment_setting.model)if inference_model is None:return- concept_library = get_concept_library()- image_concept = get_native_concept(native_concept=NativeConceptCode.IMAGE)- document_concept = get_native_concept(native_concept=NativeConceptCode.DOCUMENT)- for input_name, stuff_spec in self.inputs.items:- if NativeConceptCode.is_dynamic_concept(concept_code=stuff_spec.concept.code):- # `Dynamic` is compatible with every concept, so its declaration says nothing about files:- # the worker checks what such an input actually holds when the step runs.- continue- if concept_library.is_compatible(tested_concept=stuff_spec.concept, wanted_concept=image_concept, strict=True):- if not inference_model.is_vision_supported:- raise PipeJudgeInputCapabilityError(- pipe_code=self.code,- input_name=input_name,- file_kind="image",- model_name=inference_model.name,- model_inputs=inference_model.inputs,- )- elif concept_library.is_compatible(tested_concept=stuff_spec.concept, wanted_concept=document_concept, strict=True):- if not inference_model.is_document_supported:- raise PipeJudgeInputCapabilityError(- pipe_code=self.code,- input_name=input_name,- file_kind="document",- model_name=inference_model.name,- model_inputs=inference_model.inputs,- )+ if self.prompt_content.image_references and not inference_model.is_vision_supported:+ raise PipeJudgeInputCapabilityError(+ pipe_code=self.code,+ variable_path=self.prompt_content.image_references[0].variable_path,+ file_kind="image",+ model_name=inference_model.name,+ model_inputs=inference_model.inputs,+ )+ if self.prompt_content.document_references and not inference_model.is_document_supported:+ raise PipeJudgeInputCapabilityError(+ pipe_code=self.code,+ variable_path=self.prompt_content.document_references[0].variable_path,+ file_kind="document",+ model_name=inference_model.name,+ model_inputs=inference_model.inputs,+ )@overridedef validate_output_static(self):@@ -187,14 +215,14 @@ class PipeJudge(PipeOperator[PipeJudgeOutput]):judgment_setting = resolve_judgment_setting(judgment_choice=self.judgment_choice, pipe_code=self.code, is_dry=pipe_run_params.run_mode.is_dry)judgment_result = await run_judgment(memory=working_memory,+ prompt_content=self.prompt_content,question=self.judgment_question,- input_names=self.inputs.variables,judgment_setting=judgment_setting,concept=self.output.concept,job_metadata=job_metadata,cogt_run_params=pipe_run_params.cogt_run_params,- # No authored style on this operator: a question template takes the runtime default, the- # same one an LLM pipe that declares nothing gets.+ # No authored style on this operator: its prompt and question templates take the runtime+ # default, the same one an LLM pipe that declares nothing gets.templating_style=resolve_templating_style(authored=None),threshold=self.threshold,result_name=output_name,@@ -202,7 +230,11 @@ class PipeJudge(PipeOperator[PipeJudgeOutput]):# Capture execution data for the graph tracer. The raw answer rides along because it carries# what the verdict native's rendering does not show, such as a choice's whole distribution.+ judgment_prompt = judgment_result.promptexecution_data_dict: dict[str, Any] = {+ "rendered_prompt": judgment_prompt.text,+ "nb_prompt_images": len(judgment_prompt.images),+ "nb_prompt_documents": len(judgment_prompt.documents),"rendered_question": judgment_result.rendered_question,"resolved_model": judgment_result.judgment_setting.model,"judgment_kind": self.judgment_kind.value,@@ -236,9 +268,8 @@ def make_judgment_question(*,question_template: str,options: dict[str, str] | None,- levels: list[str] | None,- yes_criterion: str | None,- no_criterion: str | None,+ levels: list[RatingLevel] | None,+ criteria: YesNoCriteria | None,) -> JudgmentQuestion:"""The cogt question frame for a step, its instructions the authored template, its kind read off the fields declared.@@ -248,4 +279,4 @@ def make_judgment_question(return ChoiceQuestion(instructions=question_template, options={key: description or None for key, description in options.items()})if levels is not None:return RatingQuestion(instructions=question_template, levels=levels)- return YesNoQuestion(instructions=question_template, yes_criterion=yes_criterion, no_criterion=no_criterion)+ return YesNoQuestion(instructions=question_template, criteria=criteria)diff --git a/pipelex/pipe_operators/judge/pipe_judge_blueprint.py b/pipelex/pipe_operators/judge/pipe_judge_blueprint.pyindex 28eaad79f..5c623e93b 100644--- a/pipelex/pipe_operators/judge/pipe_judge_blueprint.py+++ b/pipelex/pipe_operators/judge/pipe_judge_blueprint.py@@ -1,92 +1,133 @@-from typing import Any, Literal, Self+from typing import Any, Literal, Self, castfrom pydantic import BaseModel, ConfigDict, model_validatorfrom typing_extensions import override-from pipelex.cogt.judgment.judgment_models import JudgmentKind+from pipelex.cogt.judgment.judgment_models import JudgmentKind, RatingLevelfrom pipelex.cogt.judgment.judgment_setting import JudgmentModelChoicefrom pipelex.cogt.templating.exceptions import TemplateSigilSyntaxErrorfrom pipelex.cogt.templating.template_preprocessor import preprocess_templatefrom pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicityfrom pipelex.pipe_machinery.pipe_blueprint import PipeBlueprint-from pipelex.pipe_machinery.validation import check_variables_are_declared+from pipelex.pipe_machinery.validation import check_inputs_match_variablesfrom pipelex.tools.jinja2.exceptions import Jinja2TemplateSyntaxErrorfrom pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsingfrom pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variablesfrom pipelex.tools.jinja2.template_category import TemplateCategoryfrom pipelex.tools.misc.string_utils import get_root_from_dotted_path-# The one synonym the language admits: every other inference operator calls its template `prompt`.-QUESTION_SYNONYM = "prompt"+class JudgeYesNoCriteria(BaseModel):+ """What a yes and a no mean, for a yes/no question: both sides, each non-empty. Closed: a key other than `yes` or `no` is refused.-def read_prompt_as_question(*, values: dict[str, Any]) -> dict[str, Any]:- """The fields with `prompt` renamed `question`, refusing both spellings; the caller's dict is never changed."""- if QUESTION_SYNONYM not in values:- return values- if "question" in values:- msg = f"A PipeJudge sets `question`, or `{QUESTION_SYNONYM}` as its synonym, but not both: remove one of the two."- raise ValueError(msg)- fields = dict(values)- fields["question"] = fields.pop(QUESTION_SYNONYM)- return fields+ The judging vendors silently ignore a criterion key they do not know, so this model is the only+ place a typo in one is caught. Both sides are required because a lone side cannot be carried with+ its meaning on every backend: one sends a single criterion as one of a pair of options, where the+ side left out would be a blank.+ """+ model_config = ConfigDict(extra="forbid")-def _admit_the_question_synonym(schema: dict[str, Any]) -> None: # kw-only: ignore — pydantic calls `json_schema_extra` positionally- """Give the JSON schema the synonym the before-validator reads, requiring exactly one of the two spellings.+ yes: str+ no: str- The synonym is not a field, so without this the schema would require `question` and, forbidding- extra keys, refuse a table that writes `prompt`, which the blueprint loads.- """- properties = schema["properties"]- properties[QUESTION_SYNONYM] = {**properties["question"], "title": "Prompt", "description": "A synonym of `question`, read exactly as it."}- schema["required"] = [name for name in schema.get("required", []) if name != "question"]- schema["oneOf"] = [{"required": ["question"]}, {"required": [QUESTION_SYNONYM]}]+ @model_validator(mode="before")+ @classmethod+ def refuse_a_lone_side(cls, values: Any) -> Any:+ if not isinstance(values, dict):+ return values+ raw_criteria = cast("dict[str, Any]", values)+ if not raw_criteria:+ msg = "Criteria describe both answers, and this table declares neither: write both `yes` and `no`, or remove the table."+ raise ValueError(msg)+ declared_sides = [side for side in ("yes", "no") if side in raw_criteria]+ if len(declared_sides) == 1:+ declared_side = declared_sides[0]+ missing_side = "no" if declared_side == "yes" else "yes"+ msg = (+ f"Criteria describe both answers, and these declare `{declared_side}` without `{missing_side}`: "+ f"write `{missing_side}` as the complement of `{declared_side}`."+ )+ raise ValueError(msg)+ return raw_criteria+ @model_validator(mode="after")+ def validate_sides_are_not_empty(self) -> Self:+ for side, description in (("yes", self.yes), ("no", self.no)):+ if not description.strip():+ msg = f"Criteria describe both answers, so `{side}` cannot be empty."+ raise ValueError(msg)+ return self-class JudgeYesNoCriteria(BaseModel):- """What a yes and a no mean, for a yes/no question. Closed: a key other than `yes` or `no` is refused.- The judging vendor silently ignores a criterion key it does not know, so this model is the only- place a typo in one is caught.+class JudgeRatingLevel(BaseModel):+ """A rating level written as a table: a `label` naming it in a few words, a `description` of the situation it stands for, or both.++ Closed, as the standard requires. A level written as a plain string is its description."""model_config = ConfigDict(extra="forbid")- yes: str | None = None- no: str | None = None+ label: str | None = None+ description: str | None = Noneclass PipeJudgeBlueprint(PipeBlueprint):- """Asks a judging model one closed question about its inputs: yes/no, a choice, or a rating.+ """Asks a judging model one closed question about the evidence its prompt presents: yes/no, a choice, or a rating.- The kind is decided by which of `options` and `levels` the pipe declares, never by a field of its- own. `prompt` is accepted in place of `question` and read exactly as it, so nothing after parsing- sees the synonym. Every declared input is material to judge, so an input the question never names- is still read; only a variable the question names must be declared.+ `prompt` is the evidence template, rendered and checked as a PipeLLM's prompt is, the images and+ documents it reads presented to the model as files. `question` is the instruction, a plain text+ template. The kind is decided by which of `options` and `levels` the pipe declares, never by a+ field of its own. Every declared input is read by the prompt or by the question."""- model_config = ConfigDict(json_schema_extra=_admit_the_question_synonym)-type: Literal["PipeJudge"] = "PipeJudge"pipe_category: Literal["PipeOperator"] = "PipeOperator"+ prompt: strquestion: strmodel: JudgmentModelChoice | None = Noneoptions: dict[str, str] | None = None- levels: list[str] | None = None+ levels: list[str | JudgeRatingLevel] | None = Nonecriteria: JudgeYesNoCriteria | None = Nonethreshold: float | None = None@model_validator(mode="before")@classmethod- def read_prompt_synonym(cls, values: dict[str, Any]) -> dict[str, Any]:- return read_prompt_as_question(values=values)+ def refuse_a_missing_prompt_or_question(cls, values: Any) -> Any:+ """Name the field a bundle written for the old operator gets wrong, where pydantic would only say one is missing.++ Before this version `prompt` was a synonym of `question`, so a table setting `prompt` alone most+ likely holds a question. An empty prompt or question is refused here too.+ """+ if not isinstance(values, dict):+ return values+ raw_table = cast("dict[str, Any]", values)+ if "question" not in raw_table:+ msg = (+ "A PipeJudge asks a question about the evidence of its prompt, and this one sets no `question`: "+ "`prompt` holds the evidence the question is asked over, and the question is written in `question`."+ )+ raise ValueError(msg)+ if "prompt" not in raw_table:+ msg = (+ "A PipeJudge judges the evidence its `prompt` presents, and this one sets no `prompt`: write the evidence "+ 'as a template that reads the inputs, such as `prompt = "@message"`.'+ )+ raise ValueError(msg)+ # Refused here rather than after validation, so an empty template is named before the input check+ # reports the inputs it leaves unread.+ prompt = raw_table["prompt"]+ if isinstance(prompt, str) and not prompt.strip():+ msg = "A PipeJudge judges the evidence its prompt presents, so its `prompt` cannot be empty."+ raise ValueError(msg)+ question = raw_table["question"]+ if isinstance(question, str) and not question.strip():+ msg = "A PipeJudge asks one question, so its `question` cannot be empty."+ raise ValueError(msg)+ return raw_table@model_validator(mode="after")def validate_question_kind(self) -> Self:- if not self.question.strip():- msg = "A PipeJudge asks one question, so its `question` cannot be empty."- raise ValueError(msg)if self.options is not None and self.levels is not None:msg = "A PipeJudge declares `options` for a choice question or `levels` for a rating question, not both."raise ValueError(msg)@@ -107,13 +148,7 @@ class PipeJudgeBlueprint(PipeBlueprint):msg = "Every key of `options` names an option, so none may be empty."raise ValueError(msg)if self.levels is not None:- if len(self.levels) < 2:- msg = f"A rating question needs at least two `levels` on its scale, and this one declares {len(self.levels)}."- raise ValueError(msg)- empty_indices = [index for index, level in enumerate(self.levels) if not level.strip()]- if empty_indices:- msg = f"Every level of a rating question describes a situation, so none may be empty: level {empty_indices[0]} is."- raise ValueError(msg)+ _validate_levels(levels=self.levels)if self.threshold is not None and not 0 < self.threshold < 1:msg = f"`threshold` is a probability of yes strictly between 0 and 1, and this PipeJudge declares {self.threshold}."raise ValueError(msg)@@ -127,33 +162,30 @@ class PipeJudgeBlueprint(PipeBlueprint):return JudgmentKind.RATINGreturn JudgmentKind.YES_NO+ @property+ def rating_levels(self) -> list[RatingLevel] | None:+ """The scale as the judgment family carries it: a string level is its description."""+ if self.levels is None:+ return None+ rating_levels: list[RatingLevel] = []+ for level in self.levels:+ if isinstance(level, str):+ rating_levels.append(RatingLevel(description=level))+ else:+ rating_levels.append(RatingLevel(label=level.label, description=level.description))+ return rating_levels+@overridedef validate_inputs(self):- template_category = TemplateCategory.BASIC+ """The two-way check every operator that reads its inputs through templates applies, over the prompt and the question together."""declared_inputs: set[str] = set(self.inputs.keys()) if self.inputs else set()- try:- preprocessed_template = preprocess_template(self.question, declared_inputs=declared_inputs)- except TemplateSigilSyntaxError as exc:- msg = f"Template sigil error in PipeJudge question: {exc}"- raise ValueError(msg) from exc- try:- check_jinja2_parsing(- template_source=preprocessed_template,- template_category=template_category,- )- except Jinja2TemplateSyntaxError as exc:- msg = f"Could not parse the question template for PipeJudge: {exc}"- raise ValueError(msg) from exc-- full_paths = detect_jinja2_required_variables(- template_category=template_category,- template_source=preprocessed_template,+ variable_paths = _template_variable_paths(+ template_source=self.prompt, template_category=TemplateCategory.LLM_PROMPT, declared_inputs=declared_inputs, field_name="prompt")- # Names starting with an underscore are internal and never count as read inputs- variable_paths = {path for path in full_paths if not get_root_from_dotted_path(path).startswith("_")}- # Every declared input is material to judge, read whether the question names it or not: only- # the variables the question does name must be declared.- check_variables_are_declared(declared_inputs=declared_inputs, variable_paths=variable_paths, reader="question")+ variable_paths |= _template_variable_paths(+ template_source=self.question, template_category=TemplateCategory.BASIC, declared_inputs=declared_inputs, field_name="question"+ )+ check_inputs_match_variables(declared_inputs=declared_inputs, variable_paths=variable_paths, reader="prompt or question")@overridedef validate_output(self):@@ -164,3 +196,50 @@ class PipeJudgeBlueprint(PipeBlueprint):"To judge each item of a list, map the PipeJudge over it with a PipeBatch.")raise ValueError(msg)+++def _validate_levels(*, levels: list[str | JudgeRatingLevel]) -> None:+ """A scale of at least two levels, none empty, its labels all or none and distinct, since a label is what the verdict reports."""+ if len(levels) < 2:+ msg = f"A rating question needs at least two `levels` on its scale, and this one declares {len(levels)}."+ raise ValueError(msg)+ labels: list[str] = []+ for index_level, level in enumerate(levels):+ if isinstance(level, str):+ if not level.strip():+ msg = f"Every level of a rating question describes a situation, so none may be empty: level {index_level} is."+ raise ValueError(msg)+ continue+ if level.label is None and level.description is None:+ msg = f"A rating level written as a table carries a `label`, a `description` or both, and level {index_level} carries neither."+ raise ValueError(msg)+ for field_name, field_value in (("label", level.label), ("description", level.description)):+ if field_value is not None and not field_value.strip():+ msg = f"A rating level's `{field_name}` cannot be empty, and level {index_level}'s is."+ raise ValueError(msg)+ if level.label is not None:+ labels.append(level.label)+ if labels and len(labels) != len(levels):+ msg = "On a rating scale every level carries a `label` or none does, and this one labels some of its levels only."+ raise ValueError(msg)+ duplicated_labels = sorted({label for label in labels if labels.count(label) > 1})+ if duplicated_labels:+ repeated = ", ".join(f"'{label}'" for label in duplicated_labels)+ msg = f"A rating verdict reports the label of its level, so the labels of one scale are distinct, and {repeated} is repeated."+ raise ValueError(msg)+++def _template_variable_paths(*, template_source: str, template_category: TemplateCategory, declared_inputs: set[str], field_name: str) -> set[str]:+ """The full dotted paths a template reads, rid of the internal names that start with an underscore."""+ try:+ preprocessed_template = preprocess_template(template_source, declared_inputs=declared_inputs)+ except TemplateSigilSyntaxError as exc:+ msg = f"Template sigil error in PipeJudge {field_name}: {exc}"+ raise ValueError(msg) from exc+ try:+ check_jinja2_parsing(template_source=preprocessed_template, template_category=template_category)+ except Jinja2TemplateSyntaxError as exc:+ msg = f"Could not parse the {field_name} template for PipeJudge: {exc}"+ raise ValueError(msg) from exc+ full_paths = detect_jinja2_required_variables(template_category=template_category, template_source=preprocessed_template)+ return {path for path in full_paths if not get_root_from_dotted_path(path).startswith("_")}diff --git a/pipelex/pipe_operators/judge/pipe_judge_factory.py b/pipelex/pipe_operators/judge/pipe_judge_factory.pyindex 7ef0616af..9beaa24ec 100644--- a/pipelex/pipe_operators/judge/pipe_judge_factory.py+++ b/pipelex/pipe_operators/judge/pipe_judge_factory.py@@ -2,11 +2,16 @@ from typing import Anyfrom typing_extensions import override+from pipelex.cogt.judgment.judgment_models import YesNoCriteria+from pipelex.cogt.templating.template_blueprint import TemplateBlueprintfrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec+from pipelex.kernel.prompt_assembly import UserPromptContentfrom pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocolfrom pipelex.pipe_operators.judge.pipe_judge import PipeJudge, make_judgment_questionfrom pipelex.pipe_operators.judge.pipe_judge_blueprint import PipeJudgeBlueprint+from pipelex.pipe_operators.shared.template_file_references import analyze_template_file_references+from pipelex.tools.jinja2.template_category import TemplateCategoryclass PipeJudgeFactory(PipeFactoryProtocol[PipeJudgeBlueprint, PipeJudge]):@@ -24,13 +29,27 @@ class PipeJudgeFactory(PipeFactoryProtocol[PipeJudgeBlueprint, PipeJudge]):output: StuffSpec,blueprint: PipeJudgeBlueprint,) -> PipeJudge:+ # Template analyzers read the slot grammar only, so they get the concept-spec projection.+ input_specs = blueprint.inputs_concept_specs or {}+ # The evidence is built exactly as a PipeLLM's prompt is: its template and the files it references.+ prompt_files = analyze_template_file_references(template_source=blueprint.prompt, input_specs=input_specs, domain_code=domain_code)+ prompt_content = UserPromptContent(+ template=TemplateBlueprint(template=blueprint.prompt, category=TemplateCategory.LLM_PROMPT),+ image_references=prompt_files.image_references or None,+ document_references=prompt_files.document_references or None,+ )+ # The question presents no file: whatever image or document it reads is held for the load to refuse.+ question_files = analyze_template_file_references(+ template_source=blueprint.question, input_specs=input_specs, domain_code=domain_code, template_category=TemplateCategory.BASIC+ )+ question_image_references = question_files.image_references or None+ question_document_references = question_files.document_references or Nonecriteria = blueprint.criteriajudgment_question = make_judgment_question(question_template=blueprint.question,options=blueprint.options,- levels=blueprint.levels,- yes_criterion=criteria.yes if criteria is not None else None,- no_criterion=criteria.no if criteria is not None else None,+ levels=blueprint.rating_levels,+ criteria=YesNoCriteria(yes=criteria.yes, no=criteria.no) if criteria is not None else None,)return PipeJudge(domain_code=domain_code,@@ -39,6 +58,9 @@ class PipeJudgeFactory(PipeFactoryProtocol[PipeJudgeBlueprint, PipeJudge]):output=output,inputs=inputs,judgment_choice=blueprint.model,+ prompt_content=prompt_content,judgment_question=judgment_question,+ question_image_references=question_image_references,+ question_document_references=question_document_references,threshold=blueprint.threshold,)diff --git a/pipelex/pipe_operators/llm/pipe_llm_factory.py b/pipelex/pipe_operators/llm/pipe_llm_factory.pyindex b30807bca..811c953da 100644--- a/pipelex/pipe_operators/llm/pipe_llm_factory.py+++ b/pipelex/pipe_operators/llm/pipe_llm_factory.py@@ -15,8 +15,7 @@ from pipelex.pipe_operators.llm.exceptions import PipeLLMFactoryErrorfrom pipelex.pipe_operators.llm.llm_prompt_blueprint import LLMPromptBlueprintfrom pipelex.pipe_operators.llm.pipe_llm import PipeLLMfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint-from pipelex.pipe_operators.llm.template_document_analyzer import TemplateDocumentAnalyzer-from pipelex.pipe_operators.shared.template_image_analyzer import TemplateImageAnalyzer+from pipelex.pipe_operators.shared.template_file_references import analyze_template_file_referencesfrom pipelex.tools.jinja2.template_category import TemplateCategoryfrom pipelex.tools.templating.templating_style import TagStyle, TemplatingStyle@@ -90,61 +89,26 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):# Template analyzers read the slot grammar only, so they get the concept-spec projection.blueprint_input_specs = blueprint.inputs_concept_specs or {}- # Analyze template for image references- user_image_references = None- if blueprint.prompt and blueprint.inputs:- user_image_references = (- TemplateImageAnalyzer.analyze_template_for_images(- template_source=blueprint.prompt,- input_specs=blueprint_input_specs,- domain_code=domain_code,- )- or None- )-- # Analyze template for document references- user_document_references = None- if blueprint.prompt and blueprint.inputs:- user_document_references = (- TemplateDocumentAnalyzer.analyze_template_for_documents(- template_source=blueprint.prompt,- input_specs=blueprint_input_specs,- domain_code=domain_code,- )- or None- )-- # Analyze system prompt for image references- system_image_references = None- if blueprint.system_prompt and blueprint.inputs:- system_image_references = (- TemplateImageAnalyzer.analyze_template_for_images(- template_source=blueprint.system_prompt,- input_specs=blueprint_input_specs,- domain_code=domain_code,- )- or None- )-- # Analyze system prompt for document references- system_document_references = None- if blueprint.system_prompt and blueprint.inputs:- system_document_references = (- TemplateDocumentAnalyzer.analyze_template_for_documents(- template_source=blueprint.system_prompt,- input_specs=blueprint_input_specs,- domain_code=domain_code,- )- or None- )+ # The user and system prompts are analyzed apart, each keeping its own references; the assembly+ # registers the system prompt's first and numbers both prompts' files in one sequence.+ user_files = (+ analyze_template_file_references(template_source=blueprint.prompt, input_specs=blueprint_input_specs, domain_code=domain_code)+ if blueprint.prompt and blueprint.inputs+ else None+ )+ system_files = (+ analyze_template_file_references(template_source=blueprint.system_prompt, input_specs=blueprint_input_specs, domain_code=domain_code)+ if blueprint.system_prompt and blueprint.inputs+ else None+ )llm_prompt_spec = LLMPromptBlueprint(system_prompt_blueprint=system_prompt_jinja2_blueprint,prompt_blueprint=user_text_jinja2_blueprint,- user_image_references=user_image_references,- user_document_references=user_document_references,- system_image_references=system_image_references,- system_document_references=system_document_references,+ user_image_references=(user_files.image_references or None) if user_files else None,+ user_document_references=(user_files.document_references or None) if user_files else None,+ system_image_references=(system_files.image_references or None) if system_files else None,+ system_document_references=(system_files.document_references or None) if system_files else None,)llm_choices = LLMSettingChoices(diff --git a/pipelex/pipe_operators/llm/template_document_analyzer.py b/pipelex/pipe_operators/shared/template_document_analyzer.pysimilarity index 97%rename from pipelex/pipe_operators/llm/template_document_analyzer.pyrename to pipelex/pipe_operators/shared/template_document_analyzer.pyindex b4e99a682..ec642eeac 100644--- a/pipelex/pipe_operators/llm/template_document_analyzer.py+++ b/pipelex/pipe_operators/shared/template_document_analyzer.py@@ -34,6 +34,7 @@ class TemplateDocumentAnalyzer:*,input_specs: dict[str, str],domain_code: str,+ template_category: TemplateCategory = TemplateCategory.LLM_PROMPT,) -> list[DocumentReference]:"""Analyze a template to find document references.@@ -41,6 +42,7 @@ class TemplateDocumentAnalyzer:template_source: The Jinja2 template sourceinput_specs: Mapping of variable names to concept codes (e.g., {"doc": "Document"})domain_code: The domain code for resolving concepts+ template_category: The template category, a prompt's by defaultReturns:List of DocumentReference objects describing how to extract documents@@ -50,7 +52,7 @@ class TemplateDocumentAnalyzer:# Parse template to get variable references with filtersvariable_refs = detect_jinja2_variable_references(- template_category=TemplateCategory.LLM_PROMPT,+ template_category=template_category,template_source=preprocessed,)diff --git a/pipelex/pipe_operators/shared/template_file_references.py b/pipelex/pipe_operators/shared/template_file_references.pynew file mode 100644index 000000000..3048fbaa2--- /dev/null+++ b/pipelex/pipe_operators/shared/template_file_references.py@@ -0,0 +1,76 @@+"""The images and documents a prompt template reads, analyzed the same way for every prompt-shaped operator.++PipeLLM, PipeImgGen and PipeJudge each turn a template into the file references the shared assembly+(`pipelex.kernel.prompt_assembly`) fetches when the prompt is built. They call this one analysis so that+what a reference carries cannot differ between them, in particular whether the input it starts from is+declared optional, which decides whether an absent file is skipped or refused. It lives beside the+analyzers rather than beside the assembly because analyzing resolves concepts through the library, which+the kernel may not import.+"""++from pydantic import BaseModel, ConfigDict++from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity+from pipelex.kernel.prompt_references import DocumentReference, ImageReference+from pipelex.pipe_operators.shared.template_document_analyzer import TemplateDocumentAnalyzer+from pipelex.pipe_operators.shared.template_image_analyzer import TemplateImageAnalyzer+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.misc.string_utils import get_root_from_dotted_path+++class TemplateFileReferences(BaseModel):+ """The image and document references one template makes, each in the order the template reads it."""++ model_config = ConfigDict(frozen=True, extra="forbid")++ image_references: list[ImageReference]+ document_references: list[DocumentReference]+++def analyze_template_file_references(+ *,+ template_source: str,+ input_specs: dict[str, str],+ domain_code: str,+ template_category: TemplateCategory = TemplateCategory.LLM_PROMPT,+) -> TemplateFileReferences:+ """Find the images and documents a template reads, each marked with whether its root input is optional.++ Args:+ template_source: The template, in its authored syntax (`@photo`, `$note`, Jinja2).+ input_specs: Each input's concept spec, presence marker included (`{"photo": "Image?"}`).+ domain_code: The domain the template's concepts resolve in.+ template_category: The template's category, which decides the filters it may use.+ """+ image_references = TemplateImageAnalyzer.analyze_template_for_images(+ template_source=template_source,+ input_specs=input_specs,+ domain_code=domain_code,+ template_category=template_category,+ )+ document_references = TemplateDocumentAnalyzer.analyze_template_for_documents(+ template_source=template_source,+ input_specs=input_specs,+ domain_code=domain_code,+ template_category=template_category,+ )+ return TemplateFileReferences(+ image_references=[+ image_reference.model_copy(+ update={"is_optional": _is_root_input_optional(variable_path=image_reference.variable_path, input_specs=input_specs)}+ )+ for image_reference in image_references+ ],+ document_references=[+ document_reference.model_copy(+ update={"is_optional": _is_root_input_optional(variable_path=document_reference.variable_path, input_specs=input_specs)}+ )+ for document_reference in document_references+ ],+ )+++def _is_root_input_optional(*, variable_path: str, input_specs: dict[str, str]) -> bool:+ # The analyzers only return references whose root is a declared input.+ input_spec = input_specs[get_root_from_dotted_path(variable_path)]+ return parse_concept_with_multiplicity(input_spec).presence.is_optionaldiff --git a/pipelex/pipeline/file_input_consumers.py b/pipelex/pipeline/file_input_consumers.pyindex 00b6949f5..4ebc6e97a 100644--- a/pipelex/pipeline/file_input_consumers.py+++ b/pipelex/pipeline/file_input_consumers.py@@ -25,13 +25,14 @@ resolving sub-pipes through the hub:unresolved cross-package reference consume nothing as far as the walk knows: it never guesses.- **Consumers.** A `PipeExtract` consumes its document input, and reads the formats of the model itsextract choice resolves to. A `PipeLLM` consumes the documents its prompt references by variable- path, and reads its resolved model's document types. A `PipeJudge` consumes every document input- it declares, since every input is material to judge, and reads its resolved judgment model's- document types. A waterfall reads a format when any of its members reads it.+ path, and reads its resolved model's document types. A `PipeJudge` consumes the documents its prompt+ references by variable path, as a PipeLLM does, and reads its resolved judgment model's document+ types. A waterfall reads a format when any of its members reads it.The walk depends only on the library and the model deck, never on the input values."""+from collections.abc import Sequencefrom enum import StrEnumfrom typing import Final, NamedTuple@@ -47,6 +48,7 @@ from pipelex.interpreter_hub import get_concept_library, get_native_concept, getfrom pipelex.kernel.extract_ops import resolve_extract_settingfrom pipelex.kernel.judgment_ops import judgment_setting_of_choicefrom pipelex.kernel.llm_ops import resolve_llm_setting_for_object, resolve_llm_setting_for_text+from pipelex.kernel.prompt_references import DocumentReferencefrom pipelex.pipe_controllers.batch.pipe_batch import PipeBatchfrom pipelex.pipe_controllers.binding.binding_step import BindingStepfrom pipelex.pipe_controllers.condition.pipe_condition import PipeCondition@@ -507,65 +509,68 @@ def _record_llm_document_consumers(*, pipe_llm: PipeLLM, frame: _Frame, is_condidocument_references = [*(prompt_spec.user_document_references or []), *(prompt_spec.system_document_references or [])]if not document_references:return- resolved_model = _resolve_llm_model(pipe_llm=pipe_llm)- readable_formats: frozenset[str] | None = None- if resolved_model is not None:- readable_formats = frozenset[str]().union(*(model_spec.supported_document_types for model_spec in resolved_model.specs))- # A model that reads no documents at all is the method author's choice of model, which the- # worker reports as a capability error: there is no caller's format to refuse.- if not readable_formats:- return- for document_reference in document_references:- consumed_path = _tracked_path(frame=frame, variable_path=document_reference.variable_path)- if consumed_path is None:- continue- consumers.append(- FileInputConsumer(- slot_name=consumed_path[0],- consumed_path=consumed_path,- pipe_ref=pipe_llm.pipe_ref,- pipe_code=pipe_llm.code,- kind=FileConsumerKind.LLM_DOCUMENT,- model=resolved_model.model if resolved_model else None,- readable_formats=readable_formats,- reads_web_pages=False,- is_conditional=is_conditional,- )- )+ _record_document_consumers(+ document_references=document_references,+ resolved_model=_resolve_llm_model(pipe_llm=pipe_llm),+ kind=FileConsumerKind.LLM_DOCUMENT,+ pipe=pipe_llm,+ frame=frame,+ is_conditional=is_conditional,+ consumers=consumers,+ )def _record_judgment_document_consumers(*, pipe_judge: PipeJudge, frame: _Frame, is_conditional: bool, consumers: list[FileInputConsumer]) -> None:- # A `Dynamic` input declares no kind of value, and an image input is an image by the setup check,- # so only the inputs declared as documents are followed, as the operator's load-time check does.- concept_library = get_concept_library()- document_concept = get_native_concept(native_concept=NativeConceptCode.DOCUMENT)- document_input_names = [- input_name- for input_name, stuff_spec in pipe_judge.inputs.items- if not NativeConceptCode.is_dynamic_concept(concept_code=stuff_spec.concept.code)- and concept_library.is_compatible(tested_concept=stuff_spec.concept, wanted_concept=document_concept, strict=True)- ]- if not document_input_names:+ # The prompt presents the documents it references, as a PipeLLM's does, and an input reaches the+ # judging model only through a template, so a document input the prompt does not reference is not+ # sent: only the prompt's references are followed.+ document_references = pipe_judge.prompt_content.document_references or []+ if not document_references:return- resolved_model = _resolve_judgment_model(pipe_judge=pipe_judge)+ _record_document_consumers(+ document_references=document_references,+ resolved_model=_resolve_judgment_model(pipe_judge=pipe_judge),+ kind=FileConsumerKind.JUDGMENT_DOCUMENT,+ pipe=pipe_judge,+ frame=frame,+ is_conditional=is_conditional,+ consumers=consumers,+ )+++def _record_document_consumers(+ *,+ document_references: Sequence[DocumentReference],+ resolved_model: _ResolvedModel | None,+ kind: FileConsumerKind,+ pipe: PipeAbstract,+ frame: _Frame,+ is_conditional: bool,+ consumers: list[FileInputConsumer],+) -> None:+ """Record one consumer for each document a prompt presents that comes from a tracked entry input.++ A prompt-shaped operator hands its documents to its model as files, so each reference whose root is+ tracked consumes that input's files, read by the formats the model reads.+ """readable_formats: frozenset[str] | None = Noneif resolved_model is not None:readable_formats = frozenset[str]().union(*(model_spec.supported_document_types for model_spec in resolved_model.specs))- # A model that reads no documents at all is the method author's choice of model, which the- # method's load refuses: there is no caller's format to refuse.+ # A model that reads no documents at all is the method author's choice of model, which the load or+ # the worker reports as a capability error: there is no caller's format to refuse.if not readable_formats:return- for input_name in document_input_names:- consumed_path = _tracked_path(frame=frame, variable_path=input_name)+ for document_reference in document_references:+ consumed_path = _tracked_path(frame=frame, variable_path=document_reference.variable_path)if consumed_path is None:continueconsumers.append(FileInputConsumer(slot_name=consumed_path[0],consumed_path=consumed_path,- pipe_ref=pipe_judge.pipe_ref,- pipe_code=pipe_judge.code,- kind=FileConsumerKind.JUDGMENT_DOCUMENT,+ pipe_ref=pipe.pipe_ref,+ pipe_code=pipe.code,+ kind=kind,model=resolved_model.model if resolved_model else None,readable_formats=readable_formats,reads_web_pages=False,diff --git a/pipelex/providers/typesafe/typesafe_judgment_worker.py b/pipelex/providers/typesafe/typesafe_judgment_worker.pyindex 770224abc..cb715b366 100644--- a/pipelex/providers/typesafe/typesafe_judgment_worker.py+++ b/pipelex/providers/typesafe/typesafe_judgment_worker.py@@ -6,7 +6,7 @@ from pipelex.cogt.exceptions import JudgmentCapabilityErrorfrom pipelex.cogt.inference.error_classification import extract_typesafe_metadatafrom pipelex.cogt.inference.error_render import InferenceErrorFamily, render_inference_errorfrom pipelex.cogt.judgment.judgment_job import JudgmentJob-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer+from pipelex.cogt.judgment.judgment_models import JudgmentOutcomefrom pipelex.cogt.judgment.judgment_worker_abstract import JudgmentWorkerAbstractfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory@@ -22,7 +22,9 @@ class TypesafeJudgmentWorker(JudgmentWorkerAbstract):One request carries the whole job, which is the reason the family's contract is batch-shaped:three questions over one state cost 506 input tokens together against 1138 apart, and answeredin 0.27 s against 0.66 s. The state is paid for once, so a caller that asks everything it wants- to know in one job is rewarded for it.+ to know in one job is rewarded for it. The state is the job's rendered prompt, sent as a string:+ the campaign's spike found a JSON object state and the same object as a string answered alike.+ This backend never refuses a question, so it returns an answer for every one."""def __init__(@@ -39,14 +41,15 @@ class TypesafeJudgmentWorker(JudgmentWorkerAbstract):async def _judge(self,judgment_job: JudgmentJob,- ) -> dict[str, JudgmentAnswer]:- if judgment_job.images or judgment_job.documents:- # System One judges a JSON state and takes no file, so a request built from this job would- # drop the files and the verdict would be given over less than was asked. The kit's spec- # says the model reads text alone, which refuses files before this; a spec that claims more- # is refused here rather than believed.+ ) -> dict[str, JudgmentOutcome]:+ judgment_prompt = judgment_job.prompt+ if judgment_prompt.images or judgment_prompt.documents:+ # System One judges text and takes no file, so a request built from this job would drop the+ # files and the verdict would be given over less than was asked. The kit's spec says the+ # model reads text alone, which refuses files before this; a spec that claims more is+ # refused here rather than believed.msg = (- f"TypeSafe judges a JSON state alone, and this judgment carries files for model '{self.inference_model.name}': "+ f"TypeSafe judges text alone, and this judgment's prompt presents files for model '{self.inference_model.name}': ""its spec must not declare that it reads images or documents.")raise JudgmentCapabilityError(msg)@@ -54,7 +57,7 @@ class TypesafeJudgmentWorker(JudgmentWorkerAbstract):try:# The SDK types its state as a recursive JSON alias that pyright cannot resolve to the end.response = await self._typesafe_client.system_one( # pyright: ignore[reportUnknownMemberType]- judgment_job.state,+ judgment_prompt.text,typesafe_questions,model=self.inference_model.model_id,)@@ -76,7 +79,8 @@ class TypesafeJudgmentWorker(JudgmentWorkerAbstract):self._log_request_id(response=response)self._warn_if_another_model_answered(response=response)self._record_usage(judgment_job=judgment_job, response=response)- return from_typesafe_response(questions=judgment_job.questions, response=response)+ answers = from_typesafe_response(questions=judgment_job.questions, response=response)+ return dict(answers)def _log_request_id(self, *, response: SystemOneResponse) -> None:"""Trace the provider's own request id, which it returns on success as well as on failure.diff --git a/pipelex/providers/typesafe/typesafe_translation.py b/pipelex/providers/typesafe/typesafe_translation.pyindex fccd526be..7c910d322 100644--- a/pipelex/providers/typesafe/typesafe_translation.py+++ b/pipelex/providers/typesafe/typesafe_translation.py@@ -2,6 +2,11 @@The family speaks yes/no, choice and rating; this vendor speaks *noul*, *choice* and *score*. Themapping is one-to-one, and it lives here alone — nothing outside this module knows a vendor word.+The evidence is the rendered prompt, which the worker sends as the vendor's state, a string.++**A level is one string on this wire.** A labelled level goes out as `label: description`, or as+whichever of the two it declares, and the vendor answers with the level's index, so the label never+needs reading back: the operator looks it up on the declared scale.**The vendor's bounds are enforced here, not in the blueprint.** A rating scale longer than thisAPI answers is legal MTHDS that this one backend refuses, exactly as the design's Part 5 says: the@@ -47,6 +52,7 @@ from pipelex.cogt.judgment.judgment_models import (JudgmentAnswer,JudgmentQuestion,RatingAnswer,+ RatingLevel,RatingQuestion,YesNoAnswer,YesNoQuestion,@@ -91,24 +97,35 @@ def to_typesafe_question(*, question_key: str, question: JudgmentQuestion) -> Ty),provider_metadata=None,)- return TypesafeScore(instructions=question.instructions, criteria=list(question.levels))+ return TypesafeScore(instructions=question.instructions, criteria=[_score_criterion(level=level) for level in question.levels])case _:assert_never(question)def _noul_criteria(*, question: YesNoQuestion) -> TypesafeNoulCriteria | None:- """The vendor's yes/no criteria, or nothing at all when the question declared neither side.+ """The vendor's yes/no criteria, both sides, or nothing at all when the question declared none.- Both sides are individually optional and an absent map is legal as long as the instructions are- there. A key the vendor does not know is accepted and silently ignored, so only the two names- it does know are ever sent.+ The family's question carries both sides or none, so a lone side never reaches this vendor. A key+ the vendor does not know is accepted and silently ignored, so only the two names it does know are+ ever sent."""- criteria: TypesafeNoulCriteria = {}- if question.yes_criterion is not None:- criteria["true"] = question.yes_criterion- if question.no_criterion is not None:- criteria["false"] = question.no_criterion- return criteria or None+ if question.criteria is None:+ return None+ return {"true": question.criteria.yes, "false": question.criteria.no}+++def _score_criterion(*, level: RatingLevel) -> str:+ """One level as the vendor's one string: `label: description` when both are declared, else the one that is."""+ if level.label is not None and level.description is not None:+ return f"{level.label}: {level.description}"+ if level.label is not None:+ return level.label+ if level.description is not None:+ return level.description+ # `RatingLevel`'s own validator refuses a level carrying neither, so a validated level never reaches+ # this; it is stated for the type checker, which cannot see a validator.+ msg = "A rating level carries neither a label nor a description"+ raise TypesafeQuestionUnsupportedError(msg, error_category=InferenceErrorCategory.CONTENT, provider_metadata=None)def from_typesafe_response(*, questions: dict[str, JudgmentQuestion], response: SystemOneResponse) -> dict[str, JudgmentAnswer]:diff --git a/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_catalog_review/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_catalog_review/bundle.mthdsindex 980006a8a..37a06899f 100644--- a/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_catalog_review/bundle.mthds+++ b/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_catalog_review/bundle.mthds@@ -50,6 +50,7 @@ description = "Judges whether a price looks too low to be true"inputs = { cheapest_price = "Number" }output = "YesNo"model = "@default-judgment"+prompt = "The cheapest price printed in a furniture catalog, in euros: $cheapest_price"question = "Does this price, in euros, look too low to be true for furniture?"criteria = { yes = "The price is below five euros", no = "The price is five euros or more" }threshold = 0.7diff --git a/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_judge_amount/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_judge_amount/bundle.mthdsindex b66351bae..217348cfb 100644--- a/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_judge_amount/bundle.mthds+++ b/pipelex/test_extras/mthds_corpus/entries/feature_binding_step_judge_amount/bundle.mthds@@ -25,6 +25,7 @@ description = "Judges whether an amount is high enough to need a second signaturinputs = { total_amount = "Number" }output = "YesNo"model = "@default-judgment"+prompt = "The total of an invoice, in euros: $total_amount"question = "Is this amount high enough to need a second signature?"criteria = { yes = "The amount is above five thousand euros", no = "The amount is five thousand euros or less" }threshold = 0.7diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_judge_roof_condition/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/operator_judge_roof_condition/bundle.mthdsnew file mode 100644index 000000000..22b01a76b--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_judge_roof_condition/bundle.mthds@@ -0,0 +1,20 @@+domain = "roof_inspection"+description = "Rating the condition of a roof from an inspector's site note"+main_pipe = "rate_roof_condition"++[pipe.rate_roof_condition]+type = "PipeJudge"+description = "Rates the condition of a roof on a labelled scale"+inputs = { site_note = "Text" }+output = "Rating"+model = "@default-judgment"+prompt = """+An inspector's note from a roof survey:+@site_note+"""+question = "What condition is this roof in?"+levels = [+ { label = "Sound", description = "No repair is needed before the next survey" },+ { label = "Worn", description = "Some tiles or flashing need repair within the year" },+ { label = "Failing", description = "Water is getting in, or will at the next storm" },+]diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_judge_roof_condition/entry.toml b/pipelex/test_extras/mthds_corpus/entries/operator_judge_roof_condition/entry.tomlnew file mode 100644index 000000000..ae59f84c1--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_judge_roof_condition/entry.toml@@ -0,0 +1,10 @@+name = "operator_judge_roof_condition"+description = "A rating of a roof's condition, on a scale whose levels carry labels, from an inspector's site note"++validity = "valid"+tier = "inference"+granularity = "focused"++covers = [+ "operator.judge",+]diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthdsindex 0779dbea2..9dc9aeb9b 100644--- a/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthds+++ b/pipelex/test_extras/mthds_corpus/entries/operator_judge_urgent_message/bundle.mthds@@ -3,11 +3,15 @@ description = "Judging whether an inbox message needs an answer today"main_pipe = "judge_is_urgent"[pipe.judge_is_urgent]-type = "PipeJudge"+type = "PipeJudge"description = "Judges whether a message needs an answer today"-inputs = { message = "Text" }-output = "YesNo"-model = "@default-judgment"-question = "Does this message need an answer today?"-criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }-threshold = 0.7+inputs = { message = "Text" }+output = "YesNo"+model = "@default-judgment"+prompt = """+A message from someone writing to the inbox:+@message+"""+question = "Does this message need an answer today?"+criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }+threshold = 0.7diff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.pyindex 483227133..2bf2efff6 100644--- a/pipelex/tools/misc/context_provider_abstract.py+++ b/pipelex/tools/misc/context_provider_abstract.py@@ -14,3 +14,7 @@ class ContextProviderAbstract(ABC):@abstractmethoddef generate_context(self) -> dict[str, Any]:pass++ @abstractmethod+ def is_variable_present(self, *, name: str) -> bool:+ """Whether the context holds a value under this top-level name, an alias included; an absent optional input does not."""diff --git a/subject_grants.toml b/subject_grants.tomlindex b2919c9bb..4b8a11c71 100644--- a/subject_grants.toml+++ b/subject_grants.toml@@ -3544,6 +3544,10 @@ rationale = "One post-processing pass of the schema pipeline, called as a verb oparam = "schema"rationale = "Schema-pipeline stage transforming the schema dict; single operand."+["pipelex/language/mthds_schema_generator.py::_constrain_rating_levels"]+param = "schema"+rationale = "One post-processing pass of the schema pipeline, called as a verb on the schema it rewrites, like its sibling passes."+["pipelex/language/mthds_schema_generator.py::_constrain_stored_names"]param = "schema"rationale = "One post-processing pass of the schema pipeline, called as a verb on the schema it rewrites, like its sibling passes."@@ -4248,15 +4252,15 @@ rationale = "Verb-object: formats the exception; context stays keyword."param = "pipe_run_params"rationale = "Derivation from the run params; context stays keyword."-["pipelex/pipe_operators/llm/template_document_analyzer.py::TemplateDocumentAnalyzer._resolve_concept"]+["pipelex/pipe_operators/shared/template_document_analyzer.py::TemplateDocumentAnalyzer._resolve_concept"]param = "concept_ref_or_code"rationale = "Verb-object: resolves the ref-or-code; single operand."-["pipelex/pipe_operators/llm/template_document_analyzer.py::TemplateDocumentAnalyzer._resolve_variable_type"]+["pipelex/pipe_operators/shared/template_document_analyzer.py::TemplateDocumentAnalyzer._resolve_variable_type"]param = "var_path"rationale = "Query accessor keyed by the variable path; context stays keyword."-["pipelex/pipe_operators/llm/template_document_analyzer.py::TemplateDocumentAnalyzer.analyze_template_for_documents"]+["pipelex/pipe_operators/shared/template_document_analyzer.py::TemplateDocumentAnalyzer.analyze_template_for_documents"]param = "template_source"rationale = "Verb-object: analyzes the template source; context stays keyword."diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex b4a0e7941..9e13da2cf 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -152,6 +152,7 @@ JudgmentHandleNotFoundError | Judgment handle not found | https://docs.pipelex.cJudgmentJobFailureError | Judgment job failure | https://docs.pipelex.com/latest/errors/judgment-job-failure-error/JudgmentModelMissingError | Judgment model missing | https://docs.pipelex.com/latest/errors/judgment-model-missing-error/JudgmentModelNotFoundError | Judgment model not found | https://docs.pipelex.com/latest/errors/judgment-model-not-found-error/+JudgmentRefusedError | Judgment refused | https://docs.pipelex.com/latest/errors/judgment-refused-error/KitError | Kit | https://docs.pipelex.com/latest/errors/kit-error/KitIndexLoadingError | Kit index loading | https://docs.pipelex.com/latest/errors/kit-index-loading-error/LLMAssignmentError | LLM assignment | https://docs.pipelex.com/latest/errors/llm-assignment-error/diff --git a/tests/data/judgment/wire/typesafe/bare_questions/case.json b/tests/data/judgment/wire/typesafe/bare_questions/case.jsonnew file mode 100644index 000000000..77cd102f2--- /dev/null+++ b/tests/data/judgment/wire/typesafe/bare_questions/case.json@@ -0,0 +1,4 @@+{+ "summary": "Each kind with nothing optional: a yes/no without criteria, options without descriptions, levels with labels alone.",+ "source": "Recorded live against TypeSafe for this phase, for exactly this vendor request."+}diff --git a/tests/data/judgment/wire/typesafe/bare_questions/request.json b/tests/data/judgment/wire/typesafe/bare_questions/request.jsonnew file mode 100644index 000000000..2897d4513--- /dev/null+++ b/tests/data/judgment/wire/typesafe/bare_questions/request.json@@ -0,0 +1,36 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nI was charged twice for order 4417 this morning. Please refund one of the two charges.\n</message>"+ },+ "questions": {+ "refund": {+ "kind": "yes_no",+ "instructions": "Does the customer ask for a refund?"+ },+ "tone": {+ "kind": "choice",+ "instructions": "What is the tone of the message?",+ "options": {+ "calm": null,+ "angry": null,+ "anxious": null+ }+ },+ "priority": {+ "kind": "rating",+ "instructions": "What priority should this message get?",+ "levels": [+ {+ "label": "Low"+ },+ {+ "label": "Normal"+ },+ {+ "label": "High"+ }+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/bare_questions/response.json b/tests/data/judgment/wire/typesafe/bare_questions/response.jsonnew file mode 100644index 000000000..c14960c32--- /dev/null+++ b/tests/data/judgment/wire/typesafe/bare_questions/response.json@@ -0,0 +1,36 @@+{+ "status": 200,+ "body": {+ "answers": {+ "refund": {+ "kind": "yes_no",+ "probability": 0.99+ },+ "tone": {+ "kind": "choice",+ "choice": "calm",+ "confidence": 0.98,+ "probabilities": {+ "calm": 0.99,+ "angry": 0.01,+ "anxious": 0.0+ }+ },+ "priority": {+ "kind": "rating",+ "level": 2,+ "position": 1.71,+ "confidence": 0.58,+ "probabilities": {+ "0": 0.0,+ "1": 0.28,+ "2": 0.72+ }+ }+ },+ "usage": {+ "input_tokens": 388,+ "output_tokens": 73+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/bare_questions/vendor_request.json b/tests/data/judgment/wire/typesafe/bare_questions/vendor_request.jsonnew file mode 100644index 000000000..bf178d85b--- /dev/null+++ b/tests/data/judgment/wire/typesafe/bare_questions/vendor_request.json@@ -0,0 +1,28 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nI was charged twice for order 4417 this morning. Please refund one of the two charges.\n</message>",+ "questions": {+ "refund": {+ "type": "noul",+ "instructions": "Does the customer ask for a refund?"+ },+ "tone": {+ "type": "choice",+ "instructions": "What is the tone of the message?",+ "criteria": {+ "calm": null,+ "angry": null,+ "anxious": null+ }+ },+ "priority": {+ "type": "score",+ "instructions": "What priority should this message get?",+ "criteria": [+ "Low",+ "Normal",+ "High"+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/bare_questions/vendor_response.json b/tests/data/judgment/wire/typesafe/bare_questions/vendor_response.jsonnew file mode 100644index 000000000..f6b69da6a--- /dev/null+++ b/tests/data/judgment/wire/typesafe/bare_questions/vendor_response.json@@ -0,0 +1,41 @@+{+ "status": 200,+ "body": {+ "model": "jev-1.13.0",+ "answers": {+ "refund": {+ "type": "noul",+ "noul": 0.99+ },+ "tone": {+ "type": "choice",+ "choice": "calm",+ "confidence": 0.98,+ "probabilities": {+ "calm": 0.99,+ "angry": 0.01,+ "anxious": 0.0+ }+ },+ "priority": {+ "type": "score",+ "score": 1.71,+ "confidence": 0.58,+ "legend": {+ "0": "Low",+ "1": "Normal",+ "2": "High"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.28,+ "2": 0.72+ }+ }+ },+ "usage": {+ "input_tokens": 388,+ "output_tokens": 73+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/checkout_down/case.json b/tests/data/judgment/wire/typesafe/checkout_down/case.jsonnew file mode 100644index 000000000..7e3e86d88--- /dev/null+++ b/tests/data/judgment/wire/typesafe/checkout_down/case.json@@ -0,0 +1,4 @@+{+ "summary": "One question of each kind, the yes/no with both criteria, the rating's levels labelled and described.",+ "source": "Jev's answer recorded by the Decisions spike on 2026-10-08 (wip/judgment-family/spike-decisions/responses/12_side_by_side/jev_checkout_down.json), for exactly this vendor request."+}diff --git a/tests/data/judgment/wire/typesafe/checkout_down/request.json b/tests/data/judgment/wire/typesafe/checkout_down/request.jsonnew file mode 100644index 000000000..4b33be85f--- /dev/null+++ b/tests/data/judgment/wire/typesafe/checkout_down/request.json@@ -0,0 +1,44 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "urgent": {+ "kind": "yes_no",+ "instructions": "Is the message urgent?",+ "criteria": {+ "yes": "The customer needs an answer today",+ "no": "The message can wait for the normal queue"+ }+ },+ "team": {+ "kind": "choice",+ "instructions": "Which team should handle this message?",+ "options": {+ "payments": "Charges, invoices, payment processing failures",+ "shipping": "Delivery status, delays, lost packages",+ "accounts": "Sign-in, passwords, account settings",+ "other": "None of the above"+ }+ },+ "severity": {+ "kind": "rating",+ "instructions": "How severe is the reported issue?",+ "levels": [+ {+ "label": "Cosmetic",+ "description": "Appearance only; no lost functionality"+ },+ {+ "label": "Workaround available",+ "description": "A task fails, but another way works"+ },+ {+ "label": "Fully blocked",+ "description": "A task fails with no workaround"+ }+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/checkout_down/response.json b/tests/data/judgment/wire/typesafe/checkout_down/response.jsonnew file mode 100644index 000000000..6f00d556c--- /dev/null+++ b/tests/data/judgment/wire/typesafe/checkout_down/response.json@@ -0,0 +1,37 @@+{+ "status": 200,+ "body": {+ "answers": {+ "urgent": {+ "kind": "yes_no",+ "probability": 0.98+ },+ "team": {+ "kind": "choice",+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "shipping": 0.0,+ "payments": 1.0+ }+ },+ "severity": {+ "kind": "rating",+ "level": 2,+ "position": 2.0,+ "confidence": 1.0,+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ }+ }+ },+ "usage": {+ "input_tokens": 518,+ "output_tokens": 75+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/checkout_down/vendor_request.json b/tests/data/judgment/wire/typesafe/checkout_down/vendor_request.jsonnew file mode 100644index 000000000..f5d0b8127--- /dev/null+++ b/tests/data/judgment/wire/typesafe/checkout_down/vendor_request.json@@ -0,0 +1,33 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>",+ "questions": {+ "urgent": {+ "type": "noul",+ "instructions": "Is the message urgent?",+ "criteria": {+ "true": "The customer needs an answer today",+ "false": "The message can wait for the normal queue"+ }+ },+ "team": {+ "type": "choice",+ "instructions": "Which team should handle this message?",+ "criteria": {+ "payments": "Charges, invoices, payment processing failures",+ "shipping": "Delivery status, delays, lost packages",+ "accounts": "Sign-in, passwords, account settings",+ "other": "None of the above"+ }+ },+ "severity": {+ "type": "score",+ "instructions": "How severe is the reported issue?",+ "criteria": [+ "Cosmetic: Appearance only; no lost functionality",+ "Workaround available: A task fails, but another way works",+ "Fully blocked: A task fails with no workaround"+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/checkout_down/vendor_response.json b/tests/data/judgment/wire/typesafe/checkout_down/vendor_response.jsonnew file mode 100644index 000000000..a078e5980--- /dev/null+++ b/tests/data/judgment/wire/typesafe/checkout_down/vendor_response.json@@ -0,0 +1,42 @@+{+ "status": 200,+ "body": {+ "model": "jev-1.13.0",+ "answers": {+ "urgent": {+ "type": "noul",+ "noul": 0.98+ },+ "team": {+ "type": "choice",+ "choice": "payments",+ "confidence": 1.0,+ "probabilities": {+ "accounts": 0.0,+ "other": 0.0,+ "shipping": 0.0,+ "payments": 1.0+ }+ },+ "severity": {+ "type": "score",+ "score": 2.0,+ "confidence": 1.0,+ "legend": {+ "0": "Cosmetic: Appearance only; no lost functionality",+ "1": "Workaround available: A task fails, but another way works",+ "2": "Fully blocked: A task fails with no workaround"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 1.0+ }+ }+ },+ "usage": {+ "input_tokens": 518,+ "output_tokens": 75+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/context_too_long/case.json b/tests/data/judgment/wire/typesafe/context_too_long/case.jsonnew file mode 100644index 000000000..0c8404ae7--- /dev/null+++ b/tests/data/judgment/wire/typesafe/context_too_long/case.json@@ -0,0 +1,4 @@+{+ "summary": "Evidence over the model's context limit: TypeSafe's max_tokens_exceeded envelope, a 400 with no message.",+ "source": "The envelope the phase 1 spike recorded on 2026-09-19 (04b_state_200k_chars_error) for a 200,000-character state. It does not depend on the request, so it is paired with a short one rather than carrying the evidence that provoked it."+}diff --git a/tests/data/judgment/wire/typesafe/context_too_long/request.json b/tests/data/judgment/wire/typesafe/context_too_long/request.jsonnew file mode 100644index 000000000..1d2bb14d1--- /dev/null+++ b/tests/data/judgment/wire/typesafe/context_too_long/request.json@@ -0,0 +1,16 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "urgent": {+ "kind": "yes_no",+ "instructions": "Is the message urgent?",+ "criteria": {+ "yes": "The customer needs an answer today",+ "no": "The message can wait for the normal queue"+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/context_too_long/response.json b/tests/data/judgment/wire/typesafe/context_too_long/response.jsonnew file mode 100644index 000000000..069fadc7d--- /dev/null+++ b/tests/data/judgment/wire/typesafe/context_too_long/response.json@@ -0,0 +1,7 @@+{+ "status": 400,+ "error": {+ "type": "invalid_request_error",+ "code": "typesafe_context_too_long"+ }+}diff --git a/tests/data/judgment/wire/typesafe/context_too_long/vendor_request.json b/tests/data/judgment/wire/typesafe/context_too_long/vendor_request.jsonnew file mode 100644index 000000000..e98be6e29--- /dev/null+++ b/tests/data/judgment/wire/typesafe/context_too_long/vendor_request.json@@ -0,0 +1,14 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>",+ "questions": {+ "urgent": {+ "type": "noul",+ "instructions": "Is the message urgent?",+ "criteria": {+ "true": "The customer needs an answer today",+ "false": "The message can wait for the normal queue"+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/context_too_long/vendor_response.json b/tests/data/judgment/wire/typesafe/context_too_long/vendor_response.jsonnew file mode 100644index 000000000..4fa30d13d--- /dev/null+++ b/tests/data/judgment/wire/typesafe/context_too_long/vendor_response.json@@ -0,0 +1,8 @@+{+ "status": 400,+ "body": {+ "detail": {+ "error_type": "max_tokens_exceeded"+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/eleven_levels_refused/case.json b/tests/data/judgment/wire/typesafe/eleven_levels_refused/case.jsonnew file mode 100644index 000000000..6a2a14745--- /dev/null+++ b/tests/data/judgment/wire/typesafe/eleven_levels_refused/case.json@@ -0,0 +1,4 @@+{+ "summary": "One level past TypeSafe's cap is refused by the gateway before any call; there is no vendor request.",+ "source": "No vendor exchange. TypeSafe itself answers 400 'Too many score levels. Must have at most 10 levels.' (phase 1 spike, 04_score_eleven_levels_error)."+}diff --git a/tests/data/judgment/wire/typesafe/eleven_levels_refused/request.json b/tests/data/judgment/wire/typesafe/eleven_levels_refused/request.jsonnew file mode 100644index 000000000..c9eb91c72--- /dev/null+++ b/tests/data/judgment/wire/typesafe/eleven_levels_refused/request.json@@ -0,0 +1,47 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "severity": {+ "kind": "rating",+ "instructions": "How severe is the reported issue?",+ "levels": [+ {+ "description": "Severity level 0"+ },+ {+ "description": "Severity level 1"+ },+ {+ "description": "Severity level 2"+ },+ {+ "description": "Severity level 3"+ },+ {+ "description": "Severity level 4"+ },+ {+ "description": "Severity level 5"+ },+ {+ "description": "Severity level 6"+ },+ {+ "description": "Severity level 7"+ },+ {+ "description": "Severity level 8"+ },+ {+ "description": "Severity level 9"+ },+ {+ "description": "Severity level 10"+ }+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/eleven_levels_refused/response.json b/tests/data/judgment/wire/typesafe/eleven_levels_refused/response.jsonnew file mode 100644index 000000000..0f4d10627--- /dev/null+++ b/tests/data/judgment/wire/typesafe/eleven_levels_refused/response.json@@ -0,0 +1,7 @@+{+ "status": 400,+ "error": {+ "type": "invalid_request_error",+ "code": "typesafe_too_many_levels"+ }+}diff --git a/tests/data/judgment/wire/typesafe/password_reset/case.json b/tests/data/judgment/wire/typesafe/password_reset/case.jsonnew file mode 100644index 000000000..ccd0d7256--- /dev/null+++ b/tests/data/judgment/wire/typesafe/password_reset/case.json@@ -0,0 +1,4 @@+{+ "summary": "The same three questions on evidence Jev is unsure about: a spread distribution, the rating's level its most probable one.",+ "source": "Jev's answer recorded by the Decisions spike on 2026-10-08 (wip/judgment-family/spike-decisions/responses/12_side_by_side/jev_password_reset.json), for exactly this vendor request."+}diff --git a/tests/data/judgment/wire/typesafe/password_reset/request.json b/tests/data/judgment/wire/typesafe/password_reset/request.jsonnew file mode 100644index 000000000..00f3f5b98--- /dev/null+++ b/tests/data/judgment/wire/typesafe/password_reset/request.json@@ -0,0 +1,44 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nI asked for a password reset an hour ago and the email never came. I'll try again tomorrow if it doesn't show up.\n</message>"+ },+ "questions": {+ "urgent": {+ "kind": "yes_no",+ "instructions": "Is the message urgent?",+ "criteria": {+ "yes": "The customer needs an answer today",+ "no": "The message can wait for the normal queue"+ }+ },+ "team": {+ "kind": "choice",+ "instructions": "Which team should handle this message?",+ "options": {+ "payments": "Charges, invoices, payment processing failures",+ "shipping": "Delivery status, delays, lost packages",+ "accounts": "Sign-in, passwords, account settings",+ "other": "None of the above"+ }+ },+ "severity": {+ "kind": "rating",+ "instructions": "How severe is the reported issue?",+ "levels": [+ {+ "label": "Cosmetic",+ "description": "Appearance only; no lost functionality"+ },+ {+ "label": "Workaround available",+ "description": "A task fails, but another way works"+ },+ {+ "label": "Fully blocked",+ "description": "A task fails with no workaround"+ }+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/password_reset/response.json b/tests/data/judgment/wire/typesafe/password_reset/response.jsonnew file mode 100644index 000000000..8b55aaedb--- /dev/null+++ b/tests/data/judgment/wire/typesafe/password_reset/response.json@@ -0,0 +1,37 @@+{+ "status": 200,+ "body": {+ "answers": {+ "urgent": {+ "kind": "yes_no",+ "probability": 0.21+ },+ "team": {+ "kind": "choice",+ "choice": "accounts",+ "confidence": 1.0,+ "probabilities": {+ "payments": 0.0,+ "shipping": 0.0,+ "accounts": 1.0,+ "other": 0.0+ }+ },+ "severity": {+ "kind": "rating",+ "level": 1,+ "position": 1.42,+ "confidence": 0.36,+ "probabilities": {+ "0": 0.0,+ "1": 0.58,+ "2": 0.42+ }+ }+ },+ "usage": {+ "input_tokens": 503,+ "output_tokens": 75+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/password_reset/vendor_request.json b/tests/data/judgment/wire/typesafe/password_reset/vendor_request.jsonnew file mode 100644index 000000000..87ddabd67--- /dev/null+++ b/tests/data/judgment/wire/typesafe/password_reset/vendor_request.json@@ -0,0 +1,33 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nI asked for a password reset an hour ago and the email never came. I'll try again tomorrow if it doesn't show up.\n</message>",+ "questions": {+ "urgent": {+ "type": "noul",+ "instructions": "Is the message urgent?",+ "criteria": {+ "true": "The customer needs an answer today",+ "false": "The message can wait for the normal queue"+ }+ },+ "team": {+ "type": "choice",+ "instructions": "Which team should handle this message?",+ "criteria": {+ "payments": "Charges, invoices, payment processing failures",+ "shipping": "Delivery status, delays, lost packages",+ "accounts": "Sign-in, passwords, account settings",+ "other": "None of the above"+ }+ },+ "severity": {+ "type": "score",+ "instructions": "How severe is the reported issue?",+ "criteria": [+ "Cosmetic: Appearance only; no lost functionality",+ "Workaround available: A task fails, but another way works",+ "Fully blocked: A task fails with no workaround"+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/password_reset/vendor_response.json b/tests/data/judgment/wire/typesafe/password_reset/vendor_response.jsonnew file mode 100644index 000000000..9b9cd7de9--- /dev/null+++ b/tests/data/judgment/wire/typesafe/password_reset/vendor_response.json@@ -0,0 +1,42 @@+{+ "status": 200,+ "body": {+ "model": "jev-1.13.0",+ "answers": {+ "urgent": {+ "type": "noul",+ "noul": 0.21+ },+ "team": {+ "type": "choice",+ "choice": "accounts",+ "confidence": 1.0,+ "probabilities": {+ "payments": 0.0,+ "shipping": 0.0,+ "accounts": 1.0,+ "other": 0.0+ }+ },+ "severity": {+ "type": "score",+ "score": 1.42,+ "confidence": 0.36,+ "legend": {+ "0": "Cosmetic: Appearance only; no lost functionality",+ "1": "Workaround available: A task fails, but another way works",+ "2": "Fully blocked: A task fails with no workaround"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.58,+ "2": 0.42+ }+ }+ },+ "usage": {+ "input_tokens": 503,+ "output_tokens": 75+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/case.json b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/case.jsonnew file mode 100644index 000000000..84939ede9--- /dev/null+++ b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/case.json@@ -0,0 +1,4 @@+{+ "summary": "A rating at TypeSafe's ten-level cap, its levels described and unlabelled: the level is the most probable one, not the rounded position.",+ "source": "Recorded live against TypeSafe for this phase, for exactly this vendor request."+}diff --git a/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/request.json b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/request.jsonnew file mode 100644index 000000000..e103ec7c5--- /dev/null+++ b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/request.json@@ -0,0 +1,44 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "severity": {+ "kind": "rating",+ "instructions": "How severe is the reported issue?",+ "levels": [+ {+ "description": "Severity level 0"+ },+ {+ "description": "Severity level 1"+ },+ {+ "description": "Severity level 2"+ },+ {+ "description": "Severity level 3"+ },+ {+ "description": "Severity level 4"+ },+ {+ "description": "Severity level 5"+ },+ {+ "description": "Severity level 6"+ },+ {+ "description": "Severity level 7"+ },+ {+ "description": "Severity level 8"+ },+ {+ "description": "Severity level 9"+ }+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/response.json b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/response.jsonnew file mode 100644index 000000000..2ac5a0221--- /dev/null+++ b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/response.json@@ -0,0 +1,29 @@+{+ "status": 200,+ "body": {+ "answers": {+ "severity": {+ "kind": "rating",+ "level": 9,+ "position": 8.43,+ "confidence": 0.8,+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 0.0,+ "3": 0.0,+ "4": 0.01,+ "5": 0.03,+ "6": 0.01,+ "7": 0.05,+ "8": 0.2,+ "9": 0.7+ }+ }+ },+ "usage": {+ "input_tokens": 424,+ "output_tokens": 17+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/vendor_request.json b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/vendor_request.jsonnew file mode 100644index 000000000..5ad44c1af--- /dev/null+++ b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/vendor_request.json@@ -0,0 +1,22 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>",+ "questions": {+ "severity": {+ "type": "score",+ "instructions": "How severe is the reported issue?",+ "criteria": [+ "Severity level 0",+ "Severity level 1",+ "Severity level 2",+ "Severity level 3",+ "Severity level 4",+ "Severity level 5",+ "Severity level 6",+ "Severity level 7",+ "Severity level 8",+ "Severity level 9"+ ]+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/vendor_response.json b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/vendor_response.jsonnew file mode 100644index 000000000..8689488a3--- /dev/null+++ b/tests/data/judgment/wire/typesafe/ten_unlabelled_levels/vendor_response.json@@ -0,0 +1,41 @@+{+ "status": 200,+ "body": {+ "model": "jev-1.13.0",+ "answers": {+ "severity": {+ "type": "score",+ "score": 8.43,+ "confidence": 0.8,+ "legend": {+ "0": "Severity level 0",+ "1": "Severity level 1",+ "2": "Severity level 2",+ "3": "Severity level 3",+ "4": "Severity level 4",+ "5": "Severity level 5",+ "6": "Severity level 6",+ "7": "Severity level 7",+ "8": "Severity level 8",+ "9": "Severity level 9"+ },+ "probabilities": {+ "0": 0.0,+ "1": 0.0,+ "2": 0.0,+ "3": 0.0,+ "4": 0.01,+ "5": 0.03,+ "6": 0.01,+ "7": 0.05,+ "8": 0.2,+ "9": 0.7+ }+ }+ },+ "usage": {+ "input_tokens": 424,+ "output_tokens": 17+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/unknown_model/case.json b/tests/data/judgment/wire/typesafe/unknown_model/case.jsonnew file mode 100644index 000000000..467fc8232--- /dev/null+++ b/tests/data/judgment/wire/typesafe/unknown_model/case.json@@ -0,0 +1,4 @@+{+ "summary": "A provider model TypeSafe does not serve: its api_usage_error envelope, a 400 naming the model.",+ "source": "Recorded live against TypeSafe for this phase, for exactly this vendor request."+}diff --git a/tests/data/judgment/wire/typesafe/unknown_model/request.json b/tests/data/judgment/wire/typesafe/unknown_model/request.jsonnew file mode 100644index 000000000..3beb4a632--- /dev/null+++ b/tests/data/judgment/wire/typesafe/unknown_model/request.json@@ -0,0 +1,16 @@+{+ "model": "jev-does-not-exist",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "urgent": {+ "kind": "yes_no",+ "instructions": "Is the message urgent?",+ "criteria": {+ "yes": "The customer needs an answer today",+ "no": "The message can wait for the normal queue"+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/unknown_model/response.json b/tests/data/judgment/wire/typesafe/unknown_model/response.jsonnew file mode 100644index 000000000..e435c3de2--- /dev/null+++ b/tests/data/judgment/wire/typesafe/unknown_model/response.json@@ -0,0 +1,7 @@+{+ "status": 400,+ "error": {+ "type": "invalid_request_error",+ "code": "typesafe_unknown_model"+ }+}diff --git a/tests/data/judgment/wire/typesafe/unknown_model/vendor_request.json b/tests/data/judgment/wire/typesafe/unknown_model/vendor_request.jsonnew file mode 100644index 000000000..1fc636279--- /dev/null+++ b/tests/data/judgment/wire/typesafe/unknown_model/vendor_request.json@@ -0,0 +1,14 @@+{+ "model": "jev-does-not-exist",+ "state": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>",+ "questions": {+ "urgent": {+ "type": "noul",+ "instructions": "Is the message urgent?",+ "criteria": {+ "true": "The customer needs an answer today",+ "false": "The message can wait for the normal queue"+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/unknown_model/vendor_response.json b/tests/data/judgment/wire/typesafe/unknown_model/vendor_response.jsonnew file mode 100644index 000000000..4db4b8a91--- /dev/null+++ b/tests/data/judgment/wire/typesafe/unknown_model/vendor_response.json@@ -0,0 +1,9 @@+{+ "status": 400,+ "body": {+ "detail": {+ "error_type": "api_usage_error",+ "message": "Unknown model: jev-does-not-exist"+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/vendor_refuses_question/case.json b/tests/data/judgment/wire/typesafe/vendor_refuses_question/case.jsonnew file mode 100644index 000000000..4f2d2585c--- /dev/null+++ b/tests/data/judgment/wire/typesafe/vendor_refuses_question/case.json@@ -0,0 +1,4 @@+{+ "summary": "TypeSafe refusing a question's shape answers a bare sentence: a 400 that is the gateway's defect, not the caller's.",+ "source": "The envelope the phase 1 spike recorded on 2026-09-19 (04_choice_empty_error) for a choice with no option, which the route refuses before any call. It is paired with a request the route admits to replay the bare-sentence shape, which no request the route admits is known to provoke."+}diff --git a/tests/data/judgment/wire/typesafe/vendor_refuses_question/request.json b/tests/data/judgment/wire/typesafe/vendor_refuses_question/request.jsonnew file mode 100644index 000000000..67f040433--- /dev/null+++ b/tests/data/judgment/wire/typesafe/vendor_refuses_question/request.json@@ -0,0 +1,15 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "q": {+ "kind": "choice",+ "instructions": "Which team?",+ "options": {+ "payments": null+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/vendor_refuses_question/response.json b/tests/data/judgment/wire/typesafe/vendor_refuses_question/response.jsonnew file mode 100644index 000000000..65f2c058b--- /dev/null+++ b/tests/data/judgment/wire/typesafe/vendor_refuses_question/response.json@@ -0,0 +1,7 @@+{+ "status": 400,+ "error": {+ "type": "invalid_request_error",+ "code": "typesafe_invalid_question"+ }+}diff --git a/tests/data/judgment/wire/typesafe/vendor_refuses_question/vendor_request.json b/tests/data/judgment/wire/typesafe/vendor_refuses_question/vendor_request.jsonnew file mode 100644index 000000000..0c84decc6--- /dev/null+++ b/tests/data/judgment/wire/typesafe/vendor_refuses_question/vendor_request.json@@ -0,0 +1,13 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "Which team?",+ "criteria": {+ "payments": null+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/vendor_refuses_question/vendor_response.json b/tests/data/judgment/wire/typesafe/vendor_refuses_question/vendor_response.jsonnew file mode 100644index 000000000..6d5922dc3--- /dev/null+++ b/tests/data/judgment/wire/typesafe/vendor_refuses_question/vendor_response.json@@ -0,0 +1,6 @@+{+ "status": 400,+ "body": {+ "detail": "Choice question must have at least one choice: q"+ }+}diff --git a/tests/data/judgment/wire/typesafe/wrong_key/case.json b/tests/data/judgment/wire/typesafe/wrong_key/case.jsonnew file mode 100644index 000000000..04946638a--- /dev/null+++ b/tests/data/judgment/wire/typesafe/wrong_key/case.json@@ -0,0 +1,4 @@+{+ "summary": "A key TypeSafe does not know: its authentication envelope, a 401.",+ "source": "Recorded live against TypeSafe for this phase with the placeholder key 'ts-not-a-real-key', for exactly this vendor request."+}diff --git a/tests/data/judgment/wire/typesafe/wrong_key/request.json b/tests/data/judgment/wire/typesafe/wrong_key/request.jsonnew file mode 100644index 000000000..1d2bb14d1--- /dev/null+++ b/tests/data/judgment/wire/typesafe/wrong_key/request.json@@ -0,0 +1,16 @@+{+ "model": "jev-1.13.0",+ "prompt": {+ "text": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ },+ "questions": {+ "urgent": {+ "kind": "yes_no",+ "instructions": "Is the message urgent?",+ "criteria": {+ "yes": "The customer needs an answer today",+ "no": "The message can wait for the normal queue"+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/wrong_key/response.json b/tests/data/judgment/wire/typesafe/wrong_key/response.jsonnew file mode 100644index 000000000..4fbe6c56f--- /dev/null+++ b/tests/data/judgment/wire/typesafe/wrong_key/response.json@@ -0,0 +1,7 @@+{+ "status": 401,+ "error": {+ "type": "authentication_error",+ "code": "typesafe_authentication_error"+ }+}diff --git a/tests/data/judgment/wire/typesafe/wrong_key/vendor_request.json b/tests/data/judgment/wire/typesafe/wrong_key/vendor_request.jsonnew file mode 100644index 000000000..e98be6e29--- /dev/null+++ b/tests/data/judgment/wire/typesafe/wrong_key/vendor_request.json@@ -0,0 +1,14 @@+{+ "model": "jev-1.13.0",+ "state": "A message from a customer:\n<message>\nOur production checkout has been returning 500s for every card payment since 09:14 UTC. Nothing is going through. I have three customers on the phone right now. Please help.\n</message>",+ "questions": {+ "urgent": {+ "type": "noul",+ "instructions": "Is the message urgent?",+ "criteria": {+ "true": "The customer needs an answer today",+ "false": "The message can wait for the normal queue"+ }+ }+ }+}diff --git a/tests/data/judgment/wire/typesafe/wrong_key/vendor_response.json b/tests/data/judgment/wire/typesafe/wrong_key/vendor_response.jsonnew file mode 100644index 000000000..90b3d0d8f--- /dev/null+++ b/tests/data/judgment/wire/typesafe/wrong_key/vendor_response.json@@ -0,0 +1,9 @@+{+ "status": 401,+ "body": {+ "detail": {+ "error_type": "authentication_error",+ "message": "Cannot authenticate with the server. Please check your API key and try again."+ }+ }+}diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthdsindex 16d8c385d..e6a121357 100644--- a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthds+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/pipe_judge.mthds@@ -1,15 +1,19 @@domain = "pipe_judge_e2e"-description = "Test PipeJudge with one question of each kind"+description = "Test PipeJudge with one question of each kind, each asked about the evidence its prompt presents"[pipe.judge_urgent_e2e]-type = "PipeJudge"+type = "PipeJudge"description = "Judge whether a message needs an answer today"-inputs = { message = "Text" }-output = "YesNo"-model = "@default-judgment"-question = "Does this message need an answer today?"-criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }-threshold = 0.7+inputs = { message = "Text" }+output = "YesNo"+model = "@default-judgment"+prompt = """+A message from someone writing to the inbox:+@message+"""+question = "Does this message need an answer today?"+criteria = { yes = "Something is broken or blocked for the sender right now", no = "The sender can wait for a reply" }+threshold = 0.7[pipe.route_ticket_e2e]type = "PipeSequence"@@ -52,13 +56,17 @@ output = "Text"template = "Sent to the front desk: $ticket"[pipe.judge_team_e2e]-type = "PipeJudge"+type = "PipeJudge"description = "Judge which team should handle a support ticket"-inputs = { ticket = "Text" }-output = "Choice"-model = "@default-judgment"-question = "Which team should handle this support ticket?"-options = { billing = "Charges, invoices, refunds and subscriptions", technical = "Errors, outages and features that do not work", other = "" }+inputs = { ticket = "Text" }+output = "Choice"+model = "@default-judgment"+prompt = """+A support ticket, as the customer wrote it:+@ticket+"""+question = "Which team should handle this support ticket?"+options = { billing = "Charges, invoices, refunds and subscriptions", technical = "Errors, outages and features that do not work", other = "" }[pipe.judge_severity_e2e]type = "PipeJudge"@@ -66,9 +74,13 @@ description = "Rate how severe a bug report is"inputs = { report = "Text" }output = "Rating"model = "@default-judgment"+prompt = """+A bug report filed by a user:+@report+"""question = "How severe is the bug this report describes?"levels = [- "Cosmetic: nothing stops working",- "Degraded: a workaround exists",- "Blocking: users cannot do their work at all",+ { label = "Cosmetic", description = "nothing stops working" },+ { label = "Degraded", description = "a workaround exists" },+ { label = "Blocking", description = "users cannot do their work at all" },]diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.pyindex 52a7bef31..7c58f178d 100644--- a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.py+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_data.py@@ -4,8 +4,24 @@ from typing import ClassVarclass PipeJudgeTestCases:- """Inputs pinned with the verdicts a live judgment returns for them."""+ """Inputs pinned with the evidence and the question each judge sends, and the verdicts a live judgment returns for them."""URGENT_MESSAGE: ClassVar[str] = "Our production database has been down for an hour and no customer can log in. Please help."BILLING_TICKET: ClassVar[str] = "I was charged twice for my subscription this month and I would like a refund for the second charge."BLOCKING_REPORT: ClassVar[str] = "Since this morning's release, clicking Save on any invoice crashes the app, so nobody can issue an invoice."++ # The evidence each prompt renders over its input, and the question each judge asks about it, exactly as sent.+ URGENT_EVIDENCE: ClassVar[str] = f"A message from someone writing to the inbox:\n<message>\n{URGENT_MESSAGE}\n</message>"+ URGENT_QUESTION: ClassVar[str] = "Does this message need an answer today?"+ BILLING_EVIDENCE: ClassVar[str] = f"A support ticket, as the customer wrote it:\n<ticket>\n{BILLING_TICKET}\n</ticket>"+ BILLING_QUESTION: ClassVar[str] = "Which team should handle this support ticket?"+ BLOCKING_EVIDENCE: ClassVar[str] = f"A bug report filed by a user:\n<report>\n{BLOCKING_REPORT}\n</report>"+ BLOCKING_QUESTION: ClassVar[str] = "How severe is the bug this report describes?"++ SEVERITY_LABELS: ClassVar[list[str]] = ["Cosmetic", "Degraded", "Blocking"]++ # The verdicts a live judgment returns, measured on the deck's default judgment model.+ EXPECTED_URGENT_PROBABILITY: ClassVar[float] = 0.98+ EXPECTED_BILLING_PROBABILITY: ClassVar[float] = 1.0+ EXPECTED_BLOCKING_LEVEL: ClassVar[int] = 2+ EXPECTED_BLOCKING_POSITION: ClassVar[float] = 2.0diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.pyindex db91d33be..6805fd3a3 100644--- a/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.py+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_judge/test_pipe_judge.py@@ -1,8 +1,9 @@-"""E2E tests for the PipeJudge operator: one question of each kind, run on the deck's default judgment model."""+"""E2E tests for PipeJudge: one question of each kind, asked about the evidence its prompt presents, on the default judgment model."""import pytestfrom pipelex import pretty_print+from pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.pipeline.runner import PipelexMTHDSProtocolfrom pipelex.system.pipe_run_mode import PipeRunMode@@ -14,6 +15,17 @@ LIBRARY_DIRS = ["tests/e2e/pipelex/pipes/pipe_operators/pipe_judge"]PROBABILITY_TOLERANCE = 0.05+def _assert_sent(*, pipe_output: PipeOutput, pipe_code: str, evidence: str, question: str) -> None:+ """The judge sent exactly this evidence and this question, as its execution data records them."""+ graph_spec = pipe_output.graph_spec+ assert graph_spec is not None+ (judge_node,) = [node for node in graph_spec.nodes if node.pipe_code == pipe_code]+ assert judge_node.execution_data["rendered_prompt"] == evidence+ assert judge_node.execution_data["rendered_question"] == question+ assert judge_node.execution_data["nb_prompt_images"] == 0+ assert judge_node.execution_data["nb_prompt_documents"] == 0++@pytest.mark.judgment@pytest.mark.inference@pytest.mark.dry_runnable@@ -28,13 +40,19 @@ class TestPipeJudge:pipe_output = pipeline_response.pipe_outputassert pipe_output is not None+ _assert_sent(+ pipe_output=pipe_output,+ pipe_code="judge_urgent_e2e",+ evidence=PipeJudgeTestCases.URGENT_EVIDENCE,+ question=PipeJudgeTestCases.URGENT_QUESTION,+ )verdict = pipe_output.main_stuff_as_yes_noif pipe_run_mode.is_live:pretty_print(verdict, title="Yes/no judgment")# The declared threshold of 0.7 decides the verdict from the probability.assert verdict.yes_no is Trueassert verdict.probability is not None- assert abs(verdict.probability - 0.98) <= PROBABILITY_TOLERANCE+ assert abs(verdict.probability - PipeJudgeTestCases.EXPECTED_URGENT_PROBABILITY) <= PROBABILITY_TOLERANCEasync def test_choice_question(self, pipe_run_mode: PipeRunMode) -> None:runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)@@ -45,13 +63,19 @@ class TestPipeJudge:pipe_output = pipeline_response.pipe_outputassert pipe_output is not None+ _assert_sent(+ pipe_output=pipe_output,+ pipe_code="judge_team_e2e",+ evidence=PipeJudgeTestCases.BILLING_EVIDENCE,+ question=PipeJudgeTestCases.BILLING_QUESTION,+ )verdict = pipe_output.main_stuff_as_choiceif pipe_run_mode.is_live:pretty_print(verdict, title="Choice judgment")assert verdict.choice == "billing"assert verdict.probabilities is not Noneassert set(verdict.probabilities) == {"billing", "technical", "other"}- assert abs(verdict.probabilities["billing"] - 1.0) <= PROBABILITY_TOLERANCE+ assert abs(verdict.probabilities["billing"] - PipeJudgeTestCases.EXPECTED_BILLING_PROBABILITY) <= PROBABILITY_TOLERANCEasync def test_choice_routes_a_condition(self, pipe_run_mode: PipeRunMode) -> None:runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)@@ -66,7 +90,7 @@ class TestPipeJudge:if pipe_run_mode.is_live:assert acknowledgement.text.startswith("Sent to billing:")- async def test_rating_question(self, pipe_run_mode: PipeRunMode) -> None:+ async def test_rating_question_on_a_labelled_scale(self, pipe_run_mode: PipeRunMode) -> None:runner = PipelexMTHDSProtocol(library_dirs=LIBRARY_DIRS, pipe_run_mode=pipe_run_mode)pipeline_response = await runner.execute(pipe_code="judge_severity_e2e",@@ -75,11 +99,20 @@ class TestPipeJudge:pipe_output = pipeline_response.pipe_outputassert pipe_output is not None+ _assert_sent(+ pipe_output=pipe_output,+ pipe_code="judge_severity_e2e",+ evidence=PipeJudgeTestCases.BLOCKING_EVIDENCE,+ question=PipeJudgeTestCases.BLOCKING_QUESTION,+ )verdict = pipe_output.main_stuff_as_rating+ # Dry or live, the verdict carries the label the scale declares for the level it lands on.+ assert verdict.label == PipeJudgeTestCases.SEVERITY_LABELS[verdict.level]if pipe_run_mode.is_live:pretty_print(verdict, title="Rating judgment")- assert verdict.level == 2+ assert verdict.level == PipeJudgeTestCases.EXPECTED_BLOCKING_LEVEL+ assert verdict.label == "Blocking"assert verdict.probabilities is not Noneassert set(verdict.probabilities) == {"0", "1", "2"}assert verdict.position is not None- assert abs(verdict.position - 2.0) <= PROBABILITY_TOLERANCE * 2+ assert abs(verdict.position - PipeJudgeTestCases.EXPECTED_BLOCKING_POSITION) <= PROBABILITY_TOLERANCE * 2diff --git a/tests/integration/pipelex/cogt/config_coverage/test_cc_judgment.py b/tests/integration/pipelex/cogt/config_coverage/test_cc_judgment.pyindex 951c5d36b..5204cd378 100644--- a/tests/integration/pipelex/cogt/config_coverage/test_cc_judgment.py+++ b/tests/integration/pipelex/cogt/config_coverage/test_cc_judgment.py@@ -21,7 +21,7 @@ class TestConfigCoverageJudgment:inference_model = get_model_deck().get_required_inference_model(model_handle=judgment_combo.handle, model_type=ModelType.JUDGMENT)worker = JudgmentWorkerFactory.make_judgment_worker(inference_model)judgment_job = JudgmentJobFactory.make_judgment_job(- state=JudgmentTestCases.STATE,+ prompt=JudgmentTestCases.PROMPT,questions={"is_urgent": JudgmentTestCases.IS_URGENT},judgment_setting=JudgmentSetting(model=judgment_combo.handle),job_metadata=job_metadata,diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.pyindex 315cbdce9..c15136935 100644--- a/tests/integration/pipelex/cogt/test_data.py+++ b/tests/integration/pipelex/cogt/test_data.py@@ -6,18 +6,24 @@ from pydantic import BaseModel, Field, field_validatorfrom pydantic.dataclasses import dataclassfrom pipelex.cogt.image.prompt_image import PromptImageUri-from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentKind, JudgmentQuestion, JudgmentState, RatingQuestion, YesNoQuestion+from pipelex.cogt.judgment.judgment_models import (+ ChoiceQuestion,+ JudgmentKind,+ JudgmentPrompt,+ JudgmentQuestion,+ RatingLevel,+ RatingQuestion,+ YesNoCriteria,+ YesNoQuestion,+)from pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.core.stuffs.choice_content import ChoiceContentfrom pipelex.core.stuffs.date_content import DateContent-from pipelex.core.stuffs.html_content import HtmlContent-from pipelex.core.stuffs.json_content import JSONContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.structured_content import StructuredContentfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.text_content import TextContent-from pipelex.core.stuffs.time_content import TimeContentfrom pipelex.core.stuffs.yes_no_content import YesNoContentfrom tests.cases import ImageTestCasesfrom tests.cases.documents import DocumentTestCases@@ -301,29 +307,29 @@ class LLMReasoningTestCases:class JudgmentTestCases:- """Live judgment material, and every string in it is pinned on purpose.+ """Live judgment evidence and questions, and every string in them is pinned on purpose.The spike measured what moves an answer: repeating the same request moved a borderline case by0.01, while rewording a single instruction moved it from 0.11 to 0.30. So the wording here is- part of the test, copied from the spike's recorded three-question probe, and a failure after an- innocent-looking edit to it is telling the truth. Verdicts are asserted exactly; a probability- gets ``PROBABILITY_MARGIN``, fifteen times the noise the spike observed.+ part of the test, and it is the shared wire fixture `checkout_down`'s, recorded against the live+ API for exactly this evidence and these questions. A failure after an innocent-looking edit to it+ is telling the truth. Verdicts are asserted exactly; a probability gets ``PROBABILITY_MARGIN``,+ fifteen times the noise the spike observed."""PROBABILITY_MARGIN = 0.05- STATE: ClassVar[dict[str, Any]] = {- "message": (+ PROMPT = JudgmentPrompt(+ text=(+ "A message from a customer:\n<message>\n""Our production checkout has been returning 500s for every card payment since 09:14 UTC. "- "Nothing is going through. I have three customers on the phone right now. Please help."- ),- "channel": "support_email",- }+ "Nothing is going through. I have three customers on the phone right now. Please help.\n</message>"+ )+ )IS_URGENT = YesNoQuestion(instructions="Is the message urgent?",- yes_criterion="Needs attention now",- no_criterion="Can wait until the next working day",+ criteria=YesNoCriteria(yes="The customer needs an answer today", no="The message can wait for the normal queue"),)TEAM = ChoiceQuestion(instructions="Which team should handle this message?",@@ -337,9 +343,9 @@ class JudgmentTestCases:SEVERITY = RatingQuestion(instructions="How severe is the reported issue?",levels=[- "Cosmetic; no impact on functionality",- "A feature is degraded, but a workaround exists",- "Blocking issue; no workaround exists",+ RatingLevel(label="Cosmetic", description="Appearance only; no lost functionality"),+ RatingLevel(label="Workaround available", description="A task fails, but another way works"),+ RatingLevel(label="Fully blocked", description="A task fails with no workaround"),],)@@ -356,7 +362,7 @@ class JudgmentTestCases:("rating", SEVERITY, JudgmentKind.RATING),]- # What the spike recorded for this material, bit-identical across eight repeats.+ # What the recording holds for this material.EXPECTED_IS_URGENT_PROBABILITY = 0.98EXPECTED_TEAM = "payments"EXPECTED_SEVERITY_LEVEL = 2@@ -374,80 +380,66 @@ class JudgmentTestOrder(StructuredContent):@dataclass(frozen=True)-class JudgmentStateShapeCase:- """One question over two states that differ only in the value under test, and the exact state each side must produce."""+class JudgmentPromptShapeCase:+ """One evidence template and one question over two memories that differ only in the value under test."""+ prompt: strquestion: YesNoQuestionyes_inputs: dict[str, StuffContent]no_inputs: dict[str, StuffContent]- expected_yes_state: JudgmentState- expected_no_state: JudgmentStatedef _judgment_order(*, charges: list[JudgmentTestCharge], tracking_number: str | None = None) -> JudgmentTestOrder:return JudgmentTestOrder(order_id="A-104", charges=charges, tracking_number=tracking_number)-class JudgmentStateShapeCases:- """Every JSON shape a PipeJudge puts in its judgment material, as yes/no pairs for the live judging model.+class JudgmentPromptShapeCases:+ """Each way an evidence prompt renders an input, as yes/no pairs for the live judging model.+ The evidence is the rendered prompt, so what the model reads is what the template makes of each+ input: a text block under its name, a number or a date inline, a structure or a list as a block.Like `JudgmentTestCases`, the wording is part of the test: a question that makes the model do- arithmetic over the material measures its arithmetic, not whether it read the shape. Asking- "Does the refund cover every duplicate charge?" over three charges landed at 0.66 where the- simpler question below lands at 0.99.+ arithmetic over the evidence measures its arithmetic, not whether it read what was rendered."""- # A verdict must clear the middle by this much on its side: a shape the model half-read would hover near 0.5.+ # A verdict must clear the middle by this much on its side: evidence the model half-read would hover near 0.5.CLEAR_YES = 0.8CLEAR_NO = 0.2CAPTURED = JudgmentTestCharge(amount_usd=49, status="captured")- CAPTURED_STATE: ClassVar[dict[str, Any]] = {"amount_usd": 49, "status": "captured"}- SHAPE_CASES: ClassVar[list[tuple[str, JudgmentStateShapeCase]]] = [ # topic, case+ SHAPE_CASES: ClassVar[list[tuple[str, JudgmentPromptShapeCase]]] = [ # topic, case(- "text_as_string",- JudgmentStateShapeCase(+ "text_block",+ JudgmentPromptShapeCase(+ prompt="A message from a customer:\n@message",question=YesNoQuestion(instructions="Does the customer ask for a refund?"),yes_inputs={"message": TextContent(text="I was charged twice for my order. Please refund the duplicate charge.")},no_inputs={"message": TextContent(text="Thanks, my parcel arrived this morning and everything is fine.")},- expected_yes_state={"message": "I was charged twice for my order. Please refund the duplicate charge."},- expected_no_state={"message": "Thanks, my parcel arrived this morning and everything is fine."},),),(- "integer_as_number",- JudgmentStateShapeCase(+ "number_inline",+ JudgmentPromptShapeCase(+ prompt="The order total is $order_total_usd USD.",question=YesNoQuestion(instructions="Is the order total above 1000 USD?"),yes_inputs={"order_total_usd": NumberContent(number=1240)},no_inputs={"order_total_usd": NumberContent(number=40)},- expected_yes_state={"order_total_usd": 1240},- expected_no_state={"order_total_usd": 40},),),(- "float_as_number",- JudgmentStateShapeCase(- question=YesNoQuestion(instructions="Is the account balance negative?"),- yes_inputs={"balance_usd": NumberContent(number=-12.75)},- no_inputs={"balance_usd": NumberContent(number=312.5)},- expected_yes_state={"balance_usd": -12.75},- expected_no_state={"balance_usd": 312.5},- ),- ),- (- "yes_no_as_boolean",- JudgmentStateShapeCase(+ "yes_no_inline",+ JudgmentPromptShapeCase(+ prompt="Refund approved: $refund_approved",question=YesNoQuestion(instructions="Has the refund been approved?"),yes_inputs={"refund_approved": YesNoContent(yes_no=True)},no_inputs={"refund_approved": YesNoContent(yes_no=False)},- expected_yes_state={"refund_approved": True},- expected_no_state={"refund_approved": False},),),(- "dates_as_iso_strings",- JudgmentStateShapeCase(+ "dates_inline",+ JudgmentPromptShapeCase(+ prompt="Ordered on $ordered_on, delivered on $delivered_on.",question=YesNoQuestion(instructions="Was the parcel delivered more than a week after the order was placed?"),yes_inputs={"ordered_on": DateContent(date=datetime.date(2026, 9, 1)),@@ -457,54 +449,31 @@ class JudgmentStateShapeCases:"ordered_on": DateContent(date=datetime.date(2026, 9, 1)),"delivered_on": DateContent(date=datetime.date(2026, 9, 3)),},- expected_yes_state={"ordered_on": "2026-09-01", "delivered_on": "2026-09-20"},- expected_no_state={"ordered_on": "2026-09-01", "delivered_on": "2026-09-03"},- ),- ),- (- "time_as_iso_string",- JudgmentStateShapeCase(- question=YesNoQuestion(instructions="Did the call take place in the middle of the night?"),- yes_inputs={"call_time": TimeContent(time=datetime.time(3, 15))},- no_inputs={"call_time": TimeContent(time=datetime.time(14, 30))},- expected_yes_state={"call_time": "03:15:00"},- expected_no_state={"call_time": "14:30:00"},),),(- "json_as_nested_object",- JudgmentStateShapeCase(- question=YesNoQuestion(instructions="Did the fraud check flag this payment?"),- yes_inputs={"payment_checks": JSONContent(json_obj={"fraud": {"flagged": True, "score": 0.97}, "3ds": "passed"})},- no_inputs={"payment_checks": JSONContent(json_obj={"fraud": {"flagged": False, "score": 0.02}, "3ds": "passed"})},- expected_yes_state={"payment_checks": {"fraud": {"flagged": True, "score": 0.97}, "3ds": "passed"}},- expected_no_state={"payment_checks": {"fraud": {"flagged": False, "score": 0.02}, "3ds": "passed"}},- ),- ),- (- "structure_as_nested_object_with_array",- JudgmentStateShapeCase(+ "structure_block",+ JudgmentPromptShapeCase(+ prompt="@order",question=YesNoQuestion(instructions="Was the customer charged more than once for this order?"),yes_inputs={"order": _judgment_order(charges=[CAPTURED, CAPTURED])},no_inputs={"order": _judgment_order(charges=[CAPTURED])},- expected_yes_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}},- expected_no_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE]}},),),(- # The no side is the point: the unset member is left out of the object, never sent as a null.- "absent_member_left_out",- JudgmentStateShapeCase(+ # The no side is the point: an unset member renders as nothing the model could read as a value.+ "absent_member",+ JudgmentPromptShapeCase(+ prompt="@order",question=YesNoQuestion(instructions="Has a tracking number been assigned to the order?"),yes_inputs={"order": _judgment_order(charges=[CAPTURED], tracking_number="1Z999AA10123456784")},no_inputs={"order": _judgment_order(charges=[CAPTURED])},- expected_yes_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE], "tracking_number": "1Z999AA10123456784"}},- expected_no_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE]}},),),(- "list_of_texts_as_array",- JudgmentStateShapeCase(+ "list_block",+ JudgmentPromptShapeCase(+ prompt="The customer's messages, oldest first:\n@messages",question=YesNoQuestion(instructions="Did the customer threaten to cancel their subscription?"),yes_inputs={"messages": ListContent(items=[TextContent(text="This is the third time this happens."), TextContent(text="Fix it or I cancel.")])@@ -514,88 +483,24 @@ class JudgmentStateShapeCases:items=[TextContent(text="This is the third time this happens."), TextContent(text="Thanks for the quick fix.")])},- expected_yes_state={"messages": ["This is the third time this happens.", "Fix it or I cancel."]},- expected_no_state={"messages": ["This is the third time this happens.", "Thanks for the quick fix."]},- ),- ),- (- "list_of_structures_as_array_of_objects",- JudgmentStateShapeCase(- question=YesNoQuestion(instructions="Was any of these charges refunded?"),- yes_inputs={"charges": ListContent(items=[CAPTURED, JudgmentTestCharge(amount_usd=49, status="refunded")])},- no_inputs={"charges": ListContent(items=[CAPTURED, CAPTURED])},- expected_yes_state={"charges": [CAPTURED_STATE, {"amount_usd": 49, "status": "refunded"}]},- expected_no_state={"charges": [CAPTURED_STATE, CAPTURED_STATE]},),),(- "earlier_verdict_as_object",- JudgmentStateShapeCase(+ "earlier_verdict_inline",+ JudgmentPromptShapeCase(+ prompt="The message was routed to the $routing team.",question=YesNoQuestion(instructions="Was the message routed to the payments team?"),yes_inputs={"routing": ChoiceContent(choice="payments", confidence=0.93)},no_inputs={"routing": ChoiceContent(choice="shipping", confidence=0.93)},- expected_yes_state={"routing": {"choice": "payments", "confidence": 0.93}},- expected_no_state={"routing": {"choice": "shipping", "confidence": 0.93}},- ),- ),- (- "html_as_wrapped_object",- JudgmentStateShapeCase(- question=YesNoQuestion(instructions="Does the page ask the visitor for a password?"),- yes_inputs={"page": HtmlContent(inner_html='<form><label>Password</label><input type="password" name="pw"></form>')},- no_inputs={"page": HtmlContent(inner_html="<p>Our offices are closed on public holidays.</p>")},- expected_yes_state={"page": {"inner_html": '<form><label>Password</label><input type="password" name="pw"></form>'}},- expected_no_state={"page": {"inner_html": "<p>Our offices are closed on public holidays.</p>"}},),),("several_inputs_related",- JudgmentStateShapeCase(+ JudgmentPromptShapeCase(+ prompt="@order\nThe refund issued: $refund_usd USD",question=YesNoQuestion(instructions="Is the refund amount equal to the amount of one charge on the order?"),yes_inputs={"order": _judgment_order(charges=[CAPTURED, CAPTURED]), "refund_usd": NumberContent(number=49)},no_inputs={"order": _judgment_order(charges=[CAPTURED, CAPTURED]), "refund_usd": NumberContent(number=5)},- expected_yes_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}, "refund_usd": 49},- expected_no_state={"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}, "refund_usd": 5},),),]-- # The same fact in the typed shape a PipeJudge sends, and in the stringified or flattened shape it could have- # sent instead. Each pair must land on the same side: the typed shape loses nothing to the alternative.- EQUIVALENT_SHAPES: ClassVar[list[tuple[str, YesNoQuestion, JudgmentState, JudgmentState, bool]]] = [- # topic, question, typed_state, alternative_state, expected_yes- (- "boolean_true_vs_word",- YesNoQuestion(instructions="Has the refund been approved?"),- {"refund_approved": True},- {"refund_approved": "yes"},- True,- ),- (- "boolean_false_vs_word",- YesNoQuestion(instructions="Has the refund been approved?"),- {"refund_approved": False},- {"refund_approved": "no"},- False,- ),- (- "number_vs_numeric_string",- YesNoQuestion(instructions="Is the order total above 1000 USD?"),- {"order_total_usd": 1240},- {"order_total_usd": "1240"},- True,- ),- (- "nested_vs_flattened_strings",- YesNoQuestion(instructions="Was the customer charged more than once for this order?"),- {"order": {"order_id": "A-104", "charges": [CAPTURED_STATE, CAPTURED_STATE]}},- {- "order.order_id": "A-104",- "order.charges.0.amount_usd": "49",- "order.charges.0.status": "captured",- "order.charges.1.amount_usd": "49",- "order.charges.1.status": "captured",- },- True,- ),- ]diff --git a/tests/integration/pipelex/cogt/test_handle_per_model_type_end_to_end.py b/tests/integration/pipelex/cogt/test_handle_per_model_type_end_to_end.pyindex 669db66b6..6dc03e818 100644--- a/tests/integration/pipelex/cogt/test_handle_per_model_type_end_to_end.py+++ b/tests/integration/pipelex/cogt/test_handle_per_model_type_end_to_end.py@@ -93,7 +93,8 @@ description = "Judge the reply"inputs = {{ reply = "Text" }}output = "YesNo"model = "{TWIN_HANDLE}"-question = "Is $reply polite?"+prompt = "@reply"+question = "Is the reply polite?""""diff --git a/tests/integration/pipelex/cogt/test_judgment.py b/tests/integration/pipelex/cogt/test_judgment.pyindex b6b71967b..c8bebaa08 100644--- a/tests/integration/pipelex/cogt/test_judgment.py+++ b/tests/integration/pipelex/cogt/test_judgment.py@@ -6,9 +6,10 @@ from pipelex.cogt.judgment.judgment_job import JudgmentJobfrom pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactoryfrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,- JudgmentAnswer,JudgmentKind,+ JudgmentOutcome,JudgmentQuestion,+ JudgmentRefusal,RatingAnswer,YesNoAnswer,)@@ -29,14 +30,14 @@ def _inference_model(judgment_combo: ModelCombo) -> InferenceModelSpec:def _job(*, judgment_combo: ModelCombo, job_metadata: JobMetadata, questions: dict[str, JudgmentQuestion]) -> JudgmentJob:return JudgmentJobFactory.make_judgment_job(- state=JudgmentTestCases.STATE,+ prompt=JudgmentTestCases.PROMPT,questions=questions,judgment_setting=JudgmentSetting(model=judgment_combo.handle),job_metadata=job_metadata,)-def _assert_expected_verdict(*, answer: JudgmentAnswer) -> None:+def _assert_expected_verdict(*, answer: JudgmentOutcome) -> None:"""The verdict exactly, the probability within the margin the spike's stability measure sets."""match answer:case YesNoAnswer():@@ -51,6 +52,8 @@ def _assert_expected_verdict(*, answer: JudgmentAnswer) -> None:assert answer.level == JudgmentTestCases.EXPECTED_SEVERITY_LEVELassert answer.probabilities is not Noneassert set(answer.probabilities) == set(range(len(JudgmentTestCases.SEVERITY.levels)))+ case JudgmentRefusal():+ pytest.fail("The unambiguous case is answered, never declined")@pytest.mark.judgmentdiff --git a/tests/integration/pipelex/cogt/test_judgment_prompt_shapes.py b/tests/integration/pipelex/cogt/test_judgment_prompt_shapes.pynew file mode 100644index 000000000..15d309b04--- /dev/null+++ b/tests/integration/pipelex/cogt/test_judgment_prompt_shapes.py@@ -0,0 +1,89 @@+"""Each way an evidence prompt renders an input, proven live against the judging model.++A PipeJudge's evidence is its rendered prompt, so the judging model reads each input as the template+renders it: a text block under its name, a number, a yes/no or a date inline, a structure or a list as+a block. Each case is a pair of memories that differ only in the value under test, rendered by the same+user-prompt assembly the operator calls, so the evidence sent is the evidence a run sends. The verdict+must land clearly on the yes side for one and clearly on the no side for the other: a model that+ignored or misread the rendering could not flip with the value.+"""++import pytest++from pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory+from pipelex.cogt.judgment.judgment_models import JudgmentPrompt, YesNoAnswer, YesNoQuestion+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.cogt.judgment.judgment_worker_factory import JudgmentWorkerFactory+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.kernel.prompt_assembly import UserPromptContent, assemble_user_prompt+from pipelex.kernel.templating_style_ops import resolve_templating_style+from pipelex.runtime_hub import get_model_deck+from pipelex.system.job_metadata import JobMetadata+from pipelex.tools.jinja2.template_category import TemplateCategory+from tests.integration.pipelex.cogt.test_data import JudgmentPromptShapeCase, JudgmentPromptShapeCases+from tests.integration.pipelex.fixtures.model_combo import ModelCombo+++async def _evidence(*, prompt: str, inputs: dict[str, StuffContent]) -> JudgmentPrompt:+ """The evidence a PipeJudge with this prompt would send over these inputs, assembled by the operator's own assembly."""+ anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)+ stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in inputs.items()]+ memory = WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)+ assembled = await assemble_user_prompt(+ prompt_content=UserPromptContent(template=TemplateBlueprint(template=prompt, category=TemplateCategory.LLM_PROMPT)),+ context_provider=memory,+ templating_style=resolve_templating_style(authored=None),+ )+ assert not assembled.images+ assert not assembled.documents+ return JudgmentPrompt(text=assembled.text)+++async def _probability_of_yes(*, judgment_combo: ModelCombo, job_metadata: JobMetadata, evidence: JudgmentPrompt, question: YesNoQuestion) -> float:+ inference_model = get_model_deck().get_required_inference_model(model_handle=judgment_combo.handle, model_type=ModelType.JUDGMENT)+ worker = JudgmentWorkerFactory.make_judgment_worker(inference_model)+ job = JudgmentJobFactory.make_judgment_job(+ prompt=evidence,+ questions={"question": question},+ judgment_setting=JudgmentSetting(model=judgment_combo.handle),+ job_metadata=job_metadata,+ )+ answer = (await worker.judge(job))["question"]+ assert isinstance(answer, YesNoAnswer)+ assert answer.probability is not None+ return answer.probability++[email protected](loop_scope="class")+class TestJudgmentPromptShapes:+ @pytest.mark.parametrize(("topic", "case"), JudgmentPromptShapeCases.SHAPE_CASES)+ async def test_the_verdict_follows_the_value_the_prompt_renders(+ self,+ judgment_combo: ModelCombo,+ job_metadata: JobMetadata,+ topic: str,+ case: JudgmentPromptShapeCase,+ ) -> None:+ """Both renderings differ only in the value under test, and the verdict flips with it."""+ yes_evidence = await _evidence(prompt=case.prompt, inputs=case.yes_inputs)+ no_evidence = await _evidence(prompt=case.prompt, inputs=case.no_inputs)+ assert yes_evidence.text != no_evidence.text++ yes_probability = await _probability_of_yes(+ judgment_combo=judgment_combo, job_metadata=job_metadata, evidence=yes_evidence, question=case.question+ )+ no_probability = await _probability_of_yes(+ judgment_combo=judgment_combo, job_metadata=job_metadata, evidence=no_evidence, question=case.question+ )+ print(f"{topic}: yes side {yes_probability:.3f} over {yes_evidence.text!r}, no side {no_probability:.3f} over {no_evidence.text!r}")++ assert yes_probability >= JudgmentPromptShapeCases.CLEAR_YES, f"The yes-side evidence was not read as yes: {yes_probability:.3f}"+ assert no_probability <= JudgmentPromptShapeCases.CLEAR_NO, f"The no-side evidence was not read as no: {no_probability:.3f}"diff --git a/tests/integration/pipelex/cogt/test_judgment_state_shapes.py b/tests/integration/pipelex/cogt/test_judgment_state_shapes.pydeleted file mode 100644index 0bba2b430..000000000--- a/tests/integration/pipelex/cogt/test_judgment_state_shapes.py+++ /dev/null@@ -1,110 +0,0 @@-"""Every JSON shape a PipeJudge puts in its judgment state, proven live against the judging model.--A PipeJudge sends its inputs as one JSON object keyed by input name, each value read off its content's-class: a string, a number, a boolean, an ISO date or time, an array, or a nested object without its-absent members. Nothing is flattened and nothing is stringified. These tests prove the judging model-reads each of those shapes for what it says, rather than assuming it from the vendor's schema.--Each case is a pair of states that differ only in the value under test, built by the same-`build_judgment_material` the operator calls, so the state sent is the state a run sends. The verdict-must land clearly on the yes side for one and clearly on the no side for the other: a model that-ignored or misread the shape could not flip with the value.-"""--import pytest--from pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory-from pipelex.cogt.judgment.judgment_models import JudgmentState, YesNoAnswer, YesNoQuestion-from pipelex.cogt.judgment.judgment_setting import JudgmentSetting-from pipelex.cogt.judgment.judgment_worker_factory import JudgmentWorkerFactory-from pipelex.cogt.model_backends.model_type import ModelType-from pipelex.core.concepts.concept_factory import ConceptFactory-from pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory-from pipelex.core.stuffs.stuff_content import StuffContent-from pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.kernel.judgment_ops import build_judgment_material-from pipelex.runtime_hub import get_model_deck-from pipelex.system.job_metadata import JobMetadata-from tests.integration.pipelex.cogt.test_data import JudgmentStateShapeCase, JudgmentStateShapeCases-from tests.integration.pipelex.fixtures.model_combo import ModelCombo---def _state(inputs: dict[str, StuffContent]) -> JudgmentState:- """The state a PipeJudge declaring exactly these inputs would send, built by the operator's own material builder."""- anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)- stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in inputs.items()]- memory = WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)- state, images, documents = build_judgment_material(memory=memory, input_names=list(inputs))- assert not images- assert not documents- return state---async def _probability_of_yes(*, judgment_combo: ModelCombo, job_metadata: JobMetadata, state: JudgmentState, question: YesNoQuestion) -> float:- inference_model = get_model_deck().get_required_inference_model(model_handle=judgment_combo.handle, model_type=ModelType.JUDGMENT)- worker = JudgmentWorkerFactory.make_judgment_worker(inference_model)- job = JudgmentJobFactory.make_judgment_job(- state=state,- questions={"question": question},- judgment_setting=JudgmentSetting(model=judgment_combo.handle),- job_metadata=job_metadata,- )- answer = (await worker.judge(job))["question"]- assert isinstance(answer, YesNoAnswer)- assert answer.probability is not None- return answer.probability---def _assert_clear_verdict(*, label: str, probability: float, expected_yes: bool) -> None:- if expected_yes:- assert probability >= JudgmentStateShapeCases.CLEAR_YES, f"The {label} state was not read as yes: {probability:.3f}"- else:- assert probability <= JudgmentStateShapeCases.CLEAR_NO, f"The {label} state was not read as no: {probability:.3f}"--[email protected](loop_scope="class")-class TestJudgmentStateShapes:- @pytest.mark.parametrize(("topic", "case"), JudgmentStateShapeCases.SHAPE_CASES)- async def test_the_verdict_follows_the_value_in_each_shape(- self,- judgment_combo: ModelCombo,- job_metadata: JobMetadata,- topic: str,- case: JudgmentStateShapeCase,- ) -> None:- """Both states a PipeJudge sends have the expected shape, and the verdict flips with the value inside them."""- yes_state = _state(case.yes_inputs)- no_state = _state(case.no_inputs)- assert yes_state == case.expected_yes_state- assert no_state == case.expected_no_state-- yes_probability = await _probability_of_yes(judgment_combo=judgment_combo, job_metadata=job_metadata, state=yes_state, question=case.question)- no_probability = await _probability_of_yes(judgment_combo=judgment_combo, job_metadata=job_metadata, state=no_state, question=case.question)- print(f"{topic}: yes side {yes_probability:.3f}, no side {no_probability:.3f}")-- _assert_clear_verdict(label="yes-side", probability=yes_probability, expected_yes=True)- _assert_clear_verdict(label="no-side", probability=no_probability, expected_yes=False)-- @pytest.mark.parametrize(("topic", "question", "typed_state", "alternative_state", "expected_yes"), JudgmentStateShapeCases.EQUIVALENT_SHAPES)- async def test_the_typed_shape_reads_like_its_alternative(- self,- judgment_combo: ModelCombo,- job_metadata: JobMetadata,- topic: str,- question: YesNoQuestion,- typed_state: JudgmentState,- alternative_state: JudgmentState,- expected_yes: bool,- ) -> None:- """The typed shape and its stringified or flattened alternative give the same clear verdict."""- typed_probability = await _probability_of_yes(judgment_combo=judgment_combo, job_metadata=job_metadata, state=typed_state, question=question)- alternative_probability = await _probability_of_yes(- judgment_combo=judgment_combo, job_metadata=job_metadata, state=alternative_state, question=question- )- print(f"{topic}: typed {typed_probability:.3f}, alternative {alternative_probability:.3f}")-- _assert_clear_verdict(label="typed", probability=typed_probability, expected_yes=expected_yes)- _assert_clear_verdict(label="alternative", probability=alternative_probability, expected_yes=expected_yes)diff --git a/tests/integration/pipelex/error_handling/bundles/native_judge.mthds b/tests/integration/pipelex/error_handling/bundles/native_judge.mthdsindex 3191a2874..0cad8ee82 100644--- a/tests/integration/pipelex/error_handling/bundles/native_judge.mthds+++ b/tests/integration/pipelex/error_handling/bundles/native_judge.mthds@@ -1,9 +1,10 @@domain = "native_judge_test"-description = "Single PipeJudge step over a fixed question (no dynamic classes)"+description = "Single PipeJudge step over a fixed evidence and a fixed question (no dynamic classes)"[pipe.native_judge]type = "PipeJudge"description = "Judge whether a fixed claim holds"output = "YesNo"model = "jev-1.13.0"-question = "Is the sky blue on a clear day?"+prompt = "A weather report: the sky over Brest is cloudless this afternoon."+question = "Is the sky blue over Brest this afternoon?"diff --git a/tests/integration/pipelex/pipeline/test_file_input_consumers.py b/tests/integration/pipelex/pipeline/test_file_input_consumers.pyindex 06b92ad89..ffac65ee0 100644--- a/tests/integration/pipelex/pipeline/test_file_input_consumers.py+++ b/tests/integration/pipelex/pipeline/test_file_input_consumers.py@@ -26,6 +26,7 @@ from pipelex.system.registries.func_registry import func_registryPDF_ONLY_MODEL = "pypdfium2-extract-pdf"DOCLING_MODEL = "docling-extract-text"+TEXT_ONLY_LLM = "gemini-2.5-flash"DOCLING_FORMATS = frozenset({"pdf", "docx", "pptx", "xlsx", "html", "md", "csv", "txt", "vtt", "eml", "image"})_EXTRACTORS_MTHDS = f"""@@ -458,8 +459,8 @@ prompt = "Compare these transcripts: $transcripts, given their summaries: $summaassert consumer.covers(file_input_path=("transcripts", 1))assert not consumer.covers(file_input_path=("notes", 1))- def test_a_pipe_judge_consumes_its_document_inputs(self, load_empty_library: Callable[[], str], mocker: MockerFixture):- """Every input is material to judge, so a document input is sent as a file whether or not the question names it."""+ def test_a_pipe_judge_consumes_the_documents_its_prompt_references(self, load_empty_library: Callable[[], str], mocker: MockerFixture):+ """The prompt presents the documents it references by variable path, a single document and a list alike, as a PipeLLM's does."""model_deck = get_model_deck()judgment_model = model_deck.judgment_aliases["default-judgment"]judgment_specs = model_deck.inference_models.root[ModelType.JUDGMENT]@@ -468,20 +469,74 @@ prompt = "Compare these transcripts: $transcripts, given their summaries: $summapipes = """[pipe.main]type = "PipeJudge"-description = "Judge whether the claims are complete"-inputs = { claims = "Document[]", note = "Text", photo = "Image" }+description = "Judge whether a claim is complete"+inputs = { form = "Document", scans = "Document[]", note = "Text", photo = "Image" }output = "YesNo"model = "@default-judgment"-question = "Are these claims complete, given $note?"+prompt = "The signed claim form: $form, the scans attached to it: $scans, and a photo of the damage: $photo"+question = "Is this claim complete, given $note?""""consumers = _consumers(load_empty_library=load_empty_library, domain="fic_judge", pipes=pipes)- assert set(consumers) == {"claims"}- (consumer,) = consumers["claims"]- assert consumer.kind == FileConsumerKind.JUDGMENT_DOCUMENT- assert consumer.model == judgment_model- assert consumer.readable_formats == frozenset({"pdf"})- assert consumer.covers(file_input_path=("claims", 0))+ assert set(consumers) == {"form", "scans"}+ assert _summary(consumers["form"]) == [("main", "form", False)]+ assert _summary(consumers["scans"]) == [("main", "scans", False)]+ for consumer in (*consumers["form"], *consumers["scans"]):+ assert consumer.kind == FileConsumerKind.JUDGMENT_DOCUMENT+ assert consumer.pipe_ref == "fic_judge.main"+ assert consumer.model == judgment_model+ assert consumer.readable_formats == frozenset({"pdf"})+ assert consumers["scans"][0].covers(file_input_path=("scans", 0))+ assert not consumers["scans"][0].covers(file_input_path=("form",))++ def test_a_prompt_whose_model_reads_no_documents_consumes_nothing(self, load_empty_library: Callable[[], str], mocker: MockerFixture):+ """A model reading no documents at all is the method author's choice, so there is no caller's format to refuse."""+ llm_specs = get_model_deck().inference_models.root[ModelType.LLM]+ booted_spec = llm_specs[TEXT_ONLY_LLM]+ mocker.patch.dict(llm_specs, {TEXT_ONLY_LLM: booted_spec.model_copy(update={"inputs": ["text"]})})+ pipes = f"""+[pipe.main]+type = "PipeLLM"+description = "Summarize a transcript with a model reading text only"+inputs = {{ transcript = "Document" }}+output = "Text"+model = "{TEXT_ONLY_LLM}"+prompt = "Summarize this transcript: $transcript"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_text_only", pipes=pipes)++ assert not any(consumers.values())++ def test_a_document_the_prompt_reads_from_no_entry_input_is_not_followed(self, load_empty_library: Callable[[], str]):+ """A prompt's document that an earlier step produced is no file the caller sends."""+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Make a new document from the transcript, then summarize the new one"+inputs = { transcript = "Document" }+output = "Text"+steps = [+ { pipe = "replace_transcript", result = "fresh_document" },+ { pipe = "summarize_fresh_document", result = "summary" },+]++[pipe.replace_transcript]+type = "PipeFunc"+description = "Make a new document from the transcript"+inputs = { transcript = "Document" }+output = "Document"+function_name = "file_consumers_replace_transcript"++[pipe.summarize_fresh_document]+type = "PipeLLM"+description = "Summarize the new document"+inputs = { fresh_document = "Document" }+output = "Text"+prompt = "Summarize this document: $fresh_document"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_untracked", pipes=pipes)++ assert not any(consumers.values())def test_a_waterfall_reads_a_format_when_any_member_reads_it(self, load_empty_library: Callable[[], str], mocker: MockerFixture):mocker.patch.dict(get_model_deck().extract_waterfalls, {"fic-mixed-extractors": [PDF_ONLY_MODEL, DOCLING_MODEL]})diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_material.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_evidence.pysimilarity index 58%rename from tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_material.pyrename to tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_evidence.pyindex 8f15b78d0..8b6bfe239 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_material.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_binding_judge_evidence.py@@ -4,6 +4,7 @@ import pytestfrom pytest_mock import MockerFixturefrom pipelex.kernel import judgment_ops+from pipelex.kernel.prompt_assembly import AssembledUserPromptfrom pipelex.pipeline.runner import PipelexMTHDSProtocolfrom pipelex.system.pipe_run_mode import PipeRunModefrom pipelex.test_extras.mthds_corpus.loader import load_entry@@ -11,19 +12,21 @@ from pipelex.test_extras.mthds_corpus.resources import entries_root@pytest.mark.asyncio(loop_scope="class")-class TestBindingJudgeMaterial:+class TestBindingJudgeEvidence:async def test_v28_a_judge_reading_a_bound_total_sees_it_alone(self, mocker: MockerFixture) -> None:- """The judgment's material is keyed by the judge's plain input names, so it holds the bound total and nothing else of the invoice."""+ """The evidence is the judge's prompt rendered over its plain input names, so it shows the bound total and nothing else of the invoice."""entry = load_entry(directory=entries_root() / "feature_binding_step_judge_amount")- material_spy = mocker.spy(judgment_ops, "build_judgment_material")+ assembly_spy = mocker.spy(judgment_ops, "assemble_user_prompt")inputs: dict[str, Any] = {"invoice": {"concept": "invoice_approval.Invoice", "content": {"supplier": "Chantier Naval Le Bihan", "total": 7420.0}},}await PipelexMTHDSProtocol(library_dirs=[str(entry.directory)], pipe_run_mode=PipeRunMode.DRY).execute(pipe_code="check_total", inputs=inputs)- assert material_spy.call_count == 1- state, images, documents = material_spy.spy_return- assert state == {"total_amount": 7420.0}- assert images == {}- assert documents == {}+ assert assembly_spy.call_count == 1+ evidence = assembly_spy.spy_return+ assert isinstance(evidence, AssembledUserPrompt)+ assert evidence.text == "The total of an invoice, in euros: 7420.0"+ assert "Chantier Naval Le Bihan" not in evidence.text+ assert not evidence.images+ assert not evidence.documentsdiff --git a/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_optional_files.py b/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_optional_files.pynew file mode 100644index 000000000..472b2f198--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_img_gen/test_pipe_img_gen_optional_files.py@@ -0,0 +1,41 @@+"""A PipeImgGen whose prompt reads an optional image runs when the image is not given, the shared assembly leaving it out."""++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt+from pipelex.core.stuffs.text_content import TextContent+from pipelex.pipe_operators.img_gen import img_gen_prompt_blueprint+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.system.pipe_run_mode import PipeRunMode++_BUNDLE = """+domain = "img_gen_optional_files"+description = "A PipeImgGen whose prompt may hold a reference image"++[pipe.draw_parcel]+type = "PipeImgGen"+description = "Draw a parcel, after a reference photo when one is given"+inputs = { scene = "Text", reference = "Image?" }+output = "Image"+model = "$gen-image-testing-img2img"+prompt = "Draw $scene{% if reference %}, in the style of $reference{% endif %}"+"""++[email protected](loop_scope="class")+class TestPipeImgGenOptionalFiles:+ async def test_an_absent_optional_image_is_left_out_of_the_prompt(self, mocker: MockerFixture) -> None:+ assembly_spy = mocker.spy(img_gen_prompt_blueprint, "assemble_img_gen_prompt")++ response = await PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.DRY).execute(+ mthds_contents=[_BUNDLE],+ pipe_code="draw_parcel",+ inputs={"scene": TextContent(text="a parcel with one corner crushed")},+ )++ img_gen_prompt = assembly_spy.spy_return+ assert isinstance(img_gen_prompt, ImgGenPrompt)+ assert img_gen_prompt.positive_text == "Draw a parcel with one corner crushed"+ assert not img_gen_prompt.input_images+ assert response.pipe_output is not Nonediff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.pyindex 1047a1514..845fa2ce3 100644--- a/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.py+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_data.py@@ -1,10 +1,28 @@+import json+from typing import ClassVar++class PipeJudgeLoadTestData:"""A bundle with one PipeJudge step, built from the step's own fields."""JUDGMENT_MODEL = "jev-1.13.0"@classmethod- def bundle(cls, *, inputs: str = '{ message = "Text" }', output: str = "YesNo", step_fields: str = "") -> str:+ def bundle(+ cls,+ *,+ inputs: str = '{ message = "Text" }',+ output: str = "YesNo",+ prompt: str | None = "@message",+ question: str | None = "Is it urgent?",+ step_fields: str = "",+ ) -> str:+ """The step's fields as written: a `prompt` or a `question` given as `None` is left out of the table.++ A JSON string is a TOML basic string, escapes included, so the two templates are written that way.+ """+ prompt_line = f"prompt = {json.dumps(prompt)}" if prompt is not None else ""+ question_line = f"question = {json.dumps(question)}" if question is not None else ""return f"""domain = "judge_load"description = "Loading a PipeJudge step"@@ -15,9 +33,47 @@ Ticket = "A support ticket"[pipe.judge_it]type = "PipeJudge"-description = "Judge the material"+description = "Judge the evidence"inputs = {inputs}output = "{output}"-question = "Is it urgent?"+{prompt_line}+{question_line}{step_fields}"""+++class PipeJudgeOptionalFileTestData:+ """A PipeJudge whose evidence prompt reads an optional file, run with that file absent or present."""++ PIPE_CODE = "judge_damage"+ NOTE = "The parcel arrived with one corner crushed."+ PHOTO_URL = "https://example.com/parcel-corner.png"++ # (topic, the optional input's declaration, the evidence prompt reading it behind a guard)+ ABSENT_CASES: ClassVar[list[tuple[str, str, str]]] = [+ ("image", 'photo = "Image?"', "A claim note: $note\n@?photo\n"),+ ("document", 'claim = "Document?"', "A claim note: $note\n@?claim\n"),+ ("list_of_images", 'album = "Album?"', "A claim note: $note{% if album %}\nThe photos: $album.photos{% endif %}"),+ ]++ @classmethod+ def bundle(cls, *, optional_input: str, prompt: str) -> str:+ return f"""+domain = "judge_optional_files"+description = "A PipeJudge whose evidence may hold a file"++[concept.Album]+description = "Photos taken when a parcel was delivered"++[concept.Album.structure]+photos = {{ type = "list", item_type = "concept", item_concept_ref = "native.Image", description = "The photos", required = true }}++[pipe.{cls.PIPE_CODE}]+type = "PipeJudge"+description = "Judge whether a parcel's damage is serious"+inputs = {{ note = "Text", {optional_input} }}+output = "YesNo"+model = "{PipeJudgeLoadTestData.JUDGMENT_MODEL}"+prompt = {json.dumps(prompt)}+question = "Is the damage serious?"+"""diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.pyindex cb7d25eeb..79d724674 100644--- a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.py+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_acceptances.py@@ -15,20 +15,42 @@ class TestPipeJudgeLoadAcceptances:result = await validate_bundle(mthds_contents=[bundle])assert "judge_load.judge_it" in {pipe.pipe_ref for pipe in result.pipes}- async def test_an_input_the_question_never_names_is_material(self) -> None:- bundle = PipeJudgeLoadTestData.bundle(inputs='{ message = "Text", ticket = "Ticket" }', step_fields=_MODEL)+ async def test_an_input_read_by_the_question_alone_is_admitted(self) -> None:+ """The two-way check runs over the prompt and the question together, so a short parameter may live in the question."""+ bundle = PipeJudgeLoadTestData.bundle(+ inputs='{ message = "Text", topic = "Text" }', question="Is the message about $topic?", step_fields=_MODEL+ )await validate_bundle(mthds_contents=[bundle])@pytest.mark.usefixtures("judgment_model_reading_files")- @pytest.mark.parametrize("inputs", ['{ photo = "Image" }', '{ photos = "Image[]", claim = "Document" }'])- async def test_a_file_input_is_admitted_for_a_model_that_reads_files(self, inputs: str) -> None:- await validate_bundle(mthds_contents=[PipeJudgeLoadTestData.bundle(inputs=inputs, step_fields=_MODEL)])+ @pytest.mark.parametrize(+ ("inputs", "prompt"),+ [+ pytest.param('{ photo = "Image" }', "Inspect this photo: $photo", id="image"),+ pytest.param('{ photos = "Image[]", claim = "Document" }', "The photos: $photos\nThe claim: $claim", id="images_and_a_document"),+ ],+ )+ async def test_a_file_the_prompt_presents_is_admitted_for_a_model_that_reads_files(self, inputs: str, prompt: str) -> None:+ await validate_bundle(mthds_contents=[PipeJudgeLoadTestData.bundle(inputs=inputs, prompt=prompt, step_fields=_MODEL)])async def test_a_dynamic_input_is_admitted_for_a_text_only_model(self) -> None:- # `Dynamic` is compatible with every concept, `Image` included, so its value is checked when it runs, never at load.- await validate_bundle(mthds_contents=[PipeJudgeLoadTestData.bundle(inputs='{ payload = "Dynamic" }', step_fields=_MODEL)])+ # `Dynamic` is compatible with every concept, `Image` included, so the prompt renders it as text, never as a file.+ bundle = PipeJudgeLoadTestData.bundle(inputs='{ payload = "Dynamic" }', prompt="@payload", step_fields=_MODEL)+ await validate_bundle(mthds_contents=[bundle])++ async def test_an_optional_input_guarded_in_the_prompt_is_admitted(self) -> None:+ """Absence is the template's business: a guarded reference renders nothing when the input is absent."""+ bundle = PipeJudgeLoadTestData.bundle(inputs='{ message = "Text", note = "Text?" }', prompt="@message\n@?note", step_fields=_MODEL)+ await validate_bundle(mthds_contents=[bundle])++ async def test_a_labelled_scale_is_admitted(self) -> None:+ levels = '[{ label = "Cosmetic", description = "Appearance only" }, { label = "Fully blocked" }]'+ bundle = PipeJudgeLoadTestData.bundle(output="Rating", question="How severe is it?", step_fields=f"{_MODEL}\nlevels = {levels}")+ await validate_bundle(mthds_contents=[bundle])async def test_a_waterfall_no_backend_serves_is_left_to_the_run(self, unserved_judgment_waterfall: str) -> None:# Its model's spec is not on this boot, so whether it reads files is checked when the step runs, as on a keyless boot.- bundle = PipeJudgeLoadTestData.bundle(inputs='{ photo = "Image" }', step_fields=f'model = "{unserved_judgment_waterfall}"')+ bundle = PipeJudgeLoadTestData.bundle(+ inputs='{ photo = "Image" }', prompt="Inspect this photo: $photo", step_fields=f'model = "{unserved_judgment_waterfall}"'+ )await validate_bundle(mthds_contents=[bundle])diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.pyindex af93d3544..5e52157ed 100644--- a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.py+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_load_refusals.py@@ -54,12 +54,50 @@ class TestPipeJudgeLoadRefusals:pytest.param('{ photo = "Document" }', "documents", id="document"),],)- async def test_a_file_input_is_refused_for_a_text_only_model(self, inputs: str, file_kind: str) -> None:- report = await _refusal_report(PipeJudgeLoadTestData.bundle(inputs=inputs, step_fields=_MODEL))+ async def test_a_file_the_prompt_presents_is_refused_for_a_text_only_model(self, inputs: str, file_kind: str) -> None:+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(inputs=inputs, prompt="Inspect this: $photo", step_fields=_MODEL))assert f"does not read {file_kind}" in report- assert "'photo'" in report+ assert "presents 'photo' in its prompt" in reportassert PipeJudgeLoadTestData.JUDGMENT_MODEL in report+ @pytest.mark.usefixtures("judgment_model_reading_files")+ @pytest.mark.parametrize(+ "inputs",+ [+ pytest.param('{ message = "Text", photo = "Image" }', id="image"),+ pytest.param('{ message = "Text", photo = "Document" }', id="document"),+ ],+ )+ async def test_a_file_read_by_the_question_is_refused(self, inputs: str) -> None:+ """A question is plain text, so a file it read would reach the model as a token naming nothing, whatever the model reads."""+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(inputs=inputs, question="Is $photo damaged?", step_fields=_MODEL))+ assert "reads 'photo' in its question" in report+ assert "only the prompt presents files" in report+ assert "'input_stuff_spec_mismatch'" in report++ @pytest.mark.parametrize(+ ("prompt", "question"),+ [+ pytest.param("@message\n$note", "Is it urgent?", id="in_the_prompt"),+ pytest.param("@message", "Is it urgent, given $note?", id="in_the_question"),+ ],+ )+ async def test_an_unguarded_optional_input_is_refused(self, prompt: str, question: str) -> None:+ """Both templates are linted, so an optional input read without a guard is refused wherever it is read."""+ bundle = PipeJudgeLoadTestData.bundle(inputs='{ message = "Text", note = "Text?" }', prompt=prompt, question=question, step_fields=_MODEL)+ report = await _refusal_report(bundle)+ assert "'optional_input_unguarded'" in report+ assert "note" in report++ async def test_an_input_neither_template_reads_is_refused(self) -> None:+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(inputs='{ message = "Text", ticket = "Ticket" }', step_fields=_MODEL))+ assert "Input 'ticket' is declared but never read by the prompt or question" in report++ async def test_a_bundle_writing_its_question_in_prompt_is_told_where_each_goes(self) -> None:+ """Before MTHDS 5.0.0 `prompt` was a synonym of `question`, so a bundle written then holds its question in `prompt`."""+ report = await _refusal_report(PipeJudgeLoadTestData.bundle(prompt="Is the message urgent?\n@message", question=None, step_fields=_MODEL))+ assert "`prompt` holds the evidence the question is asked over, and the question is written in `question`" in report+@pytest.mark.parametrize(("output", "step_fields", "asks"),[diff --git a/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_optional_files.py b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_optional_files.pynew file mode 100644index 000000000..4333a2e1e--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_judge/test_pipe_judge_optional_files.py@@ -0,0 +1,88 @@+"""A PipeJudge whose evidence prompt reads an optional file runs whether the file is given or not."""++from typing import Any++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.content_generation.assignment_models import JudgmentAssignment+from pipelex.cogt.content_generation.content_generator import ContentGenerator+from pipelex.cogt.judgment.judgment_models import JudgmentOutcome, YesNoAnswer+from pipelex.core.stuffs.text_content import TextContent+from pipelex.kernel import judgment_ops+from pipelex.kernel.prompt_assembly import AssembledUserPrompt+from pipelex.pipeline.exceptions import PipelineExecutionError+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.system.pipe_run_mode import PipeRunMode+from tests.integration.pipelex.pipes.operator.pipe_judge.test_data import PipeJudgeOptionalFileTestData+++def _answer_yes(*, judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentOutcome]:+ return dict.fromkeys(judgment_assignment.questions, YesNoAnswer(probability=0.9))++[email protected]("judgment_model_reading_files")[email protected](loop_scope="class")+class TestPipeJudgeOptionalFiles:+ @pytest.mark.parametrize("pipe_run_mode", [PipeRunMode.DRY, PipeRunMode.LIVE])+ @pytest.mark.parametrize(("topic", "optional_input", "prompt"), PipeJudgeOptionalFileTestData.ABSENT_CASES)+ async def test_an_absent_optional_file_is_left_out_of_the_evidence(+ self,+ mocker: MockerFixture,+ pipe_run_mode: PipeRunMode,+ topic: str, # ruff: ignore[unused-method-argument]+ optional_input: str,+ prompt: str,+ ) -> None:+ """The guard renders the absent file out, and the judgment runs over the rest of the evidence with no file to present."""+ assembly_spy = mocker.spy(judgment_ops, "assemble_user_prompt")+ if pipe_run_mode.is_live:+ # The live run's content generator answers here without calling any model; the dry run keeps its own mock.+ mocker.patch.object(ContentGenerator, "make_judgment_answers", side_effect=_answer_yes)++ response = await PipelexMTHDSProtocol(pipe_run_mode=pipe_run_mode).execute(+ mthds_contents=[PipeJudgeOptionalFileTestData.bundle(optional_input=optional_input, prompt=prompt)],+ pipe_code=PipeJudgeOptionalFileTestData.PIPE_CODE,+ inputs={"note": TextContent(text=PipeJudgeOptionalFileTestData.NOTE)},+ )++ evidence = assembly_spy.spy_return+ assert isinstance(evidence, AssembledUserPrompt)+ assert evidence.text.strip() == f"A claim note: {PipeJudgeOptionalFileTestData.NOTE}"+ assert evidence.images == []+ assert evidence.documents == []+ assert response.pipe_output is not None+ verdict = response.pipe_output.main_stuff_as_yes_no+ if pipe_run_mode.is_live:+ assert verdict.yes_no is True++ async def test_a_present_optional_file_is_attached_and_numbered(self, mocker: MockerFixture) -> None:+ assembly_spy = mocker.spy(judgment_ops, "assemble_user_prompt")+ _topic, optional_input, prompt = PipeJudgeOptionalFileTestData.ABSENT_CASES[0]+ inputs: dict[str, Any] = {+ "note": TextContent(text=PipeJudgeOptionalFileTestData.NOTE),+ "photo": {"concept": "native.Image", "content": {"url": PipeJudgeOptionalFileTestData.PHOTO_URL}},+ }++ await PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.DRY).execute(+ mthds_contents=[PipeJudgeOptionalFileTestData.bundle(optional_input=optional_input, prompt=prompt)],+ pipe_code=PipeJudgeOptionalFileTestData.PIPE_CODE,+ inputs=inputs,+ )++ evidence = assembly_spy.spy_return+ assert isinstance(evidence, AssembledUserPrompt)+ assert "[Image 1]" in evidence.text+ assert len(evidence.images) == 1++ @pytest.mark.parametrize("pipe_run_mode", [PipeRunMode.DRY, PipeRunMode.LIVE])+ async def test_a_missing_required_file_is_refused(self, mocker: MockerFixture, pipe_run_mode: PipeRunMode) -> None:+ judge_spy = mocker.patch.object(ContentGenerator, "make_judgment_answers", side_effect=_answer_yes)+ with pytest.raises(PipelineExecutionError) as exc_info:+ await PipelexMTHDSProtocol(pipe_run_mode=pipe_run_mode).execute(+ mthds_contents=[PipeJudgeOptionalFileTestData.bundle(optional_input='photo = "Image"', prompt="A claim note: $note\n@photo\n")],+ pipe_code=PipeJudgeOptionalFileTestData.PIPE_CODE,+ inputs={"note": TextContent(text=PipeJudgeOptionalFileTestData.NOTE)},+ )+ assert "missing required inputs: photo" in str(exc_info.value)+ judge_spy.assert_not_called()diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm_optional_files.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm_optional_files.pynew file mode 100644index 000000000..1de89e33e--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm_optional_files.py@@ -0,0 +1,47 @@+"""A PipeLLM whose prompt reads an optional file runs when the file is not given, the shared assembly leaving it out."""++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.llm.llm_prompt import LLMPrompt+from pipelex.core.stuffs.text_content import TextContent+from pipelex.kernel import llm_ops+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.system.pipe_run_mode import PipeRunMode++_BUNDLE = """+domain = "llm_optional_files"+description = "A PipeLLM whose prompt may hold a file"++[pipe.describe_damage]+type = "PipeLLM"+description = "Describe a parcel's damage"+inputs = { note = "Text", photo = "Image?", claim = "Document?" }+output = "Text"+model = "gemini-2.5-flash"+prompt = \"\"\"+A claim note: $note+@?photo+@?claim+\"\"\"+"""++[email protected](loop_scope="class")+class TestPipeLLMOptionalFiles:+ async def test_absent_optional_files_are_left_out_of_the_prompt(self, mocker: MockerFixture) -> None:+ assembly_spy = mocker.spy(llm_ops, "assemble_llm_prompt")++ response = await PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.DRY).execute(+ mthds_contents=[_BUNDLE],+ pipe_code="describe_damage",+ inputs={"note": TextContent(text="The parcel arrived with one corner crushed.")},+ )++ llm_prompt = assembly_spy.spy_return+ assert isinstance(llm_prompt, LLMPrompt)+ assert llm_prompt.user_text is not None+ assert llm_prompt.user_text.strip() == "A claim note: The parcel arrived with one corner crushed."+ assert llm_prompt.user_images == []+ assert llm_prompt.user_documents == []+ assert response.pipe_output is not Nonediff --git a/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py b/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.pyindex 56fef8812..81db5ff13 100644--- a/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py+++ b/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py@@ -40,7 +40,7 @@ from pipelex.cogt.extract.extract_job_components import ExtractJobConfig, Extracfrom pipelex.cogt.image.prompt_image import PromptImageUrifrom pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt-from pipelex.cogt.judgment.judgment_models import YesNoQuestion+from pipelex.cogt.judgment.judgment_models import JudgmentPrompt, YesNoQuestionfrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSetting@@ -65,7 +65,7 @@ URL_FIELD_INVENTORY: dict[type[BaseModel], set[str]] = {RenderDocumentAssignment: {"composition.layout.blocks.url", "composition.layout.blocks.blocks.url"},SearchAssignment: set(),SearchObjectAssignment: set(),- JudgmentAssignment: {"images.uri", "documents.uri"},+ JudgmentAssignment: {"prompt.images.uri", "prompt.documents.uri"},}@@ -202,9 +202,11 @@ URL_BEARING_SAMPLES: dict[type[BaseModel], tuple[_DeclaresReads, set[str]]] = {JudgmentAssignment(job_metadata=_job_metadata(),cogt_run_params=CogtRunParams(run_mode=PipeRunMode.DRY),- state={"message": "the url https://example.com/in-the-state is data, never read"},- images={"before": [PromptImageUri(uri="pipelex-storage://s/before.png")], "after": [PromptImageUri(uri="pipelex-storage://s/after.png")]},- documents={"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf")]},+ prompt=JudgmentPrompt(+ text="Before: [Image 1] After: [Image 2] Claim: [Document 1], and the url https://example.com/in-the-text is data, never read",+ images=[PromptImageUri(uri="pipelex-storage://s/before.png"), PromptImageUri(uri="pipelex-storage://s/after.png")],+ documents=[PromptDocumentUri(uri="pipelex-storage://s/claim.pdf")],+ ),questions={"is_urgent": YesNoQuestion(instructions="Is it urgent?")},judgment_setting=JudgmentSetting(model="h"),),diff --git a/tests/unit/pipelex/cogt/content_generation/test_judgment_generate_dry_branch.py b/tests/unit/pipelex/cogt/content_generation/test_judgment_generate_dry_branch.pyindex 3f499a5de..475defd96 100644--- a/tests/unit/pipelex/cogt/content_generation/test_judgment_generate_dry_branch.py+++ b/tests/unit/pipelex/cogt/content_generation/test_judgment_generate_dry_branch.py@@ -14,7 +14,9 @@ from pipelex.cogt.content_generation.judgment_generate import judgment_gen_answefrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,ChoiceQuestion,+ JudgmentPrompt,RatingAnswer,+ RatingLevel,RatingQuestion,YesNoAnswer,YesNoQuestion,@@ -31,11 +33,13 @@ class TestJudgmentGenerateDryBranch:run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_judgment_dry")),cogt_run_params=CogtRunParams(run_mode=run_mode),- state={"message": "the roof is on fire"},+ prompt=JudgmentPrompt(text="A message from a customer: the roof is on fire"),questions={"is_urgent": YesNoQuestion(instructions="Does this need an answer today?"),"topic": ChoiceQuestion(instructions="What is this about?", options={"fire": None, "flood": "water damage"}),- "severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ "severity": RatingQuestion(+ instructions="How severe?", levels=[RatingLevel(label="Mild"), RatingLevel(label="Bad"), RatingLevel(label="Critical")]+ ),},judgment_setting=JudgmentSetting(model="mock-judgment-handle"),)diff --git a/tests/unit/pipelex/cogt/inference/test_inference_call_summary.py b/tests/unit/pipelex/cogt/inference/test_inference_call_summary.pyindex 0e7aa3ae4..1fb8a61d4 100644--- a/tests/unit/pipelex/cogt/inference/test_inference_call_summary.py+++ b/tests/unit/pipelex/cogt/inference/test_inference_call_summary.py@@ -46,7 +46,7 @@ from pipelex.cogt.image.prompt_image import PromptImageUrifrom pipelex.cogt.img_gen.img_gen_job_factory import ImgGenJobFactoryfrom pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstractfrom pipelex.cogt.inference.inference_call_summary import INFERENCE_CALL_ENDS_MESSAGE, InferenceOperation-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, YesNoAnswer, YesNoQuestion+from pipelex.cogt.judgment.judgment_models import JudgmentOutcome, YesNoAnswer, YesNoQuestionfrom pipelex.cogt.judgment.judgment_worker_abstract import JudgmentWorkerAbstractfrom pipelex.cogt.llm.llm_job import LLMJobfrom pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams@@ -216,7 +216,7 @@ class _StandInJudgmentWorker(JudgmentWorkerAbstract):JudgmentWorkerAbstract.__init__(self, inference_model=_model(model_type=ModelType.JUDGMENT, name="judge-test", model_id="judge-test-2026"))@override- async def _judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentAnswer]:+ async def _judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentOutcome]:_record_usage(tokens_usage=judgment_job.job_report.judgment_tokens_usage, usage=OTHER_USAGE)return {"is_urgent": YesNoAnswer(yes_no=True)}diff --git a/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.py b/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.pyindex cced838d7..a57ef4005 100644--- a/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.py+++ b/tests/unit/pipelex/cogt/judgment/fake_judgment_worker.py@@ -7,7 +7,7 @@ from pipelex.cogt.exceptions import CogtErrorfrom pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.judgment.judgment_job import JudgmentJobfrom pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, JudgmentQuestion+from pipelex.cogt.judgment.judgment_models import JudgmentOutcome, JudgmentPrompt, JudgmentQuestionfrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.judgment.judgment_worker_abstract import JudgmentWorkerAbstractfrom pipelex.cogt.llm.thinking_mode import ThinkingMode@@ -25,7 +25,7 @@ class FakeJudgmentWorker(JudgmentWorkerAbstract):self,inference_model: InferenceModelSpec,*,- answers: dict[str, JudgmentAnswer] | None = None,+ answers: dict[str, JudgmentOutcome] | None = None,raises: CogtError | None = None,reporting_delegate: ReportingProtocol | None = None,) -> None:@@ -35,7 +35,7 @@ class FakeJudgmentWorker(JudgmentWorkerAbstract):self.was_called = False@override- async def _judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentAnswer]:+ async def _judge(self, judgment_job: JudgmentJob) -> dict[str, JudgmentOutcome]:self.was_called = Trueif self._raises is not None:raise self._raises@@ -45,13 +45,11 @@ class FakeJudgmentWorker(JudgmentWorkerAbstract):def make_fake_judgment_job(questions: dict[str, JudgmentQuestion],*,- images: dict[str, list[PromptImage]] | None = None,- documents: dict[str, list[PromptDocument]] | None = None,+ images: list[PromptImage] | None = None,+ documents: list[PromptDocument] | None = None,) -> JudgmentJob:return JudgmentJobFactory.make_judgment_job(- state={"message": "the roof is on fire"},- images=images,- documents=documents,+ prompt=JudgmentPrompt(text="A message from a customer: the roof is on fire", images=images or [], documents=documents or []),questions=questions,judgment_setting=JudgmentSetting(model="fake-judgment-handle"),job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_judgment")),diff --git a/tests/unit/pipelex/cogt/judgment/test_data.py b/tests/unit/pipelex/cogt/judgment/test_data.pyindex 6c7efec4b..6badf158f 100644--- a/tests/unit/pipelex/cogt/judgment/test_data.py+++ b/tests/unit/pipelex/cogt/judgment/test_data.py@@ -3,23 +3,29 @@ from typing import ClassVarfrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,ChoiceQuestion,- JudgmentAnswer,+ JudgmentOutcome,JudgmentQuestion,RatingAnswer,+ RatingLevel,RatingQuestion,YesNoAnswer,YesNoQuestion,)+def described_levels(*descriptions: str) -> list[RatingLevel]:+ """An unlabelled scale, each level described."""+ return [RatingLevel(description=description) for description in descriptions]++class JudgmentTestCases:THREE_QUESTIONS: ClassVar[dict[str, JudgmentQuestion]] = {"is_urgent": YesNoQuestion(instructions="Does this message need an answer today?"),"topic": ChoiceQuestion(instructions="What is this message about?", options={"fire": None, "flood": "water damage"}),- "severity": RatingQuestion(instructions="How severe is the situation?", levels=["mild", "bad", "critical"]),+ "severity": RatingQuestion(instructions="How severe is the situation?", levels=described_levels("mild", "bad", "critical")),}- THREE_ANSWERS: ClassVar[dict[str, JudgmentAnswer]] = {+ THREE_ANSWERS: ClassVar[dict[str, JudgmentOutcome]] = {"is_urgent": YesNoAnswer(probability=0.93),"topic": ChoiceAnswer(choice="fire", confidence=0.88, probabilities={"fire": 0.88, "flood": 0.12}),"severity": RatingAnswer(level=2, position=1.82, confidence=0.7, probabilities={0: 0.02, 1: 0.14, 2: 0.84}),diff --git a/tests/unit/pipelex/cogt/judgment/test_judgment_models.py b/tests/unit/pipelex/cogt/judgment/test_judgment_models.pyindex bc62e98d0..8c7e11e9f 100644--- a/tests/unit/pipelex/cogt/judgment/test_judgment_models.py+++ b/tests/unit/pipelex/cogt/judgment/test_judgment_models.py@@ -1,8 +1,9 @@"""What the question and answer models refuse.-The one rule that is not a type is ``YesNoAnswer``'s: an answer carrying neither a probability nor a-boolean is not an answer at all. Everything else here pins the optionality that lets a backend which-measured nothing stay honest.+The rules that are not types: ``YesNoAnswer`` carries a verdict in one form or the other, a yes/no+question's criteria declare both sides or none, a level carries a label, a description or both, and+a scale's labels are all or none and distinct. Everything else here pins the optionality that lets a+backend which measured nothing stay honest, and the wire round trip a distributed run makes."""import pytest@@ -10,19 +11,26 @@ from pydantic import TypeAdapter, ValidationErrorfrom pipelex.cogt.content_generation.assignment_models import JudgmentAssignmentfrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParams+from pipelex.cogt.document.prompt_document import PromptDocumentUri+from pipelex.cogt.image.prompt_image import PromptImageUrifrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,ChoiceQuestion,- JudgmentAnswer,JudgmentKind,+ JudgmentOutcome,+ JudgmentPrompt,+ JudgmentRefusal,RatingAnswer,+ RatingLevel,RatingQuestion,YesNoAnswer,+ YesNoCriteria,YesNoQuestion,)from pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.system.job_metadata import JobMetadata, RunMetadatafrom pipelex.system.pipe_run_mode import PipeRunMode+from tests.unit.pipelex.cogt.judgment.test_data import JudgmentTestCasesclass TestJudgmentModels:@@ -46,6 +54,25 @@ class TestJudgmentModels:with pytest.raises(ValidationError):YesNoAnswer(probability=1.5)+ @pytest.mark.parametrize(+ "criteria",+ [+ pytest.param({"yes": "The customer needs an answer today"}, id="yes_alone"),+ pytest.param({"no": "The message can wait"}, id="no_alone"),+ ],+ )+ def test_a_lone_criterion_cannot_be_constructed(self, criteria: dict[str, str]) -> None:+ """Criteria describe both answers, so no worker ever meets a lone side."""+ with pytest.raises(ValidationError):+ YesNoQuestion.model_validate({"instructions": "Is it urgent?", "criteria": criteria})++ def test_a_yes_no_question_carries_both_criteria_or_none(self) -> None:+ bare = YesNoQuestion(instructions="Is it urgent?")+ with_criteria = YesNoQuestion(instructions="Is it urgent?", criteria=YesNoCriteria(yes="today", no="can wait"))++ assert bare.criteria is None+ assert with_criteria.criteria == YesNoCriteria(yes="today", no="can wait")+def test_a_choice_question_needs_an_option(self) -> None:with pytest.raises(ValidationError):ChoiceQuestion(instructions="Which one?", options={})@@ -54,6 +81,22 @@ class TestJudgmentModels:with pytest.raises(ValidationError):RatingQuestion(instructions="How much?", levels=[])+ def test_a_level_carries_a_label_a_description_or_both(self) -> None:+ with pytest.raises(ValidationError, match="label, a description or both"):+ RatingLevel()++ @pytest.mark.parametrize(+ ("levels", "message_fragment"),+ [+ pytest.param([RatingLevel(label="Low"), RatingLevel(description="High")], "every level carries a label or none does", id="mixed"),+ pytest.param([RatingLevel(label="Low"), RatingLevel(label="Low")], "'Low'", id="duplicated_label"),+ ],+ )+ def test_a_scales_labels_are_all_or_none_and_distinct(self, levels: list[RatingLevel], message_fragment: str) -> None:+ """The label is what a rating verdict reports, so two equal labels would make it ambiguous."""+ with pytest.raises(ValidationError, match=message_fragment):+ RatingQuestion(instructions="How much?", levels=levels)+def test_an_option_may_be_declared_without_a_description(self) -> None:question = ChoiceQuestion(instructions="Which one?", options={"fire": None, "flood": "water damage"})assert question.options["fire"] is None@@ -69,17 +112,23 @@ class TestJudgmentModels:assert sorted(answer.probabilities) == [0, 1, 2]def test_a_rating_answer_coerces_the_wire_string_keys_it_is_handed(self) -> None:- """The vendor's wire form keys a distribution by strings; the model holds integers."""+ """The wire keys a distribution by strings; the model holds integers."""answer = RatingAnswer.model_validate({"level": 2, "probabilities": {"0": 0.02, "2": 0.98}})assert answer.probabilities == {0: 0.02, 2: 0.98}def test_every_question_carries_its_kind(self) -> None:assert YesNoQuestion(instructions="?").kind == JudgmentKind.YES_NOassert ChoiceQuestion(instructions="?", options={"a": None}).kind == JudgmentKind.CHOICE- assert RatingQuestion(instructions="?", levels=["low"]).kind == JudgmentKind.RATING+ assert RatingQuestion(instructions="?", levels=[RatingLevel(description="low")]).kind == JudgmentKind.RATING++ def test_a_prompt_defaults_to_no_files(self) -> None:+ """The wire leaves `images` out of a text-only prompt, and the prompt reads it as no files."""+ prompt = JudgmentPrompt.model_validate({"text": "A message from a customer"})+ assert prompt.images == []+ assert prompt.documents == []def test_an_assignment_survives_the_round_trip_a_distributed_boundary_makes_it_take(self) -> None:- """Questions and answers are the whole wire payload, so the discriminator must work both ways.+ """The prompt, its files and the questions are the whole wire payload, so the discriminators must work both ways.Nothing else guards this: under a distributed orchestrator the assignment is dumped on thesubmitter and validated on a worker, and a discriminated union that dumps but does not@@ -90,35 +139,40 @@ class TestJudgmentModels:run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_judgment_wire")),cogt_run_params=CogtRunParams(run_mode=PipeRunMode.LIVE),- state={"message": "the roof is on fire", "read_count": 3, "starred": True, "labels": ["home", "urgent"]},- questions={- "is_urgent": YesNoQuestion(instructions="Does this need an answer today?"),- "topic": ChoiceQuestion(instructions="What is this about?", options={"fire": None, "flood": "water damage"}),- "severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),- },+ prompt=JudgmentPrompt(+ text="Inspect the product in this photo: [Image 1]\nIts claim form: [Document 1]",+ images=[PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")],+ documents=[PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")],+ ),+ questions=JudgmentTestCases.THREE_QUESTIONS,judgment_setting=JudgmentSetting(model="some-judgment-handle"),)restored = JudgmentAssignment.model_validate(assignment.model_dump(mode="json"))- assert restored.state == assignment.state+ assert restored == assignmentassert isinstance(restored.questions["is_urgent"], YesNoQuestion)assert isinstance(restored.questions["topic"], ChoiceQuestion)assert isinstance(restored.questions["severity"], RatingQuestion)assert restored.judgment_handle == "some-judgment-handle"- def test_answers_survive_the_same_round_trip(self) -> None:- answers: dict[str, JudgmentAnswer] = {- "is_urgent": YesNoAnswer(probability=0.93),- "topic": ChoiceAnswer(choice="fire", confidence=0.88, probabilities={"fire": 0.88, "flood": 0.12}),- "severity": RatingAnswer(level=2, position=1.82, confidence=0.7, probabilities={0: 0.02, 1: 0.14, 2: 0.84}),- }- adapter: TypeAdapter[dict[str, JudgmentAnswer]] = TypeAdapter(dict[str, JudgmentAnswer])+ def test_outcomes_survive_the_same_round_trip_a_refusal_included(self) -> None:+ outcomes: dict[str, JudgmentOutcome] = {**JudgmentTestCases.THREE_ANSWERS, "refused": JudgmentRefusal()}+ adapter: TypeAdapter[dict[str, JudgmentOutcome]] = TypeAdapter(dict[str, JudgmentOutcome])- restored = adapter.validate_python(adapter.dump_python(answers, mode="json"))+ restored = adapter.validate_python(adapter.dump_python(outcomes, mode="json"))- assert isinstance(restored["is_urgent"], YesNoAnswer)- assert isinstance(restored["topic"], ChoiceAnswer)+ assert restored == outcomes+ assert isinstance(restored["refused"], JudgmentRefusal)severity = restored["severity"]assert isinstance(severity, RatingAnswer)assert severity.probabilities == {0: 0.02, 1: 0.14, 2: 0.84}++ def test_a_refusal_carries_nothing_but_its_kind(self) -> None:+ assert JudgmentRefusal().model_dump(mode="json") == {"kind": "refusal"}++ def test_a_refusal_carrying_a_score_is_refused(self) -> None:+ """A refusal crosses the distributed boundary, so a worker's refusal smuggling a score must fail there rather than be dropped."""+ adapter: TypeAdapter[JudgmentOutcome] = TypeAdapter(JudgmentOutcome)+ with pytest.raises(ValidationError, match="score"):+ adapter.validate_python({"kind": "refusal", "score": 0.4})diff --git a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.py b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.pyindex bb2773c21..287cfaf65 100644--- a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.py+++ b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_contract.py@@ -18,6 +18,7 @@ from pipelex.cogt.judgment.judgment_models import (ChoiceQuestion,JudgmentAnswer,JudgmentQuestion,+ JudgmentRefusal,RatingAnswer,RatingQuestion,YesNoAnswer,@@ -26,7 +27,7 @@ from pipelex.cogt.judgment.judgment_models import (from pipelex.reporting.reporting_protocol import ReportingNoOpfrom pipelex.system.job_metadata import JobCategory, UnitJobIdfrom tests.unit.pipelex.cogt.judgment.fake_judgment_worker import FakeJudgmentWorker, make_fake_judgment_job, make_fake_judgment_model-from tests.unit.pipelex.cogt.judgment.test_data import JudgmentTestCases+from tests.unit.pipelex.cogt.judgment.test_data import JudgmentTestCases, described_levelsclass _RecordingDelegate(ReportingNoOp):@@ -66,6 +67,27 @@ class TestJudgmentWorkerContract:assert "is_spam" in str(exc_info.value)+ async def test_it_accepts_a_refusal_for_any_question(self) -> None:+ """A refusal is an outcome, not an error: the worker hands it up for the operator's policy to read."""+ job = make_fake_judgment_job(JudgmentTestCases.THREE_QUESTIONS)+ worker = FakeJudgmentWorker(+ make_fake_judgment_model(),+ answers={"is_urgent": JudgmentRefusal(), "topic": JudgmentRefusal(), "severity": JudgmentTestCases.THREE_ANSWERS["severity"]},+ )++ outcomes = await worker.judge(job)++ assert outcomes == {"is_urgent": JudgmentRefusal(), "topic": JudgmentRefusal(), "severity": JudgmentTestCases.THREE_ANSWERS["severity"]}++ async def test_it_refuses_a_refusal_under_a_key_nobody_asked(self) -> None:+ job = make_fake_judgment_job({"is_urgent": YesNoQuestion(instructions="Is it urgent?")})+ worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"is_urgent": YesNoAnswer(probability=0.9), "is_spam": JudgmentRefusal()})++ with pytest.raises(JudgmentAnswerMismatchError) as exc_info:+ await worker.judge(job)++ assert "unasked ['is_spam']" in str(exc_info.value)+async def test_it_refuses_a_dropped_answer(self) -> None:job = make_fake_judgment_job(JudgmentTestCases.THREE_QUESTIONS)worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"is_urgent": YesNoAnswer(probability=0.9)})@@ -86,7 +108,7 @@ class TestJudgmentWorkerContract:id="choice_answered_by_a_rating",),pytest.param(- RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical")),ChoiceAnswer(choice="mild"),"rating",id="rating_answered_by_a_choice",@@ -129,7 +151,7 @@ class TestJudgmentWorkerContract:],)async def test_it_refuses_a_level_beyond_the_scale(self, answer: RatingAnswer, named: str) -> None:- job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})+ job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical"))})worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"severity": answer})with pytest.raises(JudgmentAnswerMismatchError) as exc_info:@@ -148,19 +170,19 @@ class TestJudgmentWorkerContract:id="choice_probability_above_one",),pytest.param(- RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical")),RatingAnswer(level=1, probabilities={0: -0.1, 1: 1.1}),"0",id="rating_probability_below_zero",),pytest.param(- RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical")),RatingAnswer(level=2, position=float("inf")),"position",id="rating_infinite_position",),pytest.param(- RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical")),RatingAnswer(level=2, position=2.5),"position of 2.5",id="rating_position_past_the_last_level",@@ -181,7 +203,7 @@ class TestJudgmentWorkerContract:async def test_it_accepts_the_top_level_and_a_full_distribution(self) -> None:job = make_fake_judgment_job({- "severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"]),+ "severity": RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical")),"topic": ChoiceQuestion(instructions="Which topic?", options={"fire": None, "flood": "water damage"}),})@@ -243,7 +265,7 @@ class TestJudgmentWorkerContract:assert answer.choice == "fire"async def test_a_rating_answer_carries_its_level(self) -> None:- job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=["mild", "bad", "critical"])})+ job = make_fake_judgment_job({"severity": RatingQuestion(instructions="How severe?", levels=described_levels("mild", "bad", "critical"))})worker = FakeJudgmentWorker(make_fake_judgment_model(), answers={"severity": RatingAnswer(level=2, position=1.8)})answers = await worker.judge(job)diff --git a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.py b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.pyindex 6c3a6a81f..c5b9ca317 100644--- a/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.py+++ b/tests/unit/pipelex/cogt/judgment/test_judgment_worker_file_capability.py@@ -1,4 +1,4 @@-"""The files a judgment carries are refused by a model that does not read them, before its backend is called.+"""The files a judgment's prompt presents are refused by a model that does not read them, before its backend is called.Each guard in `JudgmentWorkerAbstract._check_can_read_files` is mutation-tested: break it and exactlyone of these goes red.@@ -12,13 +12,13 @@ from pipelex.cogt.document.prompt_document import PromptDocument, PromptDocumentfrom pipelex.cogt.exceptions import JudgmentCapabilityError, PromptDocumentFormatError, PromptImageFormatErrorfrom pipelex.cogt.image.prompt_image import PromptImage, PromptImageUrifrom pipelex.cogt.judgment.judgment_job import JudgmentJob-from pipelex.cogt.judgment.judgment_models import JudgmentAnswer, JudgmentQuestion, YesNoAnswer, YesNoQuestion+from pipelex.cogt.judgment.judgment_models import JudgmentOutcome, JudgmentQuestion, YesNoAnswer, YesNoQuestionfrom tests.unit.pipelex.cogt.judgment.fake_judgment_worker import FakeJudgmentWorker, make_fake_judgment_job, make_fake_judgment_model@pytest.mark.asyncioclass TestJudgmentWorkerFileCapability:- """The files a judgment carries are refused by a model that does not read them, before the backend is called."""+ """The files a judgment's prompt presents are refused by a model that does not read them, before the backend is called."""_IMAGE = PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")_PDF = PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")@@ -27,19 +27,19 @@ class TestJudgmentWorkerFileCapability:mime_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",)_QUESTION: ClassVar[dict[str, JudgmentQuestion]] = {"is_urgent": YesNoQuestion(instructions="Is it urgent?")}- _ANSWER: ClassVar[dict[str, JudgmentAnswer]] = {"is_urgent": YesNoAnswer(probability=0.9)}+ _ANSWER: ClassVar[dict[str, JudgmentOutcome]] = {"is_urgent": YesNoAnswer(probability=0.9)}def _worker(self, *, inputs: list[str], max_prompt_images: int | None = None) -> FakeJudgmentWorker:return FakeJudgmentWorker(make_fake_judgment_model(inputs=inputs, max_prompt_images=max_prompt_images), answers=self._ANSWER)- def _job(self, *, images: dict[str, list[PromptImage]] | None = None, documents: dict[str, list[PromptDocument]] | None = None) -> JudgmentJob:+ def _job(self, *, images: list[PromptImage] | None = None, documents: list[PromptDocument] | None = None) -> JudgmentJob:return make_fake_judgment_job(self._QUESTION, images=images, documents=documents)async def test_a_text_only_model_refuses_an_image(self) -> None:worker = self._worker(inputs=["text"])with pytest.raises(JudgmentCapabilityError, match="does not read images") as exc_info:- await worker.judge(self._job(images={"photo": [self._IMAGE]}))+ await worker.judge(self._job(images=[self._IMAGE]))assert not worker.was_called# The message names the model as the deck does, with none of the console's markup escapes.@@ -49,14 +49,14 @@ class TestJudgmentWorkerFileCapability:worker = self._worker(inputs=["text"])with pytest.raises(JudgmentCapabilityError, match="does not read documents"):- await worker.judge(self._job(documents={"claim": [self._PDF]}))+ await worker.judge(self._job(documents=[self._PDF]))assert not worker.was_calledasync def test_a_model_reading_images_and_pdf_judges_both(self) -> None:worker = self._worker(inputs=["text", "images", "pdf"])- answers = await worker.judge(self._job(images={"photo": [self._IMAGE]}, documents={"claim": [self._PDF]}))+ answers = await worker.judge(self._job(images=[self._IMAGE], documents=[self._PDF]))assert set(answers) == {"is_urgent"}assert worker.was_called@@ -65,14 +65,14 @@ class TestJudgmentWorkerFileCapability:worker = self._worker(inputs=["text", "pdf"])with pytest.raises(PromptDocumentFormatError, match="does not read"):- await worker.judge(self._job(documents={"claim": [self._PDF, self._DOCX]}))+ await worker.judge(self._job(documents=[self._PDF, self._DOCX]))assert not worker.was_calledasync def test_a_document_of_unknown_format_is_left_to_the_provider(self) -> None:worker = self._worker(inputs=["text", "pdf"])- await worker.judge(self._job(documents={"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim")]}))+ await worker.judge(self._job(documents=[PromptDocumentUri(uri="pipelex-storage://s/claim")]))assert worker.was_called@@ -80,7 +80,7 @@ class TestJudgmentWorkerFileCapability:worker = self._worker(inputs=["text", "images"])with pytest.raises(PromptImageFormatError, match="not an image"):- await worker.judge(self._job(images={"photo": [PromptImageUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]}))+ await worker.judge(self._job(images=[PromptImageUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]))assert not worker.was_called@@ -88,6 +88,6 @@ class TestJudgmentWorkerFileCapability:worker = self._worker(inputs=["text", "images"], max_prompt_images=1)with pytest.raises(JudgmentCapabilityError, match="at most 1 images"):- await worker.judge(self._job(images={"before": [self._IMAGE], "after": [self._IMAGE]}))+ await worker.judge(self._job(images=[self._IMAGE, self._IMAGE]))assert not worker.was_calleddiff --git a/tests/unit/pipelex/core/memory/test_absence_ledger.py b/tests/unit/pipelex/core/memory/test_absence_ledger.pyindex 452805604..c34e44f34 100644--- a/tests/unit/pipelex/core/memory/test_absence_ledger.py+++ b/tests/unit/pipelex/core/memory/test_absence_ledger.py@@ -150,6 +150,21 @@ class TestAbsenceLedger:assert working_memory.resolve_stuff("draft_alias") == recordassert working_memory.list_missing_names({"draft_alias"}) == []+ def test_is_variable_present_answers_whether_a_stuff_is_held(self):+ """Presence follows `get_optional_stuff`: a name, or an alias whose target, holds a stuff; an alias to a resolved-absent slot does not."""+ working_memory = WorkingMemory()+ working_memory.set_stuff(name="note", stuff=_make_text_stuff("note"))+ working_memory.add_alias("note_alias", target="note")+ working_memory.set_stuff(name="draft", stuff=_make_text_stuff("draft"))+ working_memory.add_alias("draft_alias", target="draft")+ working_memory.record_resolved_absence(_make_record("draft"))++ assert working_memory.is_variable_present(name="note") is True+ assert working_memory.is_variable_present(name="note_alias") is True+ assert working_memory.is_variable_present(name="draft") is False+ assert working_memory.is_variable_present(name="draft_alias") is False+ assert working_memory.is_variable_present(name="never_declared") is False+def test_record_absence_is_a_note_beside_a_value(self):"""The plain record_absence stays a ledger NOTE: a value under the same name is kept andwins in resolve_stuff (the D4 plural empty-list note depends on this).diff --git a/tests/unit/pipelex/core/memory/test_working_memory_generate_context.py b/tests/unit/pipelex/core/memory/test_working_memory_generate_context.pyindex 3213f533f..d2c73f749 100644--- a/tests/unit/pipelex/core/memory/test_working_memory_generate_context.py+++ b/tests/unit/pipelex/core/memory/test_working_memory_generate_context.py@@ -1,3 +1,4 @@+from pipelex.core.memory.absence import AbsenceKind, AbsenceRecordfrom pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.html_content import HtmlContent@@ -148,6 +149,17 @@ class TestWorkingMemoryGenerateContext:assert context["backup_text"] is context["secondary_text"]assert context[MAIN_STUFF_NAME] is context["primary_text"]+ def test_generate_context_leaves_out_an_alias_whose_target_was_resolved_absent(self, memory_with_aliases: WorkingMemory):+ """The alias outlives its target's resolution as an absence; the context holds neither, so a template reads both as absent."""+ memory_with_aliases.record_resolved_absence(+ AbsenceRecord(variable_name="secondary_text", kind=AbsenceKind.DECLARED_ABSENT, reason="nothing to back up", producing_pipe="backup")+ )++ context = memory_with_aliases.generate_context()++ assert sorted(context) == sorted(["primary_text", "main_text", MAIN_STUFF_NAME])+ assert context["main_text"] is context["primary_text"]+def test_generate_context_complex_list(self, complex_list_memory: WorkingMemory):"""Test generate_context with complex list content."""context = complex_list_memory.generate_context()diff --git a/tests/unit/pipelex/kernel/test_judgment_material.py b/tests/unit/pipelex/kernel/test_judgment_material.pydeleted file mode 100644index 7a0564a2a..000000000--- a/tests/unit/pipelex/kernel/test_judgment_material.py+++ /dev/null@@ -1,177 +0,0 @@-"""The material a judgment is asked over: one entry per input, read off its content's class, files beside the state."""--import datetime-from typing import Any--import pytest-from pydantic import Field--from pipelex.cogt.document.prompt_document import PromptDocumentUri-from pipelex.cogt.image.prompt_image import PromptImageUri-from pipelex.core.concepts.concept_factory import ConceptFactory-from pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.memory.absence import AbsenceKind, AbsenceRecord-from pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory-from pipelex.core.stuffs.choice_content import ChoiceContent-from pipelex.core.stuffs.date_content import DateContent-from pipelex.core.stuffs.document_content import DocumentContent-from pipelex.core.stuffs.html_content import HtmlContent-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.core.stuffs.json_content import JSONContent-from pipelex.core.stuffs.list_content import ListContent-from pipelex.core.stuffs.markdown_content import MarkdownContent-from pipelex.core.stuffs.number_content import NumberContent-from pipelex.core.stuffs.rating_content import RatingContent-from pipelex.core.stuffs.structured_content import StructuredContent-from pipelex.core.stuffs.stuff_content import StuffContent-from pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.core.stuffs.text_content import TextContent-from pipelex.core.stuffs.time_content import TimeContent-from pipelex.core.stuffs.yes_no_content import YesNoContent-from pipelex.kernel.judgment_ops import build_judgment_material---class _ReportWithNestedContents(StructuredContent):- """A structure whose fields are typed as the content base, so they hold whichever content subclass a run put there."""-- payload: StuffContent- extras: list[StuffContent]---class _Invoice(StructuredContent):- """A structured input, judged as one whole value under its plain name."""-- total: float- note: str | None = None- scan: ImageContent | None = None- lines: list[TextContent] = Field(default_factory=list[TextContent])---def _memory(contents: dict[str, StuffContent]) -> WorkingMemory:- """A memory holding each content under its name.-- The concept is `Anything` for every one, because the material is read off the content's class and- never off its concept: that is what keeps the kernel free of the concept library.- """- anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)- stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in contents.items()]- return WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)---class TestJudgmentMaterial:- @pytest.mark.parametrize(- ("content", "expected"),- [- pytest.param(TextContent(text="the roof leaks"), "the roof leaks", id="text"),- pytest.param(MarkdownContent(text="# Leak"), "# Leak", id="text_refinement"),- pytest.param(NumberContent(number=42), 42, id="number"),- pytest.param(YesNoContent(yes_no=False, probability=0.2), False, id="yes_no_as_its_boolean"),- pytest.param(DateContent(date=datetime.date(2026, 10, 4)), "2026-10-04", id="date"),- pytest.param(TimeContent(time=datetime.time(15, 40)), "15:40:00", id="time"),- pytest.param(JSONContent(json_obj={"a": 1}), {"a": 1}, id="json"),- pytest.param(- ChoiceContent(choice="billing", confidence=0.7), {"choice": "billing", "confidence": 0.7}, id="choice_without_absent_members"- ),- pytest.param(RatingContent(level=2), {"level": 2}, id="rating"),- pytest.param(HtmlContent(inner_html="<p>hi</p>"), {"inner_html": "<p>hi</p>"}, id="other_structure"),- pytest.param(ListContent(items=[TextContent(text="a"), TextContent(text="b")]), ["a", "b"], id="list"),- ],- )- def test_a_non_file_input_is_a_member_of_the_state(self, content: StuffContent, expected: Any) -> None:- state, images, documents = build_judgment_material(memory=_memory({"material": content}), input_names=["material"])-- assert state == {"material": expected}- assert images == {}- assert documents == {}-- def test_the_state_is_keyed_by_the_declared_names_in_their_order(self) -> None:- memory = _memory({"message": TextContent(text="help"), "count": NumberContent(number=3), "unasked": TextContent(text="ignored")})-- state, _, _ = build_judgment_material(memory=memory, input_names=["count", "message"])-- assert list(state) == ["count", "message"]-- def test_an_absent_optional_input_is_left_out(self) -> None:- memory = _memory({"message": TextContent(text="help")})- memory.record_resolved_absence(AbsenceRecord(variable_name="context", kind=AbsenceKind.NOT_PROVIDED, reason="not provided"))-- state, images, documents = build_judgment_material(memory=memory, input_names=["message", "context"])-- assert state == {"message": "help"}- assert images == {}- assert documents == {}-- def test_an_optional_input_never_written_is_left_out(self) -> None:- """An optional input with neither a value nor a recorded absence reaches the step: the presence scan lets it through."""- memory = _memory({"message": TextContent(text="help")})-- state, images, documents = build_judgment_material(memory=memory, input_names=["message", "context"])-- assert state == {"message": "help"}- assert images == {}- assert documents == {}-- def test_an_image_and_a_document_go_to_the_file_channel(self) -> None:- memory = _memory(- {- "photo": ImageContent(url="pipelex-storage://s/photo.png", mime_type="image/png"),- "claim": DocumentContent(url="pipelex-storage://s/claim.pdf", mime_type="application/pdf"),- "message": TextContent(text="see attached"),- }- )-- state, images, documents = build_judgment_material(memory=memory, input_names=["photo", "claim", "message"])-- assert state == {"message": "see attached"}- assert images == {"photo": [PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")]}- assert documents == {"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]}-- def test_a_list_of_images_and_a_list_of_documents_go_to_the_file_channel_as_lists(self) -> None:- memory = _memory(- {- "photos": ListContent(items=[ImageContent(url="pipelex-storage://s/a.png"), ImageContent(url="pipelex-storage://s/b.png")]),- "claims": ListContent(items=[DocumentContent(url="pipelex-storage://s/a.pdf")]),- }- )-- state, images, documents = build_judgment_material(memory=memory, input_names=["photos", "claims"])-- assert state == {}- assert [image.uri for image in images["photos"] if isinstance(image, PromptImageUri)] == [- "pipelex-storage://s/a.png",- "pipelex-storage://s/b.png",- ]- assert [document.uri for document in documents["claims"] if isinstance(document, PromptDocumentUri)] == ["pipelex-storage://s/a.pdf"]-- def test_an_empty_list_is_an_empty_array_in_the_state(self) -> None:- state, images, _ = build_judgment_material(memory=_memory({"photos": ListContent[ImageContent](items=[])}), input_names=["photos"])-- assert state == {"photos": []}- assert images == {}-- def test_a_structure_keeps_the_fields_of_the_contents_nested_in_it(self) -> None:- report = _ReportWithNestedContents(payload=TextContent(text="The roof is on fire"), extras=[NumberContent(number=3)])- state, _, _ = build_judgment_material(memory=_memory({"report": report}), input_names=["report"])- assert state == {"report": {"payload": {"text": "The roof is on fire"}, "extras": [{"number": 3}]}}-- def test_a_structured_input_is_its_whole_value_keyed_by_its_plain_name(self) -> None:- """The material holds each declared input whole, under its plain name, and nothing it did not declare."""- invoice = _Invoice(total=1250.0, note="rush", lines=[TextContent(text="roof"), TextContent(text="gutter")])- memory = _memory({"invoice": invoice, "message": TextContent(text="help")})-- state, images, documents = build_judgment_material(memory=memory, input_names=["invoice"])-- assert state == {"invoice": {"total": 1250.0, "note": "rush", "lines": [{"text": "roof"}, {"text": "gutter"}]}}- assert images == {}- assert documents == {}-- def test_a_dotted_name_is_never_walked_as_a_path(self) -> None:- """The dotted branch is gone, so a dotted name is looked up as a name, finds no stuff, and adds nothing."""- memory = _memory({"invoice": _Invoice(total=1.0, scan=ImageContent(url="pipelex-storage://s/scan.png", mime_type="image/png"))})-- state, images, documents = build_judgment_material(memory=memory, input_names=["invoice.total", "invoice.scan"])-- assert state == {}- assert images == {}- assert documents == {}diff --git a/tests/unit/pipelex/kernel/test_judgment_verdict_content.py b/tests/unit/pipelex/kernel/test_judgment_verdict_content.pyindex 449f03716..61cf71e73 100644--- a/tests/unit/pipelex/kernel/test_judgment_verdict_content.py+++ b/tests/unit/pipelex/kernel/test_judgment_verdict_content.py@@ -1,11 +1,17 @@-"""The verdict native an answer becomes, and what became of a declared threshold."""+"""The verdict native an answer becomes, the label a rating takes from its scale, and what became of a declared threshold."""++from typing import ClassVarimport pytestfrom pipelex.cogt.judgment.judgment_models import (ChoiceAnswer,+ ChoiceQuestion,RatingAnswer,+ RatingLevel,+ RatingQuestion,YesNoAnswer,+ YesNoQuestion,)from pipelex.core.stuffs.choice_content import ChoiceContentfrom pipelex.core.stuffs.rating_content import RatingContent@@ -14,6 +20,20 @@ from pipelex.kernel.judgment_ops import make_verdict_contentclass TestVerdictContent:+ _YES_NO: ClassVar[YesNoQuestion] = YesNoQuestion(instructions="Is it urgent?")+ _CHOICE: ClassVar[ChoiceQuestion] = ChoiceQuestion(instructions="Which team?", options={"billing": None, "technical": None})+ _UNLABELLED_SCALE: ClassVar[RatingQuestion] = RatingQuestion(+ instructions="How severe?", levels=[RatingLevel(description="mild"), RatingLevel(description="bad"), RatingLevel(description="critical")]+ )+ _LABELLED_SCALE: ClassVar[RatingQuestion] = RatingQuestion(+ instructions="How severe?",+ levels=[+ RatingLevel(label="Cosmetic", description="Appearance only"),+ RatingLevel(label="Workaround available"),+ RatingLevel(label="Fully blocked", description="A task fails with no workaround"),+ ],+ )+@pytest.mark.parametrize(("probability", "threshold", "expected_yes_no"),[@@ -27,13 +47,15 @@ class TestVerdictContent:def test_a_probability_decides_the_verdict_against_the_threshold(self, probability: float, threshold: float | None, expected_yes_no: bool) -> None:- content, threshold_applied = make_verdict_content(answer=YesNoAnswer(probability=probability), threshold=threshold, is_dry=False)+ content, threshold_applied = make_verdict_content(+ question=self._YES_NO, answer=YesNoAnswer(probability=probability), threshold=threshold, is_dry=False+ )assert content == YesNoContent(yes_no=expected_yes_no, probability=probability)assert threshold_applied is (True if threshold is not None else None)def test_a_probability_outranks_the_models_own_verdict(self) -> None:- content, _ = make_verdict_content(answer=YesNoAnswer(probability=0.3, yes_no=True), threshold=0.5, is_dry=False)+ content, _ = make_verdict_content(question=self._YES_NO, answer=YesNoAnswer(probability=0.3, yes_no=True), threshold=0.5, is_dry=False)assert content == YesNoContent(yes_no=False, probability=0.3)@@ -48,7 +70,7 @@ class TestVerdictContent:def test_without_a_probability_the_models_verdict_stands(self, threshold: float | None, is_dry: bool, expected_threshold_applied: bool | None) -> None:- content, threshold_applied = make_verdict_content(answer=YesNoAnswer(yes_no=False), threshold=threshold, is_dry=is_dry)+ content, threshold_applied = make_verdict_content(question=self._YES_NO, answer=YesNoAnswer(yes_no=False), threshold=threshold, is_dry=is_dry)assert content == YesNoContent(yes_no=False)assert threshold_applied is expected_threshold_applied@@ -56,7 +78,7 @@ class TestVerdictContent:def test_a_choice_is_carried_as_reported(self) -> None:answer = ChoiceAnswer(choice="billing", confidence=0.8, probabilities={"billing": 0.8, "technical": 0.2})- content, threshold_applied = make_verdict_content(answer=answer, threshold=None, is_dry=False)+ content, threshold_applied = make_verdict_content(question=self._CHOICE, answer=answer, threshold=None, is_dry=False)assert content == ChoiceContent(choice="billing", confidence=0.8, probabilities={"billing": 0.8, "technical": 0.2})assert threshold_applied is None@@ -64,11 +86,25 @@ class TestVerdictContent:def test_a_ratings_distribution_is_keyed_by_the_level_index_as_text(self) -> None:answer = RatingAnswer(level=2, position=1.8, confidence=0.7, probabilities={0: 0.1, 1: 0.2, 2: 0.7})- content, _ = make_verdict_content(answer=answer, threshold=None, is_dry=False)+ content, _ = make_verdict_content(question=self._UNLABELLED_SCALE, answer=answer, threshold=None, is_dry=False)assert content == RatingContent(level=2, position=1.8, confidence=0.7, probabilities={"0": 0.1, "1": 0.2, "2": 0.7})def test_nothing_absent_is_synthesised(self) -> None:- content, _ = make_verdict_content(answer=RatingAnswer(level=0), threshold=None, is_dry=False)+ content, _ = make_verdict_content(question=self._UNLABELLED_SCALE, answer=RatingAnswer(level=0), threshold=None, is_dry=False)assert content == RatingContent(level=0)++ @pytest.mark.parametrize(+ ("level", "expected_label"),+ [+ pytest.param(0, "Cosmetic", id="bottom"),+ pytest.param(1, "Workaround available", id="label_alone"),+ pytest.param(2, "Fully blocked", id="top"),+ ],+ )+ def test_a_rating_takes_the_label_of_the_declared_level_its_answer_names(self, level: int, expected_label: str) -> None:+ """No backend reports a label: the kernel looks it up on the scale the step declared."""+ content, _ = make_verdict_content(question=self._LABELLED_SCALE, answer=RatingAnswer(level=level), threshold=None, is_dry=False)++ assert content == RatingContent(level=level, label=expected_label)diff --git a/tests/unit/pipelex/kernel/test_kernel_boot_contract.py b/tests/unit/pipelex/kernel/test_kernel_boot_contract.pyindex 63c06904e..2b60152e6 100644--- a/tests/unit/pipelex/kernel/test_kernel_boot_contract.py+++ b/tests/unit/pipelex/kernel/test_kernel_boot_contract.py@@ -113,6 +113,7 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(shape_inputs,)from pipelex.kernel.pipelex_kernel import PipelexKernel+ from pipelex.kernel.prompt_assembly import UserPromptContentfrom pipelex.kernel.prompt_references import ImageReference, ImageReferenceKindfrom pipelex.kernel.search_ops import run_searchfrom pipelex.kernel.templating_style_ops import resolve_templating_style@@ -375,8 +376,10 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(judgment_result = asyncio.run(run_judgment(memory=judgment_memory,+ prompt_content=UserPromptContent(+ template=TemplateBlueprint(template="A message from a customer: {{ message }}", category=TemplateCategory.LLM_PROMPT)+ ),question=ChoiceQuestion(instructions="Which team handles {{ message }}?", options={"billing": None, "technical": None}),- input_names=["message"],judgment_setting=JudgmentSetting(model="kernel-boot-contract-judgment-model"),concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.CHOICE),job_metadata=kernel.make_step_metadata(),@@ -386,7 +389,9 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent())- # The rendered question proves the templating ran against memory rather than being skipped.+ # The rendered prompt and question prove the templating ran against memory rather than being skipped.+ if judgment_result.prompt.text != "A message from a customer: My invoice is wrong.":+ fail(f"run_judgment assembled {judgment_result.prompt.text!r}, not the evidence over its memory")if judgment_result.rendered_question != "Which team handles My invoice is wrong.?":fail(f"run_judgment rendered {judgment_result.rendered_question!r}, not the question over its memory")check_stored(judgment_result, "run_judgment", ChoiceContent)diff --git a/tests/unit/pipelex/kernel/test_kernel_log_context.py b/tests/unit/pipelex/kernel/test_kernel_log_context.pyindex 7ef362673..a7ed57046 100644--- a/tests/unit/pipelex/kernel/test_kernel_log_context.py+++ b/tests/unit/pipelex/kernel/test_kernel_log_context.py@@ -26,6 +26,7 @@ from pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.cogt.search.search_setting import SearchSetting+from pipelex.cogt.templating.template_blueprint import TemplateBlueprintfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactory@@ -39,6 +40,7 @@ from pipelex.kernel.judgment_ops import run_judgmentfrom pipelex.kernel.llm_ops import generate_object_content, run_llm_object, run_llm_textfrom pipelex.kernel.llm_prompt_content import LlmPromptContentfrom pipelex.kernel.pipelex_kernel import PipelexKernel+from pipelex.kernel.prompt_assembly import UserPromptContentfrom pipelex.kernel.search_ops import run_searchfrom pipelex.kernel.templating_style_ops import resolve_templating_stylefrom pipelex.system.pipe_run_mode import PipeRunMode@@ -213,8 +215,8 @@ class TestKernelLogContext:await run_judgment(memory=WorkingMemoryFactory.make_empty(),+ prompt_content=UserPromptContent(template=TemplateBlueprint(template="No evidence at all.", category=TemplateCategory.LLM_PROMPT)),question=YesNoQuestion(instructions="Does the log context bind?"),- input_names=[],judgment_setting=JudgmentSetting(model="kernel-log-context-judgment-model"),concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.YES_NO),job_metadata=step,diff --git a/tests/unit/pipelex/kernel/test_prompt_assembly.py b/tests/unit/pipelex/kernel/test_prompt_assembly.pynew file mode 100644index 000000000..82f24a34a--- /dev/null+++ b/tests/unit/pipelex/kernel/test_prompt_assembly.py@@ -0,0 +1,252 @@+"""The user-prompt assembly PipeLLM, PipeImgGen and PipeJudge share: tokens, files and their order.++What is easy to get wrong here, and what these cases pin, is the correspondence between the numbered+tokens in the text and the order of the files handed to the model: the registry numbers both, and a+mismatch mislabels which file the prompt is talking about with nothing downstream able to notice.+"""++from typing import Any++import pytest++from pipelex.cogt.document.prompt_document import PromptDocumentUri+from pipelex.cogt.image.prompt_image import PromptImageUri+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.absence import AbsenceKind, AbsenceRecord+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.page_content import PageContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.kernel.exceptions import PromptContentError+from pipelex.kernel.prompt_assembly import PromptFiles, UserPromptContent, assemble_user_prompt+from pipelex.kernel.prompt_references import DocumentReference, DocumentReferenceKind, ImageReference, ImageReferenceKind+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.templating.templating_style import TagStyle, TemplatingStyle+from pipelex.tools.templating.text_format import TextFormat++# Stated rather than resolved: these are unit tests over the assembly itself.+_STYLE = TemplatingStyle(tag_style=TagStyle.NO_TAG, text_format=TextFormat.PLAIN)++_PHOTO = "pipelex-storage://org/photo.png"+_ALBUM_FIRST = "pipelex-storage://org/album-1.png"+_ALBUM_SECOND = "pipelex-storage://org/album-2.png"+_PAGE_VIEW = "pipelex-storage://org/page-view.png"+_CLAIM = "pipelex-storage://org/claim.pdf"+_ANNEX_FIRST = "pipelex-storage://org/annex-1.pdf"+_ANNEX_SECOND = "pipelex-storage://org/annex-2.pdf"+++def _memory(contents: dict[str, StuffContent]) -> WorkingMemory:+ anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)+ stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in contents.items()]+ return WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)+++def _template(source: str) -> TemplateBlueprint:+ return TemplateBlueprint(template=source, category=TemplateCategory.LLM_PROMPT)+++def _every_kind_of_file() -> WorkingMemory:+ return _memory(+ {+ "note": TextContent(text="Handle with care."),+ "photo": ImageContent(url=_PHOTO, mime_type="image/png"),+ "album": ListContent[ImageContent](items=[ImageContent(url=_ALBUM_FIRST), ImageContent(url=_ALBUM_SECOND)]),+ "page": PageContent(text_and_images=TextAndImagesContent(text=TextContent(text="page text")), page_view=ImageContent(url=_PAGE_VIEW)),+ "claim": DocumentContent(url=_CLAIM, mime_type="application/pdf"),+ "annexes": ListContent[DocumentContent](items=[DocumentContent(url=_ANNEX_FIRST), DocumentContent(url=_ANNEX_SECOND)]),+ }+ )++[email protected](loop_scope="class")+class TestPromptAssembly:+ async def test_every_reference_kind_gets_the_token_of_its_position(self) -> None:+ """Direct, list and dotted images, and direct and list documents, each numbered where it is handed over."""+ content = UserPromptContent(+ template=_template("$note Photo: $photo Album: $album View: {{ page.page_view }} Claim: $claim Annexes: $annexes"),+ image_references=[+ ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT),+ ImageReference(variable_path="album", kind=ImageReferenceKind.DIRECT_LIST),+ ImageReference(variable_path="page.page_view", kind=ImageReferenceKind.DIRECT),+ ],+ document_references=[+ DocumentReference(variable_path="claim", kind=DocumentReferenceKind.DIRECT),+ DocumentReference(variable_path="annexes", kind=DocumentReferenceKind.DIRECT_LIST),+ ],+ )++ assembled = await assemble_user_prompt(prompt_content=content, context_provider=_every_kind_of_file(), templating_style=_STYLE)++ assert assembled.text == (+ "Handle with care. Photo: [Image 1] Album: [Image 2], [Image 3] View: [Image 4] Claim: [Document 1] Annexes: [Document 2]\n[Document 3]"+ )+ assert [image.uri for image in assembled.images if isinstance(image, PromptImageUri)] == [_PHOTO, _ALBUM_FIRST, _ALBUM_SECOND, _PAGE_VIEW]+ assert [document.uri for document in assembled.documents if isinstance(document, PromptDocumentUri)] == [+ _CLAIM,+ _ANNEX_FIRST,+ _ANNEX_SECOND,+ ]++ async def test_each_file_keeps_the_format_its_content_established(self) -> None:+ content = UserPromptContent(+ template=_template("$photo $claim"),+ image_references=[ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT)],+ document_references=[DocumentReference(variable_path="claim", kind=DocumentReferenceKind.DIRECT)],+ )++ assembled = await assemble_user_prompt(prompt_content=content, context_provider=_every_kind_of_file(), templating_style=_STYLE)++ (image,) = assembled.images+ (document,) = assembled.documents+ assert isinstance(image, PromptImageUri)+ assert image.mime_type == "image/png"+ assert isinstance(document, PromptDocumentUri)+ assert document.mime_type == "application/pdf"++ async def test_a_text_only_prompt_has_no_files(self) -> None:+ assembled = await assemble_user_prompt(+ prompt_content=UserPromptContent(template=_template("Note: $note")),+ context_provider=_every_kind_of_file(),+ templating_style=_STYLE,+ )++ assert assembled.text == "Note: Handle with care."+ assert assembled.images == []+ assert assembled.documents == []++ async def test_templates_sharing_one_registry_number_one_sequence(self) -> None:+ """A PipeLLM's system prompt registers first, so its image takes the lower number in both texts."""+ memory = _every_kind_of_file()+ prompt_files = PromptFiles.make_from_references(+ context_provider=memory,+ image_references=[+ ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT),+ ImageReference(variable_path="album", kind=ImageReferenceKind.DIRECT_LIST),+ ],+ document_references=[],+ )+ template_params = prompt_files.template_params(extra_params=None)++ first_text = await prompt_files.render(+ template_blueprint=_template("First: $photo"), context_provider=memory, extra_params=template_params, templating_style=_STYLE+ )+ second_text = await prompt_files.render(+ template_blueprint=_template("Second: $album"), context_provider=memory, extra_params=template_params, templating_style=_STYLE+ )++ assert first_text == "First: [Image 1]"+ assert second_text == "Second: [Image 2], [Image 3]"+ assert len(prompt_files.prompt_images()) == 3++ async def test_the_callers_extra_params_are_left_untouched(self) -> None:+ extra_params: dict[str, Any] = {"suffix": "!"}+ content = UserPromptContent(+ template=_template("$photo{{ suffix }}"),+ image_references=[ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT)],+ )++ assembled = await assemble_user_prompt(+ prompt_content=content, context_provider=_every_kind_of_file(), templating_style=_STYLE, extra_params=extra_params+ )++ assert assembled.text == "[Image 1]!"+ assert extra_params == {"suffix": "!"}++ @pytest.mark.parametrize(+ ("template_source", "image_references", "document_references"),+ [+ pytest.param(+ "Note: $note{% if snapshot %} $snapshot{% endif %}",+ [ImageReference(variable_path="snapshot", kind=ImageReferenceKind.DIRECT, is_optional=True)],+ [],+ id="image",+ ),+ pytest.param(+ "Note: $note{% if receipt %} $receipt{% endif %}",+ [],+ [DocumentReference(variable_path="receipt", kind=DocumentReferenceKind.DIRECT, is_optional=True)],+ id="document",+ ),+ pytest.param(+ "Note: $note{% if case %} $case.photos{% endif %}",+ [ImageReference(variable_path="case.photos", kind=ImageReferenceKind.DIRECT_LIST, is_optional=True)],+ [],+ id="list_of_images",+ ),+ pytest.param(+ "Note: $note{% if case %} $case.annexes{% endif %}",+ [],+ [DocumentReference(variable_path="case.annexes", kind=DocumentReferenceKind.DIRECT_LIST, is_optional=True)],+ id="list_of_documents",+ ),+ ],+ )+ async def test_an_absent_optional_file_is_skipped_so_its_guard_renders_it_out(+ self, template_source: str, image_references: list[ImageReference], document_references: list[DocumentReference]+ ) -> None:+ """An optional input that holds no value gets no number and no file, and the template's guard leaves it out."""+ content = UserPromptContent(template=_template(template_source), image_references=image_references, document_references=document_references)++ assembled = await assemble_user_prompt(prompt_content=content, context_provider=_every_kind_of_file(), templating_style=_STYLE)++ assert assembled.text == "Note: Handle with care."+ assert assembled.images == []+ assert assembled.documents == []++ async def test_an_optional_image_reached_through_an_alias_to_a_resolved_absent_slot_is_skipped(self) -> None:+ """The alias outlives its target's resolution as an absence, and the slot it names holds nothing to present."""+ memory = _every_kind_of_file()+ memory.add_alias("snapshot", target="photo")+ memory.record_resolved_absence(+ AbsenceRecord(variable_name="photo", kind=AbsenceKind.DECLARED_ABSENT, reason="no photo was taken", producing_pipe="take_photo")+ )+ content = UserPromptContent(+ template=_template("Note: $note{% if snapshot %} $snapshot{% endif %}"),+ image_references=[ImageReference(variable_path="snapshot", kind=ImageReferenceKind.DIRECT, is_optional=True)],+ )++ assembled = await assemble_user_prompt(prompt_content=content, context_provider=memory, templating_style=_STYLE)++ assert assembled.text == "Note: Handle with care."+ assert assembled.images == []++ async def test_a_present_optional_file_is_attached_and_numbered(self) -> None:+ """Being declared optional changes nothing for a file that is there, a dotted path into a present input included."""+ content = UserPromptContent(+ template=_template("$photo, {{ page.page_view }}{% if claim %} $claim{% endif %}"),+ image_references=[+ ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT, is_optional=True),+ ImageReference(variable_path="page.page_view", kind=ImageReferenceKind.DIRECT, is_optional=True),+ ],+ document_references=[DocumentReference(variable_path="claim", kind=DocumentReferenceKind.DIRECT, is_optional=True)],+ )++ assembled = await assemble_user_prompt(prompt_content=content, context_provider=_every_kind_of_file(), templating_style=_STYLE)++ assert assembled.text == "[Image 1], [Image 2] [Document 1]"+ assert [image.uri for image in assembled.images if isinstance(image, PromptImageUri)] == [_PHOTO, _PAGE_VIEW]+ assert [document.uri for document in assembled.documents if isinstance(document, PromptDocumentUri)] == [_CLAIM]++ @pytest.mark.parametrize(+ ("image_reference", "message_fragment"),+ [+ pytest.param(+ ImageReference(variable_path="absent", kind=ImageReferenceKind.DIRECT), "Could not find image 'absent'", id="missing_required"+ ),+ pytest.param(ImageReference(variable_path="note", kind=ImageReferenceKind.DIRECT), "Could not find image 'note'", id="not_an_image"),+ ],+ )+ async def test_a_reference_the_context_cannot_satisfy_is_refused(self, image_reference: ImageReference, message_fragment: str) -> None:+ content = UserPromptContent(template=_template("$note"), image_references=[image_reference])++ with pytest.raises(PromptContentError, match=message_fragment):+ await assemble_user_prompt(prompt_content=content, context_provider=_every_kind_of_file(), templating_style=_STYLE)diff --git a/tests/unit/pipelex/kernel/test_run_judgment.py b/tests/unit/pipelex/kernel/test_run_judgment.pyindex 7aa2648d0..8701cb1d8 100644--- a/tests/unit/pipelex/kernel/test_run_judgment.py+++ b/tests/unit/pipelex/kernel/test_run_judgment.py@@ -1,4 +1,4 @@-"""A whole judgment step: the rendered question over the material, the stored verdict, and the threshold warning."""+"""A whole judgment step: the assembled evidence, the rendered question, the stored verdict, the refusal and the threshold warning."""from typing import Any@@ -6,64 +6,96 @@ import pytestfrom pytest_mock import MockerFixturefrom pipelex.cogt.content_generation.assignment_models import JudgmentAssignment+from pipelex.cogt.content_generation.dry_mock import dry_judgment_gen_answers+from pipelex.cogt.exceptions import JudgmentRefusedError+from pipelex.cogt.image.prompt_image import PromptImageUrifrom pipelex.cogt.judgment.judgment_models import (- JudgmentAnswer,+ JudgmentOutcome,+ JudgmentQuestion,+ JudgmentRefusal,+ RatingAnswer,+ RatingLevel,+ RatingQuestion,YesNoAnswer,YesNoQuestion,)from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.cogt.templating.template_blueprint import TemplateBlueprintfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.rating_content import RatingContentfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.core.stuffs.yes_no_content import YesNoContentfrom pipelex.kernel.judgment_ops import run_judgmentfrom pipelex.kernel.pipelex_kernel import PipelexKernel+from pipelex.kernel.prompt_assembly import UserPromptContent+from pipelex.kernel.prompt_references import ImageReference, ImageReferenceKindfrom pipelex.kernel.templating_style_ops import resolve_templating_stylefrom pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.tools.jinja2.template_category import TemplateCategory+_ROOF_PHOTO = "pipelex-storage://s/roof.png"++_EVIDENCE = UserPromptContent(+ template=TemplateBlueprint(+ template="A message from a customer: {{ message }}\nThe photo they sent: $photo", category=TemplateCategory.LLM_PROMPT+ ),+ image_references=[ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT)],+)++_SEVERITY = RatingQuestion(+ instructions="How severe is it?",+ levels=[RatingLevel(label="Cosmetic"), RatingLevel(label="Workaround available"), RatingLevel(label="Fully blocked")],+)-def _memory(contents: dict[str, StuffContent]) -> WorkingMemory:- """A memory holding each content under its name.- The concept is `Anything` for every one, because the material is read off the content's class and- never off its concept: that is what keeps the kernel free of the concept library.- """+def _memory(contents: dict[str, StuffContent]) -> WorkingMemory:anything = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.ANYTHING)stuffs = [StuffFactory.make_stuff(concept=anything, content=content, name=name) for name, content in contents.items()]return WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=stuffs)class _JudgmentProbe:- """Stands in for the content generator: records the assignment and answers what it was told to."""+ """Stands in for the content generator: records the assignment and answers what it was told to, or as the dry leaf does."""- def __init__(self, answer: JudgmentAnswer) -> None:- self.answer = answer+ def __init__(self, outcome: JudgmentOutcome | None) -> None:+ self.outcome = outcomeself.assignments: list[JudgmentAssignment] = []- async def make_judgment_answers(self, *, judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentAnswer]:+ async def make_judgment_answers(self, *, judgment_assignment: JudgmentAssignment) -> dict[str, JudgmentOutcome]:self.assignments.append(judgment_assignment)- return {"question": self.answer}+ if self.outcome is None:+ return dry_judgment_gen_answers(judgment_assignment)+ return {"question": self.outcome}@pytest.mark.asyncio(loop_scope="class")class TestRunJudgment:async def _run(- self, mocker: MockerFixture, *, answer: JudgmentAnswer, threshold: float | None, run_mode: PipeRunMode+ self,+ mocker: MockerFixture,+ *,+ outcome: JudgmentOutcome | None,+ threshold: float | None = None,+ run_mode: PipeRunMode = PipeRunMode.LIVE,+ question: JudgmentQuestion | None = None,) -> tuple[_JudgmentProbe, Any]:- probe = _JudgmentProbe(answer)+ probe = _JudgmentProbe(outcome)mocker.patch("pipelex.kernel.judgment_ops.get_content_generator", return_value=probe)kernel = PipelexKernel.make(storage_scope="test/scope", read_scope=None, run_mode=run_mode, user_id="judgment-ops")result = await run_judgment(- memory=_memory({"message": TextContent(text="The roof is on fire."), "photo": ImageContent(url="pipelex-storage://s/roof.png")}),- question=YesNoQuestion(instructions="Is this urgent: {{ message }}"),- input_names=["message", "photo"],+ memory=_memory({"message": TextContent(text="The roof is on fire."), "photo": ImageContent(url=_ROOF_PHOTO, mime_type="image/png")}),+ prompt_content=_EVIDENCE,+ question=question or YesNoQuestion(instructions="Is this urgent, given that it says '{{ message }}'?"),judgment_setting=JudgmentSetting(model="judgment-ops-model"),- concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.YES_NO),+ concept=ConceptFactory.make_native_concept(+ native_concept_code=NativeConceptCode.RATING if isinstance(question, RatingQuestion) else NativeConceptCode.YES_NO+ ),job_metadata=kernel.make_step_metadata(pipe_code="is_urgent"),cogt_run_params=kernel.cogt_run_params,templating_style=resolve_templating_style(authored=None),@@ -72,22 +104,46 @@ class TestRunJudgment:)return probe, result- async def test_it_sends_the_rendered_question_over_the_material_and_stores_the_verdict(self, mocker: MockerFixture) -> None:- probe, result = await self._run(mocker, answer=YesNoAnswer(probability=0.9), threshold=0.8, run_mode=PipeRunMode.LIVE)+ async def test_it_sends_the_assembled_evidence_and_the_rendered_question_and_stores_the_verdict(self, mocker: MockerFixture) -> None:+ probe, result = await self._run(mocker, outcome=YesNoAnswer(probability=0.9), threshold=0.8)(assignment,) = probe.assignments- assert assignment.questions["question"].instructions == "Is this urgent: The roof is on fire."- assert assignment.state == {"message": "The roof is on fire."}- assert list(assignment.images) == ["photo"]- assert result.rendered_question == "Is this urgent: The roof is on fire."+ assert list(assignment.questions) == ["question"]+ assert assignment.questions["question"].instructions == "Is this urgent, given that it says 'The roof is on fire.'?"+ assert assignment.prompt.text == "A message from a customer: The roof is on fire.\nThe photo they sent: [Image 1]"+ assert assignment.prompt.images == [PromptImageUri(uri=_ROOF_PHOTO, mime_type="image/png")]+ assert assignment.prompt.documents == []+ assert result.prompt == assignment.prompt+ assert result.rendered_question == "Is this urgent, given that it says 'The roof is on fire.'?"assert result.content == YesNoContent(yes_no=True, probability=0.9)assert result.threshold_applied is Trueassert result.memory.get_stuff("verdict").content == YesNoContent(yes_no=True, probability=0.9)+ async def test_a_refusal_raises_naming_the_step_and_the_model(self, mocker: MockerFixture) -> None:+ """A single question has nowhere to leave a refused answer absent, so the step fails, as a content error."""+ with pytest.raises(JudgmentRefusedError) as exc_info:+ await self._run(mocker, outcome=JudgmentRefusal())++ message = str(exc_info.value)+ assert "PipeJudge 'is_urgent'" in message+ assert "'judgment-ops-model'" in message+ assert "reword the question" in message+ assert exc_info.value.error_category == "content"++ async def test_a_rating_carries_the_label_of_the_level_answered(self, mocker: MockerFixture) -> None:+ _, result = await self._run(mocker, outcome=RatingAnswer(level=1, confidence=0.6), question=_SEVERITY)++ assert result.content == RatingContent(level=1, label="Workaround available", confidence=0.6)++ async def test_a_dry_run_answers_the_lowest_level_with_its_declared_label(self, mocker: MockerFixture) -> None:+ _, result = await self._run(mocker, outcome=None, question=_SEVERITY, run_mode=PipeRunMode.DRY)++ assert result.content == RatingContent(level=0, label="Cosmetic")+async def test_a_live_threshold_without_a_probability_warns(self, mocker: MockerFixture) -> None:warning = mocker.patch("pipelex.kernel.judgment_ops.log.warning")- _, result = await self._run(mocker, answer=YesNoAnswer(yes_no=True), threshold=0.8, run_mode=PipeRunMode.LIVE)+ _, result = await self._run(mocker, outcome=YesNoAnswer(yes_no=True), threshold=0.8)assert result.threshold_applied is Falsewarning.assert_called_once()@@ -96,7 +152,7 @@ class TestRunJudgment:async def test_a_dry_threshold_without_a_probability_neither_warns_nor_records(self, mocker: MockerFixture) -> None:warning = mocker.patch("pipelex.kernel.judgment_ops.log.warning")- _, result = await self._run(mocker, answer=YesNoAnswer(yes_no=True), threshold=0.8, run_mode=PipeRunMode.DRY)+ _, result = await self._run(mocker, outcome=YesNoAnswer(yes_no=True), threshold=0.8, run_mode=PipeRunMode.DRY)assert result.threshold_applied is Nonewarning.assert_not_called()@@ -104,7 +160,7 @@ class TestRunJudgment:async def test_no_threshold_without_a_probability_does_not_warn(self, mocker: MockerFixture) -> None:warning = mocker.patch("pipelex.kernel.judgment_ops.log.warning")- _, result = await self._run(mocker, answer=YesNoAnswer(yes_no=True), threshold=None, run_mode=PipeRunMode.LIVE)+ _, result = await self._run(mocker, outcome=YesNoAnswer(yes_no=True), threshold=None)assert result.threshold_applied is Nonewarning.assert_not_called()diff --git a/tests/unit/pipelex/language/test_mthds_schema.py b/tests/unit/pipelex/language/test_mthds_schema.pyindex bc2c85de4..a1cc402ba 100644--- a/tests/unit/pipelex/language/test_mthds_schema.py+++ b/tests/unit/pipelex/language/test_mthds_schema.py@@ -26,7 +26,7 @@ _PIPE_KIND_EXTRA_FIELDS: dict[str, dict[str, Any]] = {"PipeSearch": {"prompt": "find it"},"PipeStructure": {},"PipeDocGen": {"format": "pdf"},- "PipeJudge": {"question": "is it?"},+ "PipeJudge": {"prompt": "@message", "question": "is it?"},"PipeBatch": {"branch_pipe_code": "sub_pipe", "input_list_name": "items", "input_item_name": "item"},"PipeCondition": {"default_outcome": "fallback_pipe", "outcomes": {"yes": "yes_pipe"}},"PipeParallel": {"branches": [{"pipe": "sub_pipe"}]},@@ -286,21 +286,42 @@ class TestMthdsSchemaGeneration:assert not errors, f"{pipe_type} table should match exactly one oneOf arm, got errors: {[e.message for e in errors]}"@pytest.mark.parametrize(- ("question_fields", "should_validate"),+ ("judge_fields", "should_validate"),[- pytest.param({"question": "is it?"}, True, id="question"),- pytest.param({"prompt": "is it?"}, True, id="prompt-synonym"),- pytest.param({"question": "is it?", "prompt": "is it?"}, False, id="both"),+ pytest.param({"prompt": "@message", "question": "is it?"}, True, id="prompt-and-question"),+ pytest.param({"question": "is it?"}, False, id="question-alone"),+ pytest.param({"prompt": "@message"}, False, id="prompt-alone"),pytest.param({}, False, id="neither"),],)- def test_pipe_judge_takes_its_question_or_the_prompt_synonym(- self, schema: dict[str, Any], question_fields: dict[str, Any], should_validate: bool+ def test_pipe_judge_takes_its_evidence_prompt_and_its_question(+ self, schema: dict[str, Any], judge_fields: dict[str, Any], should_validate: bool) -> None:- """A PipeJudge writes its question as `question` or as `prompt`, exactly one, as its blueprint reads it."""+ """A PipeJudge writes the evidence in `prompt` and asks about it in `question`, both required, as its blueprint reads them."""validator = _pipe_union_oneof_validator(schema)- table = {"type": "PipeJudge", "description": "A judge", "output": "YesNo", **question_fields}- assert validator.is_valid(table) is should_validate, f"{sorted(question_fields)} should {'' if should_validate else 'not '}validate"+ table = {"type": "PipeJudge", "description": "A judge", "output": "YesNo", **judge_fields}+ assert validator.is_valid(table) is should_validate, f"{sorted(judge_fields)} should {'' if should_validate else 'not '}validate"++ @pytest.mark.parametrize(+ ("levels", "should_validate"),+ [+ pytest.param(["Minor", "Major"], True, id="descriptions-as-strings"),+ pytest.param([{"label": "Low", "description": "A cosmetic flaw"}, {"label": "High"}], True, id="labelled-tables"),+ pytest.param(["Minor", {"description": "Nothing works"}], True, id="strings-and-unlabelled-tables"),+ pytest.param(["Minor", {"label": "High", "description": "Nothing works"}], False, id="strings-and-labelled-tables"),+ pytest.param([{"label": "Low"}, {"description": "Nothing works"}], False, id="labelled-and-unlabelled-tables"),+ pytest.param([{"label": "Low", "colour": "green"}], False, id="table-with-an-unknown-key"),+ pytest.param([{"label": 3}], False, id="label-not-a-string"),+ ],+ )+ def test_pipe_judge_levels_are_strings_or_closed_tables(self, schema: dict[str, Any], levels: list[Any], should_validate: bool) -> None:+ """A rating level is written as its description or as a closed `{label, description}` table, and a scale labels every level or none.++ The load refuses a scale labelling some of its levels only (`_validate_levels`), so the schema refuses it first.+ """+ validator = _pipe_union_oneof_validator(schema)+ table = {**_minimal_pipe_table("PipeJudge"), "output": "Rating", "levels": levels}+ assert validator.is_valid(table) is should_validate, f"levels={levels!r} should {'' if should_validate else 'not '}validate"@pytest.mark.parametrize(("size_value", "should_validate"),diff --git a/tests/unit/pipelex/pipe_machinery/test_data.py b/tests/unit/pipelex/pipe_machinery/test_data.pyindex b7db98d98..92cb27138 100644--- a/tests/unit/pipelex/pipe_machinery/test_data.py+++ b/tests/unit/pipelex/pipe_machinery/test_data.py@@ -58,6 +58,7 @@ filename = "invoice-{{ invoice.number }}"description = "Judges whether an invoice is overdue"inputs = { invoice = "Invoice" }output = "YesNo"+prompt = "@invoice"question = "Is this invoice overdue?"""",PipeType.PIPE_BATCH: """type = "PipeBatch"diff --git a/tests/unit/pipelex/pipe_operators/pipe_judge/data.py b/tests/unit/pipelex/pipe_operators/pipe_judge/data.pyindex 9f4ec101d..644f0968a 100644--- a/tests/unit/pipelex/pipe_operators/pipe_judge/data.py+++ b/tests/unit/pipelex/pipe_operators/pipe_judge/data.py@@ -4,8 +4,17 @@ from pipelex.cogt.judgment.judgment_models import JudgmentKinddef _judge(**fields: Any) -> dict[str, Any]:- """A PipeJudge blueprint's fields: a yes/no question over one message, with the given fields on top."""- blueprint: dict[str, Any] = {"description": "d", "inputs": {"message": "Text"}, "output": "YesNo", "question": "Is it urgent?"}+ """A PipeJudge blueprint's fields: a yes/no question over the evidence of one message, with the given fields on top.++ A field given as `None` is left out, which is how a case drops one of the defaults.+ """+ blueprint: dict[str, Any] = {+ "description": "d",+ "inputs": {"message": "Text"},+ "output": "YesNo",+ "prompt": "A message from a customer:\n@message",+ "question": "Is it urgent?",+ }blueprint.update(fields)return {key: value for key, value in blueprint.items() if value is not None}@@ -14,14 +23,22 @@ class PipeJudgeBlueprintTestCases:"""Each refusal is (test_id, blueprint fields, a fragment of its message); each acceptance is (test_id, fields, the kind it resolves to)."""REFUSED: ClassVar[list[tuple[str, dict[str, Any], str]]] = [- ("both_spellings", _judge(prompt="Is it urgent?"), "sets `question`, or `prompt` as its synonym, but not both"),+ (+ "prompt_as_the_question",+ _judge(question=None, prompt="Is the message urgent?"),+ "`prompt` holds the evidence the question is asked over, and the question is written in `question`",+ ),+ ("no_question_at_all", _judge(question=None), "the question is written in `question`"),+ ("no_prompt", _judge(prompt=None), "judges the evidence its `prompt` presents, and this one sets no `prompt`"),+ ("empty_prompt", _judge(prompt=""), "`prompt` cannot be empty"),+ ("blank_prompt", _judge(prompt=" \n "), "`prompt` cannot be empty"),("empty_question", _judge(question=""), "`question` cannot be empty"),("blank_question", _judge(question=" \n "), "`question` cannot be empty"),("dotted_input", _judge(inputs={"invoice.total": "Number"}), "Input 'invoice.total' is not a plain input name"),("options_and_levels", _judge(output="Choice", options={"a": "", "b": ""}, levels=["low", "high"]), "not both"),("criteria_beside_options",- _judge(output="Choice", options={"a": "", "b": ""}, criteria={"yes": "y"}),+ _judge(output="Choice", options={"a": "", "b": ""}, criteria={"yes": "y", "no": "n"}),"`criteria` applies to a yes/no question only",),("threshold_beside_levels", _judge(output="Rating", levels=["low", "high"], threshold=0.7), "`threshold` applies to a yes/no question only"),@@ -29,17 +46,68 @@ class PipeJudgeBlueprintTestCases:("empty_option_key", _judge(output="Choice", options={"": "nothing", "b": ""}), "none may be empty"),("one_level", _judge(output="Rating", levels=["only"]), "at least two `levels`"),("empty_level", _judge(output="Rating", levels=["low", " "]), "level 1 is"),+ ("empty_level_table", _judge(output="Rating", levels=["low", {}]), "carries a `label`, a `description` or both"),+ ("blank_level_label", _judge(output="Rating", levels=[{"label": "Low"}, {"label": " ", "description": "High"}]), "level 1"),+ ("unknown_level_key", _judge(output="Rating", levels=[{"label": "Low"}, {"label": "High", "score": 3}]), "score"),+ (+ "labels_mixed_with_unlabelled_levels",+ _judge(output="Rating", levels=[{"label": "Low", "description": "Barely"}, "Severe"]),+ "every level carries a `label` or none does",+ ),+ (+ "duplicated_labels",+ _judge(output="Rating", levels=[{"label": "Low"}, {"label": "High"}, {"label": "Low"}]),+ "the labels of one scale are distinct, and 'Low' is repeated",+ ),("threshold_zero", _judge(threshold=0), "strictly between 0 and 1"),("threshold_one", _judge(threshold=1), "strictly between 0 and 1"),- ("unknown_criteria_key", _judge(criteria={"yes": "y", "maybe": "m"}), "maybe"),+ ("unknown_criteria_key", _judge(criteria={"yes": "y", "no": "n", "maybe": "m"}), "maybe"),+ (+ "criteria_yes_alone",+ _judge(criteria={"yes": "The customer needs an answer today"}),+ "Criteria describe both answers, and these declare `yes` without `no`: write `no` as the complement of `yes`",+ ),+ (+ "criteria_no_alone",+ _judge(criteria={"no": "The message can wait"}),+ "Criteria describe both answers, and these declare `no` without `yes`: write `yes` as the complement of `no`",+ ),+ (+ "criteria_empty_table",+ _judge(criteria={}),+ "Criteria describe both answers, and this table declares neither: write both `yes` and `no`, or remove the table",+ ),+ ("criteria_empty_side", _judge(criteria={"yes": "Today", "no": " "}), "`no` cannot be empty"),("output_with_brackets", _judge(output="YesNo[]"), "produces one verdict"),- ("undeclared_variable", _judge(question="Is $message about $topic?"), "Variable 'topic' is read by the question but not declared"),+ ("undeclared_variable_in_question", _judge(question="Is it about $topic?"), "Variable 'topic' is read by the prompt or question"),+ ("undeclared_variable_in_prompt", _judge(prompt="@message\nFrom: $sender"), "Variable 'sender' is read by the prompt or question"),+ (+ "unread_input",+ _judge(inputs={"message": "Text", "history": "Text"}),+ "Input 'history' is declared but never read by the prompt or question",+ ),]ACCEPTED: ClassVar[list[tuple[str, dict[str, Any], JudgmentKind]]] = [- ("question_yes_no", _judge(threshold=0.8, criteria={"yes": "needs an answer today", "no": "can wait"}), JudgmentKind.YES_NO),- ("prompt_alone", {**_judge(question=None), "prompt": "Is it urgent?"}, JudgmentKind.YES_NO),- ("unreferenced_input", _judge(inputs={"message": "Text", "history": "Text"}, question="Is $message urgent?"), JudgmentKind.YES_NO),+ ("yes_no_with_both_criteria", _judge(threshold=0.8, criteria={"yes": "needs an answer today", "no": "can wait"}), JudgmentKind.YES_NO),+ (+ "input_read_by_the_question_alone",+ _judge(inputs={"message": "Text", "topic": "Text"}, question="Is it about $topic?"),+ JudgmentKind.YES_NO,+ ),("empty_string_option_description", _judge(output="Choice", options={"billing": "", "technical": "Bugs"}), JudgmentKind.CHOICE),- ("rating", _judge(output="Rating", levels=["cosmetic", "a workaround exists", "blocking"]), JudgmentKind.RATING),+ ("rating_described_by_strings", _judge(output="Rating", levels=["cosmetic", "a workaround exists", "blocking"]), JudgmentKind.RATING),+ (+ "rating_labelled_and_described",+ _judge(+ output="Rating",+ levels=[+ {"label": "Cosmetic", "description": "Appearance only"},+ {"label": "Workaround available"},+ {"label": "Fully blocked", "description": "A task fails with no workaround"},+ ],+ ),+ JudgmentKind.RATING,+ ),+ ("rating_unlabelled_tables_and_strings", _judge(output="Rating", levels=[{"description": "Barely"}, "Severe"]), JudgmentKind.RATING),]diff --git a/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.py b/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.pyindex ca369136e..77a34bb82 100644--- a/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.py+++ b/tests/unit/pipelex/pipe_operators/pipe_judge/test_pipe_judge_blueprint.py@@ -4,7 +4,7 @@ import pytestfrom pydantic import ValidationErrorfrom pipelex.cogt.judgment.judgment_models import JudgmentKind-from pipelex.pipe_operators.judge.pipe_judge_blueprint import PipeJudgeBlueprint+from pipelex.pipe_operators.judge.pipe_judge_blueprint import JudgeRatingLevel, PipeJudgeBlueprintfrom tests.unit.pipelex.pipe_operators.pipe_judge.data import PipeJudgeBlueprintTestCases@@ -20,7 +20,23 @@ class TestPipeJudgeBlueprint:blueprint = PipeJudgeBlueprint.model_validate(blueprint_fields)assert blueprint.judgment_kind == expected_kind, test_id- def test_prompt_is_read_as_question(self):- blueprint = PipeJudgeBlueprint.model_validate({"description": "d", "output": "YesNo", "prompt": "Is it urgent?"})+ def test_the_prompt_and_the_question_are_two_fields(self):+ """`prompt` is the evidence and `question` the instruction, so neither is read as the other."""+ blueprint = PipeJudgeBlueprint.model_validate(+ {"description": "d", "inputs": {"message": "Text"}, "output": "YesNo", "prompt": "@message", "question": "Is it urgent?"}+ )+ assert blueprint.prompt == "@message"assert blueprint.question == "Is it urgent?"- assert "prompt" not in blueprint.model_dump()++ def test_a_level_string_and_a_level_table_both_parse(self):+ blueprint = PipeJudgeBlueprint.model_validate(+ {+ "description": "d",+ "inputs": {"report": "Text"},+ "output": "Rating",+ "prompt": "@report",+ "question": "How severe?",+ "levels": ["Barely", {"description": "Severe"}],+ }+ )+ assert blueprint.levels == ["Barely", JudgeRatingLevel(description="Severe")]diff --git a/tests/unit/pipelex/pipe_operators/pipe_llm/test_template_document_analyzer.py b/tests/unit/pipelex/pipe_operators/pipe_llm/test_template_document_analyzer.pyindex 4077706fc..1eabd96c1 100644--- a/tests/unit/pipelex/pipe_operators/pipe_llm/test_template_document_analyzer.py+++ b/tests/unit/pipelex/pipe_operators/pipe_llm/test_template_document_analyzer.py@@ -4,7 +4,7 @@ from pathlib import Pathfrom typing import Callablefrom pipelex.kernel.prompt_references import DocumentReferenceKind-from pipelex.pipe_operators.llm.template_document_analyzer import TemplateDocumentAnalyzer+from pipelex.pipe_operators.shared.template_document_analyzer import TemplateDocumentAnalyzerclass TestTemplateDocumentAnalyzer:diff --git a/tests/unit/pipelex/pipe_operators/shared/test_template_file_references.py b/tests/unit/pipelex/pipe_operators/shared/test_template_file_references.pynew file mode 100644index 000000000..74b7cd7f7--- /dev/null+++ b/tests/unit/pipelex/pipe_operators/shared/test_template_file_references.py@@ -0,0 +1,52 @@+"""The shared analysis of the files a prompt template reads, which every prompt-shaped factory calls."""++from pathlib import Path+from typing import Callable++import pytest++from pipelex.kernel.prompt_references import DocumentReferenceKind, ImageReferenceKind+from pipelex.pipe_operators.shared.template_file_references import analyze_template_file_references+++class TestTemplateFileReferences:+ def test_images_and_documents_are_found_in_one_pass(self, load_test_library: Callable[[list[Path]], None]) -> None:+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])++ file_references = analyze_template_file_references(+ template_source="Compare $note with $photo and $claim, then $album and $annexes",+ input_specs={"note": "Text", "photo": "Image", "claim": "Document", "album": "Image[]", "annexes": "Document[]"},+ domain_code="test_pipes",+ )++ assert [(reference.variable_path, reference.kind) for reference in file_references.image_references] == [+ ("photo", ImageReferenceKind.DIRECT),+ ("album", ImageReferenceKind.DIRECT_LIST),+ ]+ assert [(reference.variable_path, reference.kind) for reference in file_references.document_references] == [+ ("claim", DocumentReferenceKind.DIRECT),+ ("annexes", DocumentReferenceKind.DIRECT_LIST),+ ]++ @pytest.mark.parametrize(+ ("presence_suffix", "is_optional"),+ [+ pytest.param("?", True, id="optional"),+ pytest.param("", False, id="required"),+ pytest.param("!", False, id="forced"),+ ],+ )+ def test_each_reference_says_whether_its_root_input_is_optional(+ self, load_test_library: Callable[[list[Path]], None], presence_suffix: str, is_optional: bool+ ) -> None:+ """The assembly skips an absent optional input's file and refuses an absent required one, so the mark must be exact."""+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])++ file_references = analyze_template_file_references(+ template_source="@?photo\n@?claim\n",+ input_specs={"photo": f"Image{presence_suffix}", "claim": f"Document{presence_suffix}"},+ domain_code="test_pipes",+ )++ assert [reference.is_optional for reference in file_references.image_references] == [is_optional]+ assert [reference.is_optional for reference in file_references.document_references] == [is_optional]diff --git a/tests/unit/pipelex/providers/typesafe/test_data.py b/tests/unit/pipelex/providers/typesafe/test_data.pyindex 4a2109a87..548a04b84 100644--- a/tests/unit/pipelex/providers/typesafe/test_data.py+++ b/tests/unit/pipelex/providers/typesafe/test_data.py@@ -1,7 +1,13 @@"""Replay material for the TypeSafe backend, recorded against the live API.-Every file under ``tests/data/typesafe/responses/`` was captured by the campaign's spike and is-kept verbatim. A success is the **wire body**, taken from the raw HTTP response rather than from+Two sets live here. The shared judge wire fixtures under ``tests/data/judgment/wire/typesafe/`` are a+byte-identical copy of the hosted gateway's, laid out as its ``docs/judge-wire-fixtures.md`` says: per+case, the neutral request, the vendor request a translation must send, the vendor response and the+neutral outcome a translation must return. Both implementations of this vendor replay them, so a case+this worker disagrees with is a disagreement between the two, settled by the fixture and the design.++The older recordings under ``tests/data/typesafe/responses/`` were captured by the campaign's spike and+are kept verbatim. A success is the **wire body**, taken from the raw HTTP response rather than fromthe SDK's own dump, so a replay test checks our reading of the API and not the SDK's reading ofitself — which matters here, because a rating answer's keys are strings on the wire and integersonly after the SDK has parsed them. A refusal is saved as an envelope describing the exception,@@ -11,13 +17,123 @@ with the wire body nested under ``body``, and is rebuilt into a real SDK exceptiimport jsonimport refrom pathlib import Path-from typing import Any, ClassVar, cast+from typing import Any, ClassVar, NamedTuple, castimport typesafe_sdkfrom httpx2 import Headers+from pydantic import BaseModel, ConfigDictfrom typesafe_sdk import SystemOneResponse, TypeSafeAPIError, TypeSafeAPITimeoutError, TypeSafeError+from pipelex.cogt.exceptions import CogtError, InferenceErrorCategory, JudgmentJobFailureError, JudgmentModelNotFoundError+from pipelex.cogt.inference.error_classification import UserActionKind+from pipelex.cogt.judgment.judgment_models import JudgmentOutcome, JudgmentPrompt, JudgmentQuestion+from pipelex.providers.typesafe.typesafe_exceptions import TypesafeQuestionUnsupportedError+RESPONSES_DIR = Path(__file__).parents[4] / "data" / "typesafe" / "responses"+WIRE_DIR = Path(__file__).parents[4] / "data" / "judgment" / "wire" / "typesafe"+++class WireJudgeRequest(BaseModel):+ """A case's ``request.json``: the neutral request as a client posts it to the gateway's judge route."""++ model_config = ConfigDict(extra="forbid")++ model: str+ prompt: JudgmentPrompt+ questions: dict[str, JudgmentQuestion]+++class WireUsage(BaseModel):+ model_config = ConfigDict(extra="forbid")++ input_tokens: int | None = None+ output_tokens: int | None = None+++class WireJudgeAnswers(BaseModel):+ model_config = ConfigDict(extra="forbid")++ answers: dict[str, JudgmentOutcome]+ usage: WireUsage+++class WireJudgeError(BaseModel):+ model_config = ConfigDict(extra="forbid")++ type: str+ code: str+++class WireJudgeResponse(BaseModel):+ """A case's ``response.json``: ``{status, body}`` for an answer, ``{status, error}`` for a failure."""++ model_config = ConfigDict(extra="forbid")++ status: int+ body: WireJudgeAnswers | None = None+ error: WireJudgeError | None = None+++class WireVendorResponse(BaseModel):+ """A case's ``vendor_response.json``: what TypeSafe answered, its body untouched."""++ model_config = ConfigDict(extra="forbid")++ status: int+ body: Any+ headers: dict[str, str] | None = None+++class WireCase(NamedTuple):+ name: str+ request: WireJudgeRequest+ response: WireJudgeResponse+ vendor_request: dict[str, Any] | None+ vendor_response: WireVendorResponse | None+++def _read_wire_json(*, case_dir: Path, file_name: str) -> Any:+ return json.loads((case_dir / file_name).read_text(encoding="utf-8"))+++def load_wire_case(name: str) -> WireCase:+ case_dir = WIRE_DIR / name+ has_vendor_exchange = (case_dir / "vendor_request.json").exists()+ return WireCase(+ name=name,+ request=WireJudgeRequest.model_validate(_read_wire_json(case_dir=case_dir, file_name="request.json")),+ response=WireJudgeResponse.model_validate(_read_wire_json(case_dir=case_dir, file_name="response.json")),+ vendor_request=_read_wire_json(case_dir=case_dir, file_name="vendor_request.json") if has_vendor_exchange else None,+ vendor_response=(+ WireVendorResponse.model_validate(_read_wire_json(case_dir=case_dir, file_name="vendor_response.json")) if has_vendor_exchange else None+ ),+ )+++def wire_case_names() -> list[str]:+ return sorted(entry.name for entry in WIRE_DIR.iterdir() if entry.is_dir())+++class WireErrorMapping(NamedTuple):+ """What this worker raises for a gateway error code: the gateway answers HTTP, the worker raises its own error."""++ status: int+ error_class: type[CogtError]+ category: InferenceErrorCategory+ user_action_kind: UserActionKind+++# The mapping is pipelex's to keep, as the fixtures' documentation says. The status is the gateway's,+# held here so a case whose status moves is noticed rather than silently re-read.+WIRE_ERROR_MAPPINGS: dict[str, WireErrorMapping] = {+ "typesafe_context_too_long": WireErrorMapping(400, JudgmentJobFailureError, InferenceErrorCategory.CONTENT, UserActionKind.CHANGE_INPUT),+ "typesafe_too_many_levels": WireErrorMapping(400, TypesafeQuestionUnsupportedError, InferenceErrorCategory.CONTENT, UserActionKind.CHANGE_INPUT),+ "typesafe_authentication_error": WireErrorMapping(+ 401, JudgmentJobFailureError, InferenceErrorCategory.CONFIGURATION, UserActionKind.CHECK_CREDENTIALS+ ),+ "typesafe_unknown_model": WireErrorMapping(400, JudgmentModelNotFoundError, InferenceErrorCategory.CONFIGURATION, UserActionKind.CHANGE_MODEL),+ "typesafe_invalid_question": WireErrorMapping(400, JudgmentJobFailureError, InferenceErrorCategory.CONFIGURATION, UserActionKind.CONTACT_SUPPORT),+}def load_recorded(name: str) -> dict[str, Any]:diff --git a/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.py b/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.pyindex 452d8b5a8..187c903cb 100644--- a/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.py+++ b/tests/unit/pipelex/providers/typesafe/test_typesafe_judgment_worker.py@@ -12,7 +12,7 @@ from pipelex.cogt.image.prompt_image import PromptImage, PromptImageUrifrom pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstractfrom pipelex.cogt.judgment.judgment_job import JudgmentJobfrom pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory-from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentQuestion, RatingQuestion, YesNoAnswer, YesNoQuestion+from pipelex.cogt.judgment.judgment_models import ChoiceQuestion, JudgmentPrompt, JudgmentQuestion, RatingQuestion, YesNoAnswer, YesNoQuestionfrom pipelex.cogt.judgment.judgment_setting import JudgmentSettingfrom pipelex.cogt.llm.thinking_mode import ThinkingModefrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec@@ -22,6 +22,7 @@ from pipelex.cogt.usage.token_category import TokenCategoryfrom pipelex.providers.typesafe.typesafe_judgment_worker import TypesafeJudgmentWorkerfrom pipelex.reporting.reporting_protocol import ReportingNoOpfrom pipelex.system.job_metadata import JobMetadata, RunMetadata+from tests.unit.pipelex.cogt.judgment.test_data import described_levelsfrom tests.unit.pipelex.providers.typesafe.test_data import TestData, load_recorded_response, rebuild_recorded_exceptionTHREE_SHAPES_QUESTIONS: dict[str, JudgmentQuestion] = {@@ -32,7 +33,7 @@ THREE_SHAPES_QUESTIONS: dict[str, JudgmentQuestion] = {),"severity": RatingQuestion(instructions=TestData.THREE_SHAPES_INSTRUCTIONS["severity"],- levels=["Cosmetic", "Degraded with a workaround", "Blocking"],+ levels=described_levels("Cosmetic", "Degraded with a workaround", "Blocking"),),}@@ -64,7 +65,9 @@ def _model(*, inputs: list[str] | None = None) -> InferenceModelSpec:def _job(questions: dict[str, JudgmentQuestion]) -> JudgmentJob:return JudgmentJobFactory.make_judgment_job(- state={"message": "Our production checkout has been returning 500s for every card payment since 09:14 UTC."},+ prompt=JudgmentPrompt(+ text="A message from a customer: Our production checkout has been returning 500s for every card payment since 09:14 UTC."+ ),questions=questions,judgment_setting=JudgmentSetting(model="jev-1.13.0"),job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id="run_typesafe")),@@ -76,29 +79,23 @@ class TestTypesafeJudgmentWorker:@pytest.mark.parametrize(("inputs", "images", "documents"),[- pytest.param(- ["text", "images"], {"photo": [PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")]}, None, id="image"- ),- pytest.param(- ["text", "pdf"], None, {"claim": [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")]}, id="document"- ),+ pytest.param(["text", "images"], [PromptImageUri(uri="pipelex-storage://s/photo.png", mime_type="image/png")], [], id="image"),+ pytest.param(["text", "pdf"], [], [PromptDocumentUri(uri="pipelex-storage://s/claim.pdf", mime_type="application/pdf")], id="document"),],)async def test_a_job_carrying_files_is_refused_whatever_the_spec_claims(self,mocker: MockerFixture,inputs: list[str],- images: dict[str, list[PromptImage]] | None,- documents: dict[str, list[PromptDocument]] | None,+ images: list[PromptImage],+ documents: list[PromptDocument],) -> None:- """TypeSafe judges a JSON state alone, so a spec claiming it reads files would have its files dropped from the request."""+ """TypeSafe judges text alone, so a spec claiming it reads files would have its files dropped from the request."""client = mocker.Mock(spec=AsyncTypeSafeClient)client.system_one = mocker.AsyncMock(return_value=load_recorded_response(TestData.THREE_SHAPES))worker = TypesafeJudgmentWorker(sdk_instance=cast("AsyncTypeSafeClient", client), inference_model=_model(inputs=inputs))job = JudgmentJobFactory.make_judgment_job(- state={"message": "See the attached file."},- images=images,- documents=documents,+ prompt=JudgmentPrompt(text="See the attached file: [Image 1] [Document 1]", images=images, documents=documents),questions={"is_urgent": YesNoQuestion(instructions=TestData.THREE_SHAPES_INSTRUCTIONS["is_urgent"])},judgment_setting=JudgmentSetting(model="jev-1.13.0"),job_metadata=JobMetadata(@@ -125,7 +122,8 @@ class TestTypesafeJudgmentWorker:call = client.system_one.await_argsassert call is not Nonesent_state, sent_questions = call.args- assert sent_state == job.state+ # The rendered prompt goes out as the vendor's state, a string.+ assert sent_state == job.prompt.textassert set(sent_questions) == set(THREE_SHAPES_QUESTIONS)assert isinstance(sent_questions["is_urgent"], TypesafeNoul)assert call.kwargs == {"model": "jev-1.13.0"}diff --git a/tests/unit/pipelex/providers/typesafe/test_typesafe_translation.py b/tests/unit/pipelex/providers/typesafe/test_typesafe_translation.pyindex 5a8ca2e08..564951dbf 100644--- a/tests/unit/pipelex/providers/typesafe/test_typesafe_translation.py+++ b/tests/unit/pipelex/providers/typesafe/test_typesafe_translation.py@@ -9,8 +9,10 @@ from pipelex.cogt.judgment.judgment_models import (ChoiceQuestion,JudgmentQuestion,RatingAnswer,+ RatingLevel,RatingQuestion,YesNoAnswer,+ YesNoCriteria,YesNoQuestion,)from pipelex.providers.typesafe.typesafe_exceptions import TypesafeJudgmentResponseError, TypesafeQuestionUnsupportedError@@ -19,28 +21,27 @@ from pipelex.providers.typesafe.typesafe_translation import (from_typesafe_response,to_typesafe_question,)+from tests.unit.pipelex.cogt.judgment.test_data import described_levelsfrom tests.unit.pipelex.providers.typesafe.test_data import TestData, load_recorded, load_recorded_response-SEVERITY_LEVELS = [+SEVERITY_DESCRIPTIONS = ["Cosmetic; no impact on functionality","A feature is degraded, but a workaround exists","Blocking issue; no workaround exists",]+SEVERITY_LEVELS = described_levels(*SEVERITY_DESCRIPTIONS)class TestTypesafeTranslation:- def test_yes_no_sends_only_the_criteria_it_declares(self) -> None:- """Both sides go out under the vendor's own keys, one side alone goes out alone, and neither sends no map at all."""+ def test_yes_no_sends_both_criteria_or_no_map_at_all(self) -> None:+ """Both sides go out under the vendor's own keys; a question without criteria sends no map at all."""both = to_typesafe_question(- question_key="q", question=YesNoQuestion(instructions="Is it urgent?", yes_criterion="Needs attention now", no_criterion="Can wait")+ question_key="q", question=YesNoQuestion(instructions="Is it urgent?", criteria=YesNoCriteria(yes="Needs attention now", no="Can wait")))- yes_only = to_typesafe_question(question_key="q", question=YesNoQuestion(instructions="Is it urgent?", yes_criterion="Needs attention now"))neither = to_typesafe_question(question_key="q", question=YesNoQuestion(instructions="Is it urgent?"))assert isinstance(both, TypesafeNoul)assert both.model_dump()["criteria"] == {"true": "Needs attention now", "false": "Can wait"}- assert isinstance(yes_only, TypesafeNoul)- assert yes_only.model_dump()["criteria"] == {"true": "Needs attention now"}assert isinstance(neither, TypesafeNoul)assert "criteria" not in neither.model_dump()assert neither.model_dump()["instructions"] == "Is it urgent?"@@ -56,7 +57,31 @@ class TestTypesafeTranslation:rendered = to_typesafe_question(question_key="severity", question=RatingQuestion(instructions="How severe?", levels=SEVERITY_LEVELS))assert isinstance(rendered, TypesafeScore)- assert rendered.model_dump()["criteria"] == SEVERITY_LEVELS+ assert rendered.model_dump()["criteria"] == SEVERITY_DESCRIPTIONS++ def test_a_level_goes_out_as_its_label_and_description_or_whichever_it_declares(self) -> None:+ """The vendor takes one string per level, so a labelled level is joined as `label: description`."""+ question = RatingQuestion(+ instructions="How severe?",+ levels=[+ RatingLevel(label="Cosmetic", description="Appearance only"),+ RatingLevel(label="Workaround available"),+ RatingLevel(label="Fully blocked", description="A task fails with no workaround"),+ ],+ )+ unlabelled = RatingQuestion(instructions="How severe?", levels=described_levels("Appearance only", "A task fails"))++ rendered = to_typesafe_question(question_key="severity", question=question)+ rendered_unlabelled = to_typesafe_question(question_key="severity", question=unlabelled)++ assert isinstance(rendered, TypesafeScore)+ assert rendered.model_dump()["criteria"] == [+ "Cosmetic: Appearance only",+ "Workaround available",+ "Fully blocked: A task fails with no workaround",+ ]+ assert isinstance(rendered_unlabelled, TypesafeScore)+ assert rendered_unlabelled.model_dump()["criteria"] == ["Appearance only", "A task fails"]@pytest.mark.parametrize(("nb_levels", "is_accepted"),@@ -67,7 +92,7 @@ class TestTypesafeTranslation:)def test_the_vendor_rating_cap_is_enforced_at_its_boundary(self, nb_levels: int, is_accepted: bool) -> None:"""Ten levels go out and eleven are refused before any request is built, as the live API refuses them."""- question = RatingQuestion(instructions="How severe?", levels=[f"Level {index_level}" for index_level in range(nb_levels)])+ question = RatingQuestion(instructions="How severe?", levels=described_levels(*(f"Level {index_level}" for index_level in range(nb_levels))))if is_accepted:assert isinstance(to_typesafe_question(question_key="q", question=question), TypesafeScore)return@@ -110,7 +135,11 @@ class TestTypesafeTranslation:def test_a_spread_rating_names_its_most_probable_level_not_its_position(self) -> None:"""The recorded ten-level answer sits at position 7.86 but is most probable at level 9 — the verdict is 9."""answers = from_typesafe_response(- questions={"q": RatingQuestion(instructions="How severe?", levels=[f"Severity level {index_level}" for index_level in range(10)])},+ questions={+ "q": RatingQuestion(+ instructions="How severe?", levels=described_levels(*(f"Severity level {index_level}" for index_level in range(10)))+ )+ },response=load_recorded_response(TestData.SCORE_TEN_LEVELS),)answer = answers["q"]@@ -137,7 +166,7 @@ class TestTypesafeTranslation:f'"probabilities": {wire_probabilities}}}}}}}')answers = from_typesafe_response(- questions={"q": RatingQuestion(instructions="How severe?", levels=["a", "b", "c"])},+ questions={"q": RatingQuestion(instructions="How severe?", levels=described_levels("a", "b", "c"))},response=SystemOneResponse.model_validate_json(wire_body),)answer = answers["q"]@@ -151,7 +180,11 @@ class TestTypesafeTranslation:assert all(isinstance(level, str) for level in wire_probabilities)answers = from_typesafe_response(- questions={"q": RatingQuestion(instructions="How severe?", levels=[f"Severity level {index_level}" for index_level in range(10)])},+ questions={+ "q": RatingQuestion(+ instructions="How severe?", levels=described_levels(*(f"Severity level {index_level}" for index_level in range(10)))+ )+ },response=load_recorded_response(TestData.SCORE_TEN_LEVELS),)answer = answers["q"]diff --git a/tests/unit/pipelex/providers/typesafe/test_typesafe_wire_replay.py b/tests/unit/pipelex/providers/typesafe/test_typesafe_wire_replay.pynew file mode 100644index 000000000..952806582--- /dev/null+++ b/tests/unit/pipelex/providers/typesafe/test_typesafe_wire_replay.py@@ -0,0 +1,102 @@+import json+from typing import Any++import httpx2+import pytest+from typesafe_sdk import AsyncTypeSafeClient, RetryPolicy++from pipelex.cogt.exceptions import CogtError+from pipelex.cogt.judgment.judgment_job_factory import JudgmentJobFactory+from pipelex.cogt.judgment.judgment_setting import JudgmentSetting+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.cogt.usage.token_category import TokenCategory+from pipelex.providers.typesafe.typesafe_judgment_worker import TypesafeJudgmentWorker+from pipelex.system.job_metadata import JobMetadata, RunMetadata+from tests.unit.pipelex.providers.typesafe.test_data import WIRE_ERROR_MAPPINGS, WireCase, load_wire_case, wire_case_names++# Not a credential: the replay transport never leaves the process, and the SDK refuses to build a client without some key.+_REPLAY_ONLY_KEY = "wire-replay-sends-nothing"+++class _VendorReplay:+ """An HTTP transport that answers TypeSafe's half of one recorded exchange and keeps the request it was sent."""++ def __init__(self, *, case: WireCase) -> None:+ self.case = case+ self.sent_bodies: list[Any] = []++ def handle(self, request: httpx2.Request) -> httpx2.Response:+ self.sent_bodies.append(json.loads(request.content))+ vendor_response = self.case.vendor_response+ if vendor_response is None:+ msg = f"Wire case '{self.case.name}' records no vendor exchange, and the worker called the vendor"+ raise AssertionError(msg)+ return httpx2.Response(status_code=vendor_response.status, json=vendor_response.body, headers=vendor_response.headers)+++def _worker(*, case: WireCase, replay: _VendorReplay) -> TypesafeJudgmentWorker:+ client = AsyncTypeSafeClient(api_key=_REPLAY_ONLY_KEY, transport=httpx2.MockTransport(replay.handle), retry=RetryPolicy(max_retries=0))+ inference_model = InferenceModelSpec(+ backend_name="typesafe",+ name="wire-replay-judgment-model",+ sdk="typesafe",+ model_type=ModelType.JUDGMENT,+ model_id=case.request.model,+ inputs=["text"],+ outputs=["judgments"],+ costs={CostCategory.INPUT: 0.042, CostCategory.OUTPUT: 0},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ )+ return TypesafeJudgmentWorker(sdk_instance=client, inference_model=inference_model)++[email protected](loop_scope="class")+class TestTypesafeWireReplay:+ """Every shared wire case, replayed through the worker with a real SDK client over a recorded transport."""++ @pytest.mark.parametrize("case_name", wire_case_names())+ async def test_the_worker_sends_and_answers_exactly_what_the_case_records(self, case_name: str) -> None:+ case = load_wire_case(case_name)+ replay = _VendorReplay(case=case)+ worker = _worker(case=case, replay=replay)+ job = JudgmentJobFactory.make_judgment_job(+ prompt=case.request.prompt,+ questions=case.request.questions,+ judgment_setting=JudgmentSetting(model="wire-replay-judgment-model"),+ job_metadata=JobMetadata(+ run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="u", pipeline_run_id=f"run_wire_{case_name}")+ ),+ )+ expected = case.response++ if expected.error is not None:+ mapping = WIRE_ERROR_MAPPINGS[expected.error.code]+ assert expected.status == mapping.status+ with pytest.raises(CogtError) as exc_info:+ await worker.judge(job)+ assert type(exc_info.value) is mapping.error_class+ assert exc_info.value.error_category == mapping.category+ assert exc_info.value.user_action is not None+ assert exc_info.value.user_action.kind == mapping.user_action_kind+ else:+ assert expected.body is not None+ outcomes = await worker.judge(job)+ assert outcomes == expected.body.answers+ usage = job.job_report.judgment_tokens_usage+ assert usage is not None+ expected_tokens: dict[TokenCategory, int] = {}+ if expected.body.usage.input_tokens is not None:+ expected_tokens[TokenCategory.INPUT] = expected.body.usage.input_tokens+ if expected.body.usage.output_tokens is not None:+ expected_tokens[TokenCategory.OUTPUT] = expected.body.usage.output_tokens+ assert usage.nb_tokens_by_category == expected_tokens++ if case.vendor_request is None:+ assert replay.sent_bodies == []+ else:+ assert replay.sent_bodies == [case.vendor_request]