Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Render-pdf-local · L-260929-a1effd (#1305)
Louis Choquel512975da3c8b
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.tomlindex c95e9c9ba..7272d2ce2 100644--- a/.drift/acks/cli-docs.toml+++ b/.drift/acks/cli-docs.toml@@ -1,12 +1,12 @@contract = "cli-docs"-digest = "sha256:c6dad185bafd093de8ccfc4083426af035c8e8e738c7412e9ecbc387ba3e6955"+digest = "sha256:edfad164dcb9959a90b014f8779348db0d43a1caaa23efa75b3eea3503b6f3ab"reviewed_by = "Louis Choquel"-reviewed_at = "2026-09-29T11:38:40Z"-rationale = "The fetched-method structures warning now gives the shared remedy; docs/tools/cli/run-by-address.md updated to state the same route, agent CLI CLAUDE.md does not describe this warning"+reviewed_at = "2026-09-30T16:40:04Z"+rationale = "clean-pass: the main-output file copy now compares the names already in the results folder without case on every file system, which the CI failure on Linux exposed; docs/tools/cli/run.md already states the rule (a taken name gets a -1 suffix) and the agent CLI CLAUDE.md names none of this behaviour, so no doc change is needed."[trigger_files]"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"-"pipelex/cli/_cli.py" = "blob:899a40b2440a27a433a14f18d22ac8574855a358"+"pipelex/cli/_cli.py" = "blob:22b4a3719e54a0c5593bbb98fb2737293df34d8c""pipelex/cli/agent_cli/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/_agent_cli.py" = "blob:e49ea3d36aa0e964e6d2cc66e901f8362c9ae76b""pipelex/cli/agent_cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"@@ -25,7 +25,7 @@ rationale = "The fetched-method structures warning now gives the shared remedy;"pipelex/cli/agent_cli/commands/fix/app.py" = "blob:e6190d9bde6a05755c04bd21806f699461038ac0""pipelex/cli/agent_cli/commands/fix/bundle_cmd.py" = "blob:37b13a1ead7b55d601f53b23647c285fc85418cd""pipelex/cli/agent_cli/commands/fmt_cmd.py" = "blob:97d6fb6c328e0b718b1293bd6b73cba03edcb7dc"-"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:2ac42caed527dcd58c6acd82e2b697acefcfe046"+"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:f10c15f5f1c220cb5f9509336258eaa2d2a11967""pipelex/cli/agent_cli/commands/inputs/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/inputs/_inputs_core.py" = "blob:d37826d96099028a1ad5dbc5cbf8ce0cfdb01e30""pipelex/cli/agent_cli/commands/inputs/app.py" = "blob:7e19284687b334c684fb03381a8d9d20af1aa152"@@ -35,7 +35,7 @@ rationale = "The fetched-method structures warning now gives the shared remedy;"pipelex/cli/agent_cli/commands/lint_cmd.py" = "blob:91ad9cb4d618ba4c0b7da366cde2f74f38d350df""pipelex/cli/agent_cli/commands/migrate_cmd.py" = "blob:ce733b78df03e65c3c8a10d46b27b57c05ec054c""pipelex/cli/agent_cli/commands/models_cmd.py" = "blob:dde4c246a28eba9f54cd582f0b171c9c26194be8"-"pipelex/cli/agent_cli/commands/pipe_cmd.py" = "blob:e5b313aad1dbf88ce31ed7e2f6ad4a0e85337453"+"pipelex/cli/agent_cli/commands/pipe_cmd.py" = "blob:9bf0480fcd1164ec1120b2514724fc0f5fefd30e""pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:a1c530d110482485ce462f199b9e79fac29f8896""pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/run/_output_helpers.py" = "blob:40ed1700e0d5e86d7696eadd7f9887baeb8059f6"@@ -82,7 +82,7 @@ rationale = "The fetched-method structures warning now gives the shared remedy;"pipelex/cli/commands/codegen/inputs_cmd.py" = "blob:20738a9e42f0e5ef7ffc489093348af3cb4643a0""pipelex/cli/commands/codegen/types_cmd.py" = "blob:b7c9c7a3664c9fbe66f9b4c3c5ebf9ea1322445c""pipelex/cli/commands/crate_loading.py" = "blob:3d7d651931ecf1a15a3e2d72411b3212b55d74ef"-"pipelex/cli/commands/doctor_cmd.py" = "blob:26db32ff20d1c064bc421093825a05e980c9b770"+"pipelex/cli/commands/doctor_cmd.py" = "blob:930e8fc7b3d87fef36ce369fd2715b1e4aea9132""pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/fix/_diff_sandbox.py" = "blob:d39c0df92b95567f74881deb01f8c1b3119fc8ef""pipelex/cli/commands/fix/_fix_core.py" = "blob:dac7056fcaf50210c910e4cad1ba14328b7f8adc"@@ -91,7 +91,7 @@ rationale = "The fetched-method structures warning now gives the shared remedy;"pipelex/cli/commands/graph_cmd.py" = "blob:d54737af7efc9ab9bbf3f978050d017af0e27a41""pipelex/cli/commands/init/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/init/backends.py" = "blob:7823d8f323e826e36ca389bf3491576723786163"-"pipelex/cli/commands/init/command.py" = "blob:cc3449142cd39d6276a7b420f30e4ad0b0611409"+"pipelex/cli/commands/init/command.py" = "blob:15bc95ae53684bef41605cabde677508bcb58934""pipelex/cli/commands/init/config_files.py" = "blob:c13be6b8d2a8b211ce9ef0f4889ee131ffc3d2d8""pipelex/cli/commands/init/credentials.py" = "blob:7475c3082b8ee7530a76a52a8ab8bf18a7e21209""pipelex/cli/commands/init/ide_extension.py" = "blob:9f0ec1494c07062246ecc906bc55b7f2cf95c715"@@ -111,14 +111,15 @@ rationale = "The fetched-method structures warning now gives the shared remedy;"pipelex/cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/run/_inputs_file_loader.py" = "blob:dde781f846968b667a0acfb4b3e3cea01cef7049""pipelex/cli/commands/run/_inputs_path_resolver.py" = "blob:c8ec89d98718e18af7aaaded44c9f545be60adab"-"pipelex/cli/commands/run/_run_core.py" = "blob:5d654266156eac27505530d83cea2a55bc1c5e75"+"pipelex/cli/commands/run/_main_stuff_file.py" = "blob:ca806e55085aa2e5748492751521296ce1f42960"+"pipelex/cli/commands/run/_run_core.py" = "blob:12958ebb9bc89fec6f0e6faf0b79d031091f02b3""pipelex/cli/commands/run/app.py" = "blob:a7090b0ff6bf2f1d47827fa27d2a138096a2e148""pipelex/cli/commands/run/bundle_cmd.py" = "blob:cb420bf4879c863d64c5e20ed535c42732b720a2""pipelex/cli/commands/run/exceptions.py" = "blob:e9552226e597922b6ac0757f7b52eb880a2e1472""pipelex/cli/commands/run/method_cmd.py" = "blob:9f5c7b6ad8f7fc7d086a9e6253049c6e2ec3e7d8""pipelex/cli/commands/run/pipe_cmd.py" = "blob:f92fcb94e3ad55cf5aeec366a59fa7a74bcc9667""pipelex/cli/commands/show_cmd.py" = "blob:9492c9af966877f694ca61f7b1db3d151a1ddf56"-"pipelex/cli/commands/update_cmd.py" = "blob:06100ec3b9d6c66ca89a8c1536aa27983ae409ca"+"pipelex/cli/commands/update_cmd.py" = "blob:b2db6170ddd9c67f17d88096ba1f1796a897557d""pipelex/cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/validate/_validate_core.py" = "blob:237c57bd815b6fa75e1fee28069f75ec6c5b7726""pipelex/cli/commands/validate/app.py" = "blob:309061423af13f2f6c496a024306ad696af567ab"diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.tomlindex 2d3da1921..dbf1c75e1 100644--- a/.drift/acks/config-docs.toml+++ b/.drift/acks/config-docs.toml@@ -1,8 +1,8 @@contract = "config-docs"-digest = "sha256:05a48390a2fdf7bf2c9aad6905b900118d0b8f17b5d42fd2e534e1966fa89ba7"-reviewed_by = "Claude (Opus 5.5)"-reviewed_at = "2026-09-30T10:30:17Z"-rationale = "clean-pass: the only trigger change is one docstring sentence in pipelex/tools/secrets/secrets_config.py that stops citing a private design label (S4) for the external-provider config passthrough; SecretsConfig's fields, types and defaults are unchanged, so the pages under docs/configuration/, configs.py, config_cogt.py and the migration ledgers need nothing."+digest = "sha256:10a3b527deabf7461808de985a1961fadfafaa59ada33d90cc0ffd53cb2b4034"+reviewed_by = "Louis Choquel"+reviewed_at = "2026-09-30T15:10:39Z"+rationale = "clean-pass: merging dev into feature/Render-pdf-local joins two separately acked trigger changes, the kit manifest stamping for backends/internal.toml in config_loader.py (docs updated in inference-backend-config.md) and a docstring-only edit in secrets_config.py; the merge adds no new trigger content and no config model, default or validator changed, so the configuration docs and migration ledgers need nothing further."[trigger_files]"pipelex/cogt/config_cogt.py" = "blob:02f71cb4a6b01aebda59c2e31f262ef9c865e17b"@@ -13,7 +13,7 @@ rationale = "clean-pass: the only trigger change is one docstring sentence in pi"pipelex/pipelex.toml" = "blob:ca10eca8a5c04a73601ed1fc1f5fae0221ffadb0""pipelex/system/configuration/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc""pipelex/system/configuration/config_check.py" = "blob:2d5de29aba3d2ce205d3d82b4623f37b53108332"-"pipelex/system/configuration/config_loader.py" = "blob:ac384ce993933656682999c26afffb4c9f9b1975"+"pipelex/system/configuration/config_loader.py" = "blob:b1940ff11d9be7f9b8e113ec564d041ec53bb0ce""pipelex/system/configuration/config_model.py" = "blob:83b29dc6e93393c9b98c4353636caf0135c80339""pipelex/system/configuration/config_root.py" = "blob:c7d43851270912d17413cfc6a511247d3c7cc160""pipelex/system/configuration/config_surface.py" = "blob:96f685656a3f4355e8a073e8bf6184e020fea8f5"diff --git a/.drift/acks/hub-layering-convention.toml b/.drift/acks/hub-layering-convention.tomlindex d403d60c7..97994d9b0 100644--- a/.drift/acks/hub-layering-convention.toml+++ b/.drift/acks/hub-layering-convention.toml@@ -1,8 +1,8 @@contract = "hub-layering-convention"digest = "sha256:75768fa677bc8550119a8b20ce80c22e5edfa1040eee6b32fe574f7ae2019435"reviewed_by = "Louis Choquel"-reviewed_at = "2026-09-17T17:57:19Z"-rationale = "The runtime hub no longer imports Rich at module level. Its console accessor builds the Rich console lazily after checking that the cli extra is installed, and the console print target now stores the stream instead of building a console. No accessor was added, moved or removed and no layer placement changed, so the partition table in the hub-layering doc, which lists get_console as the console accessor, is still accurate. The new Rich import guard reuses the import graph of the hub-layering guard without changing it, and the rich-imports contribute doc says so."+reviewed_at = "2026-09-30T00:36:45Z"+rationale = "The document renderer registry is removed from the runtime hub again, since document engines register as inference backends of the doc_gen family, so runtime_hub.py and docs/contribute/hub-layering.md are both back to dev's state. No layer placement changed."[trigger_files]"pipelex/cli/dev_cli/commands/hub_layering_guard.py" = "blob:2dbadcbab7d44364a7b5f1d16268b23b26700a00"diff --git a/.pipelex/inference/backends/.kit_manifest.json b/.pipelex/inference/backends/.kit_manifest.jsonnew file mode 100644index 000000000..7d6f49bef--- /dev/null+++ b/.pipelex/inference/backends/.kit_manifest.json@@ -0,0 +1,6 @@+{+ "files": {+ "internal.toml": "30d0f6f9de1c4cba586e04acc370e431123954bef5cb05a54eef05134d590d4c"+ },+ "kit_version": "0.70.0"+}diff --git a/.pipelex/inference/backends/internal.toml b/.pipelex/inference/backends/internal.tomlindex 8fcc38d85..1503a8457 100644--- a/.pipelex/inference/backends/internal.toml+++ b/.pipelex/inference/backends/internal.toml@@ -9,6 +9,10 @@# - Each model is defined in its own section with the model name as the header# - Model costs are in USD per million tokens (input/output)#+# Managed by `pipelex update`, which refreshes this file from the kit and backs+# up a locally edited copy to <file>.bak.<timestamp> first, so local edits will+# not survive future runs. Declare your own models in a backend of your own.+## Documentation: https://docs.pipelex.com# Support: https://go.pipelex.com/discord#@@ -42,3 +46,23 @@ model_id = "extract-text"inputs = ["pdf", "image"]outputs = ["pages"]costs = {}++################################################################################+# DOCUMENT GENERATION ENGINES+################################################################################+#+# The engines a PipeDocGen step prints with. Their inputs are the sources they+# print from: "layout" is the auto-layout of the step's inputs, "html" an HTML+# template, "template_file" an office template file. Their outputs are the+# formats they print. reportlab-pdf is built into Pipelex. The other engines+# come with the Pipelex document generation plugin, pipelex-doc-gen, which+# declares them itself when it loads: they are never listed in this file.++# --- ReportLab PDF ------------------------------------------------------------+[reportlab-pdf]+model_type = "doc_gen"+sdk = "reportlab"+model_id = "print-pdf"+inputs = ["layout"]+outputs = ["pdf"]+costs = {}diff --git a/.pipelex/inference/deck/.kit_manifest.json b/.pipelex/inference/deck/.kit_manifest.jsonindex 30faa294f..e9c32f088 100644--- a/.pipelex/inference/deck/.kit_manifest.json+++ b/.pipelex/inference/deck/.kit_manifest.json@@ -3,7 +3,8 @@"1_llm_deck.toml": "1316c59eb52db62820d313ff92569b736a3f5427ab8c10f495a02b463f37c97a","2_img_gen_deck.toml": "00d570907cee9261882922afebc53556013e46d29419fce4cf189135e2849fd6","3_extract_deck.toml": "560a40bd3254b1be011cfb35340fef6c40d70136e5ddce33a961899f92fd70ba",- "4_search_deck.toml": "08e3bd1bee0f9f8ed0ffc735a932705392d0ecfbdeefce64c9815d4b2d8d7e22"+ "4_search_deck.toml": "08e3bd1bee0f9f8ed0ffc735a932705392d0ecfbdeefce64c9815d4b2d8d7e22",+ "5_doc_gen_deck.toml": "3f48b443eb9245a87c6d8051223982bf6134155868bb44a3b167c8e05077c372"},"kit_version": "0.61.0"}diff --git a/.pipelex/inference/deck/5_doc_gen_deck.toml b/.pipelex/inference/deck/5_doc_gen_deck.tomlnew file mode 100644index 000000000..cdb0ee676--- /dev/null+++ b/.pipelex/inference/deck/5_doc_gen_deck.toml@@ -0,0 +1,41 @@+####################################################################################################+# Pipelex Model Deck - Document Generation Configuration+####################################################################################################+#+# This file defines the default engine a PipeDocGen step prints with, for each format and source,+# when the step names none with `model`. A source is where the document comes from: "layout" is the+# auto-layout of the step's inputs, "html" an HTML template, "template_file" an office template file.+#+# Open Pipelex prints a pdf from the auto-layout, on its built-in engine reportlab-pdf, so that is+# the one default this file sets. The Pipelex document generation plugin, pipelex-doc-gen, declares+# the defaults for a pdf from a template and for xlsx, docx and pptx itself when it loads. You can+# still set any default yourself, for any format and source, in an x_custom_*.toml file, which+# overrides both this file and the plugin.+#+# Managed by `pipelex update`. To customize without conflict, create an+# x_custom_*.toml file in this directory — pipelex never tracks or+# overwrites x_custom_* files. Local edits to this file are backed up+# to <file>.bak.<timestamp> on update but will not survive future runs.+#+# Model Reference Syntax:+# - Preset: $preset_name or preset:preset_name+# - Alias: @alias_name or alias:alias_name+#+# Documentation: https://docs.pipelex.com+# Support: https://go.pipelex.com/discord+#+####################################################################################################++####################################################################################################+# Document Generation Default Choices, keyed "<format>.<source>"+####################################################################################################++[doc_gen.choice_defaults]+"pdf.layout" = "@default-pdf"++####################################################################################################+# Aliases+####################################################################################################++[doc_gen.aliases]+default-pdf = "reportlab-pdf"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6e2eb20ef..ea7e8b3b1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,20 @@# Changelog+## [Unreleased]++### Added++- **`PipeDocGen` generates a document file**: a new operator turns its inputs into a stored `Document` named by its `filename` expression, which `pipelex run` copies into the run's results folder. A `pdf` without a template is printed by a built-in engine on ReportLab from the auto-layout of the inputs, and a `pdf` with an HTML template or any `xlsx`, `docx` or `pptx` step is refused when the method loads, with an error naming the Pipelex document generation plugin. A template's field paths are checked against the inputs' concepts at load, and its templates and filename render strictly, so a missing value fails instead of printing as empty text.+- **Document engines are models**: the engine that prints a `PipeDocGen` step is a model of a new `doc_gen` family, which a step names with `model = "pipelex-pdf"` or leaves to the default for its format and source in the new deck file `5_doc_gen_deck.toml`, and whose worker a plugin registers with `add_inference_backend(family=InferenceFamily.DOC_GEN, …)`, declaring its model with `add_internal_model` and its deck defaults with `add_doc_gen_default`. A step whose engine is not in the deck, does not print its format from its source, or has no installed worker is refused when the method loads, so a runtime that disables the built-in `reportlab` plugin in `[runtime.plugins] disabled` refuses every `PipeDocGen` step. An engine receives a `RenderJob` of plain data, reads the files its document names through `RenderResources` under the run's read scope, and may check a step's template file at the dry run. `pipelex update` now also refreshes `backends/internal.toml`, backing up a locally edited copy first, so run it to get the deck file and the built-in `reportlab-pdf` engine.+- **The `markdown` template filter**: an HTML template turns Markdown held in a text field into HTML with `{{ order.notes | markdown }}`. Raw HTML in the source is shown as text, and only URLs with a scheme become links.+- **The `Markdown` native concept**: `Markdown` refines `Text` and holds a text written in Markdown, which a `PipeLLM` with `output = "Markdown"` writes as free text and a `Markdown` input takes as a string or as `{"concept": "Markdown", "content": "…"}`; Pipelex defines it ahead of the MTHDS standard. A `Markdown` value is accepted wherever a `Text` is, while a `Text`, or any concept refining `Text`, is refused where a `Markdown` is required. `pipelex run --save-main-stuff` saves it verbatim as `main_stuff.md`, an HTML template prints `{{ report }}` and `$report` as HTML with raw HTML escaped, and `Stuff.as_markdown` and `PipeOutput.main_stuff_as_markdown` read it from Python.++### Changed++- **`Markdown` is a reserved native concept code (Breaking)**: a bundle declaring its own concept named `Markdown` is refused with `native_concept_redeclaration`; rename it, or use the native.+- **A string under a concept refining `Text` keeps the concept's class**: a bare string or a `{"concept": …, "content": "…"}` envelope for such a concept now holds the concept's own content class where it held a plain `TextContent`, as the `YesNo`, `Date` and `Time` envelopes already did.+- **`markdown-it-py` replaces `markdown` in the dependencies**: the `markdown` package, which pipelex no longer imported, is dropped, and `markdown-it-py` with its `linkify` extra renders the Markdown of `PipeDocGen`, the `markdown` filter and the `Markdown` concept.+## [v0.70.0] - 2026-09-29### Highlightsdiff --git a/docs/building-methods/concepts/define_your_concepts.md b/docs/building-methods/concepts/define_your_concepts.mdindex ab9774ccb..816a016d6 100644--- a/docs/building-methods/concepts/define_your_concepts.md+++ b/docs/building-methods/concepts/define_your_concepts.md@@ -147,7 +147,7 @@ The refined concept inherits the structure of the base concept while adding sema## Native Concepts-Pipelex includes these built-in native concepts: `Text`, `Image`, `Document`, `TextAndImages`, `Number`, `YesNo`, `Date`, `Time`, `Page`, `Dynamic`, `JSON`, `Html`, `SearchResult`, `Composite`, and `Anything`.+Pipelex includes these built-in native concepts: `Text`, `Markdown`, `Image`, `Document`, `TextAndImages`, `Number`, `YesNo`, `Date`, `Time`, `Page`, `Dynamic`, `JSON`, `Html`, `SearchResult`, `Composite`, and `Anything`.These concepts are automatically available in all pipelines—no setup required. You can use them directly in your pipes or refine them to create more specific concepts.diff --git a/docs/building-methods/concepts/native-concepts.md b/docs/building-methods/concepts/native-concepts.mdindex 263d10ed1..de3e3df62 100644--- a/docs/building-methods/concepts/native-concepts.md+++ b/docs/building-methods/concepts/native-concepts.md@@ -26,6 +26,7 @@ Here are all the native concepts you can use out of the box:| Concept | Description | Content Class Name ||-----------------|-------------|---------------------|| `Text` | A text | `TextContent` |+| `Markdown` | A text written in Markdown, which refines `Text` | `MarkdownContent` || `Image` | An image | `ImageContent` || `Document` | A document (PDF, DOCX, PPTX, web page) | `DocumentContent` || `TextAndImages` | Text with its associated images | `TextAndImagesContent` |@@ -86,6 +87,51 @@ class TextContent(StuffContent):**Use for:** Plain text outputs, summaries, descriptions, etc.+### MarkdownContent++A text written in Markdown. `Markdown` refines `Text`, and its content class keeps the single field of `TextContent`, which holds the Markdown source:++```python+class MarkdownContent(TextContent):+ text: str+```++What sets it apart from `Text` is how it is shown: the HTML view, an HTML template and a PDF laid out without a template format a `Markdown` value and show a `Text` value as it is, so a stray `#` or `1.` in a plain text never turns into a heading or a list there. The terminal's pretty view is the exception: it renders both through Markdown.++As a `PipeLLM` output, `output = "Markdown"` gives a report whose headings, bold text, lists, tables and links are formatted downstream. The LLM writes free text, exactly as it does for `Text`:++```toml+[pipe.write_inspection_report]+type = "PipeLLM"+description = "Write site notes up as an inspection report"+inputs = { notes = "Text" }+output = "Markdown"+prompt = """+Write these site notes up as an inspection report in Markdown, with a heading for each area inspected.++@notes+"""+```++Read the report from a Python caller via `pipe_output.main_stuff_as_markdown.text`.++**Refinement.** A `Markdown` value is accepted wherever a `Text` is: a pipe whose input is `Text` takes a `Markdown` report, and its prompt receives the source. The reverse does not hold: a `Text` value, or a concept that refines `Text`, is not accepted where a `Markdown` is required, since nothing says it was written as Markdown. To name a specific kind of report, declare a concept with `refines = "Markdown"`.++**As an input**, `Markdown` takes a string, as `Text` does: `"report": "# Roof\n\n- two cracked tiles"`, or the envelope form `{"concept": "Markdown", "content": "# Roof"}`.++**How it renders:**++- In a prompt, and in the plain and Markdown views, it is its source as it is. `pipelex run --save-main-stuff` writes that source verbatim as `main_stuff.md`.+- Its HTML view converts the Markdown to HTML rather than escaping it, so `main_stuff.html` and the HTML tab of `main_stuff_viewer.html` show the formatted report. Raw HTML inside the source is shown as text rather than passed through, and only URLs with a scheme, such as `https://example.com`, become links, so a file name like `README.md` stays text.+- The pretty view in the terminal renders it as Markdown.+- Its JSON form is `{"text": "..."}`, the same as a `Text`.++**Inside an HTML template**, such as a `PipeCompose` template with `category = "html"`, `{{ report }}` prints a `Markdown` value as its converted HTML with no filter, while a `Text` value keeps being escaped. A `Markdown` field of a structure converts the same way, as in `{{ digest.summary }}`. `{{ report.text }}` is the Markdown source, escaped like any string. The `$report` sigil and `{{ report | format }}` print the converted HTML too, unless the template names another format, as in `{{ report | format("plain") }}`, which prints the Markdown source, escaped. The `@report` sigil wraps the source in tags for a prompt and is not meant for HTML templates.++**In a document**, a [`PipeDocGen`](../pipes/pipe-operators/PipeDocGen.md) step that prints a PDF without a template formats a `Markdown` value, with its headings, lists, tables, code and links, and prints a `Text` value as plain paragraphs.++**Use for:** Reports, summaries and write-ups an LLM writes to be read formatted, in an HTML page or a PDF.+### ImageContentRepresents an image with optional metadata:diff --git a/docs/building-methods/configure-ai-llm-to-optimize-methods.md b/docs/building-methods/configure-ai-llm-to-optimize-methods.mdindex 126fa938d..0c6428a71 100644--- a/docs/building-methods/configure-ai-llm-to-optimize-methods.md+++ b/docs/building-methods/configure-ai-llm-to-optimize-methods.md@@ -114,6 +114,7 @@ The Model Deck is your central configuration hub for all LLM-related settings. I- `2_img_gen_deck.toml`: Image generation configuration- `3_extract_deck.toml`: Document extraction configuration- `4_search_deck.toml`: Search configuration+- `5_doc_gen_deck.toml`: The document engines `PipeDocGen` prints with, by format and source- `x_custom_extract_deck.toml`: Custom extraction waterfalls/overrides (loaded last)- `x_custom_llm_deck.toml`: Custom LLM waterfalls/overrides (loaded last)@@ -133,6 +134,7 @@ The Model Deck is your central configuration hub for all LLM-related settings. I├── 2_img_gen_deck.toml # Image generation config├── 3_extract_deck.toml # Document extraction config├── 4_search_deck.toml # Search config+ ├── 5_doc_gen_deck.toml # Document engines, by format and source├── x_custom_extract_deck.toml # Custom extraction waterfalls/overrides└── x_custom_llm_deck.toml # Custom LLM waterfalls/overrides```diff --git a/docs/building-methods/pipes/pipe-operators/PipeCompose.md b/docs/building-methods/pipes/pipe-operators/PipeCompose.mdindex d443c0f45..a5ff723ce 100644--- a/docs/building-methods/pipes/pipe-operators/PipeCompose.md+++ b/docs/building-methods/pipes/pipe-operators/PipeCompose.md@@ -109,6 +109,8 @@ template = """"""```+**The `markdown` filter.** An HTML template can turn Markdown held in a plain text field into HTML with `{{ order.notes | markdown }}`. Raw HTML inside the Markdown is shown as text rather than passed through. A bare URL becomes a link only when it has a scheme, such as `https://example.com`, so a file name like `README.md` stays text; a link written in Markdown, such as `[the guide](docs/guide.md)`, is a link whatever its target. A [`Markdown`](../../concepts/native-concepts.md) input needs no filter: it renders as HTML by itself in an HTML template.+## Construct ModeConstruct mode builds structured objects by mapping fields from inputs. Use this when you need to assemble a complex output concept from multiple inputs without using an LLM.diff --git a/docs/building-methods/pipes/pipe-operators/PipeDocGen.md b/docs/building-methods/pipes/pipe-operators/PipeDocGen.mdnew file mode 100644index 000000000..ff90f4397--- /dev/null+++ b/docs/building-methods/pipes/pipe-operators/PipeDocGen.md@@ -0,0 +1,142 @@+---+description: "Generate a document file, such as a PDF, from a method's structured result with PipeDocGen. It calls no AI model: a document engine lays out or fills what its inputs already hold."+---++# PipeDocGen++The `PipeDocGen` operator turns a method's result into a document file: a PDF, and with the Pipelex document generation plugin an Excel workbook, a Word document or a PowerPoint deck. The file is stored like any other file a run produces, and the step outputs a `Document`.++`PipeDocGen` calls no AI model, unlike `PipeImgGen`, whose name it echoes. It lays out, or fills a template with, what its inputs already hold, so it costs nothing to run and produces the same file every time for the same inputs. The AI work, such as an LLM writing a report or extracting an invoice, happens in the steps before it. Its `model` is the **document engine** that prints the file, such as `reportlab-pdf` or, with the plugin, `pipelex-pdf`, chosen the way a step chooses any model.++## How it works++The step runs in two stages:++1. **Compose.** Pipelex lays the inputs out, or renders the step's HTML template against them, and renders the file's name. Every template here renders strictly: a field the inputs do not have fails the step instead of printing as empty text.+2. **Print.** The step's document engine turns the composition into the file's bytes, and Pipelex stores them. The output is a `Document` whose URL ends in the file's own name, such as `invoice-INV-2026-0142.pdf`.++With no template, the step lays its inputs out by itself, which is called the auto-layout:++- the scalar fields of a structure, such as a number, a date or a short text, make a grid of labels and values, labelled from the fields' titles;+- a list of structures the method declares, whose fields are all flat, makes a table, whose header row repeats on every page;+- a nested structure makes a section with a heading, and so does a nested object of a `JSON` input; a `Composite` lays out its components like a structure's fields;+- a `Text` input prints as paragraphs, and a [`Markdown`](../../concepts/native-concepts.md) input prints formatted, with its headings, emphasis, lists, tables, code blocks and links;+- an `Html` input prints as its text with the tags removed, since the auto-layout does not interpret HTML, a table row on one line with its cells separated by `|`;+- an `Image` prints as a picture, with its caption;+- a list of images, of `Markdown` texts or of other native values prints item by item, each as it would print alone.++A single input is the document itself: an invoice's fields fill the page. Several inputs each get a section, in the order the step declares them. Every page carries the document's title in a running header and "Page N of M" in its footer, on A4 portrait, in a font bundled with Pipelex so that a PDF looks the same on every machine.++## Document engines++A document engine is a model of the `doc_gen` family. A step names one with `model`; a step that names none prints on the model deck's default for its format and for whether it has a template:++| The step asks for | Default engine | Other engines that print it | Comes with |+| --- | --- | --- | --- |+| `pdf` without a template | `reportlab-pdf` | `pipelex-pdf` | Pipelex; `pipelex-pdf` with the plugin |+| `pdf` with an HTML template | `pipelex-pdf` | | The Pipelex document generation plugin |+| `xlsx`, with or without a template file | `pipelex-xlsx` | | The plugin |+| `docx`, with or without a template file | `pipelex-docx` | | The plugin |+| `pptx` with a template file | `pipelex-pptx` | | The plugin |++Open Pipelex prints a `pdf` without a template on its built-in `reportlab-pdf`, whose default is in `.pipelex/inference/deck/5_doc_gen_deck.toml`. The plugin declares its engines and their defaults itself when it loads, so installing it is all it takes. A step that names its engine prints the same file wherever it runs; a step that names none follows the deck of the install it runs on, and a default set in an `x_custom_*.toml` deck file overrides both the kit's and the plugin's.++A step whose engine is not installed is refused **when the method loads**, before a run spends anything on inference, with an error naming the engine and the plugin that provides it:++```+PipeDocGen 'render_invoice_xlsx' asks for an xlsx from the auto-layout of its inputs, and no document engine for it is installed in this runtime. It comes with the Pipelex document generation plugin, pipelex-doc-gen.+```++An install set up before `reportlab-pdf` existed declares neither the engine in its `internal.toml` nor its default in the model deck, so a `pdf` step that would print on `reportlab-pdf`, by naming it or by default, is refused with an error saying to run `pipelex update`, which refreshes those files from the kit. A step that prints on the plugin's `pipelex-pdf` is unaffected.++A step that names an engine which does not print its format from its source, such as `reportlab-pdf` with an HTML template, is refused at load as well.++## Configuration++```toml+[concept.InvoicePdf]+description = "The invoice laid out as a printable PDF"+refines = "Document"++[pipe.render_invoice_pdf]+type = "PipeDocGen"+description = "Lay out the invoice as a paginated PDF from its structure alone"+inputs = { invoice = "Invoice" }+output = "InvoicePdf"+format = "pdf"+filename = "invoice-{{ invoice.number }}"+```++To print the same invoice on the plugin's `pipelex-pdf`, the step names it: `model = "pipelex-pdf"`.++### MTHDS parameters++| Parameter | Type | Description | Required |+| --- | --- | --- | --- |+| `type` | string | `PipeDocGen` | Yes |+| `description` | string | What the document is | Yes |+| `inputs` | dictionary | The inputs the document shows. Every declared input is in the template's context, so several inputs combine into one document. | Yes |+| `output` | string | `Document`, or a concept that refines it. A single file: no multiplicity. | Yes |+| `format` | string | `pdf`, `xlsx`, `docx` or `pptx`. It sets the file's type and suffix. | Yes |+| `model` | string | The document engine that prints the file, a `doc_gen` model, alias or preset, such as `reportlab-pdf` or, with the plugin, `pipelex-pdf`. It defaults to the model deck's engine for the format and source. | No |+| `template` | string | An inline HTML and Jinja2 template, for `pdf` only. A `pdf` with a template is printed by the plugin. | No |+| `template_file` | string | A template file beside the bundle, relative to the bundle's file: `.html` for `pdf`, `.xlsx`, `.docx` or `.pptx`. It excludes `template`. `pptx` requires one. | No |+| `filename` | string | A Jinja expression over the inputs for the file's name, without its suffix, such as `invoice-{{ invoice.number }}`. It defaults to the pipe's code. | No |++### An LLM report as a PDF++The most common request needs no template: the LLM step declares its output as `Markdown`, and the `PipeDocGen` step formats it.++```toml+[pipe.write_report]+type = "PipeLLM"+description = "Write a structured report in Markdown"+inputs = { topic = "Text" }+output = "Markdown"+prompt = "Write a report on this topic, in Markdown: @topic"++[pipe.render_report_pdf]+type = "PipeDocGen"+description = "Print the Markdown report as a PDF, formatted"+inputs = { report = "Markdown" }+output = "Document"+format = "pdf"+filename = "report"+```++A `Text` input prints as plain paragraphs, so a report whose output is `Text` prints its `#` and `**` as they are. Declare it `Markdown` to have it formatted.++## Templates++### HTML templates++A `pdf` step's `template`, or its `.html` `template_file`, is rendered with exactly the stack of a `PipeCompose` HTML template: `$x` and `@x`, the `format` and `tag` filters, HTML autoescaping, and the [template sandbox](../../../under-the-hood/template-sandbox.md). A template that is only a fragment is wrapped in the plugin's print stylesheet (A4, running headers, page numbers); a whole HTML document prints as written, with any page setup its CSS gives. Three differences from `PipeCompose` matter:++- **A field a template names must exist.** Each path a template reads, such as `invoice.customer.name`, is checked against the inputs' concepts when the method loads, including a path through a loop variable: in `{% for item in invoice.line_items %}`, `item.amount` is checked against the line item's concept. What the load-time check cannot follow, such as a value set with `{% set %}`, fails when the template renders at the dry run, since a missing value is an error rather than empty text. An absent optional input can still be tested with `{% if notes %}` or `is defined`.+- **Pipelex's `format` filter shadows Jinja's own.** To format a number, write `{{ '%.2f' % invoice.total }}` rather than `{{ '%.2f' | format(invoice.total) }}`.+- **The `markdown` filter** turns Markdown held in a plain text field into HTML: `{{ invoice.notes | markdown }}`. A `Markdown` input needs no filter.++### Office template files++An `.xlsx`, `.docx` or `.pptx` template file is filled by its engine from the inputs as plain data, so it sees fields, not the sigils and filters of HTML templates. Its contract is documented with the plugin. When the plugin's engine checks templates, `pipelex validate` and the dry run compare the file with the inputs' concepts: a misspelled name is reported before any run.++A template file is found beside the bundle, so it works for a bundle loaded from a directory, a dependency package's included. A bundle loaded from a string, as the hosted API does, cannot name one: the step is refused at load. An inline `template` works on every load path.++## Running and iterating++`pipelex run` copies the file into the run's results folder, under its own name, and prints its path:++```bash+pipelex run bundle invoice/ --pipe render_invoice_pdf --inputs invoice/inputs.json+```++Running the `PipeDocGen` step alone on inputs saved from an earlier run is the loop for working on a document's layout: it spends nothing on inference. Write the inputs file in the shape `pipelex run` takes, one entry per input with its concept and content.++A dry run composes for real against mock inputs, which checks the templates and the file's name, prints nothing, and outputs a mock `Document`.++## Related documentation++- [PipeCompose](./PipeCompose.md), which renders a template into a `Text` or `Html` output+- [Native concepts](../../concepts/native-concepts.md), for `Document` and `Markdown`+- [`pipelex run`](../../../tools/cli/run.md)+- [Document Engine Plugins](../../../under-the-hood/document-engine-plugins.md), for how a runtime finds the engine that prints a stepdiff --git a/docs/building-methods/pipes/pipe-operators/index.md b/docs/building-methods/pipes/pipe-operators/index.mdindex c32e6a2ef..7a3be34d5 100644--- a/docs/building-methods/pipes/pipe-operators/index.md+++ b/docs/building-methods/pipes/pipe-operators/index.md@@ -22,6 +22,7 @@ Here are the primary pipe operators available in Pipelex:- [**`PipeSearch`**](./PipeSearch.md): Searches the web using a configurable search provider and returns structured results with an answer and source citations.- [**`PipeFunc`**](./PipeFunc.md): An escape hatch that allows you to execute any custom Python function, giving you maximum flexibility.- [**`PipeCompose`**](./PipeCompose.md): Composes outputs deterministically from working memory — renders Jinja2 templates for formatted reports or complex prompts, or constructs structured objects by mapping fields from inputs, without an LLM.+- [**`PipeDocGen`**](./PipeDocGen.md): Generates a document file from its inputs, calling no AI model: a PDF laid out from a structure or formatted from a Markdown report, and with the Pipelex document generation plugin a PDF from an HTML template, an Excel workbook, a Word document or a PowerPoint deck.## Overview@@ -34,6 +35,7 @@ Pipelex provides the following pipe operators:- `PipeImgGen`: For AI-powered image generation- `PipeSearch`: For web search with structured results- `PipeStructure`: For turning free-form text into structured data+- `PipeDocGen`: For generating a document file, such as a PDF, from structured data## PipeLLM@@ -93,6 +95,17 @@ Generates and manipulates images.- Batch processing- Parameter customization+## PipeDocGen++Generates a document file from its inputs, calling no AI model.++### Key Features++- A PDF laid out from the inputs' structure, with tables, sections and page numbers+- Markdown reports formatted, from a `Markdown` input+- A file name built from the inputs+- With the Pipelex document generation plugin: PDFs from HTML templates, Excel, Word and PowerPoint+## PipeSearchSearches the web and returns structured results with sources.diff --git a/docs/building-methods/pipes/pipe-output.md b/docs/building-methods/pipes/pipe-output.mdindex c19e4a219..1eb3b75d4 100644--- a/docs/building-methods/pipes/pipe-output.md+++ b/docs/building-methods/pipes/pipe-output.md@@ -164,6 +164,14 @@ verdict = pipe_output.main_stuff_as_yes_no.yes_no # a boolThis works only if the main_stuff is (or refines) a `YesNoContent` concept.+**`main_stuff_as_markdown`** - Returns `MarkdownContent` object:++```python+report = pipe_output.main_stuff_as_markdown.text # the Markdown source+```++This works only if the main_stuff is (or refines) a `MarkdownContent` concept.+**`main_stuff_as_html`** - Returns `HtmlContent` object:```pythondiff --git a/docs/configuration/config-technical/inference-backend-config.md b/docs/configuration/config-technical/inference-backend-config.mdindex ede243735..68f6f4eb2 100644--- a/docs/configuration/config-technical/inference-backend-config.md+++ b/docs/configuration/config-technical/inference-backend-config.md@@ -72,13 +72,14 @@ All inference backend configurations are stored in the `.pipelex/inference/` dir│ ├── vertexai.toml # Google Vertex AI models (LLMs)│ ├── fal.toml # FAL models (image generation)│ ├── linkup.toml # Linkup models (web search)- │ ├── internal.toml # Internal/local models (OCR)+ │ ├── internal.toml # Internal/local models (text extraction, the built-in document engine), managed by `pipelex update`│ └── ...└── deck/ # Model deck configurations├── 1_llm_deck.toml # LLM aliases & presets├── 2_img_gen_deck.toml # Image generation config├── 3_extract_deck.toml # Document extraction config├── 4_search_deck.toml # Web search config+ ├── 5_doc_gen_deck.toml # Document engines, by format and source├── x_custom_llm_deck.toml # Custom LLM waterfalls/overrides└── x_custom_extract_deck.toml # Custom extract waterfalls```@@ -86,7 +87,7 @@ All inference backend configurations are stored in the `.pipelex/inference/` dirDeck files are loaded in order by their numeric prefix (`1_`, `2_`, `3_`), with custom/override files (`x_` prefix) loaded last.!!! tip "Numbered files are pipelex-managed; overrides go in `x_custom_*.toml`"- The numbered deck files (`1_llm_deck.toml`...`4_search_deck.toml`) are refreshed by `pipelex update` when a new release ships an updated deck. Local edits to those files are preserved with a timestamped `.bak` backup but will not survive future updates.+ The numbered deck files (`1_llm_deck.toml`...`5_doc_gen_deck.toml`) are refreshed by `pipelex update` when a new release ships an updated deck. Local edits to those files are preserved with a timestamped `.bak` backup but will not survive future updates.To customize aliases, presets, or default choices without conflict, edit (or create) any file in this directory whose name starts with `x_custom_` — Pipelex never tracks or overwrites those. See [`pipelex update`](../../tools/cli/update.md) for the full workflow.@@ -473,6 +474,8 @@ Unlike other backends, internal backend models are **always available** regardleThis behavior is automatic and requires no additional configuration. To see which models are available from the internal backend, check `.pipelex/inference/backends/internal.toml`.+`internal.toml` is the one backend file Pipelex manages: it declares the software-only models open Pipelex ships, so `pipelex update` refreshes it from the kit, and an existing install receives the models a release adds, such as the built-in document engine `reportlab-pdf`. A locally edited copy is backed up to `<file>.bak.<UTC timestamp>` first, unless you pass `--no-backup`, and your edits will not survive future updates, so declare models of your own in a backend of your own. Every other backend file is yours and is never touched. A plugin that ships a software-only engine declares its model in the internal backend itself when it loads, rather than in this file.+## Model DeckThe Model Deck is the unified configuration hub for all AI model-related settings, including LLMs, OCR models, and image generation models.@@ -592,6 +595,20 @@ Search presets support the following options:- `include_images`: Whether to include images in search results- `include_inline_citations`: Whether to include inline citations in the answer+### Document Engines++The engines a `PipeDocGen` step prints with are models of the `doc_gen` family in the `internal` backend. `reportlab-pdf` is built into Pipelex, declared in `internal.toml`, and prints a `pdf` from the auto-layout of the step's inputs. `pipelex-pdf`, `pipelex-xlsx`, `pipelex-docx` and `pipelex-pptx` come with the Pipelex document generation plugin, which declares them when it loads, so no file of yours lists them. Each lists the sources it prints from as its `inputs` (`layout`, `html` or `template_file`) and its format as its `outputs`. `.pipelex/inference/deck/5_doc_gen_deck.toml` names the engine a step prints with when it names none, for the one format and source open Pipelex prints:++```toml+[doc_gen.choice_defaults]+"pdf.layout" = "@default-pdf"++[doc_gen.aliases]+default-pdf = "reportlab-pdf"+```++The plugin declares the defaults for a `pdf` from a template and for `xlsx`, `docx` and `pptx` itself, beneath the deck files. You can still set any default yourself, for any format and source, in an `x_custom_*.toml` deck file, which overrides both. A step names another engine with `model`, such as `model = "pipelex-pdf"` for a PDF without a template. See [PipeDocGen](../../building-methods/pipes/pipe-operators/PipeDocGen.md).+### Default ChoicesSet default models for different types of AI operations:@@ -736,7 +753,7 @@ pipelex migrate # ask, then rewrite in placeWhat it touches and what it leaves alone:- **Every `*.toml` directly in `inference/backends/`** — the files this page describes, in both the global `~/.pipelex/` and a project's `.pipelex/`.-- **Not** `inference/backends.toml`, which sits beside that directory rather than in it, and **not** the model deck under `inference/deck/`. The deck has its own `pipelex update`.+- **Not** `inference/backends.toml`, which sits beside that directory rather than in it, and **not** the model deck under `inference/deck/`. The deck has its own `pipelex update`, which also refreshes `internal.toml` from the kit.- **Not a key you added yourself.** The history only describes keys *we* removed or renamed. An unknown key of your own — a misspelled `maxx_tokens`, an extra header that is not header-shaped — is still an error, and it names the file, the key and what to do. That is deliberate: silently dropping a key you meant to set would change which model you get.- **Every file it rewrites is copied first**, beside itself, as `<file>.bak.<UTC timestamp>`. Running the command twice is the same as running it once — a file already up to date comes back byte for byte identical.diff --git a/docs/errors/authoring-and-language.md b/docs/errors/authoring-and-language.mdindex 434e7974b..eedfb7bdb 100644--- a/docs/errors/authoring-and-language.md+++ b/docs/errors/authoring-and-language.md@@ -73,6 +73,10 @@ own page. Classes are grouped by subsystem.- [`ConstructFieldBlueprintValueError`](construct-field-blueprint-value-error.md) — Construct field blueprint value- [`PipeComposeError`](pipe-compose-error.md) — Pipe compose- [`PipeComposeFactoryError`](pipe-compose-factory-error.md) — Pipe compose factory+- [`PipeDocGenFactoryError`](pipe-doc-gen-factory-error.md) — Pipe doc gen factory+- [`PipeDocGenRunError`](pipe-doc-gen-run-error.md) — Pipe doc gen run+- [`PipeDocGenTemplateCheckError`](pipe-doc-gen-template-check-error.md) — Pipe doc gen template check+- [`PipeDocGenUndefinedValueError`](pipe-doc-gen-undefined-value-error.md) — Pipe doc gen undefined value- [`PipeExtractFactoryError`](pipe-extract-factory-error.md) — Pipe extract factory- [`PipeFuncExecutionError`](pipe-func-execution-error.md) — Pipe func execution- [`PipeFuncTransportError`](pipe-func-transport-error.md) — Pipe func transportdiff --git a/docs/errors/doc-gen-engine-missing-error.md b/docs/errors/doc-gen-engine-missing-error.mdnew file mode 100644index 000000000..040d52ae9--- /dev/null+++ b/docs/errors/doc-gen-engine-missing-error.md@@ -0,0 +1,21 @@+---+title: "Doc gen engine missing"+description: "Reference for the `DocGenEngineMissingError` Pipelex error class."+---++<!-- pipelex:generated -->++# Doc gen engine missing++The document engine a `PipeDocGen` step prints with is not available in this runtime.++| Field | Value |+|---|---|+| `error_type` | `DocGenEngineMissingError` |+| `title` | Doc gen engine missing |+| `type_uri` | `https://docs.pipelex.com/latest/errors/doc-gen-engine-missing-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.cogt.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/doc-gen-handle-not-found-error.md b/docs/errors/doc-gen-handle-not-found-error.mdnew file mode 100644index 000000000..4751acdc8--- /dev/null+++ b/docs/errors/doc-gen-handle-not-found-error.md@@ -0,0 +1,19 @@+---+title: "Doc gen handle not found"+description: "Reference for the `DocGenHandleNotFoundError` Pipelex error class."+---++<!-- pipelex:generated -->++# Doc gen handle not found++| Field | Value |+|---|---|+| `error_type` | `DocGenHandleNotFoundError` |+| `title` | Doc gen handle not found |+| `type_uri` | `https://docs.pipelex.com/latest/errors/doc-gen-handle-not-found-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`CogtError`](cogt-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/doc-gen-model-capability-error.md b/docs/errors/doc-gen-model-capability-error.mdnew file mode 100644index 000000000..c6a49b91b--- /dev/null+++ b/docs/errors/doc-gen-model-capability-error.md@@ -0,0 +1,21 @@+---+title: "Doc gen model capability"+description: "Reference for the `DocGenModelCapabilityError` Pipelex error class."+---++<!-- pipelex:generated -->++# Doc gen model capability++The document engine a `PipeDocGen` step names does not print the step's format from its source.++| Field | Value |+|---|---|+| `error_type` | `DocGenModelCapabilityError` |+| `title` | Doc gen model capability |+| `type_uri` | `https://docs.pipelex.com/latest/errors/doc-gen-model-capability-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.cogt.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/doc-gen-render-error.md b/docs/errors/doc-gen-render-error.mdnew file mode 100644index 000000000..066b68cc7--- /dev/null+++ b/docs/errors/doc-gen-render-error.md@@ -0,0 +1,21 @@+---+title: "Doc gen render"+description: "Reference for the `DocGenRenderError` Pipelex error class."+---++<!-- pipelex:generated -->++# Doc gen render++A document engine could not print a render job.++| Field | Value |+|---|---|+| `error_type` | `DocGenRenderError` |+| `title` | Doc gen render |+| `type_uri` | `https://docs.pipelex.com/latest/errors/doc-gen-render-error/` |+| `error_domain` | `runtime` |+| Defined in | `pipelex.cogt.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/duplicate-doc-gen-default-error.md b/docs/errors/duplicate-doc-gen-default-error.mdnew file mode 100644index 000000000..9cc661476--- /dev/null+++ b/docs/errors/duplicate-doc-gen-default-error.md@@ -0,0 +1,21 @@+---+title: "Duplicate doc gen default"+description: "Reference for the `DuplicateDocGenDefaultError` Pipelex error class."+---++<!-- pipelex:generated -->++# Duplicate doc gen default++Two plugins declared the model deck's default document engine for the same format and source.++| Field | Value |+|---|---|+| `error_type` | `DuplicateDocGenDefaultError` |+| `title` | Duplicate doc gen default |+| `type_uri` | `https://docs.pipelex.com/latest/errors/duplicate-doc-gen-default-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.plugins.exceptions` |+| Parent class | [`PluginError`](plugin-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/duplicate-internal-model-error.md b/docs/errors/duplicate-internal-model-error.mdnew file mode 100644index 000000000..e9213b2e3--- /dev/null+++ b/docs/errors/duplicate-internal-model-error.md@@ -0,0 +1,21 @@+---+title: "Duplicate internal model"+description: "Reference for the `DuplicateInternalModelError` Pipelex error class."+---++<!-- pipelex:generated -->++# Duplicate internal model++Two plugins declared a model of the same name in the internal backend.++| Field | Value |+|---|---|+| `error_type` | `DuplicateInternalModelError` |+| `title` | Duplicate internal model |+| `type_uri` | `https://docs.pipelex.com/latest/errors/duplicate-internal-model-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.plugins.exceptions` |+| Parent class | [`PluginError`](plugin-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/inference-and-providers.md b/docs/errors/inference-and-providers.mdindex d6b72aaba..b65b2ea9a 100644--- a/docs/errors/inference-and-providers.md+++ b/docs/errors/inference-and-providers.md@@ -14,6 +14,10 @@ own page. Classes are grouped by subsystem.- [`CogtError`](cogt-error.md) — AI inference failed- [`CostRegistryError`](cost-registry-error.md) — Cost registry+- [`DocGenEngineMissingError`](doc-gen-engine-missing-error.md) — Doc gen engine missing+- [`DocGenHandleNotFoundError`](doc-gen-handle-not-found-error.md) — Doc gen handle not found+- [`DocGenModelCapabilityError`](doc-gen-model-capability-error.md) — Doc gen model capability+- [`DocGenRenderError`](doc-gen-render-error.md) — Doc gen render- [`DryRunMockBuildError`](dry-run-mock-build-error.md) — Dry run mock build- [`DryRunObjectFidelityError`](dry-run-object-fidelity-error.md) — Dry run object fidelity- [`ExtractCapabilityError`](extract-capability-error.md) — Extract capability@@ -59,6 +63,7 @@ own page. Classes are grouped by subsystem.- [`ModelWaterfallError`](model-waterfall-error.md) — Model waterfall- [`NeitherUrlNorDataError`](neither-url-nor-data-error.md) — Neither url nor data- [`OutputStructureSchemaError`](output-structure-schema-error.md) — Output structure schema+- [`PluginModelDeclarationError`](plugin-model-declaration-error.md) — Plugin model declaration- [`PromptDocumentFactoryError`](prompt-document-factory-error.md) — Prompt document factory- [`PromptImageFactoryError`](prompt-image-factory-error.md) — Prompt image factory- [`PromptImageFormatError`](prompt-image-format-error.md) — Prompt image format@@ -118,8 +123,10 @@ own page. Classes are grouped by subsystem.- [`BrokenPluginError`](broken-plugin-error.md) — Broken plugin- [`CoreUnconditionalPluginDisabledError`](core-unconditional-plugin-disabled-error.md) — Core unconditional plugin disabled- [`DuplicateBundleValidatorError`](duplicate-bundle-validator-error.md) — Duplicate bundle validator+- [`DuplicateDocGenDefaultError`](duplicate-doc-gen-default-error.md) — Duplicate doc gen default- [`DuplicateHttpErrorMapperError`](duplicate-http-error-mapper-error.md) — Duplicate http error mapper- [`DuplicateInferenceBackendError`](duplicate-inference-backend-error.md) — Duplicate inference backend+- [`DuplicateInternalModelError`](duplicate-internal-model-error.md) — Duplicate internal model- [`DuplicateLogSinkError`](duplicate-log-sink-error.md) — Duplicate log sink- [`DuplicateModelListerError`](duplicate-model-lister-error.md) — Duplicate model lister- [`DuplicateOrchestratorError`](duplicate-orchestrator-error.md) — Duplicate orchestratordiff --git a/docs/errors/pipe-doc-gen-factory-error.md b/docs/errors/pipe-doc-gen-factory-error.mdnew file mode 100644index 000000000..58bb5c506--- /dev/null+++ b/docs/errors/pipe-doc-gen-factory-error.md@@ -0,0 +1,21 @@+---+title: "Pipe doc gen factory"+description: "Reference for the `PipeDocGenFactoryError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe doc gen factory++Raised when a ``PipeDocGen`` cannot be built from its blueprint.++| Field | Value |+|---|---|+| `error_type` | `PipeDocGenFactoryError` |+| `title` | Pipe doc gen factory |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-doc-gen-factory-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.pipe_operators.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-doc-gen-run-error.md b/docs/errors/pipe-doc-gen-run-error.mdnew file mode 100644index 000000000..bc2d3db63--- /dev/null+++ b/docs/errors/pipe-doc-gen-run-error.md@@ -0,0 +1,21 @@+---+title: "Pipe doc gen run"+description: "Reference for the `PipeDocGenRunError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe doc gen run++Raised when a ``PipeDocGen`` step fails while it composes its document: a template or the filename does not render.++| Field | Value |+|---|---|+| `error_type` | `PipeDocGenRunError` |+| `title` | Pipe doc gen run |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-doc-gen-run-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.pipe_operators.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-doc-gen-template-check-error.md b/docs/errors/pipe-doc-gen-template-check-error.mdnew file mode 100644index 000000000..e77ee658c--- /dev/null+++ b/docs/errors/pipe-doc-gen-template-check-error.md@@ -0,0 +1,21 @@+---+title: "Pipe doc gen template check"+description: "Reference for the `PipeDocGenTemplateCheckError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe doc gen template check++Raised by a ``PipeDocGen`` dry run when the engine's template checker reports errors in the step's template file.++| Field | Value |+|---|---|+| `error_type` | `PipeDocGenTemplateCheckError` |+| `title` | Pipe doc gen template check |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-doc-gen-template-check-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.pipe_operators.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-doc-gen-undefined-value-error.md b/docs/errors/pipe-doc-gen-undefined-value-error.mdnew file mode 100644index 000000000..cc9b1a80f--- /dev/null+++ b/docs/errors/pipe-doc-gen-undefined-value-error.md@@ -0,0 +1,21 @@+---+title: "Pipe doc gen undefined value"+description: "Reference for the `PipeDocGenUndefinedValueError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe doc gen undefined value++Raised when a ``PipeDocGen`` template or filename reads a value its inputs do not have.++| Field | Value |+|---|---|+| `error_type` | `PipeDocGenUndefinedValueError` |+| `title` | Pipe doc gen undefined value |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-doc-gen-undefined-value-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.pipe_operators.doc_gen.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/plugin-model-declaration-error.md b/docs/errors/plugin-model-declaration-error.mdnew file mode 100644index 000000000..3cc84ad89--- /dev/null+++ b/docs/errors/plugin-model-declaration-error.md@@ -0,0 +1,21 @@+---+title: "Plugin model declaration"+description: "Reference for the `PluginModelDeclarationError` Pipelex error class."+---++<!-- pipelex:generated -->++# Plugin model declaration++A model a plugin declares, or a model deck default it sets, cannot be merged into this installation's inference configuration.++| Field | Value |+|---|---|+| `error_type` | `PluginModelDeclarationError` |+| `title` | Plugin model declaration |+| `type_uri` | `https://docs.pipelex.com/latest/errors/plugin-model-declaration-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`CogtError`](cogt-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/features/cli.md b/docs/features/cli.mdindex 5539d27bc..9b6840137 100644--- a/docs/features/cli.md+++ b/docs/features/cli.md@@ -17,7 +17,7 @@ The `pipelex` CLI is the primary tool for working with Pipelex methods. It cover|---------|-------------|| **`pipelex login`** | Authenticate with Pipelex Gateway via the browser and save your API key || **`pipelex init`** | Initialize configuration, backends, credentials, routing, and telemetry |-| **`pipelex update`** | Refresh the model deck to match the installed pipelex version |+| **`pipelex update`** | Refresh the model deck and `backends/internal.toml` to match the installed pipelex version || **`pipelex doctor`** | Check configuration health and suggest fixes || **`pipelex build`** | AI-powered method generation from natural language requirements || **`pipelex validate`** | Check pipeline syntax, structure, and run dry-run validation |diff --git a/docs/features/concepts.md b/docs/features/concepts.mdindex 986e941b9..1b3370a86 100644--- a/docs/features/concepts.md+++ b/docs/features/concepts.md@@ -17,6 +17,7 @@ Pre-built universal concepts available in every Pipelex project:- **Dynamic** — Dynamic typing, resolved at runtime- **Text** — Plain text content+- **Markdown** — A text written in Markdown, which refines Text and is formatted when shown- **Image** — Image data (base64, URL, or file path)- **Document** — A document container (PDF, image, or other file)- **Html** — HTML contentdiff --git a/docs/get-started/configure-ai-providers.md b/docs/get-started/configure-ai-providers.mdindex cb3e99c3f..24701f260 100644--- a/docs/get-started/configure-ai-providers.md+++ b/docs/get-started/configure-ai-providers.md@@ -158,6 +158,7 @@ By default, this creates the global `~/.pipelex/` directory with:│ ├── 2_img_gen_deck.toml # Image generation config│ ├── 3_extract_deck.toml # Document extraction config│ ├── 4_search_deck.toml # Search config+ │ ├── 5_doc_gen_deck.toml # Document engines, by format and source│ ├── x_custom_llm_deck.toml # Custom LLM configurations│ └── x_custom_extract_deck.toml # Custom extract configurations└── routing_profiles.toml # Model routing configurationdiff --git a/docs/tools/cli/index.md b/docs/tools/cli/index.mdindex f4087abe9..992b501b9 100644--- a/docs/tools/cli/index.md+++ b/docs/tools/cli/index.md@@ -24,7 +24,7 @@ The Pipelex CLI is organized into several command groups:| Command | Description ||---------|-------------|| [**init**](init.md) | Initialize Pipelex configuration |-| [**update**](update.md) | Refresh the model deck to match the installed pipelex version |+| [**update**](update.md) | Refresh the model deck and `backends/internal.toml` to match the installed pipelex version || [**migrate**](migrate.md) | Bring your configuration files up to the schema the installed version expects || [**validate**](validate.md) | Validate configuration and pipelines || [**fix**](fix.md) | Apply deterministic safe fixes to a bundle and re-validate |diff --git a/docs/tools/cli/init.md b/docs/tools/cli/init.mdindex e1a1aeafa..02a0b902f 100644--- a/docs/tools/cli/init.md+++ b/docs/tools/cli/init.md@@ -62,8 +62,8 @@ This command creates or resets a Pipelex config directory with:- **inference/** - AI backend and routing configuration- `backends.toml` - Backend provider settings- `routing_profiles.toml` - Model routing rules- - `backends/` - Individual backend configuration files- - `deck/` - AI model aliases and presets+ - `backends/` - Individual backend configuration files, of which [`pipelex update`](update.md) keeps `internal.toml` current+ - `deck/` - AI model aliases and presets, which [`pipelex update`](update.md) keeps current- **telemetry.toml** - Telemetry and observability settings- **.gitignore** - Keeps Pipelex's own transient copies out of your `git status` — the timestamped `.bak` files [`pipelex migrate`](migrate.md) leaves beside each file it rewrites. Commit it so your teammates get the same. It is written only when the directory has no `.gitignore`; one already there is never modified.diff --git a/docs/tools/cli/run.md b/docs/tools/cli/run.mdindex 2aa89d1c6..fc7acee6a 100644--- a/docs/tools/cli/run.md+++ b/docs/tools/cli/run.md@@ -214,7 +214,7 @@ The output JSON contains the complete working memory after pipeline execution, i### The results directory-Each run that saves anything gets its own numbered directory under `--output-dir` (`results/<pipe_code>_output_01/`, then `_02`, and so on). With `--save-working-memory` it holds `working_memory.json`, with `--save-main-stuff` the `main_stuff.*` renders, and with `--graph` the graph outputs that `graphs_inclusion` enables: `graphspec.json`, the Mermaid code and viewer, and the ReactFlow viewer. In `working_memory.json` each stuff names its concept by ref — `{"stuff_code": …, "stuff_name": …, "concept": "<domain>.<Code>", "content": …}` — and carries no concept definition; see [Working Memory](../../building-methods/pipes/working-memory.md#in-memory-and-on-the-wire).+Each run that saves anything gets its own numbered directory under `--output-dir` (`results/<pipe_code>_output_01/`, then `_02`, and so on). With `--save-working-memory` it holds `working_memory.json`, with `--save-main-stuff` the `main_stuff.*` renders, and with `--graph` the graph outputs that `graphs_inclusion` enables: `graphspec.json`, the Mermaid code and viewer, and the ReactFlow viewer. When the main output is a Document or an Image, `--save-main-stuff` also copies the file itself into the directory under its own name, so a PDF that a [`PipeDocGen`](../../building-methods/pipes/pipe-operators/PipeDocGen.md) step produced lands as `invoice-INV-2026-0142.pdf` and opens with a double-click, and the run prints its path. It never replaces a file the run writes in the directory: a file whose name is taken, such as a document passed through from an earlier run as `main_stuff.json`, is saved as `main_stuff-1.json`. A dry run produces no file, so it copies nothing. In `working_memory.json` each stuff names its concept by ref — `{"stuff_code": …, "stuff_name": …, "concept": "<domain>.<Code>", "content": …}` — and carries no concept definition; see [Working Memory](../../building-methods/pipes/working-memory.md#in-memory-and-on-the-wire).Beside `graphspec.json`, and gated by the same `graphspec_json` flag, the run writes the three files that describe the data the graph carries:diff --git a/docs/tools/cli/update.md b/docs/tools/cli/update.mdindex 9f528be4a..c8b9ef59d 100644--- a/docs/tools/cli/update.md+++ b/docs/tools/cli/update.md@@ -1,17 +1,17 @@----description: "Refresh the installed Pipelex model deck — keep `1_llm_deck.toml` and friends in sync with the kit shipped by your installed pipelex version, while preserving your `x_custom_*.toml` overrides."+description: "Refresh the kit-managed files of a Pipelex install — keep the model deck (`1_llm_deck.toml` and friends) and `backends/internal.toml` in sync with the kit shipped by your installed pipelex version, while preserving your `x_custom_*.toml` overrides and your own backend files."---# Update Command-Refresh the installed model deck so it matches the kit shipped with your current `pipelex` version.+Refresh the files Pipelex manages in your install, the model deck and `backends/internal.toml`, so they match the kit shipped with your current `pipelex` version.```bashpipelex update [OPTIONS]pipelex update --local [OPTIONS]```-When a new `pipelex` release ships changes to the model deck — new models, new aliases, retired entries — your previously installed deck under `~/.pipelex/inference/deck/` will fall behind. `pipelex update` brings it up to date without nuking your overrides.+When a new `pipelex` release ships changes to the model deck — new models, new aliases, retired entries — your previously installed deck under `~/.pipelex/inference/deck/` will fall behind. The same goes for `~/.pipelex/inference/backends/internal.toml`, which declares the software-only models open Pipelex ships, such as the built-in document engine `reportlab-pdf`: an install made before a release added a model does not declare it. `pipelex update` brings both up to date without nuking your overrides. `pipelex init` is no substitute: it is a full reset that rewrites every configuration file the kit ships, the backend files and the kit's `x_custom_*.toml` deck files included, so this command is how an install catches up without losing its settings.## When to run it@@ -27,11 +27,18 @@ Run `pipelex update` to clear the warning. To silence it permanently for a sessiexport PIPELEX_NO_DECK_NOTICE=1```+A `PipeDocGen` step that prints on `reportlab-pdf` in an install whose `internal.toml` predates it is refused when its method loads, with an error that says to run `pipelex update`.+## What it does-Pipelex tracks the deck install state in a small JSON manifest at `~/.pipelex/inference/deck/.kit_manifest.json` (or `.pipelex/inference/deck/.kit_manifest.json` for project-local installs). The manifest stores the kit version that produced the install and a SHA-256 of each managed deck file at install time.+Pipelex manages two areas of an install, each tracked by its own small JSON manifest, `.kit_manifest.json`, in its own directory:++- the model deck, `~/.pipelex/inference/deck/` (or `.pipelex/inference/deck/` for a project-local install);+- the backends directory, `~/.pipelex/inference/backends/` (or `.pipelex/inference/backends/`), where only `internal.toml` is managed.++Each area is found the way the runtime finds it: a project's `.pipelex/` directory wins when it has one, and the global `~/.pipelex/` one is used otherwise, so the command refreshes the files your runs actually read. A manifest stores the kit version that produced the install and a SHA-256 of each managed file at install time.-On `update`, Pipelex compares three states — what the kit ships, what is on disk, what the manifest recorded — and produces a per-file plan:+On `update`, Pipelex compares three states for each area — what the kit ships, what is on disk, what the manifest recorded — and produces a per-file plan:| Status | Meaning | Action ||---|---|---|@@ -41,14 +48,15 @@ On `update`, Pipelex compares three states — what the kit ships, what is on di| `locally modified` | Installed file was edited after install — kit version differs | back up + overwrite || `removed upstream` | Kit no longer ships this file | back up + remove |-Locally-modified files are preserved as `<file>.bak.<UTC-timestamp>` before the kit version is written in their place. Removed files get the same backup treatment before deletion. After the run, Pipelex writes a fresh manifest stamped with the new kit version.+Locally-modified files are preserved as `<file>.bak.<UTC-timestamp>` before the kit version is written in their place, unless you pass `--no-backup`. Removed files get the same backup treatment before deletion. After the run, Pipelex writes a fresh manifest, stamped with the new kit version, for each area it updated.-## Managed files vs. user overrides+## Managed files vs. user files-Pipelex draws a sharp line between deck files it owns and deck files you own.+Pipelex draws a sharp line between the files it owns and the files you own.-- **Managed (pipelex-owned)** — `1_llm_deck.toml`, `2_img_gen_deck.toml`, `3_extract_deck.toml`, `4_search_deck.toml`. These are refreshed by `pipelex update`. Local edits are preserved with a `.bak.<timestamp>` backup but will not survive future updates.-- **User overrides** — any file in the deck directory whose name starts with `x_custom_` (e.g. `x_custom_llm_deck.toml`, `x_custom_extract_deck.toml`). Pipelex never tracks, hashes, copies, or removes these. Add new ones whenever you need to override aliases, presets, or default choices for a backend.+- **Managed (pipelex-owned)** — the numbered deck files, `1_llm_deck.toml`, `2_img_gen_deck.toml`, `3_extract_deck.toml`, `4_search_deck.toml` and `5_doc_gen_deck.toml`, and `backends/internal.toml`. These are refreshed by `pipelex update`. Local edits are preserved with a `.bak.<timestamp>` backup but will not survive future updates.+- **User overrides** — any file in the deck directory whose name starts with `x_custom_` (e.g. `x_custom_llm_deck.toml`, `x_custom_extract_deck.toml`). `pipelex update` never tracks, hashes, copies, or removes these, while `pipelex init`, a full reset, rewrites the two the kit ships. Add new ones whenever you need to override aliases, presets, or default choices for a backend.+- **Every other backend file** — `openai.toml`, `anthropic.toml`, a backend of your own, and the rest of `backends/`. `pipelex update` never touches them; `pipelex init`, a full reset, rewrites the ones the kit ships, such as `openai.toml`, and leaves a backend of your own in place. Declare models of your own in a backend of your own rather than in `internal.toml`.The deck loader merges all `*.toml` files under the deck directory in alphabetical order via deep-merge, so your `x_custom_*.toml` always wins over the numbered defaults.@@ -56,10 +64,10 @@ The deck loader merges all `*.toml` files under the deck directory in alphabetic| Flag | Description ||---|---|-| `--local` / `-l` | Update the project-local `.pipelex/` deck instead of the global `~/.pipelex/` |+| `--local` / `-l` | Update the project-local `.pipelex/` files instead of the ones the layered resolution picks || `--yes` / `-y` | Apply updates non-interactively (skip the confirmation prompt) || `--dry-run` | Show the plan without modifying any file |-| `--no-backup` | Do not create `.bak` files for locally-modified deck files |+| `--no-backup` | Do not create `.bak` files for locally-modified managed files |## Examples@@ -67,13 +75,13 @@ The deck loader merges all `*.toml` files under the deck directory in alphabetic# Preview what would changepipelex update --dry-run-# Interactive update of the global deck+# Interactive update of the global deck and internal.tomlpipelex update# Non-interactive update — useful for CI or scriptspipelex update --yes-# Project-local deck only+# Project-local files onlypipelex update --local --yes# Skip the .bak backups (advanced — your edits will be lost permanently)@@ -82,18 +90,18 @@ pipelex update --yes --no-backup## Migration: existing installs without a manifest-If your deck was installed before the `update` command existed, there will be no `.kit_manifest.json` next to your deck files. The first `pipelex update` run reports each managed file as `locally modified` (no provenance proof) and asks before overwriting. Two ways to migrate cleanly:+If your deck was installed before the `update` command existed, there will be no `.kit_manifest.json` next to your deck files, and an install made before `pipelex update` managed `internal.toml` has none in its backends directory. The first `pipelex update` run reports each managed file that differs from the kit as `locally modified` (no provenance proof) and asks before overwriting. Two ways to migrate cleanly:-1. **Trust the kit** — run `pipelex update --yes` to take the upstream version of every managed file. Anything you want to keep can be moved into an `x_custom_*.toml` afterwards using the `.bak.<timestamp>` files as a reference.-2. **Move overrides first** — copy any custom aliases, presets, or defaults from your numbered files into a new `x_custom_<area>_deck.toml`, then run `pipelex update --yes`. Your customizations will live in the file Pipelex never touches.+1. **Trust the kit** — run `pipelex update --yes` to take the upstream version of every managed file. Anything you want to keep can be moved into an `x_custom_*.toml`, or for a model into a backend of your own, afterwards using the `.bak.<timestamp>` files as a reference.+2. **Move overrides first** — copy any custom aliases, presets, or defaults from your numbered files into a new `x_custom_<area>_deck.toml`, and any model you added to `internal.toml` into a backend of your own, then run `pipelex update --yes`. Your customizations will live in files Pipelex never touches.After either path, subsequent updates land cleanly without prompting on the unchanged files.## Diagnostics-`pipelex doctor` includes a **Model Deck** section that reports the same per-file status as `update`. Run `pipelex doctor --fix` to be offered an interactive `pipelex update` as part of the standard fix flow.+`pipelex doctor` includes a **Model Deck** section that reports the same per-file status as `update`, for the deck and for `backends/internal.toml`. Run `pipelex doctor --fix` to be offered an interactive `pipelex update` as part of the standard fix flow.## Related-- [Init Commands](init.md) — first-time setup that materializes the deck and its manifest-- [LLM Providers & Models Configuration](../../configuration/config-technical/inference-backend-config.md) — what the deck files actually configure+- [Init Commands](init.md) — first-time setup that materializes the deck, the backend files and their manifests+- [LLM Providers & Models Configuration](../../configuration/config-technical/inference-backend-config.md) — what the deck files and backend files actually configurediff --git a/docs/under-the-hood/document-engine-plugins.md b/docs/under-the-hood/document-engine-plugins.mdnew file mode 100644index 000000000..bcc578dd2--- /dev/null+++ b/docs/under-the-hood/document-engine-plugins.md@@ -0,0 +1,152 @@+---+title: "Document Engine Plugins"+description: "How a PipeDocGen step finds the engine that prints it, a model of the doc_gen family, how a plugin declares its engines and their defaults, the render job an engine receives, the template checker it may offer, and how a runtime refuses at load what it cannot print."+---++# Document Engine Plugins++A [`PipeDocGen`](../building-methods/pipes/pipe-operators/PipeDocGen.md) step turns its inputs into a document file in two stages. The **compose stage** is Pipelex's own and is pure: it lays the inputs out, or renders the step's HTML template, or gathers the inputs as plain data for a template file, and it renders the file's name. The **print stage** hands the result to a **document engine**, which returns the file's bytes, and Pipelex stores them.++A document engine is a **model** of the `doc_gen` family, chosen the way a step chooses any model: by the step's `model`, or by the model deck's default. Its worker is contributed by a plugin, through the same `add_inference_backend` seam the inference backends use. Open Pipelex ships one engine, `reportlab-pdf`, which prints a `pdf` from the auto-layout of a step's inputs, and declares it in the kit's `internal.toml`. The Pipelex document generation plugin, `pipelex-doc-gen`, ships the engines for a `pdf` from an HTML template and for the office formats from outside this repository, and declares their models and their deck defaults itself when it loads.++---++## Sources, and the engines that print them++A step asks for a **format** (`pdf`, `xlsx`, `docx`, `pptx`), and its template decides the **source** the compose stage hands an engine:++| The step has | Source | What the engine receives |+| --- | --- | --- |+| No template (a `pdf`, `xlsx` or `docx`; a `pptx` needs a template file) | `layout` | The layout tree of the inputs |+| An HTML `template` or `.html` `template_file` (a `pdf` only) | `html` | The template rendered against the inputs |+| An office `template_file` | `template_file` | The file's bytes and the inputs as plain data |++Each engine is a model of the `internal` backend, declared with the sources it prints from as its `inputs` and its format as its `outputs`. The built-in engine is declared in the kit's `internal.toml`, which `pipelex update` refreshes, so an existing install receives it:++```toml+[reportlab-pdf]+model_type = "doc_gen"+sdk = "reportlab"+model_id = "print-pdf"+inputs = ["layout"]+outputs = ["pdf"]+costs = {}+```++| Engine | Prints | From | Declared by |+| --- | --- | --- | --- |+| `reportlab-pdf` | `pdf` | `layout` | Pipelex, in the kit's `internal.toml` |+| `pipelex-pdf` | `pdf` | `html`, `layout` | `pipelex-doc-gen`, when it loads |+| `pipelex-xlsx` | `xlsx` | `layout`, `template_file` | `pipelex-doc-gen`, when it loads |+| `pipelex-docx` | `docx` | `layout`, `template_file` | `pipelex-doc-gen`, when it loads |+| `pipelex-pptx` | `pptx` | `template_file` | `pipelex-doc-gen`, when it loads |++The model deck names the default engine for each format and source, keyed `<format>.<source>`, and a key naming a format and source no step can ask for fails when the deck loads. The kit's `5_doc_gen_deck.toml` sets the one default open Pipelex prints:++```toml+[doc_gen.choice_defaults]+"pdf.layout" = "@default-pdf"++[doc_gen.aliases]+default-pdf = "reportlab-pdf"+```++The plugin declares the defaults for the formats it prints, beneath the deck files: a deck file that sets the same format and source overrides it, and a user sets any default of their own in an `x_custom_*.toml` deck file, which overrides both. A step that names no engine prints on the deck's default, so it follows the install's deck, as an LLM step does. A step that names one, such as `model = "pipelex-pdf"` on a `pdf` without a template, prints on that engine wherever it runs.++---++## Registering an engine++A plugin registers a worker factory for its engine's sdk in the `doc_gen` family, declares the engine's model in the `internal` backend with `add_internal_model`, and declares the deck default for each format and source it prints with `add_doc_gen_default`:++```python+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.inference_backend_registry import InferenceFamily+from pipelex.plugins.registrar import PluginRegistrar+++class MyDocGenPlugin:+ name = "my_doc_gen"+ targets_api = PLUGIN_API_VERSION++ def register(self, registrar: PluginRegistrar) -> None:+ registrar.add_inference_backend(family=InferenceFamily.DOC_GEN, sdk="openpyxl", make_worker=make_xlsx_worker)+ registrar.add_internal_model(+ name="pipelex-xlsx",+ spec={+ "model_type": "doc_gen",+ "sdk": "openpyxl",+ "model_id": "write-xlsx",+ "inputs": ["layout", "template_file"],+ "outputs": ["xlsx"],+ "costs": {},+ },+ )+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT, model="pipelex-xlsx")+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.TEMPLATE_FILE, model="pipelex-xlsx")+```++The model's `spec` is exactly the table a backend file would hold for it (`model_type`, `sdk`, `model_id`, `inputs`, `outputs`, `costs`, and any other model-spec field), and it is complete on its own: the `[defaults]` table of `internal.toml` is not applied to it. A default names a model, directly or through any reference the deck resolves.++`register` stores these declarations and validates nothing. The model manager merges them at boot, before it builds the deck, and refuses what it cannot merge with `PluginModelDeclarationError`, naming the plugin: a model whose name the install's `internal.toml` already declares (the message names the file too), a table that is not a valid model spec, or a default for a format and source no step composes. Model names are not global across backends, so a name another backend declares is left to the routing profile, as it is for a model a file declares. When the install disables the `internal` backend, or declares none, the plugin's models are not merged, as a disabled backend's own models are not loaded, and its defaults are left out with them, so a plugin never makes a boot fail that would succeed without it. A second plugin declaring the same model fails at registration with `DuplicateInternalModelError`, and the same format and source with `DuplicateDocGenDefaultError`, each naming both plugins. `pipelex plugins list` shows each declaration on the plugin's row.++A document engine is a kernel-layer capability, and so are its model and its defaults, so an out-of-tree engine publishes under the `pipelex.plugins.kernel` entry-point group (see [Inference Backend Plugins](inference-backend-plugins.md#shipping-it-as-an-out-of-tree-plugin) for the groups). `make_worker` is called for each print, with the model's spec, and imports the engine's library inside it, so `register` stays import-light and a process that never prints never imports the library. An engine loads its library, fonts and styles once per process, in the module that holds its worker, rather than per worker. A second registration of the same sdk fails loud at boot with `DuplicateInferenceBackendError`, naming both plugins.++---++## Refused at load, not at run++When a method loads, each `PipeDocGen` step resolves its engine: the model it names, or the deck's default for its format and source. The step is refused right there, before a run spends anything on inference, when:++- the deck names no engine for its format and source, and it names none (`DocGenEngineMissingError`, which names `pipelex-doc-gen` for every format and source but a `pdf` from the auto-layout, and says to run `pipelex update` for that one);+- the engine is `reportlab-pdf`, named by the step or through a default, and the install's `internal.toml` predates it (`DocGenEngineMissingError`, which says to run `pipelex update`);+- the engine it names is one of the plugin's, `pipelex-pdf`, `pipelex-xlsx`, `pipelex-docx` or `pipelex-pptx`, and the plugin is not installed (`DocGenEngineMissingError`, naming `pipelex-doc-gen`);+- the engine it names is any other model the deck does not define (an unknown model, located on the step's `model` field);+- the engine does not print its format from its source, by the `inputs` and `outputs` it declares (`DocGenModelCapabilityError`);+- no installed plugin registers the engine's sdk (`DocGenEngineMissingError`, naming the engine and `pipelex-doc-gen`, or saying the built-in engine is disabled).++The other model families find a missing worker only when they build it, at run time; a document engine is checked when the method loads as well. A runtime therefore needs no rule of its own about what it prints: a host that must not print documents disables the built-in plugin with `runtime.plugins.disabled = ["reportlab"]` and installs no other engine, and every `PipeDocGen` step is refused at load.++---++## The contract: a render job in, bytes out++An engine's worker subclasses `DocGenWorkerAbstract` from `pipelex.cogt.doc_gen.doc_gen_worker_abstract`:++```python+def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument: ...+```++- **`RenderJob`** is plain data, which Pipelex builds in the print stage from the step's composition and hands to the worker in the same process. It has a JSON round trip for an engine that prints in another process, in which the template's bytes are URL-safe base64 and a date or a time is its ISO text. It holds the format and source, the file's name with its suffix, the document's title, and exactly the payload its source names, `layout` (a `LayoutDocument` from `layout_tree.py`), `html`, or `template` with `data`, the inputs as plain data by input name. It carries no Pipelex object, so an ordinary Pipelex release does not break an engine; a change to it, or to the worker, is a change of the plugin contract.+- **`RenderResources.load(uri=…, position=…)`** is how an engine reads a file its document names, such as an image. It resolves `pipelex-storage://` keys through the run's storage provider, decodes `data:` URLs, fetches `https://` through the SSRF guard, and refuses what the run's read scope does not allow, a local path included, with `UriReadRefusedError`. An engine reads nothing any other way.+- **`RenderedDocument`** holds the bytes. Their MIME type and suffix are the format's.++`render` is synchronous and runs on a thread of a print pool of its own, never on the event loop's default executor, which the engine's reads need, while `RenderResources.load` hands each read back to the event loop the print started from. An engine keeps no state from one print into the next. It signals a document it cannot print by raising `DocGenRenderError`, which passes through; anything else it raises, beyond a Pipelex error, is reported as a `DocGenRenderError` naming the engine and the file.++---++## The template checker++An office template names the fields it is filled with in its own way, which Pipelex cannot read: an Excel workbook through its defined names and Tables, a Word document through its tags. So an engine that fills a template file overrides its worker's `check_template` (the request and findings are in `pipelex.cogt.doc_gen.template_check`):++```python+def check_template(self, *, request: TemplateCheckRequest) -> list[TemplateFinding]: ...+```++The request carries the template file's bytes and name, the shape of each input as an `InputShape` tree (which fields exist, which are lists, and what their items hold), and the dry run's mock inputs as plain data, so the checker can also fill the template in memory. The dry run, and so `pipelex validate`, calls it for every step with an office template file: a finding of severity `error` fails the step with `PipeDocGenTemplateCheckError`, listing the findings, and a `warning` is logged. The default finds nothing. An HTML template, inline or in a `.html` template file, never reaches an engine's checker: Pipelex checks it itself, against the inputs' concepts, when the method loads. The built-in engine takes no template.++---++## Where the print stage runs++The print stage is a content-generation leaf, like image generation or extraction: `ContentGenerator.make_rendered_document` hands a `RenderDocumentAssignment`, which carries the composition and the resolved engine, to `render_document_and_store`. The leaf authorizes the images the document names against the run's read scope, takes the dry-run branch, builds the engine's worker, prints and stores the file, and returns only the `DocumentContent` pointing at it, so the file's bytes never cross a workflow boundary. The dry run composes for real, which checks the templates against mock inputs, runs the template checker, and prints and stores nothing. See [Content Generation Across Boundaries](distributed-content-generation.md) for the leaves.++---++## Related++- [PipeDocGen](../building-methods/pipes/pipe-operators/PipeDocGen.md) — the operator, its formats, its templates and its `model`+- [Inference Backend Plugins](inference-backend-plugins.md) — plugin discovery, the entry-point groups and the `runtime.plugins.disabled` denylist+- [Storage Provider Plugins](storage-provider-plugins.md) — where the printed file is stored+- [Template Sandbox](template-sandbox.md) — what a composed template may readdiff --git a/docs/under-the-hood/inference-backend-plugins.md b/docs/under-the-hood/inference-backend-plugins.mdindex 79fbd4853..54ef49c0d 100644--- a/docs/under-the-hood/inference-backend-plugins.md+++ b/docs/under-the-hood/inference-backend-plugins.md@@ -29,7 +29,7 @@ run time (e.g. LLMWorkerFactory.make_llm_worker)└─ worker = make_worker(inference_model=…, backend=…, sdk_clients=…, reporting_delegate=…)```-The worker factories (`LLMWorkerFactory`, `ImgGenWorkerFactory`, `ExtractWorkerFactory`, `SearchWorkerFactory`) hold **no** `match` over SDK strings. They build a `ModelHandle`, resolve the `InferenceBackend` config, look up the backend's `make_worker` by `(family, sdk)`, and call it. A lookup miss raises a friendly `InferenceBackendNotFoundError` ("… Is its plugin installed and enabled?").+The worker factories (`LLMWorkerFactory`, `ImgGenWorkerFactory`, `ExtractWorkerFactory`, `SearchWorkerFactory`, `DocGenWorkerFactory`) hold **no** `match` over SDK strings. They build a `ModelHandle`, resolve the `InferenceBackend` config, look up the backend's `make_worker` by `(family, sdk)`, and call it. A lookup miss raises a friendly `InferenceBackendNotFoundError` ("… Is its plugin installed and enabled?").---@@ -57,7 +57,7 @@ A backend plugin's `register` calls one menu method per `(family, sdk)` it serve```pythonregistrar.add_inference_backend(- family=InferenceFamily.LLM, # LLM | IMG_GEN | EXTRACT | SEARCH+ family=InferenceFamily.LLM, # LLM | IMG_GEN | EXTRACT | SEARCH | DOC_GENsdk="acme", # the model's `sdk` stringmake_worker=_make_acme_worker, # a MakeWorkerFn (a plain callable))@@ -65,7 +65,9 @@ registrar.add_inference_backend(A registry key is `(family, sdk)`. The same `sdk` string may appear in two families (e.g. `google` serves both `LLM` and `IMG_GEN`); they are distinct keys. A duplicate `(family, sdk)` fails loud with `DuplicateInferenceBackendError` naming **both** contributing plugins.-One plugin may register across several families from a single `register` — the built-in `gateway` plugin serves all four, `mistral` serves `LLM` + `EXTRACT`, `linkup` serves `EXTRACT` + `SEARCH`. This is the cross-family-vendor coordination point: one plugin, many backends.+One plugin may register across several families from a single `register` — the built-in `gateway` plugin serves the four inference families, `mistral` serves `LLM` + `EXTRACT`, `linkup` serves `EXTRACT` + `SEARCH`. This is the cross-family-vendor coordination point: one plugin, many backends. `DOC_GEN` is the family of the document engines a `PipeDocGen` step prints with, local libraries rather than inference, which the built-in `reportlab` plugin and the Pipelex document generation plugin serve (see [Document Engine Plugins](document-engine-plugins.md)).++A plugin whose backend runs without an external service, such as a document engine, may also declare the models it serves rather than leave them to a backend file. `add_internal_model(name=…, spec=…)` declares a model in the `internal` backend, the spec being the table a backend file would hold for it, and `add_doc_gen_default(doc_gen_format=…, source=…, model=…)` declares the model deck's default engine for a document format and source. Both are stored at registration and merged by the model manager at boot, and [Document Engine Plugins](document-engine-plugins.md#registering-an-engine) gives their rules.---diff --git a/docs/under-the-hood/input-form-descriptor.md b/docs/under-the-hood/input-form-descriptor.mdindex 4a79b1e60..acacad60d 100644--- a/docs/under-the-hood/input-form-descriptor.md+++ b/docs/under-the-hood/input-form-descriptor.md@@ -34,11 +34,11 @@ Kinds are decided by chain membership and declared types — never by sniffing aThe crate the deriver reads is the current library's accumulated one, which holds the validated bundle and every `library_dirs` bundle loaded beside it — so a concept from a library dir, or a local concept refining one, follows the same rules as a local concept. A concept absent from that crate altogether is `unknown`.-The table and rules above ARE the no-hint kind assignment — stated rules, not heuristics: with no applicable intent hint, a node's kind is exactly what they produce. An applicable authored `intent` (spec: MTHDS `intent-hints.md`) *feeds* that assignment, never competes with it: on a **text-valued** node — one whose site is a `text` field, a `native.Text`-chained or description-only concept, judged per item on plural sites — an effective `intent = "prose"` yields `kind: "prose"` and `intent = "label"` yields `kind: "text"`; an absent, unknown, or inapplicable intent leaves the no-hint kind untouched. On a **number-valued** node, `rating` and `quantity` never change `kind` (both are `number`; the union has no finer kind) — they ride the `hints` slot for the renderer to honor. A time-formatted `text` node is *not* a text-valued site, so no intent word applies to it (and `native.Html`, being an `object` node, never reaches the text-valued judgment at all).+The table and rules above ARE the no-hint kind assignment — stated rules, not heuristics: with no applicable intent hint, a node's kind is exactly what they produce. An applicable authored `intent` (spec: MTHDS `intent-hints.md`) *feeds* that assignment, never competes with it: on a **text-valued** node — one whose site is a `text` field, a concept chained to `native.Text` or `native.Markdown`, or a description-only concept, judged per item on plural sites — an effective `intent = "prose"` yields `kind: "prose"` and `intent = "label"` yields `kind: "text"`; an absent, unknown, or inapplicable intent leaves the no-hint kind untouched. On a **number-valued** node, `rating` and `quantity` never change `kind` (both are `number`; the union has no finer kind) — they ride the `hints` slot for the renderer to honor. A time-formatted `text` node is *not* a text-valued site, so no intent word applies to it (and `native.Html`, being an `object` node, never reaches the text-valued judgment at all).| Native concept | Kind ||---|---|-| `Text` | `prose` |+| `Text`, `Markdown` | `prose` || `Number` | `number` with `integer: false` || `YesNo` | `boolean` || `Time` | `text` with `format: "time"` |diff --git a/docs/under-the-hood/writing-an-inference-plugin.md b/docs/under-the-hood/writing-an-inference-plugin.mdindex aba5fa3a5..50628bf9e 100644--- a/docs/under-the-hood/writing-an-inference-plugin.md+++ b/docs/under-the-hood/writing-an-inference-plugin.md@@ -124,6 +124,8 @@ A plugin is any object with a `name`, a `targets_api` and a `register` method. ``add_model_lister` is optional. It registers the callable behind `pipelex show models`, which Pipelex awaits when you list the models of a backend whose models use the `hello` token.+The registrar offers two more menu methods, which this plugin does not need: `add_internal_model` declares a model in the `internal` backend, and `add_doc_gen_default` declares the model deck's default document engine for a format and source. A document engine uses them to ship its models with its code rather than in a backend file, as [Document Engine Plugins](document-engine-plugins.md#registering-an-engine) describes. The hello model stays in a backend file of its own, which Step 5 writes.+## Step 3: Write the worker`hello_llm_worker.py` holds the model itself:diff --git a/mkdocs.yml b/mkdocs.ymlindex 071d473f7..d610575c6 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -278,6 +278,7 @@ plugins:- building-methods/pipes/pipe-operators/PipeImgGen.md: "PipeImgGen"- building-methods/pipes/pipe-operators/PipeSearch.md: "PipeSearch"- building-methods/pipes/pipe-operators/PipeCompose.md: "PipeCompose"+ - building-methods/pipes/pipe-operators/PipeDocGen.md: "PipeDocGen"- building-methods/pipes/pipe-operators/PipeFunc.md: "PipeFunc"- building-methods/pipes/pipe-controllers/index.md: "Pipe Controllers Overview"- building-methods/pipes/pipe-controllers/PipeSequence.md: "PipeSequence"@@ -354,6 +355,7 @@ plugins:- under-the-hood/storage-provider-plugins.md: "Storage Provider Plugins"- under-the-hood/secrets-provider-plugins.md: "Secrets Provider Plugins"- under-the-hood/log-sink-plugins.md: "Log Sink Plugins"+ - under-the-hood/document-engine-plugins.md: "Document Engine Plugins"- under-the-hood/error-model.md: "Error Model"- under-the-hood/runtime-bridge-and-transport.md: "Runtime Bridge & Transport"- under-the-hood/distributed-content-generation.md: "Content Generation Across Boundaries"@@ -493,6 +495,7 @@ nav:- PipeImgGen: building-methods/pipes/pipe-operators/PipeImgGen.md- PipeSearch: building-methods/pipes/pipe-operators/PipeSearch.md- PipeCompose: building-methods/pipes/pipe-operators/PipeCompose.md+ - PipeDocGen: building-methods/pipes/pipe-operators/PipeDocGen.md- PipeFunc: building-methods/pipes/pipe-operators/PipeFunc.md- Pipe Controllers:- Overview: building-methods/pipes/pipe-controllers/index.md@@ -591,6 +594,7 @@ nav:- Storage Provider Plugins: under-the-hood/storage-provider-plugins.md- Secrets Provider Plugins: under-the-hood/secrets-provider-plugins.md- Log Sink Plugins: under-the-hood/log-sink-plugins.md+ - Document Engine Plugins: under-the-hood/document-engine-plugins.md- Error Model: under-the-hood/error-model.md- Runtime Bridge & Transport: under-the-hood/runtime-bridge-and-transport.md- Content Generation Across Boundaries: under-the-hood/distributed-content-generation.mddiff --git a/pipelex/builder/concept/concept_spec.py b/pipelex/builder/concept/concept_spec.pyindex 8fc372cfc..c51b7b925 100644--- a/pipelex/builder/concept/concept_spec.py+++ b/pipelex/builder/concept/concept_spec.py@@ -263,10 +263,10 @@ class ConceptSpec(StructuredContent):default=None,description=("If applicable: the native concept this concept extends "- "(Text, Html, Image, Document, Number, Page, TextAndImages, JSON, Anything, Dynamic) "+ "(Text, Markdown, Html, Image, Document, Number, Page, TextAndImages, JSON, Anything, Dynamic) ""in PascalCase format. Cannot be used together with 'structure'."),- examples=["Text", "Html", "Image", "Document", "Number", "Page", "TextAndImages", "JSON"],+ examples=["Text", "Markdown", "Html", "Image", "Document", "Number", "Page", "TextAndImages", "JSON"],)@field_validator("concept_code", mode="before")diff --git a/pipelex/builder/operations/pipe_ops.py b/pipelex/builder/operations/pipe_ops.pyindex d324bd573..a0d9307e6 100644--- a/pipelex/builder/operations/pipe_ops.py+++ b/pipelex/builder/operations/pipe_ops.py@@ -16,6 +16,7 @@ from pipelex.builder.pipe.exceptions import PipeSpecErrorfrom pipelex.builder.pipe.pipe_batch_spec import PipeBatchSpecfrom pipelex.builder.pipe.pipe_compose_spec import PipeComposeSpecfrom pipelex.builder.pipe.pipe_condition_spec import PipeConditionSpec+from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec@@ -247,6 +248,17 @@ def add_type_specific_fields(*, pipe_spec: PipeSpec, pipe_table: Table) -> None:if pipe_spec.model:pipe_table.add("model", pipe_spec.model)+ elif isinstance(pipe_spec, PipeDocGenSpec):+ pipe_table.add("format", str(pipe_spec.format))+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ if pipe_spec.template is not None:+ pipe_table.add("template", pipe_spec.template)+ if pipe_spec.template_file is not None:+ pipe_table.add("template_file", pipe_spec.template_file)+ if pipe_spec.filename is not None:+ pipe_table.add("filename", pipe_spec.filename)+elif isinstance(pipe_spec, PipeComposeSpec):if pipe_spec.construct_spec is not None:# Construct mode: serialize the construct block as a nested TOML tablediff --git a/pipelex/builder/pipe/pipe_doc_gen_spec.py b/pipelex/builder/pipe/pipe_doc_gen_spec.pynew file mode 100644index 000000000..d598ea3e4--- /dev/null+++ b/pipelex/builder/pipe/pipe_doc_gen_spec.py@@ -0,0 +1,95 @@+from typing import TYPE_CHECKING, Literal++from pydantic import Field, field_validator+from typing_extensions import override++from pipelex.builder.pipe.pipe_spec import PipeSpec+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat+from pipelex.pipe_operators.doc_gen.pipe_doc_gen_blueprint import PipeDocGenBlueprint+from pipelex.tools.misc.pretty import require_rich_for_rendering++if TYPE_CHECKING:+ from pipelex.tools.misc.pretty import PrettyPrintable+++class PipeDocGenSpec(PipeSpec):+ """Spec for the PipeDocGen operator, which generates a document file from its inputs and calls no AI model.++ With no template, it lays the inputs out by itself: a `pdf` step without a template is what open Pipelex+ prints, on its `reportlab-pdf` engine, and a `Markdown` input comes out formatted. A `pdf` with an HTML+ template, and the `xlsx`, `docx` and `pptx` formats, need the Pipelex document generation plugin, and a+ runtime without it refuses them when the method loads. `model` names the engine; without it, the model+ deck's default for the format prints the file.++ Validation Rules:+ - the output must be a single Document, or a concept refining Document.+ - `template` (inline HTML and Jinja2) is for `pdf` only; `template_file` names a file beside the bundle+ whose suffix matches the format (`.html`, `.xlsx`, `.docx`, `.pptx`); never both.+ - `pptx` needs a `template_file`.+ """++ type: Literal["PipeDocGen"] = "PipeDocGen"+ pipe_category: Literal["PipeOperator"] = "PipeOperator"+ format: DocGenFormat = Field(strict=False, description="The file format to generate: pdf, xlsx, docx or pptx.")+ model: str | None = Field(+ default=None,+ description=(+ "The document engine that prints the file, a doc_gen model such as 'reportlab-pdf' or, "+ "with the document generation plugin, 'pipelex-pdf'. Omit it for the model deck's default for the format."+ ),+ )+ template: str | None = Field(+ default=None,+ description="An inline HTML and Jinja2 template, for pdf only. Omit it for the auto-layout of the inputs.",+ )+ template_file: str | None = Field(+ default=None,+ description="A template file beside the bundle, relative to its file: .html for pdf, .xlsx, .docx or .pptx.",+ )+ filename: str | None = Field(+ default=None,+ description="A Jinja expression over the inputs for the file's name, without its suffix, such as 'invoice-{{ invoice.number }}'.",+ )++ @field_validator("model", mode="before")+ @classmethod+ def reject_empty_model(cls, value: str | None) -> str | None:+ if isinstance(value, str) and not value.strip():+ msg = "Model cannot be an empty string; omit the field to use the default engine"+ raise ValueError(msg)+ return value++ @override+ def rendered_pretty(self, *, title: str | None = None, depth: int = 0) -> "PrettyPrintable":+ require_rich_for_rendering()+ from rich.console import Group+ from rich.markup import escape+ from rich.text import Text++ base_group = super().rendered_pretty(title=title, depth=depth)++ doc_gen_group = Group()+ doc_gen_group.renderables.append(base_group)++ doc_gen_group.renderables.append(Text()) # Blank line+ doc_gen_group.renderables.append(Text.from_markup(f"Format: [bold yellow]{escape(self.format)}[/bold yellow]"))+ doc_gen_group.renderables.append(Text.from_markup(f"Model: [bold yellow]{escape(self.model or '(default)')}[/bold yellow]"))+ if self.template_file is not None:+ doc_gen_group.renderables.append(Text.from_markup(f"Template file: [bold]{escape(self.template_file)}[/bold]"))++ return doc_gen_group++ @override+ def to_blueprint(self) -> PipeDocGenBlueprint:+ base_blueprint = super().to_blueprint()++ return PipeDocGenBlueprint(+ description=base_blueprint.description,+ inputs=base_blueprint.inputs_concept_specs,+ output=base_blueprint.output,+ format=self.format,+ model=self.model,+ template=self.template,+ template_file=self.template_file,+ filename=self.filename,+ )diff --git a/pipelex/builder/pipe/pipe_spec_map.py b/pipelex/builder/pipe/pipe_spec_map.pyindex 3406e7863..a48807cd9 100644--- a/pipelex/builder/pipe/pipe_spec_map.py+++ b/pipelex/builder/pipe/pipe_spec_map.py@@ -1,6 +1,7 @@from pipelex.builder.pipe.pipe_batch_spec import PipeBatchSpecfrom pipelex.builder.pipe.pipe_compose_spec import PipeComposeSpecfrom pipelex.builder.pipe.pipe_condition_spec import PipeConditionSpec+from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec@@ -23,6 +24,7 @@ pipe_type_to_spec_class: dict[str, type[PipeSpec]] = {"PipeExtract": PipeExtractSpec,"PipeSearch": PipeSearchSpec,"PipeStructure": PipeStructureSpec,+ "PipeDocGen": PipeDocGenSpec,"PipeBatch": PipeBatchSpec,"PipeCondition": PipeConditionSpec,"PipeParallel": PipeParallelSpec,diff --git a/pipelex/builder/pipe/pipe_spec_union.py b/pipelex/builder/pipe/pipe_spec_union.pyindex 87cae5151..c8164f6d5 100644--- a/pipelex/builder/pipe/pipe_spec_union.py+++ b/pipelex/builder/pipe/pipe_spec_union.py@@ -5,6 +5,7 @@ from pydantic import Fieldfrom pipelex.builder.pipe.pipe_batch_spec import PipeBatchSpecfrom pipelex.builder.pipe.pipe_compose_spec import PipeComposeSpecfrom pipelex.builder.pipe.pipe_condition_spec import PipeConditionSpec+from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec@@ -23,6 +24,7 @@ PipeSpecUnion = Annotated[| PipeExtractSpec| PipeSearchSpec| PipeStructureSpec+ | PipeDocGenSpec| PipeBatchSpec| PipeConditionSpec| PipeParallelSpecdiff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 899a40b24..22b4a3719 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -210,21 +210,21 @@ def doctor_command(doctor_cmd(fix=fix)[email protected](name="update", help="Update the model deck to match the installed pipelex version")[email protected](name="update", help="Update the model deck and backends/internal.toml to match the installed pipelex version")def update_command(local: Annotated[bool,typer.Option("--local","-l",- help="Force the project-local .pipelex/ deck. Default targets the resolved deck dir (project if .pipelex/ exists, else global)",+ help="Force the project-local .pipelex/ files. Default targets the resolved dirs (project if .pipelex/ has them, else global)",),] = False,yes: Annotated[bool, typer.Option("--yes", "-y", help="Apply updates without the interactive confirmation prompt")] = False,dry_run: Annotated[bool, typer.Option("--dry-run", help="Show the planned actions without modifying any file")] = False,- no_backup: Annotated[bool, typer.Option("--no-backup", help="Skip .bak files when overwriting locally-modified deck files")] = False,+ no_backup: Annotated[bool, typer.Option("--no-backup", help="Skip .bak files when overwriting locally-modified managed files")] = False,) -> None:- """Refresh the installed deck to match the kit shipped with the running pipelex version."""+ """Refresh the installed deck and backends/internal.toml to match the kit shipped with the running pipelex version."""update_cmd(local=local, yes=yes, dry_run=dry_run, no_backup=no_backup)diff --git a/pipelex/cli/agent_cli/commands/init_cmd.py b/pipelex/cli/agent_cli/commands/init_cmd.pyindex 2ac42caed..f10c15f5f 100644--- a/pipelex/cli/agent_cli/commands/init_cmd.py+++ b/pipelex/cli/agent_cli/commands/init_cmd.py@@ -20,7 +20,7 @@ from pipelex.cli.commands.init.config_files import init_configfrom pipelex.cli.commands.init.ui.backends_ui import get_backend_options_from_tomlfrom pipelex.cogt.model_backends.backend import MANAGED_GATEWAY_BACKEND_NAMES, PipelexBackendfrom pipelex.cogt.model_routing.routing_profile import PipelexRoutingProfile-from pipelex.cogt.models.deck_manifest import compute_kit_manifest, write_manifest+from pipelex.cogt.models.deck_manifest import stamp_kit_manifestsfrom pipelex.kit.paths import get_kit_configs_dirfrom pipelex.system.configuration.config_loader import config_managerfrom pipelex.system.pipelex_service.pipelex_service_agreement import (@@ -142,7 +142,7 @@ def _copy_inference_templates(target_dir: Path) -> None:if deck_file.suffix == ".toml":shutil.copy2(deck_file, target_deck_dir / deck_file.name)- write_manifest(compute_kit_manifest(), deck_dir=target_deck_dir)+ stamp_kit_manifests(inference_dir=target_inference_dir)# Copy routing_profiles.tomltemplate_routing_path = template_inference_dir / "routing_profiles.toml"diff --git a/pipelex/cli/agent_cli/commands/pipe_cmd.py b/pipelex/cli/agent_cli/commands/pipe_cmd.pyindex e5b313aad..9bf0480fc 100644--- a/pipelex/cli/agent_cli/commands/pipe_cmd.py+++ b/pipelex/cli/agent_cli/commands/pipe_cmd.py@@ -17,6 +17,7 @@ from pipelex.builder.operations.pipe_ops import parse_pipe_specfrom pipelex.builder.pipe.pipe_batch_spec import PipeBatchSpecfrom pipelex.builder.pipe.pipe_compose_spec import PipeComposeSpecfrom pipelex.builder.pipe.pipe_condition_spec import PipeConditionSpec+from pipelex.builder.pipe.pipe_doc_gen_spec import PipeDocGenSpecfrom pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpecfrom pipelex.builder.pipe.pipe_func_spec import PipeFuncSpecfrom pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec@@ -94,6 +95,17 @@ def _add_type_specific_fields(*, pipe_spec: PipeSpec, pipe_table: tomlkit.TOMLDoif pipe_spec.model:pipe_table.add("model", pipe_spec.model)+ elif isinstance(pipe_spec, PipeDocGenSpec):+ pipe_table.add("format", str(pipe_spec.format))+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ if pipe_spec.template is not None:+ pipe_table.add("template", format_toml_string(pipe_spec.template))+ if pipe_spec.template_file is not None:+ pipe_table.add("template_file", pipe_spec.template_file)+ if pipe_spec.filename is not None:+ pipe_table.add("filename", pipe_spec.filename)+elif isinstance(pipe_spec, PipeComposeSpec):if pipe_spec.construct_spec is not None:# Construct mode: serialize the construct block as a nested TOML tablediff --git a/pipelex/cli/commands/doctor_cmd.py b/pipelex/cli/commands/doctor_cmd.pyindex 26db32ff2..930e8fc7b 100644--- a/pipelex/cli/commands/doctor_cmd.py+++ b/pipelex/cli/commands/doctor_cmd.py@@ -32,13 +32,21 @@ from pipelex.cogt.exceptions import (InferenceBackendLibraryValidationError,ModelDeckNotFoundError,ModelDeckValidationError,+ PluginModelDeclarationError,RoutingProfileDisabledBackendError,RoutingProfileLibraryError,RoutingProfileLibraryNotFoundError,)from pipelex.cogt.model_backends.backend_credentials import BackendCredentialsErrorMsgFactory, BackendCredentialsReportfrom pipelex.cogt.model_backends.backend_library import InferenceBackendLibrary-from pipelex.cogt.models.deck_manifest import DeckFileStatus, DeckSyncReport, compute_deck_sync_report, status_rich_label+from pipelex.cogt.models.deck_manifest import (+ MANAGED_BACKEND_FILENAME,+ DeckFileStatus,+ DeckSyncReport,+ KitManagedArea,+ compute_sync_report,+ status_rich_label,+)from pipelex.cogt.models.model_manager import ModelManagerfrom pipelex.config import get_configfrom pipelex.core.validation import MIGRATE_COMMAND, raise_config_setup_error, report_validation_error@@ -80,6 +88,7 @@ if TYPE_CHECKING:from collections.abc import Mappingfrom pipelex.cogt.model_backends.gateway_config import GatewayConfig+ from pipelex.plugins.registrar import PluginRegistrarfrom pipelex.system.pipelex_service.types import RemoteConfigSource@@ -676,6 +685,9 @@ def display_health_report(deck_healthy: bool,deck_message: str,deck_report: DeckSyncReport,+ internal_backend_healthy: bool,+ internal_backend_message: str,+ internal_backend_report: DeckSyncReport,config_location: ConfigLocationInfo,fix_mode: bool = False,models_skipped: bool = False,@@ -699,6 +711,9 @@ def display_health_report(deck_healthy: Whether the model deck is in sync with the kit shipped by this pipelex versiondeck_message: Summary message about deck sync statusdeck_report: Per-file sync status report (used to render the per-file detail when not healthy)+ internal_backend_healthy: Whether backends/internal.toml is in sync with the kit, like the deck+ internal_backend_message: Summary message about its sync status+ internal_backend_report: Its sync status report, rendered like the deck'sconfig_location: Resolved configuration location informationfix_mode: Whether we're in interactive fix mode (--fix flag)models_skipped: True when check_models was bypassed because config is broken — render@@ -718,6 +733,7 @@ def display_health_report(and backends_healthyand models_healthyand deck_healthy+ and internal_backend_healthyand log_sink_healthyand plugins_healthy)@@ -851,18 +867,10 @@ def display_health_report(console.print(f" [dim]{escape(error_lines[0][:100])}[/dim]")console.print()- # Model Deck section+ # Model Deck section: the files `pipelex update` manages, the deck and backends/internal.tomlconsole.print("[bold]Model Deck[/bold]")- if deck_healthy:- console.print(f" [green]✓[/green] {escape(deck_message)}")- else:- console.print(f" [yellow]⚠[/yellow] {escape(deck_message)}")- # Per-file detail when there are pending actions- actionable_statuses = {name: status for name, status in deck_report.files.items() if status != DeckFileStatus.UP_TO_DATE}- if actionable_statuses:- for filename in sorted(actionable_statuses):- status = actionable_statuses[filename]- console.print(f" [dim]{escape(filename)}[/dim] — {status_rich_label(status)}")+ _print_kit_sync_row(healthy=deck_healthy, message=deck_message, report=deck_report)+ _print_kit_sync_row(healthy=internal_backend_healthy, message=internal_backend_message, report=internal_backend_report)console.print()# Recommended actions@@ -895,7 +903,7 @@ def display_health_report(has_custom_backend_issues = any(not report.has_kit_template for report in invalid_backends.values())# Determine if we have any recommendations to show- has_deck_drift = not deck_healthy+ has_deck_drift = not deck_healthy or not internal_backend_healthyhas_recommendations = (can_auto_fix_configor not log_sink_healthy@@ -944,7 +952,7 @@ def display_health_report(console.print(f" • Fix validation errors in [cyan]{escape(config_location.config_dir)}/telemetry.toml[/cyan]")if has_deck_drift:- console.print(" • Run [cyan]pipelex update[/cyan] to refresh the model deck from the current kit")+ console.print(" • Run [cyan]pipelex update[/cyan] to refresh the model deck and backends/internal.toml from the current kit")if not plugins_healthy:console.print(@@ -1010,6 +1018,18 @@ def display_health_report(console.print()+def _print_kit_sync_row(*, healthy: bool, message: str, report: DeckSyncReport) -> None:+ """Render one kit-managed area's sync row, with the per-file detail when files need action."""+ console = get_console()+ if healthy:+ console.print(f" [green]✓[/green] {escape(message)}")+ return+ console.print(f" [yellow]⚠[/yellow] {escape(message)}")+ actionable_statuses = {name: status for name, status in report.files.items() if status != DeckFileStatus.UP_TO_DATE}+ for filename in sorted(actionable_statuses):+ console.print(f" [dim]{escape(filename)}[/dim] — {status_rich_label(actionable_statuses[filename])}")++def _print_pending_migrations(*, check: PendingMigrationsCheck) -> None:"""Render the configuration-migration row: the verdict, then the files it is about.@@ -1050,20 +1070,47 @@ def check_deck_sync(*, config_dir: Path | None = None) -> tuple[bool, DeckSyncRe# since the missing-init advisory is handled by the config_files / backends checks above.return True, DeckSyncReport(kit_version="", installed_kit_version=None, manifest_present=False, files={}), "Deck directory not present"- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)+ healthy, message = _summarize_kit_sync(report=report, subject="Deck", file_noun="deck file(s)")+ return healthy, report, message+++def check_internal_backend_sync(*, config_dir: Path | None = None) -> tuple[bool, DeckSyncReport, str]:+ """Check whether the installed ``backends/internal.toml`` is in sync with the kit shipped by this pipelex version.++ The one backend file ``pipelex update`` manages, reported the way the deck is: it declares the software-only+ models open Pipelex ships, so an installation that has not refreshed it lacks the ones a release added.++ Args:+ config_dir: Explicit config directory override. If None, uses layered resolution.++ Returns:+ Tuple of (is_healthy, sync_report, summary_message)+ """+ # The backends directory boot actually reads, resolved on its own as it is at boot.+ backends_dir = Path(config_dir) / "inference" / "backends" if config_dir is not None else config_manager.backends_dir_path++ if not backends_dir.exists():+ # As for the deck: an uninitialized area is the config checks' to report.+ return True, DeckSyncReport(kit_version="", installed_kit_version=None, manifest_present=False, files={}), "Backends directory not present"++ report = compute_sync_report(backends_dir, area=KitManagedArea.BACKENDS)+ healthy, message = _summarize_kit_sync(report=report, subject=f"backends/{MANAGED_BACKEND_FILENAME}", file_noun="internal backend file(s)")+ return healthy, report, message+++def _summarize_kit_sync(*, report: DeckSyncReport, subject: str, file_noun: str) -> tuple[bool, str]:+ """Whether one kit-managed area is in sync, and the sentence the report says about it."""if report.is_clean():- return True, report, f"Deck is up to date with pipelex {report.kit_version}"+ return True, f"{subject} is up to date with pipelex {report.kit_version}"actionable = [name for name, status in report.files.items() if status.needs_action]if not report.manifest_present:- return False, report, "Deck manifest missing — run `pipelex update` to materialize a baseline"+ return False, f"{subject} manifest missing — run `pipelex update` to materialize a baseline"if report.installed_kit_version != report.kit_version:- return (- False,- report,- f"Deck installed for pipelex {report.installed_kit_version}, current is {report.kit_version} ({len(actionable)} file(s) need action)",- )- return False, report, f"{len(actionable)} deck file(s) need action"+ versions = f"installed for pipelex {report.installed_kit_version}, current is {report.kit_version}"+ return False, f"{subject} {versions} ({len(actionable)} file(s) need action)"+ return False, f"{len(actionable)} {file_noun} need action"class LogSinkCheck(BaseModel):@@ -1165,6 +1212,24 @@ def install_doctor_log_sink(*, registry: LogSinkRegistry | None, log_config: Logreturn LogSinkCheck(is_healthy=True, message=f"Log sink '{log_config.sink}' installed")+def build_doctor_registrar() -> PluginRegistrar:+ """Discover the plugins the way a full boot does, with no boot orchestrator.++ Pure and safe to run more than once, like ``build_registrar`` itself: the doctor runs it for the log sink row and+ again for the models row, which needs the internal models and deck defaults the plugins declare.++ Raises:+ PluginError: a plugin could not be discovered or registered.+ """+ return build_registrar(+ config=get_config(),+ boot_orchestrator=None,+ builtin_plugins=BUILTIN_PLUGINS,+ core_unconditional_plugin_names=CORE_UNCONDITIONAL_PLUGIN_NAMES,+ entry_point_groups=ENTRY_POINT_GROUPS,+ )++def discover_plugins_and_install_doctor_log_sink(*, log_config: LogConfig) -> DoctorRuntimeSetup:"""Discover the plugins the way boot does, then install the configured sink, each a row rather than the end of the report.@@ -1173,13 +1238,7 @@ def discover_plugins_and_install_doctor_log_sink(*, log_config: LogConfig) -> Doa configuration error and which the doctor exists to diagnose rather than die on."""try:- registrar = build_registrar(- config=get_config(),- boot_orchestrator=None,- builtin_plugins=BUILTIN_PLUGINS,- core_unconditional_plugin_names=CORE_UNCONDITIONAL_PLUGIN_NAMES,- entry_point_groups=ENTRY_POINT_GROUPS,- )+ registrar = build_doctor_registrar()except PluginError as exc:return DoctorRuntimeSetup(plugins=PluginsCheck(is_healthy=False, message=f"The plugin registry did not build: {exc.message}"),@@ -1327,6 +1386,17 @@ def check_models(*, config_dir: Path | None = None) -> tuple[bool, str, dict[strrouting_profile_override = config_manager.routing_profiles_file_paths(config_dir=config_dir) if config_dir is not None else Nonedeck_dir_override = str(config_dir / "inference" / "deck") if config_dir is not None else None+ # The plugins' internal models and deck defaults are part of the deck boot builds, so the check builds it with them.+ # A registry that does not build is the Plugins row's to report; here it only means the deck cannot be checked.+ try:+ plugin_model_declarations = build_doctor_registrar().make_model_declarations()+ except PluginError as exc:+ return (+ False,+ f"Error checking models: the plugin registry did not build, so the model deck cannot be checked: {exc.message}",+ backend_file_reports,+ )+models_manager = ModelManager()secrets_provider = EnvSecretsProvider()try:@@ -1334,6 +1404,7 @@ def check_models(*, config_dir: Path | None = None) -> tuple[bool, str, dict[strsecrets_provider=secrets_provider,managed_gateway_configs=managed_gateway_configs,gateway_config_source=gateway_config_source,+ plugin_model_declarations=plugin_model_declarations,backends_library_paths=backends_library_override,backends_dir_path=backends_dir_override,routing_profile_library_paths=routing_profile_override,@@ -1360,6 +1431,7 @@ def check_models(*, config_dir: Path | None = None) -> tuple[bool, str, dict[strModelDeckValidationError,InferenceBackendCredentialsError,GatewayUnknownModelError,+ PluginModelDeclarationError,) as exc:return False, f"Error checking models: {exc}", backend_file_reports@@ -1452,6 +1524,7 @@ def do_doctor_cmd(models_skipped = Truedeck_healthy, deck_report, deck_message = check_deck_sync()+ internal_backend_healthy, internal_backend_report, internal_backend_message = check_internal_backend_sync()# Display reportdisplay_health_report(@@ -1470,6 +1543,9 @@ def do_doctor_cmd(deck_healthy=deck_healthy,deck_message=deck_message,deck_report=deck_report,+ internal_backend_healthy=internal_backend_healthy,+ internal_backend_message=internal_backend_message,+ internal_backend_report=internal_backend_report,config_location=config_location,fix_mode=fix,log_sink_check=log_sink_check,@@ -1483,6 +1559,7 @@ def do_doctor_cmd(and backends_healthyand models_healthyand deck_healthy+ and internal_backend_healthyand (log_sink_check is None or log_sink_check.is_healthy)and (plugins_check is None or plugins_check.is_healthy))@@ -1506,7 +1583,10 @@ def do_doctor_cmd(fixable_backends = [(name, report) for name, report in invalid_backends if report.has_kit_template]can_fix_backends = len(fixable_backends) > 0- can_fix_deck = not deck_healthy and deck_report.kit_version != ""+ # `pipelex update` refreshes both kit-managed areas, so either one out of sync is the same fix.+ can_fix_deck = (not deck_healthy and deck_report.kit_version != "") or (+ not internal_backend_healthy and internal_backend_report.kit_version != ""+ )has_auto_fixable_issues = can_fix_config or can_fix_migrations or can_fix_telemetry or can_fix_backends or can_fix_deck@@ -1567,11 +1647,11 @@ def do_doctor_cmd(# Fix outdated model deckif can_fix_deck:- if Confirm.ask("[bold]Update the model deck now?[/bold]", default=True):+ if Confirm.ask("[bold]Update the model deck and backends/internal.toml now?[/bold]", default=True):try:console.print()update_cmd(yes=True)- console.print("[green]✓[/green] Model deck updated")+ console.print("[green]✓[/green] Model deck and backends/internal.toml updated")except Exception as exc: # ruff: ignore[blind-except]# Doctor --fix handler: wraps the whole update_cmd sub-command; a fix failure is reported and the doctor run continues.console.print(f"[red]Failed to update deck: {escape(str(exc))}[/red]")diff --git a/pipelex/cli/commands/init/command.py b/pipelex/cli/commands/init/command.pyindex cc3449142..15bc95ae5 100644--- a/pipelex/cli/commands/init/command.py+++ b/pipelex/cli/commands/init/command.py@@ -27,7 +27,7 @@ from pipelex.cli.commands.init.ui.gateway_ui import ()from pipelex.cli.commands.init.ui.general_ui import build_initialization_panelfrom pipelex.cli.commands.init.ui.types import InitFocus-from pipelex.cogt.models.deck_manifest import compute_kit_manifest, write_manifest+from pipelex.cogt.models.deck_manifest import stamp_kit_manifestsfrom pipelex.kit.paths import get_kit_configs_dirfrom pipelex.runtime_hub import get_consolefrom pipelex.system.configuration.config_loader import config_manager@@ -375,9 +375,9 @@ def execute_initialization(dst_path = target_deck_dir / deck_file.nameshutil.copy2(deck_file, dst_path)- # Stamp the deck manifest so future updates can detect drift and- # `pipelex update` knows the exact kit version this install came from.- write_manifest(compute_kit_manifest(), deck_dir=target_deck_dir)+ # Stamp the kit manifests of the deck and of backends/internal.toml so future updates can+ # detect drift and `pipelex update` knows the exact kit version this install came from.+ stamp_kit_manifests(inference_dir=target_inference_dir)# Reset routing_profiles.tomltemplate_routing_path = template_inference_dir / "routing_profiles.toml"diff --git a/pipelex/cli/commands/run/_main_stuff_file.py b/pipelex/cli/commands/run/_main_stuff_file.pynew file mode 100644index 000000000..ca806e550--- /dev/null+++ b/pipelex/cli/commands/run/_main_stuff_file.py@@ -0,0 +1,79 @@+"""Copy a file-shaped main output into the results folder, so the result opens with a double-click.++A Document or Image main output is otherwise saved only as JSON holding its URL, which for a local run is a+`pipelex-storage://` URI nobody can open. The bytes are loaded through the run's storage provider and+written beside `main_stuff.json` under the file's own name: `invoice-INV-2026-0142.pdf`, not a hash.+"""++import mimetypes+import urllib.parse+from pathlib import Path, PurePosixPath++from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.runtime_hub import get_storage_provider+from pipelex.tools.uri.uri_bytes import load_bytes_from_any_uri+from pipelex.tools.uri.uri_resolver import BASE64_DATA_URL_PREFIX++_FALLBACK_STEM = "main_stuff_file"+++def _filename_from_url(url: str) -> str | None:+ if url.startswith(BASE64_DATA_URL_PREFIX):+ return None+ name = PurePosixPath(urllib.parse.urlparse(url).path).name+ return name or None+++def _main_stuff_file_name(*, url: str, filename: str | None, mime_type: str | None) -> str:+ name = Path(filename).name if filename else _filename_from_url(url)+ if name:+ return name+ extension = mimetypes.guess_extension(mime_type or "") or ".bin"+ return f"{_FALLBACK_STEM}{extension}"+++def _free_target_path(*, output_dir: Path, name: str, reserved_names: frozenset[str]) -> Path:+ """`name` in `output_dir`, or a numbered variant of it when a file of that name is there or the run writes one later.++ The results folder already holds the run's own artifacts, such as `main_stuff.json`, when the file is copied, and+ a Document passed through from an earlier run can bear one of their names. Names compare without case, the files+ already there included, so the folder gets the same names on a case-sensitive file system as on a case-insensitive+ one, where `MAIN_STUFF.json` would overwrite `main_stuff.json`.+ """+ taken = {reserved_name.casefold() for reserved_name in reserved_names}+ if output_dir.is_dir():+ taken.update(existing_path.name.casefold() for existing_path in output_dir.iterdir())+ stem = PurePosixPath(name).stem+ suffix = PurePosixPath(name).suffix+ target_path = output_dir / name+ index = 1+ while target_path.name.casefold() in taken:+ target_path = output_dir / f"{stem}-{index}{suffix}"+ index += 1+ return target_path+++async def save_main_stuff_file(*, content: StuffContent, output_dir: Path, reserved_names: frozenset[str]) -> Path | None:+ """Write a Document or Image main output's bytes into `output_dir`, and return the file's path.++ The file never replaces one already in `output_dir`, nor takes one of `reserved_names`, the files the run writes+ there afterwards. Returns None when the content is not file-shaped. Loading can fail like any fetch, which the+ caller reports without failing the run that already succeeded.+ """+ url: str+ name: str+ match content:+ case DocumentContent():+ url = content.url+ name = _main_stuff_file_name(url=url, filename=content.filename, mime_type=content.mime_type)+ case ImageContent():+ url = content.url+ name = _main_stuff_file_name(url=url, filename=content.filename, mime_type=content.mime_type)+ case _:+ return None+ raw_bytes = await load_bytes_from_any_uri(url, storage_provider=get_storage_provider())+ target_path = _free_target_path(output_dir=output_dir, name=name, reserved_names=reserved_names)+ target_path.write_bytes(raw_bytes)+ return target_pathdiff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.pyindex 5d6542661..12958ebb9 100644--- a/pipelex/cli/commands/run/_run_core.py+++ b/pipelex/cli/commands/run/_run_core.py@@ -7,12 +7,14 @@ from typing import TYPE_CHECKING, castimport typerfrom posthog import tag+from rich.markup import escapefrom pipelex import logfrom pipelex.base_exceptions import PipelexErrorfrom pipelex.cli.cli_factory import make_pipelex_for_clifrom pipelex.cli.commands.run._inputs_file_loader import load_inputs_dict_from_pathfrom pipelex.cli.commands.run._inputs_path_resolver import resolve_inputs_paths+from pipelex.cli.commands.run._main_stuff_file import save_main_stuff_filefrom pipelex.cli.error_handlers import (ErrorContext,handle_dedicated_failure_panel,@@ -54,6 +56,7 @@ if TYPE_CHECKING:from pipelex.core.stuffs.stuff_content import StuffContentCOMMAND = "run"+_WORKING_MEMORY_FILENAME = "working_memory.json"def validate_run_flag_combination(*, dry_run: bool, mock_usage: bool, mock_inputs: bool) -> None:@@ -311,6 +314,7 @@ async def _execute_run(# Save main_stuff files if enabled. An absent main output saves the explicit absence# artifact (json + md, no interactive viewer — nothing to view), never value renders.saved_main_stuff_formats: list[str] = []+ saved_main_stuff_file: Path | None = Noneif save_main_stuff and output_path:if isinstance(main_resolved, AbsenceRecord):absence_json_path = output_path / "main_stuff.json"@@ -348,13 +352,23 @@ async def _execute_run(log.verbose(f"Main stuff HTML viewer saved to: {main_stuff_viewer_path}")saved_main_stuff_formats.append("html_viewer")+ # A Document or Image main output also lands as the file itself, under its own name. A dry run+ # produced a placeholder, not a file, so there is nothing to copy.+ if not dry_run:+ try:+ saved_main_stuff_file = await save_main_stuff_file(+ content=main_stuff.content, output_dir=output_path, reserved_names=frozenset({_WORKING_MEMORY_FILENAME})+ )+ except (PipelexError, OSError, ValueError) as file_exc:+ typer.secho(f"Could not copy the main output's file into {output_path}: {file_exc}", fg=typer.colors.YELLOW, err=True)+# Save working memory to JSON if enabledworking_memory_output_path: str | None = Noneif save_working_memory and output_path:if working_memory_path:working_memory_output_path = working_memory_pathelse:- working_memory_output_path = str(output_path / "working_memory.json")+ working_memory_output_path = str(output_path / _WORKING_MEMORY_FILENAME)working_memory_dict = pipe_output.working_memory.smart_dump()save_as_json_to_path(object_to_save=working_memory_dict, path=Path(working_memory_output_path))log.verbose(f"Working memory saved to: {working_memory_output_path}")@@ -435,20 +449,23 @@ async def _execute_run(console.print("\n[yellow]✓[/yellow] [bold]Dry run completed successfully[/bold]")else:console.print("\n[green]✓[/green] [bold]Pipeline execution completed successfully[/bold]")+ # Paths are escaped: a file name or an --output-dir can hold brackets, which Rich would read as markup.if output_path:- console.print(f" Output saved to [bold magenta]{output_path}[/bold magenta]:")+ console.print(f" Output saved to [bold magenta]{escape(str(output_path))}[/bold magenta]:")if saved_graphs:console.print(f" [green]✓[/green] graphs: {', '.join(saved_graphs)}")if saved_main_stuff_formats:console.print(f" [green]✓[/green] main_stuff: {', '.join(saved_main_stuff_formats)}")+ if saved_main_stuff_file:+ console.print(f" [green]✓[/green] file: [bold magenta]{escape(str(saved_main_stuff_file))}[/bold magenta]")if working_memory_output_path:if Path(working_memory_output_path).is_relative_to(output_path):- console.print(" [green]✓[/green] working_memory.json")+ console.print(f" [green]✓[/green] {_WORKING_MEMORY_FILENAME}")else:- console.print(f" [green]✓[/green] working_memory: {working_memory_output_path}")+ console.print(f" [green]✓[/green] working_memory: {escape(working_memory_output_path)}")# CSV output is written to a literal cwd-relative path (CQ1), independent of --output-dir.if save_csv is not None:- console.print(f" [green]✓[/green] CSV saved to [bold magenta]{save_csv}[/bold magenta]")+ console.print(f" [green]✓[/green] CSV saved to [bold magenta]{escape(save_csv)}[/bold magenta]")def execute_run(diff --git a/pipelex/cli/commands/update_cmd.py b/pipelex/cli/commands/update_cmd.pyindex 06100ec3b..b2db6170d 100644--- a/pipelex/cli/commands/update_cmd.py+++ b/pipelex/cli/commands/update_cmd.py@@ -1,8 +1,10 @@-"""``pipelex update`` — bring the installed model deck up to date with the kit-shipped templates.+"""``pipelex update`` — bring the kit-managed files of an installation up to date with the kit-shipped templates.-Numbered deck files are managed by pipelex. ``x_custom_*.toml`` files are user-owned and never-touched. Locally-modified numbered files are backed up to ``<file>.bak.<UTC-timestamp>`` before-being overwritten, unless ``--no-backup`` is passed.+Two areas are managed, each with its own manifest (see ``pipelex.cogt.models.deck_manifest``): the numbered+model deck files, and ``backends/internal.toml``, which declares the software-only models open Pipelex ships, so+an existing install receives the ones a release adds. ``x_custom_*.toml`` deck files and every other backend file+are user-owned and never touched. Locally-modified managed files are backed up to ``<file>.bak.<UTC-timestamp>``+before being overwritten, unless ``--no-backup`` is passed."""from __future__ import annotations@@ -11,6 +13,7 @@ import shutilimport sysfrom datetime import UTC, datetimefrom pathlib import Path+from typing import NamedTuplefrom rich.markup import escapefrom rich.panel import Panel@@ -20,9 +23,10 @@ from rich.table import Tablefrom pipelex.cogt.models.deck_manifest import (DeckFileStatus,DeckSyncReport,- compute_deck_sync_report,+ KitManagedArea,compute_kit_manifest,- kit_deck_dir,+ compute_sync_report,+ kit_area_dir,status_rich_label,suggest_x_custom_filename,write_manifest,@@ -31,6 +35,14 @@ from pipelex.runtime_hub import get_consolefrom pipelex.system.configuration.config_loader import config_manager+class _AreaUpdate(NamedTuple):+ """One kit-managed area of this update: where it is installed, and how it compares to the kit."""++ area: KitManagedArea+ installed_dir: Path+ report: DeckSyncReport++def update_cmd(*,local: bool = False,@@ -38,13 +50,13 @@ def update_cmd(dry_run: bool = False,no_backup: bool = False,) -> None:- """Apply pending model-deck updates from the running pipelex's kit.+ """Apply pending updates of the kit-managed files from the running pipelex's kit.Args:local: Update the project-local ``.pipelex/`` instead of the resolved layered config dir.yes: Skip the interactive confirmation.dry_run: Print the planned actions without modifying any file.- no_backup: Do not create ``.bak`` files for locally-modified numbered deck files.+ no_backup: Do not create ``.bak`` files for locally-modified managed files."""console = get_console()deck_dir = _resolve_deck_dir(local=local)@@ -54,21 +66,34 @@ def update_cmd(console.print()sys.exit(1)- report = compute_deck_sync_report(deck_dir)- _print_status_table(report, deck_dir=deck_dir)-- if report.is_clean():- console.print(_summary_panel("Model deck is up to date.", style="green"))+ area_dirs: list[tuple[KitManagedArea, Path]] = [(KitManagedArea.DECK, deck_dir)]+ # The backends directory resolves on its own, as boot resolves it: a project `.pipelex/` that carries a deck but+ # no backends reads the global ones, and that is the internal.toml to refresh. A directory that does not exist is+ # an installation `pipelex init` has not set up, which is `init`'s to fix, not this command's.+ backends_dir = _resolve_backends_dir(local=local)+ if backends_dir.is_dir():+ area_dirs.append((KitManagedArea.BACKENDS, backends_dir))++ area_updates = [+ _AreaUpdate(area=area, installed_dir=installed_dir, report=compute_sync_report(installed_dir, area=area)) for area, installed_dir in area_dirs+ ]+ for area_update in area_updates:+ _print_status_table(area_update.report, area=area_update.area, installed_dir=area_update.installed_dir)++ pending_updates = [area_update for area_update in area_updates if not area_update.report.is_clean()]+ if not pending_updates:+ console.print(_summary_panel("Model deck and internal backend are up to date.", style="green"))console.print()return- if not report.manifest_present:- # Migration: surface what we found so the user can review before we materialize a baseline.- console.print(- "[dim]No deck manifest found — this looks like an existing install. Running [cyan]pipelex update[/cyan] "- "will install the latest deck content and write a baseline manifest.[/dim]"- )- console.print()+ for area_update in pending_updates:+ if not area_update.report.manifest_present:+ # Migration: surface what we found so the user can review before we materialize a baseline.+ console.print(+ f"[dim]No {escape(area_update.area.display_name.lower())} manifest found — this looks like an existing install. "+ "Running [cyan]pipelex update[/cyan] will install the latest kit content and write a baseline manifest.[/dim]"+ )+ console.print()if dry_run:console.print("[dim]Dry run — no changes written.[/dim]")@@ -80,28 +105,43 @@ def update_cmd(console.print()return- actions_applied = _apply_updates(deck_dir=deck_dir, report=report, no_backup=no_backup)- write_manifest(compute_kit_manifest(), deck_dir=deck_dir)+ actions_applied = 0+ for area_update in pending_updates:+ actions_applied += _apply_updates(area_update.installed_dir, area=area_update.area, report=area_update.report, no_backup=no_backup)+ write_manifest(compute_kit_manifest(area=area_update.area), installed_dir=area_update.installed_dir)console.print()- console.print(_summary_panel(f"Model deck updated ({actions_applied} file change(s) applied).", style="green"))+ console.print(_summary_panel(f"Kit-managed files updated ({actions_applied} file change(s) applied).", style="green"))console.print()def _resolve_deck_dir(*, local: bool) -> Path:"""Pick the deck directory to operate on, mirroring the ``--local`` semantics of ``pipelex init``."""if local:- project_root = config_manager.project_root- base = project_root / ".pipelex" if project_root is not None else Path.cwd() / ".pipelex"- return base / "inference" / "deck"+ return _local_inference_dir() / "deck"# Match runtime resolution: fall through to the global deck when the project .pipelex/# does not contain inference/deck, so update targets the deck actually in use.return config_manager.model_decks_dir_path-def _print_status_table(report: DeckSyncReport, *, deck_dir: Path) -> None:- """Render the per-file sync status as a Rich table."""- table = Table(title="Pipelex Model Deck — Update Plan", show_lines=False)+def _resolve_backends_dir(*, local: bool) -> Path:+ """Pick the backends directory to operate on, the way ``_resolve_deck_dir`` picks the deck's."""+ if local:+ return _local_inference_dir() / "backends"+ # Match runtime resolution, as for the deck: the backends directory boot actually reads.+ return config_manager.backends_dir_path+++def _local_inference_dir() -> Path:+ """The project-local ``.pipelex/inference/`` directory, for ``--local``."""+ project_root = config_manager.project_root+ base = project_root / ".pipelex" if project_root is not None else Path.cwd() / ".pipelex"+ return base / "inference"+++def _print_status_table(report: DeckSyncReport, *, area: KitManagedArea, installed_dir: Path) -> None:+ """Render the per-file sync status of one area as a Rich table."""+ table = Table(title=f"Pipelex {area.display_name} — Update Plan", show_lines=False)table.add_column("File", style="cyan", no_wrap=True)table.add_column("Status", style="bold")table.add_column("Action")@@ -111,7 +151,7 @@ def _print_status_table(report: DeckSyncReport, *, deck_dir: Path) -> None:table.add_row(filename, status_rich_label(status), _action_description(status))get_console().print()- get_console().print(f"Deck directory: [cyan]{escape(str(deck_dir))}[/cyan]")+ get_console().print(f"{area.display_name} directory: [cyan]{escape(str(installed_dir))}[/cyan]")installed_version_label = report.installed_kit_version or "(none)"get_console().print(f"Installed kit version: [cyan]{escape(installed_version_label)}[/cyan] Kit version: [cyan]{escape(report.kit_version)}[/cyan]"@@ -135,22 +175,32 @@ def _action_description(status: DeckFileStatus) -> str:return "back up + overwrite from kit"-def _apply_updates(deck_dir: Path, *, report: DeckSyncReport, no_backup: bool) -> int:- """Apply the per-file actions described by ``report``. Returns the count of files changed."""- kit_dir = kit_deck_dir()+def _local_edits_tip(*, area: KitManagedArea, filename: str) -> str:+ """Where edits the update overwrote belong, so the next update leaves them in place."""+ match area:+ case KitManagedArea.DECK:+ x_custom_filename = escape(suggest_x_custom_filename(filename))+ return f"Tip: move custom aliases/presets into [cyan]{x_custom_filename}[/cyan] so future updates leave them in place."+ case KitManagedArea.BACKENDS:+ return "Tip: declare your own models in a backend of your own, so future updates leave them in place."+++def _apply_updates(installed_dir: Path, *, area: KitManagedArea, report: DeckSyncReport, no_backup: bool) -> int:+ """Apply the per-file actions described by ``report`` to one area. Returns the count of files changed."""+ kit_dir = kit_area_dir(area=area)timestamp = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ")actions_applied = 0for filename in sorted(report.files):status = report.files[filename]- installed_path = deck_dir / filename+ installed_path = installed_dir / filenamekit_path = kit_dir / filenamematch status:case DeckFileStatus.UP_TO_DATE:continuecase DeckFileStatus.KIT_ADDED:- deck_dir.mkdir(parents=True, exist_ok=True)+ installed_dir.mkdir(parents=True, exist_ok=True)shutil.copy2(kit_path, installed_path)get_console().print(f" [green]+[/green] installed [cyan]{escape(filename)}[/cyan]")actions_applied += 1@@ -166,11 +216,7 @@ def _apply_updates(deck_dir: Path, *, report: DeckSyncReport, no_backup: bool) -backup_note = f" (backed up to [dim]{escape(backup_path.name)}[/dim])"shutil.copy2(kit_path, installed_path)get_console().print(f" [red]↑[/red] overwrote local edits in [cyan]{escape(filename)}[/cyan]{backup_note}")- suggested_override = suggest_x_custom_filename(filename)- get_console().print(- f" [dim]Tip: move custom aliases/presets into [cyan]{escape(suggested_override)}[/cyan] "- "so future updates leave them in place.[/dim]"- )+ get_console().print(f" [dim]{_local_edits_tip(area=area, filename=filename)}[/dim]")actions_applied += 1case DeckFileStatus.KIT_REMOVED:backup_path = installed_path.with_name(f"{filename}.bak.{timestamp}")diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.pyindex 8621402a4..3b74fde42 100644--- a/pipelex/cogt/content_generation/assignment_models.py+++ b/pipelex/cogt/content_generation/assignment_models.py@@ -4,6 +4,8 @@ from pydantic import BaseModel, Fieldfrom typing_extensions import overridefrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParams+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenSetting+from pipelex.cogt.doc_gen.document_composition import DocumentCompositionfrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams@@ -158,6 +160,22 @@ class RenderPageViewsAssignment(BaseModel):return [UriReference(uri=self.document_uri, position="the document to render as page views")]+class RenderDocumentAssignment(BaseModel):+ """Serializable unit for a single document print: the composed document and the engine that prints it.++ ``doc_gen_setting.model`` is the resolved handle of the ``doc_gen`` model, which is also the routing key.+ """++ job_metadata: JobMetadata+ cogt_run_params: CogtRunParams+ composition: DocumentComposition+ doc_gen_setting: DocGenSetting++ def referenced_uris(self) -> list[UriReference]:+ """The URLs the leaf reads for this assignment: the images the composed document names."""+ return self.composition.referenced_uris()++class SearchAssignment(BaseModel):"""Serializable unit for a single web-search leaf call.diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex eada6e155..92e065ea5 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -8,12 +8,14 @@ from pipelex.cogt.content_generation.assignment_models import (ImgGenAssignment,LLMAssignment,ObjectAssignment,+ RenderDocumentAssignment,RenderPageViewsAssignment,SearchAssignment,TemplatingAssignment,)from pipelex.cogt.content_generation.cogt_run_params import CogtRunParamsfrom pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol, update_job_metadata+from pipelex.cogt.content_generation.doc_gen_generate import render_document_and_storefrom pipelex.cogt.content_generation.extract_generate import extract_gen_pages_and_storefrom pipelex.cogt.content_generation.generated_content_factory import GeneratedContentFactoryfrom pipelex.cogt.content_generation.img_gen_generate import img_gen_image_list_and_store, img_gen_single_image_and_store@@ -22,6 +24,8 @@ from pipelex.cogt.content_generation.object_revalidation import revalidate_leaf_from pipelex.cogt.content_generation.render_generate import render_page_views_and_storefrom pipelex.cogt.content_generation.search_generate import search_gen_sourced_answer, search_gen_structured_objectfrom pipelex.cogt.content_generation.templating_generate import templating_gen_text+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenSetting+from pipelex.cogt.doc_gen.document_composition import DocumentCompositionfrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams@@ -29,6 +33,7 @@ from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.config import get_config+from pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.page_content import PageContentfrom pipelex.core.stuffs.search_result_content import SearchResultContent@@ -237,6 +242,27 @@ class ContentGenerator(ContentGeneratorProtocol):generated_content_factory=self._generated_content_factory,)+ @override+ @update_job_metadata+ async def make_rendered_document(+ self,+ *,+ job_metadata: JobMetadata,+ cogt_run_params: CogtRunParams,+ composition: DocumentComposition,+ doc_gen_setting: DocGenSetting,+ ) -> DocumentContent:+ render_assignment = RenderDocumentAssignment(+ job_metadata=job_metadata,+ cogt_run_params=cogt_run_params,+ composition=composition,+ doc_gen_setting=doc_gen_setting,+ )+ return await render_document_and_store(+ render_assignment=render_assignment,+ generated_content_factory=self._generated_content_factory,+ )+@override@update_job_metadataasync def make_extract_pages(diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.pyindex 7135e82bd..b67e6365c 100644--- a/pipelex/cogt/content_generation/content_generator_protocol.py+++ b/pipelex/cogt/content_generation/content_generator_protocol.py@@ -4,12 +4,15 @@ from typing import Any, ParamSpec, Protocol, TypeVarfrom pipelex.cogt.content_generation.assignment_models import SearchAssignmentfrom pipelex.cogt.content_generation.cogt_run_params import CogtRunParams+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenSetting+from pipelex.cogt.doc_gen.document_composition import DocumentCompositionfrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParamsfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_setting import LLMSetting+from pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.page_content import PageContentfrom pipelex.core.stuffs.search_result_content import SearchResultContent@@ -128,6 +131,17 @@ class ContentGeneratorProtocol(Protocol):extract_job_config: ExtractJobConfig | None = None,) -> Coroutine[Any, Any, list[ImageContent]]: ...+ def make_rendered_document(+ self,+ *,+ job_metadata: JobMetadata,+ cogt_run_params: CogtRunParams,+ composition: DocumentComposition,+ doc_gen_setting: DocGenSetting,+ ) -> Coroutine[Any, Any, DocumentContent]:+ """Print a `PipeDocGen` step's composed document on the engine `doc_gen_setting` names and store it: the step's print stage."""+ ...+def make_extract_pages(self,*,diff --git a/pipelex/cogt/content_generation/doc_gen_generate.py b/pipelex/cogt/content_generation/doc_gen_generate.pynew file mode 100644index 000000000..68cb665c4--- /dev/null+++ b/pipelex/cogt/content_generation/doc_gen_generate.py@@ -0,0 +1,139 @@+"""Framework-agnostic document printing leaf, sibling of ``render_generate``.++Single home for "print a composed document and store it": the direct ``ContentGenerator`` calls it inline, and+a Temporal activity would call it on a worker. Printing and storage happen inside the leaf so the file's bytes+never cross a workflow boundary: only the URL-bearing ``DocumentContent`` is returned. The engine that prints is+the worker of the ``doc_gen`` model the assignment names, resolved when the step ran, and it runs in a worker+thread, since engines are synchronous; a file its document names, such as an image, it reads back through the+``RenderResources`` it is handed, under the run's read scope.++Engines run on a thread pool of their own rather than the event loop's default executor, because an engine's+thread waits while the loop reads a file for it, and that read may itself need a default-executor thread (a+local file read, a DNS lookup, a cloud storage call). Were the engines on the default executor, as many+concurrent prints as it has threads would hold every one of them, and each read would wait behind them until+it timed out.+"""++import asyncio+import contextvars+import threading+from concurrent.futures import ThreadPoolExecutor++from typing_extensions import override++from pipelex.base_exceptions import PipelexError+from pipelex.cogt.content_generation.assignment_models import RenderDocumentAssignment+from pipelex.cogt.content_generation.dry_mock import dry_render_document+from pipelex.cogt.content_generation.generated_content_factory import GeneratedContentFactory+from pipelex.cogt.content_generation.read_authorization import authorize_assignment_reads+from pipelex.cogt.doc_gen.doc_gen_engine import get_doc_gen_inference_model, require_doc_gen_engine_installed+from pipelex.cogt.doc_gen.doc_gen_worker_factory import DocGenWorkerFactory+from pipelex.cogt.doc_gen.exceptions import DocGenRenderError+from pipelex.cogt.doc_gen.render_job import RenderedDocument, RenderResources+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.runtime_hub import get_report_delegate+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract+from pipelex.tools.uri.uri_bytes import load_bytes_from_any_uri+from pipelex.tools.uri.uri_read_scope import authorize_uri_read++# How long an engine may wait for one file it reads, such as an image fetched over https.+_RESOURCE_LOAD_TIMEOUT_SECONDS = 120+# How many documents print at once; more wait their turn. A print is CPU-bound apart from its reads.+_MAX_CONCURRENT_PRINTS = 4+_PRINT_EXECUTOR = ThreadPoolExecutor(max_workers=_MAX_CONCURRENT_PRINTS, thread_name_prefix="pipelex-doc-gen")+++class RunRenderResources(RenderResources):+ """The `RenderResources` of one print: files read through the run's storage provider, under its read scope.++ An engine runs in a worker thread and reads synchronously, while storage and fetching are async, so each+ read is handed back to the event loop the print was started from, and the engine's thread waits for it.+ """++ def __init__(self, *, storage_provider: StorageProviderAbstract, read_scope: str | None, loop: asyncio.AbstractEventLoop):+ self._storage_provider = storage_provider+ self._read_scope = read_scope+ self._loop = loop+ self._loop_thread_id = threading.get_ident()++ async def _load(self, *, uri: str, position: str) -> bytes:+ authorize_uri_read(uri=uri, read_scope=self._read_scope, position=position)+ return await load_bytes_from_any_uri(uri, storage_provider=self._storage_provider)++ @override+ def load(self, *, uri: str, position: str) -> bytes:+ if threading.get_ident() == self._loop_thread_id:+ msg = "An engine reads its resources from the worker thread it prints in, never from the event loop's own thread."+ raise RuntimeError(msg)+ future = asyncio.run_coroutine_threadsafe(self._load(uri=uri, position=position), self._loop)+ try:+ return future.result(timeout=_RESOURCE_LOAD_TIMEOUT_SECONDS)+ except TimeoutError as exc:+ # Stop the read too, which would otherwise go on running on the loop after the print has failed.+ future.cancel()+ msg = f"Reading {position} took longer than {_RESOURCE_LOAD_TIMEOUT_SECONDS} seconds."+ raise DocGenRenderError(msg) from exc+++async def render_document_and_store(+ render_assignment: RenderDocumentAssignment,+ *,+ generated_content_factory: GeneratedContentFactory,+) -> DocumentContent:+ """Print the composed document with its engine, store the file, and return the `DocumentContent` pointing at it.++ The read scope is authorized first, then the DRY branch, which prints and stores nothing, so a dry run+ refuses what a live one would.++ Raises:+ UriReadRefusedError: an image the document names is outside the run's read scope.+ ModelNotFoundError: the assignment's engine is not a model served here.+ DocGenEngineMissingError: the engine is the built-in one or one of the plugin's and is no longer declared here, or+ no installed plugin registers it.+ DocGenRenderError: the engine could not print it, or failed in a way it did not report.+ """+ authorize_assignment_reads(job_metadata=render_assignment.job_metadata, uri_references=render_assignment.referenced_uris())+ if render_assignment.cogt_run_params.run_mode.is_dry:+ return dry_render_document(render_assignment)+ composition = render_assignment.composition+ inference_model = get_doc_gen_inference_model(+ model_handle=render_assignment.doc_gen_setting.model,+ doc_gen_format=composition.format,+ source=composition.source,+ pipe_code=render_assignment.job_metadata.pipe_code,+ )+ require_doc_gen_engine_installed(+ inference_model=inference_model,+ doc_gen_format=composition.format,+ source=composition.source,+ pipe_code=render_assignment.job_metadata.pipe_code,+ )+ worker = DocGenWorkerFactory.make_doc_gen_worker(inference_model=inference_model, reporting_delegate=get_report_delegate())+ render_job = composition.make_render_job()+ resources = RunRenderResources(+ storage_provider=generated_content_factory.storage_provider,+ read_scope=render_assignment.job_metadata.run_metadata.read_scope,+ loop=asyncio.get_running_loop(),+ )+ # The context is carried into the thread, as `asyncio.to_thread` would, so the engine logs under the run.+ run_context = contextvars.copy_context()++ def _print() -> RenderedDocument:+ return run_context.run(worker.render, job=render_job, resources=resources)++ try:+ rendered = await asyncio.get_running_loop().run_in_executor(_PRINT_EXECUTOR, _print)+ except PipelexError:+ # Already classified: a refused read, or a failure the engine reported as a `DocGenRenderError`.+ raise+ except Exception as exc:+ # Dynamic plugin dispatch: an engine is plugin code whose exceptions cannot be enumerated, and anything+ # else it raises is a failure to print this document.+ msg = f"The engine '{inference_model.name}' could not print '{composition.filename}': {exc}"+ raise DocGenRenderError(msg) from exc+ return await generated_content_factory.make_document_content(+ storage_scope=render_assignment.job_metadata.run_metadata.storage_scope,+ data=rendered.data,+ mime_type=composition.format.mime_type,+ filename=composition.filename,+ )diff --git a/pipelex/cogt/content_generation/dry_mock.py b/pipelex/cogt/content_generation/dry_mock.pyindex 5a30b80cb..849c1b2a2 100644--- a/pipelex/cogt/content_generation/dry_mock.py+++ b/pipelex/cogt/content_generation/dry_mock.py@@ -47,6 +47,7 @@ from pipelex.cogt.content_generation.assignment_models import (ImgGenAssignment,LLMAssignment,ObjectAssignment,+ RenderDocumentAssignment,RenderPageViewsAssignment,SearchAssignment,SearchObjectAssignment,@@ -408,6 +409,16 @@ def dry_render_page_views(render_assignment: RenderPageViewsAssignment) -> list[return [_dry_image_content(image_url=image_urls[page_index % len(image_urls)]) for page_index in range(nb_pages)]+def dry_render_document(render_assignment: RenderDocumentAssignment) -> DocumentContent:+ """Dry leaf for document printing: a mock Document named as the live one would be, no engine, no storage IO.++ Its URL is the blank one-page PDF every mocked document carries, whatever the format asked for.+ """+ composition = render_assignment.composition+ log.verbose(f"🤡 DRY RUN: render_document '{composition.filename}' for '{render_assignment.job_metadata.run_metadata.pipeline_run_id}'")+ return DocumentContent(url=DryRunFactory.generate_mock_document_url(), mime_type=composition.format.mime_type, filename=composition.filename)++def dry_search_gen_sourced_answer(search_assignment: SearchAssignment) -> SearchResultContent:"""Dry leaf for sourced-answer search: polyfactory-built result with mock sources, no provider."""log.verbose(f"🤡 DRY RUN: search_gen_sourced_answer for '{search_assignment.search_handle}'")diff --git a/pipelex/cogt/content_generation/generated_content_factory.py b/pipelex/cogt/content_generation/generated_content_factory.pyindex 6ba2d4b92..7a0428ff5 100644--- a/pipelex/cogt/content_generation/generated_content_factory.py+++ b/pipelex/cogt/content_generation/generated_content_factory.py@@ -6,6 +6,7 @@ from pipelex.cogt.content_generation.exceptions import NeitherUrlNorDataErrorfrom pipelex.cogt.extract.extract_output import ExtractOutputfrom pipelex.cogt.image.generated_image import GeneratedImageRawDetailsfrom pipelex.config import get_config+from pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.page_content import PageContentfrom pipelex.core.stuffs.text_and_images_content import TextAndImagesContent@@ -112,6 +113,27 @@ class GeneratedContentFactory:return raw_bytes, media_typereturn raw_bytes, None+ async def make_document_content(+ self,+ *,+ storage_scope: str,+ data: bytes,+ mime_type: str,+ filename: str,+ ) -> DocumentContent:+ """Store a produced file's bytes and return the `DocumentContent` that points at them.++ The key is `{storage_scope}/generated/{hash}/{filename}`: the hash keeps two different files with the+ same name apart, and ending in the human filename means every backend's URL, and so every download, is+ called `invoice-INV-2026-0142.pdf` rather than a hash. The caller owns the filename's safety: it must be+ one path segment.+ """+ hash_digest = hashlib.sha256(data).hexdigest()[:16]+ storage_key = f"{storage_scope}/{GENERATED_CONTENT_LEAF}/{hash_digest}/{filename}"+ url = await self.storage_provider.store(data=data, key=storage_key, content_type=mime_type)+ public_url = await self.storage_provider.public_url(uri=url)+ return DocumentContent(url=url, public_url=public_url, mime_type=mime_type, filename=filename)+async def make_image_content(self,storage_scope: str,diff --git a/pipelex/cogt/doc_gen/__init__.py b/pipelex/cogt/doc_gen/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/cogt/doc_gen/doc_gen_engine.py b/pipelex/cogt/doc_gen/doc_gen_engine.pynew file mode 100644index 000000000..7627b001b--- /dev/null+++ b/pipelex/cogt/doc_gen/doc_gen_engine.py@@ -0,0 +1,161 @@+"""Which document engine prints a `PipeDocGen` step: the `doc_gen` model it names, or the deck's default for it.++An engine is a model of the `doc_gen` family, declared with the sources it prints from as its `inputs` and its format+as its `outputs`, in the kit's `internal.toml` for the built-in engine or by the plugin that ships it for the others,+and its worker is registered by a plugin. A step may name one in its+`model`; a step that names none prints on the model deck's default for its format and source. The same resolution+runs when the method loads, so a step no engine here can print is refused before a run spends anything, and when+the step runs, where the resolved handle rides into the print stage as its routing key.++A missing engine is refused naming its remedy. Core knows the name of the built-in engine, `reportlab-pdf`, and the+names of the engines the Pipelex document generation plugin declares when it loads, so a step that asks for one+this installation does not declare is told to run `pipelex update` or to install the plugin, rather than that its+model is unknown.+"""++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice, DocGenSetting+from pipelex.cogt.doc_gen.exceptions import KNOWN_DOC_GEN_MODEL_NAMES, DocGenEngineGap, DocGenEngineMissingError, DocGenModelCapabilityError+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_reference import ModelReferenceKind, ensure_model_reference+from pipelex.plugins.inference_backend_registry import InferenceFamily+from pipelex.runtime_hub import get_inference_backend_registry, get_model_deck+++def prints_document(*, inference_model: InferenceModelSpec, doc_gen_format: DocGenFormat, source: DocGenSource) -> bool:+ """Whether a `doc_gen` model prints this format from this source, by the `inputs` and `outputs` it declares."""+ return str(source) in inference_model.inputs and str(doc_gen_format) in inference_model.outputs+++def require_doc_gen_engine_installed(+ *,+ inference_model: InferenceModelSpec,+ doc_gen_format: DocGenFormat,+ source: DocGenSource,+ pipe_code: str | None,+) -> None:+ """Refuse a model whose engine no plugin registers in this runtime.++ The other model families find a missing worker only when they build it, at run time; a document engine is+ checked when the method loads too, so the refusal comes before any inference is spent.++ Raises:+ DocGenEngineMissingError: no installed plugin registers the model's sdk.+ """+ if not get_inference_backend_registry().has(family=InferenceFamily.DOC_GEN, sdk=inference_model.sdk):+ raise DocGenEngineMissingError(+ gap=DocGenEngineGap.NOT_REGISTERED,+ doc_gen_format=doc_gen_format,+ source=source,+ model=inference_model.name,+ sdk=inference_model.sdk,+ pipe_code=pipe_code,+ )+++def _refuse_undeclared_known_engine(*, model_handle: str, doc_gen_format: DocGenFormat, source: DocGenSource, pipe_code: str | None) -> None:+ """Refuse the built-in engine or one of the plugin's, named by its model name, where this installation does not declare it.++ Only these names have a remedy to name: the plugin declares its own engines, and `pipelex update` refreshes the+ kit's `internal.toml`, which declares the built-in one. Any other handle is left to the deck's own checks.++ Raises:+ DocGenEngineMissingError: the handle is a known engine name the model deck does not serve.+ """+ if model_handle not in KNOWN_DOC_GEN_MODEL_NAMES:+ return+ if get_model_deck().get_optional_inference_model(model_handle=model_handle, model_type=ModelType.DOC_GEN) is not None:+ return+ raise DocGenEngineMissingError(+ gap=DocGenEngineGap.NOT_DECLARED,+ doc_gen_format=doc_gen_format,+ source=source,+ model=model_handle,+ sdk=None,+ pipe_code=pipe_code,+ )+++def require_doc_gen_engine_declared(+ *,+ doc_gen_choice: DocGenModelChoice,+ doc_gen_format: DocGenFormat,+ source: DocGenSource,+ pipe_code: str | None,+) -> None:+ """Refuse a choice that names the built-in engine or one of the plugin's directly, where this installation does not declare it.++ Runs before the deck resolves the choice, which would otherwise refuse such a name as an unknown model and name+ no remedy. An alias, a waterfall or a preset resolves through the deck first, and the model it lands on is+ checked by `get_doc_gen_inference_model`.++ Raises:+ DocGenEngineMissingError: the choice is a known engine name the model deck does not serve.+ """+ model_handle: str+ if isinstance(doc_gen_choice, DocGenSetting):+ model_handle = doc_gen_choice.model+ else:+ ref = ensure_model_reference(doc_gen_choice)+ match ref.kind:+ case ModelReferenceKind.HANDLE:+ model_handle = ref.name+ case ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL | ModelReferenceKind.PRESET:+ return+ _refuse_undeclared_known_engine(model_handle=model_handle, doc_gen_format=doc_gen_format, source=source, pipe_code=pipe_code)+++def get_doc_gen_inference_model(+ *,+ model_handle: str,+ doc_gen_format: DocGenFormat,+ source: DocGenSource,+ pipe_code: str | None,+) -> InferenceModelSpec:+ """The `doc_gen` model a handle resolves to, refusing a known engine this installation does not declare with its remedy.++ Raises:+ DocGenEngineMissingError: the handle is the built-in engine or one of the plugin's, and the deck does not serve it.+ ModelNotFoundError: any other handle that resolves to no model served here.+ """+ _refuse_undeclared_known_engine(model_handle=model_handle, doc_gen_format=doc_gen_format, source=source, pipe_code=pipe_code)+ return get_model_deck().get_required_inference_model(model_handle=model_handle, model_type=ModelType.DOC_GEN)+++def resolve_doc_gen_setting(+ *,+ doc_gen_choice: DocGenModelChoice | None,+ doc_gen_format: DocGenFormat,+ source: DocGenSource,+ pipe_code: str | None,+) -> DocGenSetting:+ """The engine that prints a step, pinned to the model handle it resolves to, checked to print the step here.++ Raises:+ DocGenEngineMissingError: the deck names no default for the step and the step names no engine, the engine is+ the built-in one or one of the plugin's and this installation does not declare it, or no installed plugin+ registers the engine.+ ModelChoiceNotFoundError: the step names a reference the deck does not define.+ ModelNotFoundError: the reference resolves to no model served here.+ DocGenModelCapabilityError: the engine does not print the step's format from its source.+ """+ model_deck = get_model_deck()+ resolved_choice = doc_gen_choice+ if resolved_choice is None:+ resolved_choice = model_deck.get_doc_gen_choice_default(doc_gen_format=doc_gen_format, source=source)+ if resolved_choice is None:+ raise DocGenEngineMissingError(+ gap=DocGenEngineGap.NO_DEFAULT, doc_gen_format=doc_gen_format, source=source, model=None, sdk=None, pipe_code=pipe_code+ )+ require_doc_gen_engine_declared(doc_gen_choice=resolved_choice, doc_gen_format=doc_gen_format, source=source, pipe_code=pipe_code)+ doc_gen_setting = model_deck.get_doc_gen_setting(doc_gen_choice=resolved_choice)+ inference_model = get_doc_gen_inference_model(+ model_handle=doc_gen_setting.model, doc_gen_format=doc_gen_format, source=source, pipe_code=pipe_code+ )+ if not prints_document(inference_model=inference_model, doc_gen_format=doc_gen_format, source=source):+ raise DocGenModelCapabilityError(doc_gen_format=doc_gen_format, source=source, model=inference_model.name, pipe_code=pipe_code)+ require_doc_gen_engine_installed(inference_model=inference_model, doc_gen_format=doc_gen_format, source=source, pipe_code=pipe_code)+ if inference_model.name != doc_gen_setting.model:+ doc_gen_setting = doc_gen_setting.model_copy(update={"model": inference_model.name})+ return doc_gen_settingdiff --git a/pipelex/cogt/doc_gen/doc_gen_format.py b/pipelex/cogt/doc_gen/doc_gen_format.pynew file mode 100644index 000000000..34c7015a9--- /dev/null+++ b/pipelex/cogt/doc_gen/doc_gen_format.py@@ -0,0 +1,151 @@+from enum import StrEnum+++class DocGenFormat(StrEnum):+ """The file formats a `PipeDocGen` step can ask for.++ The tokens are MTHDS: a method names one, and it loads wherever the standard is read. Which of them a+ runtime prints depends on the document engines installed in it, each a model of the `doc_gen` family:+ open Pipelex prints a `pdf` laid out without a template, on `reportlab-pdf`, and the Pipelex document+ generation plugin prints the rest. HTML is not a format: `PipeCompose` already produces the native+ `Html`.+ """++ PDF = "pdf"+ XLSX = "xlsx"+ DOCX = "docx"+ PPTX = "pptx"++ @property+ def mime_type(self) -> str:+ match self:+ case DocGenFormat.PDF:+ return "application/pdf"+ case DocGenFormat.XLSX:+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"+ case DocGenFormat.DOCX:+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+ case DocGenFormat.PPTX:+ return "application/vnd.openxmlformats-officedocument.presentationml.presentation"++ @property+ def with_article(self) -> str:+ """The token after its indefinite article, for a message: 'a pdf', 'an xlsx'."""+ match self:+ case DocGenFormat.XLSX:+ return f"an {self}"+ case DocGenFormat.PDF | DocGenFormat.DOCX | DocGenFormat.PPTX:+ return f"a {self}"++ @property+ def suffix(self) -> str:+ """The file suffix, without its dot."""+ return self.value++ @property+ def template_file_suffix(self) -> str:+ """The suffix a `template_file` must carry for this format: an HTML template for a PDF, the office file itself otherwise."""+ match self:+ case DocGenFormat.PDF:+ return ".html"+ case DocGenFormat.XLSX:+ return ".xlsx"+ case DocGenFormat.DOCX:+ return ".docx"+ case DocGenFormat.PPTX:+ return ".pptx"++ @property+ def is_template_html(self) -> bool:+ """Whether this format's template is HTML, composed by Pipelex before an engine prints it.++ Only a PDF's is: an office template is a file its engine fills from plain data.+ """+ match self:+ case DocGenFormat.PDF:+ return True+ case DocGenFormat.XLSX | DocGenFormat.DOCX | DocGenFormat.PPTX:+ return False++ @property+ def has_auto_layout(self) -> bool:+ """Whether a step in this format may have no template and print the auto-layout of its inputs.++ A deck generated without a design is not what anyone asks for (DB8), so `pptx` needs a template file.+ """+ match self:+ case DocGenFormat.PDF | DocGenFormat.XLSX | DocGenFormat.DOCX:+ return True+ case DocGenFormat.PPTX:+ return False+++class DocGenSource(StrEnum):+ """What the compose stage hands an engine to print, which is half of what picks the engine.++ A `pdf` from the layout tree and a `pdf` from composed HTML may print on different engines, so the model deck+ names a default engine per format and per source, and a `doc_gen` model lists the sources it prints from as+ its `inputs` and its format as its `outputs`.+ """++ LAYOUT = "layout"+ """The auto-layout of the step's inputs, as a layout tree: the step has no template."""++ HTML = "html"+ """Composed HTML: a `pdf` step's HTML template rendered against its inputs."""++ TEMPLATE_FILE = "template_file"+ """An office template file, which the engine fills from the step's inputs as plain data."""++ @property+ def desc(self) -> str:+ """How a message names this source after a format: 'pdf from the auto-layout'."""+ match self:+ case DocGenSource.LAYOUT:+ return "from the auto-layout of its inputs"+ case DocGenSource.HTML:+ return "from an HTML template"+ case DocGenSource.TEMPLATE_FILE:+ return "from a template file"++ @classmethod+ def for_step(cls, *, doc_gen_format: DocGenFormat, has_template: bool) -> "DocGenSource":+ """The source a step's format and template make: no template is the layout, a template is HTML for a PDF and a file otherwise."""+ if not has_template:+ return DocGenSource.LAYOUT+ if doc_gen_format.is_template_html:+ return DocGenSource.HTML+ return DocGenSource.TEMPLATE_FILE++ @classmethod+ def possible_for(cls, *, doc_gen_format: DocGenFormat) -> list["DocGenSource"]:+ """The sources a step in this format can compose from: the auto-layout where the format has one, and its kind of template."""+ sources: list[DocGenSource] = []+ if doc_gen_format.has_auto_layout:+ sources.append(DocGenSource.LAYOUT)+ sources.append(DocGenSource.HTML if doc_gen_format.is_template_html else DocGenSource.TEMPLATE_FILE)+ return sources+++def doc_gen_choice_key(*, doc_gen_format: DocGenFormat, source: DocGenSource) -> str:+ """The key the model deck lists a format and source's default engine under: 'pdf.layout'."""+ return f"{doc_gen_format}.{source}"+++def parse_doc_gen_choice_key(key: str) -> tuple[DocGenFormat, DocGenSource]:+ """The format and source a key of the model deck's doc-gen defaults names.++ Raises:+ ValueError: the key is not '<format>.<source>', or no step composes that format from that source.+ """+ format_token, separator, source_token = key.partition(".")+ try:+ doc_gen_format = DocGenFormat(format_token)+ source = DocGenSource(source_token)+ except ValueError as exc:+ msg = f"'{key}' is not a '<format>.<source>' key, such as 'pdf.layout'."+ raise ValueError(msg) from exc+ if not separator or source not in DocGenSource.possible_for(doc_gen_format=doc_gen_format):+ msg = f"'{key}' names no step: {doc_gen_format.with_article} is never composed {source.desc}."+ raise ValueError(msg)+ return doc_gen_format, sourcediff --git a/pipelex/cogt/doc_gen/doc_gen_setting.py b/pipelex/cogt/doc_gen/doc_gen_setting.pynew file mode 100644index 000000000..5a7991be8--- /dev/null+++ b/pipelex/cogt/doc_gen/doc_gen_setting.py@@ -0,0 +1,24 @@+from typing import Annotated, Union++from pydantic import BeforeValidator++from pipelex.cogt.models.model_reference import ModelReference, parse_model_reference+from pipelex.system.configuration.config_model import ConfigModel+++class DocGenSetting(ConfigModel):+ """The document engine a `PipeDocGen` step prints with: a model of the `doc_gen` family, such as `reportlab-pdf`."""++ model: str+ description: str | None = None++ def desc(self) -> str:+ return f"DocGenSetting(model={self.model})"+++# DocGenModelChoice accepts a DocGenSetting, a ModelReference, or a string, which is parsed into a ModelReference.+# ModelReference.model_serializer serializes it back to the string it was written as.+DocGenModelChoice = Union[+ DocGenSetting,+ Annotated[str | ModelReference, BeforeValidator(parse_model_reference)],+]diff --git a/pipelex/cogt/doc_gen/doc_gen_worker_abstract.py b/pipelex/cogt/doc_gen/doc_gen_worker_abstract.pynew file mode 100644index 000000000..b3ca64b6e--- /dev/null+++ b/pipelex/cogt/doc_gen/doc_gen_worker_abstract.py@@ -0,0 +1,50 @@+from abc import abstractmethod++from typing_extensions import override++from pipelex.cogt.doc_gen.render_job import RenderedDocument, RenderJob, RenderResources+from pipelex.cogt.doc_gen.template_check import TemplateCheckRequest, TemplateFinding+from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.reporting.reporting_protocol import ReportingProtocol+++class DocGenWorkerAbstract(InferenceWorkerAbstract):+ """A document engine: the worker of a `doc_gen` model, which prints a render job, synchronously.++ It is part of the plugin contract. An engine registers a factory for its worker with+ `registrar.add_inference_backend(family=InferenceFamily.DOC_GEN, sdk=…)`, and the kit's `internal.toml`+ declares its model, with the sources it prints from as `inputs` and its format as `outputs`. Open Pipelex+ registers `reportlab`; the Pipelex document generation plugin registers the rest from outside this repository.++ A worker is made for each print, and `render` runs on a thread of the print pool, never on the event loop, so+ an engine loads its library once per process, in the module that holds its worker, rather than per worker.+ """++ def __init__(self, *, inference_model: InferenceModelSpec, reporting_delegate: ReportingProtocol | None = None):+ super().__init__(reporting_delegate=reporting_delegate)+ self.inference_model = inference_model++ @property+ @override+ def desc(self) -> str:+ return f"Document generation using {self.inference_model.desc}"++ @abstractmethod+ def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument:+ """Print the job, reading every file its document names through `resources`.++ Raises:+ DocGenRenderError: the engine cannot print this job, such as a table cell taller than a page.+ """++ def check_template(self, *, request: TemplateCheckRequest) -> list[TemplateFinding]:+ """Compare a template file with the inputs it is filled with, for the dry run and so `pipelex validate`.++ An office template names its fields in its own way, which only its engine can read, so an engine that fills+ template files overrides this. The default finds nothing: the built-in PDF engine takes no template, and+ Pipelex checks HTML templates itself. An error finding fails the step, and a warning is logged. It raises+ only for a fault of its own: a template it cannot read at all is an error finding.+ """+ _ = request+ return []diff --git a/pipelex/cogt/doc_gen/doc_gen_worker_factory.py b/pipelex/cogt/doc_gen/doc_gen_worker_factory.pynew file mode 100644index 000000000..fa41a02f4--- /dev/null+++ b/pipelex/cogt/doc_gen/doc_gen_worker_factory.py@@ -0,0 +1,30 @@+from typing import cast++from pipelex.cogt.doc_gen.doc_gen_worker_abstract import DocGenWorkerAbstract+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.plugins.inference_backend_registry import InferenceFamily+from pipelex.plugins.model_handle import ModelHandle+from pipelex.reporting.reporting_protocol import ReportingProtocol+from pipelex.runtime_hub import get_inference_backend_registry, get_models_manager, get_sdk_client_manager+++class DocGenWorkerFactory:+ @classmethod+ def make_doc_gen_worker(+ cls,+ *,+ inference_model: InferenceModelSpec,+ reporting_delegate: ReportingProtocol | None = None,+ ) -> DocGenWorkerAbstract:+ model_handle = ModelHandle.make_for_inference_model(inference_model=inference_model)+ backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)+ make_worker = get_inference_backend_registry().lookup(family=InferenceFamily.DOC_GEN, sdk=model_handle.sdk)+ worker = make_worker(+ inference_model=inference_model,+ backend=backend,+ sdk_clients=get_sdk_client_manager().sdk_client_registry,+ reporting_delegate=reporting_delegate,+ )+ # The DOC_GEN family registry only ever holds doc gen workers, which the (family, sdk) key guarantees; the+ # uniform MakeWorkerFn return type is widened to InferenceWorkerAbstract, so narrow it back here.+ return cast("DocGenWorkerAbstract", worker)diff --git a/pipelex/cogt/doc_gen/document_composition.py b/pipelex/cogt/doc_gen/document_composition.pynew file mode 100644index 000000000..348c2273c--- /dev/null+++ b/pipelex/cogt/doc_gen/document_composition.py@@ -0,0 +1,82 @@+from pathlib import Path+from typing import Any, Self++from pydantic import BaseModel, ConfigDict, Field, model_validator++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.layout_tree import LayoutDocument+from pipelex.cogt.doc_gen.render_job import RenderJob+from pipelex.tools.uri.uri_read_scope import UriReference+++class DocumentComposition(BaseModel):+ """What a `PipeDocGen` step's compose stage produced, for its print stage to hand an engine.++ The compose stage is pure, so it can run where the pipe runs, inside a workflow; this is what crosses to+ the print stage, which reads the template file and prints. It differs from the engine's `RenderJob` by+ naming the template file by its path rather than carrying its bytes: reading a file is the print stage's IO.+ """++ model_config = ConfigDict(extra="forbid")++ format: DocGenFormat = Field(strict=False)+ source: DocGenSource = Field(strict=False)+ filename: str+ title: str+ layout: LayoutDocument | None = None+ html: str | None = None+ template_path: str | None = Field(default=None, description="The template file's resolved path, for the print stage to read")+ template_name: str | None = Field(default=None, description="The template file as the method names it")+ data: dict[str, Any] = Field(default_factory=dict)++ def _carries_payload_of(self, *, source: DocGenSource) -> bool:+ match source:+ case DocGenSource.LAYOUT:+ return self.layout is not None+ case DocGenSource.HTML:+ return self.html is not None+ case DocGenSource.TEMPLATE_FILE:+ return self.template_path is not None++ @model_validator(mode="after")+ def validate_payload(self) -> Self:+ """Exactly the payload its source names, as its `RenderJob` will need, so a wrong composition fails here and not at print."""+ payloads = {source: self._carries_payload_of(source=source) for source in DocGenSource}+ if not payloads[self.source]:+ msg = f"A document composed {self.source.desc} carries its payload."+ raise ValueError(msg)+ extra_payloads = [str(source) for source, is_set in payloads.items() if is_set and source != self.source]+ if extra_payloads:+ msg = f"A document composed {self.source.desc} carries only its own payload, not the {', '.join(extra_payloads)} one."+ raise ValueError(msg)+ return self++ def referenced_uris(self) -> list[UriReference]:+ """The files the print stage will read for the document, which it authorizes against the run's read scope first.++ The images of the layout tree. Composed HTML may name files too, but only an engine laying it out finds+ them, and it reads each through `RenderResources`, which applies the same check.+ """+ if self.layout is None:+ return []+ return [+ UriReference(uri=image_url, position=f"image {index} of the document '{self.filename}'")+ for index, image_url in enumerate(self.layout.image_urls(), start=1)+ ]++ def make_render_job(self) -> RenderJob:+ """The engine's job: this composition with the template file's bytes read in, when there is one."""+ template: bytes | None = None+ if self.template_path is not None:+ template = Path(self.template_path).read_bytes()+ return RenderJob(+ format=self.format,+ source=self.source,+ filename=self.filename,+ title=self.title,+ layout=self.layout,+ html=self.html,+ template=template,+ template_name=self.template_name,+ data=self.data,+ )diff --git a/pipelex/cogt/doc_gen/exceptions.py b/pipelex/cogt/doc_gen/exceptions.pynew file mode 100644index 000000000..4c6e18448--- /dev/null+++ b/pipelex/cogt/doc_gen/exceptions.py@@ -0,0 +1,130 @@+from enum import StrEnum++from pipelex.base_exceptions import ErrorDomain, PipelexError+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource, doc_gen_choice_key++# The package whose engines print what open Pipelex does not: a PDF from a template, Excel, Word and PowerPoint.+DOC_GEN_PLUGIN_PACKAGE = "pipelex-doc-gen"+# The sdk of the engine built into open Pipelex, `reportlab-pdf`; every other doc gen sdk is the plugin's.+BUILT_IN_DOC_GEN_SDK = "reportlab"+# The engine built into open Pipelex, declared in the kit's `internal.toml`, which `pipelex update` refreshes.+BUILT_IN_DOC_GEN_MODEL = "reportlab-pdf"+# The one format and source the built-in engine prints, keyed as the model deck keys its defaults: the kit's deck+# declares its default, and the plugin declares every other one.+BUILT_IN_DOC_GEN_CHOICE_KEY = doc_gen_choice_key(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.LAYOUT)+# The engines the plugin declares when it loads, by the model names a step may use. Core knows them only to say+# where a missing one comes from.+DOC_GEN_PLUGIN_MODEL_NAMES: frozenset[str] = frozenset({"pipelex-pdf", "pipelex-xlsx", "pipelex-docx", "pipelex-pptx"})+# Every engine name whose absence is refused naming its remedy, rather than as an unknown model.+KNOWN_DOC_GEN_MODEL_NAMES: frozenset[str] = DOC_GEN_PLUGIN_MODEL_NAMES | {BUILT_IN_DOC_GEN_MODEL}+++class DocGenEngineGap(StrEnum):+ """What is missing when a `PipeDocGen` step's engine is not available, which decides the remedy its refusal names."""++ NO_DEFAULT = "no_default"+ """The step names no engine, and the model deck names no default for its format and source."""++ NOT_DECLARED = "not_declared"+ """The step's engine is the built-in one or one of the plugin's, and this installation does not declare its model."""++ NOT_REGISTERED = "not_registered"+ """The engine's model is declared, and no installed plugin registers a worker for its sdk."""+++def _step_label(*, pipe_code: str | None) -> str:+ return f"PipeDocGen '{pipe_code}'" if pipe_code else "A PipeDocGen step"+++class DocGenEngineMissingError(PipelexError):+ """The document engine a `PipeDocGen` step prints with is not available in this runtime.++ `gap` says what is missing, and the message names its remedy. The deck may name no engine for the step's format+ and source, which the kit's deck does only for a `pdf` from the auto-layout: every other default comes with the+ Pipelex document generation plugin. The engine's model may not be declared here: open Pipelex declares+ `reportlab-pdf` in the kit's `internal.toml`, which `pipelex update` refreshes, and the plugin declares its own+ engines when it loads. Or the model may be declared with no worker registered for it: open Pipelex registers+ `reportlab-pdf`'s, and the plugin registers the rest. Raised when the method loads, before a run spends+ anything, and again by the print stage should the engine have gone since. A runtime that disables the built-in+ plugin and installs no other refuses every `PipeDocGen` step this way. Its message names only the step, the+ format, the source, the engine and the plugin, so it is kept verbatim for the caller.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = True++ def __init__(+ self,+ *,+ gap: DocGenEngineGap,+ doc_gen_format: DocGenFormat,+ source: DocGenSource,+ model: str | None,+ sdk: str | None,+ pipe_code: str | None = None,+ ):+ self.gap = gap+ self.doc_gen_format = doc_gen_format+ self.source = source+ self.model = model+ self.sdk = sdk+ self.pipe_code = pipe_code+ asks = f"{_step_label(pipe_code=pipe_code)} asks for {doc_gen_format.with_article} {source.desc}"+ comes_with_the_plugin = f"It comes with the Pipelex document generation plugin, {DOC_GEN_PLUGIN_PACKAGE}."+ message: str+ match gap:+ case DocGenEngineGap.NO_DEFAULT:+ if doc_gen_choice_key(doc_gen_format=doc_gen_format, source=source) == BUILT_IN_DOC_GEN_CHOICE_KEY:+ message = (+ f"{asks}, and the model deck names no document engine for it. Name one with the step's 'model', "+ "or run `pipelex update` to get the kit's document generation deck."+ )+ else:+ message = f"{asks}, and no document engine for it is installed in this runtime. {comes_with_the_plugin}"+ case DocGenEngineGap.NOT_DECLARED:+ if model == BUILT_IN_DOC_GEN_MODEL:+ message = (+ f"{asks}, on the engine '{model}', which is built into Pipelex and not declared in this installation's "+ "inference backends. Run `pipelex update` to refresh backends/internal.toml from the kit."+ )+ else:+ message = f"{asks}, on the engine '{model}', which is not installed in this runtime. {comes_with_the_plugin}"+ case DocGenEngineGap.NOT_REGISTERED:+ if sdk == BUILT_IN_DOC_GEN_SDK:+ message = f"{asks}, on the engine '{model}', which is built into Pipelex and disabled in this runtime."+ else:+ message = f"{asks}, on the engine '{model}', which is not installed in this runtime. {comes_with_the_plugin}"+ super().__init__(message)+++class DocGenModelCapabilityError(PipelexError):+ """The document engine a `PipeDocGen` step names does not print the step's format from its source.++ A `doc_gen` model lists the sources it prints from as its `inputs` and its format as its `outputs`, and a step+ that names one outside them is refused when the method loads.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = True++ def __init__(self, *, doc_gen_format: DocGenFormat, source: DocGenSource, model: str, pipe_code: str | None = None):+ self.doc_gen_format = doc_gen_format+ self.source = source+ self.model = model+ self.pipe_code = pipe_code+ message = (+ f"{_step_label(pipe_code=pipe_code)} asks the engine '{model}' for {doc_gen_format.with_article} {source.desc}, which it does not print. "+ "Name an engine that does in the step's 'model', or leave 'model' out for the model deck's default."+ )+ super().__init__(message)+++class DocGenRenderError(PipelexError):+ """A document engine could not print a render job.++ Engines raise it for what they cannot lay out, such as a table cell taller than a page, and the print stage+ raises it for an engine failure it did not expect. Engines outside Pipelex raise it too: it is part of the+ render contract (`render_job.py`).+ """++ error_domain = ErrorDomain.RUNTIMEdiff --git a/pipelex/cogt/doc_gen/input_shape.py b/pipelex/cogt/doc_gen/input_shape.pynew file mode 100644index 000000000..18ad6058a--- /dev/null+++ b/pipelex/cogt/doc_gen/input_shape.py@@ -0,0 +1,190 @@+"""The shape of a step's inputs as plain data, which a template checker compares a template file with.++An office template is filled from plain data (`plain_data.py`), so what a checker needs to know is the shape+of that data: which fields each input has, which of them are lists, and what their items hold. It learns it+from an `InputShape` tree per input rather than from Pipelex's concepts and classes, which keeps the checker+on the plugin contract (`template_check.py`).+"""++import datetime+import types+import typing+from collections.abc import Iterable, Sequence+from decimal import Decimal+from enum import Enum, StrEnum+from typing import Any, Literal, Union++from pydantic import BaseModel, Field++from pipelex.cogt.doc_gen.plain_data import plain_data+from pipelex.core.stuffs.composite_content import CompositeContent+from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.time_content import TimeContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+++class InputShapeKind(StrEnum):+ TEXT = "text"+ MARKDOWN = "markdown"+ HTML = "html"+ NUMBER = "number"+ BOOLEAN = "boolean"+ DATE = "date"+ DATETIME = "datetime"+ TIME = "time"+ IMAGE = "image"+ DOCUMENT = "document"+ STRUCTURE = "structure"+ LIST = "list"+ ANY = "any"+++class InputShape(BaseModel):+ """One value's shape as plain data.++ A structure lists its `fields`, which an image and a document also do (their URL among them, which an+ engine reads through `RenderResources`); a list has its `item`; anything else is a leaf. `ANY` is a value+ whose shape is not declared, such as a JSON input, so a checker cannot tell what it holds. A native `Date`+ is `DATE`, and its plain data is a `datetime.date`, or a `datetime.datetime`, which is one too, when it+ carries a time of day.+ """++ kind: InputShapeKind = Field(strict=False)+ title: str | None = None+ description: str | None = None+ fields: dict[str, "InputShape"] = Field(default_factory=dict)+ item: "InputShape | None" = None+++InputShape.model_rebuild()++_NONE_TYPE = type(None)+++def _leaf(kind: InputShapeKind, *, title: str | None = None, description: str | None = None) -> InputShape:+ return InputShape(kind=kind, title=title, description=description)+++def _content_leaf_kind(content_class: type[BaseModel]) -> InputShapeKind | None:+ """The leaf kind of a native content class, whose plain data is a single value; None for any other class."""+ if issubclass(content_class, MarkdownContent):+ return InputShapeKind.MARKDOWN+ if issubclass(content_class, TextContent):+ return InputShapeKind.TEXT+ if issubclass(content_class, HtmlContent):+ return InputShapeKind.HTML+ if issubclass(content_class, NumberContent):+ return InputShapeKind.NUMBER+ if issubclass(content_class, YesNoContent):+ return InputShapeKind.BOOLEAN+ if issubclass(content_class, DateContent):+ return InputShapeKind.DATE+ if issubclass(content_class, TimeContent):+ return InputShapeKind.TIME+ if issubclass(content_class, (JSONContent, ListContent, CompositeContent)):+ return InputShapeKind.ANY+ return None+++def shape_of_class(content_class: type[BaseModel], *, title: str | None = None, description: str | None = None) -> InputShape:+ """The plain-data shape of a content class, or of any pydantic class nested in one."""+ return _shape_of_class(content_class=content_class, title=title, description=description, seen=frozenset())+++def _shape_of_class(*, content_class: type[BaseModel], title: str | None, description: str | None, seen: frozenset[type]) -> InputShape:+ leaf_kind = _content_leaf_kind(content_class)+ if leaf_kind is not None:+ return _leaf(leaf_kind, title=title, description=description)+ if content_class in seen:+ # A recursive structure: the level that repeats is not described again.+ return _leaf(InputShapeKind.ANY, title=title, description=description)+ kind = InputShapeKind.STRUCTURE+ if issubclass(content_class, ImageContent):+ kind = InputShapeKind.IMAGE+ elif issubclass(content_class, DocumentContent):+ kind = InputShapeKind.DOCUMENT+ fields: dict[str, InputShape] = {}+ for field_name, field_info in content_class.model_fields.items():+ fields[field_name] = _shape_of_annotation(+ annotation=field_info.annotation,+ title=field_info.title,+ description=field_info.description,+ seen=seen | {content_class},+ )+ return InputShape(kind=kind, title=title, description=description, fields=fields)+++def _shape_of_annotation(*, annotation: Any, title: str | None, description: str | None, seen: frozenset[type]) -> InputShape:+ origin = typing.get_origin(annotation)+ if origin in {Union, types.UnionType}:+ members = [member for member in typing.get_args(annotation) if member is not _NONE_TYPE]+ if len(members) == 1:+ return _shape_of_annotation(annotation=members[0], title=title, description=description, seen=seen)+ return _leaf(InputShapeKind.ANY, title=title, description=description)+ if origin is typing.Annotated:+ return _shape_of_annotation(annotation=typing.get_args(annotation)[0], title=title, description=description, seen=seen)+ if origin is Literal:+ return _leaf(_kind_of_values(values=typing.get_args(annotation)), title=title, description=description)+ if origin in {list, tuple, set, frozenset, Sequence}:+ item_args = typing.get_args(annotation)+ item = _shape_of_annotation(annotation=item_args[0], title=None, description=None, seen=seen) if item_args else _leaf(InputShapeKind.ANY)+ return InputShape(kind=InputShapeKind.LIST, title=title, description=description, item=item)+ if origin is not None or not isinstance(annotation, type):+ return _leaf(InputShapeKind.ANY, title=title, description=description)+ if issubclass(annotation, BaseModel):+ return _shape_of_class(content_class=annotation, title=title, description=description, seen=seen)+ scalar_type: type = annotation # pyright: ignore[reportUnknownVariableType]+ return _leaf(_scalar_kind(scalar_type), title=title, description=description)+++def _kind_of_values(*, values: Iterable[Any]) -> InputShapeKind:+ """The kind every one of these values has as plain data, an enum member being its value; `ANY` when they differ.++ A `None` among them is left out, as an optional field's is.+ """+ kinds: set[InputShapeKind] = set()+ for value in values:+ if value is not None:+ plain_value: object = plain_data(value)+ kinds.add(_scalar_kind(type(plain_value)))+ if len(kinds) == 1:+ return kinds.pop()+ return InputShapeKind.ANY+++def _scalar_kind(annotation: type) -> InputShapeKind:+ if issubclass(annotation, Enum):+ enum_class: type[Enum] = annotation+ return _kind_of_values(values=[member.value for member in enum_class])+ if issubclass(annotation, bool):+ return InputShapeKind.BOOLEAN+ if issubclass(annotation, (int, float, Decimal)):+ return InputShapeKind.NUMBER+ if issubclass(annotation, str):+ return InputShapeKind.TEXT+ if issubclass(annotation, datetime.datetime):+ return InputShapeKind.DATETIME+ if issubclass(annotation, datetime.date):+ return InputShapeKind.DATE+ if issubclass(annotation, datetime.time):+ return InputShapeKind.TIME+ return InputShapeKind.ANY+++def shape_of_input(*, content_class: type[BaseModel] | None, is_list: bool) -> InputShape:+ """The plain-data shape of one declared input: its content class's, or a list of it for a list input.++ An input whose concept declares no content class, `Anything`, has a shape that is not declared.+ """+ shape = shape_of_class(content_class) if content_class is not None else _leaf(InputShapeKind.ANY)+ if is_list:+ return InputShape(kind=InputShapeKind.LIST, item=shape)+ return shapediff --git a/pipelex/cogt/doc_gen/layout_builder.py b/pipelex/cogt/doc_gen/layout_builder.pynew file mode 100644index 000000000..c9a7c5044--- /dev/null+++ b/pipelex/cogt/doc_gen/layout_builder.py@@ -0,0 +1,327 @@+"""Build the layout tree of a step's inputs, the auto-layout every engine writes.++What counts as a table is decided by the CSV codec's own flatness gate, `flat_field_names`, so "a list+that makes a table" means the same thing everywhere in Pipelex. The builder deliberately does not reuse+`StructuredContent.rendered_html`, which renders a list of structures as a list of small two-column+tables, so an invoice's line items would never come out as one table.+"""++import datetime+from decimal import Decimal+from enum import Enum+from html.parser import HTMLParser+from typing import Any++from pydantic import BaseModel+from typing_extensions import override++from pipelex.cogt.doc_gen.layout_tree import (+ FieldGridBlock,+ ImageBlock,+ LayoutBlock,+ LayoutColumn,+ LayoutDocument,+ LayoutField,+ LayoutScalar,+ MarkdownBlock,+ ParagraphsBlock,+ SectionBlock,+ TableBlock,+)+from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.time_content import TimeContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+from pipelex.tools.tabular.csv_codec import flat_field_names+from pipelex.tools.tabular.exceptions import CsvFlatnessError++# The tags after which the text of an HTML value starts a new paragraph when it is printed as text.+_HTML_BLOCK_TAGS = frozenset(+ {"p", "div", "br", "hr", "pre", "blockquote", "address", "details", "summary", "h1", "h2", "h3", "h4", "h5", "h6"}+ | {"ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "tr"}+ | {"section", "article", "header", "footer", "main", "nav", "aside", "figure", "figcaption"}+)+# The tags that start a table cell: a row stays one paragraph, its cells separated.+_HTML_CELL_TAGS = frozenset({"td", "th"})+_HTML_CELL_SEPARATOR = " | "+# The tags whose content is never text a reader sees.+_HTML_HIDDEN_TAGS = frozenset({"script", "style", "head", "title", "template"})+++class _HtmlTextExtractor(HTMLParser):+ """Collect an HTML value's visible text, one paragraph per block-level element."""++ def __init__(self) -> None:+ super().__init__(convert_charrefs=True)+ self._paragraphs: list[str] = []+ self._current: list[str] = []+ self._hidden_depth = 0++ def _break(self) -> None:+ paragraph = " ".join("".join(self._current).split())+ if paragraph:+ self._paragraphs.append(paragraph)+ self._current = []++ @override+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ if tag in _HTML_HIDDEN_TAGS:+ self._hidden_depth += 1+ elif tag in _HTML_BLOCK_TAGS:+ self._break()+ elif tag in _HTML_CELL_TAGS and "".join(self._current).strip():+ self._current.append(_HTML_CELL_SEPARATOR)++ @override+ def handle_endtag(self, tag: str) -> None:+ if tag in _HTML_HIDDEN_TAGS:+ self._hidden_depth = max(0, self._hidden_depth - 1)+ elif tag in _HTML_BLOCK_TAGS:+ self._break()++ @override+ def handle_data(self, data: str) -> None:+ if self._hidden_depth == 0:+ self._current.append(data)++ def paragraphs(self) -> list[str]:+ self._break()+ return self._paragraphs+++def html_to_paragraphs(html: str) -> list[str]:+ """The visible text of an HTML value, one paragraph per block-level element, with every tag removed.++ The built-in engine does not interpret HTML, so this is what an `Html` value prints as when a PDF is laid+ out without a template; a PDF that must keep the formatting uses an HTML template, which the plugin prints.+ """+ extractor = _HtmlTextExtractor()+ extractor.feed(html)+ extractor.close()+ return extractor.paragraphs()+++def humanize(name: str) -> str:+ """`unit_price` reads 'Unit price', `lineItems` stays readable as 'LineItems'."""+ words = name.replace("_", " ").strip()+ if not words:+ return name+ return words[0].upper() + words[1:]+++def _scalar(value: Any) -> tuple[bool, LayoutScalar]:+ """Whether a value is a scalar a field grid can hold, and the value as the grid holds it.++ A `Markdown` value is never a scalar: it is prose to format, so it gets a block of its own.+ """+ match value:+ case None | str() | bool() | int() | float() | datetime.datetime() | datetime.date() | datetime.time():+ return True, value+ case Decimal():+ return True, float(value)+ case Enum():+ return _scalar(value.value)+ case MarkdownContent():+ return False, None+ case TextContent():+ return True, value.text+ case NumberContent():+ return True, value.number+ case YesNoContent():+ return True, value.yes_no+ case DateContent():+ if value.time is None:+ return True, value.date+ return True, datetime.datetime.combine(value.date, value.time)+ case TimeContent():+ return True, value.time+ case _:+ return False, None+++def _collection_items(*, value: Any) -> list[Any] | None:+ """The items of a list, a tuple or a set, a set's in a stable order; None for any other value."""+ match value:+ case list() | tuple():+ return list(value) # pyright: ignore[reportUnknownArgumentType]+ case set() | frozenset():+ return sorted(value, key=str) # pyright: ignore[reportUnknownArgumentType]+ case _:+ return None+++def _is_list_of_scalars(value: Any) -> bool:+ items = _collection_items(value=value)+ if items is None:+ return False+ return all(_scalar(item)[0] for item in items)+++def _joined_scalars(items: list[Any]) -> str:+ return ", ".join("" if (scalar := _scalar(item)[1]) is None else str(scalar) for item in items)+++def _flat_columns(items: list[Any]) -> list[LayoutColumn] | None:+ """The table columns for a list of structures, or None when the list does not make a table.++ Only a structure a method declares makes a table. A native content such as an image or a Markdown text+ has flat fields too, but they are how it is stored rather than what it shows, so a list of them lays+ out item by item.+ """+ if not items:+ return None+ first_item: object = items[0]+ if not isinstance(first_item, StructuredContent):+ return None+ first_class = type(first_item)+ if not all(type(item) is first_class for item in items):+ return None+ try:+ field_names = flat_field_names(first_class)+ except CsvFlatnessError:+ return None+ columns: list[LayoutColumn] = []+ for field_name in field_names:+ field_info = first_class.model_fields[field_name]+ columns.append(LayoutColumn(key=field_name, label=field_info.title or humanize(field_name)))+ return columns+++def _table(*, title: str, path: str, items: list[Any], columns: list[LayoutColumn]) -> TableBlock:+ rows: list[dict[str, LayoutScalar]] = []+ for item in items:+ row: dict[str, LayoutScalar] = {}+ for column in columns:+ _, cell = _scalar(getattr(item, column.key))+ row[column.key] = cell+ rows.append(row)+ return TableBlock(title=title, path=path, columns=columns, rows=rows)+++def _paragraphs(text: str) -> ParagraphsBlock:+ paragraphs = [paragraph.strip() for paragraph in text.split("\n\n") if paragraph.strip()]+ return ParagraphsBlock(paragraphs=paragraphs)+++def _blocks_for_value(*, value: Any, title: str, path: str, level: int) -> list[LayoutBlock]:+ """The blocks that show one value, which is either a whole input or a field inside one."""+ is_scalar, scalar = _scalar(value)+ if is_scalar and not isinstance(value, TextContent):+ return [FieldGridBlock(fields=[LayoutField(label=title, value=scalar)])]+ match value:+ case MarkdownContent():+ return [SectionBlock(title=title, level=level, blocks=[MarkdownBlock(markdown=value.text)])]+ case TextContent():+ return [SectionBlock(title=title, level=level, blocks=[_paragraphs(value.text)])]+ case HtmlContent():+ return [SectionBlock(title=title, level=level, blocks=[ParagraphsBlock(paragraphs=html_to_paragraphs(value.inner_html))])]+ case ImageContent():+ return [ImageBlock(url=value.url, caption=value.caption or title)]+ case DocumentContent():+ label = value.title or value.filename or title+ return [FieldGridBlock(fields=[LayoutField(label=label, value=value.public_url or value.url)])]+ case JSONContent():+ return _blocks_for_value(value=value.json_obj, title=title, path=path, level=level)+ case ListContent():+ return _blocks_for_list(items=list(value.items), title=title, path=path, level=level) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType]+ case list() | tuple() | set() | frozenset():+ return _blocks_for_list(items=_collection_items(value=value) or [], title=title, path=path, level=level)+ case BaseModel():+ return [SectionBlock(title=title, level=level, blocks=_blocks_for_structure(structure=value, path=path, level=level + 1))]+ case dict():+ json_object: dict[Any, Any] = value # pyright: ignore[reportUnknownVariableType]+ named_values = [(str(key), humanize(str(key)), item) for key, item in json_object.items()]+ return [SectionBlock(title=title, level=level, blocks=_blocks_for_named_values(named_values=named_values, path=path, level=level + 1))]+ case _:+ return [FieldGridBlock(fields=[LayoutField(label=title, value=str(value))])]+++def _blocks_for_list(*, items: list[Any], title: str, path: str, level: int) -> list[LayoutBlock]:+ if _is_list_of_scalars(items):+ return [FieldGridBlock(fields=[LayoutField(label=title, value=_joined_scalars(items))])]+ columns = _flat_columns(items)+ if columns is not None:+ return [_table(title=title, path=path, items=items, columns=columns)]+ item_title = title[:-1] if title.endswith("s") and len(title) > 1 else title+ blocks: list[LayoutBlock] = []+ for index, item in enumerate(items, start=1):+ blocks.extend(_blocks_for_value(value=item, title=f"{item_title} {index}", path=f"{path}.{index - 1}", level=level + 1))+ return [SectionBlock(title=title, level=level, blocks=blocks)]+++def _named_fields(*, structure: BaseModel) -> list[tuple[str, str, Any]]:+ """A structure's fields as (name, label, value): the declared ones in order, then the public extra ones.++ The extra fields are where `CompositeContent` holds its parts. They are read from `model_extra` and never+ with getattr, so a part named like a pydantic attribute resolves to the part.+ """+ named_fields: list[tuple[str, str, Any]] = []+ for field_name, field_info in type(structure).model_fields.items():+ named_fields.append((field_name, field_info.title or humanize(field_name), getattr(structure, field_name)))+ for field_name, value in (structure.model_extra or {}).items():+ if not field_name.startswith("_"):+ named_fields.append((field_name, humanize(field_name), value))+ return named_fields+++def _blocks_for_structure(*, structure: BaseModel, path: str, level: int) -> list[LayoutBlock]:+ return _blocks_for_named_values(named_values=_named_fields(structure=structure), path=path, level=level)+++def _blocks_for_named_values(*, named_values: list[tuple[str, str, Any]], path: str, level: int) -> list[LayoutBlock]:+ """Named values, a structure's fields or a JSON object's, lay out as one grid of the scalar ones, then the others in order."""+ grid_fields: list[LayoutField] = []+ other_blocks: list[LayoutBlock] = []+ for name, label, value in named_values:+ is_scalar, scalar = _scalar(value)+ if is_scalar:+ grid_fields.append(LayoutField(label=label, value=scalar))+ elif _is_list_of_scalars(value):+ grid_fields.append(LayoutField(label=label, value=_joined_scalars(_collection_items(value=value) or [])))+ else:+ other_blocks.extend(_blocks_for_value(value=value, title=label, path=f"{path}.{name}", level=level))+ blocks: list[LayoutBlock] = []+ if grid_fields:+ blocks.append(FieldGridBlock(fields=grid_fields))+ blocks.extend(other_blocks)+ return blocks+++def _is_structure_input(content: StuffContent) -> bool:+ """Whether an input is a structure whose own fields can fill the document's top level."""+ if isinstance(content, (ListContent, TextContent, HtmlContent, ImageContent, DocumentContent, JSONContent)):+ return False+ is_scalar, _ = _scalar(content)+ return not is_scalar+++def build_layout_document(*, title: str, named_contents: list[tuple[str, StuffContent]]) -> LayoutDocument:+ """Lay out a step's inputs as one document.++ A single input is the document itself: a structure's fields fill the top level, and a text, such as a+ `Markdown` report, is the whole body, under the given title in both cases. Several inputs each get a+ section of their own, in the order the step declares them.+ """+ if len(named_contents) == 1:+ name, content = named_contents[0]+ if _is_structure_input(content):+ return LayoutDocument(title=title, blocks=_blocks_for_structure(structure=content, path=name, level=1))+ single_blocks = _blocks_for_value(value=content, title=humanize(name), path=name, level=1)+ match single_blocks:+ case [SectionBlock(blocks=[MarkdownBlock() | ParagraphsBlock() as body_block])]:+ return LayoutDocument(title=title, blocks=[body_block])+ case _:+ return LayoutDocument(title=title, blocks=single_blocks)+ blocks: list[LayoutBlock] = []+ for name, content in named_contents:+ blocks.extend(_blocks_for_value(value=content, title=humanize(name), path=name, level=1))+ return LayoutDocument(title=title, blocks=blocks)diff --git a/pipelex/cogt/doc_gen/layout_tree.py b/pipelex/cogt/doc_gen/layout_tree.pynew file mode 100644index 000000000..bdcf06ba8--- /dev/null+++ b/pipelex/cogt/doc_gen/layout_tree.py@@ -0,0 +1,102 @@+"""The layout tree: what the auto-layout of a step's inputs is, before any engine writes it.++A `PipeDocGen` step with no template lays its inputs out by itself, and every format's engine writes the+same tree: the built-in PDF engine on ReportLab, and the Excel and Word engines of the Pipelex document+generation plugin. The tree is plain data with a JSON round trip, and it is part of the engines' contract+(`render_job.py`). Every print today hands it to an engine in the same process; across a JSON boundary a+date, a time or a datetime comes back as its ISO text, since nothing in the wire format tags it. It shows:++- the scalar fields of a structure make a **field grid**, labelled from the field titles;+- a list of flat structures makes a **table**;+- a nested structure makes a **section**, with its own blocks;+- a `Text` makes **paragraphs**, a `Markdown` a **markdown** block for the engine to format, and an+ image an **image** block, which the engine reads through `RenderResources`.++`layout_builder.py` builds it from stuff contents.+"""++import datetime+from typing import Annotated, Literal++from pydantic import BaseModel, Field++LayoutScalar = str | bool | int | float | datetime.datetime | datetime.date | datetime.time | None+++class LayoutField(BaseModel):+ label: str+ value: LayoutScalar+++class LayoutColumn(BaseModel):+ key: str+ label: str+++class FieldGridBlock(BaseModel):+ kind: Literal["field_grid"] = "field_grid"+ fields: list[LayoutField]+++class TableBlock(BaseModel):+ kind: Literal["table"] = "table"+ title: str+ path: str = Field(description="The dotted path of the list this table shows, such as 'invoice.line_items'")+ columns: list[LayoutColumn]+ rows: list[dict[str, LayoutScalar]]+++class ParagraphsBlock(BaseModel):+ kind: Literal["paragraphs"] = "paragraphs"+ paragraphs: list[str]+++class MarkdownBlock(BaseModel):+ """Markdown source, which the engine formats: headings, emphasis, lists, tables, code and links."""++ kind: Literal["markdown"] = "markdown"+ markdown: str+++class ImageBlock(BaseModel):+ kind: Literal["image"] = "image"+ url: str+ caption: str | None = None+++class SectionBlock(BaseModel):+ kind: Literal["section"] = "section"+ title: str+ level: int+ blocks: list["LayoutBlock"]+++LayoutBlock = Annotated[+ FieldGridBlock | TableBlock | ParagraphsBlock | MarkdownBlock | ImageBlock | SectionBlock,+ Field(discriminator="kind"),+]+++class LayoutDocument(BaseModel):+ title: str+ blocks: list[LayoutBlock]++ def image_urls(self) -> list[str]:+ """The URL of every image block, in document order: what an engine will read to print the tree."""+ urls: list[str] = []+ _collect_image_urls(blocks=self.blocks, urls=urls)+ return urls+++def _collect_image_urls(*, blocks: list[LayoutBlock], urls: list[str]) -> None:+ for block in blocks:+ match block:+ case ImageBlock():+ urls.append(block.url)+ case SectionBlock():+ _collect_image_urls(blocks=block.blocks, urls=urls)+ case FieldGridBlock() | TableBlock() | ParagraphsBlock() | MarkdownBlock():+ pass+++SectionBlock.model_rebuild()diff --git a/pipelex/cogt/doc_gen/plain_data.py b/pipelex/cogt/doc_gen/plain_data.pynew file mode 100644index 000000000..2650469c3--- /dev/null+++ b/pipelex/cogt/doc_gen/plain_data.py@@ -0,0 +1,77 @@+"""Stuff contents as plain Python data, the shape an office template sees.++An office template is filled by its engine from plain data: a Word template reads `{{ invoice.number }}`+and an Excel template finds `invoice_number` by name, and neither knows what a stuff is. So a list stuff+becomes a list, a text stuff its text, a number its number, and a structure a dict of its fields.+Dates stay dates, so a spreadsheet cell keeps its type. That holds for an engine printing in the same process;+once serialized to JSON, a date is its ISO text.+"""++import datetime+from decimal import Decimal+from enum import Enum+from typing import Any++from pydantic import BaseModel++from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.core.stuffs.time_content import TimeContent+from pipelex.core.stuffs.yes_no_content import YesNoContent+++def plain_data(value: Any) -> Any:+ """Turn a stuff content, or any value found inside one, into plain Python data."""+ match value:+ case ListContent():+ return [plain_data(item) for item in value.items] # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]+ case TextContent():+ return value.text+ case HtmlContent():+ return value.inner_html+ case NumberContent():+ return value.number+ case YesNoContent():+ return value.yes_no+ case DateContent():+ if value.time is None:+ return value.date+ return datetime.datetime.combine(value.date, value.time)+ case TimeContent():+ return value.time+ case JSONContent():+ return value.json_obj+ case BaseModel():+ return _structure_plain_data(structure=value)+ case list() | tuple():+ return [plain_data(item) for item in value] # pyright: ignore[reportUnknownVariableType]+ case set() | frozenset():+ # A set has no order of its own, so its items are sorted for a document that reads the same on every run.+ return sorted((plain_data(item) for item in value), key=str) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]+ case dict():+ return {str(key): plain_data(item) for key, item in value.items()} # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]+ case Enum():+ return plain_data(value.value)+ case Decimal():+ return float(value)+ case _:+ return value+++def _structure_plain_data(*, structure: BaseModel) -> dict[str, Any]:+ """A structure's declared fields, then its public extra fields, which is where `CompositeContent` holds its parts.++ The extra fields are read from `model_extra` and never with getattr, so a part named like a pydantic attribute+ resolves to the part.+ """+ fields: dict[str, Any] = {}+ for field_name in type(structure).model_fields:+ fields[field_name] = plain_data(getattr(structure, field_name))+ for field_name, value in (structure.model_extra or {}).items():+ if not field_name.startswith("_"):+ fields[field_name] = plain_data(value)+ return fieldsdiff --git a/pipelex/cogt/doc_gen/render_job.py b/pipelex/cogt/doc_gen/render_job.pynew file mode 100644index 000000000..dd20d60d6--- /dev/null+++ b/pipelex/cogt/doc_gen/render_job.py@@ -0,0 +1,96 @@+"""The contract between Pipelex and a document engine: a render job in, the file's bytes out.++A `PipeDocGen` step runs in two stages. The compose stage renders the templates and builds the layout tree,+and it is pure. The print stage hands its result to an engine, as a `RenderJob`, and stores the bytes the+engine returns. An engine is the worker of a `doc_gen` model (`DocGenWorkerAbstract`), registered through+the plugin registrar's `add_inference_backend`, and the Pipelex document generation plugin registers its own+from outside this repository, so this module and the worker are that plugin's contract with Pipelex:+everything an engine needs arrives as plain data in the job, and the one thing it reads from outside the job,+a file the document names (an image), it reads through the `RenderResources` it is handed, which applies the+run's read scope. The job never carries a Pipelex object, so an ordinary Pipelex release does not break an+engine; changing this module is a change of the plugin contract, versioned by `PLUGIN_API_VERSION`.+"""++from typing import Any, Protocol, Self++from pydantic import BaseModel, ConfigDict, Field, model_validator++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.layout_tree import LayoutDocument+++class RenderJob(BaseModel):+ """Everything an engine needs to print one document, as plain data.++ Exactly one payload is set, the one the source names: `layout` for `DocGenSource.LAYOUT`, `html` for+ `DocGenSource.HTML`, and `template` with `data` for `DocGenSource.TEMPLATE_FILE`.+ """++ model_config = ConfigDict(extra="forbid", ser_json_bytes="base64", val_json_bytes="base64")++ format: DocGenFormat = Field(strict=False)+ source: DocGenSource = Field(strict=False)+ filename: str = Field(description="The file's name, suffix included, such as 'invoice-INV-2026-0142.pdf'")+ title: str = Field(description="The document's title, for a running header and the file's metadata")+ layout: LayoutDocument | None = Field(default=None, description="The auto-layout of the step's inputs")+ html: str | None = Field(default=None, description="The step's HTML template, rendered against its inputs")+ template: bytes | None = Field(default=None, description="The template file's bytes")+ template_name: str | None = Field(default=None, description="The template file as the method names it, for messages")+ data: dict[str, Any] = Field(+ default_factory=dict,+ description="The step's inputs as plain data, by input name: what a template file is filled with",+ )++ def _carries_payload_of(self, *, source: DocGenSource) -> bool:+ match source:+ case DocGenSource.LAYOUT:+ return self.layout is not None+ case DocGenSource.HTML:+ return self.html is not None+ case DocGenSource.TEMPLATE_FILE:+ return self.template is not None++ @model_validator(mode="after")+ def validate_payload(self) -> Self:+ payloads = {source: self._carries_payload_of(source=source) for source in DocGenSource}+ if not payloads[self.source]:+ msg = f"A render job {self.source.desc} carries its payload: the '{self.source}' one is missing."+ raise ValueError(msg)+ extra_payloads = [str(source) for source, is_set in payloads.items() if is_set and source != self.source]+ if extra_payloads:+ msg = f"A render job {self.source.desc} carries only its own payload, not the {', '.join(extra_payloads)} one."+ raise ValueError(msg)+ return self+++class RenderedDocument(BaseModel):+ """What an engine returns: the file's bytes.++ The MIME type and the suffix are the format's (`DocGenFormat`), which the step's filename already ends+ in, so an engine does not restate them. A model rather than bare bytes, so the contract can grow an+ optional field without breaking an engine.+ """++ data: bytes+++class RenderResources(Protocol):+ """How an engine reads a file its document names, such as the image of an image block.++ Pipelex hands one to every render. It resolves `pipelex-storage://` keys through the run's storage+ provider, decodes `data:` URLs and fetches `https://` through the SSRF guard, and it refuses what the+ run's read scope does not allow, a local path included. It is synchronous, since engines are, and it+ may be called from the thread the engine prints in.+ """++ def load(self, *, uri: str, position: str) -> bytes:+ """Return the bytes a URI points at.++ Args:+ uri: The URI the document names.+ position: Where the URI sits, in words, which a refusal names instead of the URI: "image 2 of the document".++ Raises:+ UriReadRefusedError: the run's read scope does not allow the URI.+ """+ ...diff --git a/pipelex/cogt/doc_gen/template_check.py b/pipelex/cogt/doc_gen/template_check.pynew file mode 100644index 000000000..8de1efd10--- /dev/null+++ b/pipelex/cogt/doc_gen/template_check.py@@ -0,0 +1,57 @@+"""What an engine's template checker takes and returns, part of the plugin contract.++An office template names the fields it is filled with in its own way: an Excel workbook through its defined+names and Tables, a Word document through its tags, a PowerPoint deck through its shape names. Pipelex cannot+read those, so an engine that fills a template file overrides its worker's `check_template`+(`DocGenWorkerAbstract`). The dry run, and so `pipelex validate`, calls it for every `PipeDocGen` step with a+`template_file`, before anything is spent: an error finding fails the step, and a warning is logged. The+built-in PDF engine takes no template, and Pipelex checks HTML templates itself, so neither needs one.+"""++from enum import StrEnum+from typing import Any++from pydantic import BaseModel, ConfigDict, Field++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat+from pipelex.cogt.doc_gen.input_shape import InputShape+++class TemplateFindingSeverity(StrEnum):+ WARNING = "warning"+ ERROR = "error"++ @property+ def is_error(self) -> bool:+ """Whether a finding of this severity fails the step, rather than being logged."""+ match self:+ case TemplateFindingSeverity.ERROR:+ return True+ case TemplateFindingSeverity.WARNING:+ return False+++class TemplateFinding(BaseModel):+ """One thing a checker found in a template file."""++ severity: TemplateFindingSeverity = Field(strict=False)+ message: str = Field(description="What is wrong, in the template author's words: the name, the field, the fix")+ location: str | None = Field(default=None, description="Where it is in the file, such as 'the Table LineItems on sheet Invoice'")+++class TemplateCheckRequest(BaseModel):+ """A template file and the inputs it is filled with, for a checker to compare."""++ model_config = ConfigDict(extra="forbid", ser_json_bytes="base64", val_json_bytes="base64")++ format: DocGenFormat = Field(strict=False)+ template: bytes = Field(description="The template file's bytes")+ template_name: str = Field(description="The template file as the method names it, for messages")+ inputs: dict[str, InputShape] = Field(description="The step's declared inputs by name, as the shape of their plain data")+ data: dict[str, Any] | None = Field(+ default=None,+ description=(+ "Mock inputs as plain data, shaped as `RenderJob.data` is, so the checker can fill the template in memory and report "+ "what the fill would raise; None when only the structure is to be compared"+ ),+ )diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 468e2ec44..572a5d98d 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -349,6 +349,15 @@ class SearchHandleNotFoundError(CogtError):super().__init__(message)+class DocGenHandleNotFoundError(CogtError):+ error_category = InferenceErrorCategory.CONFIGURATION++ def __init__(self, message: str, preset_id: str, model_handle: str):+ self.preset_id = preset_id+ self.model_handle = model_handle+ super().__init__(message)++class ExtractOutputError(CogtError):pass@@ -501,6 +510,22 @@ class ModelManagerError(CogtError):pass+class PluginModelDeclarationError(CogtError):+ """A model a plugin declares, or a model deck default it sets, cannot be merged into this installation's inference configuration.++ The model manager validates the plugins' declarations when it merges them at boot, since a plugin's ``register``+ only stores them: a model whose name the installation's ``internal.toml`` already declares, a table that is not a+ valid model spec, or a default for a format and source no step composes. The message names the plugin, and the+ file when one is involved.+ """++ error_category = InferenceErrorCategory.CONFIGURATION++ def __init__(self, message: str, *, plugin: str):+ self.plugin = plugin+ super().__init__(message)++class ModelListingUnsupportedError(CogtError):"""A registered lister cannot enumerate models for an SDK variant at runtime.diff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.pyindex 753b0de71..514416d15 100644--- a/pipelex/cogt/model_backends/backend_library.py+++ b/pipelex/cogt/model_backends/backend_library.py@@ -13,12 +13,14 @@ from pipelex.cogt.exceptions import (InferenceBackendLibraryNotFoundError,InferenceBackendLibraryValidationError,InferenceModelSpecError,+ PluginModelDeclarationError,)-from pipelex.cogt.model_backends.backend import InferenceBackend, resolve_model_specs_section+from pipelex.cogt.model_backends.backend import InferenceBackend, PipelexBackend, resolve_model_specs_sectionfrom pipelex.cogt.model_backends.backend_factory import (InferenceBackendBlueprint,InferenceBackendFactory,)+from pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedConstraintfrom pipelex.cogt.model_backends.gateway_config import GatewayConfig, drop_unknown_gateway_defaultsfrom pipelex.cogt.model_backends.model_spec_document import MODEL_SPEC_DEFAULTS_TABLEfrom pipelex.cogt.model_backends.model_spec_factory import (@@ -28,6 +30,7 @@ from pipelex.cogt.model_backends.model_spec_factory import ()from pipelex.cogt.model_backends.model_spec_keys import ModelSpecSource, describe_rejected_keys, split_model_spec_keysfrom pipelex.migration.plan import MigrationPlan+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarationsfrom pipelex.system.configuration.config_loader import config_managerfrom pipelex.system.configuration.config_surface import (INFERENCE_BACKEND_CONFIG_SURFACE_ID,@@ -345,7 +348,8 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):backend_name=backend_name,backend_config_source=backend_config_source,model_spec_source=model_spec_source,- backend_blueprint=backend_blueprint,+ backend_listed_constraints=backend_blueprint.listed_constraints,+ backend_valued_constraints=backend_blueprint.valued_constraints,)except InferenceBackendLibraryError:recovered = self._local_model_specs_the_ledger_can_explain(@@ -397,13 +401,15 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):backend_name: str,backend_config_source: str,model_spec_source: ModelSpecSource,- backend_blueprint: InferenceBackendBlueprint,+ backend_listed_constraints: list[ListedConstraint],+ backend_valued_constraints: dict[ValuedConstraint, Any],) -> "dict[str, InferenceModelSpec]":"""Turn one backend's raw tables into model specs: pop `[defaults]`, split, merge, validate.Its own method so the boot-tolerance retry can run it a second time over a migrated document- without duplicating a line of it. The read is non-destructive — `[defaults]` is popped from a- copy — because the caller may still need the original tables when this raises.+ without duplicating a line of it, and so a plugin's internal models are built by the very path+ a backend file's are. The read is non-destructive — `[defaults]` is popped from a copy —+ because the caller may still need the original tables when this raises."""remaining_tables = dict(model_specs_dict)defaults_dict: dict[str, Any] = remaining_tables.pop(MODEL_SPEC_DEFAULTS_TABLE, {})@@ -420,7 +426,7 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):key_split = split_model_spec_keys(model_spec_dict=model_spec_dict)if key_split.rejected:match model_spec_source:- case ModelSpecSource.LOCAL_FILE:+ case ModelSpecSource.LOCAL_FILE | ModelSpecSource.PLUGIN:# Fatal in lenient mode too: leniency covers credentials only (see the docstring),# and this is not a credentials error, so the lenient `except` in `load` lets it# through. What may still catch it is the ledger, one level up.@@ -442,8 +448,8 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):backend_name=backend_name,name=model_spec_name,blueprint=model_spec_blueprint,- backend_listed_constraints=backend_blueprint.listed_constraints,- backend_valued_constraints=backend_blueprint.valued_constraints,+ backend_listed_constraints=backend_listed_constraints,+ backend_valued_constraints=backend_valued_constraints,extra_headers=key_split.headers,)backend_model_specs[model_spec_name] = model_spec@@ -485,7 +491,8 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):— but at boot there is nothing there to carry forward."""match model_spec_source:- case ModelSpecSource.REMOTE_GATEWAY:+ case ModelSpecSource.REMOTE_GATEWAY | ModelSpecSource.PLUGIN:+ # A plugin's table is never read from a file, and never retried here: it is built after the load.return Nonecase ModelSpecSource.LOCAL_FILE:pass@@ -506,7 +513,8 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):backend_name=backend_name,backend_config_source=backend_config_source,model_spec_source=model_spec_source,- backend_blueprint=backend_blueprint,+ backend_listed_constraints=backend_blueprint.listed_constraints,+ backend_valued_constraints=backend_blueprint.valued_constraints,)except (InferenceBackendLibraryError, InferenceModelSpecError, InferenceBackendCredentialsError):return None@@ -643,6 +651,63 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):key_name=unknown_var_prefix_exc.var_name,) from unknown_var_prefix_exc+ def merge_plugin_internal_models(self, *, plugin_model_declarations: PluginModelDeclarations, backends_dir_path: str) -> bool:+ """Merge the internal models the plugins declared into the internal backend, or report there is none to merge into.++ Each table is built by the path a backend file's tables take, as a `PLUGIN` source: a key the model spec+ does not know is the plugin author's mistake and fatal, as it is in a local file. The table is complete on+ its own, so the `[defaults]` of `internal.toml` is not applied to it.++ **A name the installation's `internal.toml` already declares is refused rather than overridden**, naming+ the plugin and the file: which of the two declarations a boot would run would otherwise depend on nothing+ the user can see. Model names are not global across backends — the same name in several backend files is+ the ordinary case, and the routing profile picks one — so a name another backend declares is left to the+ routing, exactly as it is for a model the file declares.++ Returns:+ Whether this load has an internal backend. `False` when the installation disables it, or declares none:+ the plugins' models are then not merged, as a disabled backend's own models are not loaded, and the+ caller leaves the plugins' model deck defaults out for the same reason.++ Raises:+ PluginModelDeclarationError: a plugin declares a model `internal.toml` declares too, or one whose table+ is not a valid model spec.+ """+ internal_backend = self.root.get(PipelexBackend.INTERNAL)+ if internal_backend is None:+ return False+ internal_file_path = backend_toml_path(backends_dir_path=backends_dir_path, backend_name=PipelexBackend.INTERNAL)+ merged_model_specs = dict(internal_backend.model_specs)+ for model_name, plugin_model in plugin_model_declarations.internal_models.items():+ if model_name == MODEL_SPEC_DEFAULTS_TABLE:+ msg = (+ f"Plugin '{plugin_model.plugin}' declares an internal model named '{model_name}', which is the name of a backend "+ "file's table of defaults, so it cannot name a model."+ )+ raise PluginModelDeclarationError(msg, plugin=plugin_model.plugin)+ if model_name in internal_backend.model_specs:+ msg = (+ f"Plugin '{plugin_model.plugin}' declares the internal model '{model_name}', which '{internal_file_path}' declares too. "+ "Remove it from that file, or run `pipelex update` to refresh the file from the kit: the plugin's declaration is the one "+ "that ships with its engine."+ )+ raise PluginModelDeclarationError(msg, plugin=plugin_model.plugin)+ try:+ plugin_model_specs = self._build_backend_model_specs(+ model_specs_dict={model_name: plugin_model.spec},+ backend_name=PipelexBackend.INTERNAL,+ backend_config_source=f"plugin '{plugin_model.plugin}'",+ model_spec_source=ModelSpecSource.PLUGIN,+ backend_listed_constraints=internal_backend.listed_constraints,+ backend_valued_constraints=internal_backend.valued_constraints,+ )+ except InferenceBackendLibraryError as exc:+ msg = f"Plugin '{plugin_model.plugin}' declares the internal model '{model_name}' with a table that is not a valid model spec: {exc}"+ raise PluginModelDeclarationError(msg, plugin=plugin_model.plugin) from exc+ merged_model_specs.update(plugin_model_specs)+ self.root[PipelexBackend.INTERNAL] = internal_backend.model_copy(update={"model_specs": merged_model_specs})+ return True+def list_backend_names(self) -> list[str]:return list(self.root.keys())diff --git a/pipelex/cogt/model_backends/model_spec_keys.py b/pipelex/cogt/model_backends/model_spec_keys.pyindex 888bfa818..31acbf35e 100644--- a/pipelex/cogt/model_backends/model_spec_keys.py+++ b/pipelex/cogt/model_backends/model_spec_keys.py@@ -26,8 +26,8 @@ not become "whatever this version of h11 does".This module is deliberately pure and deliberately silent. It runs on the success path of every backendload, including loads that precede `runtime_hub.set_config()`, so a `log` call here would turn a datatransform into a boot-order dependency. It also does not decide what a rejected key *means*: only the-caller knows whether the table came from a local file (where a rejected key is the author's typo, and-fatal) or from the served gateway config (where it is version skew, and pruned).+caller knows whether the table came from a local file or a plugin (where a rejected key is the author's+typo, and fatal) or from the served gateway config (where it is version skew, and pruned)."""import re@@ -49,6 +49,8 @@ class ModelSpecSource(StrEnum):LOCAL_FILE = "local_file"REMOTE_GATEWAY = "remote_gateway"+ PLUGIN = "plugin"+ """A table a plugin declared with `add_internal_model`, merged into the internal backend at boot."""class ModelSpecKeyRejection(StrEnum):diff --git a/pipelex/cogt/model_backends/model_type.py b/pipelex/cogt/model_backends/model_type.pyindex 299e42077..e7344fedc 100644--- a/pipelex/cogt/model_backends/model_type.py+++ b/pipelex/cogt/model_backends/model_type.py@@ -6,6 +6,7 @@ class ModelType(StrEnum):TEXT_EXTRACTOR = "text_extractor"IMG_GEN = "img_gen"SEARCH = "search"+ DOC_GEN = "doc_gen"@propertydef indefinite_description(self) -> str:@@ -19,3 +20,5 @@ class ModelType(StrEnum):return "an image-generation model"case ModelType.SEARCH:return "a search model"+ case ModelType.DOC_GEN:+ return "a document-generation engine"diff --git a/pipelex/cogt/models/deck_manifest.py b/pipelex/cogt/models/deck_manifest.pyindex af98d3d47..6ac93b02e 100644--- a/pipelex/cogt/models/deck_manifest.py+++ b/pipelex/cogt/models/deck_manifest.py@@ -1,15 +1,20 @@-"""Model deck manifest: detect when an installed deck has drifted from the kit-shipped templates.+"""Kit manifests: detect when the kit-managed files of an installation have drifted from the kit-shipped templates.-The manifest pins, for one specific deck install, the kit version that produced it and the SHA-256 of-each managed deck file at install/update time. It enables three independent signals:+``pipelex update`` manages two areas of an installed configuration, each a ``KitManagedArea`` with its own+manifest file in its own directory: the model deck, and the one backend file the kit owns, the internal+backend's ``internal.toml``. A manifest pins, for one install of an area, the kit version that produced it and+the SHA-256 of each managed file at install/update time. It enables three independent signals:- Manifest's ``kit_version`` vs the running ``pipelex`` version → behind upstream.-- Manifest's per-file hash vs the installed file's actual hash → user has locally edited a numbered file.+- Manifest's per-file hash vs the installed file's actual hash → user has locally edited a managed file.- Kit content vs installed content → upstream changed.-Numbered deck files (``<digits>_*.toml``, e.g. ``1_llm_deck.toml``) are pipelex-managed. Anything-else is left alone — including ``x_custom_*.toml`` overrides (the recommended escape hatch) and-any other project-local additions (e.g. a cookbook's preset file).+In the deck, numbered files (``<digits>_*.toml``, e.g. ``1_llm_deck.toml``) are pipelex-managed. Anything+else is left alone — including ``x_custom_*.toml`` overrides (the recommended escape hatch) and any other+project-local additions (e.g. a cookbook's preset file). In the backends directory only ``internal.toml`` is+managed, because it declares the software-only models open Pipelex ships and an existing install must receive+the ones a release adds: every other backend file is the user's, which ``pipelex update`` never touches+(``pipelex init``, a full reset, rewrites the ones the kit ships)."""from __future__ import annotations@@ -21,15 +26,41 @@ from pathlib import Pathfrom pydantic import BaseModel, ConfigDict, Field+from pipelex.cogt.model_backends.backend import PipelexBackendfrom pipelex.kit.paths import get_kit_configs_dirfrom pipelex.tools.misc.file_utils import path_existsfrom pipelex.tools.misc.package_utils import get_package_versionMANIFEST_FILENAME = ".kit_manifest.json"+# The one backend file ``pipelex update`` manages: the internal backend's, which declares the models open Pipelex ships.+MANAGED_BACKEND_FILENAME = f"{PipelexBackend.INTERNAL}.toml"+++class KitManagedArea(StrEnum):+ """A directory of an installed configuration whose kit-shipped files ``pipelex update`` manages, each with its own manifest."""++ DECK = "deck"+ BACKENDS = "backends"++ @property+ def display_name(self) -> str:+ match self:+ case KitManagedArea.DECK:+ return "Model deck"+ case KitManagedArea.BACKENDS:+ return "Internal backend"++ def is_managed_filename(self, *, filename: str) -> bool:+ """Whether ``pipelex update`` manages a file of this name in this area's directory."""+ match self:+ case KitManagedArea.DECK:+ return _is_managed_deck_filename(filename)+ case KitManagedArea.BACKENDS:+ return filename == MANAGED_BACKEND_FILENAMEclass DeckFileStatus(StrEnum):- """Per-file sync status between the installed deck and the kit-shipped templates."""+ """Per-file sync status between an installed kit-managed area and the kit-shipped templates."""UP_TO_DATE = "up_to_date"KIT_ADDED = "kit_added"@@ -57,7 +88,7 @@ class DeckManifest(BaseModel):class DeckSyncReport(BaseModel):- """Result of comparing an installed deck dir to the currently shipping kit."""+ """Result of comparing an installed kit-managed area's directory to the currently shipping kit."""model_config = ConfigDict(extra="forbid")@@ -95,6 +126,20 @@ def kit_deck_dir() -> Path:return Path(str(get_kit_configs_dir())) / "inference" / "deck"+def kit_backends_dir() -> Path:+ """Return the kit's shipped backends directory as a real filesystem path, the way ``kit_deck_dir`` does for the deck."""+ return Path(str(get_kit_configs_dir())) / "inference" / "backends"+++def kit_area_dir(*, area: KitManagedArea) -> Path:+ """The kit's shipped directory for an area."""+ match area:+ case KitManagedArea.DECK:+ return kit_deck_dir()+ case KitManagedArea.BACKENDS:+ return kit_backends_dir()++def _is_managed_deck_filename(filename: str) -> bool:"""True for files that follow the pipelex-managed numbered convention.@@ -108,38 +153,52 @@ def _is_managed_deck_filename(filename: str) -> bool:return bool(sep) and head.isdigit()-def list_managed_kit_files() -> dict[str, str]:- """Hash every managed deck file shipped in the current pipelex wheel."""- kit_dir = kit_deck_dir()+def list_managed_kit_files(*, area: KitManagedArea) -> dict[str, str]:+ """Hash every managed file of an area shipped in the current pipelex wheel."""+ kit_dir = kit_area_dir(area=area)return {- entry.name: compute_file_sha256(entry) for entry in sorted(kit_dir.iterdir()) if entry.is_file() and _is_managed_deck_filename(entry.name)+ entry.name: compute_file_sha256(entry)+ for entry in sorted(kit_dir.iterdir())+ if entry.is_file() and area.is_managed_filename(filename=entry.name)}-def list_managed_installed_files(deck_dir: Path) -> dict[str, str]:- """Hash every managed deck file present in the user's installed deck dir."""- if not deck_dir.is_dir():+def list_managed_installed_files(installed_dir: Path, *, area: KitManagedArea) -> dict[str, str]:+ """Hash every managed file of an area present in the user's installed directory for it."""+ if not installed_dir.is_dir():return {}return {- entry.name: compute_file_sha256(entry) for entry in sorted(deck_dir.iterdir()) if entry.is_file() and _is_managed_deck_filename(entry.name)+ entry.name: compute_file_sha256(entry)+ for entry in sorted(installed_dir.iterdir())+ if entry.is_file() and area.is_managed_filename(filename=entry.name)}-def compute_kit_manifest() -> DeckManifest:- """Build the manifest that should be written after a fresh install or successful update."""- return DeckManifest(kit_version=get_package_version(), files=list_managed_kit_files())+def compute_kit_manifest(*, area: KitManagedArea) -> DeckManifest:+ """Build the manifest that should be written for an area after a fresh install or successful update."""+ return DeckManifest(kit_version=get_package_version(), files=list_managed_kit_files(area=area))++def stamp_kit_manifests(*, inference_dir: Path) -> None:+ """Write both areas' manifests for an ``inference/`` directory whose managed files were just copied from the kit.++ For the installers (``pipelex init``, the first-run materialization), so a later ``pipelex update`` tells a file+ the user edited from one the kit moved on.+ """+ for area in KitManagedArea:+ write_manifest(compute_kit_manifest(area=area), installed_dir=inference_dir / area)-def manifest_path(deck_dir: Path) -> Path:- return deck_dir / MANIFEST_FILENAME+def manifest_path(installed_dir: Path) -> Path:+ return installed_dir / MANIFEST_FILENAME-def read_manifest(deck_dir: Path) -> DeckManifest | None:++def read_manifest(installed_dir: Path) -> DeckManifest | None:"""Return the persisted manifest, or ``None`` when absent or unreadable.A corrupt manifest is treated as missing — the caller will warn the user and offer to rebuild it."""- target = manifest_path(deck_dir)+ target = manifest_path(installed_dir)if not path_exists(str(target)):return Nonetry:@@ -152,11 +211,11 @@ def read_manifest(deck_dir: Path) -> DeckManifest | None:return None-def write_manifest(manifest: DeckManifest, *, deck_dir: Path) -> None:- """Persist the manifest, creating the deck directory if needed."""- deck_dir.mkdir(parents=True, exist_ok=True)+def write_manifest(manifest: DeckManifest, *, installed_dir: Path) -> None:+ """Persist the manifest, creating the area's directory if needed."""+ installed_dir.mkdir(parents=True, exist_ok=True)payload = manifest.model_dump()- target = manifest_path(deck_dir)+ target = manifest_path(installed_dir)target.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")@@ -230,15 +289,15 @@ def suggest_x_custom_filename(numbered_filename: str) -> str:return f"x_custom_{tail}"-def compute_deck_sync_report(deck_dir: Path) -> DeckSyncReport:- """Full per-file diff between the installed deck and the running pipelex's kit.+def compute_sync_report(installed_dir: Path, *, area: KitManagedArea) -> DeckSyncReport:+ """Full per-file diff between an installed area and the running pipelex's kit.This walks every managed file in either side and assigns it a ``DeckFileStatus``. Cost is oneSHA-256 per file — only call from ``pipelex update`` and ``pipelex doctor``, never on the boot path."""- kit_files = list_managed_kit_files()- installed_files = list_managed_installed_files(deck_dir)- manifest = read_manifest(deck_dir)+ kit_files = list_managed_kit_files(area=area)+ installed_files = list_managed_installed_files(installed_dir, area=area)+ manifest = read_manifest(installed_dir)manifest_files: dict[str, str] = manifest.files if manifest is not None else {}all_filenames = set(kit_files) | set(installed_files)diff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.pyindex 9da4fe308..931f46d65 100644--- a/pipelex/cogt/models/model_deck.py+++ b/pipelex/cogt/models/model_deck.py@@ -4,7 +4,10 @@ from pydantic import Field, PrivateAttr, field_validator, model_validatorfrom pipelex import logfrom pipelex.cogt.config_cogt import ModelDeckConfig+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource, doc_gen_choice_key, parse_doc_gen_choice_key+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice, DocGenSettingfrom pipelex.cogt.exceptions import (+ DocGenHandleNotFoundError,ExtractHandleNotFoundError,ImgGenHandleNotFoundError,LLMHandleNotFoundError,@@ -73,11 +76,30 @@ class SearchDeckBlueprint(ConfigModel):choice_default: SearchModelChoice+class DocGenDeckBlueprint(ConfigModel):+ """The document engines' deck: one default engine per format and source, keyed '<format>.<source>' ('pdf.layout')."""++ aliases: dict[str, str] = Field(default_factory=dict)+ waterfalls: dict[str, list[str]] = Field(default_factory=dict)+ presets: dict[str, DocGenSetting] = Field(default_factory=dict)+ choice_defaults: dict[str, DocGenModelChoice] = Field(default_factory=dict)++ @field_validator("choice_defaults", mode="after")+ @classmethod+ def validate_choice_default_keys(cls, choice_defaults: dict[str, DocGenModelChoice]) -> dict[str, DocGenModelChoice]:+ for key in choice_defaults:+ parse_doc_gen_choice_key(key)+ return choice_defaults++class ModelDeckBlueprint(ConfigModel):llm: LLMDeckBlueprintextract: ExtractDeckBlueprintimg_gen: ImgGenDeckBlueprintsearch: SearchDeckBlueprint+ # Optional, unlike the other families: a project whose deck predates it still boots, and only its+ # `PipeDocGen` steps are refused, naming the missing default, until `pipelex update` adds the deck file.+ doc_gen: DocGenDeckBlueprint = Field(default_factory=DocGenDeckBlueprint)class ModelDeck(ConfigModel):@@ -117,6 +139,12 @@ class ModelDeck(ConfigModel):search_presets: dict[str, SearchSetting] = Field(default_factory=dict)search_choice_default: SearchModelChoice+ # DocGen-specific+ doc_gen_aliases: dict[str, str] = Field(default_factory=dict)+ doc_gen_waterfalls: dict[str, list[str]] = Field(default_factory=dict)+ doc_gen_presets: dict[str, DocGenSetting] = Field(default_factory=dict)+ doc_gen_choice_defaults: dict[str, DocGenModelChoice] = Field(default_factory=dict)+def get_aliases_and_waterfalls_for_type(self, model_type: ModelType) -> tuple[dict[str, str], dict[str, list[str]]]:"""Return the type-specific aliases and waterfalls dictionaries."""match model_type:@@ -128,6 +156,8 @@ class ModelDeck(ConfigModel):return self.img_gen_aliases, self.img_gen_waterfallscase ModelType.SEARCH:return self.search_aliases, self.search_waterfalls+ case ModelType.DOC_GEN:+ return self.doc_gen_aliases, self.doc_gen_waterfallsdef is_model_handle_defined(self, model_handle: str, *, model_type: ModelType) -> bool:"""Check if a model handle is defined in the model deck.@@ -213,12 +243,26 @@ class ModelDeck(ConfigModel):f"Bare string '{name}' matches: {', '.join(matches)}. Using it as a direct model handle. Add explicit prefix to avoid ambiguity.")+ def _warn_if_ambiguous_doc_gen(self, name: str) -> None:+ """Log a warning if a bare string handle matches presets/aliases/waterfalls."""+ matches: list[str] = []+ if name in self.doc_gen_presets:+ matches.append(f"doc gen preset (use ${name} or preset:{name})")+ if name in self.doc_gen_aliases:+ matches.append(f"alias (use @{name} or alias:{name})")+ if name in self.doc_gen_waterfalls:+ matches.append(f"waterfall (use ~{name} or waterfall:{name})")+ if matches:+ log.warning(+ f"Bare string '{name}' matches: {', '.join(matches)}. Using it as a direct model handle. Add explicit prefix to avoid ambiguity."+ )+def _raise_handle_not_found_error(self,ref: ModelReference,*,model_type: ModelType,- presets: dict[str, LLMSetting] | dict[str, ExtractSetting] | dict[str, ImgGenSetting] | dict[str, SearchSetting],+ presets: dict[str, LLMSetting] | dict[str, ExtractSetting] | dict[str, ImgGenSetting] | dict[str, SearchSetting] | dict[str, DocGenSetting],) -> NoReturn:"""Raise ModelChoiceNotFoundError with migration hints if applicable."""msg = f"Model handle '{ref.name}' was not found in the model deck"@@ -454,6 +498,59 @@ class ModelDeck(ConfigModel):presets=self.search_presets,)+ def get_doc_gen_choice_default(self, *, doc_gen_format: DocGenFormat, source: DocGenSource) -> DocGenModelChoice | None:+ """The engine the deck prints this format from this source with when a step names none, or None when it names none either."""+ return self.doc_gen_choice_defaults.get(doc_gen_choice_key(doc_gen_format=doc_gen_format, source=source))++ def get_doc_gen_setting(self, *, doc_gen_choice: DocGenModelChoice) -> DocGenSetting:+ if isinstance(doc_gen_choice, DocGenSetting):+ return doc_gen_choice++ ref = ensure_model_reference(doc_gen_choice)+ match ref.kind:+ case ModelReferenceKind.PRESET:+ if preset := self.doc_gen_presets.get(ref.name):+ return preset+ msg = f"Doc gen preset '{ref.name}' was not found in the model deck"+ raise ModelChoiceNotFoundError(+ message=msg,+ model_type=ModelType.DOC_GEN,+ model_choice=ref.raw,+ reference_kind=ModelReferenceKind.PRESET,+ available_options=list(self.doc_gen_presets.keys()),+ )+ case ModelReferenceKind.ALIAS:+ if alias_target := self.doc_gen_aliases.get(ref.name):+ return DocGenSetting(model=alias_target)+ msg = f"Alias '{ref.name}' was not found in the model deck"+ raise ModelChoiceNotFoundError(+ message=msg,+ model_type=ModelType.DOC_GEN,+ model_choice=ref.raw,+ reference_kind=ModelReferenceKind.ALIAS,+ available_options=list(self.doc_gen_aliases.keys()),+ )+ case ModelReferenceKind.WATERFALL:+ if ref.name in self.doc_gen_waterfalls:+ return DocGenSetting(model=ref.name)+ msg = f"Waterfall '{ref.name}' was not found in the model deck"+ raise ModelChoiceNotFoundError(+ message=msg,+ model_type=ModelType.DOC_GEN,+ model_choice=ref.raw,+ reference_kind=ModelReferenceKind.WATERFALL,+ available_options=list(self.doc_gen_waterfalls.keys()),+ )+ case ModelReferenceKind.HANDLE:+ self._warn_if_ambiguous_doc_gen(ref.name)+ if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.DOC_GEN):+ return DocGenSetting(model=ref.name)+ self._raise_handle_not_found_error(+ ref=ref,+ model_type=ModelType.DOC_GEN,+ presets=self.doc_gen_presets,+ )+def get_img_gen_setting(self, img_gen_choice: ImgGenModelChoice) -> ImgGenSetting:if isinstance(img_gen_choice, ImgGenSetting):return img_gen_choice@@ -580,6 +677,17 @@ class ModelDeck(ConfigModel):)return self+ def validate_doc_gen_presets(self) -> Self:+ for doc_gen_preset_id, doc_gen_setting in self.doc_gen_presets.items():+ if not self.is_model_handle_defined(model_handle=doc_gen_setting.model, model_type=ModelType.DOC_GEN):+ msg = f"Doc gen handle '{doc_gen_setting.model}' for doc gen preset '{doc_gen_preset_id}' was not found in the model deck"+ raise DocGenHandleNotFoundError(+ message=msg,+ preset_id=doc_gen_preset_id,+ model_handle=doc_gen_setting.model,+ )+ return self+def validate_registered_models(self):self.validate_inference_models()try:@@ -647,6 +755,22 @@ class ModelDeck(ConfigModel):log.warning(f"Search handle not found: {exc}")case ProblemReaction.NONE:pass+ try:+ self.validate_doc_gen_presets()+ except DocGenHandleNotFoundError as exc:+ match self.model_deck_config.missing_presets_reaction:+ case ProblemReaction.RAISE:+ msg = f"Failed to validate all DocGen presets: {exc}"+ raise ModelDeckPresetValidatonError(+ message=msg,+ model_type=ModelType.DOC_GEN,+ preset_id=exc.preset_id,+ model_handle=exc.model_handle,+ ) from exc+ case ProblemReaction.LOG:+ log.warning(f"DocGen handle not found: {exc}")+ case ProblemReaction.NONE:+ passdef validate_inference_models(self):for model_handle, model_spec in self.inference_models.items():@@ -872,8 +996,8 @@ class ModelDeck(ConfigModel):named_references: set[str] = set(aliases.values())for fallback_list in waterfalls.values():named_references.update(fallback_list)- deck_settings: list[LLMSetting | ExtractSetting | ImgGenSetting | SearchSetting]- deck_choices: list[LLMModelChoice | ExtractModelChoice | ImgGenModelChoice | SearchModelChoice | None]+ deck_settings: list[LLMSetting | ExtractSetting | ImgGenSetting | SearchSetting | DocGenSetting]+ deck_choices: list[LLMModelChoice | ExtractModelChoice | ImgGenModelChoice | SearchModelChoice | DocGenModelChoice | None]match model_type:case ModelType.LLM:deck_settings = list(self.llm_presets.values())@@ -892,9 +1016,12 @@ class ModelDeck(ConfigModel):case ModelType.SEARCH:deck_settings = list(self.search_presets.values())deck_choices = [self.search_choice_default]+ case ModelType.DOC_GEN:+ deck_settings = list(self.doc_gen_presets.values())+ deck_choices = list(self.doc_gen_choice_defaults.values())named_references.update(deck_setting.model for deck_setting in deck_settings)for deck_choice in deck_choices:- if isinstance(deck_choice, (LLMSetting, ExtractSetting, ImgGenSetting, SearchSetting)):+ if isinstance(deck_choice, (LLMSetting, ExtractSetting, ImgGenSetting, SearchSetting, DocGenSetting)):named_references.add(deck_choice.model)elif isinstance(deck_choice, ModelReference):match deck_choice.kind:diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.pyindex 57589d58b..930df875e 100644--- a/pipelex/cogt/models/model_deck_check.py+++ b/pipelex/cogt/models/model_deck_check.py@@ -2,6 +2,7 @@from typing import NoReturn+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice, DocGenSettingfrom pipelex.cogt.exceptions import ModelChoiceNotFoundErrorfrom pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting@@ -226,6 +227,68 @@ def check_search_choice_with_deck(search_choice: SearchModelChoice) -> None:)+def check_doc_gen_choice_with_deck(*, doc_gen_choice: DocGenModelChoice) -> None:+ if isinstance(doc_gen_choice, DocGenSetting):+ return++ model_deck = get_model_deck()+ ref = ensure_model_reference(doc_gen_choice)++ match ref.kind:+ case ModelReferenceKind.PRESET:+ if ref.name in model_deck.doc_gen_presets:+ return+ msg = f"Doc gen preset '{ref.name}' was not found in the model deck"+ _raise_model_choice_not_found(+ msg,+ model_deck=model_deck,+ model_type=ModelType.DOC_GEN,+ raw_choice=ref.raw,+ name=ref.name,+ reference_kind=ModelReferenceKind.PRESET,+ available_options=list(model_deck.doc_gen_presets.keys()),+ )+ case ModelReferenceKind.ALIAS:+ if ref.name in model_deck.doc_gen_aliases:+ return+ msg = f"Alias '{ref.name}' was not found in the model deck"+ _raise_model_choice_not_found(+ msg,+ model_deck=model_deck,+ model_type=ModelType.DOC_GEN,+ raw_choice=ref.raw,+ name=ref.name,+ reference_kind=ModelReferenceKind.ALIAS,+ available_options=list(model_deck.doc_gen_aliases.keys()),+ )+ case ModelReferenceKind.WATERFALL:+ if ref.name in model_deck.doc_gen_waterfalls:+ return+ msg = f"Waterfall '{ref.name}' was not found in the model deck"+ _raise_model_choice_not_found(+ msg,+ model_deck=model_deck,+ model_type=ModelType.DOC_GEN,+ raw_choice=ref.raw,+ name=ref.name,+ reference_kind=ModelReferenceKind.WATERFALL,+ available_options=list(model_deck.doc_gen_waterfalls.keys()),+ )+ case ModelReferenceKind.HANDLE:+ if model_deck.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.DOC_GEN):+ return+ msg = f"Model handle '{ref.name}' was not found in the model deck"+ _raise_model_choice_not_found(+ msg,+ model_deck=model_deck,+ model_type=ModelType.DOC_GEN,+ raw_choice=ref.raw,+ name=ref.name,+ reference_kind=ModelReferenceKind.HANDLE,+ available_options=list(model_deck.inference_models.keys()),+ )++def check_img_gen_choice_with_deck(img_gen_choice: ImgGenModelChoice) -> None:if isinstance(img_gen_choice, ImgGenSetting):returndiff --git a/pipelex/cogt/models/model_deck_loader.py b/pipelex/cogt/models/model_deck_loader.pyindex 1a406a86f..08b394cb4 100644--- a/pipelex/cogt/models/model_deck_loader.py+++ b/pipelex/cogt/models/model_deck_loader.py@@ -1,3 +1,4 @@+from collections.abc import Mappingfrom typing import Anyfrom pydantic import ValidationError@@ -9,8 +10,18 @@ from pipelex.tools.misc.toml_utils import load_toml_from_pathfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error-def load_model_deck_blueprint(model_deck_paths: list[str]) -> ModelDeckBlueprint:+def load_model_deck_blueprint(model_deck_paths: list[str], *, base_deck_dict: Mapping[str, Any] | None = None) -> ModelDeckBlueprint:+ """Merge the model deck files, in order, over an optional base document, and validate the result.++ Args:+ model_deck_paths: The deck files, in the order they are merged: a later file overrides an earlier one.+ base_deck_dict: What the files are merged over, so any file overrides it: the model deck defaults the plugins+ declare (`PluginModelDeclarations.make_deck_base`). The merged document is validated as a whole, so a base+ entry meets the same checks as a file's.+ """full_deck_dict: dict[str, Any] = {}+ if base_deck_dict:+ deep_update(full_deck_dict, updates=base_deck_dict)if not model_deck_paths:msg = "No Model deck paths found. Please run `pipelex init config` to create the set up the base deck."raise ModelDeckNotFoundError(msg)diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.pyindex a48e4ca10..46e6fe4f1 100644--- a/pipelex/cogt/models/model_manager.py+++ b/pipelex/cogt/models/model_manager.py@@ -1,10 +1,12 @@from collections.abc import Sequencefrom pathlib import Path+from typing import Anyfrom typing_extensions import overridefrom pipelex import log-from pipelex.cogt.exceptions import GatewayUnknownModelError, ModelManagerError+from pipelex.cogt.doc_gen.doc_gen_format import parse_doc_gen_choice_key+from pipelex.cogt.exceptions import GatewayUnknownModelError, ModelManagerError, PluginModelDeclarationErrorfrom pipelex.cogt.extract.extract_setting import ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenSettingfrom pipelex.cogt.llm.llm_setting import LLMSetting@@ -23,6 +25,7 @@ from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstractfrom pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKindfrom pipelex.cogt.search.search_setting import SearchSettingfrom pipelex.config import get_config+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarationsfrom pipelex.system.configuration.config_loader import config_managerfrom pipelex.system.pipelex_service.types import RemoteConfigSourcefrom pipelex.tools.misc.file_utils import find_files_in_dir@@ -69,6 +72,7 @@ class ModelManager(ModelManagerAbstract):secrets_provider: SecretsProviderAbstract,managed_gateway_configs: dict[str, GatewayConfig] | None,gateway_config_source: RemoteConfigSource | None,+ plugin_model_declarations: PluginModelDeclarations,needs_inference: bool = True,backends_library_paths: Sequence[Path] | None = None,backends_dir_path: str | None = None,@@ -79,10 +83,11 @@ class ModelManager(ModelManagerAbstract):# back to layered config_manager paths for all other callers. The two documents are# sequences — the base file, then the personal override files — see# `ConfigLoader.backends_file_paths`.+ resolved_backends_dir_path = backends_dir_path or str(config_manager.backends_dir_path)self.inference_backend_library.load(secrets_provider=secrets_provider,backends_library_paths=backends_library_paths or config_manager.backends_file_paths(),- backends_dir_path=backends_dir_path or str(config_manager.backends_dir_path),+ backends_dir_path=resolved_backends_dir_path,managed_gateway_configs=managed_gateway_configs,lenient=not needs_inference,)@@ -92,6 +97,12 @@ class ModelManager(ModelManagerAbstract):# single copy, and by here logging is configured.if (stale_warning := self.inference_backend_library.take_stale_configuration_warning()) is not None:log.warning(stale_warning)+ # The plugins' internal models join the internal backend before anything reads the library, so routing, the+ # deck and the gateway check below see them exactly as they see a model `internal.toml` declares.+ has_internal_backend = self.inference_backend_library.merge_plugin_internal_models(+ plugin_model_declarations=plugin_model_declarations,+ backends_dir_path=resolved_backends_dir_path,+ )enabled_backends = self.inference_backend_library.all_enabled_backends()self._routing_profile = load_active_routing_profile(routing_profile_library_paths=routing_profile_library_paths or config_manager.routing_profiles_file_paths(),@@ -99,7 +110,12 @@ class ModelManager(ModelManagerAbstract):lenient=not needs_inference,)model_deck_paths = ModelManager.get_model_deck_paths(deck_dir_path=deck_dir_path or str(config_manager.model_decks_dir_path))- deck_blueprint = load_model_deck_blueprint(model_deck_paths=model_deck_paths)+ deck_blueprint = load_model_deck_blueprint(+ model_deck_paths=model_deck_paths,+ base_deck_dict=self._make_plugin_deck_base(+ plugin_model_declarations=plugin_model_declarations, has_internal_backend=has_internal_backend+ ),+ )self.model_deck = self.build_deck(enabled_backends=enabled_backends, model_deck_blueprint=deck_blueprint)self._enforce_gateway_model_membership(@@ -108,6 +124,28 @@ class ModelManager(ModelManagerAbstract):enabled_backends=enabled_backends,)+ @classmethod+ def _make_plugin_deck_base(cls, *, plugin_model_declarations: PluginModelDeclarations, has_internal_backend: bool) -> dict[str, Any]:+ """The model deck document the plugins' defaults make, for the deck files to be merged over.++ Left out entirely without an internal backend, as the plugins' models are: a default pointing at a model this+ boot does not serve is exactly what a live gateway's membership check refuses, and a plugin must not make a+ boot fail that would succeed without it. The kit's own internal models set the precedent: with the backend+ disabled their files' models are not loaded, and nothing a plugin adds changes that.++ Raises:+ PluginModelDeclarationError: a plugin declares a default for a format and source no step composes.+ """+ if not has_internal_backend:+ return {}+ for doc_gen_default in plugin_model_declarations.doc_gen_defaults:+ try:+ parse_doc_gen_choice_key(doc_gen_default.choice_key)+ except ValueError as exc:+ msg = f"Plugin '{doc_gen_default.plugin}' declares a default document engine that no step can use: {exc}"+ raise PluginModelDeclarationError(msg, plugin=doc_gen_default.plugin) from exc+ return plugin_model_declarations.make_deck_base()+def _enforce_gateway_model_membership(self,managed_gateway_configs: dict[str, GatewayConfig] | None,@@ -197,10 +235,14 @@ class ModelManager(ModelManagerAbstract):def _collect_deck_referenced_handles(cls, deck: ModelDeck) -> list[tuple[str, ModelType]]:"""Gather the (handle, model_type) pairs that the deck advertises as usable.- Covers presets and choice defaults across every model type. Aliases and waterfalls- are intentionally NOT enumerated directly — they are reachable via preset/choice- references, and the resolver walks through them. Including them here would force the- check on dangling helpers the user has not actively wired into a preset.+ Covers presets and choice defaults across every model type a managed gateway can serve.+ Aliases and waterfalls are intentionally NOT enumerated directly — they are reachable via+ preset/choice references, and the resolver walks through them. Including them here would+ force the check on dangling helpers the user has not actively wired into a preset.++ Document engines (``doc_gen``) are left out: they are software on the host, which no+ gateway serves, and a step whose engine is undeclared is refused at load naming the engine+ and the fix, where a gateway error at boot would stop every method and blame the gateway."""references: list[tuple[str, ModelType]] = []for llm_setting in deck.llm_presets.values():@@ -433,6 +475,11 @@ class ModelManager(ModelManagerAbstract):search_waterfalls=model_deck_blueprint.search.waterfalls,search_presets=model_deck_blueprint.search.presets,search_choice_default=model_deck_blueprint.search.choice_default,+ # DocGen+ doc_gen_aliases=model_deck_blueprint.doc_gen.aliases,+ doc_gen_waterfalls=model_deck_blueprint.doc_gen.waterfalls,+ doc_gen_presets=model_deck_blueprint.doc_gen.presets,+ doc_gen_choice_defaults=model_deck_blueprint.doc_gen.choice_defaults,model_deck_config=get_config().inference.model_deck,)diff --git a/pipelex/cogt/models/model_manager_abstract.py b/pipelex/cogt/models/model_manager_abstract.pyindex 927903a09..abd392cb6 100644--- a/pipelex/cogt/models/model_manager_abstract.py+++ b/pipelex/cogt/models/model_manager_abstract.py@@ -5,6 +5,7 @@ from pipelex.cogt.model_backends.gateway_config import GatewayConfigfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.models.model_deck import ModelDeck+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarationsfrom pipelex.system.pipelex_service.types import RemoteConfigSourcefrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract@@ -25,9 +26,15 @@ class ModelManagerAbstract(ABC):secrets_provider: SecretsProviderAbstract,managed_gateway_configs: dict[str, GatewayConfig] | None,gateway_config_source: RemoteConfigSource | None,+ plugin_model_declarations: PluginModelDeclarations,needs_inference: bool = True,) -> None:- pass+ """Load the inference backends, the routing profile and the model deck, and build the deck.++ ``plugin_model_declarations`` carries the internal models and the model deck defaults the registered plugins+ declared (``PluginRegistrar.make_model_declarations``): an implementation merges the models into the internal+ backend and the defaults beneath the deck files. Required, so no caller can forget the plugins' engines.+ """@abstractmethoddef get_inference_model(self, model_handle: str, *, model_type: ModelType) -> InferenceModelSpec:diff --git a/pipelex/cogt/models/model_suggestion.py b/pipelex/cogt/models/model_suggestion.pyindex 0bc025d06..9934fc683 100644--- a/pipelex/cogt/models/model_suggestion.py+++ b/pipelex/cogt/models/model_suggestion.py@@ -48,6 +48,8 @@ def get_collection_keys(return list(model_deck.img_gen_presets.keys())case ModelType.SEARCH:return list(model_deck.search_presets.keys())+ case ModelType.DOC_GEN:+ return list(model_deck.doc_gen_presets.keys())case ModelReferenceKind.ALIAS:match model_type:case ModelType.LLM:@@ -58,6 +60,8 @@ def get_collection_keys(return list(model_deck.img_gen_aliases.keys())case ModelType.SEARCH:return list(model_deck.search_aliases.keys())+ case ModelType.DOC_GEN:+ return list(model_deck.doc_gen_aliases.keys())case ModelReferenceKind.WATERFALL:match model_type:case ModelType.LLM:@@ -68,6 +72,8 @@ def get_collection_keys(return list(model_deck.img_gen_waterfalls.keys())case ModelType.SEARCH:return list(model_deck.search_waterfalls.keys())+ case ModelType.DOC_GEN:+ return list(model_deck.doc_gen_waterfalls.keys())case ModelReferenceKind.HANDLE:return sorted(handle for handle, spec in model_deck.inference_models.items() if spec.model_type == model_type)diff --git a/pipelex/cogt/templating/template_rendering.py b/pipelex/cogt/templating/template_rendering.pyindex 2be065b1a..8dd25c049 100644--- a/pipelex/cogt/templating/template_rendering.py+++ b/pipelex/cogt/templating/template_rendering.py@@ -14,6 +14,7 @@ async def render_template(context: dict[str, Any],templating_style: TemplatingStyle | None = None,finalize: Callable[[Any], Any] | None = None,+ is_undefined_strict: bool = False,) -> str:template_source = rewrite_template_sigils(template)@@ -23,4 +24,5 @@ async def render_template(templating_context=context,templating_style=templating_style,finalize=finalize,+ is_undefined_strict=is_undefined_strict,)diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex d81b7090b..973dbfcc7 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -156,8 +156,10 @@ class Concept(ConceptAbstract):if resolved is not None and resolved.concept_ref == concept_2.concept_ref:return True- # If both concepts refine the same concept, they are compatible- if concept_1.refines is not None and concept_2.refines is not None:+ # If both concepts refine the same concept, they are compatible — unless the wanted one is a native.+ # A native is reached only through its own lineage: a concept beside `native.Markdown`, refining+ # `native.Text` as it does, is a text but not a Markdown.+ if concept_1.refines is not None and concept_2.refines is not None and not cls.is_native_concept(concept=concept_2):refines_1 = concept_1.refinesrefines_2 = concept_2.refines# Resolve cross-package refines through the resolverdiff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex d9c072730..65ef57a0c 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -122,6 +122,16 @@ class ConceptFactory:description="A text",structure_class_name=structure_class_name,)+ case NativeConceptCode.MARKDOWN:+ # The one native refining another (`NativeConceptCode.refined_native`): the declaration is+ # what lets a Markdown value stand in for a Text, established without resolving any class.+ return Concept(+ code=native_concept_code,+ domain_code=SpecialDomain.NATIVE,+ description="A text written in Markdown",+ structure_class_name=structure_class_name,+ refines=NativeConceptCode.TEXT.concept_ref,+ )case NativeConceptCode.IMAGE:return Concept(code=native_concept_code,diff --git a/pipelex/core/concepts/native/concept_native.py b/pipelex/core/concepts/native/concept_native.pyindex a60d8637b..9e75a3c18 100644--- a/pipelex/core/concepts/native/concept_native.py+++ b/pipelex/core/concepts/native/concept_native.py@@ -11,6 +11,7 @@ from pipelex.core.stuffs.dynamic_content import DynamicContentfrom pipelex.core.stuffs.html_content import HtmlContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.markdown_content import MarkdownContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.page_content import PageContentfrom pipelex.core.stuffs.search_result_content import SearchResultContent@@ -23,6 +24,7 @@ from pipelex.core.stuffs.yes_no_content import YesNoContentclass NativeConceptCode(StrEnum):DYNAMIC = "Dynamic"TEXT = "Text"+ MARKDOWN = "Markdown"IMAGE = "Image"DOCUMENT = "Document"HTML = "Html"@@ -50,6 +52,7 @@ class NativeConceptCode(StrEnum):case (NativeConceptCode.DYNAMIC| NativeConceptCode.TEXT+ | NativeConceptCode.MARKDOWN| NativeConceptCode.IMAGE| NativeConceptCode.DOCUMENT| NativeConceptCode.HTML@@ -65,6 +68,68 @@ class NativeConceptCode(StrEnum):):return False+ @property+ def refined_native(self) -> "NativeConceptCode | None":+ """The native concept this one refines, or None when it refines none.++ `Markdown` refines `Text`: a Markdown value is a text, so it is accepted wherever a `Text` is,+ while a plain `Text` is not a Markdown. Its content class subclasses the refined one's, which+ gives it the same shape, so compatibility with such a native is decided by lineage and never+ by shape (`ConceptLibrary.is_compatible`).+ """+ match self:+ case NativeConceptCode.MARKDOWN:+ return NativeConceptCode.TEXT+ case (+ NativeConceptCode.DYNAMIC+ | NativeConceptCode.TEXT+ | NativeConceptCode.IMAGE+ | NativeConceptCode.DOCUMENT+ | NativeConceptCode.HTML+ | NativeConceptCode.TEXT_AND_IMAGES+ | NativeConceptCode.NUMBER+ | NativeConceptCode.YES_NO+ | NativeConceptCode.DATE+ | NativeConceptCode.TIME+ | NativeConceptCode.PAGE+ | NativeConceptCode.JSON+ | NativeConceptCode.SEARCH_RESULT+ | NativeConceptCode.ANYTHING+ | NativeConceptCode.COMPOSITE+ ):+ return None++ @property+ def is_pinned_by_the_standard(self) -> bool:+ """Whether the MTHDS standard's page pins this native's definition, which `pinned_blueprints.py` copies.++ `Markdown` is the one native Pipelex defines ahead of the standard: its blueprint in+ `pinned_blueprints.py` is Pipelex's own until the standard's page defines it, and the day it+ does, this returns True for it and `PINNED_NATIVES_MTHDS_VERSION` moves to the version that+ re-pinned the set. The conformance test against the page reads only the natives this is True for.+ """+ match self:+ case NativeConceptCode.MARKDOWN:+ return False+ case (+ NativeConceptCode.DYNAMIC+ | NativeConceptCode.TEXT+ | NativeConceptCode.IMAGE+ | NativeConceptCode.DOCUMENT+ | NativeConceptCode.HTML+ | NativeConceptCode.TEXT_AND_IMAGES+ | NativeConceptCode.NUMBER+ | NativeConceptCode.YES_NO+ | NativeConceptCode.DATE+ | NativeConceptCode.TIME+ | NativeConceptCode.PAGE+ | NativeConceptCode.JSON+ | NativeConceptCode.SEARCH_RESULT+ | NativeConceptCode.ANYTHING+ | NativeConceptCode.COMPOSITE+ ):+ return True+@propertydef concept_ref(self) -> str:return f"{SpecialDomain.NATIVE}.{self.value}"@@ -85,6 +150,8 @@ class NativeConceptCode(StrEnum):return DynamicContentcase NativeConceptCode.TEXT:return TextContent+ case NativeConceptCode.MARKDOWN:+ return MarkdownContentcase NativeConceptCode.IMAGE:return ImageContentcase NativeConceptCode.DOCUMENT:@@ -151,7 +218,7 @@ class NativeConceptCode(StrEnum):return Falsematch enum_value:- case NativeConceptCode.TEXT:+ case NativeConceptCode.TEXT | NativeConceptCode.MARKDOWN:return Truecase (NativeConceptCode.DYNAMIC@@ -191,6 +258,7 @@ class NativeConceptCode(StrEnum):case (NativeConceptCode.DYNAMIC| NativeConceptCode.TEXT+ | NativeConceptCode.MARKDOWN| NativeConceptCode.IMAGE| NativeConceptCode.DOCUMENT| NativeConceptCode.HTML@@ -216,6 +284,7 @@ class NativeConceptCode(StrEnum):match enum_value:case (NativeConceptCode.TEXT+ | NativeConceptCode.MARKDOWN| NativeConceptCode.IMAGE| NativeConceptCode.DOCUMENT| NativeConceptCode.HTMLdiff --git a/pipelex/core/concepts/native/pinned_blueprints.py b/pipelex/core/concepts/native/pinned_blueprints.pyindex fb2fe02ca..ca4c98952 100644--- a/pipelex/core/concepts/native/pinned_blueprints.py+++ b/pipelex/core/concepts/native/pinned_blueprints.py@@ -5,8 +5,10 @@ descriptions — in `mthds/docs/spec/native-concepts.md`. This module is the runpinned set: crate materialization looks definitions up here instead of reflecting over the runtimecontent classes, so two independent implementations byte-agree on materialized natives (andtherefore on crate fingerprints). Any edit here is a standard change and must land in the spec-page first; the consistency test in `tests/unit/pipelex/codegen/` proves each runtime content-class still matches its pinned blueprint, so the two can never drift silently — and+page first, except for a native Pipelex defines ahead of the standard+(`NativeConceptCode.is_pinned_by_the_standard` is False for it), whose blueprint is Pipelex's own+until the page defines it in exactly these words; the consistency test in `tests/unit/pipelex/codegen/`+proves each runtime content class still matches its pinned blueprint, so the two can never drift silently — and`tests/unit/pipelex/core/concepts/test_pinned_natives_vs_standard.py` holds this set to thestandard's page itself, read live from the sibling `mthds/` checkout (the `MTHDS standardconformance` CI workflow runs it against a fresh checkout on every pull request), so the day the@@ -48,6 +50,8 @@ def _pinned_description(native_code: NativeConceptCode) -> str:return "A dynamic concept"case NativeConceptCode.TEXT:return "A text"+ case NativeConceptCode.MARKDOWN:+ return "A text written in Markdown"case NativeConceptCode.IMAGE:return "An image"case NativeConceptCode.DOCUMENT:@@ -86,6 +90,13 @@ def _pinned_structure(native_code: NativeConceptCode) -> dict[str, ConceptStructreturn {"text": _text_field(description="The text", required=True),}+ case NativeConceptCode.MARKDOWN:+ # Text's one field, holding the Markdown source. A blueprint states `structure` or `refines`,+ # never both, so the refinement of `Text` lives on the runtime concept (`ConceptFactory`), not here.+ # Pipelex defines Markdown ahead of the standard, so this is not yet a copy of the standard's page.+ return {+ "text": _text_field(description="The text, written in Markdown", required=True),+ }case NativeConceptCode.IMAGE:return {"url": _text_field(description="The image URL: a storage URI, an HTTP(S) URL, or a base64 data URL", required=True),diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.pyindex ab5f6d619..655b28e72 100644--- a/pipelex/core/memory/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -19,6 +19,7 @@ from pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.html_content import HtmlContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContentfrom pipelex.core.stuffs.mermaid_content import MermaidContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.stuff import Stuff@@ -562,6 +563,11 @@ class WorkingMemory(WorkingMemoryAbstract[Stuff], ContextProviderAbstract):"""Get main stuff content as HtmlContent if applicable."""return self.get_stuff_as_html(name=MAIN_STUFF_NAME)+ @property+ def main_stuff_as_markdown(self) -> MarkdownContent:+ """Get main stuff content as MarkdownContent if applicable."""+ return self.main_stuff_as(content_type=MarkdownContent)+@propertydef main_stuff_as_mermaid(self) -> MermaidContent:"""Get main stuff content as MermaidContent if applicable."""diff --git a/pipelex/core/pipes/pipe_output.py b/pipelex/core/pipes/pipe_output.pyindex 1a1245de5..4d417867b 100644--- a/pipelex/core/pipes/pipe_output.py+++ b/pipelex/core/pipes/pipe_output.py@@ -9,6 +9,7 @@ from pipelex.core.stuffs.date_content import DateContentfrom pipelex.core.stuffs.html_content import HtmlContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContentfrom pipelex.core.stuffs.mermaid_content import MermaidContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.stuff import Stuff@@ -139,6 +140,11 @@ class PipeOutput(PipeOutputAbstract[WorkingMemory]):"""Get main stuff content as HtmlContent if applicable."""return self.working_memory.main_stuff_as_html+ @property+ def main_stuff_as_markdown(self) -> MarkdownContent:+ """Get main stuff content as MarkdownContent if applicable."""+ return self.working_memory.main_stuff_as_markdown+@propertydef main_stuff_as_mermaid(self) -> MermaidContent:"""Get main stuff content as MermaidContent if applicable."""diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.pyindex c8f6d10c5..6be00be21 100644--- a/pipelex/core/registry_models.py+++ b/pipelex/core/registry_models.py@@ -8,6 +8,7 @@ from pipelex.core.stuffs.html_content import HtmlContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.json_content import JSONContentfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.page_content import PageContentfrom pipelex.core.stuffs.search_result_content import SearchResultContent@@ -34,6 +35,7 @@ class CoreRegistryModels(RegistryModels):STUFF: ClassVar[list[ModelType]] = [TextContent,+ MarkdownContent,NumberContent,YesNoContent,DateContent,diff --git a/pipelex/core/stuffs/markdown_content.py b/pipelex/core/stuffs/markdown_content.pynew file mode 100644index 000000000..4ef7eee7e--- /dev/null+++ b/pipelex/core/stuffs/markdown_content.py@@ -0,0 +1,51 @@+from typing import TYPE_CHECKING++from pydantic import Field+from typing_extensions import override++from pipelex.core.stuffs.text_content import TextContent+from pipelex.tools.markdown.markdown_parser import render_markdown_as_html+from pipelex.tools.misc.pretty import require_rich_for_rendering++if TYPE_CHECKING:+ from pipelex.tools.misc.pretty import PrettyPrintable+++# The content class of the native `Markdown` concept, which refines `Text`: a text written in Markdown, which+# the views and the renditions format rather than show as it is. It keeps `TextContent`'s single `text` field,+# holding the Markdown source, so a Markdown value goes wherever a text goes, and a prompt, the plain view and+# the saved `.md` file get the source as it is. The views that format are the HTML one, which converts it with+# the shared parser (`render_markdown_as_html`), and the pretty one, which Rich renders as Markdown.+#+# Inside an HTML template, a Markdown value prints as its converted HTML with no filter: the class follows+# markupsafe's `__html__` convention, which HTML autoescaping reads, and `StuffArtefact` answers it for a stuff+# whose content defines it. The conversion is safe to insert: the parser escapes raw HTML, and markdown-it+# refuses `javascript:` and similar link targets.+#+# The docstring is the concept's description, as for every native content class: pydantic publishes it as the+# schema's description, which an LLM reads when it writes a list of Markdown texts.+class MarkdownContent(TextContent):+ """A text written in Markdown"""++ text: str = Field(description="The text, written in Markdown")++ @property+ @override+ def short_desc(self) -> str:+ return f"some markdown ({len(self.text)} chars)"++ @override+ def rendered_html(self) -> str:+ return render_markdown_as_html(self.text)++ @override+ def rendered_pretty(self, *, title: str | None = None, depth: int = 0) -> "PrettyPrintable":+ # Always Markdown: `TextContent` sniffs for HTML first, but raw HTML in a Markdown source is text.+ require_rich_for_rendering()+ from rich.markdown import Markdown++ return Markdown(self.text)++ def __html__(self) -> str:+ """The HTML that markupsafe inserts as it is: the converted Markdown, which holds no raw HTML from the source."""+ return self.rendered_html()diff --git a/pipelex/core/stuffs/stuff.py b/pipelex/core/stuffs/stuff.pyindex 82911805e..d7bbfabb0 100644--- a/pipelex/core/stuffs/stuff.py+++ b/pipelex/core/stuffs/stuff.py@@ -15,6 +15,7 @@ from pipelex.core.stuffs.exceptions import StuffContentTypeError, StuffContentVafrom pipelex.core.stuffs.html_content import HtmlContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContentfrom pipelex.core.stuffs.mermaid_content import MermaidContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.stuff_artefact import StuffArtefact@@ -247,6 +248,11 @@ class Stuff(PrettyRenderable, CustomBaseModel, StuffAbstract[Concept, StuffConte"""Get content as HtmlContent if applicable."""return self.content_as(content_type=HtmlContent)+ @property+ def as_markdown(self) -> MarkdownContent:+ """Get content as MarkdownContent if applicable."""+ return self.content_as(content_type=MarkdownContent)+@propertydef as_mermaid(self) -> MermaidContent:"""Get content as MermaidContent if applicable."""diff --git a/pipelex/core/stuffs/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.pyindex b4b723d9e..431efd648 100644--- a/pipelex/core/stuffs/stuff_artefact.py+++ b/pipelex/core/stuffs/stuff_artefact.py@@ -15,17 +15,25 @@ Templates render under the Pipelex sandbox (`pipelex/tools/jinja2/jinja2_sandboxtemplate read public data and call methods of plain values only. StuffArtefact declares its templatesurface: the dict-like accessors a template may call, and the metadata fields it may read althoughthey start with an underscore.++Printed as it is, `{{ my_stuff }}`, an artefact is its content's plain rendering, which an HTML template+escapes. The exception is a content that knows its own HTML (`HtmlRenderable`, markupsafe's `__html__`),+such as a Markdown stuff: the artefact answers `__html__` for it, so HTML autoescaping inserts the content's+converted HTML instead. A template never calls `__html__` itself, since the sandbox refuses every+underscore name; markupsafe does, when it escapes."""from __future__ import annotationsfrom enum import StrEnum-from typing import TYPE_CHECKING, Any, Iterator+from typing import TYPE_CHECKING, Any, Callable, Iteratorfrom typing_extensions import overridefrom pipelex.core.stuffs.list_content import ListContent+from pipelex.tools.jinja2.html_renderable import HtmlRenderablefrom pipelex.tools.jinja2.image_renderable import ImageRenderable+from pipelex.tools.jinja2.renderable_dispatch import type_implementsfrom pipelex.tools.jinja2.template_surface import TemplateSurfacefrom pipelex.tools.templating.text_format import TextFormat@@ -94,6 +102,22 @@ def _get_template_value(*, stuff: Stuff, key: str) -> Any:raise KeyError(key)+_HTML_DUNDER = "__html__"+++def _get_content_html(*, stuff: Stuff) -> Callable[[], str]:+ """The `__html__` of the stuff's content, when the content's class defines it (`HtmlRenderable`).++ Raises AttributeError otherwise, so that `hasattr(artefact, "__html__")` is False and markupsafe escapes+ the artefact's plain rendering, as it always has. Decided from the content's class, never from the+ instance, like every renderable the filters dispatch on (`renderable_dispatch.py`).+ """+ content = stuff.content+ if isinstance(content, HtmlRenderable) and type_implements(value=content, protocol=HtmlRenderable):+ return content.__html__+ raise AttributeError(_HTML_DUNDER)++# Attributes that should NOT be intercepted and delegated to content_PASSTHROUGH_ATTRS = frozenset({@@ -149,6 +173,7 @@ class StuffArtefact:- TagRenderable protocol (render_for_tag_async, default_tag_name)- TextFormatRenderable protocol (rendered_for_template_async)- ImageRenderable protocol (render_with_images)+ - HtmlRenderable protocol (`__html__`), only when its content does: see `_get_content_html`Attributes:_stuff: The underlying Stuff object being wrapped.@@ -171,6 +196,17 @@ class StuffArtefact:"""object.__setattr__(self, "_stuff", stuff)+ @property+ def __html__(self) -> Callable[[], str]:+ """Markupsafe's `__html__`, present only for a content that knows its own HTML (a Markdown stuff).++ HTML autoescaping then inserts that HTML, and escapes every other artefact's plain rendering as before:+ for any other content this raises AttributeError, so `hasattr(artefact, "__html__")` is False. It is a+ property of the class rather than an answer of `__getattr__` so that the filters, which license a call+ by the value's class (`renderable_dispatch.py`), see it too.+ """+ return _get_content_html(stuff=self._stuff)+# -------------------------------------------------------------------------# Attribute access for Jinja2 templates# -------------------------------------------------------------------------diff --git a/pipelex/core/stuffs/stuff_content_factory.py b/pipelex/core/stuffs/stuff_content_factory.pyindex d817762e0..70de01fba 100644--- a/pipelex/core/stuffs/stuff_content_factory.py+++ b/pipelex/core/stuffs/stuff_content_factory.py@@ -22,8 +22,10 @@ class StuffContentFactory:# which rejects a bare bool. Covers native YesNoContent and the subclasses generated for concepts refining YesNo.if isinstance(value, bool) and issubclass(stuff_content_subclass, YesNoContent):return stuff_content_subclass(yes_no=value)- if isinstance(value, str) and stuff_content_subclass == TextContent:- return TextContent(text=value)+ # A string under a Text-family class is its text: native Text, native Markdown, and the classes generated+ # for concepts refining either, so a refining class is honored rather than flattened to TextContent.+ if isinstance(value, str) and issubclass(stuff_content_subclass, TextContent):+ return stuff_content_subclass(text=value)# A date/datetime object or an ISO string under a Date-family class builds via the split constructor;# a dict ({"date","time"}) still rides model_validate below. Covers native Date and concepts refining it.if issubclass(stuff_content_subclass, DateContent) and isinstance(value, (datetime.date, str)):diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.pyindex a049efbab..94a15bacd 100644--- a/pipelex/core/stuffs/stuff_factory.py+++ b/pipelex/core/stuffs/stuff_factory.py@@ -112,6 +112,20 @@ class StuffFactory:)return the_stuff+ @classmethod+ def _make_text_content(cls, *, concept: Concept, text: str) -> TextContent:+ """A string under a Text-compatible concept, held in the concept's own class when that class is a text one.++ So `{"concept": "Markdown", "content": "..."}` holds a `MarkdownContent`, which the views format, and a+ concept refining Text or Markdown keeps its generated class, as the YesNo and Date arms keep theirs. A+ concept that is Text-compatible through its shape alone, or whose class is not registered, gets a+ `TextContent`, as every Text-compatible concept did before.+ """+ the_class = get_class_registry().get_class(name=concept.structure_class_name)+ if isinstance(the_class, type) and issubclass(the_class, TextContent):+ return the_class(text=text)+ return TextContent(text=text)+@classmethoddef combine_stuffs(cls,@@ -303,12 +317,13 @@ class StuffFactory:Case 2: Dict with 'concept' AND 'content' keys (can be plain dict or DictStuff instance)2.1/2.1b: {"concept": "Text"/"native.Text", "content": str} → TextContent with Text concept- 2.1c: {"concept": "domain.Concept", "content": str} → TextContent with that concept (if compatible)+ 2.1c: {"concept": "Markdown"/"domain.Concept", "content": str} → that concept's text class, e.g. MarkdownContent+ (if Text-compatible; TextContent when the class is not a TextContent subclass)2.1d: {"concept": "YesNo"/"domain.Concept", "content": bool} → YesNoContent (if YesNo-compatible)2.1e: {"concept": "Date"/"domain.Concept", "content": date/datetime obj} → DateContent (if Date-compatible)2.1f: {"concept": "Date"/"domain.Concept", "content": ISO str} → DateContent (if Date-compatible, checked after Text)2.1g: {"concept": "Time"/"domain.Concept", "content": time obj or ISO str} → TimeContent (if Time-compatible)- 2.2/2.2b: {"concept": "...", "content": list[str]} → ListContent[TextContent]+ 2.2/2.2b: {"concept": "...", "content": list[str]} → ListContent of the concept's text class, as in 2.1c2.3: {"concept": "...", "content": StuffContent} → Use the StuffContent2.4: {"concept": "...", "content": list[StuffContent]} → ListContent[StuffContent]2.5: {"concept": "...", "content": dict} → Create StuffContent from dict@@ -531,7 +546,7 @@ class StuffFactory:if concept_provider.is_compatible(tested_concept=concept, wanted_concept=text_concept, strict=True):return cls.make_stuff(concept=concept,- content=TextContent(text=content),+ content=cls._make_text_content(concept=concept, text=content),name=name,code=code,)@@ -667,7 +682,7 @@ class StuffFactory:text_concept = concept_provider.get_native_concept(native_concept=NativeConceptCode.TEXT)if concept_provider.is_compatible(tested_concept=concept, wanted_concept=text_concept, strict=True):- items = [TextContent(text=item) for item in list_content_2]+ items = [cls._make_text_content(concept=concept, text=item) for item in list_content_2]return cls.make_stuff(concept=concept,content=ListContent(items=items),diff --git a/pipelex/kit/configs/inference/backends/.kit_manifest.json b/pipelex/kit/configs/inference/backends/.kit_manifest.jsonnew file mode 100644index 000000000..7d6f49bef--- /dev/null+++ b/pipelex/kit/configs/inference/backends/.kit_manifest.json@@ -0,0 +1,6 @@+{+ "files": {+ "internal.toml": "30d0f6f9de1c4cba586e04acc370e431123954bef5cb05a54eef05134d590d4c"+ },+ "kit_version": "0.70.0"+}diff --git a/pipelex/kit/configs/inference/backends/internal.toml b/pipelex/kit/configs/inference/backends/internal.tomlindex 8fcc38d85..1503a8457 100644--- a/pipelex/kit/configs/inference/backends/internal.toml+++ b/pipelex/kit/configs/inference/backends/internal.toml@@ -9,6 +9,10 @@# - Each model is defined in its own section with the model name as the header# - Model costs are in USD per million tokens (input/output)#+# Managed by `pipelex update`, which refreshes this file from the kit and backs+# up a locally edited copy to <file>.bak.<timestamp> first, so local edits will+# not survive future runs. Declare your own models in a backend of your own.+## Documentation: https://docs.pipelex.com# Support: https://go.pipelex.com/discord#@@ -42,3 +46,23 @@ model_id = "extract-text"inputs = ["pdf", "image"]outputs = ["pages"]costs = {}++################################################################################+# DOCUMENT GENERATION ENGINES+################################################################################+#+# The engines a PipeDocGen step prints with. Their inputs are the sources they+# print from: "layout" is the auto-layout of the step's inputs, "html" an HTML+# template, "template_file" an office template file. Their outputs are the+# formats they print. reportlab-pdf is built into Pipelex. The other engines+# come with the Pipelex document generation plugin, pipelex-doc-gen, which+# declares them itself when it loads: they are never listed in this file.++# --- ReportLab PDF ------------------------------------------------------------+[reportlab-pdf]+model_type = "doc_gen"+sdk = "reportlab"+model_id = "print-pdf"+inputs = ["layout"]+outputs = ["pdf"]+costs = {}diff --git a/pipelex/kit/configs/inference/deck/.kit_manifest.json b/pipelex/kit/configs/inference/deck/.kit_manifest.jsonindex 30faa294f..e9c32f088 100644--- a/pipelex/kit/configs/inference/deck/.kit_manifest.json+++ b/pipelex/kit/configs/inference/deck/.kit_manifest.json@@ -3,7 +3,8 @@"1_llm_deck.toml": "1316c59eb52db62820d313ff92569b736a3f5427ab8c10f495a02b463f37c97a","2_img_gen_deck.toml": "00d570907cee9261882922afebc53556013e46d29419fce4cf189135e2849fd6","3_extract_deck.toml": "560a40bd3254b1be011cfb35340fef6c40d70136e5ddce33a961899f92fd70ba",- "4_search_deck.toml": "08e3bd1bee0f9f8ed0ffc735a932705392d0ecfbdeefce64c9815d4b2d8d7e22"+ "4_search_deck.toml": "08e3bd1bee0f9f8ed0ffc735a932705392d0ecfbdeefce64c9815d4b2d8d7e22",+ "5_doc_gen_deck.toml": "3f48b443eb9245a87c6d8051223982bf6134155868bb44a3b167c8e05077c372"},"kit_version": "0.61.0"}diff --git a/pipelex/kit/configs/inference/deck/5_doc_gen_deck.toml b/pipelex/kit/configs/inference/deck/5_doc_gen_deck.tomlnew file mode 100644index 000000000..cdb0ee676--- /dev/null+++ b/pipelex/kit/configs/inference/deck/5_doc_gen_deck.toml@@ -0,0 +1,41 @@+####################################################################################################+# Pipelex Model Deck - Document Generation Configuration+####################################################################################################+#+# This file defines the default engine a PipeDocGen step prints with, for each format and source,+# when the step names none with `model`. A source is where the document comes from: "layout" is the+# auto-layout of the step's inputs, "html" an HTML template, "template_file" an office template file.+#+# Open Pipelex prints a pdf from the auto-layout, on its built-in engine reportlab-pdf, so that is+# the one default this file sets. The Pipelex document generation plugin, pipelex-doc-gen, declares+# the defaults for a pdf from a template and for xlsx, docx and pptx itself when it loads. You can+# still set any default yourself, for any format and source, in an x_custom_*.toml file, which+# overrides both this file and the plugin.+#+# Managed by `pipelex update`. To customize without conflict, create an+# x_custom_*.toml file in this directory — pipelex never tracks or+# overwrites x_custom_* files. Local edits to this file are backed up+# to <file>.bak.<timestamp> on update but will not survive future runs.+#+# Model Reference Syntax:+# - Preset: $preset_name or preset:preset_name+# - Alias: @alias_name or alias:alias_name+#+# Documentation: https://docs.pipelex.com+# Support: https://go.pipelex.com/discord+#+####################################################################################################++####################################################################################################+# Document Generation Default Choices, keyed "<format>.<source>"+####################################################################################################++[doc_gen.choice_defaults]+"pdf.layout" = "@default-pdf"++####################################################################################################+# Aliases+####################################################################################################++[doc_gen.aliases]+default-pdf = "reportlab-pdf"diff --git a/pipelex/kit/deck_variants/README.md b/pipelex/kit/deck_variants/README.mdindex ac4e87028..4a893b2ca 100644--- a/pipelex/kit/deck_variants/README.md+++ b/pipelex/kit/deck_variants/README.md@@ -1,6 +1,6 @@# Deck variants-A deck variant is a complete set of the numbered model deck files (`1_llm_deck.toml`, `2_img_gen_deck.toml`, `3_extract_deck.toml`, `4_search_deck.toml`) kept as an alternative edition of the deck that the kit ships in `pipelex/kit/configs/inference/deck/`. Nothing loads a variant: `pipelex init` does not copy it, `pipelex update` and `pipelex doctor` do not diff against it, and the deck manifest does not know it exists. It is source code that is kept correct, not configuration that is applied.+A deck variant is a complete set of the numbered model deck files (`1_llm_deck.toml`, `2_img_gen_deck.toml`, `3_extract_deck.toml`, `4_search_deck.toml`, `5_doc_gen_deck.toml`) kept as an alternative edition of the deck that the kit ships in `pipelex/kit/configs/inference/deck/`. Nothing loads a variant: `pipelex init` does not copy it, `pipelex update` and `pipelex doctor` do not diff against it, and the deck manifest does not know it exists. It is source code that is kept correct, not configuration that is applied.`multi_provider/` is the deck Pipelex shipped before the default moved to the models the Pipelex Gateway serves from Azure. It resolves the default aliases and presets to Anthropic, Google and OpenAI models, and it still defines the three provider-named aliases (`best-claude`, `best-gemini`, `best-mistral`) that the shipped deck no longer carries.diff --git a/pipelex/kit/deck_variants/multi_provider/5_doc_gen_deck.toml b/pipelex/kit/deck_variants/multi_provider/5_doc_gen_deck.tomlnew file mode 100644index 000000000..cdb0ee676--- /dev/null+++ b/pipelex/kit/deck_variants/multi_provider/5_doc_gen_deck.toml@@ -0,0 +1,41 @@+####################################################################################################+# Pipelex Model Deck - Document Generation Configuration+####################################################################################################+#+# This file defines the default engine a PipeDocGen step prints with, for each format and source,+# when the step names none with `model`. A source is where the document comes from: "layout" is the+# auto-layout of the step's inputs, "html" an HTML template, "template_file" an office template file.+#+# Open Pipelex prints a pdf from the auto-layout, on its built-in engine reportlab-pdf, so that is+# the one default this file sets. The Pipelex document generation plugin, pipelex-doc-gen, declares+# the defaults for a pdf from a template and for xlsx, docx and pptx itself when it loads. You can+# still set any default yourself, for any format and source, in an x_custom_*.toml file, which+# overrides both this file and the plugin.+#+# Managed by `pipelex update`. To customize without conflict, create an+# x_custom_*.toml file in this directory — pipelex never tracks or+# overwrites x_custom_* files. Local edits to this file are backed up+# to <file>.bak.<timestamp> on update but will not survive future runs.+#+# Model Reference Syntax:+# - Preset: $preset_name or preset:preset_name+# - Alias: @alias_name or alias:alias_name+#+# Documentation: https://docs.pipelex.com+# Support: https://go.pipelex.com/discord+#+####################################################################################################++####################################################################################################+# Document Generation Default Choices, keyed "<format>.<source>"+####################################################################################################++[doc_gen.choice_defaults]+"pdf.layout" = "@default-pdf"++####################################################################################################+# Aliases+####################################################################################################++[doc_gen.aliases]+default-pdf = "reportlab-pdf"diff --git a/pipelex/libraries/concept/concept_library.py b/pipelex/libraries/concept/concept_library.pyindex 25ccac14d..71487d35f 100644--- a/pipelex/libraries/concept/concept_library.py+++ b/pipelex/libraries/concept/concept_library.py@@ -120,11 +120,20 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptLibraryAbstract):# make: the declaration tier's silence is the whole answer.return False- return are_structure_classes_compatible(- class_1=self.get_structure_class(concept=tested_concept),- class_2=self.get_structure_class(concept=wanted_concept),- strict=strict,- )+ tested_class = self.get_structure_class(concept=tested_concept)+ wanted_class = self.get_structure_class(concept=wanted_concept)+ if self._is_native_refining_a_native(concept=wanted_concept):+ # `native.Markdown` refines `native.Text` and its class has Text's shape by construction, so+ # shape says nothing about being one: only its lineage does. A plain Text, or any text concept,+ # is refused where a Markdown is wanted; a concept refining Markdown, at any depth, is accepted.+ return issubclass(tested_class, wanted_class)+ return are_structure_classes_compatible(class_1=tested_class, class_2=wanted_class, strict=strict)++ @staticmethod+ def _is_native_refining_a_native(*, concept: Concept) -> bool:+ if not Concept.is_native_concept(concept=concept):+ return False+ return NativeConceptCode(concept.code).refined_native is not None@overridedef get_structure_class(self, *, concept: Concept) -> type[StuffContent]:diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 2dfa9b34a..bdab7e608 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -694,6 +694,10 @@ class LibraryManager(LibraryManagerAbstract):concept_codes_for_domain = domain_concept_codes.get(domain_code, [])source = crate.source_map.get(pipe_ref)+ # A pipe that reads a file beside its bundle (a PipeDocGen `template_file`) resolves it+ # against the bundle's own file, which the crate knows and the pipe's blueprint does not.+ if source is not None and pipe_blueprint.source is None:+ pipe_blueprint = pipe_blueprint.model_copy(update={"source": source})with _locating_pipe_build_refusals(pipe_code=pipe_code, domain_code=domain_code, source=source, elaboration=crate.elaboration_metadata.get(pipe_ref)):@@ -1187,15 +1191,17 @@ class LibraryManager(LibraryManagerAbstract):# Parse dependency blueprintsdep_blueprints: list[PipelexBundleBlueprint] = []+ # Where each bundle is on this host, by its caller-facing source, for the pipes that read a file beside it.+ bundle_files_by_source: dict[str, Path] = {}for mthds_path in resolved_dep.mthds_files:try:blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_path)- blueprint.source = _dependency_bundle_source(- package_address=package_address, package_root=resolved_dep.package_root, mthds_path=mthds_path- )except (FileNotFoundError, MthdsParserError) as exc:log.warning(f"Could not parse dependency '{alias}' bundle '{mthds_path}': {exc}")continue+ bundle_source = _dependency_bundle_source(package_address=package_address, package_root=resolved_dep.package_root, mthds_path=mthds_path)+ blueprint.source = bundle_source+ bundle_files_by_source[bundle_source] = mthds_pathdep_blueprints.append(blueprint)if not dep_blueprints:@@ -1318,13 +1324,20 @@ class LibraryManager(LibraryManagerAbstract):# If manifest has exports, only load exported pipesif has_exports and pipe_code not in all_exported:continue+ dependency_source = crate.source_map.get(pipe_ref)+ # A pipe that reads a file beside its bundle (a PipeDocGen `template_file`) finds it from the+ # bundle's file on this host, which its blueprint carries into the factory and no further. The+ # package's address stays the source every refusal names, since the host's path is not the caller's.+ bundle_file = bundle_files_by_source.get(dependency_source) if dependency_source is not None else None+ if bundle_file is not None and pipe_blueprint.source is None:+ pipe_blueprint = pipe_blueprint.model_copy(update={"source": str(bundle_file)})try:# The same location the main load path attaches, so a dependency pipe's refusal# names the dependency's own file.with _locating_pipe_build_refusals(pipe_code=pipe_code,domain_code=domain_code,- source=crate.source_map.get(pipe_ref),+ source=dependency_source,elaboration=crate.elaboration_metadata.get(pipe_ref),):pipe = PipeFactory[PipeAbstract].make_from_blueprint(diff --git a/pipelex/migration/goldens/inference-backend/[email protected] b/pipelex/migration/goldens/inference-backend/[email protected]index 525439d99..55a738271 100644--- a/pipelex/migration/goldens/inference-backend/[email protected]+++ b/pipelex/migration/goldens/inference-backend/[email protected]@@ -62,6 +62,7 @@"value_type": "enum","required": false,"enum_members": [+ "doc_gen","img_gen","llm","search",diff --git a/pipelex/mthds_parsing/pipelex_bundle_blueprint.py b/pipelex/mthds_parsing/pipelex_bundle_blueprint.pyindex 3fcea60fd..95d4d9d3a 100644--- a/pipelex/mthds_parsing/pipelex_bundle_blueprint.py+++ b/pipelex/mthds_parsing/pipelex_bundle_blueprint.py@@ -19,6 +19,7 @@ from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenfrom pipelex.pipe_machinery.pipe_blueprint import normalize_typeless_signature_sectionfrom pipelex.pipe_machinery.validation import is_pipe_code_validfrom pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint+from pipelex.pipe_operators.doc_gen.pipe_doc_gen_blueprint import PipeDocGenBlueprintfrom pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprintfrom pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprintfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint@@ -61,6 +62,7 @@ PipeBlueprintUnion = Annotated[| PipeExtractBlueprint| PipeSearchBlueprint| PipeStructureBlueprint+ | PipeDocGenBlueprint| PipeBatchBlueprint| PipeConditionBlueprint| PipeParallelBlueprintdiff --git a/pipelex/pipe_machinery/pipe_blueprint.py b/pipelex/pipe_machinery/pipe_blueprint.pyindex 30483c2ff..994716c47 100644--- a/pipelex/pipe_machinery/pipe_blueprint.py+++ b/pipelex/pipe_machinery/pipe_blueprint.py@@ -132,6 +132,7 @@ class PipeType(StrEnum):PIPE_EXTRACT = "PipeExtract"PIPE_SEARCH = "PipeSearch"PIPE_STRUCTURE = "PipeStructure"+ PIPE_DOC_GEN = "PipeDocGen"# Pipe ControllerPIPE_BATCH = "PipeBatch"PIPE_CONDITION = "PipeCondition"@@ -159,6 +160,8 @@ class PipeType(StrEnum):return PipeCategory.PIPE_OPERATORcase PipeType.PIPE_STRUCTURE:return PipeCategory.PIPE_OPERATOR+ case PipeType.PIPE_DOC_GEN:+ return PipeCategory.PIPE_OPERATORcase PipeType.PIPE_BATCH:return PipeCategory.PIPE_CONTROLLERcase PipeType.PIPE_CONDITION:diff --git a/pipelex/pipe_machinery/registry_models.py b/pipelex/pipe_machinery/registry_models.pyindex 67eaf6f9c..a2203bb10 100644--- a/pipelex/pipe_machinery/registry_models.py+++ b/pipelex/pipe_machinery/registry_models.py@@ -12,6 +12,8 @@ from pipelex.pipe_machinery.pipe_abstract import PipeAbstractTypefrom pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocolfrom pipelex.pipe_operators.compose.pipe_compose import PipeComposefrom pipelex.pipe_operators.compose.pipe_compose_factory import PipeComposeFactory+from pipelex.pipe_operators.doc_gen.pipe_doc_gen import PipeDocGen+from pipelex.pipe_operators.doc_gen.pipe_doc_gen_factory import PipeDocGenFactoryfrom pipelex.pipe_operators.extract.pipe_extract import PipeExtractfrom pipelex.pipe_operators.extract.pipe_extract_factory import PipeExtractFactoryfrom pipelex.pipe_operators.func.pipe_func import PipeFunc@@ -47,6 +49,7 @@ class PipeRegistryModels(RegistryModels):PipeExtract,PipeSearch,PipeStructure,+ PipeDocGen,]PIPE_OPERATORS_FACTORY: ClassVar[list[PipeFactoryProtocol[Any, Any]]] = [@@ -57,6 +60,7 @@ class PipeRegistryModels(RegistryModels):PipeExtractFactory,PipeSearchFactory,PipeStructureFactory,+ PipeDocGenFactory,]PIPE_CONTROLLERS: ClassVar[list[PipeAbstractType]] = [diff --git a/pipelex/pipe_machinery/rendering/output_renderer.py b/pipelex/pipe_machinery/rendering/output_renderer.pyindex f8a5dcea4..390eaf4e8 100644--- a/pipelex/pipe_machinery/rendering/output_renderer.py+++ b/pipelex/pipe_machinery/rendering/output_renderer.py@@ -116,6 +116,7 @@ def _collect_possible_outputs(| PipeType.PIPE_EXTRACT| PipeType.PIPE_SEARCH| PipeType.PIPE_STRUCTURE+ | PipeType.PIPE_DOC_GEN| PipeType.PIPE_BATCH| PipeType.PIPE_PARALLEL):diff --git a/pipelex/pipe_operators/doc_gen/__init__.py b/pipelex/pipe_operators/doc_gen/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/doc_gen/exceptions.py b/pipelex/pipe_operators/doc_gen/exceptions.pynew file mode 100644index 000000000..da0e93915--- /dev/null+++ b/pipelex/pipe_operators/doc_gen/exceptions.py@@ -0,0 +1,44 @@+from pipelex.base_exceptions import ErrorDomain, PipelexError+++class PipeDocGenFactoryError(PipelexError):+ """Raised when a ``PipeDocGen`` cannot be built from its blueprint.++ The output concept is not a Document, a template does not parse or reads a field its input does not have,+ or the template file is missing or cannot be resolved. It is raised only while the pipe is built, about the+ author's own blueprint, so it is the author's to fix and its messages name only the author's pipe, concepts,+ fields, template and file path.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = True+++class PipeDocGenRunError(PipelexError):+ """Raised when a ``PipeDocGen`` step fails while it composes its document: a template or the filename does not render."""+++class PipeDocGenUndefinedValueError(PipelexError):+ """Raised when a ``PipeDocGen`` template or filename reads a value its inputs do not have.++ Its templates render with a strict undefined, so a field the load-time check could not follow, such as one+ read through a ``{% set %}`` alias, fails here rather than printing as empty text, at the dry run when the+ template names a field its concept lacks. The template is always the method author's own, unlike the+ renders ``Jinja2TemplateRenderError`` also serves, so the message is the author's to read: it names the pipe,+ the template or the filename, and the missing name, and it never quotes the template's source.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = True+++class PipeDocGenTemplateCheckError(PipelexError):+ """Raised by a ``PipeDocGen`` dry run when the engine's template checker reports errors in the step's template file.++ The dry run is what ``pipelex validate`` runs, so a template naming a field the inputs do not have fails there,+ before anything is spent. Its message lists the checker's findings, which name only the author's template,+ names and fields.+ """++ error_domain = ErrorDomain.INPUT+ _authors_caller_facing_message = Truediff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen.pynew file mode 100644index 000000000..1cf2f2822--- /dev/null+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen.py@@ -0,0 +1,393 @@+import asyncio+import re+from pathlib import Path+from typing import Any, Literal++from jinja2.exceptions import UndefinedError+from pydantic import Field+from typing_extensions import override++from pipelex import log+from pipelex.cogt.doc_gen.doc_gen_engine import require_doc_gen_engine_declared, resolve_doc_gen_setting+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice, DocGenSetting+from pipelex.cogt.doc_gen.doc_gen_worker_factory import DocGenWorkerFactory+from pipelex.cogt.doc_gen.document_composition import DocumentComposition+from pipelex.cogt.doc_gen.input_shape import InputShape, shape_of_input+from pipelex.cogt.doc_gen.layout_builder import build_layout_document, humanize+from pipelex.cogt.doc_gen.plain_data import plain_data+from pipelex.cogt.doc_gen.template_check import TemplateCheckRequest+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck_check import check_doc_gen_choice_with_deck+from pipelex.cogt.templating.template_preprocessor import rewrite_template_sigils+from pipelex.cogt.templating.template_rendering import render_template+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs+from pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.kernel.compose_ops import build_compose_context+from pipelex.kernel.memory_ops import store_result+from pipelex.kernel.templating_style_ops import resolve_templating_style+from pipelex.pipe_machinery.template_guard_lint import lint_authored_template+from pipelex.pipe_operators.doc_gen.exceptions import PipeDocGenRunError, PipeDocGenTemplateCheckError, PipeDocGenUndefinedValueError+from pipelex.pipe_operators.doc_gen.template_field_check import check_template_field_paths+from pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipe_run.pipe_run_params import PipeRunParams+from pipelex.runtime_hub import get_class_registry, get_content_generator, get_model_deck, get_report_delegate+from pipelex.system.job_metadata import JobMetadata+from pipelex.tools.jinja2.exceptions import Jinja2TemplateRenderError+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.misc.string_utils import get_root_from_dotted_path+from pipelex.tools.templating.templating_style import TemplatingStyle++_UNSAFE_FILENAME_CHARACTERS = re.compile(r"[^\w.\-]+")+_MAX_FILENAME_STEM_LENGTH = 120+++def safe_filename(*, rendered: str, fallback: str, suffix: str) -> str:+ """One storage-safe path segment ending in the format's suffix, from whatever the filename template rendered."""+ stem = _UNSAFE_FILENAME_CHARACTERS.sub("-", rendered.strip()).strip(".-_")+ if stem.lower().endswith(f".{suffix}"):+ stem = stem[: -(len(suffix) + 1)]+ stem = stem[:_MAX_FILENAME_STEM_LENGTH].strip(".-_") or fallback+ return f"{stem}.{suffix}"+++class PipeDocGenOutput(PipeOutput):+ pass+++class PipeDocGen(PipeOperator[PipeDocGenOutput]):+ """Turn the step's inputs into a stored document file, in two stages. It calls no AI model.++ The **compose stage** is pure: it builds the layout tree of the inputs, or renders the HTML template, or+ gathers the inputs as plain data for a template file, and it renders the filename, every template with a+ strict undefined so a missing field fails rather than printing as empty text. The **print stage** is the+ content generator's `make_rendered_document`: the step's engine prints the composition, and the bytes are+ stored through the run's storage provider as a `DocumentContent`. The engine is a `doc_gen` model, the one the+ step names or the model deck's default for its format and source, checked when the method loads. A dry run+ composes for real, which checks the templates against mock inputs, runs the engine's template checker on a+ template file, and prints and stores nothing.+ """++ type: Literal["PipeDocGen"] = "PipeDocGen"+ doc_gen_format: DocGenFormat = Field(strict=False)+ doc_gen_choice: DocGenModelChoice | None = Field(default=None, description="The engine the step names, or None for the deck's default")+ template: str | None = Field(default=None, description="The HTML template of a pdf step, inline or read from its template file")+ template_file: str | None = Field(default=None, description="The template file as the method names it")+ template_path: str | None = Field(default=None, description="The resolved path of an office template file, which its engine fills")+ filename: str | None = None++ @property+ def source(self) -> DocGenSource:+ return DocGenSource.for_step(doc_gen_format=self.doc_gen_format, has_template=self.template is not None or self.template_path is not None)++ @override+ def needed_inputs(self, *, visited_pipes: set[str] | None = None) -> InputStuffSpecs:+ return self.inputs++ @override+ def required_variables(self) -> set[str]:+ roots: set[str] = set()+ for template_source, category in ((self.template, TemplateCategory.HTML), (self.filename, TemplateCategory.BASIC)):+ if template_source is None:+ continue+ for path in detect_jinja2_required_variables(template_category=category, template_source=rewrite_template_sigils(template_source)):+ root = get_root_from_dotted_path(path)+ if not root.startswith("_") and root != "place_holder":+ roots.add(root)+ return roots++ def resolve_engine(self) -> DocGenSetting:+ """The engine this step prints on, resolved in the model deck and checked to print its format from its source here."""+ return resolve_doc_gen_setting(+ doc_gen_choice=self.doc_gen_choice, doc_gen_format=self.doc_gen_format, source=self.source, pipe_code=self.code+ )++ @override+ def validate_inputs_static(self):+ # The engine is resolved when the method loads, so a step no engine here prints is refused before a run spends anything.+ if self.doc_gen_choice is not None:+ with self.locating_model_choice(field_name="model"):+ # The built-in engine or one of the plugin's, named where this installation does not declare it, is refused+ # naming its remedy before the deck check would call it an unknown model.+ require_doc_gen_engine_declared(+ doc_gen_choice=self.doc_gen_choice, doc_gen_format=self.doc_gen_format, source=self.source, pipe_code=self.code+ )+ check_doc_gen_choice_with_deck(doc_gen_choice=self.doc_gen_choice)+ self.resolve_engine()++ # The same template lints as PipeCompose: no private names, and every reference to a declared-optional input+ # guarded. The filename gets both too, since it renders as strictly as the template does.+ if self.template is not None:+ lint_authored_template(+ pipe_code=self.code,+ domain_code=self.domain_code,+ inputs=self.inputs,+ template_source=self.template,+ template_category=TemplateCategory.HTML,+ template_label=f"template file '{self.template_file}'" if self.template_file is not None else "template",+ )+ if self.filename is not None:+ lint_authored_template(+ pipe_code=self.code,+ domain_code=self.domain_code,+ inputs=self.inputs,+ template_source=self.filename,+ template_category=TemplateCategory.BASIC,+ template_label="filename",+ )++ @override+ def validate_inputs_with_library(self):+ # Every field a template reads exists on its input's concept, which is known once the library is loaded.+ if self.template is not None:+ check_template_field_paths(+ pipe_code=self.code,+ template_source=self.template,+ template_category=TemplateCategory.HTML,+ label=f"template file '{self.template_file}'" if self.template_file is not None else "template",+ inputs=self.inputs,+ )+ if self.filename is not None:+ check_template_field_paths(+ pipe_code=self.code,+ template_source=self.filename,+ template_category=TemplateCategory.BASIC,+ label="filename",+ inputs=self.inputs,+ )++ @override+ def validate_output_static(self):+ pass++ @override+ def validate_output_with_library(self):+ pass++ ####################################################################################################################+ # Compose stage+ ####################################################################################################################++ def _named_contents(self, working_memory: WorkingMemory) -> list[tuple[str, StuffContent]]:+ """The declared inputs present in memory, in declared order; an absent optional input is skipped."""+ named_contents: list[tuple[str, StuffContent]] = []+ for input_name in self.inputs.variables:+ stuff = working_memory.get_optional_stuff(input_name)+ if stuff is not None:+ named_contents.append((input_name, stuff.content))+ return named_contents++ def _title(self, named_contents: list[tuple[str, StuffContent]]) -> str:+ """The document's title: its one input's name, or the step's code when it has several."""+ if len(named_contents) == 1:+ return humanize(named_contents[0][0])+ return humanize(self.code)++ async def _render(+ self, *, template: str, category: TemplateCategory, context: dict[str, Any], label: str, templating_style: TemplatingStyle | None = None+ ) -> str:+ try:+ return await render_template(+ template=template,+ category=category,+ context=context,+ templating_style=templating_style,+ is_undefined_strict=True,+ )+ except Jinja2TemplateRenderError as exc:+ undefined_error = exc.__cause__+ if isinstance(undefined_error, UndefinedError):+ # Raised from Jinja's own error rather than from the tools-layer wrapper, whose message quotes the+ # template and is not caller-facing, so that this is the root fault a verdict or a run report names.+ msg = (+ f"PipeDocGen '{self.code}' could not render its {label}: {undefined_error.message}. "+ "Every value a document's template reads must exist: check the name against the input's concept, "+ "or test an optional value first with {% if ... %}."+ )+ raise PipeDocGenUndefinedValueError(msg) from undefined_error+ msg = f"PipeDocGen '{self.code}' could not render its {label}: {exc}"+ raise PipeDocGenRunError(msg) from exc++ async def compose(self, *, working_memory: WorkingMemory, pipe_run_params: PipeRunParams) -> DocumentComposition:+ """The pure half of the step: everything the print stage needs, and nothing printed yet."""+ context = build_compose_context(memory=working_memory, runtime_params=pipe_run_params.params)+ rendered_filename = self.code+ if self.filename is not None:+ # A `$name` sigil in the filename expands to the `format` filter, which prints under the render's templating style.+ rendered_filename = await self._render(+ template=self.filename,+ category=TemplateCategory.BASIC,+ context=context,+ label="filename",+ templating_style=resolve_templating_style(authored=None),+ )+ filename = safe_filename(rendered=rendered_filename, fallback=self.code, suffix=self.doc_gen_format.suffix)+ named_contents = self._named_contents(working_memory)+ title = self._title(named_contents)+ source = self.source+ match source:+ case DocGenSource.LAYOUT:+ return DocumentComposition(+ format=self.doc_gen_format,+ source=source,+ filename=filename,+ title=title,+ layout=build_layout_document(title=title, named_contents=named_contents),+ )+ case DocGenSource.HTML:+ if self.template is None:+ msg = f"PipeDocGen '{self.code}' composes from HTML but has no HTML template."+ raise PipeDocGenRunError(msg)+ html = await self._render(+ template=self.template,+ category=TemplateCategory.HTML,+ context=context,+ label="template",+ templating_style=resolve_templating_style(authored=None),+ )+ return DocumentComposition(format=self.doc_gen_format, source=source, filename=filename, title=title, html=html)+ case DocGenSource.TEMPLATE_FILE:+ return DocumentComposition(+ format=self.doc_gen_format,+ source=source,+ filename=filename,+ title=title,+ template_path=self.template_path,+ template_name=self.template_file,+ data={name: plain_data(content) for name, content in named_contents},+ )++ ####################################################################################################################+ # Template file check, at the dry run+ ####################################################################################################################++ def _input_shapes(self) -> dict[str, InputShape]:+ shapes: dict[str, InputShape] = {}+ for input_name, stuff_spec in self.inputs.root.items():+ content_class: type[StuffContent] | None = None+ if stuff_spec.concept.declares_a_structure_class:+ content_class = get_class_registry().get_required_subclass(name=stuff_spec.concept.structure_class_name, base_class=StuffContent)+ shapes[input_name] = shape_of_input(content_class=content_class, is_list=stuff_spec.is_multiple())+ return shapes++ async def check_template_file(self, *, working_memory: WorkingMemory) -> None:+ """Run the engine's template checker on the step's template file, when its engine has one.++ The inputs in memory are the dry run's mocks, so the checker can fill the template in memory with them.++ Raises:+ PipeDocGenTemplateCheckError: the checker reported errors.+ """+ if self.template_path is None or self.template_file is None:+ return+ doc_gen_setting = self.resolve_engine()+ inference_model = get_model_deck().get_required_inference_model(model_handle=doc_gen_setting.model, model_type=ModelType.DOC_GEN)+ worker = DocGenWorkerFactory.make_doc_gen_worker(inference_model=inference_model, reporting_delegate=get_report_delegate())+ request = TemplateCheckRequest(+ format=self.doc_gen_format,+ template=Path(self.template_path).read_bytes(),+ template_name=self.template_file,+ inputs=self._input_shapes(),+ data={name: plain_data(content) for name, content in self._named_contents(working_memory)},+ )+ findings = await asyncio.to_thread(worker.check_template, request=request)+ errors = [finding for finding in findings if finding.severity.is_error]+ for finding in findings:+ if not finding.severity.is_error:+ location = f" ({finding.location})" if finding.location else ""+ log.warning(f"PipeDocGen '{self.code}', template file '{self.template_file}'{location}: {finding.message}")+ if errors:+ listed = "\n".join(f"- {finding.message}" + (f" ({finding.location})" if finding.location else "") for finding in errors)+ msg = f"PipeDocGen '{self.code}': the template file '{self.template_file}' does not fit the step's inputs:\n{listed}"+ raise PipeDocGenTemplateCheckError(msg)++ ####################################################################################################################+ # Runs+ ####################################################################################################################++ def _store_document(+ self,+ *,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ document: DocumentContent,+ doc_gen_setting: DocGenSetting,+ output_name: str | None,+ ) -> PipeDocGenOutput:+ # The output concept refines Document, so its class is DocumentContent or a subclass of it.+ output_class = get_class_registry().get_required_subclass(name=self.output.concept.structure_class_name, base_class=DocumentContent)+ content = output_class.model_validate(document.model_dump())+ working_memory = store_result(memory=working_memory, concept=self.output.concept, content=content, result_name=output_name)+ self._register_execution_data(+ job_metadata=job_metadata,+ execution_data={+ "format": self.doc_gen_format,+ "source": self.source,+ "resolved_model": doc_gen_setting.model,+ "filename": document.filename,+ "url": document.url,+ },+ )+ return PipeDocGenOutput(working_memory=working_memory, pipeline_run_id=job_metadata.run_metadata.pipeline_run_id)++ @override+ async def _live_run_operator_pipe(+ self,+ *,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeDocGenOutput:+ composition = await self.compose(working_memory=working_memory, pipe_run_params=pipe_run_params)+ doc_gen_setting = self.resolve_engine()+ document = await get_content_generator().make_rendered_document(+ job_metadata=job_metadata,+ cogt_run_params=pipe_run_params.cogt_run_params,+ composition=composition,+ doc_gen_setting=doc_gen_setting,+ )+ log.verbose(f"PipeDocGen '{self.code}' stored {document.filename} at {document.url}")+ return self._store_document(+ job_metadata=job_metadata,+ working_memory=working_memory,+ document=document,+ doc_gen_setting=doc_gen_setting,+ output_name=output_name,+ )++ @override+ async def _dry_run_operator_pipe(+ self,+ *,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeDocGenOutput:+ # The live path with the dry mode riding into the leaf: the compose stage runs for real against the mock+ # inputs, which checks the templates, and the leaf prints and stores nothing. The template file's checker+ # runs first, since only a dry run has inputs to fill it with and nothing to spend.+ await self.check_template_file(working_memory=working_memory)+ return await self._live_run_operator_pipe(+ job_metadata=job_metadata,+ working_memory=working_memory,+ pipe_run_params=pipe_run_params,+ output_name=output_name,+ )++ @override+ async def _validate_before_run(+ self, *, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ pass++ @override+ async def _validate_after_run(+ self, *, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None+ ):+ passdiff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.pynew file mode 100644index 000000000..11044cd87--- /dev/null+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.py@@ -0,0 +1,127 @@+from typing import Literal, Self++from pydantic import Field, model_validator+from typing_extensions import override++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice+from pipelex.cogt.templating.exceptions import TemplateSigilSyntaxError+from pipelex.cogt.templating.template_preprocessor import preprocess_template+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity+from pipelex.pipe_machinery.pipe_blueprint import PipeBlueprint+from pipelex.tools.jinja2.exceptions import Jinja2TemplateSyntaxError+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.misc.string_utils import get_root_from_dotted_path+++def template_required_roots(*, template_source: str, template_category: TemplateCategory, declared_inputs: set[str], label: str) -> set[str]:+ """Check that a `PipeDocGen` template parses, and return the input names it reads.++ Sigils are rewritten first, exactly as PipeCompose does, so `$invoice` and `@invoice` count as reads of+ `invoice`. Raises `ValueError`, which pydantic turns into a validation error located on the blueprint.+ """+ try:+ preprocessed = preprocess_template(template_source, declared_inputs=declared_inputs)+ except TemplateSigilSyntaxError as exc:+ msg = f"Template sigil error in the PipeDocGen {label}: {exc}"+ raise ValueError(msg) from exc+ try:+ check_jinja2_parsing(template_source=preprocessed, template_category=template_category)+ except Jinja2TemplateSyntaxError as exc:+ msg = f"Could not parse the PipeDocGen {label}: {exc}"+ raise ValueError(msg) from exc+ roots: set[str] = set()+ for path in detect_jinja2_required_variables(template_category=template_category, template_source=preprocessed):+ root = get_root_from_dotted_path(path)+ if not root.startswith("_") and root != "place_holder":+ roots.add(root)+ return roots+++class PipeDocGenBlueprint(PipeBlueprint):+ """Generate a document file from a method's structured result: a PDF, an Excel workbook, a Word document or a PowerPoint deck.++ It calls no AI model: it lays out or fills what its inputs already hold. With no template, the pipe lays its inputs+ out by itself (the auto-layout), for every format but `pptx`. `template` is an inline HTML and Jinja2 template, for+ `pdf` only; `template_file` is a path relative to the bundle's file: an `.html` for `pdf`, an `.xlsx` for `xlsx`, a+ `.docx` for `docx` and a `.pptx` for `pptx`, which `pptx` requires. `filename` is a Jinja expression over the+ inputs; the suffix is added. The output must be a single `Document`, or a concept refining it. `model` names the+ document engine that prints it, a model of the `doc_gen` family such as `reportlab-pdf` or, with the Pipelex+ document generation plugin, `pipelex-pdf`; without it, the model deck's default for the format and source prints+ it. Which engines a runtime has depends on its plugins: open Pipelex prints a `pdf` without a template on+ `reportlab-pdf`, and a runtime without the engine a step needs refuses the method when it loads.+ """++ type: Literal["PipeDocGen"] = "PipeDocGen"+ pipe_category: Literal["PipeOperator"] = "PipeOperator"++ format: DocGenFormat = Field(strict=False)+ model: DocGenModelChoice | None = None+ template: str | None = None+ template_file: str | None = None+ filename: str | None = None++ @model_validator(mode="after")+ def validate_template_sources(self) -> Self:+ if self.template is not None and self.template_file is not None:+ msg = "PipeDocGen cannot have both 'template' and 'template_file': use one or the other."+ raise ValueError(msg)+ if self.template is not None and not self.format.is_template_html:+ msg = (+ f"PipeDocGen's inline 'template' is an HTML template, for the 'pdf' format only. "+ f"For '{self.format}', use 'template_file' with a {self.format.template_file_suffix} file"+ f"{', or no template for the auto-layout' if self.format.has_auto_layout else ''}."+ )+ raise ValueError(msg)+ if self.template_file is not None and not self.template_file.lower().endswith(self.format.template_file_suffix):+ msg = (+ f"PipeDocGen's 'template_file' for the '{self.format}' format must be a {self.format.template_file_suffix} file, "+ f"got '{self.template_file}'."+ )+ raise ValueError(msg)+ if not self.format.has_auto_layout and self.template_file is None:+ msg = (+ f"PipeDocGen's '{self.format}' format has no auto-layout: name the designed {self.format.template_file_suffix} "+ "file it fills with 'template_file'."+ )+ raise ValueError(msg)+ return self++ @override+ def validate_inputs(self):+ declared_inputs = set(self.input_names)+ if self.template is not None:+ self._check_roots_are_inputs(+ roots=template_required_roots(+ template_source=self.template,+ template_category=TemplateCategory.HTML,+ declared_inputs=declared_inputs,+ label="template",+ ),+ label="template",+ )+ if self.filename is not None:+ self._check_roots_are_inputs(+ roots=template_required_roots(+ template_source=self.filename,+ template_category=TemplateCategory.BASIC,+ declared_inputs=declared_inputs,+ label="filename",+ ),+ label="filename",+ )++ def _check_roots_are_inputs(self, *, roots: set[str], label: str):+ for root in sorted(roots):+ if root not in self.input_names:+ msg = f"Variable '{root}' in the PipeDocGen {label} is not in the inputs of the pipe."+ raise ValueError(msg)++ @override+ def validate_output(self):+ parsed_output = parse_concept_with_multiplicity(concept_ref_or_code=self.output)+ if parsed_output.multiplicity:+ msg = f"PipeDocGen produces one file: its output must be a single Document, or a concept refining Document. Current output: {self.output}"+ raise ValueError(msg)diff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.pynew file mode 100644index 000000000..58cbf5c38--- /dev/null+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py@@ -0,0 +1,122 @@+from pathlib import Path+from typing import Any++from typing_extensions import override++from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs+from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec+from pipelex.interpreter_hub import get_concept_library, get_native_concept+from pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocol+from pipelex.pipe_operators.doc_gen.exceptions import PipeDocGenFactoryError+from pipelex.pipe_operators.doc_gen.pipe_doc_gen import PipeDocGen+from pipelex.pipe_operators.doc_gen.pipe_doc_gen_blueprint import PipeDocGenBlueprint, template_required_roots+from pipelex.tools.jinja2.template_category import TemplateCategory+++def bundle_dir_of_source(source: str | None) -> Path | None:+ """The directory of the bundle file a pipe was loaded from, or None for a bundle loaded from a string.++ A content-string load, such as the hosted API's `api://bundle-0.mthds`, has no directory, so nothing beside+ the bundle can be found from it.+ """+ if source is None or "://" in source:+ return None+ source_path = Path(source)+ if not source_path.is_file():+ return None+ return source_path.resolve().parent+++class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):+ @classmethod+ @override+ def make(+ cls,+ *,+ pipe_category: Any,+ pipe_type: str,+ pipe_code: str,+ domain_code: str,+ description: str,+ inputs: InputStuffSpecs,+ output: StuffSpec,+ blueprint: PipeDocGenBlueprint,+ ) -> PipeDocGen:+ document_concept = get_native_concept(native_concept=NativeConceptCode.DOCUMENT)+ if not get_concept_library().is_compatible(tested_concept=output.concept, wanted_concept=document_concept, strict=True):+ msg = (+ f"The output of PipeDocGen '{pipe_code}' is '{output.concept.concept_ref}', which is not a Document. "+ f"Declare the output as {document_concept.concept_ref}, or as a concept that refines it."+ )+ raise PipeDocGenFactoryError(msg)++ template = blueprint.template+ template_path: str | None = None+ if blueprint.template_file is not None:+ resolved_path = cls._resolve_template_file(pipe_code=pipe_code, template_file=blueprint.template_file, source=blueprint.source)+ if blueprint.format.is_template_html:+ try:+ template = resolved_path.read_text(encoding="utf-8")+ except (OSError, UnicodeError) as exc:+ msg = f"PipeDocGen '{pipe_code}' could not read the template file '{blueprint.template_file}' as UTF-8 text."+ raise PipeDocGenFactoryError(msg) from exc+ cls._check_html_template_file(pipe_code=pipe_code, template=template, inputs=inputs, template_file=blueprint.template_file)+ else:+ template_path = str(resolved_path)++ return PipeDocGen(+ domain_code=domain_code,+ code=pipe_code,+ description=description,+ inputs=inputs,+ output=output,+ doc_gen_format=blueprint.format,+ doc_gen_choice=blueprint.model,+ template=template,+ template_file=blueprint.template_file,+ template_path=template_path,+ filename=blueprint.filename,+ )++ @classmethod+ def _resolve_template_file(cls, *, pipe_code: str, template_file: str, source: str | None) -> Path:+ """The template file's path, beside the bundle and inside its directory, checked to exist.++ The messages name the template file as the method does and never the host's path to it: the bundle a+ refusal is located in already says which one, and a dependency's install path is not the caller's to read.+ """+ bundle_dir = bundle_dir_of_source(source)+ if bundle_dir is None:+ msg = (+ f"PipeDocGen '{pipe_code}' names the template file '{template_file}', but its bundle was not loaded from a file on disk, "+ "so there is no directory to find the template in. Template files are not supported there yet: they work for bundles "+ "loaded from a directory. Use an inline 'template' for a pdf, or no template for the auto-layout."+ )+ raise PipeDocGenFactoryError(msg)+ resolved_path = (bundle_dir / template_file).resolve()+ if not resolved_path.is_relative_to(bundle_dir):+ msg = f"PipeDocGen '{pipe_code}' names the template file '{template_file}', which is outside its bundle's directory."+ raise PipeDocGenFactoryError(msg)+ if not resolved_path.is_file():+ msg = f"PipeDocGen '{pipe_code}' names the template file '{template_file}', which does not exist beside its bundle."+ raise PipeDocGenFactoryError(msg)+ return resolved_path++ @classmethod+ def _check_html_template_file(cls, *, pipe_code: str, template: str, inputs: InputStuffSpecs, template_file: str) -> None:+ """Check an HTML template file at load, as an inline template is checked when its blueprint validates."""+ declared_inputs = set(inputs.variables)+ try:+ roots = template_required_roots(+ template_source=template,+ template_category=TemplateCategory.HTML,+ declared_inputs=declared_inputs,+ label=f"template file '{template_file}'",+ )+ except ValueError as exc:+ msg = f"PipeDocGen '{pipe_code}': {exc}"+ raise PipeDocGenFactoryError(msg) from exc+ for root in sorted(roots - declared_inputs):+ msg = f"Variable '{root}' in the template file '{template_file}' of PipeDocGen '{pipe_code}' is not in the inputs of the pipe."+ raise PipeDocGenFactoryError(msg)diff --git a/pipelex/pipe_operators/doc_gen/template_field_check.py b/pipelex/pipe_operators/doc_gen/template_field_check.pynew file mode 100644index 000000000..e57f55819--- /dev/null+++ b/pipelex/pipe_operators/doc_gen/template_field_check.py@@ -0,0 +1,135 @@+"""The load-time check that every field a `PipeDocGen` template reads exists on its input's concept.++The blueprint checks that each variable a template reads is a declared input; this goes further, down each+path: `invoice.nosuch` is refused when the method loads, and so is `item.nosuch` in a loop over+`invoice.line_items`, which is followed back to the list's item concept. A path is checked only as far as the+concept's classes say what it holds: once it reaches a text, a number, a JSON value or a property or method of a+class, the check stops, and anything the static check cannot follow is caught by the strict undefined when the+template renders at the dry run.+"""++import types+import typing+from collections.abc import Sequence+from typing import Any, Union++from pydantic import BaseModel++from pipelex.cogt.templating.template_preprocessor import rewrite_template_sigils+from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.stuff_artefact import StuffArtefact+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.pipe_operators.doc_gen.exceptions import PipeDocGenFactoryError+from pipelex.runtime_hub import get_class_registry+from pipelex.tools.jinja2.jinja2_field_paths import LIST_ITEM_SEGMENT, TemplateFieldPath, detect_template_field_paths+from pipelex.tools.jinja2.template_category import TemplateCategory++_NONE_TYPE = type(None)+++def _unwrap(annotation: Any) -> Any:+ """An optional or annotated type's own type; any other union is returned as is, and stops the check."""+ origin = typing.get_origin(annotation)+ if origin is typing.Annotated:+ return _unwrap(typing.get_args(annotation)[0])+ if origin in {Union, types.UnionType}:+ members = [member for member in typing.get_args(annotation) if member is not _NONE_TYPE]+ if len(members) == 1:+ return _unwrap(members[0])+ return annotation+++def _list_item_type(annotation: Any) -> Any | None:+ if typing.get_origin(annotation) in {list, tuple, set, frozenset, Sequence}:+ item_args = typing.get_args(annotation)+ return item_args[0] if item_args else None+ return None+++def _first_missing_field(*, start: Any, segments: tuple[str, ...]) -> tuple[int, type[BaseModel]] | None:+ """The index of the first segment naming no field of the class it is read on, with that class; None when the path holds.++ A segment the classes cannot answer for stops the walk without a finding.+ """+ current: Any = start+ for index, segment in enumerate(segments):+ current = _unwrap(current)+ if segment == LIST_ITEM_SEGMENT:+ item_type = _list_item_type(current)+ if item_type is None:+ return None+ current = item_type+ continue+ if not isinstance(current, type) or not issubclass(current, BaseModel):+ return None+ if current.model_config.get("extra") == "allow":+ return None+ field_info = current.model_fields.get(segment)+ if field_info is not None:+ current = field_info.annotation+ continue+ if hasattr(current, segment):+ # A property or a method of the class: what it returns is not described, so the check stops.+ return None+ return index, current+ return None+++def _describe_fields(model_class: type[BaseModel]) -> str:+ return ", ".join(model_class.model_fields) or "(none)"+++def _check_path(*, path: TemplateFieldPath, inputs: InputStuffSpecs, pipe_code: str, label: str) -> None:+ stuff_spec = inputs.root.get(path.root)+ if stuff_spec is None or not path.segments:+ return+ if not stuff_spec.concept.declares_a_structure_class:+ # An `Anything` input declares no fields to check a path against; the strict undefined catches a bad read at the dry run.+ return+ content_class = get_class_registry().get_required_subclass(name=stuff_spec.concept.structure_class_name, base_class=StuffContent)+ first_segment = path.segments[0]+ if stuff_spec.is_multiple():+ # A list input iterates over its items, and otherwise reads as the list stuff itself.+ if first_segment == LIST_ITEM_SEGMENT:+ start: Any = list[content_class] # type: ignore[valid-type]+ else:+ start = ListContent+ else:+ start = content_class+ if first_segment != LIST_ITEM_SEGMENT and (first_segment.startswith("_") or hasattr(StuffArtefact, first_segment)):+ is_content_field = isinstance(start, type) and issubclass(start, BaseModel) and first_segment in start.model_fields+ if not is_content_field:+ # The artefact's own accessors and metadata (`get`, `_stuff_name`), which templates may read too.+ return+ missing = _first_missing_field(start=start, segments=path.segments)+ if missing is None:+ return+ index, model_class = missing+ missing_field = path.segments[index]+ if index == 0:+ holder = f"the concept '{stuff_spec.concept.concept_ref}' of input '{path.root}'"+ else:+ holder = f"'{model_class.__name__}'"+ reached = ""+ loop_variable = path.written.split(".")[0]+ if loop_variable != path.root:+ iterated = ".".join(segment for segment in path.segments[:index] if segment != LIST_ITEM_SEGMENT)+ reached = f" ({loop_variable} is an item of {path.root}.{iterated})"+ msg = (+ f"PipeDocGen '{pipe_code}': the {label} reads '{path.written}'{reached}, but {holder} has no field '{missing_field}'. "+ f"Its fields are: {_describe_fields(model_class)}."+ )+ raise PipeDocGenFactoryError(msg)+++def check_template_field_paths(+ *, pipe_code: str, template_source: str, template_category: TemplateCategory, label: str, inputs: InputStuffSpecs+) -> None:+ """Refuse a template that reads a field its input's concept does not have.++ Raises:+ PipeDocGenFactoryError: a path names a missing field.+ """+ for path in detect_template_field_paths(template_source=rewrite_template_sigils(template_source), template_category=template_category):+ _check_path(path=path, inputs=inputs, pipe_code=pipe_code, label=label)diff --git a/pipelex/pipeline/hint_warnings.py b/pipelex/pipeline/hint_warnings.pyindex eda0f667b..237edd358 100644--- a/pipelex/pipeline/hint_warnings.py+++ b/pipelex/pipeline/hint_warnings.py@@ -60,15 +60,16 @@ def elide_authored_token(*, token: str) -> str:def _native_class_value_kind(*, class_name: str) -> HintSiteValueKind:"""The value kind of a class-backed structure, by native-class identity.- Mirrors the deriver's `_class_backed_node` lookup: `TextContent` is the native Text payload- (text-valued), `NumberContent` the native Number payload (number-valued). Every other class —- the remaining natives and registered project classes alike — is not a hint-applicable site.+ Mirrors the deriver's `_class_backed_node` lookup: `TextContent` is the native Text payload and+ `MarkdownContent` the native Markdown one, which refines Text (both text-valued, both `prose` in the+ descriptor), `NumberContent` the native Number payload (number-valued). Every other class — the+ remaining natives and registered project classes alike — is not a hint-applicable site.One divergence is open (L-260930-173b03): the deriver reflects a registered `RootModel` over ascalar as its root value, which this lookup still classifies as OTHER."""native_code = next((code for code in NativeConceptCode if code.structure_class_name == class_name), None)match native_code:- case NativeConceptCode.TEXT:+ case NativeConceptCode.TEXT | NativeConceptCode.MARKDOWN:return HintSiteValueKind.TEXT_VALUEDcase NativeConceptCode.NUMBER:return HintSiteValueKind.NUMBER_VALUED@@ -243,6 +244,11 @@ class _HintLinter:seen.add(current_ref)if NativeConceptCode.is_native_concept_ref_or_code(concept_ref_or_code=current_ref):native_ref = NativeConceptCode.get_validated_native_concept_ref(concept_ref_or_code=current_ref)+ refined_native = NativeConceptCode(QualifiedRef.parse(native_ref).local_code).refined_native+ if refined_native is not None:+ # A native refining another (`Markdown` refines `Text`) classifies like the native it refines.+ current_ref = refined_native.concept_ref+ continueif native_ref == _NATIVE_TEXT_REF:return HintSiteValueKind.TEXT_VALUEDif native_ref == _NATIVE_NUMBER_REF:diff --git a/pipelex/pipeline/input_form.py b/pipelex/pipeline/input_form.pyindex ff8237bf2..6e11940b2 100644--- a/pipelex/pipeline/input_form.py+++ b/pipelex/pipeline/input_form.py@@ -392,7 +392,8 @@ class InputFormDeriver:node_ref = concept_ref or native_code.concept_reftext = description or pinned.descriptionmatch native_code:- case NativeConceptCode.TEXT:+ case NativeConceptCode.TEXT | NativeConceptCode.MARKDOWN:+ # A Markdown input is prose too: its source is typed as text, and only the views format it.return ProseField(name=name, concept_ref=node_ref, refines=refines, description=text, required=True)case NativeConceptCode.NUMBER:return NumberField(name=name, concept_ref=node_ref, refines=refines, description=text, required=True, integer=False)diff --git a/pipelex/plugins/contract.py b/pipelex/plugins/contract.pyindex 9f6366a3b..9014fa046 100644--- a/pipelex/plugins/contract.py+++ b/pipelex/plugins/contract.py@@ -24,6 +24,15 @@ if TYPE_CHECKING:# ``add_log_sink`` — a third config-selected, process-global registry (``runtime.log.sink`` picks the# factory at boot) — joined the menu under v4 without a bump: a menu addition breaks no plugin that# targets v4, and a bump would have made every installed plugin re-declare ``targets_api`` for nothing.+#+# ``InferenceFamily.DOC_GEN`` — the document engines a ``PipeDocGen`` step prints with, each a model of the+# ``doc_gen`` family registered through ``add_inference_backend`` — joined under v4 on the same reasoning. Its+# worker (``DocGenWorkerAbstract``), render job and template check request (``pipelex.cogt.doc_gen``) are part+# of the contract: a breaking change to any of them is a bump.+#+# ``add_internal_model`` and ``add_doc_gen_default`` — plain data a plugin declares for the model manager to merge at+# boot: a model of the internal backend, as the table a backend file would hold, and the model deck's default engine for+# one document format and source — joined under v4 without a bump, on the same reasoning as ``add_log_sink``.PLUGIN_API_VERSION: int = 4@@ -31,8 +40,8 @@ PLUGIN_API_VERSION: int = 4class PipelexPlugin(Protocol):"""A unit of optional capability discovered at startup.- A plugin contributes inference backends, model listers, orchestrators,- hub-slot claims, HTTP-error mappers and teardown callbacks by calling the menu+ A plugin contributes inference backends, internal models and model deck defaults, model listers,+ orchestrators, hub-slot claims, HTTP-error mappers and teardown callbacks by calling the menumethods on the ``PluginRegistrar`` it is handed.**Invariant — a plugin belongs to exactly one layer, and it is the highest tier it contributes@@ -41,8 +50,9 @@ class PipelexPlugin(Protocol):interpreter-layer plugin and publishes under ``PluginGroup.INTERPRETER``. It may contributekernel-tier capabilities alongside them, and ours does: it registers an orchestrator(interpreter-tier) *and* an HTTP-error mapper (kernel-tier). Do not split such a plugin in two.- A plugin contributing only kernel-tier capabilities — an inference backend, a model lister, a- storage or secrets provider, an HTTP-error mapper — is a kernel-layer plugin and publishes under+ A plugin contributing only kernel-tier capabilities — an inference backend, an internal model or a+ model deck default, a model lister, a storage or secrets provider, an HTTP-error mapper — is a+ kernel-layer plugin and publishes under``PluginGroup.KERNEL``.An external plugin declares its layer by the group it publishes under, and the registrar enforcesdiff --git a/pipelex/plugins/exceptions.py b/pipelex/plugins/exceptions.pyindex b414ab972..00f2c0727 100644--- a/pipelex/plugins/exceptions.py+++ b/pipelex/plugins/exceptions.py@@ -142,6 +142,34 @@ class DuplicateLogSinkError(PluginError):super().__init__(message)+class DuplicateInternalModelError(PluginError):+ """Two plugins declared a model of the same name in the internal backend."""++ def __init__(self, *, name: str, first_plugin: str, second_plugin: str):+ self.name = name+ self.first_plugin = first_plugin+ self.second_plugin = second_plugin+ message = (+ f"Internal model '{name}' is declared by both plugin '{first_plugin}' and plugin '{second_plugin}'. "+ "Each internal model must be declared by a single plugin."+ )+ super().__init__(message)+++class DuplicateDocGenDefaultError(PluginError):+ """Two plugins declared the model deck's default document engine for the same format and source."""++ def __init__(self, *, choice_key: str, first_plugin: str, second_plugin: str):+ self.choice_key = choice_key+ self.first_plugin = first_plugin+ self.second_plugin = second_plugin+ message = (+ f"The default document engine for '{choice_key}' is declared by both plugin '{first_plugin}' and plugin '{second_plugin}'. "+ "Each format and source must have its default declared by a single plugin; a user sets their own in an x_custom_*.toml deck file."+ )+ super().__init__(message)++class DuplicatePipeFuncExecutorError(PluginError):"""Two plugins registered a PipeFunc executor for the same execution mode."""diff --git a/pipelex/plugins/inference_backend_registry.py b/pipelex/plugins/inference_backend_registry.pyindex b900c1458..b09386bf8 100644--- a/pipelex/plugins/inference_backend_registry.py+++ b/pipelex/plugins/inference_backend_registry.py@@ -13,6 +13,7 @@ class InferenceFamily(StrEnum):IMG_GEN = "img_gen"EXTRACT = "extract"SEARCH = "search"+ DOC_GEN = "doc_gen"# The uniform inference-backend factory. A backend plugin registers one of these@@ -79,6 +80,15 @@ class InferenceBackendRegistry:def has(self, *, family: InferenceFamily, sdk: str) -> bool:return (family, sdk) in self._backends+ def with_family(self, *, family: InferenceFamily, backends: dict[str, MakeWorkerFn]) -> "InferenceBackendRegistry":+ """A copy whose backends of one family are exactly `backends`, keyed by sdk, the other families' kept as they are.++ What a test uses to stand stub workers in for a family, or to remove it, without rebooting the runtime.+ """+ kept = {key: make_worker for key, make_worker in self._backends.items() if key[0] != family}+ kept.update({(family, sdk): make_worker for sdk, make_worker in backends.items()})+ return InferenceBackendRegistry(kept)+@propertydef keys(self) -> list[tuple[InferenceFamily, str]]:return list(self._backends)diff --git a/pipelex/plugins/plugin_model_declarations.py b/pipelex/plugins/plugin_model_declarations.pynew file mode 100644index 000000000..f2b4c64eb--- /dev/null+++ b/pipelex/plugins/plugin_model_declarations.py@@ -0,0 +1,66 @@+"""What plugins declare to the model manager: the internal models they ship, and the model deck defaults they set.++A plugin that ships an engine running inside Pipelex declares its model itself, rather than leaving it to the kit's+`internal.toml`, which only describes what open Pipelex ships. The registrar collects the declarations+(`add_internal_model`, `add_doc_gen_default`), `PluginRegistrar.make_model_declarations` freezes them into this value+object, and boot hands it to `ModelManagerAbstract.setup`, which merges the models into the internal backend and the+defaults beneath the model deck files. Nothing is validated when a plugin registers: the model manager validates each+declaration as it merges it, and names the plugin when one is refused.+"""++from typing import Any, Self++from pydantic import BaseModel, ConfigDict, Field++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource, doc_gen_choice_key+++class PluginInternalModel(BaseModel):+ """One model a plugin declares in the internal backend, and the plugin that declared it.++ `spec` is exactly the table a backend file would hold for the model (`model_type`, `sdk`, `model_id`, `inputs`,+ `outputs`, `costs`…). It is complete on its own: no backend file's `[defaults]` table is applied to it.+ """++ model_config = ConfigDict(frozen=True)++ spec: dict[str, Any]+ plugin: str+++class PluginDocGenDefault(BaseModel):+ """The engine a plugin declares as the model deck's default for one format and source, and the plugin that declared it."""++ model_config = ConfigDict(frozen=True)++ doc_gen_format: DocGenFormat+ source: DocGenSource+ model: str+ plugin: str++ @property+ def choice_key(self) -> str:+ """The key the model deck lists this default under: 'xlsx.layout'."""+ return doc_gen_choice_key(doc_gen_format=self.doc_gen_format, source=self.source)+++class PluginModelDeclarations(BaseModel):+ """Every internal model and model deck default the registered plugins declared, frozen for the model manager to merge."""++ model_config = ConfigDict(frozen=True)++ internal_models: dict[str, PluginInternalModel] = Field(default_factory=dict)+ """The plugins' internal models, keyed by model name."""+ doc_gen_defaults: tuple[PluginDocGenDefault, ...] = ()++ @classmethod+ def make_empty(cls) -> Self:+ """No declarations: what a boot without such plugins, or a test that builds no registrar, hands the model manager."""+ return cls()++ def make_deck_base(self) -> dict[str, Any]:+ """The model deck document the defaults make, which the deck files, a user's `x_custom_*.toml` included, are merged over."""+ if not self.doc_gen_defaults:+ return {}+ choice_defaults = {doc_gen_default.choice_key: doc_gen_default.model for doc_gen_default in self.doc_gen_defaults}+ return {"doc_gen": {"choice_defaults": choice_defaults}}diff --git a/pipelex/plugins/registrar.py b/pipelex/plugins/registrar.pyindex a3baf8150..394bedf29 100644--- a/pipelex/plugins/registrar.py+++ b/pipelex/plugins/registrar.py@@ -1,15 +1,19 @@-from collections.abc import Callable+import copy+from collections.abc import Callable, Mappingfrom enum import StrEnumfrom typing import TYPE_CHECKING, Any, NamedTuple, TypeVarfrom pydantic import BaseModel, Fieldfrom pipelex.base_exceptions import ErrorReport+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource, doc_gen_choice_keyfrom pipelex.plugins.bundle_validator_registry import BundleValidatorProtocolfrom pipelex.plugins.exceptions import (DuplicateBundleValidatorError,+ DuplicateDocGenDefaultError,DuplicateHttpErrorMapperError,DuplicateInferenceBackendError,+ DuplicateInternalModelError,DuplicateLogSinkError,DuplicateModelListerError,DuplicateOrchestratorError,@@ -25,6 +29,7 @@ from pipelex.plugins.model_lister_registry import ListModelsFnfrom pipelex.plugins.orchestrator_registry import OrchestratorProtocolfrom pipelex.plugins.pipe_func_executor_registry import PipeFuncExecutorFactoryFnfrom pipelex.plugins.plugin_group import PluginGroup+from pipelex.plugins.plugin_model_declarations import PluginDocGenDefault, PluginInternalModel, PluginModelDeclarationsfrom pipelex.plugins.secrets_provider_registry import SecretsProviderFactoryFnfrom pipelex.plugins.storage_provider_registry import StorageProviderFactoryFnfrom pipelex.runtime_bridge.orchestration_mode import OrchestrationMode@@ -162,6 +167,9 @@ class PluginRegistrar:self.secrets_providers: dict[str, SecretsProviderFactoryFn] = {}self.log_sinks: dict[str, LogSinkFactoryFn] = {}self.pipe_func_executors: dict[str, PipeFuncExecutorFactoryFn] = {}+ # Plain data, read once by ``make_model_declarations`` for the model manager to merge at boot.+ self.internal_models: dict[str, dict[str, Any]] = {}+ self.doc_gen_defaults: dict[tuple[DocGenFormat, DocGenSource], str] = {}# Ordered list (not a type-keyed dict) because the exception types are# resolved lazily — only ``get_http_error_mappers`` invokes the providers,# so duplicate-by-type detection is deferred to resolution time too.@@ -177,6 +185,8 @@ class PluginRegistrar:self._secrets_provider_sources: dict[str, str] = {}self._log_sink_sources: dict[str, str] = {}self._pipe_func_executor_sources: dict[str, str] = {}+ self._internal_model_sources: dict[str, str] = {}+ self._doc_gen_default_sources: dict[tuple[DocGenFormat, DocGenSource], str] = {}self._slot_sources: dict[HubSlot, str] = {}# Reassigned per plugin by build_registrar; the floating default keeps the# menu methods safe to call outside a registration loop (e.g. a focused unit test).@@ -309,6 +319,52 @@ class PluginRegistrar:),)+ def add_internal_model(self, *, name: str, spec: Mapping[str, Any]) -> None:+ """Declare one model of the internal backend, the software-only backend that runs inside Pipelex.++ ``spec`` is exactly the table a backend file would hold for the model (``model_type``, ``sdk``, ``model_id``,+ ``inputs``, ``outputs``, ``costs``…), and it is complete on its own: no backend file's ``[defaults]`` table+ is applied to it. A plugin that ships an engine declares its model here rather than leaving it to the kit's+ ``internal.toml``, and registers the engine's worker with ``add_inference_backend`` for the same sdk.++ Plain data, so it is stored and nothing else: the model manager validates the table when it merges it into+ the internal backend at boot, and refuses one whose name the installation's ``internal.toml`` already+ declares. Fail-loud on a name another plugin declared, naming both plugins.+ """+ self._add(+ store=self.internal_models,+ sources=self._internal_model_sources,+ key=name,+ value=copy.deepcopy(dict(spec)),+ contribution=f"internal model {name}",+ on_duplicate=lambda first_plugin, second_plugin: DuplicateInternalModelError(+ name=name, first_plugin=first_plugin, second_plugin=second_plugin+ ),+ )++ def add_doc_gen_default(self, *, doc_gen_format: DocGenFormat, source: DocGenSource, model: str) -> None:+ """Declare the model deck's default document engine for one format and source, such as an ``xlsx`` from the auto-layout.++ ``model`` names the engine a ``PipeDocGen`` step prints with when it names none: a model name, or any model+ reference the deck resolves. The default sits beneath the deck files, so a deck file that sets the same+ format and source, a user's ``x_custom_*.toml`` included, overrides it. Stored and nothing else: the model+ manager checks that a step composes this format from this source when it merges the default at boot.+ Fail-loud on a format and source another plugin declared, naming both plugins.+ """+ key = (doc_gen_format, source)+ self._add(+ store=self.doc_gen_defaults,+ sources=self._doc_gen_default_sources,+ key=key,+ value=model,+ contribution=f"doc_gen default {doc_gen_choice_key(doc_gen_format=doc_gen_format, source=source)} = {model}",+ on_duplicate=lambda first_plugin, second_plugin: DuplicateDocGenDefaultError(+ choice_key=doc_gen_choice_key(doc_gen_format=doc_gen_format, source=source),+ first_plugin=first_plugin,+ second_plugin=second_plugin,+ ),+ )+def add_pipe_func_executor(self, *, mode: str, factory: PipeFuncExecutorFactoryFn) -> None:"""Contribute a factory for one PipeFunc execution mode, keyed by an open ``mode`` token.@@ -391,6 +447,24 @@ class PluginRegistrar:"""return {discovery.name for discovery in self.discoveries if discovery.status.is_registered}+ def make_model_declarations(self) -> PluginModelDeclarations:+ """Freeze the internal models and model deck defaults the plugins declared, for ``ModelManagerAbstract.setup``.++ A fresh value object with deep copies of the tables, so neither the model manager nor a plugin holding on to+ the mapping it passed, or a list inside it, can change what the registrar recorded.+ """+ internal_models = {+ name: PluginInternalModel(spec=copy.deepcopy(spec), plugin=self._internal_model_sources[name])+ for name, spec in self.internal_models.items()+ }+ doc_gen_defaults = tuple(+ PluginDocGenDefault(+ doc_gen_format=doc_gen_format, source=source, model=model, plugin=self._doc_gen_default_sources[doc_gen_format, source]+ )+ for (doc_gen_format, source), model in self.doc_gen_defaults.items()+ )+ return PluginModelDeclarations(internal_models=internal_models, doc_gen_defaults=doc_gen_defaults)+def get_http_error_mappers(self) -> dict[type[Exception], HttpErrorMapperFn]:"""Resolve every contributed exc-type provider into a ``{exc_type: mapper}`` dict.diff --git a/pipelex/providers/builtins.py b/pipelex/providers/builtins.pyindex 384ab1b35..3185617df 100644--- a/pipelex/providers/builtins.py+++ b/pipelex/providers/builtins.py@@ -17,6 +17,7 @@ from pipelex.providers.openai.openai_plugin import OpenAIPluginfrom pipelex.providers.openrouter.openrouter_plugin import OpenRouterPluginfrom pipelex.providers.portkey.portkey_plugin import PortkeyPluginfrom pipelex.providers.pypdfium2.pypdfium2_plugin import Pypdfium2Plugin+from pipelex.providers.reportlab.reportlab_plugin import ReportlabDocGenPluginfrom pipelex.providers.secrets.secrets_plugin import SecretsPluginfrom pipelex.providers.storage.storage_plugin import StoragePlugin@@ -54,6 +55,7 @@ KERNEL_BUILTIN_PLUGINS: list[PipelexPlugin] = [DoclingPlugin(),Pypdfium2Plugin(),LinkupPlugin(),+ ReportlabDocGenPlugin(),]# Kernel-layer built-ins that core requires unconditionally — naming one in ``runtime.plugins.disabled`` is adiff --git a/pipelex/providers/reportlab/__init__.py b/pipelex/providers/reportlab/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/providers/reportlab/fonts/open-sans/OFL.txt b/pipelex/providers/reportlab/fonts/open-sans/OFL.txtnew file mode 100644index 000000000..2e76eefd4--- /dev/null+++ b/pipelex/providers/reportlab/fonts/open-sans/OFL.txt@@ -0,0 +1,88 @@+Copyright 2020 The Open Sans Project Authors (https://github.com/googlefonts/opensans)++-----------------------------------------------------------+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007+-----------------------------------------------------------++PREAMBLE+The goals of the Open Font License (OFL) are to stimulate worldwide+development of collaborative font projects, to support the font+creation efforts of academic and linguistic communities, and to+provide a free and open framework in which fonts may be shared and+improved in partnership with others.++The OFL allows the licensed fonts to be used, studied, modified and+redistributed freely as long as they are not sold by themselves. The+fonts, including any derivative works, can be bundled, embedded,+redistributed and/or sold with any software provided that any reserved+names are not used by derivative works. The fonts and derivatives,+however, cannot be released under any other type of license. The+requirement for fonts to remain under this license does not apply to+any document created using the fonts or their derivatives.++DEFINITIONS+"Font Software" refers to the set of files released by the Copyright+Holder(s) under this license and clearly marked as such. This may+include source files, build scripts and documentation.++"Reserved Font Name" refers to any names specified as such after the+copyright statement(s).++"Original Version" refers to the collection of Font Software+components as distributed by the Copyright Holder(s).++"Modified Version" refers to any derivative made by adding to,+deleting, or substituting -- in part or in whole -- any of the+components of the Original Version, by changing formats or by porting+the Font Software to a new environment.++"Author" refers to any designer, engineer, programmer, technical+writer or other person who contributed to the Font Software.++PERMISSION & CONDITIONS+Permission is hereby granted, free of charge, to any person obtaining+a copy of the Font Software, to use, study, copy, merge, embed,+modify, redistribute, and sell modified and unmodified copies of the+Font Software, subject to the following conditions:++1) Neither the Font Software nor any of its individual components, in+Original or Modified Versions, may be sold by itself.++2) Original or Modified Versions of the Font Software may be bundled,+redistributed and/or sold with any software, provided that each copy+contains the above copyright notice and this license. These can be+included either as stand-alone text files, human-readable headers or+in the appropriate machine-readable metadata fields within text or+binary files as long as those fields can be easily viewed by the user.++3) No Modified Version of the Font Software may use the Reserved Font+Name(s) unless explicit written permission is granted by the+corresponding Copyright Holder. This restriction only applies to the+primary font name as presented to the users.++4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font+Software shall not be used to promote, endorse or advertise any+Modified Version, except to acknowledge the contribution(s) of the+Copyright Holder(s) and the Author(s) or with their explicit written+permission.++5) The Font Software, modified or unmodified, in part or in whole,+must be distributed entirely under this license, and must not be+distributed under any other license. The requirement for fonts to+remain under this license does not apply to any document created using+the Font Software.++TERMINATION+This license becomes null and void if any of the above conditions are+not met.++DISCLAIMER+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM+OTHER DEALINGS IN THE FONT SOFTWARE.diff --git a/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Bold.ttf b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Bold.ttfnew file mode 100644index 000000000..b7fadfa4aBinary files /dev/null and b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Bold.ttf differdiff --git a/pipelex/providers/reportlab/fonts/open-sans/OpenSans-BoldItalic.ttf b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-BoldItalic.ttfnew file mode 100644index 000000000..136a4b4c0Binary files /dev/null and b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-BoldItalic.ttf differdiff --git a/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Italic.ttf b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Italic.ttfnew file mode 100644index 000000000..e99cb92d4Binary files /dev/null and b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Italic.ttf differdiff --git a/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Regular.ttf b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Regular.ttfnew file mode 100644index 000000000..8529c432cBinary files /dev/null and b/pipelex/providers/reportlab/fonts/open-sans/OpenSans-Regular.ttf differdiff --git a/pipelex/providers/reportlab/fonts/roboto-mono/OFL.txt b/pipelex/providers/reportlab/fonts/roboto-mono/OFL.txtnew file mode 100644index 000000000..8e7df338d--- /dev/null+++ b/pipelex/providers/reportlab/fonts/roboto-mono/OFL.txt@@ -0,0 +1,93 @@+Copyright 2015 The Roboto Mono Project Authors (https://github.com/googlefonts/robotomono)++This Font Software is licensed under the SIL Open Font License, Version 1.1.+This license is copied below, and is also available with a FAQ at:+https://openfontlicense.org+++-----------------------------------------------------------+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007+-----------------------------------------------------------++PREAMBLE+The goals of the Open Font License (OFL) are to stimulate worldwide+development of collaborative font projects, to support the font creation+efforts of academic and linguistic communities, and to provide a free and+open framework in which fonts may be shared and improved in partnership+with others.++The OFL allows the licensed fonts to be used, studied, modified and+redistributed freely as long as they are not sold by themselves. The+fonts, including any derivative works, can be bundled, embedded,+redistributed and/or sold with any software provided that any reserved+names are not used by derivative works. The fonts and derivatives,+however, cannot be released under any other type of license. The+requirement for fonts to remain under this license does not apply+to any document created using the fonts or their derivatives.++DEFINITIONS+"Font Software" refers to the set of files released by the Copyright+Holder(s) under this license and clearly marked as such. This may+include source files, build scripts and documentation.++"Reserved Font Name" refers to any names specified as such after the+copyright statement(s).++"Original Version" refers to the collection of Font Software components as+distributed by the Copyright Holder(s).++"Modified Version" refers to any derivative made by adding to, deleting,+or substituting -- in part or in whole -- any of the components of the+Original Version, by changing formats or by porting the Font Software to a+new environment.++"Author" refers to any designer, engineer, programmer, technical+writer or other person who contributed to the Font Software.++PERMISSION & CONDITIONS+Permission is hereby granted, free of charge, to any person obtaining+a copy of the Font Software, to use, study, copy, merge, embed, modify,+redistribute, and sell modified and unmodified copies of the Font+Software, subject to the following conditions:++1) Neither the Font Software nor any of its individual components,+in Original or Modified Versions, may be sold by itself.++2) Original or Modified Versions of the Font Software may be bundled,+redistributed and/or sold with any software, provided that each copy+contains the above copyright notice and this license. These can be+included either as stand-alone text files, human-readable headers or+in the appropriate machine-readable metadata fields within text or+binary files as long as those fields can be easily viewed by the user.++3) No Modified Version of the Font Software may use the Reserved Font+Name(s) unless explicit written permission is granted by the corresponding+Copyright Holder. This restriction only applies to the primary font name as+presented to the users.++4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font+Software shall not be used to promote, endorse or advertise any+Modified Version, except to acknowledge the contribution(s) of the+Copyright Holder(s) and the Author(s) or with their explicit written+permission.++5) The Font Software, modified or unmodified, in part or in whole,+must be distributed entirely under this license, and must not be+distributed under any other license. The requirement for fonts to+remain under this license does not apply to any document created+using the Font Software.++TERMINATION+This license becomes null and void if any of the above conditions are+not met.++DISCLAIMER+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM+OTHER DEALINGS IN THE FONT SOFTWARE.diff --git a/pipelex/providers/reportlab/fonts/roboto-mono/RobotoMono-Regular.ttf b/pipelex/providers/reportlab/fonts/roboto-mono/RobotoMono-Regular.ttfnew file mode 100644index 000000000..eb1fde544Binary files /dev/null and b/pipelex/providers/reportlab/fonts/roboto-mono/RobotoMono-Regular.ttf differdiff --git a/pipelex/providers/reportlab/markdown_flowables.py b/pipelex/providers/reportlab/markdown_flowables.pynew file mode 100644index 000000000..67ea63e30--- /dev/null+++ b/pipelex/providers/reportlab/markdown_flowables.py@@ -0,0 +1,280 @@+"""Markdown as ReportLab flowables, for the built-in PDF engine.++The Markdown is parsed by the one parser Pipelex formats Markdown with (`get_markdown_parser`), so a PDF reads it+exactly as the `markdown` filter of HTML templates does, and the converter walks markdown-it's syntax tree rather+than its flat token stream, so a nested list lives inside its item and each list numbers its own items.++What it prints:++- headings 1 to 6, paragraphs with bold, italics, strikethrough, inline code and links, soft breaks as spaces and+ hard breaks as line breaks;+- bullet and ordered lists, nested, an ordered list starting at its own `start`;+- tables, whose header row repeats on every page, with the column alignments the Markdown sets;+- fenced and indented code blocks in the monospace face, printed as written, long lines wrapped;+- blockquotes, indented behind a rule, and horizontal rules.++Every text is escaped into ReportLab's paragraph markup (`escape_text`), and only `http`, `https` and `mailto` links+become links: any other target prints its text alone. A Markdown image is never fetched: its alt text prints in+italics, as `[image: alt]`. A node the converter does not know prints its text, and never fails the document.+"""++from urllib.parse import urlsplit++from markdown_it.tree import SyntaxTreeNode+from reportlab.lib.styles import ParagraphStyle # type: ignore[import-untyped]+from reportlab.pdfbase import pdfmetrics # type: ignore[import-untyped]+from reportlab.platypus import Flowable, HRFlowable, ListFlowable, Paragraph, Preformatted, Spacer # type: ignore[import-untyped]++from pipelex.providers.reportlab.pdf_elements import (+ BOX_PADDING,+ LINK_COLOR_HEX,+ MONO_FONT,+ RULE_COLOR,+ SANS_FONT,+ TEXT_COLOR,+ ColumnExtent,+ PdfStyles,+ TableCell,+ boxed,+ data_table,+ escape_attribute,+ escape_text,+ fit_column_widths,+ heading_break,+ measure_column,+)+from pipelex.tools.markdown.markdown_parser import get_markdown_parser++LINKED_SCHEMES = frozenset({"http", "https", "mailto"})++# The bullets of nested bullet lists, by depth, alternating: both are in the bundled sans face.+_BULLETS = ("•", "–")+_MINIMUM_LIST_INDENT = 14.0+_BULLET_GAP = 6.0+_QUOTE_INDENT = 2 * BOX_PADDING+_MINIMUM_CODE_LINE_LENGTH = 20+++def markdown_to_flowables(*, markdown_text: str, styles: PdfStyles, available_width: float) -> list[Flowable]:+ """The flowables that print a Markdown text in a frame `available_width` points wide."""+ tree = SyntaxTreeNode(get_markdown_parser().parse(markdown_text))+ return markdown_nodes_to_flowables(nodes=tree.children, styles=styles, available_width=available_width)+++def markdown_nodes_to_flowables(*, nodes: list[SyntaxTreeNode], styles: PdfStyles, available_width: float) -> list[Flowable]:+ """The flowables that print the top-level block nodes of a parsed Markdown text."""+ return _MarkdownWriter(styles=styles).blocks(nodes=nodes, available_width=available_width, list_depth=0, is_contained=False)+++class _MarkdownWriter:+ def __init__(self, *, styles: PdfStyles):+ self._styles = styles++ def blocks(self, *, nodes: list[SyntaxTreeNode], available_width: float, list_depth: int, is_contained: bool) -> list[Flowable]:+ flowables: list[Flowable] = []+ for node in nodes:+ flowables.extend(self._block(node=node, available_width=available_width, list_depth=list_depth, is_contained=is_contained))+ return flowables++ def _block(self, *, node: SyntaxTreeNode, available_width: float, list_depth: int, is_contained: bool) -> list[Flowable]:+ """The flowables of one block node; `is_contained` when it sits inside a list item or a quotation, where no page break can go."""+ match node.type:+ case "paragraph":+ style = self._styles.list_body if list_depth else self._styles.body+ return [Paragraph(inline_markup(nodes=node.children), style)]+ case "heading":+ heading = Paragraph(inline_markup(nodes=node.children), self._styles.heading(level=_heading_level(tag=node.tag)))+ if is_contained:+ return [heading]+ return [heading_break(), heading]+ case "bullet_list" | "ordered_list":+ return self._list(node=node, available_width=available_width, list_depth=list_depth)+ case "table":+ return self._table(node=node, available_width=available_width)+ case "fence" | "code_block":+ return self._code(text=node.content, available_width=available_width)+ case "blockquote":+ inner = self.blocks(nodes=node.children, available_width=available_width - _QUOTE_INDENT, list_depth=list_depth, is_contained=True)+ if not inner:+ return []+ return [boxed(flowables=inner, width=available_width, is_code=False)]+ case "hr":+ return [HRFlowable(width="100%", thickness=0.6, color=RULE_COLOR, spaceBefore=4, spaceAfter=8)]+ case _:+ text = plain_text(nodes=[node]).strip()+ if not text:+ return []+ return [Paragraph(escape_text(text=text), self._styles.body)]++ def _list(self, *, node: SyntaxTreeNode, available_width: float, list_depth: int) -> list[Flowable]:+ items = [child for child in node.children if child.type == "list_item"]+ if not items:+ return []+ is_ordered = node.type == "ordered_list"+ start = _list_start(node=node) if is_ordered else 1+ bullet = _BULLETS[list_depth % len(_BULLETS)]+ labels = [f"{start + index}." for index in range(len(items))] if is_ordered else [bullet]+ font_size = self._styles.body.fontSize+ label_width = max(pdfmetrics.stringWidth(label, SANS_FONT, font_size) for label in labels)+ indent = max(label_width + _BULLET_GAP, _MINIMUM_LIST_INDENT)+ # Each item is the list of its own flowables: ReportLab bullets the first and numbers the items from `start`,+ # so a nested list, which sits inside its item, neither takes a number nor shifts the outer numbering.+ list_items: list[list[Flowable]] = []+ for item in items:+ content = self.blocks(nodes=item.children, available_width=available_width - indent, list_depth=list_depth + 1, is_contained=True)+ list_items.append(content or [Spacer(1, self._styles.list_body.leading)])+ list_flowable = ListFlowable(+ list_items,+ bulletType="1" if is_ordered else "bullet",+ start=start if is_ordered else bullet,+ bulletFormat="%s." if is_ordered else None,+ leftIndent=indent,+ bulletFontName=SANS_FONT,+ bulletFontSize=font_size,+ bulletColor=TEXT_COLOR,+ spaceAfter=3,+ )+ return [list_flowable]++ def _table(self, *, node: SyntaxTreeNode, available_width: float) -> list[Flowable]:+ header_rows: list[list[SyntaxTreeNode]] = []+ body_rows: list[list[SyntaxTreeNode]] = []+ for section in node.children:+ rows = header_rows if section.type == "thead" else body_rows+ rows.extend([cell for cell in row.children if cell.type in {"th", "td"}] for row in section.children if row.type == "tr")+ all_rows = header_rows[:1] + body_rows+ column_count = max((len(row) for row in all_rows), default=0)+ if column_count == 0:+ return []+ header_row = header_rows[0] if header_rows else None+ extents: list[ColumnExtent] = []+ for column_index in range(column_count):+ header_text = plain_text(nodes=[header_row[column_index]]) if header_row and column_index < len(header_row) else None+ cell_texts = (plain_text(nodes=[row[column_index]]) for row in body_rows if column_index < len(row))+ extents.append(measure_column(header=header_text, cells=cell_texts, available_width=available_width, column_count=column_count))+ col_widths = fit_column_widths(extents=extents, available_width=available_width)+ table_rows: list[list[TableCell]] = []+ for row_index, row in enumerate(all_rows):+ is_header = header_row is not None and row_index == 0+ cells: list[TableCell] = []+ for column_index in range(column_count):+ if column_index < len(row):+ cell = row[column_index]+ style = self._cell_style(alignment=_cell_alignment(node=cell), is_header=is_header)+ cells.append(Paragraph(inline_markup(nodes=cell.children), style))+ else:+ cells.append("")+ table_rows.append(cells)+ return [data_table(rows=table_rows, col_widths=col_widths, has_header=header_row is not None, right_aligned_columns=[])]++ def _cell_style(self, *, alignment: str | None, is_header: bool) -> ParagraphStyle:+ match alignment:+ case "right":+ return self._styles.cell_header_right if is_header else self._styles.cell_right+ case "center":+ return self._styles.cell_header_center if is_header else self._styles.cell_center+ case _:+ return self._styles.cell_header if is_header else self._styles.cell++ def _code(self, *, text: str, available_width: float) -> list[Flowable]:+ code = text.rstrip("\n").expandtabs(4)+ if not code.strip():+ return []+ code_style = self._styles.code+ character_width = pdfmetrics.stringWidth("0", MONO_FONT, code_style.fontSize)+ line_length = max(int((available_width - 2 * BOX_PADDING) // character_width), _MINIMUM_CODE_LINE_LENGTH)+ preformatted = Preformatted(code, code_style, maxLineLength=line_length, newLineChars="")+ return [boxed(flowables=[preformatted], width=available_width, is_code=True)]+++def inline_markup(*, nodes: list[SyntaxTreeNode]) -> str:+ """Inline Markdown as ReportLab paragraph markup: every text escaped, emphasis, code and safe links as tags."""+ parts: list[str] = []+ for node in nodes:+ match node.type:+ case "text":+ parts.append(escape_text(text=node.content))+ case "softbreak":+ parts.append(" ")+ case "hardbreak":+ parts.append("<br/>")+ case "strong":+ parts.append(f"<b>{inline_markup(nodes=node.children)}</b>")+ case "em":+ parts.append(f"<i>{inline_markup(nodes=node.children)}</i>")+ case "s":+ parts.append(f"<strike>{inline_markup(nodes=node.children)}</strike>")+ case "code_inline":+ parts.append(f'<font face="{MONO_FONT}">{escape_text(text=node.content)}</font>')+ case "link":+ parts.append(_link_markup(node=node))+ case "image":+ alt_text = plain_text(nodes=node.children).strip() or node.content+ parts.append(f"<i>[image: {escape_text(text=alt_text)}]</i>")+ case _:+ if node.children:+ parts.append(inline_markup(nodes=node.children))+ else:+ parts.append(escape_text(text=node.content))+ return "".join(parts)+++def _link_markup(*, node: SyntaxTreeNode) -> str:+ label = inline_markup(nodes=node.children)+ href = node.attrs.get("href")+ if not isinstance(href, str) or not is_linked_href(href=href):+ return label+ return f'<a href="{escape_attribute(value=href)}" color="{LINK_COLOR_HEX}">{label}</a>'+++def is_linked_href(*, href: str) -> bool:+ """Whether a link target becomes a link in the PDF: only `http`, `https` and `mailto` do."""+ try:+ scheme = urlsplit(href.strip()).scheme+ except ValueError:+ return False+ return scheme.lower() in LINKED_SCHEMES+++def plain_text(*, nodes: list[SyntaxTreeNode]) -> str:+ """The text of Markdown nodes without their markup, as it reads once printed: what a table column is measured by."""+ parts: list[str] = []+ for node in nodes:+ match node.type:+ case "softbreak":+ parts.append(" ")+ case "hardbreak":+ parts.append("\n")+ case "image":+ parts.append(f"[image: {plain_text(nodes=node.children).strip() or node.content}]")+ case _:+ if node.children:+ parts.append(plain_text(nodes=node.children))+ else:+ parts.append(node.content)+ return "".join(parts)+++def _heading_level(*, tag: str) -> int:+ if len(tag) == 2 and tag[0] == "h" and tag[1].isdigit():+ return int(tag[1])+ return 6+++def _list_start(*, node: SyntaxTreeNode) -> int:+ start = node.attrs.get("start", 1)+ try:+ return int(start)+ except (TypeError, ValueError):+ return 1+++def _cell_alignment(*, node: SyntaxTreeNode) -> str | None:+ style = node.attrs.get("style")+ if not isinstance(style, str):+ return None+ if "text-align:right" in style:+ return "right"+ if "text-align:center" in style:+ return "center"+ return Nonediff --git a/pipelex/providers/reportlab/pdf_elements.py b/pipelex/providers/reportlab/pdf_elements.pynew file mode 100644index 000000000..76a896894--- /dev/null+++ b/pipelex/providers/reportlab/pdf_elements.py@@ -0,0 +1,377 @@+"""What both writers of the built-in PDF engine share: the bundled fonts, the paragraph styles, the escaping every+text goes through, and the styled tables.++**The bundled fonts.** ReportLab's own fonts are the PDF standard fonts, which print Western European text only, so+the engine bundles open-licensed ones (SIL Open Font License 1.1, whose text sits beside each family in `fonts/`):++- **Open Sans** 3.003, in its four static faces, for every text: Latin with Central European and Vietnamese,+ Greek, Cyrillic and Hebrew. From `googlefonts/opensans` at commit `bd7e37632246368c60fdcbd374dbf9bad11969b6`,+ `fonts/ttf/`.+- **Roboto Mono** 3.001, its regular face only, for code: the scripts of Open Sans except Hebrew, so a comment in+ Polish or Greek inside a code block prints, which ReportLab's Courier would not. From `googlefonts/RobotoMono`+ at commit `895ec691990d041dd727c7b5afa3ce56525d98e6`, `fonts/ttf/`. Its family maps bold and italic to the+ regular face, so inline code inside bold text still resolves.++Right-to-left scripts need ReportLab's optional extras for bidirectional text and shaping (`reportlab[bidi,shaping]`,+which bring `rlbidi` and `uharfbuzz`), which pipelex does not install: without them a Hebrew line prints its letters+left to right. Arabic, Chinese, Japanese and Korean are not in the bundled fonts at all and print as empty boxes.+"""++import re+import threading+from collections.abc import Iterable+from functools import cache+from pathlib import Path+from typing import NamedTuple, TypeAlias+from xml.sax.saxutils import escape++from reportlab.lib import colors # type: ignore[import-untyped]+from reportlab.lib.enums import TA_CENTER, TA_RIGHT # type: ignore[import-untyped]+from reportlab.lib.styles import ParagraphStyle # type: ignore[import-untyped]+from reportlab.pdfbase import pdfmetrics # type: ignore[import-untyped]+from reportlab.pdfbase.ttfonts import TTFont # type: ignore[import-untyped]+from reportlab.platypus import CondPageBreak, Flowable, Paragraph, Table, TableStyle # type: ignore[import-untyped]++FONTS_DIR = Path(__file__).parent / "fonts"++# The names the bundled faces are registered under with ReportLab, which are the names paragraph markup uses.+SANS_FONT = "PipelexSans"+SANS_BOLD_FONT = "PipelexSans-Bold"+SANS_ITALIC_FONT = "PipelexSans-Italic"+SANS_BOLD_ITALIC_FONT = "PipelexSans-BoldItalic"+MONO_FONT = "PipelexMono"++TEXT_COLOR = colors.HexColor("#1f2328")+MUTED_COLOR = colors.HexColor("#6b7280")+RULE_COLOR = colors.HexColor("#d0d7de")+HEADER_BACKGROUND_COLOR = colors.HexColor("#f1f3f5")+CODE_BACKGROUND_COLOR = colors.HexColor("#f6f8fa")+LINK_COLOR_HEX = "#1d4ed8"++BODY_FONT_SIZE = 9.5+CELL_FONT_SIZE = 8.5+CELL_LEADING = 11.0+CELL_HORIZONTAL_PADDING = 4.0+CELL_VERTICAL_PADDING = 3.0+BOX_PADDING = 6.0++# A heading starts a new page when less than this height is left below it, so it is never left alone at the foot of a+# page. ReportLab's `keepWithNext` is not used for that: it keeps a heading with everything that follows it, so a+# heading before a long table would push the whole table onto the next page.+HEADING_ROOM = 72.0++# What a table cell holds: a flowable, or a text ReportLab draws as it is, in the table's own font.+TableCell: TypeAlias = Flowable | str+++class BundledFace(NamedTuple):+ font_name: str+ file_path: Path+++BUNDLED_FACES: tuple[BundledFace, ...] = (+ BundledFace(font_name=SANS_FONT, file_path=FONTS_DIR / "open-sans" / "OpenSans-Regular.ttf"),+ BundledFace(font_name=SANS_BOLD_FONT, file_path=FONTS_DIR / "open-sans" / "OpenSans-Bold.ttf"),+ BundledFace(font_name=SANS_ITALIC_FONT, file_path=FONTS_DIR / "open-sans" / "OpenSans-Italic.ttf"),+ BundledFace(font_name=SANS_BOLD_ITALIC_FONT, file_path=FONTS_DIR / "open-sans" / "OpenSans-BoldItalic.ttf"),+ BundledFace(font_name=MONO_FONT, file_path=FONTS_DIR / "roboto-mono" / "RobotoMono-Regular.ttf"),+)++_FONT_REGISTRATION_LOCK = threading.Lock()+++def register_bundled_fonts() -> None:+ """Register the bundled faces and their families with ReportLab, once per process, whichever thread asks first.++ ReportLab keeps registered fonts in a process-wide table, so a second registration would swap the font objects+ under a render already using them: the lock and the cache make it happen exactly once.+ """+ with _FONT_REGISTRATION_LOCK:+ _register_bundled_fonts_once()+++@cache+def _register_bundled_fonts_once() -> None:+ # The stubs leave some of these functions' parameters untyped, hence the ignores.+ for face in BUNDLED_FACES:+ pdfmetrics.registerFont(TTFont(face.font_name, str(face.file_path))) # pyright: ignore[reportUnknownMemberType]+ pdfmetrics.registerFontFamily( # pyright: ignore[reportUnknownMemberType]+ SANS_FONT, normal=SANS_FONT, bold=SANS_BOLD_FONT, italic=SANS_ITALIC_FONT, boldItalic=SANS_BOLD_ITALIC_FONT+ )+ pdfmetrics.registerFontFamily(MONO_FONT, normal=MONO_FONT, bold=MONO_FONT, italic=MONO_FONT, boldItalic=MONO_FONT) # pyright: ignore[reportUnknownMemberType]+++class PdfStyles(NamedTuple):+ """The paragraph styles of the engine, built once per renderer, after the fonts are registered."""++ title: ParagraphStyle+ headings: tuple[ParagraphStyle, ...]+ body: ParagraphStyle+ list_body: ParagraphStyle+ cell: ParagraphStyle+ cell_right: ParagraphStyle+ cell_center: ParagraphStyle+ cell_header: ParagraphStyle+ cell_header_right: ParagraphStyle+ cell_header_center: ParagraphStyle+ grid_label: ParagraphStyle+ caption: ParagraphStyle+ muted: ParagraphStyle+ code: ParagraphStyle++ def heading(self, *, level: int) -> ParagraphStyle:+ """The heading style of a level, clamped to the levels there are: 1 is the largest, deeper ones the smallest."""+ clamped_level = min(max(level, 1), len(self.headings))+ return self.headings[clamped_level - 1]+++def build_pdf_styles() -> PdfStyles:+ body = ParagraphStyle(+ "pipelex-body",+ fontName=SANS_FONT,+ fontSize=BODY_FONT_SIZE,+ leading=13.5,+ textColor=TEXT_COLOR,+ spaceAfter=5,+ )+ heading_sizes = (16.0, 13.5, 12.0, 11.0, 10.0, 9.5)+ headings = tuple(+ ParagraphStyle(+ f"pipelex-heading-{index}",+ parent=body,+ fontName=SANS_BOLD_FONT,+ fontSize=size,+ leading=size * 1.3,+ spaceBefore=size * 0.7,+ spaceAfter=size * 0.35,+ )+ for index, size in enumerate(heading_sizes, start=1)+ )+ cell = ParagraphStyle("pipelex-cell", parent=body, fontSize=CELL_FONT_SIZE, leading=CELL_LEADING, spaceAfter=0)+ cell_header = ParagraphStyle("pipelex-cell-header", parent=cell, fontName=SANS_BOLD_FONT)+ return PdfStyles(+ title=ParagraphStyle("pipelex-title", parent=body, fontName=SANS_BOLD_FONT, fontSize=20, leading=25, spaceAfter=10),+ headings=headings,+ body=body,+ list_body=ParagraphStyle("pipelex-list-body", parent=body, spaceAfter=2),+ cell=cell,+ cell_right=ParagraphStyle("pipelex-cell-right", parent=cell, alignment=TA_RIGHT),+ cell_center=ParagraphStyle("pipelex-cell-center", parent=cell, alignment=TA_CENTER),+ cell_header=cell_header,+ cell_header_right=ParagraphStyle("pipelex-cell-header-right", parent=cell_header, alignment=TA_RIGHT),+ cell_header_center=ParagraphStyle("pipelex-cell-header-center", parent=cell_header, alignment=TA_CENTER),+ grid_label=ParagraphStyle("pipelex-grid-label", parent=body, fontName=SANS_BOLD_FONT, spaceAfter=0),+ caption=ParagraphStyle(+ "pipelex-caption",+ parent=body,+ fontName=SANS_ITALIC_FONT,+ fontSize=8.5,+ leading=11,+ textColor=MUTED_COLOR,+ alignment=TA_CENTER,+ spaceBefore=3,+ spaceAfter=10,+ ),+ muted=ParagraphStyle("pipelex-muted", parent=body, fontName=SANS_ITALIC_FONT, textColor=MUTED_COLOR),+ code=ParagraphStyle("pipelex-code", parent=body, fontName=MONO_FONT, fontSize=8, leading=10.5, spaceAfter=0),+ )+++# C0 control characters other than tab, line feed and carriage return, and DEL: they have no glyph and no meaning in+# a printed page, so they are dropped rather than handed to ReportLab's parser.+_CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")+++def escape_text(*, text: str) -> str:+ """A text as ReportLab paragraph markup that prints it literally, its line breaks kept.++ Every text a writer puts into a paragraph goes through here, so a value such as `<b>Tom & Jerry</b>` prints as+ written and can never open a tag of ReportLab's markup.+ """+ cleaned = _CONTROL_CHARACTERS.sub("", text).replace("\r\n", "\n").replace("\r", "\n")+ return escape(cleaned).replace("\n", "<br/>")+++def escape_attribute(*, value: str) -> str:+ """A value for a double-quoted attribute of ReportLab's paragraph markup, such as a link's `href`."""+ return escape(_CONTROL_CHARACTERS.sub("", value), {'"': """})+++class TextExtent(NamedTuple):+ """How wide a text is when set on one line per line break, and how wide its widest word is, in points."""++ line_width: float+ word_width: float+++def measure_text(*, text: str, font_name: str, font_size: float) -> TextExtent:+ line_width = 0.0+ word_width = 0.0+ for line in text.splitlines() or [""]:+ line_width = max(line_width, pdfmetrics.stringWidth(line, font_name, font_size))+ for word in line.split():+ word_width = max(word_width, pdfmetrics.stringWidth(word, font_name, font_size))+ return TextExtent(line_width=line_width, word_width=word_width)+++class ColumnExtent(NamedTuple):+ """The widths a table column would like, in points, cell padding included."""++ preferred: float+ minimum: float+++# A table measures at most this many rows to size its columns, so a table of thousands of rows costs no more to+# size than one of a few hundred.+_MEASURED_ROWS_LIMIT = 300+_MINIMUM_COLUMN_WIDTH = 24.0+++def measure_column(*, header: str | None, cells: Iterable[str], available_width: float, column_count: int) -> ColumnExtent:+ """The preferred and minimum widths of a column, from its header and the text of its cells.++ The minimum is the widest word, which is capped at an even share of the width, so a long URL wraps inside its+ cell rather than squeezing every other column.+ """+ padding = 2 * CELL_HORIZONTAL_PADDING+ extents: list[TextExtent] = []+ if header is not None:+ extents.append(measure_text(text=header, font_name=SANS_BOLD_FONT, font_size=CELL_FONT_SIZE))+ for index, cell_text in enumerate(cells):+ if index >= _MEASURED_ROWS_LIMIT:+ break+ extents.append(measure_text(text=cell_text, font_name=SANS_FONT, font_size=CELL_FONT_SIZE))+ preferred = max((extent.line_width for extent in extents), default=0.0) + padding+ minimum = max((extent.word_width for extent in extents), default=0.0) + padding+ share = available_width / max(column_count, 1)+ minimum = min(max(minimum, _MINIMUM_COLUMN_WIDTH), share)+ return ColumnExtent(preferred=max(preferred, minimum), minimum=minimum)+++def fit_column_widths(*, extents: list[ColumnExtent], available_width: float) -> list[float]:+ """Column widths that fit the available width.++ Columns keep their preferred widths when they all fit. Otherwise every column gets its minimum, and the width+ left is shared out evenly: a column that wants less than an even share takes what it wants, and the columns that+ want more split what remains. So a column of long text wraps, and the short columns beside it do not. When even+ the minimums do not fit, they shrink in proportion.+ """+ total_preferred = sum(extent.preferred for extent in extents)+ if total_preferred <= available_width:+ return [extent.preferred for extent in extents]+ total_minimum = sum(extent.minimum for extent in extents)+ if total_minimum >= available_width:+ return [extent.minimum * available_width / total_minimum for extent in extents]+ widths = [extent.minimum for extent in extents]+ extra_width = available_width - total_minimum+ wanting = list(range(len(extents)))+ while wanting:+ share = extra_width / len(wanting)+ satisfied = [index for index in wanting if extents[index].preferred - widths[index] <= share]+ if not satisfied:+ for index in wanting:+ widths[index] += share+ break+ for index in satisfied:+ extra_width -= extents[index].preferred - widths[index]+ widths[index] = extents[index].preferred+ wanting = [index for index in wanting if index not in satisfied]+ return widths+++def heading_break() -> Flowable:+ """What goes before a heading so that it starts a new page rather than end one (`HEADING_ROOM`)."""+ return CondPageBreak(HEADING_ROOM)+++def text_cell(*, text: str, width: float, style: ParagraphStyle) -> TableCell:+ """A table cell printing a text in a column `width` points wide.++ A text that fits on one line is the cell itself, which ReportLab draws as it is, in the font the table style+ sets for the cell (`data_table`), with no paragraph to lay out: that is what keeps a long table fast. A text+ that does not fit becomes a paragraph, escaped, which wraps.+ """+ cleaned = _CONTROL_CHARACTERS.sub("", text)+ if "\n" not in cleaned and "\r" not in cleaned:+ if pdfmetrics.stringWidth(cleaned, style.fontName, style.fontSize) <= width - 2 * CELL_HORIZONTAL_PADDING:+ return cleaned+ return Paragraph(escape_text(text=cleaned), style)+++def data_table(*, rows: list[list[TableCell]], col_widths: list[float], has_header: bool, right_aligned_columns: list[int]) -> Table:+ """A table of cells: its header row repeats on every page, and a row taller than a page splits.++ Its style sets the font of the cells that are plain texts (`text_cell`): the cell face and size, bold in the+ header row, and right-aligned in `right_aligned_columns`. Paragraph cells carry their own style.+ """+ table = Table(+ rows,+ colWidths=col_widths,+ repeatRows=1 if has_header else 0,+ splitInRow=1,+ hAlign="LEFT",+ spaceBefore=2,+ spaceAfter=8,+ )+ table_style = TableStyle(+ [+ ("VALIGN", (0, 0), (-1, -1), "TOP"),+ ("LEFTPADDING", (0, 0), (-1, -1), CELL_HORIZONTAL_PADDING),+ ("RIGHTPADDING", (0, 0), (-1, -1), CELL_HORIZONTAL_PADDING),+ ("TOPPADDING", (0, 0), (-1, -1), CELL_VERTICAL_PADDING),+ ("BOTTOMPADDING", (0, 0), (-1, -1), CELL_VERTICAL_PADDING),+ ("LINEBELOW", (0, 0), (-1, -1), 0.25, RULE_COLOR),+ ("FONTNAME", (0, 0), (-1, -1), SANS_FONT),+ ("FONTSIZE", (0, 0), (-1, -1), CELL_FONT_SIZE),+ ("LEADING", (0, 0), (-1, -1), CELL_LEADING),+ ("TEXTCOLOR", (0, 0), (-1, -1), TEXT_COLOR),+ ]+ )+ for column_index in right_aligned_columns:+ table_style.add("ALIGN", (column_index, 0), (column_index, -1), "RIGHT")+ if has_header:+ table_style.add("FONTNAME", (0, 0), (-1, 0), SANS_BOLD_FONT)+ table_style.add("BACKGROUND", (0, 0), (-1, 0), HEADER_BACKGROUND_COLOR)+ table_style.add("LINEBELOW", (0, 0), (-1, 0), 0.6, MUTED_COLOR)+ table.setStyle(table_style)+ return table+++def field_grid_table(*, rows: list[list[Flowable]], col_widths: list[float]) -> Table:+ """A two-column table of labels and values, with no header, which splits between rows and inside a tall row."""+ table = Table(rows, colWidths=col_widths, splitInRow=1, hAlign="LEFT", spaceBefore=2, spaceAfter=8)+ table.setStyle(+ TableStyle(+ [+ ("VALIGN", (0, 0), (-1, -1), "TOP"),+ ("LEFTPADDING", (0, 0), (-1, -1), 0),+ ("RIGHTPADDING", (0, 0), (-1, -1), 2 * CELL_HORIZONTAL_PADDING),+ ("TOPPADDING", (0, 0), (-1, -1), 2),+ ("BOTTOMPADDING", (0, 0), (-1, -1), 2),+ ]+ )+ )+ return table+++def boxed(*, flowables: list[Flowable], width: float, is_code: bool) -> Table:+ """Flowables in a box the width of the frame: shaded for code, with a rule on the left for a quotation.++ A one-cell table rather than a frame of its own, so the box splits across pages with its content.+ """+ table = Table([[flowables]], colWidths=[width], splitInRow=1, hAlign="LEFT", spaceBefore=2, spaceAfter=8)+ table_style = TableStyle(+ [+ ("VALIGN", (0, 0), (-1, -1), "TOP"),+ ("LEFTPADDING", (0, 0), (-1, -1), BOX_PADDING),+ ("RIGHTPADDING", (0, 0), (-1, -1), BOX_PADDING),+ ("TOPPADDING", (0, 0), (-1, -1), BOX_PADDING if is_code else 1),+ ("BOTTOMPADDING", (0, 0), (-1, -1), BOX_PADDING if is_code else 1),+ ]+ )+ if is_code:+ table_style.add("BACKGROUND", (0, 0), (-1, -1), CODE_BACKGROUND_COLOR)+ else:+ table_style.add("LINEBEFORE", (0, 0), (0, -1), 2, RULE_COLOR)+ table.setStyle(table_style)+ return tablediff --git a/pipelex/providers/reportlab/reportlab_pdf_renderer.py b/pipelex/providers/reportlab/reportlab_pdf_renderer.pynew file mode 100644index 000000000..b81f44421--- /dev/null+++ b/pipelex/providers/reportlab/reportlab_pdf_renderer.py@@ -0,0 +1,451 @@+"""The built-in PDF engine: it prints the layout tree of a `PipeDocGen` step with no template, on ReportLab.++It writes the tree as ReportLab flowables on A4 portrait pages:++- the document's title at the top, and the title of the render job in a running header on every page, with+ "Page N of M" in the footer, which is why the document is built twice, the first pass counting the pages;+- a field grid as a two-column table of bold labels and their values;+- a table block as a table whose header row repeats on every page, its cells wrapping, its columns fitted to the+ page, and its numeric columns aligned right;+- a section as a heading whose size follows its level, then its own blocks;+- paragraphs as they are, Markdown through its converter (`markdown_flowables`), and an image from the bytes the+ render's resources read, scaled to the page with its caption under it.++Every text is escaped before it enters ReportLab's paragraph markup, so a value prints as written. The engine+fetches nothing itself: an image is read through `RenderResources`, which applies the run's read scope, and a+Markdown image is not read at all. It is the worker of the `reportlab-pdf` model, made for each print; it registers+its bundled fonts once per process (`pdf_elements`) and builds its styles once, and it keeps nothing of one render+into the next, so renders on any thread do not meet.++Renders are built one at a time in a process (`_BUILD_LOCK`): ReportLab subsets a registered TrueType font through a+cursor the font keeps on its file, so two documents saved at once corrupt each other's fonts. That costs no+throughput, since a build is pure Python and would hold the interpreter lock anyway, and the images are read before+the lock is taken, so a slow image does not hold up another render.+"""++import datetime+import io+import threading+from functools import cache+from typing import NamedTuple++from PIL import Image as PilImage+from reportlab.lib.pagesizes import A4 # type: ignore[import-untyped]+from reportlab.lib.units import mm # type: ignore[import-untyped]+from reportlab.pdfbase import pdfmetrics # type: ignore[import-untyped]+from reportlab.platypus import BaseDocTemplate, Flowable, Frame, Image, KeepTogether, PageTemplate, Paragraph, Spacer # type: ignore[import-untyped]+from reportlab.platypus.doctemplate import LayoutError # type: ignore[import-untyped]+from typing_extensions import override++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.doc_gen_worker_abstract import DocGenWorkerAbstract+from pipelex.cogt.doc_gen.exceptions import DocGenRenderError+from pipelex.cogt.doc_gen.layout_tree import (+ FieldGridBlock,+ ImageBlock,+ LayoutBlock,+ LayoutDocument,+ LayoutScalar,+ MarkdownBlock,+ ParagraphsBlock,+ SectionBlock,+ TableBlock,+)+from pipelex.cogt.doc_gen.render_job import RenderedDocument, RenderJob, RenderResources+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.providers.reportlab.markdown_flowables import markdown_to_flowables+from pipelex.providers.reportlab.pdf_elements import (+ MUTED_COLOR,+ RULE_COLOR,+ SANS_BOLD_FONT,+ SANS_FONT,+ PdfStyles,+ TableCell,+ build_pdf_styles,+ data_table,+ escape_text,+ field_grid_table,+ fit_column_widths,+ heading_break,+ measure_column,+ register_bundled_fonts,+ text_cell,+)+from pipelex.reporting.reporting_protocol import ReportingProtocol++PAGE_WIDTH, PAGE_HEIGHT = A4+LEFT_MARGIN = RIGHT_MARGIN = 18 * mm+TOP_MARGIN = 22 * mm+BOTTOM_MARGIN = 20 * mm+FRAME_WIDTH = PAGE_WIDTH - LEFT_MARGIN - RIGHT_MARGIN+FRAME_HEIGHT = PAGE_HEIGHT - TOP_MARGIN - BOTTOM_MARGIN++# An image is printed at 96 pixels to the inch unless that is wider than the frame or taller than this share of it.+_IMAGE_POINTS_PER_PIXEL = 72 / 96+_IMAGE_MAX_HEIGHT = FRAME_HEIGHT * 0.6+_IMAGE_SPACE_BEFORE = 6.0++# A field grid's label column takes the width of its widest label, up to this share of the frame.+_GRID_LABEL_MAX_SHARE = 0.4++_RUNNING_TEXT_FONT_SIZE = 8.0+_HEADER_BASELINE = PAGE_HEIGHT - 12 * mm+_FOOTER_BASELINE = 10 * mm++_PDF_CREATOR = "Pipelex"++_BUILD_LOCK = threading.Lock()+++@cache+def _pdf_styles() -> PdfStyles:+ """The engine's paragraph and table styles, built once per process, after the fonts they name are registered."""+ register_bundled_fonts()+ return build_pdf_styles()+++class ReportlabPdfWorker(DocGenWorkerAbstract):+ """The worker of `reportlab-pdf`, the engine that prints a `pdf` from the layout tree.++ Made for each print by its plugin's factory; `render` keeps its state in objects of its own call, so concurrent+ renders do not meet.+ """++ def __init__(self, *, inference_model: InferenceModelSpec, reporting_delegate: ReportingProtocol | None = None):+ super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)+ self._styles = _pdf_styles()++ @override+ def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument:+ """Print a render job's layout tree as a PDF.++ Raises:+ DocGenRenderError: the job is not a PDF from the layout, an image cannot be read as one, or something in+ the document cannot be laid out on an A4 page.+ UriReadRefusedError: the run's read scope does not allow an image the document names.+ """+ layout = _layout_of_pdf_job(job=job)+ images = _RenderImages(resources=resources)+ images.preload(layout=layout)+ with _BUILD_LOCK:+ first_pass = self._build(job=job, layout=layout, images=images, page_count=None)+ second_pass = self._build(job=job, layout=layout, images=images, page_count=first_pass.page_count)+ return RenderedDocument(data=second_pass.data)++ def _build(self, *, job: RenderJob, layout: LayoutDocument, images: "_RenderImages", page_count: int | None) -> "_BuiltPass":+ origins: dict[int, str] = {}+ story = _LayoutWriter(styles=self._styles, images=images, origins=origins).document(layout=layout)+ buffer = io.BytesIO()+ doc_template = _PagedDocTemplate(buffer=buffer, title=job.title, page_count=page_count, origins=origins)+ try:+ doc_template.build(story)+ except LayoutError as exc:+ where = doc_template.last_origin or "an element of the document"+ msg = (+ f"The PDF engine cannot print '{job.filename}': {where} does not fit on an A4 page, "+ "as part of it is taller than a page and cannot be split across pages."+ )+ raise DocGenRenderError(msg).as_caller_fault() from exc+ return _BuiltPass(data=buffer.getvalue(), page_count=doc_template.page)+++def _layout_of_pdf_job(*, job: RenderJob) -> LayoutDocument:+ """The layout tree of a job this engine prints, which is a PDF from the layout: its model declares no other."""+ refusal = f"The built-in PDF engine prints a pdf from the auto-layout of its inputs, not {job.format.with_article} {job.source.desc}."+ match job.format:+ case DocGenFormat.PDF:+ pass+ case DocGenFormat.XLSX | DocGenFormat.DOCX | DocGenFormat.PPTX:+ raise DocGenRenderError(refusal)+ match job.source:+ case DocGenSource.LAYOUT:+ pass+ case DocGenSource.HTML | DocGenSource.TEMPLATE_FILE:+ raise DocGenRenderError(refusal)+ if job.layout is None:+ raise DocGenRenderError(refusal)+ return job.layout+++class _BuiltPass(NamedTuple):+ data: bytes+ page_count: int+++class _LoadedImage(NamedTuple):+ url: str+ data: bytes+ width_px: int+ height_px: int+++class _RenderImages:+ """The images of one render, each read once through the render's resources, and kept for both build passes.++ They are numbered in document order, the order `LayoutDocument.image_urls` and the writer both follow, and each+ read names its number as its position: "image 2 of the document".+ """++ def __init__(self, *, resources: RenderResources):+ self._resources = resources+ self._images: dict[int, _LoadedImage] = {}++ def preload(self, *, layout: LayoutDocument) -> None:+ """Read every image of the layout, before the build takes its lock."""+ for number, url in enumerate(layout.image_urls(), start=1):+ self.get(number=number, url=url)++ def get(self, *, number: int, url: str) -> _LoadedImage:+ loaded = self._images.get(number)+ if loaded is None or loaded.url != url:+ position = f"image {number} of the document"+ data = self._resources.load(uri=url, position=position)+ loaded = _decode_image(url=url, data=data, position=position)+ self._images[number] = loaded+ return loaded+++def _decode_image(*, url: str, data: bytes, position: str) -> _LoadedImage:+ """Decode an image fully, so a file that is not an image, or a truncated one, fails here and names its position."""+ try:+ with PilImage.open(io.BytesIO(data)) as pil_image:+ pil_image.load()+ width_px, height_px = pil_image.size+ except (OSError, ValueError, SyntaxError, PilImage.DecompressionBombError) as exc:+ msg = f"The PDF engine cannot print {position}: its file is not an image it reads, such as PNG, JPEG, GIF or WebP."+ raise DocGenRenderError(msg).as_caller_fault() from exc+ if width_px <= 0 or height_px <= 0:+ msg = f"The PDF engine cannot print {position}: the image is empty."+ raise DocGenRenderError(msg).as_caller_fault()+ return _LoadedImage(url=url, data=data, width_px=width_px, height_px=height_px)+++class _PagedDocTemplate(BaseDocTemplate):+ """An A4 document whose pages carry a running header and "Page N of M", and which remembers what it was laying out.++ `page_count` is None on the first pass, which is the pass that counts the pages. `origins` names, in the author's+ terms, the flowables the writer made, so a layout error can say which part of the document did not fit.+ """++ def __init__(self, *, buffer: io.BytesIO, title: str, page_count: int | None, origins: dict[int, str]):+ super().__init__( # pyright: ignore[reportUnknownMemberType] - the stubs leave BaseDocTemplate.__init__ partly untyped+ buffer,+ pagesize=A4,+ leftMargin=LEFT_MARGIN,+ rightMargin=RIGHT_MARGIN,+ topMargin=TOP_MARGIN,+ bottomMargin=BOTTOM_MARGIN,+ title=title,+ creator=_PDF_CREATOR,+ )+ # One frame with no padding, so the frame is exactly `FRAME_WIDTH` wide, which the writers size tables to.+ body_frame = Frame(+ LEFT_MARGIN, BOTTOM_MARGIN, FRAME_WIDTH, FRAME_HEIGHT, leftPadding=0, rightPadding=0, topPadding=0, bottomPadding=0, id="body"+ )+ self.addPageTemplates([PageTemplate(id="page", frames=[body_frame])])+ self._running_title = title+ self._page_count = page_count+ self._origins = origins+ self.last_origin: str | None = None++ @override+ def handle_flowable(self, flowables: list[Flowable]) -> None:+ # The part of a table or a paragraph split over a page is a flowable the writer did not make, and it follows+ # the one it was split from: the last origin seen is still the right one.+ if flowables:+ origin = self._origins.get(id(flowables[0]))+ if origin is not None:+ self.last_origin = origin+ super().handle_flowable(flowables)++ @override+ def beforePage(self) -> None:+ # Drawn as the page begins, so the header and the page number come first in the page's text.+ canvas = self.canv+ canvas.saveState()+ canvas.setFont(SANS_FONT, _RUNNING_TEXT_FONT_SIZE)+ canvas.setFillColor(MUTED_COLOR)+ canvas.drawString(+ LEFT_MARGIN,+ _HEADER_BASELINE,+ _fit_line(text=self._running_title, font_name=SANS_FONT, font_size=_RUNNING_TEXT_FONT_SIZE, max_width=FRAME_WIDTH),+ )+ canvas.setStrokeColor(RULE_COLOR)+ canvas.setLineWidth(0.4)+ canvas.line(LEFT_MARGIN, _HEADER_BASELINE - 3, PAGE_WIDTH - RIGHT_MARGIN, _HEADER_BASELINE - 3)+ page_label = f"Page {self.page}" if self._page_count is None else f"Page {self.page} of {self._page_count}"+ canvas.drawRightString(PAGE_WIDTH - RIGHT_MARGIN, _FOOTER_BASELINE, page_label)+ canvas.restoreState()+++def _fit_line(*, text: str, font_name: str, font_size: float, max_width: float) -> str:+ """A text on one line, cut with an ellipsis when it is wider than `max_width`."""+ line = " ".join(text.split())+ if pdfmetrics.stringWidth(line, font_name, font_size) <= max_width:+ return line+ ellipsis = "…"+ kept_length = 0+ too_long_length = len(line)+ while too_long_length - kept_length > 1:+ middle_length = (kept_length + too_long_length) // 2+ if pdfmetrics.stringWidth(line[:middle_length] + ellipsis, font_name, font_size) <= max_width:+ kept_length = middle_length+ else:+ too_long_length = middle_length+ return line[:kept_length].rstrip() + ellipsis+++class _LayoutWriter:+ """Writes one layout tree as the flowables of one build pass."""++ def __init__(self, *, styles: PdfStyles, images: _RenderImages, origins: dict[int, str]):+ self._styles = styles+ self._images = images+ self._origins = origins+ self._image_count = 0++ def document(self, *, layout: LayoutDocument) -> list[Flowable]:+ story: list[Flowable] = [Paragraph(escape_text(text=layout.title), self._styles.title)]+ story.extend(self._blocks(blocks=layout.blocks, level=1, section_title=None))+ return story++ def _blocks(self, *, blocks: list[LayoutBlock], level: int, section_title: str | None) -> list[Flowable]:+ """The flowables of a list of blocks, `level` being the heading level of a titled block among them."""+ where = f"'{section_title}'" if section_title is not None else "the document"+ flowables: list[Flowable] = []+ for block in blocks:+ match block:+ case FieldGridBlock():+ if block.fields:+ flowables.append(self._tagged(flowable=self._field_grid(block=block), origin=f"the fields of {where}"))+ case TableBlock():+ flowables.extend(self._heading(title=block.title, level=level))+ flowables.extend(self._table(block=block))+ case ParagraphsBlock():+ for paragraph in block.paragraphs:+ flowables.append(+ self._tagged(flowable=Paragraph(escape_text(text=paragraph), self._styles.body), origin=f"the text of {where}")+ )+ case MarkdownBlock():+ for flowable in markdown_to_flowables(markdown_text=block.markdown, styles=self._styles, available_width=FRAME_WIDTH):+ flowables.append(self._tagged(flowable=flowable, origin=f"the Markdown of {where}"))+ case ImageBlock():+ flowables.append(self._image(block=block))+ case SectionBlock():+ flowables.extend(self._heading(title=block.title, level=block.level))+ flowables.extend(self._blocks(blocks=block.blocks, level=block.level + 1, section_title=block.title))+ return flowables++ def _tagged(self, *, flowable: Flowable, origin: str) -> Flowable:+ self._origins[id(flowable)] = origin+ return flowable++ def _heading(self, *, title: str, level: int) -> list[Flowable]:+ return [heading_break(), Paragraph(escape_text(text=title), self._styles.heading(level=level))]++ def _field_grid(self, *, block: FieldGridBlock) -> Flowable:+ label_style = self._styles.grid_label+ widest_label = max(+ pdfmetrics.stringWidth(word, SANS_BOLD_FONT, label_style.fontSize)+ for field in block.fields+ for word in [field.label, *field.label.split()]+ )+ label_width = min(widest_label + 12, FRAME_WIDTH * _GRID_LABEL_MAX_SHARE)+ rows: list[list[Flowable]] = [+ [Paragraph(escape_text(text=field.label), label_style), Paragraph(escape_text(text=display_scalar(value=field.value)), self._styles.body)]+ for field in block.fields+ ]+ return field_grid_table(rows=rows, col_widths=[label_width, FRAME_WIDTH - label_width])++ def _table(self, *, block: TableBlock) -> list[Flowable]:+ if not block.columns:+ return []+ column_count = len(block.columns)+ displayed_rows = [[display_scalar(value=row.get(column.key)) for column in block.columns] for row in block.rows]+ extents = [+ measure_column(+ header=column.label,+ cells=(displayed_row[index] for displayed_row in displayed_rows),+ available_width=FRAME_WIDTH,+ column_count=column_count,+ )+ for index, column in enumerate(block.columns)+ ]+ col_widths = fit_column_widths(extents=extents, available_width=FRAME_WIDTH)+ numeric_columns = [_is_numeric_column(values=[row.get(column.key) for row in block.rows]) for column in block.columns]+ styles = self._styles+ header_row: list[TableCell] = [+ text_cell(text=column.label, width=col_widths[index], style=styles.cell_header_right if numeric_columns[index] else styles.cell_header)+ for index, column in enumerate(block.columns)+ ]+ body_rows: list[list[TableCell]] = [+ [+ text_cell(text=cell_text, width=col_widths[index], style=styles.cell_right if numeric_columns[index] else styles.cell)+ for index, cell_text in enumerate(displayed_row)+ ]+ for displayed_row in displayed_rows+ ]+ right_aligned_columns = [index for index, is_numeric in enumerate(numeric_columns) if is_numeric]+ table = data_table(rows=[header_row, *body_rows], col_widths=col_widths, has_header=True, right_aligned_columns=right_aligned_columns)+ flowables: list[Flowable] = [self._tagged(flowable=table, origin=f"the table '{block.title}'")]+ if not block.rows:+ flowables.append(Paragraph("No rows.", styles.muted))+ return flowables++ def _image(self, *, block: ImageBlock) -> Flowable:+ self._image_count += 1+ number = self._image_count+ loaded = self._images.get(number=number, url=block.url)+ natural_width = loaded.width_px * _IMAGE_POINTS_PER_PIXEL+ natural_height = loaded.height_px * _IMAGE_POINTS_PER_PIXEL+ scale = min(1.0, FRAME_WIDTH / natural_width, _IMAGE_MAX_HEIGHT / natural_height)+ image = Image(io.BytesIO(loaded.data), width=natural_width * scale, height=natural_height * scale, hAlign="CENTER")+ members: list[Flowable] = [Spacer(1, _IMAGE_SPACE_BEFORE), image]+ if block.caption:+ members.append(Paragraph(escape_text(text=block.caption), self._styles.caption))+ return self._tagged(flowable=KeepTogether(members), origin=f"image {number} of the document")+++def _is_numeric_column(*, values: list[LayoutScalar]) -> bool:+ """Whether a column holds numbers only, blanks aside, and at least one: such a column is aligned right."""+ present = [value for value in values if value is not None]+ return bool(present) and all(isinstance(value, (int, float)) and not isinstance(value, bool) for value in present)+++def _utc_offset_suffix(*, value: datetime.datetime | datetime.time) -> str:+ """The UTC offset a value states, as ' UTC' or ' +02:00', or nothing for a value that states none."""+ offset = value.utcoffset()+ if offset is None:+ return ""+ total_minutes = int(offset.total_seconds()) // 60+ if total_minutes == 0:+ return " UTC"+ sign = "+" if total_minutes > 0 else "-"+ hours, minutes = divmod(abs(total_minutes), 60)+ return f" {sign}{hours:02d}:{minutes:02d}"+++def display_scalar(*, value: LayoutScalar) -> str:+ """A scalar as the document prints it: blank for nothing, Yes or No, numbers plainly, dates and times in ISO order.++ A time of day prints to the minute, with the UTC offset the value states, if any.+ """+ match value:+ case None:+ return ""+ case bool():+ return "Yes" if value else "No"+ case int():+ return str(value)+ case float():+ if value.is_integer():+ return str(int(value))+ return format(value, ".15g")+ case datetime.datetime():+ return value.strftime("%Y-%m-%d %H:%M") + _utc_offset_suffix(value=value)+ case datetime.date():+ return value.isoformat()+ case datetime.time():+ return value.strftime("%H:%M") + _utc_offset_suffix(value=value)+ case str():+ return valuediff --git a/pipelex/providers/reportlab/reportlab_plugin.py b/pipelex/providers/reportlab/reportlab_plugin.pynew file mode 100644index 000000000..84d85ab2e--- /dev/null+++ b/pipelex/providers/reportlab/reportlab_plugin.py@@ -0,0 +1,35 @@+from pipelex.cogt.doc_gen.exceptions import BUILT_IN_DOC_GEN_SDK+from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract+from pipelex.cogt.model_backends.backend import InferenceBackend+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.inference_backend_registry import InferenceFamily+from pipelex.plugins.registrar import PluginRegistrar+from pipelex.plugins.sdk_client_registry import SdkClientRegistry+from pipelex.reporting.reporting_protocol import ReportingProtocol+++def _make_reportlab_pdf_worker(+ *,+ inference_model: InferenceModelSpec,+ backend: InferenceBackend, # ruff: ignore[unused-function-argument] - a local engine, with no backend settings+ sdk_clients: SdkClientRegistry, # ruff: ignore[unused-function-argument] - stateless worker, no SDK-client caching+ reporting_delegate: ReportingProtocol | None,+) -> InferenceWorkerAbstract:+ from pipelex.providers.reportlab.reportlab_pdf_renderer import ReportlabPdfWorker # ruff: ignore[import-outside-top-level]++ return ReportlabPdfWorker(inference_model=inference_model, reporting_delegate=reporting_delegate)+++class ReportlabDocGenPlugin:+ """Built-in document engine `reportlab-pdf`: it prints a `pdf` from the auto-layout of a `PipeDocGen` step's inputs.++ Registering imports nothing of ReportLab: the factory imports the worker's module, which imports ReportLab, the+ first time a document is printed with it, and the worker registers the bundled fonts once per process.+ """++ name = "reportlab"+ targets_api = PLUGIN_API_VERSION++ def register(self, registrar: PluginRegistrar) -> None:+ registrar.add_inference_backend(family=InferenceFamily.DOC_GEN, sdk=BUILT_IN_DOC_GEN_SDK, make_worker=_make_reportlab_pdf_worker)diff --git a/pipelex/runtime_boot.py b/pipelex/runtime_boot.pyindex e72d07624..4f7d323be 100644--- a/pipelex/runtime_boot.py+++ b/pipelex/runtime_boot.py@@ -656,10 +656,18 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.# typed as, and which is a public injection point. Widening that interface is a decision of# its own, so the gap is documented rather than half-closed. The docstrings say exactly# this; do not read ``config_dir`` as "only this directory is read" for inference.+ #+ # The interface WAS widened once, for ``plugin_model_declarations``, and that decision is+ # the reason it is required rather than optional: a plugin that ships a document engine+ # declares the engine's model and its deck defaults on the registrar, and the model deck+ # is built here, so an implementation that could be set up without them would boot a deck+ # missing every plugin engine and refuse those steps with a misleading "not installed".+ # The registrar was built above, before any of this, so its declarations are final here.self.models_manager.setup(secrets_provider=secrets_provider,managed_gateway_configs=managed_gateway_configs,gateway_config_source=gateway_config_source,+ plugin_model_declarations=plugin_registrar.make_model_declarations(),needs_inference=needs_inference,)except RoutingProfileLibraryNotFoundError as routing_not_found_exc:diff --git a/pipelex/system/configuration/config_loader.py b/pipelex/system/configuration/config_loader.pyindex ac384ce99..b1940ff11 100644--- a/pipelex/system/configuration/config_loader.py+++ b/pipelex/system/configuration/config_loader.py@@ -310,11 +310,12 @@ class ConfigLoader:copy_directory_structure(src_dir=config_template_dir, dst_dir=global_dir)- # Stamp the deck manifest so the boot-time staleness check has a baseline.+ # Stamp the kit manifests of the deck and of backends/internal.toml so the boot-time staleness+ # check and `pipelex update` have a baseline.# Imported lazily to avoid a circular import — config_loader is loaded very early.- from pipelex.cogt.models.deck_manifest import compute_kit_manifest, write_manifest # ruff: ignore[import-outside-top-level]+ from pipelex.cogt.models.deck_manifest import stamp_kit_manifests # ruff: ignore[import-outside-top-level]- write_manifest(compute_kit_manifest(), deck_dir=global_dir / "inference" / "deck")+ stamp_kit_manifests(inference_dir=global_dir / "inference")@classmethoddef _override_files_for_dir(cls, config_dir: Path, *, include_run_mode: bool) -> list[Path]:diff --git a/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/bundle.mthdsnew file mode 100644index 000000000..9b699496f--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/bundle.mthds@@ -0,0 +1,35 @@+domain = "building_inspections"+description = "Write a building inspector's site notes up as a formatted report, then log it in one line"+main_pipe = "report_and_log"++[pipe.report_and_log]+type = "PipeSequence"+description = "Writes the formatted inspection report, then the one-line entry for the site log"+inputs = { notes = "Text" }+output = "Text"+steps = [+ { pipe = "write_inspection_report", result = "report" },+ { pipe = "log_report", result = "log_entry" },+]++[pipe.write_inspection_report]+type = "PipeLLM"+description = "Writes the site notes up as a report with a heading per area and a list of the defects found"+inputs = { notes = "Text" }+output = "Markdown"+prompt = """+Write these site notes up as an inspection report in Markdown: a heading for each area inspected, and a bulleted list of the defects found there.++@notes+"""++[pipe.log_report]+type = "PipeLLM"+description = "Sums up an inspection report in one line for the site log"+inputs = { report = "Text" }+output = "Text"+prompt = """+Sum up this inspection report in one line for the site log.++@report+"""diff --git a/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/entry.toml b/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/entry.tomlnew file mode 100644index 000000000..49c7dcafd--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/entry.toml@@ -0,0 +1,10 @@+name = "native_markdown_inspection_report"+description = "An inspection report written in Markdown from site notes, then read back as text for the site log"++validity = "valid"+tier = "dry"+granularity = "focused"++covers = [+ "native.markdown",+]diff --git a/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/inputs.json b/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/inputs.jsonnew file mode 100644index 000000000..1eb317539--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/native_markdown_inspection_report/inputs.json@@ -0,0 +1,8 @@+{+ "notes": {+ "concept": "native.Text",+ "content": {+ "text": "Roof: two cracked tiles above the north gable, gutter sagging at the east corner. Cellar: damp patch on the back wall, no standing water. Stairs: loose handrail between the first and second floors."+ }+ }+}diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/bundle.mthdsnew file mode 100644index 000000000..b5a66ebff--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/bundle.mthds@@ -0,0 +1,23 @@+domain = "shop_notices"+description = "Printing the notice a shop tapes to its door"+main_pipe = "print_notice"++[concept.OpeningNotice]+description = "What a shop wants passers-by to know about its opening hours"++[concept.OpeningNotice.structure]+shop_name = { type = "text", description = "The name painted above the shop's door", required = true }+weekdays = { type = "text", description = "The opening hours from Monday to Friday", required = true }+weekend = { type = "text", description = "The opening hours on Saturday and Sunday", required = true }++[concept.PrintedNotice]+description = "The notice laid out as a page to print and tape to the door"+refines = "Document"++[pipe.print_notice]+type = "PipeDocGen"+description = "Lays the opening hours out as a page to print"+inputs = { notice = "OpeningNotice" }+output = "PrintedNotice"+format = "pdf"+filename = "notice-{{ notice.shop_name }}"diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/entry.toml b/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/entry.tomlnew file mode 100644index 000000000..e4946f8be--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/entry.toml@@ -0,0 +1,10 @@+name = "operator_doc_gen_door_notice"+description = "A shop's opening hours laid out as a notice page to print and tape to the door"++validity = "valid"+tier = "offline"+granularity = "focused"++covers = [+ "operator.doc_gen",+]diff --git a/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/inputs.json b/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/inputs.jsonnew file mode 100644index 000000000..3e91e3bf9--- /dev/null+++ b/pipelex/test_extras/mthds_corpus/entries/operator_doc_gen_door_notice/inputs.json@@ -0,0 +1,10 @@+{+ "notice": {+ "concept": "shop_notices.OpeningNotice",+ "content": {+ "shop_name": "Boulangerie Kerlann",+ "weekdays": "7:00 to 19:30",+ "weekend": "7:30 to 13:00"+ }+ }+}diff --git a/pipelex/test_extras/mthds_corpus/vocabulary.toml b/pipelex/test_extras/mthds_corpus/vocabulary.tomlindex b56f95602..b043e0591 100644--- a/pipelex/test_extras/mthds_corpus/vocabulary.toml+++ b/pipelex/test_extras/mthds_corpus/vocabulary.toml@@ -20,6 +20,9 @@ code = "Dynamic"[native.text]code = "Text"+[native.markdown]+code = "Markdown"+[native.image]code = "Image"@@ -81,6 +84,9 @@ code = "PipeSearch"[operator.structure]code = "PipeStructure"+[operator.doc_gen]+code = "PipeDocGen"+[controller.batch]code = "PipeBatch"diff --git a/pipelex/tools/jinja2/html_renderable.py b/pipelex/tools/jinja2/html_renderable.pynew file mode 100644index 000000000..f42d383aa--- /dev/null+++ b/pipelex/tools/jinja2/html_renderable.py@@ -0,0 +1,31 @@+"""Protocol for values that know their own HTML: markupsafe's `__html__` convention.++HTML autoescaping (`TemplateCategory.HTML`) escapes every value it prints, except a value that defines+`__html__`: markupsafe calls it and inserts what it returns as it is. A content class defines it when its+HTML rendering is both the right thing to print and safe to insert, which is the case of `MarkdownContent`,+whose conversion escapes any raw HTML in the source. `StuffArtefact` answers `__html__` for a stuff whose+content does, so `{{ report }}` prints a Markdown stuff as formatted HTML with no filter, while a Text stuff,+whose content defines no `__html__`, keeps being escaped.++markupsafe looks `__html__` up on the value itself, so the protocol is checked against the class with+`type_implements` (`renderable_dispatch.py`): a template can build instances, never a class.++This module lives in `tools/` and imports nothing from `core/`, like the other template protocols.+"""++from __future__ import annotations++from typing import Protocol, runtime_checkable+++@runtime_checkable+class HtmlRenderable(Protocol):+ """A value whose `__html__` returns HTML that is safe to insert as it is.++ Implementations:+ - MarkdownContent: the Markdown converted by the shared parser, raw HTML escaped+ """++ def __html__(self) -> str:+ """The HTML to insert as it is, which HTML autoescaping will not escape again."""+ ...diff --git a/pipelex/tools/jinja2/jinja2_environment.py b/pipelex/tools/jinja2/jinja2_environment.pyindex f4662b254..7dbdb1ee8 100644--- a/pipelex/tools/jinja2/jinja2_environment.py+++ b/pipelex/tools/jinja2/jinja2_environment.py@@ -4,6 +4,7 @@ from jinja2 import BaseLoaderfrom pipelex.tools.jinja2.jinja2_sandbox import PipelexTemplateEnvironmentfrom pipelex.tools.jinja2.jinja2_template_registry import TemplateRegistry+from pipelex.tools.jinja2.jinja2_undefined import PresenceProbingStrictUndefinedfrom pipelex.tools.jinja2.template_category import TemplateCategory@@ -12,8 +13,13 @@ def make_jinja2_env_from_loader(template_category: TemplateCategory,loader: BaseLoader,enable_async: bool = True,+ is_undefined_strict: bool = False,) -> PipelexTemplateEnvironment:- """Build the environment for one template category. Every template renders sandboxed (`jinja2_sandbox.py`)."""+ """Build the environment for one template category. Every template renders sandboxed (`jinja2_sandbox.py`).++ With `is_undefined_strict`, a missing value fails the render instead of printing as empty text+ (`jinja2_undefined.py`); `PipeDocGen` asks for it, and every other template keeps Jinja's lenient default.+ """autoescape: booltrim_blocks: boollstrip_blocks: bool@@ -47,6 +53,15 @@ def make_jinja2_env_from_loader(trim_blocks = Falselstrip_blocks = False+ if is_undefined_strict:+ return PipelexTemplateEnvironment(+ loader=loader,+ enable_async=enable_async,+ autoescape=autoescape,+ trim_blocks=trim_blocks,+ lstrip_blocks=lstrip_blocks,+ undefined=PresenceProbingStrictUndefined,+ )return PipelexTemplateEnvironment(loader=loader,enable_async=enable_async,@@ -84,12 +99,14 @@ def make_jinja2_env_without_loader(template_category: TemplateCategory,*,enable_async: bool = True,+ is_undefined_strict: bool = False,) -> PipelexTemplateEnvironment:loader = BaseLoader()jinja2_env = make_jinja2_env_from_loader(template_category=template_category,loader=loader,enable_async=enable_async,+ is_undefined_strict=is_undefined_strict,)_register_filters(jinja2_env, template_category=template_category, enable_async=enable_async)@@ -100,6 +117,7 @@ def make_jinja2_env_from_registry(template_category: TemplateCategory,*,enable_async: bool = True,+ is_undefined_strict: bool = False,) -> PipelexTemplateEnvironment:"""Create Environment with DictLoader from pre-loaded registry.@@ -110,6 +128,7 @@ def make_jinja2_env_from_registry(Args:template_category: The category of templates being rendered.enable_async: Whether to enable async mode for the environment.+ is_undefined_strict: Whether a missing value fails the render instead of printing as empty text.Returns:A Jinja2 Environment with DictLoader and appropriate filters.@@ -119,6 +138,7 @@ def make_jinja2_env_from_registry(template_category=template_category,loader=loader,enable_async=enable_async,+ is_undefined_strict=is_undefined_strict,)_register_filters(jinja2_env, template_category=template_category, enable_async=enable_async)diff --git a/pipelex/tools/jinja2/jinja2_field_paths.py b/pipelex/tools/jinja2/jinja2_field_paths.pynew file mode 100644index 000000000..bb3f40242--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_field_paths.py@@ -0,0 +1,149 @@+"""The field paths a template reads from its inputs, through loop variables included, for checking them at load.++`detect_jinja2_required_variables` answers which inputs a template needs; this answers which fields of them it+reads, so a load-time check can compare each path with the input's concept and refuse `invoice.nosuch` before+any run. A loop variable is followed back to what it iterates: in `{% for item in invoice.line_items %}`,+`item.amount` reads `invoice.line_items[].amount`, the `[]` standing for an item of the list.++It is deliberately conservative, since what it cannot follow is left to the strict undefined at the dry run: a+name set with `{% set %}`, a macro argument, a tuple loop target, and a chain that starts from a subscript, a+call or a filter give no path. Only attribute chains (`a.b.c`) are followed; `a['b']` is not.+"""++from typing import NamedTuple++from jinja2 import nodes+from jinja2.exceptions import TemplateSyntaxError++from pipelex.tools.jinja2.exceptions import Jinja2DetectVariablesError+from pipelex.tools.jinja2.jinja2_environment import make_jinja2_env_without_loader+from pipelex.tools.jinja2.template_category import TemplateCategory++# The segment standing for "an item of the list" in a path reached through a loop variable.+LIST_ITEM_SEGMENT = "[]"+++class TemplateFieldPath(NamedTuple):+ """One attribute chain a template reads, resolved back to the input it starts from."""++ root: str+ segments: tuple[str, ...]+ written: str+ """The chain as the template writes it, such as `item.amount`, for messages."""+++# What a name stands for where it is read: the path it is bound to, or None when it is declared locally and+# cannot be followed (a `set`, a macro argument, `loop`). A name absent from the bindings is an input or a global.+_Bindings = dict[str, tuple[str, ...] | None]+++def _chain(node: nodes.Node) -> tuple[str, list[str]] | None:+ """The name and attribute names of a pure attribute chain (`a.b.c`), or None for any other shape."""+ attributes: list[str] = []+ current = node+ while isinstance(current, nodes.Getattr):+ attributes.append(current.attr)+ current = current.node+ if isinstance(current, nodes.Name):+ return current.name, list(reversed(attributes))+ return None+++def _resolve(*, name: str, attributes: list[str], bindings: _Bindings) -> tuple[str, ...] | None:+ """The full path a chain reads, through the bindings: None when it starts from a name that cannot be followed."""+ if name in bindings:+ bound = bindings[name]+ if bound is None:+ return None+ return (*bound, *attributes)+ return (name, *attributes)+++def _target_names(target: nodes.Node) -> list[str]:+ if isinstance(target, nodes.Name):+ return [target.name]+ if isinstance(target, nodes.Tuple):+ return [name for item in target.items for name in _target_names(item)]+ return []+++def _walk(*, node: nodes.Node, bindings: _Bindings, paths: list[TemplateFieldPath]) -> None:+ if isinstance(node, (nodes.Template, nodes.Scope)):+ _walk_body(body=node.body, bindings=bindings, paths=paths)+ return+ if isinstance(node, nodes.With):+ for value in node.values:+ _walk(node=value, bindings=bindings, paths=paths)+ with_bindings: _Bindings = dict(bindings)+ for target in node.targets:+ for target_name in _target_names(target):+ with_bindings[target_name] = None+ _walk_body(body=node.body, bindings=with_bindings, paths=paths)+ return+ if isinstance(node, nodes.For):+ _walk(node=node.iter, bindings=bindings, paths=paths)+ loop_bindings: _Bindings = {**bindings, "loop": None}+ iterated = _chain(node.iter)+ iterated_path = _resolve(name=iterated[0], attributes=iterated[1], bindings=bindings) if iterated is not None else None+ if isinstance(node.target, nodes.Name) and iterated_path is not None:+ loop_bindings[node.target.name] = (*iterated_path, LIST_ITEM_SEGMENT)+ else:+ for target_name in _target_names(node.target):+ loop_bindings[target_name] = None+ _walk_body(body=node.body, bindings=loop_bindings, paths=paths)+ if node.test is not None:+ _walk(node=node.test, bindings=loop_bindings, paths=paths)+ _walk_body(body=node.else_, bindings=bindings, paths=paths)+ return+ if isinstance(node, nodes.Macro):+ macro_bindings: _Bindings = {**bindings, "caller": None, "varargs": None, "kwargs": None}+ for argument in node.args:+ macro_bindings[argument.name] = None+ for default in node.defaults:+ _walk(node=default, bindings=bindings, paths=paths)+ _walk_body(body=node.body, bindings=macro_bindings, paths=paths)+ return+ if isinstance(node, (nodes.Name, nodes.Getattr)):+ chain = _chain(node)+ if chain is not None:+ if isinstance(node, nodes.Name) and node.ctx != "load":+ return+ name, attributes = chain+ path = _resolve(name=name, attributes=attributes, bindings=bindings)+ if path is not None:+ written = ".".join([name, *attributes])+ paths.append(TemplateFieldPath(root=path[0], segments=path[1:], written=written))+ return+ for child in node.iter_child_nodes():+ _walk(node=child, bindings=bindings, paths=paths)+++def _walk_body(*, body: list[nodes.Node], bindings: _Bindings, paths: list[TemplateFieldPath]) -> None:+ """Walk statements in order: a `{% set %}` declares its name for the statements after it, in this scope."""+ scope_bindings: _Bindings = dict(bindings)+ for statement in body:+ if isinstance(statement, nodes.Macro):+ scope_bindings[statement.name] = None+ _walk(node=statement, bindings=scope_bindings, paths=paths)+ if isinstance(statement, (nodes.Assign, nodes.AssignBlock)):+ for target_name in _target_names(statement.target):+ scope_bindings[target_name] = None+++def detect_template_field_paths(*, template_source: str, template_category: TemplateCategory) -> list[TemplateFieldPath]:+ """Every attribute chain the template reads, in source order, resolved back to the name it starts from.++ The root of a path is an input name or a global of the environment; the caller keeps the ones it knows.++ Raises:+ Jinja2DetectVariablesError: the template does not parse.+ """+ jinja2_env = make_jinja2_env_without_loader(template_category=template_category)+ try:+ parsed_ast = jinja2_env.parse(template_source)+ except TemplateSyntaxError as syntax_error:+ msg = f"Jinja2 detect field paths — syntax error: '{syntax_error}', template_category: {template_category}"+ raise Jinja2DetectVariablesError(msg) from syntax_error+ paths: list[TemplateFieldPath] = []+ _walk(node=parsed_ast, bindings={}, paths=paths)+ return pathsdiff --git a/pipelex/tools/jinja2/jinja2_filters.py b/pipelex/tools/jinja2/jinja2_filters.pyindex d72e7cb60..32607ed12 100644--- a/pipelex/tools/jinja2/jinja2_filters.py+++ b/pipelex/tools/jinja2/jinja2_filters.py@@ -3,13 +3,16 @@ from typing import Anyfrom jinja2 import pass_contextfrom jinja2.runtime import Context, Undefined+from markupsafe import Markupfrom pipelex.tools.jinja2.exceptions import Jinja2ContextError+from pipelex.tools.jinja2.html_renderable import HtmlRenderablefrom pipelex.tools.jinja2.image_registry import ImageRegistryfrom pipelex.tools.jinja2.jinja2_models import Jinja2ContextKeyfrom pipelex.tools.jinja2.renderable_dispatch import type_implementsfrom pipelex.tools.jinja2.tag_renderable import TagRenderablefrom pipelex.tools.jinja2.text_format_renderable import TextFormatRenderable+from pipelex.tools.markdown.markdown_parser import render_markdown_as_htmlfrom pipelex.tools.templating.templating_style import TagStylefrom pipelex.tools.templating.text_format import TextFormat@@ -17,7 +20,7 @@ from pipelex.tools.templating.text_format import TextFormat# Jinja2 filters########################################################################################-ALLOWED_FILTERS = ["tag", "format", "default", "escape_script_tag", "with_images"]+ALLOWED_FILTERS = ["tag", "format", "default", "escape_script_tag", "with_images", "markdown"]def require_templating_style_value(*, context: Context, jinja2_context_key: Jinja2ContextKey) -> str:@@ -50,18 +53,35 @@ async def text_format(context: Context, value: Any, text_format: TextFormat | stif placeholder is not None:return placeholder- applied_text_format: TextFormat+ named_text_format: TextFormat | Noneif text_format:# A template can name its own format — `{{ x | format("markdown") }}` — and Jinja2 hands the# argument over as a raw string, so both that and a `TextFormat` member normalise here. An# unknown name is a template error, reported as one rather than as a bare `ValueError`.try:- applied_text_format = TextFormat(text_format)+ named_text_format = TextFormat(text_format)except ValueError as exc:msg = f"Invalid text format: '{text_format}'"raise Jinja2ContextError(msg) from excelse:- applied_text_format = TextFormat(require_templating_style_value(context=context, jinja2_context_key=Jinja2ContextKey.TEXT_FORMAT))+ named_text_format = None++ # In an HTML template, a value that knows its own HTML (a Markdown stuff) prints as that HTML unless the+ # template names another format, so `$report` prints what `{{ report }}` prints. The result is inserted+ # unescaped, so the value's class must define `__html__`, which a template cannot forge, and the value must+ # answer it: a `StuffArtefact`'s class defines it for every stuff, and only a content that knows its HTML+ # answers it. `isinstance` cannot ask the second question, since from Python 3.12 it reads a protocol's+ # members off the class too.+ if context.eval_ctx.autoescape and type_implements(value=value, protocol=HtmlRenderable) and hasattr(value, "__html__"):+ match named_text_format:+ case None | TextFormat.HTML:+ return Markup(value) # ruff: ignore[unsafe-markup-use] - Markup inserts the __html__ the value's class vouches for+ case TextFormat.PLAIN | TextFormat.MARKDOWN | TextFormat.JSON:+ pass++ applied_text_format = named_text_format or TextFormat(+ require_templating_style_value(context=context, jinja2_context_key=Jinja2ContextKey.TEXT_FORMAT)+ )# Protocol-based rendering, licensed by the value's type (see renderable_dispatch.py)if isinstance(value, TextFormatRenderable) and type_implements(value=value, protocol=TextFormatRenderable):@@ -179,3 +199,21 @@ def escape_script_tag(value: Any) -> Any:if not isinstance(value, str):return valuereturn value.replace("<", "\\u003c")+++def markdown_to_html(value: Any) -> Markup:+ """Render a Markdown text as HTML: the `markdown` filter of HTML templates.++ It is for Markdown held in a plain text field, such as an invoice's notes; a `Markdown` stuff renders as+ HTML by itself. A text stuff renders its text, anything else its string form, and None nothing. An+ undefined value prints the way the template prints one anywhere else: nothing in a lenient template, and a+ render error in a strict one, which is how a misspelled field reached through an alias still fails.++ The parser is Pipelex's one Markdown parser (`markdown_parser.py`): raw HTML in the source is escaped+ rather than passed through, and only URLs with a scheme become links, so the markup is marked safe.+ """+ if value is None:+ return Markup("")+ # `str()` of a strict undefined raises, and of a lenient one is empty.+ source_text = value if isinstance(value, str) else str(value)+ return Markup(render_markdown_as_html(source_text)) # ruff: ignore[unsafe-markup-use] - raw HTML in the source is escaped by the parserdiff --git a/pipelex/tools/jinja2/jinja2_models.py b/pipelex/tools/jinja2/jinja2_models.pyindex 36686c32a..4494f69e5 100644--- a/pipelex/tools/jinja2/jinja2_models.py+++ b/pipelex/tools/jinja2/jinja2_models.py@@ -6,6 +6,7 @@ class Jinja2FilterName(StrEnum):TAG = "tag"ESCAPE_SCRIPT_TAG = "escape_script_tag"WITH_IMAGES = "with_images"+ MARKDOWN = "markdown"class Jinja2ContextKey(StrEnum):diff --git a/pipelex/tools/jinja2/jinja2_rendering.py b/pipelex/tools/jinja2/jinja2_rendering.pyindex 14965551c..4a3bd8ec2 100644--- a/pipelex/tools/jinja2/jinja2_rendering.py+++ b/pipelex/tools/jinja2/jinja2_rendering.py@@ -44,16 +44,19 @@ def _compile_jinja2_template(use_registry: bool = False,enable_async: bool = True,finalize: Callable[[Any], Any] | None = None,+ is_undefined_strict: bool = False,) -> _Jinja2Template:if use_registry:jinja2_env = make_jinja2_env_from_registry(template_category=template_category,enable_async=enable_async,+ is_undefined_strict=is_undefined_strict,)else:jinja2_env = make_jinja2_env_without_loader(template_category=template_category,enable_async=enable_async,+ is_undefined_strict=is_undefined_strict,)if finalize is not None:@@ -206,12 +209,14 @@ async def render_jinja2_async(templating_style: TemplatingStyle | None = None,use_registry: bool = False,finalize: Callable[[Any], Any] | None = None,+ is_undefined_strict: bool = False,) -> str:template = _compile_jinja2_template(template_source=template_source,template_category=template_category,use_registry=use_registry,finalize=finalize,+ is_undefined_strict=is_undefined_strict,)prepared_templating_context = _prepare_templating_context(templating_context=templating_context,diff --git a/pipelex/tools/jinja2/jinja2_undefined.py b/pipelex/tools/jinja2/jinja2_undefined.pynew file mode 100644index 000000000..3a3567a18--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_undefined.py@@ -0,0 +1,25 @@+"""The strict undefined of templates whose missing value must fail the render rather than print as empty text.++Jinja's default undefined prints as nothing, which suits a prompt that may leave a part out but not a document:+a misspelled field in an invoice template would print an invoice with a hole in it. `PipeDocGen` renders its+templates and its filename with this one instead, and so does the environment Pipelex hands a document engine+for filling a Word template, so the misspelling fails at the dry run.++It is Jinja's `StrictUndefined` with one difference: it is falsy rather than raising when tested. Truthiness is+how a Pipelex template probes whether a declared-optional input is present (`{% if notes %}`, which the+optional-input guard lint accepts as a guard, and which the `@?notes` sigil rewrites to), so a strict undefined+that raised on `bool()` would break every guarded template. Printing, iterating, comparing and reading an+attribute of it still raise, and `is defined` and the `default` filter behave as usual.+"""++from jinja2 import StrictUndefined+from typing_extensions import override+++class PresenceProbingStrictUndefined(StrictUndefined):+ """A strict undefined that answers a truthiness test with False instead of raising."""++ # Jinja types StrictUndefined's `__bool__` as never returning, since it raises; answering is this class's point.+ @override+ def __bool__(self) -> bool: # type: ignore[override]+ return Falsediff --git a/pipelex/tools/jinja2/template_category.py b/pipelex/tools/jinja2/template_category.pyindex 527e31c42..c0942246d 100644--- a/pipelex/tools/jinja2/template_category.py+++ b/pipelex/tools/jinja2/template_category.py@@ -2,7 +2,7 @@ from collections.abc import Callablefrom enum import StrEnumfrom typing import Any-from pipelex.tools.jinja2.jinja2_filters import escape_script_tag, tag, text_format+from pipelex.tools.jinja2.jinja2_filters import escape_script_tag, markdown_to_html, tag, text_formatfrom pipelex.tools.jinja2.jinja2_models import Jinja2FilterNamefrom pipelex.tools.jinja2.jinja2_with_images_filter import with_images@@ -26,7 +26,14 @@ class TemplateCategory(StrEnum):}case TemplateCategory.EXPRESSION:return {}- case TemplateCategory.HTML | TemplateCategory.MARKDOWN:+ case TemplateCategory.HTML:+ return {+ Jinja2FilterName.FORMAT: text_format,+ Jinja2FilterName.TAG: tag,+ Jinja2FilterName.ESCAPE_SCRIPT_TAG: escape_script_tag,+ Jinja2FilterName.MARKDOWN: markdown_to_html,+ }+ case TemplateCategory.MARKDOWN:return {Jinja2FilterName.FORMAT: text_format,Jinja2FilterName.TAG: tag,diff --git a/pipelex/tools/markdown/__init__.py b/pipelex/tools/markdown/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/tools/markdown/markdown_parser.py b/pipelex/tools/markdown/markdown_parser.pynew file mode 100644index 000000000..e075c5bce--- /dev/null+++ b/pipelex/tools/markdown/markdown_parser.py@@ -0,0 +1,33 @@+"""The one Markdown parser Pipelex formats Markdown with, whatever it turns it into.++The `markdown` filter of HTML templates, the `Markdown` concept's HTML view and the built-in PDF engine all+read Markdown an LLM or a person wrote, so they parse it the same way, here:++- CommonMark, with tables and strikethrough, which LLM reports use;+- raw HTML disabled, so HTML inside the Markdown shows as text rather than entering the document;+- bare URLs linked, but only those with a scheme: markdown-it's default fuzzy matching links `README.md` to+ `http://README.md` and `config.py` to `http://config.py`, `.md` and `.py` being country domains, so fuzzy+ links and fuzzy emails are off, and `www.` addresses and bare emails stay text.+"""++from functools import cache++from markdown_it import MarkdownIt+++@cache+def get_markdown_parser() -> MarkdownIt:+ """The shared parser, built once. Parsing and rendering keep no state between calls, so it is shared freely."""+ parser = MarkdownIt("commonmark", {"html": False, "linkify": True}).enable(["table", "strikethrough", "linkify"])+ linkify = parser.linkify+ if linkify is None:+ msg = "markdown-it-py found no linkify-it-py: Pipelex requires markdown-it-py[linkify]."+ raise RuntimeError(msg)+ linkify.set({"fuzzy_link": False, "fuzzy_email": False}) # pyright: ignore[reportUnknownMemberType]+ return parser+++def render_markdown_as_html(markdown_text: str) -> str:+ """Markdown as an HTML fragment, with any raw HTML in the source escaped."""+ html: str = get_markdown_parser().render(markdown_text)+ return htmldiff --git a/pipelex/tools/uri/uri_bytes.py b/pipelex/tools/uri/uri_bytes.pynew file mode 100644index 000000000..9920ab56d--- /dev/null+++ b/pipelex/tools/uri/uri_bytes.py@@ -0,0 +1,55 @@+"""Load the bytes behind any URI a value carries, fetching remote content as needed.++It sits beside `uri_base64` and apart from `uri_resolver` for the same reason: resolving a URI is a pure+function the kernel layer needs everywhere, while loading one reaches the network through the fetch helper,+which is kept out of the kernel's import closure. Loading does not check the run's read scope: a caller that+reads a value's URL authorizes it first (`uri_read_scope`).+"""++import base64+from pathlib import Path++from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx+from pipelex.tools.misc.file_utils import load_binary_async+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract+from pipelex.tools.uri.resolved_uri import (+ ResolvedBase64DataUrl,+ ResolvedHttpUrl,+ ResolvedLocalPath,+ ResolvedPipelexStorage,+)+from pipelex.tools.uri.uri_resolver import resolve_uri+++async def load_bytes_from_any_uri(+ uri: str,+ *,+ storage_provider: StorageProviderAbstract | None = None,+) -> bytes:+ """Load the bytes a URI points at, whichever kind of URI it is.++ Args:+ uri: A URI string (http(s)://, a local path, a data: URL, or pipelex-storage://)+ storage_provider: The storage provider that resolves pipelex-storage:// URIs.+ Required when the URI is a pipelex-storage:// URI.++ Raises:+ ValueError: If the URI is a pipelex-storage:// URI and no storage provider is given.+ RemoteFileFetchError: The server answered with an error status, refused the connection, or did not answer in time.+ SsrfBlockedError: The destination, or a redirect hop's, is a private, loopback, link-local or metadata address.+ """+ raw_bytes: bytes+ resolved_uri = resolve_uri(uri)+ match resolved_uri:+ case ResolvedBase64DataUrl():+ raw_bytes = base64.b64decode(resolved_uri.base64_data)+ case ResolvedHttpUrl():+ raw_bytes = await fetch_file_from_url_httpx(url=resolved_uri.url)+ case ResolvedLocalPath():+ raw_bytes = await load_binary_async(path=Path(resolved_uri.path))+ case ResolvedPipelexStorage():+ if storage_provider is None:+ msg = f"Cannot load a pipelex-storage:// URI without a storage provider: {uri}"+ raise ValueError(msg)+ raw_bytes = await storage_provider.load(uri=resolved_uri.storage_uri)+ return raw_bytesdiff --git a/pyproject.toml b/pyproject.tomlindex 37ad30243..221ff6f16 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -30,7 +30,7 @@ dependencies = ["jinja2>=3.1.6","json2html>=1.3.0","kajson==0.7.1",- "markdown>=3.6",+ "markdown-it-py[linkify]>=3.0.0", # the Markdown concept, the `markdown` filter and the built-in PDF engine"mthds==0.17.0","networkx>=3.4.2","openai>=2.0.0",@@ -48,7 +48,7 @@ dependencies = ["pydantic>=2.10.6,<3.0.0","python-dotenv>=1.2.2","PyYAML>=6.0.2",- "reportlab>=4.0,<5", # for generating synthetic PDFs+ "reportlab>=4.0,<5", # the built-in PDF engine of PipeDocGen, and synthetic PDFs in tests"requests>=2.33.0","semantic-version>=2.10.0","shortuuid>=1.0.13",@@ -138,7 +138,6 @@ dev = ["types-aiobotocore[bedrock,bedrock-runtime]>=3.0.0","types-aiofiles>=24.1.0.20240626","types-jsonschema>=4.20.0",- "types-markdown>=3.6.0.20240316","types-networkx>=3.3.0.20241020","types-PyYAML>=6.0.12.20250326",]diff --git a/subject_grants.toml b/subject_grants.tomlindex a87cc8209..1a09c2150 100644--- a/subject_grants.toml+++ b/subject_grants.toml@@ -432,6 +432,10 @@ rationale = "Operates on the inputs_dict; single operand."param = "value"rationale = "Operates on the value; single operand."+["pipelex/cli/commands/run/_main_stuff_file.py::_filename_from_url"]+param = "url"+rationale = "Derives a file name from one URL, its sole operand."+["pipelex/cli/commands/show_cmd.py::do_show_pipe"]param = "pipe_code"rationale = "Shows the pipe identified by pipe_code; single operand."@@ -441,8 +445,8 @@ param = "status"rationale = "Renders the status; single operand."["pipelex/cli/commands/update_cmd.py::_apply_updates"]-param = "deck_dir"-rationale = "Applies deck updates in the deck dir; single operand."+param = "installed_dir"+rationale = "Applies one kit-managed area's updates in its installed dir; single operand."["pipelex/cli/commands/update_cmd.py::_print_status_table"]param = "report"@@ -1280,6 +1284,10 @@ rationale = "Factory from source: builds the result from the search_assignment;param = "func"rationale = "Decorator: wraps the single func operand."+["pipelex/cogt/content_generation/doc_gen_generate.py::render_document_and_store"]+param = "render_assignment"+rationale = "Content-generation leaf: renders and stores the one render assignment, like its sibling leaves."+["pipelex/cogt/content_generation/dry_mock.py::_dry_image_content"]param = "image_url"rationale = "Consumes the image_url; single operand."@@ -1328,6 +1336,10 @@ rationale = "Consumes the object_assignment; single operand."param = "llm_assignment"rationale = "Consumes the llm_assignment; single operand."+["pipelex/cogt/content_generation/dry_mock.py::dry_render_document"]+param = "render_assignment"+rationale = "Dry-run mock of the render leaf: mocks the one render assignment it is handed."+["pipelex/cogt/content_generation/dry_mock.py::dry_render_page_views"]param = "render_assignment"rationale = "Consumes the render_assignment; single operand."@@ -1536,6 +1548,62 @@ rationale = "Worker verb–object: processes the search_assignment; single operaparam = "templating_assignment"rationale = "Processes the templating assignment; single operand."+["pipelex/cogt/doc_gen/doc_gen_format.py::parse_doc_gen_choice_key"]+param = "key"+rationale = "Verb–object: parses the key of a doc gen default; single operand."++["pipelex/cogt/doc_gen/input_shape.py::_content_leaf_kind"]+param = "content_class"+rationale = "Classifies one content class as a leaf kind."++["pipelex/cogt/doc_gen/input_shape.py::_leaf"]+param = "kind"+rationale = "Builds the leaf shape of one kind."++["pipelex/cogt/doc_gen/input_shape.py::_scalar_kind"]+param = "annotation"+rationale = "Classifies one field annotation as a scalar kind."++["pipelex/cogt/doc_gen/input_shape.py::shape_of_class"]+param = "content_class"+rationale = "Describes the shape of one content class."++["pipelex/cogt/doc_gen/layout_builder.py::_flat_columns"]+param = "items"+rationale = "Derives table columns from one list of items."++["pipelex/cogt/doc_gen/layout_builder.py::_is_list_of_scalars"]+param = "value"+rationale = "Predicate over one value."++["pipelex/cogt/doc_gen/layout_builder.py::_is_structure_input"]+param = "content"+rationale = "Predicate over one input content."++["pipelex/cogt/doc_gen/layout_builder.py::_joined_scalars"]+param = "items"+rationale = "Joins one list of scalar items into text."++["pipelex/cogt/doc_gen/layout_builder.py::_paragraphs"]+param = "text"+rationale = "Splits one text into paragraphs."++["pipelex/cogt/doc_gen/layout_builder.py::_scalar"]+param = "value"+rationale = "Converts one value into a layout scalar."++["pipelex/cogt/doc_gen/layout_builder.py::html_to_paragraphs"]+param = "html"+rationale = "Converts one HTML string into paragraphs."++["pipelex/cogt/doc_gen/layout_builder.py::humanize"]+param = "name"+rationale = "Turns one field name into a label."++["pipelex/cogt/doc_gen/plain_data.py::plain_data"]+param = "value"+rationale = "Converts one value into plain data."+["pipelex/cogt/document/prompt_document_utils.py::prep_prompt_documents"]param = "prompt_documents"rationale = "Prepares the prompt documents; single operand."@@ -2084,10 +2152,6 @@ rationale = "Predicate on the manifest_version; single operand."param = "version"rationale = "Deck/file op on the version; single operand."-["pipelex/cogt/models/deck_manifest.py::compute_deck_sync_report"]-param = "deck_dir"-rationale = "Deck/file op on the deck_dir; single operand."-["pipelex/cogt/models/deck_manifest.py::compute_file_sha256"]param = "path"rationale = "Deck/file op on the path; single operand."@@ -2096,21 +2160,25 @@ rationale = "Deck/file op on the path; single operand."param = "data"rationale = "Deck/file op on the data; single operand."+["pipelex/cogt/models/deck_manifest.py::compute_sync_report"]+param = "installed_dir"+rationale = "Kit-managed area op on the installed_dir; single operand."+["pipelex/cogt/models/deck_manifest.py::is_deck_stale_fast"]param = "deck_dir"rationale = "Predicate on the deck_dir; single operand."["pipelex/cogt/models/deck_manifest.py::list_managed_installed_files"]-param = "deck_dir"-rationale = "Deck/file op on the deck_dir; single operand."+param = "installed_dir"+rationale = "Kit-managed area op on the installed_dir; single operand."["pipelex/cogt/models/deck_manifest.py::manifest_path"]-param = "deck_dir"-rationale = "Deck/file op on the deck_dir; single operand."+param = "installed_dir"+rationale = "Kit-managed area op on the installed_dir; single operand."["pipelex/cogt/models/deck_manifest.py::read_manifest"]-param = "deck_dir"-rationale = "Deck/file op on the deck_dir; single operand."+param = "installed_dir"+rationale = "Kit-managed area op on the installed_dir; single operand."["pipelex/cogt/models/deck_manifest.py::status_rich_label"]param = "status"@@ -2140,6 +2208,10 @@ rationale = "Operates on the ref; single operand (qualifiers keyword-only)."param = "waterfall_name"rationale = "Operates on the waterfall_name; single operand (qualifiers keyword-only)."+["pipelex/cogt/models/model_deck.py::ModelDeck._warn_if_ambiguous_doc_gen"]+param = "name"+rationale = "Verb–object: validates/checks the name; single operand."+["pipelex/cogt/models/model_deck.py::ModelDeck._warn_if_ambiguous_extract"]param = "name"rationale = "Verb–object: validates/checks the name; single operand."@@ -4252,6 +4324,30 @@ rationale = "Verb-object: resolves the field blueprint; context stays keyword."param = "item"rationale = "Verb-object: validates the item; expectations stay keyword."+["pipelex/pipe_operators/doc_gen/pipe_doc_gen.py::PipeDocGen._named_contents"]+param = "working_memory"+rationale = "Reads the step's inputs out of one working memory."++["pipelex/pipe_operators/doc_gen/pipe_doc_gen.py::PipeDocGen._title"]+param = "named_contents"+rationale = "Derives the document's title from its one mapping of named contents."++["pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py::bundle_dir_of_source"]+param = "source"+rationale = "Resolves the bundle directory of one source."++["pipelex/pipe_operators/doc_gen/template_field_check.py::_describe_fields"]+param = "model_class"+rationale = "Lists the fields of one model class."++["pipelex/pipe_operators/doc_gen/template_field_check.py::_list_item_type"]+param = "annotation"+rationale = "Reads the item type of one list annotation."++["pipelex/pipe_operators/doc_gen/template_field_check.py::_unwrap"]+param = "annotation"+rationale = "Unwraps one annotation from Optional and Annotated."+["pipelex/pipe_operators/func/pipe_func_execution_transport.py::pipe_func_execution_result_from_response"]param = "response"rationale = "Converter: rebinds a transported response into a result; single self-labelling operand."@@ -5344,6 +5440,10 @@ rationale = "Plugin protocol register: the registrar is the sole sink the pluginparam = "pdf_uri"rationale = "Resolves the PDF URI to a readable local source; single operand."+["pipelex/providers/reportlab/reportlab_plugin.py::ReportlabDocGenPlugin.register"]+param = "registrar"+rationale = "Plugin protocol register: the registrar is the sole sink the plugin populates, as for every built-in plugin."+["pipelex/providers/secrets/secrets_plugin.py::SecretsPlugin.register"]param = "registrar"rationale = "Duck-typed plugin protocol: discovery invokes plugin.register(registrar) positionally."@@ -5948,10 +6048,22 @@ rationale = "Factory from source: builds the result from the template_category;param = "template_category"rationale = "Factory from source: builds the result from the template_category; single operand."+["pipelex/tools/jinja2/jinja2_field_paths.py::_chain"]+param = "node"+rationale = "Reads the attribute chain of one Jinja node."++["pipelex/tools/jinja2/jinja2_field_paths.py::_target_names"]+param = "target"+rationale = "Lists the names one assignment target binds."+["pipelex/tools/jinja2/jinja2_filters.py::escape_script_tag"]param = "value"rationale = "Transforms the value; single operand."+["pipelex/tools/jinja2/jinja2_filters.py::markdown_to_html"]+param = "value"+rationale = "Jinja filter: Jinja passes the filtered value positionally."+["pipelex/tools/jinja2/jinja2_optional_guards.py::_GuardWalker._assign_target_names"]param = "target"rationale = "Noun-named derivation: the assigned names of the target node; single operand."@@ -6136,6 +6248,10 @@ rationale = "Logs the content; single operand (formatting keyword-only)."param = "channel_name"rationale = "Renders the channel_name; single operand."+["pipelex/tools/markdown/markdown_parser.py::render_markdown_as_html"]+param = "markdown_text"+rationale = "Renders one Markdown text as HTML."+["pipelex/tools/mermaid/mermaid_utils.py::clean_str_for_mermaid_node_title"]param = "text"rationale = "Transforms the text; single operand."@@ -7028,6 +7144,10 @@ rationale = "Factory from source: builds the result from the uri; single operandparam = "url"rationale = "Factory from source: builds the result from the url; single operand."+["pipelex/tools/uri/uri_bytes.py::load_bytes_from_any_uri"]+param = "uri"+rationale = "Loads the bytes one URI names."+["pipelex/tools/uri/uri_resolver.py::_resolve_base64_data_url"]param = "uri"rationale = "Parses/reads the uri; single operand."diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex 61c459b1f..3185abcf0 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -54,6 +54,10 @@ CsvFlatnessError | CSV flatness error | https://docs.pipelex.com/latest/errors/cCsvReadError | CSV read error | https://docs.pipelex.com/latest/errors/csv-read-error/DateContentError | Date content | https://docs.pipelex.com/latest/errors/date-content-error/DeliveryError | Delivery | https://docs.pipelex.com/latest/errors/delivery-error/+DocGenEngineMissingError | Doc gen engine missing | https://docs.pipelex.com/latest/errors/doc-gen-engine-missing-error/+DocGenHandleNotFoundError | Doc gen handle not found | https://docs.pipelex.com/latest/errors/doc-gen-handle-not-found-error/+DocGenModelCapabilityError | Doc gen model capability | https://docs.pipelex.com/latest/errors/doc-gen-model-capability-error/+DocGenRenderError | Doc gen render | https://docs.pipelex.com/latest/errors/doc-gen-render-error/DomainCodeError | Domain code | https://docs.pipelex.com/latest/errors/domain-code-error/DomainFactoryError | Domain factory | https://docs.pipelex.com/latest/errors/domain-factory-error/DomainLibraryError | Domain library | https://docs.pipelex.com/latest/errors/domain-library-error/@@ -67,8 +71,10 @@ DryRunGraphNotProducedError | Dry run graph not produced | https://docs.pipelex.DryRunMockBuildError | Dry run mock build | https://docs.pipelex.com/latest/errors/dry-run-mock-build-error/DryRunObjectFidelityError | Dry run object fidelity | https://docs.pipelex.com/latest/errors/dry-run-object-fidelity-error/DuplicateBundleValidatorError | Duplicate bundle validator | https://docs.pipelex.com/latest/errors/duplicate-bundle-validator-error/+DuplicateDocGenDefaultError | Duplicate doc gen default | https://docs.pipelex.com/latest/errors/duplicate-doc-gen-default-error/DuplicateHttpErrorMapperError | Duplicate http error mapper | https://docs.pipelex.com/latest/errors/duplicate-http-error-mapper-error/DuplicateInferenceBackendError | Duplicate inference backend | https://docs.pipelex.com/latest/errors/duplicate-inference-backend-error/+DuplicateInternalModelError | Duplicate internal model | https://docs.pipelex.com/latest/errors/duplicate-internal-model-error/DuplicateLogSinkError | Duplicate log sink | https://docs.pipelex.com/latest/errors/duplicate-log-sink-error/DuplicateModelListerError | Duplicate model lister | https://docs.pipelex.com/latest/errors/duplicate-model-lister-error/DuplicateOrchestratorError | Duplicate orchestrator | https://docs.pipelex.com/latest/errors/duplicate-orchestrator-error/@@ -216,6 +222,10 @@ PipeComposeFactoryError | Pipe compose factory | https://docs.pipelex.com/latestPipeConditionFactoryError | Pipe condition factory | https://docs.pipelex.com/latest/errors/pipe-condition-factory-error/PipeControllerError | Pipe controller | https://docs.pipelex.com/latest/errors/pipe-controller-error/PipeControllerOutputConceptMismatchError | Pipe controller output concept mismatch | https://docs.pipelex.com/latest/errors/pipe-controller-output-concept-mismatch-error/+PipeDocGenFactoryError | Pipe doc gen factory | https://docs.pipelex.com/latest/errors/pipe-doc-gen-factory-error/+PipeDocGenRunError | Pipe doc gen run | https://docs.pipelex.com/latest/errors/pipe-doc-gen-run-error/+PipeDocGenTemplateCheckError | Pipe doc gen template check | https://docs.pipelex.com/latest/errors/pipe-doc-gen-template-check-error/+PipeDocGenUndefinedValueError | Pipe doc gen undefined value | https://docs.pipelex.com/latest/errors/pipe-doc-gen-undefined-value-error/PipeExecutionError | Pipe execution | https://docs.pipelex.com/latest/errors/pipe-execution-error/PipeExtractFactoryError | Pipe extract factory | https://docs.pipelex.com/latest/errors/pipe-extract-factory-error/PipeFactoryError | Pipe factory | https://docs.pipelex.com/latest/errors/pipe-factory-error/@@ -265,6 +275,7 @@ PluginApiVersionMismatchError | Plugin api version mismatch | https://docs.pipelPluginDeclaredInMultipleGroupsError | Plugin declared in multiple groups | https://docs.pipelex.com/latest/errors/plugin-declared-in-multiple-groups-error/PluginError | Plugin error | https://docs.pipelex.com/latest/errors/plugin-error/PluginLayerViolationError | Plugin layer violation | https://docs.pipelex.com/latest/errors/plugin-layer-violation-error/+PluginModelDeclarationError | Plugin model declaration | https://docs.pipelex.com/latest/errors/plugin-model-declaration-error/PortkeyCredentialsError | Portkey credentials | https://docs.pipelex.com/latest/errors/portkey-credentials-error/PortkeyError | Portkey | https://docs.pipelex.com/latest/errors/portkey-error/PortkeyFactoryError | Portkey factory | https://docs.pipelex.com/latest/errors/portkey-factory-error/diff --git a/tests/e2e/pipelex/pipes/markdown/test_markdown_dry_run.py b/tests/e2e/pipelex/pipes/markdown/test_markdown_dry_run.pynew file mode 100644index 000000000..ff60b399e--- /dev/null+++ b/tests/e2e/pipelex/pipes/markdown/test_markdown_dry_run.py@@ -0,0 +1,40 @@+import pytest++from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.pipeline.pipeline_response import RunState+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.test_extras.mthds_corpus.loader import get_entry++# The bundle is a corpus entry, not a fixture local to this test: the corpus is the single source+# for language-level `.mthds` methods, and covering `native.markdown` is exactly why the entry exists.+_FIXTURE_DIR = get_entry(name="native_markdown_inspection_report").directory++[email protected](loop_scope="class")+class TestMarkdownDryRun:+ async def test_dry_run_markdown_report_feeds_a_text_step(self):+ """The sequence's LLM step writes a Markdown report, which the next step reads as its Text input."""+ runner = PipelexMTHDSProtocol(library_dirs=[str(_FIXTURE_DIR)], pipe_run_mode=PipeRunMode.DRY)++ response = await runner.execute(pipe_code="report_and_log", inputs={"notes": "Roof: two cracked tiles above the north gable."})++ assert response.state == RunState.COMPLETED+ report_stuff = response.pipe_output.working_memory.get_stuff("report")+ assert report_stuff.concept.concept_ref == "native.Markdown"+ assert type(report_stuff.content) is MarkdownContent+ assert report_stuff.as_markdown.text+ assert response.pipe_output.main_stuff.as_text.text++ async def test_dry_run_markdown_envelope_fills_a_text_input(self):+ """A Markdown input in the envelope form fills a Text input and stays a Markdown in memory."""+ runner = PipelexMTHDSProtocol(library_dirs=[str(_FIXTURE_DIR)], pipe_run_mode=PipeRunMode.DRY)+ report_source = "# Roof\n\n- two cracked tiles\n- sagging gutter"++ response = await runner.execute(pipe_code="log_report", inputs={"report": {"concept": "Markdown", "content": report_source}})++ assert response.state == RunState.COMPLETED+ report_stuff = response.pipe_output.working_memory.get_stuff("report")+ assert report_stuff.concept.concept_ref == "native.Markdown"+ assert report_stuff.content == MarkdownContent(text=report_source)+ assert response.pipe_output.main_stuff.as_text.textdiff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/README.md b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/README.mdnew file mode 100644index 000000000..81a8477e6--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/README.md@@ -0,0 +1,61 @@+# The PipeDocGen playground++This directory is where to try methods that end in a `PipeDocGen` step and write a PDF you can open. Open Pipelex prints a `pdf` without a template, laid out from the step's inputs by the built-in ReportLab engine, so everything here runs with nothing more than `make install`. A `pdf` from an HTML template, and the `xlsx`, `docx` and `pptx` formats, need the Pipelex document generation plugin, and without it such a step is refused when the method loads. Run the commands below from the repository's root.++## Printing without inference++Each `PipeDocGen` step runs alone on a saved inputs file, so it spends nothing on inference. The file lands in `results/<pipe>_output_NN/`, under its own name, and the command prints its path.++```bash+P=tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen+.venv/bin/pipelex run bundle $P/invoice --pipe render_invoice_pdf --inputs $P/invoice/inputs.json+.venv/bin/pipelex run bundle $P/report --pipe render_report_pdf --inputs $P/report/render_inputs.json+```++| Pipe | What comes out |+| --- | --- |+| `render_invoice_pdf` | The invoice of `invoice/inputs.json`, with sixty line items, laid out from its structure alone: the fields in a grid, the customer as a section, the line items as a table whose header repeats on every page, a running title and "Page N of M" |+| `render_report_pdf` | The Markdown report of `report/render_inputs.json`, formatted: headings, emphasis, lists, a table, a code block and links |++**This is the loop for working on a document**: change the inputs or the concept, rerun the pipe, open the file. To print other data, write an inputs file shaped like `invoice/inputs.json`, one entry per input with its concept and content.++## End to end, with inference++An LLM writes a report in Markdown on the topic in `report/inputs.json`, and the last step prints it formatted:++```bash+P=tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen+.venv/bin/pipelex run bundle $P/report --inputs $P/report/inputs.json+```++An LLM invents an invoice from the brief in `invoice/brief_inputs.json`, and the last step prints it:++```bash+P=tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen+.venv/bin/pipelex run bundle $P/invoice --inputs $P/invoice/brief_inputs.json+```++## Writing a PipeDocGen step++```toml+[concept.InvoicePdf]+description = "The invoice laid out as a printable PDF"+refines = "Document"++[pipe.render_invoice_pdf]+type = "PipeDocGen"+description = "Lay out the invoice as a paginated PDF from its structure alone"+inputs = { invoice = "Invoice" }+output = "InvoicePdf"+format = "pdf"+filename = "invoice-{{ invoice.number }}"+```++- **`format`** is `pdf`, `xlsx`, `docx` or `pptx`; open Pipelex prints a `pdf` without a template.+- **`model`** names the document engine, such as `reportlab-pdf`, or `pipelex-pdf` with the plugin. Without it, the model deck's default for the format prints the file, which for a `pdf` without a template is `reportlab-pdf`.+- **`filename`** is a Jinja expression over the inputs, and the suffix is added. It defaults to the pipe's code. It renders strictly, so a field the inputs do not have fails the step, and its field paths are checked against the inputs' concepts when the method loads.+- **The output** is `Document` or a concept that refines it, and a single file.+- **The inputs** each get a section when there are several, in the order the step declares them; a single input is the document itself.+- **A report an LLM writes** prints formatted when its output is declared `Markdown`; declared `Text`, its `#` and `**` print as they are.++The operator's page in the docs, `docs/building-methods/pipes/pipe-operators/PipeDocGen.md`, covers templates and what each format needs.diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/brief_inputs.json b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/brief_inputs.jsonnew file mode 100644index 000000000..7cbbd84d3--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/brief_inputs.json@@ -0,0 +1,8 @@+{+ "brief": {+ "concept": "native.Text",+ "content": {+ "text": "A two-week consulting engagement to design document-processing methods for a French insurance company"+ }+ }+}\ No newline at end of filediff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/inputs.json b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/inputs.jsonnew file mode 100644index 000000000..7be18165d--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/inputs.json@@ -0,0 +1,379 @@+{+ "invoice": {+ "concept": "invoice_rendering.Invoice",+ "content": {+ "number": "INV-2026-0142",+ "issue_date": "2026-09-29",+ "due_date": "2026-10-29",+ "customer": {+ "name": "Evotis S.A.S.",+ "address": "12 rue de la Paix, 75002 Paris"+ },+ "currency": "EUR",+ "line_items": [+ {+ "description": "Method design workshop — day 1",+ "quantity": 1,+ "unit_price": 850.0,+ "amount": 850.0+ },+ {+ "description": "Pipeline implementation — day 2",+ "quantity": 2,+ "unit_price": 860.0,+ "amount": 1720.0+ },+ {+ "description": "Prompt engineering review — day 3",+ "quantity": 3,+ "unit_price": 870.0,+ "amount": 2610.0+ },+ {+ "description": "Structured output hardening — day 4",+ "quantity": 1,+ "unit_price": 880.0,+ "amount": 880.0+ },+ {+ "description": "Document extraction tuning — day 5",+ "quantity": 2,+ "unit_price": 890.0,+ "amount": 1780.0+ },+ {+ "description": "Integration testing — day 6",+ "quantity": 3,+ "unit_price": 850.0,+ "amount": 2550.0+ },+ {+ "description": "Deployment support — day 7",+ "quantity": 1,+ "unit_price": 860.0,+ "amount": 860.0+ },+ {+ "description": "Training session — day 8",+ "quantity": 2,+ "unit_price": 870.0,+ "amount": 1740.0+ },+ {+ "description": "Method design workshop — day 9",+ "quantity": 3,+ "unit_price": 880.0,+ "amount": 2640.0+ },+ {+ "description": "Pipeline implementation — day 10",+ "quantity": 1,+ "unit_price": 890.0,+ "amount": 890.0+ },+ {+ "description": "Prompt engineering review — day 11",+ "quantity": 2,+ "unit_price": 850.0,+ "amount": 1700.0+ },+ {+ "description": "Structured output hardening — day 12",+ "quantity": 3,+ "unit_price": 860.0,+ "amount": 2580.0+ },+ {+ "description": "Document extraction tuning — day 13",+ "quantity": 1,+ "unit_price": 870.0,+ "amount": 870.0+ },+ {+ "description": "Integration testing — day 14",+ "quantity": 2,+ "unit_price": 880.0,+ "amount": 1760.0+ },+ {+ "description": "Deployment support — day 15",+ "quantity": 3,+ "unit_price": 890.0,+ "amount": 2670.0+ },+ {+ "description": "Training session — day 16",+ "quantity": 1,+ "unit_price": 850.0,+ "amount": 850.0+ },+ {+ "description": "Method design workshop — day 17",+ "quantity": 2,+ "unit_price": 860.0,+ "amount": 1720.0+ },+ {+ "description": "Pipeline implementation — day 18",+ "quantity": 3,+ "unit_price": 870.0,+ "amount": 2610.0+ },+ {+ "description": "Prompt engineering review — day 19",+ "quantity": 1,+ "unit_price": 880.0,+ "amount": 880.0+ },+ {+ "description": "Structured output hardening — day 20",+ "quantity": 2,+ "unit_price": 890.0,+ "amount": 1780.0+ },+ {+ "description": "Document extraction tuning — day 21",+ "quantity": 3,+ "unit_price": 850.0,+ "amount": 2550.0+ },+ {+ "description": "Integration testing — day 22",+ "quantity": 1,+ "unit_price": 860.0,+ "amount": 860.0+ },+ {+ "description": "Deployment support — day 23",+ "quantity": 2,+ "unit_price": 870.0,+ "amount": 1740.0+ },+ {+ "description": "Training session — day 24",+ "quantity": 3,+ "unit_price": 880.0,+ "amount": 2640.0+ },+ {+ "description": "Method design workshop — day 25",+ "quantity": 1,+ "unit_price": 890.0,+ "amount": 890.0+ },+ {+ "description": "Pipeline implementation — day 26",+ "quantity": 2,+ "unit_price": 850.0,+ "amount": 1700.0+ },+ {+ "description": "Prompt engineering review — day 27",+ "quantity": 3,+ "unit_price": 860.0,+ "amount": 2580.0+ },+ {+ "description": "Structured output hardening — day 28",+ "quantity": 1,+ "unit_price": 870.0,+ "amount": 870.0+ },+ {+ "description": "Document extraction tuning — day 29",+ "quantity": 2,+ "unit_price": 880.0,+ "amount": 1760.0+ },+ {+ "description": "Integration testing — day 30",+ "quantity": 3,+ "unit_price": 890.0,+ "amount": 2670.0+ },+ {+ "description": "Deployment support — day 31",+ "quantity": 1,+ "unit_price": 850.0,+ "amount": 850.0+ },+ {+ "description": "Training session — day 32",+ "quantity": 2,+ "unit_price": 860.0,+ "amount": 1720.0+ },+ {+ "description": "Method design workshop — day 33",+ "quantity": 3,+ "unit_price": 870.0,+ "amount": 2610.0+ },+ {+ "description": "Pipeline implementation — day 34",+ "quantity": 1,+ "unit_price": 880.0,+ "amount": 880.0+ },+ {+ "description": "Prompt engineering review — day 35",+ "quantity": 2,+ "unit_price": 890.0,+ "amount": 1780.0+ },+ {+ "description": "Structured output hardening — day 36",+ "quantity": 3,+ "unit_price": 850.0,+ "amount": 2550.0+ },+ {+ "description": "Document extraction tuning — day 37",+ "quantity": 1,+ "unit_price": 860.0,+ "amount": 860.0+ },+ {+ "description": "Integration testing — day 38",+ "quantity": 2,+ "unit_price": 870.0,+ "amount": 1740.0+ },+ {+ "description": "Deployment support — day 39",+ "quantity": 3,+ "unit_price": 880.0,+ "amount": 2640.0+ },+ {+ "description": "Training session — day 40",+ "quantity": 1,+ "unit_price": 890.0,+ "amount": 890.0+ },+ {+ "description": "Method design workshop — day 41",+ "quantity": 2,+ "unit_price": 850.0,+ "amount": 1700.0+ },+ {+ "description": "Pipeline implementation — day 42",+ "quantity": 3,+ "unit_price": 860.0,+ "amount": 2580.0+ },+ {+ "description": "Prompt engineering review — day 43",+ "quantity": 1,+ "unit_price": 870.0,+ "amount": 870.0+ },+ {+ "description": "Structured output hardening — day 44",+ "quantity": 2,+ "unit_price": 880.0,+ "amount": 1760.0+ },+ {+ "description": "Document extraction tuning — day 45",+ "quantity": 3,+ "unit_price": 890.0,+ "amount": 2670.0+ },+ {+ "description": "Integration testing — day 46",+ "quantity": 1,+ "unit_price": 850.0,+ "amount": 850.0+ },+ {+ "description": "Deployment support — day 47",+ "quantity": 2,+ "unit_price": 860.0,+ "amount": 1720.0+ },+ {+ "description": "Training session — day 48",+ "quantity": 3,+ "unit_price": 870.0,+ "amount": 2610.0+ },+ {+ "description": "Method design workshop — day 49",+ "quantity": 1,+ "unit_price": 880.0,+ "amount": 880.0+ },+ {+ "description": "Pipeline implementation — day 50",+ "quantity": 2,+ "unit_price": 890.0,+ "amount": 1780.0+ },+ {+ "description": "Prompt engineering review — day 51",+ "quantity": 3,+ "unit_price": 850.0,+ "amount": 2550.0+ },+ {+ "description": "Structured output hardening — day 52",+ "quantity": 1,+ "unit_price": 860.0,+ "amount": 860.0+ },+ {+ "description": "Document extraction tuning — day 53",+ "quantity": 2,+ "unit_price": 870.0,+ "amount": 1740.0+ },+ {+ "description": "Integration testing — day 54",+ "quantity": 3,+ "unit_price": 880.0,+ "amount": 2640.0+ },+ {+ "description": "Deployment support — day 55",+ "quantity": 1,+ "unit_price": 890.0,+ "amount": 890.0+ },+ {+ "description": "Training session — day 56",+ "quantity": 2,+ "unit_price": 850.0,+ "amount": 1700.0+ },+ {+ "description": "Method design workshop — day 57",+ "quantity": 3,+ "unit_price": 860.0,+ "amount": 2580.0+ },+ {+ "description": "Pipeline implementation — day 58",+ "quantity": 1,+ "unit_price": 870.0,+ "amount": 870.0+ },+ {+ "description": "Prompt engineering review — day 59",+ "quantity": 2,+ "unit_price": 880.0,+ "amount": 1760.0+ },+ {+ "description": "Structured output hardening — day 60",+ "quantity": 3,+ "unit_price": 890.0,+ "amount": 2670.0+ }+ ],+ "total": 104400.0,+ "notes": "**Payment terms**\n\n- Bank transfer within 30 days\n- Late payments bear interest at three times the legal rate\n\nThank you for your trust."+ }+ }+}\ No newline at end of filediff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/invoice.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/invoice.mthdsnew file mode 100644index 000000000..91d917e29--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/invoice/invoice.mthds@@ -0,0 +1,76 @@+domain = "invoice_rendering"+description = "One invoice generated as a PDF laid out from its structure"+main_pipe = "invent_and_render_invoice"++[concept.LineItem]+description = "One line of an invoice"++[concept.LineItem.structure]+description = { type = "text", description = "What was sold", required = true }+quantity = { type = "integer", description = "How many units", required = true }+unit_price = { type = "number", description = "The price of one unit, before tax", required = true }+amount = { type = "number", description = "The quantity times the unit price", required = true }++[concept.Customer]+description = "The customer an invoice is addressed to"++[concept.Customer.structure]+name = { type = "text", description = "The customer's company name", required = true }+address = { type = "text", description = "The customer's postal address, on one line", required = true }++[concept.Invoice]+description = "An invoice with its customer and line items"++[concept.Invoice.structure]+number = { type = "text", description = "The invoice number, such as INV-2026-0142", required = true }+issue_date = { type = "date", description = "The day the invoice was issued", required = true }+due_date = { type = "date", description = "The day payment is due", required = true }+customer = { type = "concept", concept_ref = "invoice_rendering.Customer", description = "Who the invoice is addressed to", required = true }+currency = { type = "text", description = "The ISO currency code, such as EUR", required = true }+line_items = { type = "list", item_type = "concept", item_concept_ref = "invoice_rendering.LineItem", description = "What was sold", required = true }+total = { type = "number", description = "The sum of the line items' amounts", required = true }+notes = { type = "text", description = "Payment terms and remarks", required = false }++[concept.InvoicePdf]+description = "The invoice laid out as a printable PDF"+refines = "Document"++########################################################################################################################+# End to end: an LLM invents the invoice, the last step prints it+########################################################################################################################++[pipe.invent_and_render_invoice]+type = "PipeSequence"+description = "Invent a plausible invoice from a one-line brief, then print it as a PDF"+inputs = { brief = "Text" }+output = "InvoicePdf"+steps = [+ { pipe = "invent_invoice", result = "invoice" },+ { pipe = "render_invoice_pdf", result = "invoice_pdf" },+]++[pipe.invent_invoice]+type = "PipeLLM"+description = "Invent a plausible invoice from a one-line brief"+inputs = { brief = "Text" }+output = "Invoice"+model = "$writing-factual-cheap"+prompt = """+Invent a plausible invoice for this brief:++@brief++Use between five and twelve line items. Each amount is the quantity times the unit price, and the total is the sum of the amounts. Write the notes as two short sentences on the payment terms.+"""++########################################################################################################################+# The PDF, laid out from the invoice's structure with no template+########################################################################################################################++[pipe.render_invoice_pdf]+type = "PipeDocGen"+description = "Lay out the invoice as a paginated PDF from its structure alone"+inputs = { invoice = "Invoice" }+output = "InvoicePdf"+format = "pdf"+filename = "invoice-{{ invoice.number }}"diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/inputs.json b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/inputs.jsonnew file mode 100644index 000000000..ea89479da--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/inputs.json@@ -0,0 +1,8 @@+{+ "topic": {+ "concept": "native.Text",+ "content": {+ "text": "How small manufacturers can use document-processing AI to cut the time they spend on supplier invoices"+ }+ }+}\ No newline at end of filediff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/render_inputs.json b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/render_inputs.jsonnew file mode 100644index 000000000..1445fe834--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/render_inputs.json@@ -0,0 +1,8 @@+{+ "report": {+ "concept": "native.Markdown",+ "content": {+ "text": "# Quarterly review of invoice automation\n\nThis report summarises what six sites learnt from automating supplier invoices.\n\n## 1. Findings from site 1\n\nThe **pipeline** processed *120* supplier invoices this quarter. Key points:\n\n- extraction accuracy rose to 91%\n- median handling time fell to 11 minutes\n- <script>alert('raw HTML stays text')</script> is shown, not run\n\n| Metric | Value |\n|---|---|\n| Invoices | 120 |\n| Errors | 1 |\n\n> Reviewers flagged 1 invoices for a second look.\n\n## 2. Findings from site 2\n\nThe **pipeline** processed *240* supplier invoices this quarter. Key points:\n\n- extraction accuracy rose to 92%\n- median handling time fell to 10 minutes\n- <script>alert('raw HTML stays text')</script> is shown, not run\n\n| Metric | Value |\n|---|---|\n| Invoices | 240 |\n| Errors | 2 |\n\n> Reviewers flagged 2 invoices for a second look.\n\n## 3. Findings from site 3\n\nThe **pipeline** processed *360* supplier invoices this quarter. Key points:\n\n- extraction accuracy rose to 93%\n- median handling time fell to 9 minutes\n- <script>alert('raw HTML stays text')</script> is shown, not run\n\n| Metric | Value |\n|---|---|\n| Invoices | 360 |\n| Errors | 3 |\n\n> Reviewers flagged 3 invoices for a second look.\n\n## 4. Findings from site 4\n\nThe **pipeline** processed *480* supplier invoices this quarter. Key points:\n\n- extraction accuracy rose to 94%\n- median handling time fell to 8 minutes\n- <script>alert('raw HTML stays text')</script> is shown, not run\n\n| Metric | Value |\n|---|---|\n| Invoices | 480 |\n| Errors | 4 |\n\n> Reviewers flagged 4 invoices for a second look.\n\n## 5. Findings from site 5\n\nThe **pipeline** processed *600* supplier invoices this quarter. Key points:\n\n- extraction accuracy rose to 95%\n- median handling time fell to 7 minutes\n- <script>alert('raw HTML stays text')</script> is shown, not run\n\n| Metric | Value |\n|---|---|\n| Invoices | 600 |\n| Errors | 5 |\n\n> Reviewers flagged 5 invoices for a second look.\n\n## 6. Findings from site 6\n\nThe **pipeline** processed *720* supplier invoices this quarter. Key points:\n\n- extraction accuracy rose to 96%\n- median handling time fell to 6 minutes\n- <script>alert('raw HTML stays text')</script> is shown, not run\n\n| Metric | Value |\n|---|---|\n| Invoices | 720 |\n| Errors | 6 |\n\n> Reviewers flagged 6 invoices for a second look.\n\n## Method\n\nEvery site ran the same extraction settings:\n\n```toml\nconfidence_threshold = 0.9\nsecond_look = \"flagged\"\n```\n\nThe figures behind each site are on [the automation dashboard](https://example.com/invoice-automation).\n\n## Conclusion\n\nAutomation paid for itself within the quarter at every site."+ }+ }+}\ No newline at end of filediff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/report.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/report.mthdsnew file mode 100644index 000000000..937db55bb--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/report/report.mthds@@ -0,0 +1,39 @@+domain = "report_rendering"+description = "An LLM writes a Markdown report, and the last step prints it as a PDF"+main_pipe = "write_and_render_report"++[concept.ReportPdf]+description = "A report printed as a PDF"+refines = "Document"++[pipe.write_and_render_report]+type = "PipeSequence"+description = "Write a Markdown report on the topic, then print it as a PDF"+inputs = { topic = "Text" }+output = "ReportPdf"+steps = [+ { pipe = "write_report", result = "report" },+ { pipe = "render_report_pdf", result = "report_pdf" },+]++[pipe.write_report]+type = "PipeLLM"+description = "Write a structured report in Markdown"+inputs = { topic = "Text" }+output = "Markdown"+model = "$writing-factual"+prompt = """+Write a report of about a thousand words on this topic:++@topic++Write it in Markdown: a title as a level-one heading, an executive summary, four to six sections with level-two headings, at least one bulleted list and one table, and a short conclusion. Return only the Markdown.+"""++[pipe.render_report_pdf]+type = "PipeDocGen"+description = "Print the Markdown report as a PDF, formatted"+inputs = { report = "Markdown" }+output = "ReportPdf"+format = "pdf"+filename = "report"diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_playground.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_playground.pynew file mode 100644index 000000000..f1afa01ef--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_playground.py@@ -0,0 +1,64 @@+import json+from pathlib import Path+from typing import Any++import pypdfium2 as pdfium+import pytest++from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.runtime_hub import get_storage_provider+from pipelex.tools.uri.uri_bytes import load_bytes_from_any_uri++PLAYGROUND_DIR = Path("tests/e2e/pipelex/pipes/pipe_operators/pipe_doc_gen")+++def _inputs(relative_path: str) -> dict[str, Any]:+ inputs: dict[str, Any] = json.loads((PLAYGROUND_DIR / relative_path).read_text(encoding="utf-8"))+ return inputs+++async def _print(*, bundle_dir: str, pipe_code: str, inputs_file: str) -> tuple[DocumentContent, list[str]]:+ """Run one PipeDocGen step alone, and return its document with the text of each page."""+ result = await PipelexMTHDSProtocol(library_dirs=[str(PLAYGROUND_DIR / bundle_dir)]).execute(pipe_code=pipe_code, inputs=_inputs(inputs_file))+ document = result.pipe_output.main_stuff_as(content_type=DocumentContent)+ pdf_bytes = await load_bytes_from_any_uri(document.url, storage_provider=get_storage_provider())+ assert pdf_bytes.startswith(b"%PDF-")+ pdf = pdfium.PdfDocument(pdf_bytes)+ try:+ page_texts: list[str] = []+ for page_index in range(len(pdf)):+ page_texts.append(pdf[page_index].get_textpage().get_text_bounded()) # pyright: ignore[reportUnknownMemberType]+ finally:+ pdf.close()+ return document, page_texts++[email protected](loop_scope="class")+class TestPipeDocGenPlayground:+ async def test_the_invoice_prints_over_several_pages_from_its_structure(self) -> None:+ document, page_texts = await _print(bundle_dir="invoice", pipe_code="render_invoice_pdf", inputs_file="invoice/inputs.json")++ assert document.filename == "invoice-INV-2026-0142.pdf"+ assert document.mime_type == "application/pdf"+ assert len(page_texts) > 1+ assert "INV-2026-0142" in page_texts[0]+ assert "Evotis S.A.S." in page_texts[0]+ assert f"Page 1 of {len(page_texts)}" in page_texts[0]+ assert f"Page {len(page_texts)} of {len(page_texts)}" in page_texts[-1]+ # The table's header row repeats on every page it runs over.+ assert "Description" in page_texts[1]++ async def test_the_markdown_report_prints_formatted(self) -> None:+ document, page_texts = await _print(bundle_dir="report", pipe_code="render_report_pdf", inputs_file="report/render_inputs.json")++ assert document.filename == "report.pdf"+ full_text = "\n".join(page_texts)+ assert "Quarterly review of invoice automation" in full_text+ assert "**" not in full_text+ assert "# Quarterly" not in full_text+ assert "<script>" in full_text+ assert "confidence_threshold = 0.9" in full_text+ assert "```" not in full_text+ assert "the automation dashboard" in full_text+ assert "](https://" not in full_textdiff --git a/tests/integration/pipelex/cogt/test_doc_gen_models.py b/tests/integration/pipelex/cogt/test_doc_gen_models.pynew file mode 100644index 000000000..cd97153df--- /dev/null+++ b/tests/integration/pipelex/cogt/test_doc_gen_models.py@@ -0,0 +1,51 @@+from typing import Any, cast++import pytest+from pydantic import ValidationError++from pipelex.cogt.doc_gen.doc_gen_engine import prints_document+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.exceptions import BUILT_IN_DOC_GEN_CHOICE_KEY, BUILT_IN_DOC_GEN_MODEL, DOC_GEN_PLUGIN_MODEL_NAMES+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.deck_manifest import KitManagedArea, kit_backends_dir, kit_deck_dir, list_managed_kit_files+from pipelex.cogt.models.model_deck import DocGenDeckBlueprint+from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprint+from pipelex.runtime_hub import get_model_deck+from pipelex.tools.misc.toml_utils import load_toml_from_path+++class TestDocGenModels:+ def test_the_kit_s_deck_declares_the_built_in_engine_s_default_only(self) -> None:+ """Open Pipelex declares a default for a pdf from the auto-layout alone: the plugin declares every other one when it loads.++ Read from the kit's deck files rather than the booted deck, which an installed plugin's defaults join.+ """+ kit_deck_paths = [str(kit_deck_dir() / filename) for filename in sorted(list_managed_kit_files(area=KitManagedArea.DECK))]+ kit_deck_blueprint = load_model_deck_blueprint(kit_deck_paths)+ assert list(kit_deck_blueprint.doc_gen.choice_defaults) == [BUILT_IN_DOC_GEN_CHOICE_KEY]++ def test_the_built_in_engine_is_the_default_for_a_pdf_without_a_template(self) -> None:+ model_deck = get_model_deck()+ choice = model_deck.get_doc_gen_choice_default(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.LAYOUT)+ assert choice is not None+ doc_gen_setting = model_deck.get_doc_gen_setting(doc_gen_choice=choice)+ assert doc_gen_setting.model == BUILT_IN_DOC_GEN_MODEL+ inference_model = model_deck.get_required_inference_model(model_handle=doc_gen_setting.model, model_type=ModelType.DOC_GEN)+ assert prints_document(inference_model=inference_model, doc_gen_format=DocGenFormat.PDF, source=DocGenSource.LAYOUT)++ def test_the_kit_declares_none_of_the_plugin_s_engines(self) -> None:+ """The plugin declares its own models: the kit's internal.toml names none of them, so an open install serves none.++ Read from the kit's backend file rather than the booted deck, which an installed plugin's models join.+ """+ internal_backend = load_toml_from_path(kit_backends_dir() / "internal.toml")+ doc_gen_models: set[str] = set()+ for name, table in internal_backend.items():+ if isinstance(table, dict) and cast("dict[str, Any]", table).get("model_type") == ModelType.DOC_GEN:+ doc_gen_models.add(name)+ assert doc_gen_models == {BUILT_IN_DOC_GEN_MODEL}+ assert not DOC_GEN_PLUGIN_MODEL_NAMES & set(internal_backend)++ def test_a_default_under_a_key_no_step_makes_is_refused_when_the_deck_loads(self) -> None:+ with pytest.raises(ValidationError, match="a pptx is never composed from the auto-layout"):+ DocGenDeckBlueprint.model_validate({"choice_defaults": {"pptx.layout": "pipelex-pptx"}})diff --git a/tests/integration/pipelex/cogt/test_setup_inference_workers.py b/tests/integration/pipelex/cogt/test_setup_inference_workers.pyindex 19ac73373..5d98543f8 100644--- a/tests/integration/pipelex/cogt/test_setup_inference_workers.py+++ b/tests/integration/pipelex/cogt/test_setup_inference_workers.py@@ -1,3 +1,4 @@+from pipelex.cogt.doc_gen.doc_gen_worker_factory import DocGenWorkerFactoryfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.runtime_hub import get_inference_manager, get_model_deck@@ -16,3 +17,8 @@ class TestSetupInferenceWorkers:_ = inference_manager.get_img_gen_worker(img_gen_handle=model_handle)case ModelType.SEARCH:pass+ case ModelType.DOC_GEN:+ # Every document engine the deck serves has a worker: the kit declares only the built-in one, and a+ # plugin that declares an engine also registers its worker.+ doc_gen_worker = DocGenWorkerFactory.make_doc_gen_worker(inference_model=inference_model)+ assert doc_gen_worker.inference_model == inference_modeldiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/conftest.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/conftest.pynew file mode 100644index 000000000..b09025099--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/conftest.py@@ -0,0 +1,65 @@+import shutil+from collections.abc import Iterator+from pathlib import Path++import pytest++from pipelex.cogt.doc_gen.exceptions import BUILT_IN_DOC_GEN_MODEL+from pipelex.plugins.inference_backend_registry import InferenceFamily+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations+from pipelex.runtime_hub import get_inference_backend_registry, get_models_manager, get_runtime_hub+from pipelex.system.configuration.config_loader import config_manager+from pipelex.tools.misc.toml_utils import load_toml_with_tomlkit, save_toml_to_path+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import StubEngines, make_doc_gen_backends, make_models_manager+++def stub_engines() -> Iterator[StubEngines]:+ """Install a stub document generation plugin for one test, and put the booted engines and model deck back after.++ The stub registers as the real plugin does: a worker per sdk, which replaces the runtime's document engines, and+ the plugin's models and deck defaults, which a model manager set up with its declarations merges.+ """+ booted_registry = get_inference_backend_registry()+ booted_models_manager = get_models_manager()+ engines = StubEngines()+ registrar = engines.make_registrar()+ get_runtime_hub().set_inference_backend_registry(+ booted_registry.with_family(family=InferenceFamily.DOC_GEN, backends=make_doc_gen_backends(registrar=registrar))+ )+ get_runtime_hub().set_models_manager(models_manager=make_models_manager(plugin_model_declarations=registrar.make_model_declarations()))+ try:+ yield engines+ finally:+ get_runtime_hub().set_inference_backend_registry(booted_registry)+ get_runtime_hub().set_models_manager(models_manager=booted_models_manager)+++def no_engines() -> Iterator[None]:+ """A runtime with no document engine, as a host that disables the built-in one and installs no plugin."""+ booted_registry = get_inference_backend_registry()+ get_runtime_hub().set_inference_backend_registry(booted_registry.with_family(family=InferenceFamily.DOC_GEN, backends={}))+ try:+ yield+ finally:+ get_runtime_hub().set_inference_backend_registry(booted_registry)+++def stale_internal_backend(tmp_path: Path) -> Iterator[None]:+ """An installation whose `internal.toml` predates the built-in engine: this runtime's backends, without `reportlab-pdf`."""+ backends_dir = tmp_path / "backends"+ shutil.copytree(config_manager.backends_dir_path, backends_dir)+ internal_toml_path = backends_dir / "internal.toml"+ internal_toml = load_toml_with_tomlkit(internal_toml_path)+ del internal_toml[BUILT_IN_DOC_GEN_MODEL]+ save_toml_to_path(internal_toml, path=internal_toml_path)+ booted_models_manager = get_models_manager()+ get_runtime_hub().set_models_manager(+ models_manager=make_models_manager(plugin_model_declarations=PluginModelDeclarations.make_empty(), backends_dir_path=backends_dir)+ )+ try:+ yield+ finally:+ get_runtime_hub().set_models_manager(models_manager=booted_models_manager)diff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.pynew file mode 100644index 000000000..3f6aa5eb0--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.py@@ -0,0 +1,171 @@+from pathlib import Path+from typing import Any++import pytest+from typing_extensions import override++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.doc_gen_worker_abstract import DocGenWorkerAbstract+from pipelex.cogt.doc_gen.render_job import RenderedDocument, RenderJob, RenderResources+from pipelex.cogt.doc_gen.template_check import TemplateCheckRequest, TemplateFinding+from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract+from pipelex.cogt.model_backends.backend import InferenceBackend+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.models.model_manager import ModelManager+from pipelex.config import get_config+from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.validate_bundle import validate_bundle+from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.inference_backend_registry import InferenceFamily, MakeWorkerFn+from pipelex.plugins.plugin_group import PluginGroup+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations+from pipelex.plugins.registrar import PluginOrigin, PluginRegistrar+from pipelex.plugins.sdk_client_registry import SdkClientRegistry+from pipelex.reporting.reporting_protocol import ReportingProtocol+from pipelex.runtime_hub import get_secrets_provider+from pipelex.system.pipelex_service.managed_gateway_configs import build_managed_gateway_configs+from pipelex.system.pipelex_service.pipelex_service_config import enabled_managed_gateway_sections+from pipelex.system.pipelex_service.remote_config_fetcher import RemoteConfigFetcher+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData++# The engines the stubs stand in for: the built-in pdf from the layout, and two of the document generation plugin's,+# a pdf from HTML and a docx, which the stub plugin declares as the real plugin would.+BUILT_IN_STUB_SDK = "reportlab"+PIPELEX_PDF_SPEC: dict[str, Any] = {+ "model_type": "doc_gen",+ "sdk": "weasyprint",+ "model_id": "print-pdf",+ "inputs": ["html", "layout"],+ "outputs": ["pdf"],+ "costs": {},+}+PIPELEX_DOCX_SPEC: dict[str, Any] = {+ "model_type": "doc_gen",+ "sdk": "docxtpl",+ "model_id": "write-docx",+ "inputs": ["layout", "template_file"],+ "outputs": ["docx"],+ "costs": {},+}+++class StubEngine:+ """An engine that prints the job it was handed as its bytes, and remembers it."""++ def __init__(self) -> None:+ self.jobs: list[RenderJob] = []++ def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument: # ruff: ignore[unused-method-argument]+ self.jobs.append(job)+ return RenderedDocument(data=b"%PDF-stub " + job.model_dump_json().encode())+++class StubDocGenWorker(DocGenWorkerAbstract):+ """The worker the stub sdks make: it prints on the test's engine and checks templates with the test's checker."""++ def __init__(self, *, inference_model: InferenceModelSpec, engines: "StubEngines"):+ super().__init__(inference_model=inference_model)+ self._engines = engines++ @override+ def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument:+ return self._engines.engine.render(job=job, resources=resources)++ @override+ def check_template(self, *, request: TemplateCheckRequest) -> list[TemplateFinding]:+ return self._engines.check_template(request=request)+++class StubEngines:+ """The stub engines of one test, registered for the sdks of `reportlab-pdf`, `pipelex-pdf` and `pipelex-docx`."""++ def __init__(self) -> None:+ self.engine = StubEngine()+ self.nb_builds = 0+ self.findings: list[TemplateFinding] = []+ self.check_requests: list[TemplateCheckRequest] = []+ self.models: list[str] = []++ def make_worker(+ self,+ *,+ inference_model: InferenceModelSpec,+ backend: InferenceBackend, # ruff: ignore[unused-method-argument]+ sdk_clients: SdkClientRegistry, # ruff: ignore[unused-method-argument]+ reporting_delegate: ReportingProtocol | None, # ruff: ignore[unused-method-argument]+ ) -> InferenceWorkerAbstract:+ self.nb_builds += 1+ self.models.append(inference_model.name)+ return StubDocGenWorker(inference_model=inference_model, engines=self)++ def check_template(self, *, request: TemplateCheckRequest) -> list[TemplateFinding]:+ self.check_requests.append(request)+ return self.findings++ def register(self, registrar: PluginRegistrar) -> None:+ """Register as the document generation plugin does: a worker per sdk, the plugin's models, and their deck defaults."""+ for sdk in (BUILT_IN_STUB_SDK, PIPELEX_PDF_SPEC["sdk"], PIPELEX_DOCX_SPEC["sdk"]):+ registrar.add_inference_backend(family=InferenceFamily.DOC_GEN, sdk=sdk, make_worker=self.make_worker)+ registrar.add_internal_model(name="pipelex-pdf", spec=PIPELEX_PDF_SPEC)+ registrar.add_internal_model(name="pipelex-docx", spec=PIPELEX_DOCX_SPEC)+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.HTML, model="pipelex-pdf")+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.DOCX, source=DocGenSource.LAYOUT, model="pipelex-docx")+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.DOCX, source=DocGenSource.TEMPLATE_FILE, model="pipelex-docx")++ def make_registrar(self) -> PluginRegistrar:+ """A registrar holding only the stub plugin's contributions, built the way discovery builds one."""+ registrar = PluginRegistrar(config=get_config())+ registrar.begin_plugin(name="stub-doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)+ self.register(registrar)+ return registrar+++def make_doc_gen_backends(*, registrar: PluginRegistrar) -> dict[str, MakeWorkerFn]:+ """The document engine workers a registrar holds, keyed by sdk, for `InferenceBackendRegistry.with_family`."""+ backends: dict[str, MakeWorkerFn] = {}+ for (family, sdk), make_worker in registrar.inference_backends.items():+ match family:+ case InferenceFamily.DOC_GEN:+ backends[sdk] = make_worker+ case InferenceFamily.LLM | InferenceFamily.IMG_GEN | InferenceFamily.EXTRACT | InferenceFamily.SEARCH:+ pass+ return backends+++def make_models_manager(*, plugin_model_declarations: PluginModelDeclarations, backends_dir_path: Path | None = None) -> ModelManager:+ """A model manager set up from this runtime's configuration with the given plugins' declarations.++ Set up the way a boot that needs no model specs is, with the managed gateways' placeholder specs: nothing is+ fetched, and the internal models, the plugins' included, resolve through the internal backend. A test that needs+ other backend files hands their directory in.+ """+ managed_gateway_sections = enabled_managed_gateway_sections()+ managed_gateway_configs = (+ build_managed_gateway_configs(remote_config=RemoteConfigFetcher.make_dummy_remote_config(), managed_gateway_sections=managed_gateway_sections)+ if managed_gateway_sections+ else None+ )+ models_manager = ModelManager()+ models_manager.setup(+ secrets_provider=get_secrets_provider(),+ managed_gateway_configs=managed_gateway_configs,+ gateway_config_source=None,+ plugin_model_declarations=plugin_model_declarations,+ needs_inference=False,+ backends_dir_path=str(backends_dir_path) if backends_dir_path is not None else None,+ )+ return models_manager+++async def refusal_report(*, step_fields: str) -> str:+ """The error report, as text, of validating the test bundle with the given PipeDocGen step, which must be refused."""+ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_contents=[PipeDocGenTestData.bundle(step_fields=step_fields)])+ return str(exc_info.value.to_error_report().model_dump())+++def write_bundle(*, directory: Path, step_fields: str) -> Path:+ """Write the test bundle with the given PipeDocGen step into a directory, for a step that names a template file."""+ bundle_path = directory / "invoice.mthds"+ bundle_path.write_text(PipeDocGenTestData.bundle(step_fields=step_fields), encoding="utf-8")+ return bundle_pathdiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_data.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_data.pynew file mode 100644index 000000000..9d283a245--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_data.py@@ -0,0 +1,69 @@+from typing import Any, ClassVar+++class PipeDocGenTestData:+ """An invoice bundle whose PipeDocGen step each test sets: its format, its template and its filename."""++ STEP_SLOT: ClassVar[str] = "#STEP_FIELDS#"++ BUNDLE_MTHDS: ClassVar[str] = """+domain = "doc_gen_tests"+description = "Printing an invoice"+main_pipe = "print_invoice"++[concept.LineItem]+description = "One line of an invoice"++[concept.LineItem.structure]+description = { type = "text", description = "What was sold", required = true }+amount = { type = "number", description = "The line's amount, in euros", required = true }++[concept.Invoice]+description = "An invoice to print"++[concept.Invoice.structure]+number = { type = "text", description = "The invoice number", required = true }+customer = { type = "text", description = "The customer's name", required = true }+notes = { type = "text", description = "Notes, in Markdown" }+line_items = { type = "list", item_type = "concept", item_concept_ref = "doc_gen_tests.LineItem", description = "The lines", required = true }++[concept.InvoicePdf]+description = "The invoice as a printable file"+refines = "Document"++[pipe.print_invoice]+type = "PipeDocGen"+description = "Print the invoice"+inputs = { invoice = "Invoice" }+output = "InvoicePdf"+#STEP_FIELDS#+"""++ INVOICE_INPUTS: ClassVar[dict[str, Any]] = {+ "invoice": {+ "concept": "doc_gen_tests.Invoice",+ "content": {+ "number": "INV-2026-0142",+ "customer": "Ada Lovelace",+ "notes": "Paid by **bank transfer**, see https://example.com/terms.",+ "line_items": [{"description": "Tea", "amount": 7.0}, {"description": "Cake", "amount": 4.5}],+ },+ }+ }++ INVOICE_INPUTS_WITHOUT_NOTES: ClassVar[dict[str, Any]] = {+ "invoice": {+ "concept": "doc_gen_tests.Invoice",+ "content": {+ "number": "INV-2026-0143",+ "customer": "Ada Lovelace",+ "line_items": [{"description": "Tea", "amount": 7.0}],+ },+ }+ }++ PDF_LAYOUT_STEP: ClassVar[str] = 'format = "pdf"\nfilename = "invoice-{{ invoice.number }}"'++ @classmethod+ def bundle(cls, *, step_fields: str) -> str:+ return cls.BUNDLE_MTHDS.replace(cls.STEP_SLOT, step_fields)diff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.pynew file mode 100644index 000000000..7a4cab3a7--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.py@@ -0,0 +1,62 @@+from pathlib import Path++import pytest+from mthds.package.dependency_resolver import ResolvedDependency+from mthds.package.manifest.schema import MethodsManifest+from pytest_mock import MockerFixture++from pipelex.core.pipes.exceptions import PipeLoadRefusalError+from pipelex.interpreter_hub import get_library_manager+from pipelex.libraries.library import Library+from pipelex.libraries.library_factory import LibraryFactory+from pipelex.libraries.library_manager import LibraryManager+from pipelex.pipe_operators.doc_gen.pipe_doc_gen import PipeDocGen+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData++_PACKAGE_ADDRESS = "github.com/acme/invoicing"+_TEMPLATE = "<h1>{{ invoice.number }}</h1>"++[email protected]("stub_engines")+class TestPipeDocGenDependencyTemplateFile:+ def _load_dependency(self, *, mocker: MockerFixture, package_root: Path, template: str | None) -> Library:+ """Load the invoice bundle, whose step reads `invoice.html`, as a dependency package installed at `package_root`."""+ bundle_dir = package_root / "invoice"+ bundle_dir.mkdir(parents=True)+ if template is not None:+ (bundle_dir / "invoice.html").write_text(template, encoding="utf-8")+ mthds_file = bundle_dir / "invoice.mthds"+ mthds_file.write_text(PipeDocGenTestData.bundle(step_fields='format = "pdf"\ntemplate_file = "invoice.html"'), encoding="utf-8")+ resolved_dep = ResolvedDependency(+ alias="invoicing",+ address=_PACKAGE_ADDRESS,+ manifest=MethodsManifest(address=_PACKAGE_ADDRESS, version="1.0.0", description="Invoicing methods"),+ package_root=package_root,+ mthds_files=[mthds_file],+ exported_pipe_codes=None,+ )+ library = LibraryFactory.make_empty()+ mocker.patch.object(get_library_manager(), "get_current_library", return_value=library)+ LibraryManager()._load_single_dependency( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]+ library=library, package_address=_PACKAGE_ADDRESS, resolved_dep=resolved_dep+ )+ return library++ def test_a_dependency_s_template_file_is_found_beside_its_bundle(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """A dependency's source names the package, not a path, and its template file still loads from the installed package."""+ library = self._load_dependency(mocker=mocker, package_root=tmp_path, template=_TEMPLATE)++ pipe = library.dependency_libraries["invoicing"].pipe_library.get_required_pipe(pipe_code="doc_gen_tests.print_invoice")+ assert isinstance(pipe, PipeDocGen)+ assert pipe.template == _TEMPLATE++ def test_a_dependency_s_missing_template_file_is_refused_without_the_host_path(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """The refusal is located at the package's address and names the template file as the method does, never where the host installed it."""+ with pytest.raises(PipeLoadRefusalError) as exc_info:+ self._load_dependency(mocker=mocker, package_root=tmp_path, template=None)++ refusal = exc_info.value+ assert refusal.source == f"{_PACKAGE_ADDRESS}/invoice/invoice.mthds"+ report = str(refusal.to_error_report().model_dump())+ assert "'invoice.html', which does not exist beside its bundle" in report+ assert str(tmp_path) not in reportdiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_load_refusals.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_load_refusals.pynew file mode 100644index 000000000..83d5a3ffd--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_load_refusals.py@@ -0,0 +1,122 @@+import pytest++from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.validate_bundle import validate_bundle+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import refusal_report+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData++[email protected](loop_scope="class")+class TestPipeDocGenLoadRefusals:+ @pytest.mark.parametrize(+ ("step_fields", "asks"),+ [+ ('format = "pdf"\ntemplate = "<h1>{{ invoice.number }}</h1>"', "a pdf from an HTML template"),+ ('format = "xlsx"', "an xlsx from the auto-layout of its inputs"),+ ],+ ids=["pdf_with_an_html_template", "xlsx_auto_layout"],+ )+ async def test_a_step_no_installed_engine_prints_names_the_plugin(self, step_fields: str, asks: str) -> None:+ """Open Pipelex declares a default for a pdf without a template only: any other step is refused at load, naming the plugin."""+ report = await refusal_report(step_fields=step_fields)+ assert asks in report+ assert "pipelex-doc-gen" in report+ assert "pipelex update" not in report++ async def test_a_step_naming_a_plugin_engine_is_refused_where_the_plugin_is_absent(self) -> None:+ """A step may name the plugin's `pipelex-pdf` for a pdf without a template, and open Pipelex refuses it at load, naming the plugin."""+ report = await refusal_report(step_fields='format = "pdf"\nmodel = "pipelex-pdf"')+ assert "pipelex-pdf" in report+ assert "pipelex-doc-gen" in report+ assert "was not found in the model deck" not in report++ @pytest.mark.usefixtures("stale_internal_backend")+ @pytest.mark.parametrize(+ "step_fields",+ [PipeDocGenTestData.PDF_LAYOUT_STEP, 'format = "pdf"\nmodel = "reportlab-pdf"'],+ ids=["through_the_default", "named_by_the_step"],+ )+ async def test_a_pdf_layout_step_is_refused_where_internal_toml_predates_the_built_in_engine(self, step_fields: str) -> None:+ """An installation whose internal.toml does not declare `reportlab-pdf` is told to run `pipelex update`, not that the model is unknown."""+ report = await refusal_report(step_fields=step_fields)+ assert "reportlab-pdf" in report+ assert "pipelex update" in report+ assert "was not found in the model deck" not in report++ async def test_an_engine_that_does_not_print_the_step_s_source_is_refused(self) -> None:+ """ReportLab prints only the auto-layout, so a step that names it with an HTML template is refused at load."""+ report = await refusal_report(step_fields='format = "pdf"\nmodel = "reportlab-pdf"\ntemplate = "<h1>{{ invoice.number }}</h1>"')+ assert "reportlab-pdf" in report+ assert "from an HTML template, which it does not print" in report++ async def test_an_engine_the_deck_does_not_define_is_an_unknown_model(self) -> None:+ """A misspelled engine is refused at load like any unknown model, located on the step's `model` field."""+ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_contents=[PipeDocGenTestData.bundle(step_fields='format = "pdf"\nmodel = "reportlab-pfd"')])+ report = str(exc_info.value.to_error_report().model_dump())+ assert "reportlab-pfd" in report+ assert "model" in report++ @pytest.mark.usefixtures("no_engines")+ async def test_a_pdf_layout_step_is_refused_where_the_built_in_engine_is_disabled(self) -> None:+ """A host that disables the built-in engine and installs no plugin refuses even a pdf without a template."""+ report = await refusal_report(step_fields=PipeDocGenTestData.PDF_LAYOUT_STEP)+ assert "built into Pipelex and disabled" in report++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_template_file_in_a_bundle_loaded_from_a_string_is_refused(self) -> None:+ """A bundle loaded from a string has no directory to find a template file in."""+ report = await refusal_report(step_fields='format = "docx"\ntemplate_file = "invoice.docx"')+ assert "not loaded from a file on disk" in report++ @pytest.mark.usefixtures("stub_engines")+ @pytest.mark.parametrize(+ ("step_fields", "expected"),+ [+ ('format = "pdf"\ntemplate = "<h1>{{ invoice.numbr }}</h1>"', "numbr"),+ ('format = "pdf"\ntemplate = "{% for line in invoice.line_items %}{{ line.amout }}{% endfor %}"', "amout"),+ ('format = "pdf"\nfilename = "invoice-{{ invoice.nmber }}"', "nmber"),+ ('format = "pdf"\ntemplate = "<h1>{{ invoice._stuff }}</h1>"', "_stuff"),+ ],+ ids=["misspelled_field", "misspelled_field_through_a_loop", "misspelled_field_in_the_filename", "private_name"],+ )+ async def test_a_template_mistake_is_refused_at_load(self, step_fields: str, expected: str) -> None:+ """A field a template or the filename names is checked against the input's concept, through loop variables too."""+ report = await refusal_report(step_fields=step_fields)+ assert expected in report++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_field_read_on_an_anything_input_is_left_to_the_dry_run(self) -> None:+ """An `Anything` input declares no fields for the load check to follow, so a field read on it is checked when the template renders."""+ bundle = PipeDocGenTestData.bundle(step_fields='format = "pdf"\ntemplate = "<h1>{{ invoice.number }} {{ thing.title }}</h1>"').replace(+ 'inputs = { invoice = "Invoice" }', 'inputs = { invoice = "Invoice", thing = "Anything" }'+ )+ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_contents=[bundle])+ report = str(exc_info.value.to_error_report().model_dump())+ assert "could not render its template" in report+ assert "title" in report+ assert "AnythingContent" not in report++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_misspelling_the_load_check_cannot_follow_fails_at_the_dry_run(self) -> None:+ """A field read through a `set` alias escapes the load check, and the strict undefined names it at the dry run."""+ report = await refusal_report(step_fields='format = "pdf"\ntemplate = "{% set buyer = invoice %}<h1>{{ buyer.custmer }}</h1>"')+ assert "custmer" in report+ assert "could not render its template" in report++ @pytest.mark.parametrize(+ "filename",+ ["invoice-{{ reference }}", "invoice-$reference"],+ ids=["jinja2", "sigil"],+ )+ async def test_a_filename_reading_an_optional_input_unguarded_is_refused(self, filename: str) -> None:+ """The filename renders strictly, so an optional input it reads without a guard would fail every run that leaves it out."""+ bundle = PipeDocGenTestData.bundle(step_fields=f'format = "pdf"\nfilename = "{filename}"').replace(+ 'inputs = { invoice = "Invoice" }', 'inputs = { invoice = "Invoice", reference = "Text?" }'+ )+ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_contents=[bundle])+ report = str(exc_info.value.to_error_report().model_dump())+ assert "reference" in report+ assert "filename" in reportdiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_run.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_run.pynew file mode 100644index 000000000..3a61c7781--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_run.py@@ -0,0 +1,151 @@+import threading++import pytest+from typing_extensions import override++from pipelex.cogt.doc_gen.doc_gen_format import DocGenSource+from pipelex.cogt.doc_gen.render_job import RenderedDocument, RenderJob, RenderResources+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.pipeline.exceptions import PipelineExecutionError+from pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.runtime_hub import get_storage_provider+from pipelex.system.pipe_run_mode import PipeRunMode+from pipelex.tools.uri.uri_bytes import load_bytes_from_any_uri+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import StubEngine, StubEngines+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData+++class _ThreadRecordingEngine(StubEngine):+ """A stub engine that remembers the name of the thread it printed on."""++ def __init__(self) -> None:+ super().__init__()+ self.thread_names: list[str] = []++ @override+ def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument:+ self.thread_names.append(threading.current_thread().name)+ return super().render(job=job, resources=resources)+++class _BrokenEngine(StubEngine):+ """A stub engine that fails the way third-party code can, with an exception nobody declared."""++ @override+ def render(self, *, job: RenderJob, resources: RenderResources) -> RenderedDocument:+ msg = "font cache is corrupt"+ raise RuntimeError(msg)++[email protected](loop_scope="class")+class TestPipeDocGenRun:+ async def test_a_run_hands_the_engine_the_layout_and_stores_the_file_under_its_name(self, stub_engines: StubEngines) -> None:+ """The engine gets the layout tree, and the stored file's key ends in the rendered filename."""+ result = await PipelexMTHDSProtocol().execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields=PipeDocGenTestData.PDF_LAYOUT_STEP)],+ inputs=PipeDocGenTestData.INVOICE_INPUTS,+ )++ document = result.pipe_output.main_stuff_as(content_type=DocumentContent)+ assert document.filename == "invoice-INV-2026-0142.pdf"+ assert document.url.endswith("/invoice-INV-2026-0142.pdf")+ assert document.mime_type == "application/pdf"+ (job,) = stub_engines.engine.jobs+ assert job.source == DocGenSource.LAYOUT+ assert job.title == "Invoice"+ assert job.layout is not None+ stored = await load_bytes_from_any_uri(document.url, storage_provider=get_storage_provider())+ assert stored.startswith(b"%PDF-stub ")+ assert stub_engines.models == ["reportlab-pdf"]++ async def test_a_step_that_names_its_engine_prints_on_it(self, stub_engines: StubEngines) -> None:+ """`model` chooses the engine: a pdf without a template prints on the plugin's `pipelex-pdf` when the step names it."""+ await PipelexMTHDSProtocol().execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields='format = "pdf"\nmodel = "pipelex-pdf"')],+ inputs=PipeDocGenTestData.INVOICE_INPUTS,+ )++ assert stub_engines.models == ["pipelex-pdf"]+ (job,) = stub_engines.engine.jobs+ assert job.source == DocGenSource.LAYOUT++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_sigil_in_the_filename_prints_the_input(self) -> None:+ """`$reference` in the filename prints a Text input's text, as it does in a template."""+ bundle = PipeDocGenTestData.bundle(step_fields='format = "pdf"\nfilename = "invoice-$reference"').replace(+ 'inputs = { invoice = "Invoice" }', 'inputs = { invoice = "Invoice", reference = "Text" }'+ )+ result = await PipelexMTHDSProtocol().execute(+ mthds_contents=[bundle],+ inputs={**PipeDocGenTestData.INVOICE_INPUTS, "reference": "INV-7"},+ )++ document = result.pipe_output.main_stuff_as(content_type=DocumentContent)+ assert document.filename == "invoice-INV-7.pdf"++ async def test_an_engine_prints_on_the_document_pool_not_the_default_executor(self, stub_engines: StubEngines) -> None:+ """An engine waits on the loop while it reads a file, and reads need the default executor, so engines never hold it."""+ engine = _ThreadRecordingEngine()+ stub_engines.engine = engine+ await PipelexMTHDSProtocol().execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields=PipeDocGenTestData.PDF_LAYOUT_STEP)],+ inputs=PipeDocGenTestData.INVOICE_INPUTS,+ )++ (thread_name,) = engine.thread_names+ assert thread_name.startswith("pipelex-doc-gen")++ async def test_an_undeclared_engine_failure_is_a_render_error(self, stub_engines: StubEngines) -> None:+ """Whatever an engine raises beyond a Pipelex error fails the step as a failure to print, naming the file."""+ stub_engines.engine = _BrokenEngine()+ with pytest.raises(PipelineExecutionError) as exc_info:+ await PipelexMTHDSProtocol().execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields=PipeDocGenTestData.PDF_LAYOUT_STEP)],+ inputs=PipeDocGenTestData.INVOICE_INPUTS,+ )++ message = str(exc_info.value)+ assert "could not print 'invoice-INV-2026-0142.pdf'" in message+ assert "font cache is corrupt" in message++ async def test_an_html_template_is_rendered_before_the_engine_prints_it(self, stub_engines: StubEngines) -> None:+ """Sigils and the `markdown` filter work in the template, and the engine gets the composed HTML."""+ template = "<h1>$invoice.number</h1><div>{{ invoice.notes | markdown }}</div>"+ await PipelexMTHDSProtocol().execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields=f'format = "pdf"\ntemplate = "{template}"')],+ inputs=PipeDocGenTestData.INVOICE_INPUTS,+ )++ assert stub_engines.models == ["pipelex-pdf"]+ (job,) = stub_engines.engine.jobs+ assert job.source == DocGenSource.HTML+ assert job.html is not None+ assert "INV-2026-0142" in job.html+ assert "<strong>bank transfer</strong>" in job.html+ assert '<a href="https://example.com/terms">' in job.html++ async def test_a_dry_run_outputs_a_mock_document_and_builds_no_engine(self, stub_engines: StubEngines) -> None:+ """The dry run composes for real, then the leaf returns a mock Document without printing."""+ result = await PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.DRY).execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields=PipeDocGenTestData.PDF_LAYOUT_STEP)],+ inputs=PipeDocGenTestData.INVOICE_INPUTS,+ )++ document = result.pipe_output.main_stuff_as(content_type=DocumentContent)+ assert document.mime_type == "application/pdf"+ assert document.filename is not None+ assert document.filename.endswith(".pdf")+ assert stub_engines.nb_builds == 0++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_value_absent_from_the_run_s_inputs_fails_the_step(self) -> None:+ """An optional field absent from this run's data fails the strict render, naming the step and the template."""+ with pytest.raises(PipelineExecutionError) as exc_info:+ await PipelexMTHDSProtocol().execute(+ mthds_contents=[PipeDocGenTestData.bundle(step_fields='format = "pdf"\ntemplate = "<p>{{ invoice.notes.upper() }}</p>"')],+ inputs=PipeDocGenTestData.INVOICE_INPUTS_WITHOUT_NOTES,+ )++ message = str(exc_info.value)+ assert "print_invoice" in message+ assert "could not render its template" in messagediff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.pynew file mode 100644index 000000000..c0ce082cf--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.py@@ -0,0 +1,92 @@+from pathlib import Path++import pytest++from pipelex.cogt.doc_gen.input_shape import InputShapeKind+from pipelex.cogt.doc_gen.template_check import TemplateFinding, TemplateFindingSeverity+from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.validate_bundle import validate_bundle+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import StubEngines, write_bundle+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData++_DOCX_TEMPLATE_STEP = 'format = "docx"\ntemplate_file = "invoice.docx"'++[email protected](loop_scope="class")+class TestPipeDocGenTemplateFiles:+ @pytest.mark.usefixtures("stub_engines")+ @pytest.mark.parametrize(+ ("template_file", "expected"),+ [+ ("invoice.docx", "does not exist"),+ ("../outside.docx", "outside its bundle's directory"),+ ],+ ids=["missing_file", "outside_the_bundle"],+ )+ async def test_a_template_file_is_found_inside_the_bundle_s_directory(self, tmp_path: Path, template_file: str, expected: str) -> None:+ """A template file must exist beside the bundle, and a path out of the bundle's directory is refused even when the file exists."""+ bundle_dir = tmp_path / "bundle"+ bundle_dir.mkdir()+ (tmp_path / "outside.docx").write_bytes(b"docx")+ bundle_path = write_bundle(directory=bundle_dir, step_fields=f'format = "docx"\ntemplate_file = "{template_file}"')++ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_file_path=bundle_path)++ assert expected in str(exc_info.value.to_error_report().model_dump())++ @pytest.mark.usefixtures("stub_engines")+ async def test_an_html_template_file_that_is_not_utf8_is_refused_naming_it(self, tmp_path: Path) -> None:+ """A template saved in another encoding is refused at load, naming the file, rather than crashing the load."""+ (tmp_path / "invoice.html").write_bytes("<p>Facture émise</p>".encode("latin-1"))+ bundle_path = write_bundle(directory=tmp_path, step_fields='format = "pdf"\ntemplate_file = "invoice.html"')++ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_file_path=bundle_path)++ assert "could not read the template file 'invoice.html' as UTF-8 text" in str(exc_info.value.to_error_report().model_dump())++ async def test_the_engine_s_checker_findings_fail_the_dry_run(self, tmp_path: Path, stub_engines: StubEngines) -> None:+ """The dry run hands the checker the file's bytes, the inputs' shapes and the mock data, and an error finding fails it."""+ (tmp_path / "invoice.docx").write_bytes(b"docx bytes")+ bundle_path = write_bundle(directory=tmp_path, step_fields=_DOCX_TEMPLATE_STEP)+ stub_engines.findings = [TemplateFinding(severity=TemplateFindingSeverity.ERROR, message="The tag {{ invoice.totl }} names no field")]++ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_file_path=bundle_path)++ assert "invoice.totl" in str(exc_info.value.to_error_report().model_dump())+ (request,) = stub_engines.check_requests+ assert request.template == b"docx bytes"+ assert request.template_name == "invoice.docx"+ assert set(request.inputs["invoice"].fields) == {"number", "customer", "notes", "line_items"}+ assert request.data is not None+ assert set(request.data) == {"invoice"}++ async def test_an_anything_input_reaches_the_checker_with_an_undeclared_shape(self, tmp_path: Path, stub_engines: StubEngines) -> None:+ """An `Anything` input has no content class to describe, so the checker is told its shape is not declared."""+ (tmp_path / "invoice.docx").write_bytes(b"docx bytes")+ bundle_path = tmp_path / "invoice.mthds"+ bundle_path.write_text(+ PipeDocGenTestData.bundle(step_fields=_DOCX_TEMPLATE_STEP).replace(+ 'inputs = { invoice = "Invoice" }', 'inputs = { invoice = "Invoice", thing = "Anything" }'+ ),+ encoding="utf-8",+ )++ await validate_bundle(mthds_file_path=bundle_path)++ (request,) = stub_engines.check_requests+ assert request.inputs["thing"].kind == InputShapeKind.ANY+ assert request.inputs["invoice"].kind == InputShapeKind.STRUCTURE++ async def test_warnings_alone_pass_and_print_nothing(self, tmp_path: Path, stub_engines: StubEngines) -> None:+ """A warning is logged, not raised, and the dry run prints nothing."""+ (tmp_path / "invoice.docx").write_bytes(b"docx bytes")+ bundle_path = write_bundle(directory=tmp_path, step_fields=_DOCX_TEMPLATE_STEP)+ stub_engines.findings = [TemplateFinding(severity=TemplateFindingSeverity.WARNING, message="The notes are never printed")]++ await validate_bundle(mthds_file_path=bundle_path)++ assert len(stub_engines.check_requests) == 1+ assert stub_engines.engine.jobs == []diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm_markdown_output_path.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm_markdown_output_path.pynew file mode 100644index 000000000..e96168b1e--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm_markdown_output_path.py@@ -0,0 +1,59 @@+from typing import Any, Callable++import pytest+from pytest_mock import MockerFixture++from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.interpreter_hub import get_pipe_library, get_pipe_router+from pipelex.pipe_machinery.pipe_factory import PipeFactory+from pipelex.pipe_operators.llm.pipe_llm import PipeLLM+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint+from pipelex.pipe_run.pipe_job_factory import PipeJobFactory+from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.runtime_hub import get_content_generator+from pipelex.system.job_metadata import JobMetadata+from pipelex.system.pipe_run_mode import PipeRunMode++[email protected](loop_scope="class")+class TestPipeLLMMarkdownOutputPath:+ async def test_markdown_output_takes_text_path(+ self,+ job_metadata: JobMetadata,+ mocker: MockerFixture,+ load_empty_library: Callable[[], str],+ ) -> None:+ """`output = "Markdown"` is Text-compatible: the LLM writes free text, held in a MarkdownContent, never an object."""+ load_empty_library()+ content_generator = get_content_generator()+ make_llm_text_spy = mocker.spy(content_generator, "make_llm_text")+ make_object_spy = mocker.spy(content_generator, "make_object")++ pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code="building_inspections",+ pipe_code="adhoc_markdown_output_path",+ blueprint=PipeLLMBlueprint(+ description="Write site notes up as an inspection report in Markdown",+ output=NativeConceptCode.MARKDOWN,+ prompt="Write the inspection report in Markdown, with a heading for each area inspected.",+ ),+ )+ get_pipe_library().add_new_pipe(pipe)++ pipe_job = PipeJobFactory.make_pipe_job(+ pipe=pipe,+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),+ job_metadata=job_metadata,+ )+ pipe_output = await get_pipe_router().run(pipe_job=pipe_job)++ make_llm_text_spy.assert_called_once()+ make_object_spy.assert_not_called()++ main_stuff = pipe_output.main_stuff+ assert main_stuff.concept.concept_ref == NativeConceptCode.MARKDOWN.concept_ref+ content: Any = main_stuff.content+ assert type(content) is MarkdownContent+ assert content.text+ assert pipe_output.main_stuff_as_markdown.text == content.textdiff --git a/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.py b/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.pyindex 0dae6513e..54c1b7ec0 100644--- a/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.py+++ b/tests/unit/pipelex/builder/operations/test_pipe_spec_to_toml.py@@ -202,6 +202,36 @@ class TestPipeSpecToToml:toml = pipe_spec_to_toml(spec)assert "model" not in toml+ # -- PipeDocGen -------------------------------------------------------++ def test_doc_gen_model_in_toml(self) -> None:+ spec = parse_pipe_spec(+ {+ "pipe_code": "print_invoice",+ "description": "Print the invoice",+ "inputs": {"invoice": "Invoice"},+ "output": "Document",+ "format": "docx",+ "model": "pipelex-docx",+ "template_file": "invoice.docx",+ "filename": "invoice.docx",+ },+ pipe_type="PipeDocGen",+ )+ toml = pipe_spec_to_toml(spec)+ assert 'format = "docx"' in toml+ assert 'model = "pipelex-docx"' in toml+ assert 'template_file = "invoice.docx"' in toml+ assert 'filename = "invoice.docx"' in toml++ def test_doc_gen_no_model_omits_field(self) -> None:+ spec = parse_pipe_spec(+ {"pipe_code": "print_report", "description": "Print the report", "inputs": {"report": "Markdown"}, "output": "Document", "format": "pdf"},+ pipe_type="PipeDocGen",+ )+ toml = pipe_spec_to_toml(spec)+ assert "model" not in toml+# -- PipeExtract ------------------------------------------------------def test_extract_model_in_toml(self) -> None:diff --git a/tests/unit/pipelex/cli/commands/run/test_main_stuff_file.py b/tests/unit/pipelex/cli/commands/run/test_main_stuff_file.pynew file mode 100644index 000000000..249cc4e9d--- /dev/null+++ b/tests/unit/pipelex/cli/commands/run/test_main_stuff_file.py@@ -0,0 +1,56 @@+"""Unit tests for copying a file-shaped main output into the results folder."""++from __future__ import annotations++import base64+from typing import TYPE_CHECKING++import pytest++from pipelex.cli.commands.run._main_stuff_file import save_main_stuff_file+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent++if TYPE_CHECKING:+ from pathlib import Path++ from pytest_mock import MockerFixture+++def _data_url(*, mime_type: str, data: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(data).decode()}"+++class TestMainStuffFile:+ @pytest.fixture(autouse=True)+ def _no_storage(self, mocker: MockerFixture) -> None:+ # A data URL needs no storage provider, and none is set up in a unit test.+ mocker.patch("pipelex.cli.commands.run._main_stuff_file.get_storage_provider", return_value=None)++ @pytest.mark.asyncio+ async def test_an_image_keeps_its_own_filename(self, tmp_path: Path) -> None:+ image = ImageContent(url=_data_url(mime_type="image/png", data=b"png bytes"), filename="logo.png", mime_type="image/png")++ saved_path = await save_main_stuff_file(content=image, output_dir=tmp_path, reserved_names=frozenset())++ assert saved_path == tmp_path / "logo.png"+ assert (tmp_path / "logo.png").read_bytes() == b"png bytes"++ @pytest.mark.asyncio+ async def test_the_file_never_replaces_an_artifact_of_the_run(self, tmp_path: Path) -> None:+ (tmp_path / "main_stuff.json").write_text("{}", encoding="utf-8")+ passed_through = DocumentContent(url=_data_url(mime_type="application/json", data=b"[]"), filename="MAIN_STUFF.json")++ saved_path = await save_main_stuff_file(content=passed_through, output_dir=tmp_path, reserved_names=frozenset())++ assert saved_path is not None+ assert saved_path.name.casefold() == "main_stuff-1.json"+ assert (tmp_path / "main_stuff.json").read_text(encoding="utf-8") == "{}"++ @pytest.mark.asyncio+ async def test_the_file_never_takes_a_name_the_run_writes_afterwards(self, tmp_path: Path) -> None:+ passed_through = DocumentContent(url=_data_url(mime_type="application/json", data=b"{}"), filename="working_memory.json")++ saved_path = await save_main_stuff_file(content=passed_through, output_dir=tmp_path, reserved_names=frozenset({"working_memory.json"}))++ assert saved_path == tmp_path / "working_memory-1.json"diff --git a/tests/unit/pipelex/cli/test_doctor_cmd.py b/tests/unit/pipelex/cli/test_doctor_cmd.pyindex fa487aa4a..8e31073e1 100644--- a/tests/unit/pipelex/cli/test_doctor_cmd.py+++ b/tests/unit/pipelex/cli/test_doctor_cmd.py@@ -85,6 +85,14 @@ class TestDoctorLayeredResolution:"OK",),)+ mock_check_internal_backend = mocker.patch(+ "pipelex.cli.commands.doctor_cmd.check_internal_backend_sync",+ return_value=(+ True,+ DeckSyncReport(kit_version="1.0.0", installed_kit_version="1.0.0", manifest_present=True, files={}),+ "OK",+ ),+ )mock_check_migrations = mocker.patch("pipelex.cli.commands.doctor_cmd.check_pending_migrations",return_value=NO_PENDING_MIGRATIONS,@@ -103,6 +111,7 @@ class TestDoctorLayeredResolution:mock_check_backends.assert_called_once_with()mock_check_models.assert_called_once_with()mock_check_deck.assert_called_once_with()+ mock_check_internal_backend.assert_called_once_with()# The migration row takes no directory at all — it answers for `pipelex migrate`, which# walks both configuration directories and has no way to be pointed at one.mock_check_migrations.assert_called_once_with()@@ -145,6 +154,14 @@ class TestDoctorLayeredResolution:"OK",),)+ mocker.patch(+ "pipelex.cli.commands.doctor_cmd.check_internal_backend_sync",+ return_value=(+ True,+ DeckSyncReport(kit_version="1.0.0", installed_kit_version="1.0.0", manifest_present=True, files={}),+ "OK",+ ),+ )mocker.patch("pipelex.cli.commands.doctor_cmd.check_pending_migrations",return_value=NO_PENDING_MIGRATIONS,diff --git a/tests/unit/pipelex/cli/test_doctor_deck_sync.py b/tests/unit/pipelex/cli/test_doctor_deck_sync.pyindex 0f98372bf..3bb5aef46 100644--- a/tests/unit/pipelex/cli/test_doctor_deck_sync.py+++ b/tests/unit/pipelex/cli/test_doctor_deck_sync.py@@ -36,7 +36,7 @@ class TestCheckDeckSync:manifest_present=True,files={"deck.toml": DeckFileStatus.UP_TO_DATE},)- mocker.patch("pipelex.cli.commands.doctor_cmd.compute_deck_sync_report", return_value=clean_report)+ mocker.patch("pipelex.cli.commands.doctor_cmd.compute_sync_report", return_value=clean_report)healthy, report, message = check_deck_sync(config_dir=tmp_path)@@ -53,7 +53,7 @@ class TestCheckDeckSync:manifest_present=False,files={"deck.toml": DeckFileStatus.LOCALLY_MODIFIED},)- mocker.patch("pipelex.cli.commands.doctor_cmd.compute_deck_sync_report", return_value=dirty_report)+ mocker.patch("pipelex.cli.commands.doctor_cmd.compute_sync_report", return_value=dirty_report)healthy, _, message = check_deck_sync(config_dir=tmp_path)@@ -74,7 +74,7 @@ class TestCheckDeckSync:"ok.toml": DeckFileStatus.UP_TO_DATE,},)- mocker.patch("pipelex.cli.commands.doctor_cmd.compute_deck_sync_report", return_value=dirty_report)+ mocker.patch("pipelex.cli.commands.doctor_cmd.compute_sync_report", return_value=dirty_report)healthy, _, message = check_deck_sync(config_dir=tmp_path)@@ -90,7 +90,7 @@ class TestCheckDeckSync:manifest_present=True,files={"deck.toml": DeckFileStatus.LOCALLY_MODIFIED},)- mocker.patch("pipelex.cli.commands.doctor_cmd.compute_deck_sync_report", return_value=dirty_report)+ mocker.patch("pipelex.cli.commands.doctor_cmd.compute_sync_report", return_value=dirty_report)healthy, _, message = check_deck_sync(config_dir=tmp_path)diff --git a/tests/unit/pipelex/cli/test_doctor_display_report.py b/tests/unit/pipelex/cli/test_doctor_display_report.pyindex 79ddddb08..aee750f31 100644--- a/tests/unit/pipelex/cli/test_doctor_display_report.py+++ b/tests/unit/pipelex/cli/test_doctor_display_report.py@@ -64,6 +64,9 @@ def _healthy_report_kwargs() -> dict[str, Any]:"deck_healthy": True,"deck_message": "Deck is up to date with pipelex 1.2.0","deck_report": CLEAN_DECK,+ "internal_backend_healthy": True,+ "internal_backend_message": "backends/internal.toml is up to date with pipelex 1.2.0",+ "internal_backend_report": CLEAN_DECK,"config_location": PROJECT_LOCATION,}diff --git a/tests/unit/pipelex/cli/test_doctor_fix_mode.py b/tests/unit/pipelex/cli/test_doctor_fix_mode.pyindex d673c2793..6ccfba4bc 100644--- a/tests/unit/pipelex/cli/test_doctor_fix_mode.py+++ b/tests/unit/pipelex/cli/test_doctor_fix_mode.py@@ -57,6 +57,7 @@ class TestDoctorFixMode:"backends": mocker.patch("pipelex.cli.commands.doctor_cmd.check_backend_credentials", return_value=(True, {}, "OK")),"models": mocker.patch("pipelex.cli.commands.doctor_cmd.check_models", return_value=(True, "OK", {})),"deck": mocker.patch("pipelex.cli.commands.doctor_cmd.check_deck_sync", return_value=(True, CLEAN_DECK, "OK")),+ "internal_backend": mocker.patch("pipelex.cli.commands.doctor_cmd.check_internal_backend_sync", return_value=(True, CLEAN_DECK, "OK")),"display": mocker.patch("pipelex.cli.commands.doctor_cmd.display_health_report"),"init_cmd": mocker.patch("pipelex.cli.commands.doctor_cmd.init_cmd"),"update_cmd": mocker.patch("pipelex.cli.commands.doctor_cmd.update_cmd"),@@ -203,7 +204,21 @@ class TestDoctorFixMode:self._run_doctor_expecting_exit_one()doctor_mocks["update_cmd"].assert_called_once_with(yes=True)- assert "Model deck updated" in doctor_mocks["console"].export_text()+ assert "Model deck and backends/internal.toml updated" in doctor_mocks["console"].export_text()++ def test_fix_a_stale_internal_toml_runs_update(self, doctor_mocks: dict[str, Any]) -> None:+ """A stale backends/internal.toml alone is the same fix as a stale deck: `pipelex update --yes`."""+ stale_internal_backend = DeckSyncReport(+ kit_version="1.3.0",+ installed_kit_version=None,+ manifest_present=False,+ files={"internal.toml": DeckFileStatus.LOCALLY_MODIFIED},+ )+ doctor_mocks["internal_backend"].return_value = (False, stale_internal_backend, "backends/internal.toml manifest missing")++ self._run_doctor_expecting_exit_one()++ doctor_mocks["update_cmd"].assert_called_once_with(yes=True)def _make_invalid_backend_report(self, tmp_path: Path) -> BackendFileReport:return BackendFileReport(diff --git a/tests/unit/pipelex/cli/test_doctor_internal_backend_sync.py b/tests/unit/pipelex/cli/test_doctor_internal_backend_sync.pynew file mode 100644index 000000000..b37c50357--- /dev/null+++ b/tests/unit/pipelex/cli/test_doctor_internal_backend_sync.py@@ -0,0 +1,123 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++import pytest+from rich.console import Console++from pipelex.cli.commands.doctor_cmd import (+ ConfigLocationInfo,+ PendingMigrationsCheck,+ PendingMigrationsFinding,+ TelemetryConfigCheck,+ TelemetryConfigFinding,+ check_internal_backend_sync,+ display_health_report,+)+from pipelex.cogt.models import deck_manifest+from pipelex.cogt.models.deck_manifest import DeckFileStatus, DeckSyncReport, KitManagedArea, compute_kit_manifest, write_manifest++if TYPE_CHECKING:+ from pathlib import Path++ from pytest_mock import MockerFixture++CLEAN_DECK = DeckSyncReport(kit_version="1.2.0", installed_kit_version="1.2.0", manifest_present=True, files={})+++class TestDoctorInternalBackendSync:+ @pytest.fixture+ def backends_dir(self, tmp_path: Path, mocker: MockerFixture) -> Path:+ """A kit whose internal.toml declares the built-in engine, and an installed backends dir whose copy predates it."""+ kit_backends_dir = tmp_path / "kit-backends"+ kit_backends_dir.mkdir()+ (kit_backends_dir / "internal.toml").write_text("internal-with-reportlab", encoding="utf-8")+ (kit_backends_dir / "openai.toml").write_text("openai-kit", encoding="utf-8")+ mocker.patch.object(deck_manifest, "kit_backends_dir", return_value=kit_backends_dir)+ mocker.patch.object(deck_manifest, "get_package_version", return_value="1.2.0")+ backends_dir = tmp_path / "inference" / "backends"+ backends_dir.mkdir(parents=True)+ (backends_dir / "internal.toml").write_text("internal-before-reportlab", encoding="utf-8")+ (backends_dir / "openai.toml").write_text("openai-user", encoding="utf-8")+ return backends_dir++ def test_a_missing_backends_dir_is_healthy(self, tmp_path: Path) -> None:+ """An uninitialized directory is the config checks' to report, as it is for the deck."""+ healthy, report, message = check_internal_backend_sync(config_dir=tmp_path)++ assert healthy is True+ assert report.manifest_present is False+ assert message == "Backends directory not present"++ def test_a_stale_internal_toml_is_reported_as_drift(self, tmp_path: Path, backends_dir: Path) -> None:+ """An internal.toml from before this release has no manifest and differs from the kit: the doctor points at `pipelex update`."""+ healthy, report, message = check_internal_backend_sync(config_dir=tmp_path)++ assert healthy is False+ assert message == "backends/internal.toml manifest missing — run `pipelex update` to materialize a baseline"+ assert report.files == {"internal.toml": DeckFileStatus.LOCALLY_MODIFIED}+ assert backends_dir.is_dir()++ def test_an_internal_toml_the_kit_moved_past_is_reported_as_behind(self, tmp_path: Path, backends_dir: Path, mocker: MockerFixture) -> None:+ write_manifest(+ deck_manifest.DeckManifest(+ kit_version="1.1.0", files={"internal.toml": deck_manifest.compute_file_sha256(backends_dir / "internal.toml")}+ ),+ installed_dir=backends_dir,+ )+ mocker.patch.object(deck_manifest, "get_package_version", return_value="1.2.0")++ healthy, report, message = check_internal_backend_sync(config_dir=tmp_path)++ assert healthy is False+ assert message == "backends/internal.toml installed for pipelex 1.1.0, current is 1.2.0 (1 file(s) need action)"+ assert report.files == {"internal.toml": DeckFileStatus.CLEAN_BEHIND}++ def test_an_internal_toml_in_sync_is_healthy(self, tmp_path: Path, backends_dir: Path) -> None:+ (backends_dir / "internal.toml").write_text("internal-with-reportlab", encoding="utf-8")+ write_manifest(compute_kit_manifest(area=KitManagedArea.BACKENDS), installed_dir=backends_dir)++ healthy, report, message = check_internal_backend_sync(config_dir=tmp_path)++ assert healthy is True+ assert message == "backends/internal.toml is up to date with pipelex 1.2.0"+ assert report.files == {"internal.toml": DeckFileStatus.UP_TO_DATE}++ def test_the_report_shows_the_drift_and_names_pipelex_update(self, mocker: MockerFixture) -> None:+ console = Console(width=200, record=True, color_system=None)+ mocker.patch("pipelex.cli.commands.doctor_cmd.get_console", return_value=console)+ stale_report = DeckSyncReport(+ kit_version="1.2.0", installed_kit_version=None, manifest_present=False, files={"internal.toml": DeckFileStatus.LOCALLY_MODIFIED}+ )+ report_kwargs: dict[str, Any] = {+ "config_healthy": True,+ "config_message": "All configuration files present and valid",+ "config_missing_count": 0,+ "pending_migrations_check": PendingMigrationsCheck(+ finding=PendingMigrationsFinding.UP_TO_DATE, message="Every configuration file is at the current schema"+ ),+ "telemetry_check": TelemetryConfigCheck(finding=TelemetryConfigFinding.HEALTHY, message="Telemetry configured (mode: off)"),+ "backends_healthy": True,+ "backends_message": "All backends have valid credentials",+ "backend_credential_reports": {},+ "models_healthy": True,+ "models_message": "Models are valid",+ "backend_file_reports": {},+ "deck_healthy": True,+ "deck_message": "Deck is up to date with pipelex 1.2.0",+ "deck_report": CLEAN_DECK,+ "internal_backend_healthy": False,+ "internal_backend_message": "backends/internal.toml manifest missing — run `pipelex update` to materialize a baseline",+ "internal_backend_report": stale_report,+ "config_location": ConfigLocationInfo(+ config_dir="/work/project/.pipelex", is_project_local=True, project_root="/work/project", global_config_dir="/home/user/.pipelex"+ ),+ }++ display_health_report(**report_kwargs)++ output = console.export_text()+ assert "Issues Found" in output+ assert "backends/internal.toml manifest missing" in output+ assert "internal.toml — locally modified" in output+ assert "pipelex update to refresh the model deck and backends/internal.toml" in outputdiff --git a/tests/unit/pipelex/cli/test_pipe_cmd.py b/tests/unit/pipelex/cli/test_pipe_cmd.pyindex 9506019ca..1547e87ee 100644--- a/tests/unit/pipelex/cli/test_pipe_cmd.py+++ b/tests/unit/pipelex/cli/test_pipe_cmd.py@@ -114,6 +114,23 @@ class TestCliPipeCmd:toml = _pipe_spec_to_toml(spec)assert 'model = "@default-extract-document"' in toml+ def test_doc_gen_model_in_toml(self) -> None:+ spec = parse_pipe_spec(+ {+ "pipe_code": "print_invoice",+ "description": "Print the invoice",+ "inputs": {"invoice": "Invoice"},+ "output": "Document",+ "format": "pdf",+ "model": "reportlab-pdf",+ "filename": "invoice.pdf",+ },+ pipe_type="PipeDocGen",+ )+ toml = _pipe_spec_to_toml(spec)+ assert 'model = "reportlab-pdf"' in toml+ assert 'filename = "invoice.pdf"' in toml+def test_img_gen_model_in_toml(self) -> None:spec = parse_pipe_spec({diff --git a/tests/unit/pipelex/cli/test_update_cmd.py b/tests/unit/pipelex/cli/test_update_cmd.pyindex ab79f0c67..2deacc730 100644--- a/tests/unit/pipelex/cli/test_update_cmd.py+++ b/tests/unit/pipelex/cli/test_update_cmd.py@@ -28,7 +28,13 @@ class TestUpdateCmd:for filename, content in files.items():(kit_dir / filename).write_text(content, encoding="utf-8")mocker.patch.object(deck_manifest, "kit_deck_dir", return_value=kit_dir)- mocker.patch.object(update_cmd_module, "kit_deck_dir", return_value=kit_dir)++ @staticmethod+ def _seed_kit_backends(mocker: MockerFixture, kit_backends_dir: Path, files: dict[str, str]) -> None:+ kit_backends_dir.mkdir(parents=True, exist_ok=True)+ for filename, content in files.items():+ (kit_backends_dir / filename).write_text(content, encoding="utf-8")+ mocker.patch.object(deck_manifest, "kit_backends_dir", return_value=kit_backends_dir)@staticmethoddef _seed_installed(deck_dir: Path, files: dict[str, str]) -> None:@@ -40,29 +46,57 @@ class TestUpdateCmd:def _patch_resolve_deck_dir(mocker: MockerFixture, deck_dir: Path) -> None:mocker.patch.object(update_cmd_module, "_resolve_deck_dir", return_value=deck_dir)+ @staticmethod+ def _patch_resolve_backends_dir(mocker: MockerFixture, backends_dir: Path) -> None:+ mocker.patch.object(update_cmd_module, "_resolve_backends_dir", return_value=backends_dir)+@staticmethoddef _patch_version(mocker: MockerFixture, version: str) -> None:mocker.patch.object(deck_manifest, "get_package_version", return_value=version)@pytest.fixturedef kit_and_deck(self, mocker: MockerFixture, tmp_path: Path) -> tuple[Path, Path]:- """Materialize a kit with one numbered file and an installed deck dir mirroring it."""+ """Materialize a kit with one numbered file and an installed deck dir mirroring it, and no installed backends dir."""kit_dir = tmp_path / "kit"deck_dir = tmp_path / "deck"contents = {"1_llm_deck.toml": "v1-content"}self._seed_kit(mocker, kit_dir, contents)self._seed_installed(deck_dir, contents)self._patch_resolve_deck_dir(mocker, deck_dir)+ # Never the real backends directory: an absent one is skipped, and the backends tests seed their own.+ self._patch_resolve_backends_dir(mocker, tmp_path / "no-backends")self._patch_version(mocker, "1.0.0")return kit_dir, deck_dir+ @pytest.fixture+ def kit_and_backends(self, mocker: MockerFixture, tmp_path: Path, kit_and_deck: tuple[Path, Path]) -> tuple[Path, Path]:+ """Materialize a kit backends dir and an installed one whose internal.toml predates the kit's, with no backends manifest.++ The deck is left in sync with its manifest, so only the internal backend needs an update.+ """+ _, deck_dir = kit_and_deck+ write_manifest(+ DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),+ installed_dir=deck_dir,+ )+ kit_backends_dir = tmp_path / "kit-backends"+ backends_dir = tmp_path / "backends"+ self._seed_kit_backends(+ mocker,+ kit_backends_dir,+ {"internal.toml": "internal-with-reportlab", "openai.toml": "openai-kit", "pipelex_gateway_models.md": "models-kit"},+ )+ self._seed_installed(backends_dir, {"internal.toml": "internal-before-reportlab", "openai.toml": "openai-user", "my_backend.toml": "mine"})+ self._patch_resolve_backends_dir(mocker, backends_dir)+ return kit_backends_dir, backends_dir+def test_dry_run_makes_no_changes(self, mocker: MockerFixture, kit_and_deck: tuple[Path, Path]) -> None:kit_dir, deck_dir = kit_and_deck# Kit has moved on, but the install hasn't yet.(kit_dir / "1_llm_deck.toml").write_text("v2-content", encoding="utf-8")write_manifest(DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),- deck_dir=deck_dir,+ installed_dir=deck_dir,)self._patch_version(mocker, "1.1.0")@@ -84,7 +118,7 @@ class TestUpdateCmd:kit_dir, deck_dir = kit_and_deckwrite_manifest(DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),- deck_dir=deck_dir,+ installed_dir=deck_dir,)(kit_dir / "1_llm_deck.toml").write_text("v2-content", encoding="utf-8")self._patch_version(mocker, "1.1.0")@@ -99,7 +133,7 @@ class TestUpdateCmd:kit_dir, deck_dir = kit_and_deckwrite_manifest(DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),- deck_dir=deck_dir,+ installed_dir=deck_dir,)(deck_dir / "1_llm_deck.toml").write_text("user-edits", encoding="utf-8")(kit_dir / "1_llm_deck.toml").write_text("v2-content", encoding="utf-8")@@ -117,7 +151,7 @@ class TestUpdateCmd:kit_dir, deck_dir = kit_and_deckwrite_manifest(DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),- deck_dir=deck_dir,+ installed_dir=deck_dir,)(deck_dir / "1_llm_deck.toml").write_text("user-edits", encoding="utf-8")(kit_dir / "1_llm_deck.toml").write_text("v2-content", encoding="utf-8")@@ -132,7 +166,7 @@ class TestUpdateCmd:kit_dir, deck_dir = kit_and_deckwrite_manifest(DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),- deck_dir=deck_dir,+ installed_dir=deck_dir,)(kit_dir / "5_new_deck.toml").write_text("brand-new", encoding="utf-8")self._patch_version(mocker, "1.1.0")@@ -152,7 +186,7 @@ class TestUpdateCmd:"9_retired_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "9_retired_deck.toml"),},),- deck_dir=deck_dir,+ installed_dir=deck_dir,)self._patch_version(mocker, "1.1.0")@@ -177,7 +211,7 @@ class TestUpdateCmd:kit_dir, deck_dir = kit_and_deckwrite_manifest(DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": deck_manifest.compute_file_sha256(deck_dir / "1_llm_deck.toml")}),- deck_dir=deck_dir,+ installed_dir=deck_dir,)(kit_dir / "1_llm_deck.toml").write_text("v2-content", encoding="utf-8")self._patch_version(mocker, "1.1.0")@@ -193,3 +227,55 @@ class TestUpdateCmd:with pytest.raises(SystemExit) as exit_info:update_cmd(yes=True)assert exit_info.value.code == 1++ def test_a_stale_internal_toml_is_refreshed_with_a_backup(self, kit_and_backends: tuple[Path, Path]) -> None:+ """An internal.toml from before this release has no manifest entry and differs from the kit: backed up, then refreshed."""+ _, backends_dir = kit_and_backends++ update_cmd(yes=True)++ assert (backends_dir / "internal.toml").read_text(encoding="utf-8") == "internal-with-reportlab"+ backups = sorted(backends_dir.glob("internal.toml.bak.*"))+ assert len(backups) == 1+ assert backups[0].read_text(encoding="utf-8") == "internal-before-reportlab"+ assert re.match(r"internal\.toml\.bak\.\d{8}T\d{6}Z", backups[0].name) is not None+ assert (backends_dir / MANIFEST_FILENAME).is_file()++ def test_a_stale_internal_toml_is_refreshed_without_a_backup_under_no_backup(self, kit_and_backends: tuple[Path, Path]) -> None:+ _, backends_dir = kit_and_backends++ update_cmd(yes=True, no_backup=True)++ assert (backends_dir / "internal.toml").read_text(encoding="utf-8") == "internal-with-reportlab"+ assert not list(backends_dir.glob("*.bak.*"))++ def test_other_backend_files_are_left_alone(self, kit_and_backends: tuple[Path, Path]) -> None:+ """Only internal.toml is managed: a kit backend file the user edited, and one of their own, stay as they are."""+ _, backends_dir = kit_and_backends++ update_cmd(yes=True)++ assert (backends_dir / "openai.toml").read_text(encoding="utf-8") == "openai-user"+ assert (backends_dir / "my_backend.toml").read_text(encoding="utf-8") == "mine"+ assert not (backends_dir / "pipelex_gateway_models.md").exists()+ assert sorted(path.name for path in backends_dir.glob("*.bak.*")) == [next(backends_dir.glob("internal.toml.bak.*")).name]++ def test_a_refreshed_internal_toml_is_up_to_date_afterwards(self, kit_and_backends: tuple[Path, Path]) -> None:+ """The update writes the backends manifest, so the next run finds nothing to do and the dry run changes nothing."""+ _, backends_dir = kit_and_backends+ update_cmd(yes=True)+ refreshed_mtime = (backends_dir / "internal.toml").stat().st_mtime_ns++ report = deck_manifest.compute_sync_report(backends_dir, area=deck_manifest.KitManagedArea.BACKENDS)+ update_cmd(dry_run=True)++ assert report.is_clean()+ assert (backends_dir / "internal.toml").stat().st_mtime_ns == refreshed_mtime++ def test_a_dry_run_leaves_a_stale_internal_toml_as_it_is(self, kit_and_backends: tuple[Path, Path]) -> None:+ _, backends_dir = kit_and_backends++ update_cmd(dry_run=True)++ assert (backends_dir / "internal.toml").read_text(encoding="utf-8") == "internal-before-reportlab"+ assert not (backends_dir / MANIFEST_FILENAME).exists()diff --git a/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py b/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.pyindex 071b7f199..c71a3012b 100644--- a/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py+++ b/tests/unit/pipelex/cogt/content_generation/test_assignment_read_coverage.py@@ -22,12 +22,17 @@ from pipelex.cogt.content_generation.assignment_models import (ImgGenAssignment,LLMAssignment,ObjectAssignment,+ RenderDocumentAssignment,RenderPageViewsAssignment,SearchAssignment,SearchObjectAssignment,TemplatingAssignment,)from pipelex.cogt.content_generation.cogt_run_params import CogtRunParams+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenSetting+from pipelex.cogt.doc_gen.document_composition import DocumentComposition+from pipelex.cogt.doc_gen.layout_tree import ImageBlock, LayoutDocument, SectionBlockfrom pipelex.cogt.document.prompt_document import PromptDocumentUrifrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParams@@ -53,6 +58,8 @@ URL_FIELD_INVENTORY: dict[type[BaseModel], set[str]] = {TemplatingAssignment: set(),ExtractAssignment: {"extract_input.image_uri", "extract_input.document_uri"},RenderPageViewsAssignment: {"document_uri"},+ # A section's blocks hold blocks again: the walk names the images inside one section and stops where a section would enclose itself.+ RenderDocumentAssignment: {"composition.layout.blocks.url", "composition.layout.blocks.blocks.url"},SearchAssignment: set(),SearchObjectAssignment: set(),}@@ -76,16 +83,18 @@ def _concrete_types(annotation: Any) -> list[Any]:return found-def _url_field_paths(model_class: type[BaseModel], *, prefix: str = "") -> set[str]:+def _url_field_paths(model_class: type[BaseModel], *, prefix: str = "", enclosing: frozenset[type] | None = None) -> set[str]:+ """The dotted paths of the URL-shaped fields under a model, not walking again into a model that encloses itself."""paths: set[str] = set()+ enclosing = (enclosing or frozenset()) | {model_class}for field_name, field_info in model_class.model_fields.items():if field_name in NON_PAYLOAD_FIELDS:continuefor concrete_type in _concrete_types(field_info.annotation):if concrete_type is str and URL_FIELD_NAME.search(field_name):paths.add(f"{prefix}{field_name}")- elif inspect.isclass(concrete_type) and issubclass(concrete_type, BaseModel):- paths |= _url_field_paths(concrete_type, prefix=f"{prefix}{field_name}.")+ elif inspect.isclass(concrete_type) and issubclass(concrete_type, BaseModel) and concrete_type not in enclosing:+ paths |= _url_field_paths(concrete_type, prefix=f"{prefix}{field_name}.", enclosing=enclosing)return paths@@ -164,6 +173,27 @@ URL_BEARING_SAMPLES: dict[type[BaseModel], tuple[_DeclaresReads, set[str]]] = {),{"pipelex-storage://s/doc.pdf"},),+ RenderDocumentAssignment: (+ RenderDocumentAssignment(+ job_metadata=_job_metadata(),+ cogt_run_params=CogtRunParams(run_mode=PipeRunMode.DRY),+ composition=DocumentComposition(+ format=DocGenFormat.PDF,+ source=DocGenSource.LAYOUT,+ filename="report.pdf",+ title="Report",+ layout=LayoutDocument(+ title="Report",+ blocks=[+ ImageBlock(url="pipelex-storage://s/cover.png"),+ SectionBlock(title="Figures", level=1, blocks=[ImageBlock(url="pipelex-storage://s/figure.png")]),+ ],+ ),+ ),+ doc_gen_setting=DocGenSetting(model="reportlab-pdf"),+ ),+ {"pipelex-storage://s/cover.png", "pipelex-storage://s/figure.png"},+ ),}diff --git a/tests/unit/pipelex/cogt/content_generation/test_doc_gen_render_resources.py b/tests/unit/pipelex/cogt/content_generation/test_doc_gen_render_resources.pynew file mode 100644index 000000000..f3b67eb3b--- /dev/null+++ b/tests/unit/pipelex/cogt/content_generation/test_doc_gen_render_resources.py@@ -0,0 +1,34 @@+import asyncio++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.content_generation import doc_gen_generate+from pipelex.cogt.content_generation.doc_gen_generate import RunRenderResources+from pipelex.cogt.doc_gen.exceptions import DocGenRenderError+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract++[email protected](loop_scope="class")+class TestRunRenderResources:+ async def test_a_read_that_times_out_is_a_render_error_and_is_cancelled(self, mocker: MockerFixture) -> None:+ """A slow read fails the print with a `DocGenRenderError` naming its position, and stops running on the loop."""+ was_cancelled = asyncio.Event()++ async def _never_answers(uri: str, *, storage_provider: StorageProviderAbstract) -> bytes: # ruff: ignore[unused-function-argument]+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ was_cancelled.set()+ raise+ return b""++ mocker.patch.object(doc_gen_generate, "_RESOURCE_LOAD_TIMEOUT_SECONDS", 0.05)+ mocker.patch.object(doc_gen_generate, "load_bytes_from_any_uri", _never_answers)+ resources = RunRenderResources(+ storage_provider=mocker.MagicMock(spec=StorageProviderAbstract), read_scope=None, loop=asyncio.get_running_loop()+ )++ with pytest.raises(DocGenRenderError, match="image 1 of the document"):+ await asyncio.to_thread(resources.load, uri="https://example.com/photo.png", position="image 1 of the document")+ await asyncio.wait_for(was_cancelled.wait(), timeout=5)diff --git a/tests/unit/pipelex/cogt/doc_gen/test_doc_gen_format.py b/tests/unit/pipelex/cogt/doc_gen/test_doc_gen_format.pynew file mode 100644index 000000000..a107693da--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_doc_gen_format.py@@ -0,0 +1,51 @@+import pytest++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource, doc_gen_choice_key, parse_doc_gen_choice_key+++class TestDocGenFormat:+ @pytest.mark.parametrize(+ ("doc_gen_format", "has_template", "expected_source"),+ [+ (DocGenFormat.PDF, False, DocGenSource.LAYOUT),+ (DocGenFormat.PDF, True, DocGenSource.HTML),+ (DocGenFormat.XLSX, False, DocGenSource.LAYOUT),+ (DocGenFormat.XLSX, True, DocGenSource.TEMPLATE_FILE),+ (DocGenFormat.DOCX, True, DocGenSource.TEMPLATE_FILE),+ (DocGenFormat.PPTX, True, DocGenSource.TEMPLATE_FILE),+ ],+ )+ def test_a_step_s_format_and_template_make_its_source(+ self, doc_gen_format: DocGenFormat, has_template: bool, expected_source: DocGenSource+ ) -> None:+ assert DocGenSource.for_step(doc_gen_format=doc_gen_format, has_template=has_template) == expected_source++ def test_only_a_pdf_has_an_html_template_and_only_a_deck_needs_one(self) -> None:+ assert [doc_gen_format for doc_gen_format in DocGenFormat if doc_gen_format.is_template_html] == [DocGenFormat.PDF]+ assert [doc_gen_format for doc_gen_format in DocGenFormat if not doc_gen_format.has_auto_layout] == [DocGenFormat.PPTX]+ assert DocGenFormat.PDF.template_file_suffix == ".html"+ assert DocGenFormat.DOCX.template_file_suffix == ".docx"+ assert DocGenFormat.PDF.mime_type == "application/pdf"+ assert DocGenFormat.XLSX.suffix == "xlsx"++ def test_every_step_a_format_can_make_has_a_choice_key_that_parses_back(self) -> None:+ for doc_gen_format in DocGenFormat:+ for source in DocGenSource.possible_for(doc_gen_format=doc_gen_format):+ key = doc_gen_choice_key(doc_gen_format=doc_gen_format, source=source)+ assert parse_doc_gen_choice_key(key) == (doc_gen_format, source)+ assert doc_gen_choice_key(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.LAYOUT) == "pdf.layout"++ @pytest.mark.parametrize(+ ("key", "expected"),+ [+ ("pdf", "is not a '<format>.<source>' key"),+ ("html.layout", "is not a '<format>.<source>' key"),+ ("pdf.markdown", "is not a '<format>.<source>' key"),+ ("pdf.template_file", "a pdf is never composed from a template file"),+ ("xlsx.html", "an xlsx is never composed from an HTML template"),+ ("pptx.layout", "a pptx is never composed from the auto-layout of its inputs"),+ ],+ )+ def test_a_choice_key_no_step_makes_is_refused(self, key: str, expected: str) -> None:+ with pytest.raises(ValueError, match=expected):+ parse_doc_gen_choice_key(key)diff --git a/tests/unit/pipelex/cogt/doc_gen/test_document_composition.py b/tests/unit/pipelex/cogt/doc_gen/test_document_composition.pynew file mode 100644index 000000000..aaf881f60--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_document_composition.py@@ -0,0 +1,81 @@+from pathlib import Path++import pytest+from pydantic import ValidationError++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.document_composition import DocumentComposition+from pipelex.cogt.doc_gen.layout_tree import ImageBlock, LayoutDocument, SectionBlock+++class TestDocumentComposition:+ def test_its_images_are_what_the_print_stage_reads(self) -> None:+ composition = DocumentComposition(+ format=DocGenFormat.PDF,+ source=DocGenSource.LAYOUT,+ filename="report.pdf",+ title="Report",+ layout=LayoutDocument(+ title="Report",+ blocks=[+ ImageBlock(url="pipelex-storage://s/cover.png"),+ SectionBlock(+ title="Figures",+ level=1,+ blocks=[+ ImageBlock(url="https://example.com/figure.png"),+ SectionBlock(title="Detail", level=2, blocks=[ImageBlock(url="https://example.com/detail.png")]),+ ],+ ),+ ],+ ),+ )++ uri_references = composition.referenced_uris()++ assert [uri_reference.uri for uri_reference in uri_references] == [+ "pipelex-storage://s/cover.png",+ "https://example.com/figure.png",+ "https://example.com/detail.png",+ ]+ assert uri_references[1].position == "image 2 of the document 'report.pdf'"++ def test_composed_html_declares_no_read(self) -> None:+ composition = DocumentComposition(+ format=DocGenFormat.PDF, source=DocGenSource.HTML, filename="report.pdf", title="Report", html='<img src="https://example.com/a.png">'+ )+ assert composition.referenced_uris() == []++ def test_the_render_job_carries_the_template_file_s_bytes(self, tmp_path: Path) -> None:+ template_file = tmp_path / "invoice.docx"+ template_file.write_bytes(b"docx bytes")+ composition = DocumentComposition(+ format=DocGenFormat.DOCX,+ source=DocGenSource.TEMPLATE_FILE,+ filename="invoice.docx",+ title="Invoice",+ template_path=str(template_file),+ template_name="invoice.docx",+ data={"invoice": {"number": "INV-1"}},+ )++ job = composition.make_render_job()++ assert job.template == b"docx bytes"+ assert job.template_name == "invoice.docx"+ assert job.data == {"invoice": {"number": "INV-1"}}++ def test_a_composition_without_its_payload_is_refused(self) -> None:+ with pytest.raises(ValidationError, match="carries its payload"):+ DocumentComposition(format=DocGenFormat.DOCX, source=DocGenSource.TEMPLATE_FILE, filename="invoice.docx", title="Invoice")++ def test_a_composition_with_another_source_s_payload_too_is_refused(self) -> None:+ with pytest.raises(ValidationError, match="carries only its own payload, not the template_file one"):+ DocumentComposition(+ format=DocGenFormat.PDF,+ source=DocGenSource.HTML,+ filename="report.pdf",+ title="Report",+ html="<p>Report</p>",+ template_path="/nowhere/report.docx",+ )diff --git a/tests/unit/pipelex/cogt/doc_gen/test_input_shape.py b/tests/unit/pipelex/cogt/doc_gen/test_input_shape.pynew file mode 100644index 000000000..848104e4f--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_input_shape.py@@ -0,0 +1,118 @@+import datetime+from enum import Enum, IntEnum+from typing import Literal++from pydantic import Field++from pipelex.cogt.doc_gen.input_shape import InputShape, InputShapeKind, shape_of_class, shape_of_input+from pipelex.core.stuffs.composite_content import CompositeContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.text_content import TextContent+++class _LineItem(StructuredContent):+ description: str+ amount: float = Field(title="Amount", description="In euros")+++class _Node(StructuredContent):+ name: str+ children: list["_Node"]+++class _Invoice(StructuredContent):+ number: str+ status: Literal["draft", "paid"]+ issued_at: datetime.datetime+ due_on: datetime.date | None = None+ is_paid: bool+ line_items: list[_LineItem]+ logo: ImageContent | None = None+++class _Priority(IntEnum):+ LOW = 1+ HIGH = 2+++class _Mixed(Enum):+ ONE = 1+ NAMED = "named"+++class _Ticket(StructuredContent):+ priority: _Priority+ mixed: _Mixed+ level: Literal[1, 2]+ is_flagged: Literal[True]+ status: Literal["open"] | None+ blend: Literal["open", 1]+++class TestInputShape:+ def test_a_structure_lists_its_fields_with_their_kinds(self) -> None:+ shape = shape_of_class(_Invoice)++ assert shape.kind == InputShapeKind.STRUCTURE+ kinds = {field_name: field_shape.kind for field_name, field_shape in shape.fields.items()}+ assert kinds == {+ "number": InputShapeKind.TEXT,+ "status": InputShapeKind.TEXT,+ "issued_at": InputShapeKind.DATETIME,+ "due_on": InputShapeKind.DATE,+ "is_paid": InputShapeKind.BOOLEAN,+ "line_items": InputShapeKind.LIST,+ "logo": InputShapeKind.IMAGE,+ }++ def test_a_list_field_describes_its_items(self) -> None:+ line_items = shape_of_class(_Invoice).fields["line_items"]++ assert line_items.item is not None+ amount = line_items.item.fields["amount"]+ assert amount.kind == InputShapeKind.NUMBER+ assert amount.title == "Amount"+ assert amount.description == "In euros"++ def test_native_contents_are_leaves(self) -> None:+ assert shape_of_class(TextContent).kind == InputShapeKind.TEXT+ assert shape_of_class(MarkdownContent).kind == InputShapeKind.MARKDOWN+ assert "url" in shape_of_class(ImageContent).fields++ def test_a_recursive_structure_stops_where_it_repeats(self) -> None:+ children = shape_of_class(_Node).fields["children"]+ assert children.item is not None+ assert children.item.kind == InputShapeKind.ANY++ def test_a_list_input_is_a_list_of_its_concept(self) -> None:+ shape = shape_of_input(content_class=_LineItem, is_list=True)+ assert shape.kind == InputShapeKind.LIST+ assert shape.item is not None+ assert shape.item.kind == InputShapeKind.STRUCTURE++ def test_an_input_without_a_content_class_has_an_undeclared_shape(self) -> None:+ assert shape_of_input(content_class=None, is_list=False).kind == InputShapeKind.ANY+ list_shape = shape_of_input(content_class=None, is_list=True)+ assert list_shape.kind == InputShapeKind.LIST+ assert list_shape.item is not None+ assert list_shape.item.kind == InputShapeKind.ANY++ def test_a_shape_round_trips_through_json(self) -> None:+ shape = shape_of_class(_Invoice)+ assert InputShape.model_validate_json(shape.model_dump_json()) == shape++ def test_an_enum_or_a_literal_has_the_kind_of_its_values(self) -> None:+ kinds = {field_name: field_shape.kind for field_name, field_shape in shape_of_class(_Ticket).fields.items()}+ assert kinds == {+ "priority": InputShapeKind.NUMBER,+ "mixed": InputShapeKind.ANY,+ "level": InputShapeKind.NUMBER,+ "is_flagged": InputShapeKind.BOOLEAN,+ "status": InputShapeKind.TEXT,+ "blend": InputShapeKind.ANY,+ }++ def test_a_composite_is_undeclared(self) -> None:+ assert shape_of_class(CompositeContent).kind == InputShapeKind.ANYdiff --git a/tests/unit/pipelex/cogt/doc_gen/test_layout_builder.py b/tests/unit/pipelex/cogt/doc_gen/test_layout_builder.pynew file mode 100644index 000000000..a198f34ee--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_layout_builder.py@@ -0,0 +1,183 @@+import datetime++from pydantic import Field++from pipelex.cogt.doc_gen.layout_builder import build_layout_document+from pipelex.cogt.doc_gen.layout_tree import FieldGridBlock, ImageBlock, LayoutField, MarkdownBlock, ParagraphsBlock, SectionBlock, TableBlock+from pipelex.core.stuffs.composite_content import CompositeContent+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.text_content import TextContent+++class _LineItem(StructuredContent):+ description: str+ quantity: int+ unit_price: float = Field(title="Unit price (EUR)")+++class _Customer(StructuredContent):+ name: str+ city: str+++class _Invoice(StructuredContent):+ number: str+ issued_on: datetime.date+ tags: list[str]+ customer: _Customer+ line_items: list[_LineItem]+++class _Tagged(StructuredContent):+ tags: set[str]+ codes: tuple[int, ...]+++def _invoice() -> _Invoice:+ return _Invoice(+ number="INV-1",+ issued_on=datetime.date(2026, 9, 30),+ tags=["urgent", "paid"],+ customer=_Customer(name="Ada", city="Paris"),+ line_items=[_LineItem(description="Tea", quantity=2, unit_price=3.5), _LineItem(description="Cake", quantity=1, unit_price=4.0)],+ )+++class TestLayoutBuilder:+ def test_a_single_structure_fills_the_top_level(self) -> None:+ document = build_layout_document(title="Invoice", named_contents=[("invoice", _invoice())])++ grid, customer, table = document.blocks+ assert isinstance(grid, FieldGridBlock)+ assert [(field.label, field.value) for field in grid.fields] == [+ ("Number", "INV-1"),+ ("Issued on", datetime.date(2026, 9, 30)),+ ("Tags", "urgent, paid"),+ ]+ assert isinstance(customer, SectionBlock)+ assert customer.title == "Customer"+ assert isinstance(table, TableBlock)+ assert table.path == "invoice.line_items"+ assert [column.label for column in table.columns] == ["Description", "Quantity", "Unit price (EUR)"]+ assert table.rows[1] == {"description": "Cake", "quantity": 1, "unit_price": 4.0}++ def test_a_single_markdown_input_is_the_whole_body(self) -> None:+ document = build_layout_document(title="Report", named_contents=[("report", MarkdownContent(text="# Findings\n\n**Bold** claim."))])+ assert document.blocks == [MarkdownBlock(markdown="# Findings\n\n**Bold** claim.")]++ def test_a_single_text_input_prints_as_paragraphs(self) -> None:+ document = build_layout_document(title="Notice", named_contents=[("notice", TextContent(text="First.\n\nSecond, with **stars**."))])+ assert document.blocks == [ParagraphsBlock(paragraphs=["First.", "Second, with **stars**."])]++ def test_several_inputs_get_a_section_each_in_order(self) -> None:+ document = build_layout_document(+ title="Pack",+ named_contents=[+ ("summary", TextContent(text="All good.")),+ ("cover_image", ImageContent(url="pipelex-storage://s/cover.png")),+ ("invoice", _invoice()),+ ],+ )++ summary, image, invoice = document.blocks+ assert isinstance(summary, SectionBlock)+ assert summary.title == "Summary"+ assert image == ImageBlock(url="pipelex-storage://s/cover.png", caption="Cover image")+ assert isinstance(invoice, SectionBlock)+ assert invoice.title == "Invoice"+ assert document.image_urls() == ["pipelex-storage://s/cover.png"]++ def test_a_list_input_of_flat_structures_is_a_table(self) -> None:+ items = ListContent[_LineItem](items=[_LineItem(description="Tea", quantity=2, unit_price=3.5)])+ document = build_layout_document(title="Items", named_contents=[("line_items", items)])++ (table,) = document.blocks+ assert isinstance(table, TableBlock)+ assert table.title == "Line items"++ def test_a_list_input_of_images_prints_each_image(self) -> None:+ photos = ListContent[ImageContent](+ items=[ImageContent(url="pipelex-storage://s/front.png"), ImageContent(url="pipelex-storage://s/back.png", caption="Back")]+ )+ document = build_layout_document(title="Photos", named_contents=[("photos", photos)])++ (section,) = document.blocks+ assert isinstance(section, SectionBlock)+ assert section.blocks == [+ ImageBlock(url="pipelex-storage://s/front.png", caption="Photo 1"),+ ImageBlock(url="pipelex-storage://s/back.png", caption="Back"),+ ]+ assert document.image_urls() == ["pipelex-storage://s/front.png", "pipelex-storage://s/back.png"]++ def test_a_list_input_of_markdown_formats_each_text(self) -> None:+ sections = ListContent[MarkdownContent](items=[MarkdownContent(text="# One"), MarkdownContent(text="**Two**")])+ document = build_layout_document(title="Report", named_contents=[("sections", sections)])++ (section,) = document.blocks+ assert isinstance(section, SectionBlock)+ first, second = section.blocks+ assert isinstance(first, SectionBlock)+ assert first.blocks == [MarkdownBlock(markdown="# One")]+ assert isinstance(second, SectionBlock)+ assert second.blocks == [MarkdownBlock(markdown="**Two**")]++ def test_a_structure_field_listing_images_prints_them(self) -> None:+ class _Inspection(StructuredContent):+ site: str+ photos: list[ImageContent]++ inspection = _Inspection(site="Roof", photos=[ImageContent(url="pipelex-storage://s/a.png"), ImageContent(url="pipelex-storage://s/b.png")])+ document = build_layout_document(title="Inspection", named_contents=[("inspection", inspection)])++ assert not any(isinstance(block, TableBlock) for block in document.blocks)+ assert document.image_urls() == ["pipelex-storage://s/a.png", "pipelex-storage://s/b.png"]++ def test_an_html_input_prints_its_visible_text(self) -> None:+ document = build_layout_document(+ title="Page", named_contents=[("page", HtmlContent(inner_html="<h1>Title</h1><script>alert(1)</script><p>Body <b>text</b>.</p>"))]+ )++ assert document.blocks == [ParagraphsBlock(paragraphs=["Title", "Body text."])]++ def test_a_composite_lays_out_its_components(self) -> None:+ composite = CompositeContent.model_validate(+ {"summary": TextContent(text="All good."), "score": NumberContent(number=3), "details": _Customer(name="Ada", city="Paris")}+ )++ document = build_layout_document(title="Review", named_contents=[("review", composite)])++ grid, details = document.blocks+ assert isinstance(grid, FieldGridBlock)+ assert [(field.label, field.value) for field in grid.fields] == [("Summary", "All good."), ("Score", 3)]+ assert isinstance(details, SectionBlock)+ assert details.title == "Details"++ def test_a_json_input_lays_out_nested_objects(self) -> None:+ payload = JSONContent(json_obj={"customer": {"name": "Ada"}, "tags": ["a", "b"], "lines": [{"sku": "X", "qty": 1}]})++ document = build_layout_document(title="Order", named_contents=[("order", payload)])++ (order,) = document.blocks+ assert isinstance(order, SectionBlock)+ grid, customer, lines = order.blocks+ assert isinstance(grid, FieldGridBlock)+ assert [(field.label, field.value) for field in grid.fields] == [("Tags", "a, b")]+ assert isinstance(customer, SectionBlock)+ assert customer.blocks == [FieldGridBlock(fields=[LayoutField(label="Name", value="Ada")])]+ assert isinstance(lines, SectionBlock)+ (line,) = lines.blocks+ assert isinstance(line, SectionBlock)+ assert line.blocks == [FieldGridBlock(fields=[LayoutField(label="Sku", value="X"), LayoutField(label="Qty", value=1)])]++ def test_a_set_field_lays_out_as_its_items(self) -> None:+ tagged = _Tagged(tags={"b", "a"}, codes=(1, 2))++ document = build_layout_document(title="Tagged", named_contents=[("tagged", tagged)])++ assert document.blocks == [FieldGridBlock(fields=[LayoutField(label="Tags", value="a, b"), LayoutField(label="Codes", value="1, 2")])]diff --git a/tests/unit/pipelex/cogt/doc_gen/test_layout_builder_helpers.py b/tests/unit/pipelex/cogt/doc_gen/test_layout_builder_helpers.pynew file mode 100644index 000000000..ef19c3b93--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_layout_builder_helpers.py@@ -0,0 +1,15 @@+from pipelex.cogt.doc_gen.layout_builder import html_to_paragraphs, humanize+++class TestLayoutBuilderHelpers:+ def test_humanize(self) -> None:+ assert humanize("unit_price") == "Unit price"+ assert humanize("lineItems") == "LineItems"+ assert humanize("_") == "_"++ def test_html_to_paragraphs_breaks_on_block_tags_and_hides_scripts(self) -> None:+ assert html_to_paragraphs("<ul><li>one</li><li>two & three</li></ul><style>p {}</style><script>hidden</script>") == ["one", "two & three"]++ def test_html_to_paragraphs_keeps_a_table_row_on_one_line_with_its_cells_apart(self) -> None:+ html = "<table><tr><th>Name</th><th>City</th></tr><tr><td>Ada</td><td>London</td></tr></table><dl><dt>Term</dt><dd>Meaning</dd></dl>"+ assert html_to_paragraphs(html) == ["Name | City", "Ada | London", "Term", "Meaning"]diff --git a/tests/unit/pipelex/cogt/doc_gen/test_plain_data.py b/tests/unit/pipelex/cogt/doc_gen/test_plain_data.pynew file mode 100644index 000000000..0823207ef--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_plain_data.py@@ -0,0 +1,40 @@+import datetime+from enum import StrEnum++from pipelex.cogt.doc_gen.plain_data import plain_data+from pipelex.core.stuffs.composite_content import CompositeContent+from pipelex.core.stuffs.date_content import DateContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.text_content import TextContent+++class _Color(StrEnum):+ RED = "red"+ BLUE = "blue"+++class _Tagged(StructuredContent):+ name: str+ colors: set[_Color]+ codes: frozenset[str]+++class TestPlainData:+ def test_a_structure_is_a_dict_of_its_fields(self) -> None:+ tagged = _Tagged(name="Ada", colors={_Color.RED, _Color.BLUE}, codes=frozenset({"b", "a"}))+ assert plain_data(tagged) == {"name": "Ada", "colors": ["blue", "red"], "codes": ["a", "b"]}++ def test_a_composite_is_a_dict_of_its_components(self) -> None:+ composite = CompositeContent.model_validate({"summary": TextContent(text="All good."), "score": NumberContent(number=3)})+ assert plain_data(composite) == {"summary": "All good.", "score": 3}++ def test_a_list_stuff_is_a_list(self) -> None:+ texts = ListContent[TextContent](items=[TextContent(text="one"), TextContent(text="two")])+ assert plain_data(texts) == ["one", "two"]++ def test_a_date_with_a_time_of_day_is_a_datetime(self) -> None:+ assert plain_data(DateContent(date=datetime.date(2026, 9, 30))) == datetime.date(2026, 9, 30)+ with_time = DateContent(date=datetime.date(2026, 9, 30), time=datetime.time(9, 30))+ assert plain_data(with_time) == datetime.datetime(2026, 9, 30, 9, 30)diff --git a/tests/unit/pipelex/cogt/doc_gen/test_render_job.py b/tests/unit/pipelex/cogt/doc_gen/test_render_job.pynew file mode 100644index 000000000..2f03e323a--- /dev/null+++ b/tests/unit/pipelex/cogt/doc_gen/test_render_job.py@@ -0,0 +1,58 @@+import base64+import json++import pytest+from pydantic import ValidationError++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.layout_tree import LayoutDocument, ParagraphsBlock+from pipelex.cogt.doc_gen.render_job import RenderJob+++def _layout() -> LayoutDocument:+ return LayoutDocument(title="Notice", blocks=[ParagraphsBlock(paragraphs=["Closed on Monday."])])+++class TestRenderJob:+ def test_a_job_carries_the_payload_its_source_names(self) -> None:+ job = RenderJob(format=DocGenFormat.PDF, source=DocGenSource.LAYOUT, filename="notice.pdf", title="Notice", layout=_layout())+ assert job.layout is not None++ def test_a_job_without_its_payload_is_refused(self) -> None:+ with pytest.raises(ValidationError, match="missing"):+ RenderJob(format=DocGenFormat.PDF, source=DocGenSource.HTML, filename="notice.pdf", title="Notice", layout=_layout())++ def test_a_job_with_another_source_s_payload_too_is_refused(self) -> None:+ with pytest.raises(ValidationError, match="only its own payload"):+ RenderJob(+ format=DocGenFormat.PDF,+ source=DocGenSource.HTML,+ filename="notice.pdf",+ title="Notice",+ html="<p>Closed</p>",+ layout=_layout(),+ )++ def test_a_stray_field_is_refused(self) -> None:+ with pytest.raises(ValidationError, match="extra"):+ RenderJob.model_validate(+ {"format": "pdf", "source": "html", "filename": "notice.pdf", "title": "Notice", "html": "<p/>", "engine": "weasyprint"}+ )++ def test_a_job_round_trips_through_json_with_the_template_as_base64(self) -> None:+ template = b"PK\x03\x04\xfb\xff not really a workbook"+ job = RenderJob(+ format=DocGenFormat.XLSX,+ source=DocGenSource.TEMPLATE_FILE,+ filename="invoice.xlsx",+ title="Invoice",+ template=template,+ template_name="invoice.xlsx",+ data={"invoice": {"number": "INV-1", "total": 12.5}},+ )++ json_text = job.model_dump_json()++ # Pydantic writes bytes as URL-safe base64, '-' and '_' where standard base64 has '+' and '/'.+ assert json.loads(json_text)["template"] == base64.urlsafe_b64encode(template).decode()+ assert RenderJob.model_validate_json(json_text) == jobdiff --git a/tests/unit/pipelex/cogt/models/test_deck_manifest.py b/tests/unit/pipelex/cogt/models/test_deck_manifest.pyindex 91b551358..efc4784d6 100644--- a/tests/unit/pipelex/cogt/models/test_deck_manifest.py+++ b/tests/unit/pipelex/cogt/models/test_deck_manifest.py@@ -10,14 +10,16 @@ from pipelex.cogt.models.deck_manifest import (MANIFEST_FILENAME,DeckFileStatus,DeckManifest,- compute_deck_sync_report,+ KitManagedArea,compute_file_sha256,compute_kit_manifest,+ compute_sync_report,is_deck_stale_fast,list_managed_installed_files,list_managed_kit_files,manifest_path,read_manifest,+ stamp_kit_manifests,suggest_x_custom_filename,write_manifest,)@@ -45,7 +47,7 @@ class TestDeckManifest:def test_manifest_round_trip(self, tmp_path: Path) -> None:manifest = DeckManifest(kit_version="1.2.3", files={"1_llm_deck.toml": "hash-a", "2_img_gen_deck.toml": "hash-b"})deck_dir = tmp_path / "deck"- write_manifest(manifest, deck_dir=deck_dir)+ write_manifest(manifest, installed_dir=deck_dir)assert manifest_path(deck_dir).is_file()reloaded = read_manifest(deck_dir)@@ -89,7 +91,7 @@ class TestDeckManifest:".DS_Store": "junk",},)- kit_files = list_managed_kit_files()+ kit_files = list_managed_kit_files(area=KitManagedArea.DECK)assert set(kit_files.keys()) == {"1_llm_deck.toml", "2_img_gen_deck.toml"}def test_list_managed_installed_files_filters_same_way(self, tmp_path: Path) -> None:@@ -102,7 +104,7 @@ class TestDeckManifest:"cookbook.toml": "c",},)- installed = list_managed_installed_files(deck_dir)+ installed = list_managed_installed_files(deck_dir, area=KitManagedArea.DECK)assert set(installed.keys()) == {"1_llm_deck.toml"}def test_list_managed_installed_files_ignores_non_numbered_user_tomls(self, tmp_path: Path) -> None:@@ -119,18 +121,18 @@ class TestDeckManifest:"notes.toml": "user-authored",},)- installed = list_managed_installed_files(deck_dir)+ installed = list_managed_installed_files(deck_dir, area=KitManagedArea.DECK)assert set(installed.keys()) == {"1_llm_deck.toml"}def test_list_managed_installed_files_missing_dir(self, tmp_path: Path) -> None:- assert list_managed_installed_files(tmp_path / "absent") == {}+ assert list_managed_installed_files(tmp_path / "absent", area=KitManagedArea.DECK) == {}def test_compute_kit_manifest_stamps_version(self, mocker: MockerFixture, tmp_path: Path) -> None:kit_dir = tmp_path / "kit"self._seed_kit(mocker, kit_dir, {"1_llm_deck.toml": "content-a"})mocker.patch.object(deck_manifest, "get_package_version", return_value="9.9.9")- manifest = compute_kit_manifest()+ manifest = compute_kit_manifest(area=KitManagedArea.DECK)assert manifest.kit_version == "9.9.9"assert "1_llm_deck.toml" in manifest.files@@ -141,9 +143,9 @@ class TestDeckManifest:self._seed_kit(mocker, kit_dir, contents)self._seed_installed(deck_dir, contents)mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")- write_manifest(compute_kit_manifest(), deck_dir=deck_dir)+ write_manifest(compute_kit_manifest(area=KitManagedArea.DECK), installed_dir=deck_dir)- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.is_clean()assert report.files["1_llm_deck.toml"] == DeckFileStatus.UP_TO_DATE@@ -153,13 +155,13 @@ class TestDeckManifest:self._seed_kit(mocker, kit_dir, {"1_llm_deck.toml": "old"})self._seed_installed(deck_dir, {"1_llm_deck.toml": "old"})mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")- write_manifest(compute_kit_manifest(), deck_dir=deck_dir)+ write_manifest(compute_kit_manifest(area=KitManagedArea.DECK), installed_dir=deck_dir)# Simulate a kit upgrade — the file content has moved on but the user has not edited theirs.(kit_dir / "1_llm_deck.toml").write_text("new", encoding="utf-8")mocker.patch.object(deck_manifest, "get_package_version", return_value="1.1.0")- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.files["1_llm_deck.toml"] == DeckFileStatus.CLEAN_BEHINDassert not report.is_clean()assert report.installed_kit_version == "1.0.0"@@ -170,12 +172,12 @@ class TestDeckManifest:self._seed_kit(mocker, kit_dir, {"1_llm_deck.toml": "kit-version"})self._seed_installed(deck_dir, {"1_llm_deck.toml": "kit-version"})mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")- write_manifest(compute_kit_manifest(), deck_dir=deck_dir)+ write_manifest(compute_kit_manifest(area=KitManagedArea.DECK), installed_dir=deck_dir)# User edits their installed file, kit unchanged.(deck_dir / "1_llm_deck.toml").write_text("user-edited", encoding="utf-8")- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.files["1_llm_deck.toml"] == DeckFileStatus.LOCALLY_MODIFIEDdef test_sync_report_kit_added(self, mocker: MockerFixture, tmp_path: Path) -> None:@@ -186,10 +188,10 @@ class TestDeckManifest:mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")# Manifest captures only the file that existed at install time.write_manifest(- DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": compute_file_sha256(deck_dir / "1_llm_deck.toml")}), deck_dir=deck_dir+ DeckManifest(kit_version="1.0.0", files={"1_llm_deck.toml": compute_file_sha256(deck_dir / "1_llm_deck.toml")}), installed_dir=deck_dir)- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.files["5_new_deck.toml"] == DeckFileStatus.KIT_ADDEDassert report.files["1_llm_deck.toml"] == DeckFileStatus.UP_TO_DATE@@ -207,10 +209,10 @@ class TestDeckManifest:"9_retired_deck.toml": compute_file_sha256(deck_dir / "9_retired_deck.toml"),},),- deck_dir=deck_dir,+ installed_dir=deck_dir,)- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.files["9_retired_deck.toml"] == DeckFileStatus.KIT_REMOVEDassert report.files["1_llm_deck.toml"] == DeckFileStatus.UP_TO_DATE@@ -220,9 +222,9 @@ class TestDeckManifest:self._seed_kit(mocker, kit_dir, {"1_llm_deck.toml": "kit"})self._seed_installed(deck_dir, {"1_llm_deck.toml": "kit", "cookbook.toml": "user-content"})mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")- write_manifest(compute_kit_manifest(), deck_dir=deck_dir)+ write_manifest(compute_kit_manifest(area=KitManagedArea.DECK), installed_dir=deck_dir)- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert "cookbook.toml" not in report.filesassert report.files["1_llm_deck.toml"] == DeckFileStatus.UP_TO_DATE@@ -234,7 +236,7 @@ class TestDeckManifest:self._seed_installed(deck_dir, contents)mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.manifest_present is Falseassert report.files["1_llm_deck.toml"] == DeckFileStatus.UP_TO_DATE# No manifest means the report still cannot be considered "clean" — boot warn still fires.@@ -247,7 +249,7 @@ class TestDeckManifest:self._seed_installed(deck_dir, {"1_llm_deck.toml": "drift"})mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")- report = compute_deck_sync_report(deck_dir)+ report = compute_sync_report(deck_dir, area=KitManagedArea.DECK)assert report.manifest_present is Falseassert report.files["1_llm_deck.toml"] == DeckFileStatus.LOCALLY_MODIFIED@@ -276,7 +278,7 @@ class TestDeckManifest:) -> None:deck_dir = tmp_path / "deck"deck_dir.mkdir()- write_manifest(DeckManifest(kit_version=manifest_version, files={}), deck_dir=deck_dir)+ write_manifest(DeckManifest(kit_version=manifest_version, files={}), installed_dir=deck_dir)mocker.patch.object(deck_manifest, "get_package_version", return_value=current_version)assert is_deck_stale_fast(deck_dir) is expected_stale@@ -301,3 +303,37 @@ class TestDeckManifest:)def test_suggest_x_custom_filename(self, numbered_filename: str, expected_override: str) -> None:assert suggest_x_custom_filename(numbered_filename) == expected_override++ def test_only_internal_toml_is_managed_in_the_backends_area(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """The kit's other backend files and a user's own are never the update's: only internal.toml is hashed, on either side."""+ kit_backends_dir = tmp_path / "kit-backends"+ kit_backends_dir.mkdir()+ for filename in ("internal.toml", "openai.toml", "pipelex_gateway_models.md"):+ (kit_backends_dir / filename).write_text(f"kit {filename}", encoding="utf-8")+ mocker.patch.object(deck_manifest, "kit_backends_dir", return_value=kit_backends_dir)+ backends_dir = tmp_path / "backends"+ backends_dir.mkdir()+ for filename in ("internal.toml", "openai.toml", "my_backend.toml"):+ (backends_dir / filename).write_text(f"user {filename}", encoding="utf-8")++ assert list(list_managed_kit_files(area=KitManagedArea.BACKENDS)) == ["internal.toml"]+ assert list(list_managed_installed_files(backends_dir, area=KitManagedArea.BACKENDS)) == ["internal.toml"]++ def test_stamping_writes_a_manifest_per_area(self, tmp_path: Path, mocker: MockerFixture) -> None:+ self._seed_kit(mocker, tmp_path / "kit-deck", {"1_llm_deck.toml": "deck"})+ kit_backends_dir = tmp_path / "kit-backends"+ kit_backends_dir.mkdir()+ (kit_backends_dir / "internal.toml").write_text("internal", encoding="utf-8")+ mocker.patch.object(deck_manifest, "kit_backends_dir", return_value=kit_backends_dir)+ mocker.patch.object(deck_manifest, "get_package_version", return_value="1.0.0")+ inference_dir = tmp_path / "inference"++ stamp_kit_manifests(inference_dir=inference_dir)++ deck_manifest_written = read_manifest(inference_dir / "deck")+ backends_manifest_written = read_manifest(inference_dir / "backends")+ assert deck_manifest_written is not None+ assert list(deck_manifest_written.files) == ["1_llm_deck.toml"]+ assert backends_manifest_written is not None+ assert backends_manifest_written.kit_version == "1.0.0"+ assert backends_manifest_written.files == {"internal.toml": compute_file_sha256(kit_backends_dir / "internal.toml")}diff --git a/tests/unit/pipelex/cogt/models/test_enforce_gateway_model_membership.py b/tests/unit/pipelex/cogt/models/test_enforce_gateway_model_membership.pyindex 3a5c6c46c..b328798c4 100644--- a/tests/unit/pipelex/cogt/models/test_enforce_gateway_model_membership.py+++ b/tests/unit/pipelex/cogt/models/test_enforce_gateway_model_membership.py@@ -292,3 +292,22 @@ class TestAWaterfallSpansBackends:assert refused.value.model_name == MANIFOLD_HANDLEassert refused.value.backend_name == PipelexBackend.MANIFOLD+++class TestDocumentEnginesAreNotGatewayModels:+ """A document engine is software on the host, a `doc_gen` model no managed service serves.++ The kit's deck points `pdf.layout` at `reportlab-pdf`, which the default route sends to the gateway on an+ installation whose `internal.toml` predates it or whose internal backend is disabled. Such a step is refused at+ load, naming the engine and the fix; the boot must not fail on it, blaming a gateway that was never asked.+ """++ def test_a_doc_gen_default_whose_engine_is_undeclared_does_not_fail_the_boot(self) -> None:+ manager = _make_manager()+ manager.model_deck = manager.get_model_deck().model_copy(update={"doc_gen_choice_defaults": {"pdf.layout": "reportlab-pdf"}})++ _enforce(+ manager,+ gateway_specs=_specs(*EVERY_DEFAULT_ROUTED_HANDLE),+ manifold_specs=_specs(),+ )diff --git a/tests/unit/pipelex/cogt/models/test_inference_config_overrides_end_to_end.py b/tests/unit/pipelex/cogt/models/test_inference_config_overrides_end_to_end.pyindex b74e519dc..afb34f125 100644--- a/tests/unit/pipelex/cogt/models/test_inference_config_overrides_end_to_end.py+++ b/tests/unit/pipelex/cogt/models/test_inference_config_overrides_end_to_end.py@@ -17,6 +17,7 @@ import pytestfrom pipelex.cogt.models.model_manager import ModelManagerfrom pipelex.kit.paths import get_kit_configs_dir+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarationsfrom pipelex.system.configuration.config_loader import CONFIG_DIR_NAME, INFERENCE_DIR_NAMEfrom pipelex.system.pipelex_service.pipelex_service_config import is_pipelex_gateway_enabledfrom pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider@@ -51,6 +52,7 @@ class TestInferenceConfigOverridesEndToEnd:secrets_provider=EnvSecretsProvider(),managed_gateway_configs=None,gateway_config_source=None,+ plugin_model_declarations=PluginModelDeclarations.make_empty(),needs_inference=False,)return models_manager.routing_profile.namediff --git a/tests/unit/pipelex/cogt/models/test_model_deck_references.py b/tests/unit/pipelex/cogt/models/test_model_deck_references.pyindex b51c4f570..9e2ef72ea 100644--- a/tests/unit/pipelex/cogt/models/test_model_deck_references.py+++ b/tests/unit/pipelex/cogt/models/test_model_deck_references.py@@ -8,6 +8,7 @@ from typing import castimport pytest+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenSettingfrom pipelex.cogt.extract.extract_setting import ExtractSettingfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenSettingfrom pipelex.cogt.llm.llm_setting import LLMSetting@@ -114,7 +115,7 @@ class TestModelDeckReferences:def _find_invalid_preset_references(self,- presets: dict[str, LLMSetting] | dict[str, ExtractSetting] | dict[str, ImgGenSetting] | dict[str, SearchSetting],+ presets: dict[str, LLMSetting] | dict[str, ExtractSetting] | dict[str, ImgGenSetting] | dict[str, SearchSetting] | dict[str, DocGenSetting],all_aliases: dict[str, str],all_waterfalls: dict[str, list[str]],known_model_handles: dict[str, ModelType],@@ -177,6 +178,8 @@ class TestModelDeckReferences:return model_deck_blueprint.img_gencase ModelType.SEARCH:return model_deck_blueprint.search+ case ModelType.DOC_GEN:+ return model_deck_blueprint.doc_gen@pytest.mark.parametrize(("model_type", "deck_name"),@@ -185,6 +188,7 @@ class TestModelDeckReferences:(ModelType.TEXT_EXTRACTOR, "Extract"),(ModelType.IMG_GEN, "ImgGen"),(ModelType.SEARCH, "Search"),+ (ModelType.DOC_GEN, "DocGen"),],)def test_aliases_reference_valid_targets(@@ -218,6 +222,7 @@ class TestModelDeckReferences:(ModelType.TEXT_EXTRACTOR, "Extract"),(ModelType.IMG_GEN, "ImgGen"),(ModelType.SEARCH, "Search"),+ (ModelType.DOC_GEN, "DocGen"),],)def test_presets_reference_valid_models(@@ -251,6 +256,7 @@ class TestModelDeckReferences:(ModelType.TEXT_EXTRACTOR, "Extract"),(ModelType.IMG_GEN, "ImgGen"),(ModelType.SEARCH, "Search"),+ (ModelType.DOC_GEN, "DocGen"),],)def test_waterfalls_contain_valid_models(@@ -283,6 +289,7 @@ class TestModelDeckReferences:(ModelType.TEXT_EXTRACTOR, "Extract"),(ModelType.IMG_GEN, "ImgGen"),(ModelType.SEARCH, "Search"),+ (ModelType.DOC_GEN, "DocGen"),],)def test_aliases_no_circular_references(diff --git a/tests/unit/pipelex/cogt/models/test_plugin_model_merge.py b/tests/unit/pipelex/cogt/models/test_plugin_model_merge.pynew file mode 100644index 000000000..2839fe69b--- /dev/null+++ b/tests/unit/pipelex/cogt/models/test_plugin_model_merge.py@@ -0,0 +1,173 @@+from __future__ import annotations++import shutil+from pathlib import Path+from types import SimpleNamespace+from typing import TYPE_CHECKING, Any, cast++import pytest++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.exceptions import PluginModelDeclarationError+from pipelex.cogt.model_backends.backend import PipelexBackend+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_manager import ModelManager+from pipelex.cogt.models.model_reference import ModelReference+from pipelex.kit.paths import get_kit_configs_dir+from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.plugin_group import PluginGroup+from pipelex.plugins.registrar import PluginOrigin, PluginRegistrar+from pipelex.system.configuration.config_loader import INFERENCE_DIR_NAME+from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider++if TYPE_CHECKING:+ from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations+ from pipelex.system.configuration.configs import PipelexConfig++PIPELEX_XLSX_SPEC: dict[str, Any] = {+ "model_type": "doc_gen",+ "sdk": "openpyxl",+ "model_id": "write-xlsx",+ "inputs": ["layout", "template_file"],+ "outputs": ["xlsx"],+ "costs": {},+}+++def _declarations(+ *, doc_gen_defaults: dict[tuple[DocGenFormat, DocGenSource], str], internal_models: dict[str, dict[str, Any]]+) -> PluginModelDeclarations:+ """The declarations of one plugin, built the way the registrar builds them at boot."""+ config = cast("PipelexConfig", SimpleNamespace(runtime=SimpleNamespace(plugins=SimpleNamespace(disabled=[]))))+ registrar = PluginRegistrar(config=config)+ registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)+ for name, spec in internal_models.items():+ registrar.add_internal_model(name=name, spec=spec)+ for (doc_gen_format, source), model in doc_gen_defaults.items():+ registrar.add_doc_gen_default(doc_gen_format=doc_gen_format, source=source, model=model)+ return registrar.make_model_declarations()+++def _xlsx_plugin_declarations() -> PluginModelDeclarations:+ """A plugin declaring one engine, its default for its own format, and a default for the kit's pdf.layout, which the kit's deck file overrides."""+ return _declarations(+ internal_models={"pipelex-xlsx": PIPELEX_XLSX_SPEC},+ doc_gen_defaults={(DocGenFormat.XLSX, DocGenSource.LAYOUT): "pipelex-xlsx", (DocGenFormat.PDF, DocGenSource.LAYOUT): "pipelex-xlsx"},+ )+++class TestPluginModelMerge:+ @pytest.fixture+ def inference_dir(self, tmp_path: Path) -> Path:+ """The kit's inference tree, routed to the internal backend so a lenient setup needs no credential and no gateway."""+ inference_dir = tmp_path / INFERENCE_DIR_NAME+ shutil.copytree(Path(str(get_kit_configs_dir())) / INFERENCE_DIR_NAME, inference_dir)+ routing_profiles_path = inference_dir / "routing_profiles.toml"+ routing_profiles = routing_profiles_path.read_text(encoding="utf-8")+ routing_profiles_path.write_text(routing_profiles.replace('active = "all_pipelex_gateway"', 'active = "all_internal"', 1), encoding="utf-8")+ return inference_dir++ @staticmethod+ def _setup(*, inference_dir: Path, plugin_model_declarations: PluginModelDeclarations) -> ModelManager:+ models_manager = ModelManager()+ models_manager.setup(+ secrets_provider=EnvSecretsProvider(),+ managed_gateway_configs=None,+ gateway_config_source=None,+ plugin_model_declarations=plugin_model_declarations,+ needs_inference=False,+ backends_library_paths=[inference_dir / "backends.toml"],+ backends_dir_path=str(inference_dir / "backends"),+ routing_profile_library_paths=[inference_dir / "routing_profiles.toml"],+ deck_dir_path=str(inference_dir / "deck"),+ )+ return models_manager++ def test_a_plugin_model_and_its_default_are_merged(self, inference_dir: Path) -> None:+ """The plugin's model is served by the internal backend, and its default resolves to it."""+ model_deck = self._setup(inference_dir=inference_dir, plugin_model_declarations=_xlsx_plugin_declarations()).get_model_deck()++ inference_model = model_deck.get_required_inference_model(model_handle="pipelex-xlsx", model_type=ModelType.DOC_GEN)+ assert inference_model.backend_name == PipelexBackend.INTERNAL+ assert inference_model.sdk == "openpyxl"+ assert inference_model.inputs == ["layout", "template_file"]+ assert inference_model.outputs == ["xlsx"]+ choice = model_deck.get_doc_gen_choice_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT)+ assert choice is not None+ assert model_deck.get_doc_gen_setting(doc_gen_choice=choice).model == "pipelex-xlsx"++ def test_a_deck_file_overrides_a_plugin_default(self, inference_dir: Path) -> None:+ """The plugin's defaults sit beneath the deck files: the kit's own and a user's x_custom file both win."""+ (inference_dir / "deck" / "x_custom_doc_gen_deck.toml").write_text(+ '[doc_gen.choice_defaults]\n"xlsx.layout" = "@my-xlsx"\n\n[doc_gen.aliases]\nmy-xlsx = "pipelex-xlsx"\n', encoding="utf-8"+ )++ model_deck = self._setup(inference_dir=inference_dir, plugin_model_declarations=_xlsx_plugin_declarations()).get_model_deck()++ pdf_choice = model_deck.get_doc_gen_choice_default(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.LAYOUT)+ assert pdf_choice is not None+ assert model_deck.get_doc_gen_setting(doc_gen_choice=pdf_choice).model == "reportlab-pdf"+ xlsx_choice = model_deck.get_doc_gen_choice_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT)+ assert isinstance(xlsx_choice, ModelReference)+ assert xlsx_choice.raw == "@my-xlsx"++ def test_a_plugin_model_internal_toml_declares_too_fails_the_boot_naming_the_plugin_and_the_file(self, inference_dir: Path) -> None:+ internal_toml_path = inference_dir / "backends" / "internal.toml"+ internal_toml = internal_toml_path.read_text(encoding="utf-8")+ internal_toml_path.write_text(+ f'{internal_toml}\n[pipelex-xlsx]\nmodel_type = "doc_gen"\nsdk = "openpyxl"\ninputs = ["layout"]\noutputs = ["xlsx"]\ncosts = {{}}\n',+ encoding="utf-8",+ )++ with pytest.raises(PluginModelDeclarationError) as exc_info:+ self._setup(inference_dir=inference_dir, plugin_model_declarations=_xlsx_plugin_declarations())++ assert exc_info.value.plugin == "doc-gen"+ assert "'doc-gen'" in str(exc_info.value)+ assert "'pipelex-xlsx'" in str(exc_info.value)+ assert str(internal_toml_path) in str(exc_info.value)++ def test_a_plugin_table_takes_no_file_defaults(self, inference_dir: Path) -> None:+ """The plugin's table is complete on its own: `internal.toml`'s `[defaults]` reaches the file's models only."""+ internal_toml_path = inference_dir / "backends" / "internal.toml"+ internal_toml = internal_toml_path.read_text(encoding="utf-8")+ internal_toml_path.write_text(+ internal_toml.replace('[defaults]\nthinking_mode = "none"', '[defaults]\nthinking_mode = "none"\nmax_tokens = 99', 1), encoding="utf-8"+ )++ model_deck = self._setup(inference_dir=inference_dir, plugin_model_declarations=_xlsx_plugin_declarations()).get_model_deck()++ assert model_deck.get_required_inference_model(model_handle="reportlab-pdf", model_type=ModelType.DOC_GEN).max_tokens == 99+ assert model_deck.get_required_inference_model(model_handle="pipelex-xlsx", model_type=ModelType.DOC_GEN).max_tokens is None++ def test_an_invalid_plugin_table_fails_the_boot_naming_the_plugin(self, inference_dir: Path) -> None:+ declarations = _declarations(internal_models={"pipelex-xlsx": {**PIPELEX_XLSX_SPEC, "not_a_field": 1}}, doc_gen_defaults={})++ with pytest.raises(PluginModelDeclarationError, match="Plugin 'doc-gen' declares the internal model 'pipelex-xlsx'"):+ self._setup(inference_dir=inference_dir, plugin_model_declarations=declarations)++ def test_a_plugin_default_no_step_uses_fails_the_boot_naming_the_plugin(self, inference_dir: Path) -> None:+ declarations = _declarations(internal_models={}, doc_gen_defaults={(DocGenFormat.PPTX, DocGenSource.LAYOUT): "pipelex-pptx"})++ with pytest.raises(PluginModelDeclarationError, match="Plugin 'doc-gen' declares a default document engine that no step can use"):+ self._setup(inference_dir=inference_dir, plugin_model_declarations=declarations)++ def test_without_an_internal_backend_the_plugin_adds_nothing(self, inference_dir: Path) -> None:+ """A disabled internal backend loads none of its models, the plugin's included, and the plugin's defaults stay out with them."""+ backends_toml_path = inference_dir / "backends.toml"+ backends_toml = backends_toml_path.read_text(encoding="utf-8")+ backends_toml_path.write_text(+ backends_toml.replace(+ "[internal] # software-only backend, runs internally, without AI\nenabled = true", "[internal]\nenabled = false", 1+ ),+ encoding="utf-8",+ )+ routing_profiles_path = inference_dir / "routing_profiles.toml"+ routing_profiles_path.write_text(+ routing_profiles_path.read_text(encoding="utf-8").replace('active = "all_internal"', 'active = "all_openai"', 1), encoding="utf-8"+ )++ model_deck = self._setup(inference_dir=inference_dir, plugin_model_declarations=_xlsx_plugin_declarations()).get_model_deck()++ assert "pipelex-xlsx" not in model_deck.inference_models+ assert model_deck.get_doc_gen_choice_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT) is Nonediff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_markdown_refinement.py b/tests/unit/pipelex/core/concepts/concept_factory/test_markdown_refinement.pynew file mode 100644index 000000000..556ef25b4--- /dev/null+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_markdown_refinement.py@@ -0,0 +1,89 @@+import pytest++from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.libraries.concept.concept_library import ConceptLibrary+++def _make_concept(*, code: str, blueprint: ConceptBlueprint | str) -> Concept:+ return ConceptFactory.make_from_blueprint(domain_code="site_reports", concept_code=code, blueprint_or_string_description=blueprint)+++class TestMarkdownRefinement:+ def test_native_markdown_refines_native_text(self):+ """The native Markdown concept declares its refinement of Text and resolves MarkdownContent."""+ markdown = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.MARKDOWN)+ assert markdown.concept_ref == "native.Markdown"+ assert markdown.refines == NativeConceptCode.TEXT.concept_ref+ assert markdown.structure_class_name == "MarkdownContent"+ assert NativeConceptCode.MARKDOWN.structure_class is MarkdownContent+ assert NativeConceptCode.MARKDOWN.refined_native is NativeConceptCode.TEXT+ assert NativeConceptCode.is_text_concept("Markdown")++ library = ConceptLibrary.make_empty()+ assert library.get_structure_class(concept=markdown) is MarkdownContent++ @pytest.mark.parametrize("native_code", list(NativeConceptCode))+ def test_the_factory_declares_exactly_the_enum_refinements(self, native_code: NativeConceptCode):+ """`refines` on a native concept is the enum's `refined_native`, and nothing else."""+ refined_native = native_code.refined_native+ expected_refines = refined_native.concept_ref if refined_native else None+ assert ConceptFactory.make_native_concept(native_concept_code=native_code).refines == expected_refines++ def test_refining_markdown_resolves_markdown_content(self):+ """`refines = "Markdown"` builds a concept whose structure class inherits from MarkdownContent."""+ concept = _make_concept(code="SiteReport", blueprint=ConceptBlueprint(description="A site report", refines="Markdown"))+ structure_class = ConceptLibrary.make_empty().get_structure_class(concept=concept)+ assert issubclass(structure_class, MarkdownContent)+ assert concept.refines == NativeConceptCode.MARKDOWN.concept_ref++ @pytest.mark.parametrize("strict", [True, False])+ def test_markdown_is_accepted_where_text_is_wanted(self, strict: bool):+ """Markdown refines Text, so a Markdown value stands in for a Text, strictly or loosely."""+ library = ConceptLibrary.make_empty_with_native_concepts()+ markdown = library.get_native_concept(native_concept=NativeConceptCode.MARKDOWN)+ text = library.get_native_concept(native_concept=NativeConceptCode.TEXT)+ assert library.is_compatible(tested_concept=markdown, wanted_concept=text, strict=strict) is True++ @pytest.mark.parametrize("strict", [True, False])+ def test_text_is_refused_where_markdown_is_wanted(self, strict: bool):+ """A plain Text, or any text concept beside Markdown, is not a Markdown, although its class has the same shape."""+ library = ConceptLibrary.make_empty_with_native_concepts()+ markdown = library.get_native_concept(native_concept=NativeConceptCode.MARKDOWN)+ text = library.get_native_concept(native_concept=NativeConceptCode.TEXT)+ description_only = _make_concept(code="SiteNote", blueprint="A note taken on site")+ refining_text = _make_concept(code="SiteSummary", blueprint=ConceptBlueprint(description="A summary", refines="Text"))+ declaring_text_content = _make_concept(code="SiteLine", blueprint=ConceptBlueprint(description="A line", structure="TextContent"))++ for tested in (text, description_only, refining_text, declaring_text_content):+ assert library.is_compatible(tested_concept=tested, wanted_concept=markdown, strict=strict) is False, tested.concept_ref++ @pytest.mark.parametrize("strict", [True, False])+ def test_markdown_lineage_is_accepted_at_any_depth(self, strict: bool):+ """A concept refining Markdown, directly or through another concept, is a Markdown, and so a Text."""+ library = ConceptLibrary.make_empty_with_native_concepts()+ site_report = _make_concept(code="SiteReport", blueprint=ConceptBlueprint(description="A site report", refines="Markdown"))+ library.add_new_concept(concept=site_report)+ roof_report = _make_concept(code="RoofReport", blueprint=ConceptBlueprint(description="A roof report", refines="SiteReport"))+ markdown = library.get_native_concept(native_concept=NativeConceptCode.MARKDOWN)+ text = library.get_native_concept(native_concept=NativeConceptCode.TEXT)++ for tested in (site_report, roof_report):+ assert library.is_compatible(tested_concept=tested, wanted_concept=markdown, strict=strict) is True, tested.concept_ref+ assert library.is_compatible(tested_concept=tested, wanted_concept=text, strict=strict) is True, tested.concept_ref+ assert issubclass(library.get_structure_class(concept=roof_report), MarkdownContent)++ def test_markdown_is_accepted_where_a_text_refinement_is_wanted(self):+ """Wherever a Text is accepted, a Markdown is: the relation between Text and its refinements is unchanged."""+ library = ConceptLibrary.make_empty_with_native_concepts()+ markdown = library.get_native_concept(native_concept=NativeConceptCode.MARKDOWN)+ text = library.get_native_concept(native_concept=NativeConceptCode.TEXT)+ refining_text = _make_concept(code="SiteSummary", blueprint=ConceptBlueprint(description="A summary", refines="Text"))++ assert library.is_compatible(tested_concept=text, wanted_concept=refining_text, strict=True) is True+ assert library.is_compatible(tested_concept=markdown, wanted_concept=refining_text, strict=True) is True+ assert issubclass(library.get_structure_class(concept=refining_text), TextContent)diff --git a/tests/unit/pipelex/core/concepts/test_pinned_natives_vs_standard.py b/tests/unit/pipelex/core/concepts/test_pinned_natives_vs_standard.pyindex 1cf84e693..be0931bca 100644--- a/tests/unit/pipelex/core/concepts/test_pinned_natives_vs_standard.py+++ b/tests/unit/pipelex/core/concepts/test_pinned_natives_vs_standard.py@@ -57,6 +57,10 @@ pytestmark = pytest.mark.skipif(),)+# A native Pipelex defines ahead of the standard has no definition on the page yet, so only the natives+# the standard pins are held to it.+_STANDARD_NATIVES = [native_code for native_code in NativeConceptCode if native_code.is_pinned_by_the_standard]+_DISAGREEMENT_REMEDY = ("if this change did not touch the pinned set, the MTHDS standard moved and this repo is behind it — ""that red is not this PR's fault; bring pinned_blueprints.py and the matching runtime content class "@@ -111,11 +115,11 @@ class TestPinnedNativesAgreeWithTheStandard:def test_pins_the_same_natives_in_the_pages_own_section_order(self):page_codes = [code for code, _ in read_spec_definitions()]- assert page_codes == [code.value for code in NativeConceptCode], (+ assert page_codes == [code.value for code in _STANDARD_NATIVES], (f"the standard's page and this repo's pinned set disagree on which natives exist, or in what order — {_DISAGREEMENT_REMEDY}")- @pytest.mark.parametrize("native_code", list(NativeConceptCode))+ @pytest.mark.parametrize("native_code", _STANDARD_NATIVES)def test_transcribes_the_definition_exactly_as_the_page_states_it(self, native_code: NativeConceptCode):"""Deep equality, definition for definition and field for field, descriptions included.@@ -130,7 +134,7 @@ class TestPinnedNativesAgreeWithTheStandard:f"native.{native_code.value}: the pinned blueprint disagrees with the standard's page — {_DISAGREEMENT_REMEDY}")- @pytest.mark.parametrize("native_code", list(NativeConceptCode))+ @pytest.mark.parametrize("native_code", _STANDARD_NATIVES)def test_keeps_the_structure_fields_in_the_pages_order(self, native_code: NativeConceptCode):"""Field order is normative (it governs the crate's emitted encodings), and dict equality cannot see it."""page_definition: dict[str, Any] = dict(read_spec_definitions()).get(native_code.value) or {}diff --git a/tests/unit/pipelex/core/memory/input_shaper/test_explicit_forms.py b/tests/unit/pipelex/core/memory/input_shaper/test_explicit_forms.pyindex 9776cb9df..f75919cfe 100644--- a/tests/unit/pipelex/core/memory/input_shaper/test_explicit_forms.py+++ b/tests/unit/pipelex/core/memory/input_shaper/test_explicit_forms.py@@ -12,7 +12,6 @@ from pipelex.core.memory.exceptions import ()from pipelex.core.memory.input_shaper import InputShaperfrom pipelex.core.stuffs.list_content import ListContent-from pipelex.core.stuffs.text_content import TextContentfrom pipelex.core.stuffs.time_content import TimeContentfrom pipelex.interpreter_hub import get_concept_libraryfrom tests.unit.pipelex.core.memory.input_shaper.data import OpeningTime, Question, ShaperInvoice, ShaperWeird, build_input_specs@@ -40,9 +39,10 @@ class TestInputShaperExplicitForms:stuff = working_memory.root["answer"]pretty_print(stuff, title="envelope refining wins")- # The declared lower bound is native.Text, but the caller volunteered the more specific Question.+ # The declared lower bound is native.Text, but the caller volunteered the more specific Question,+ # whose own class holds the text, as the Time envelope below keeps its refining class.assert stuff.concept.concept_ref == "shaper_test.Question"- assert stuff.content == TextContent(text="What are the fees?")+ assert stuff.content == Question(text="What are the fees?")@pytest.mark.parametrize(("concept_ref", "expected_type"),diff --git a/tests/unit/pipelex/core/memory/input_shaper/test_markdown_inputs.py b/tests/unit/pipelex/core/memory/input_shaper/test_markdown_inputs.pynew file mode 100644index 000000000..234388c36--- /dev/null+++ b/tests/unit/pipelex/core/memory/input_shaper/test_markdown_inputs.py@@ -0,0 +1,86 @@+from typing import Any++import pytest++from pipelex.core.memory.exceptions import ExplicitConceptIncompatibleError+from pipelex.core.memory.input_shaper import InputKind, InputShaper+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.core.stuffs.text_content import TextContent+from pipelex.interpreter_hub import get_concept_library+from tests.unit.pipelex.core.memory.input_shaper.data import build_input_specs++_REPORT = "# Roof\n\n- two cracked tiles"+++class TestMarkdownInputs:+ def test_markdown_takes_the_text_arm(self) -> None:+ """A declared Markdown input reads a string, as a Text input does."""+ library = get_concept_library()+ markdown = library.get_required_concept(concept_ref="native.Markdown")+ assert InputShaper.resolve_input_kind(markdown, concept_provider=library) == InputKind.TEXT++ @pytest.mark.parametrize(+ "provided",+ [+ _REPORT,+ {"concept": "Markdown", "content": _REPORT},+ {"concept": "native.Markdown", "content": _REPORT},+ {"concept": "Markdown", "content": {"text": _REPORT}},+ ],+ )+ def test_every_form_builds_markdown_content(self, provided: Any) -> None:+ """The bare string, the envelope with a string, and the envelope with the content form all hold a MarkdownContent."""+ input_specs = build_input_specs([("report", "native.Markdown", None)])++ working_memory = InputShaper.shape({"report": provided}, input_specs=input_specs, concept_provider=get_concept_library(), read_scope=None)++ stuff = working_memory.root["report"]+ assert stuff.concept.concept_ref == "native.Markdown"+ assert type(stuff.content) is MarkdownContent+ assert stuff.content == MarkdownContent(text=_REPORT)++ def test_markdown_envelope_fills_a_text_input(self) -> None:+ """A Markdown envelope is accepted where a Text is declared, and keeps being a Markdown."""+ input_specs = build_input_specs([("report", "native.Text", None)])+ provided = {"concept": "Markdown", "content": _REPORT}++ working_memory = InputShaper.shape({"report": provided}, input_specs=input_specs, concept_provider=get_concept_library(), read_scope=None)++ stuff = working_memory.root["report"]+ assert stuff.concept.concept_ref == "native.Markdown"+ assert stuff.content == MarkdownContent(text=_REPORT)++ @pytest.mark.parametrize("envelope_concept", ["Text", "shaper_test.Question"])+ def test_text_envelope_is_refused_for_a_markdown_input(self, envelope_concept: str) -> None:+ """A Text, or a concept refining Text, is not a Markdown: the envelope is refused rather than formatted."""+ input_specs = build_input_specs([("report", "native.Markdown", None)])+ provided = {"concept": envelope_concept, "content": _REPORT}++ with pytest.raises(ExplicitConceptIncompatibleError):+ InputShaper.shape({"report": provided}, input_specs=input_specs, concept_provider=get_concept_library(), read_scope=None)++ def test_factory_envelope_with_a_list_of_strings(self) -> None:+ """The envelope form with a list of strings holds one MarkdownContent per item."""+ stuff = StuffFactory.make_stuff_from_stuff_content_or_data(+ stuff_content_or_data={"concept": "Markdown", "content": ["# One", "# Two"]},+ concept_provider=get_concept_library(),+ name="reports",+ read_scope=None,+ )+ assert stuff.concept.concept_ref == "native.Markdown"+ assert stuff.content == ListContent(items=[MarkdownContent(text="# One"), MarkdownContent(text="# Two")])++ def test_factory_infers_markdown_from_the_content_class(self) -> None:+ """A MarkdownContent handed over with no concept is a Markdown stuff, and a bare string a Text one."""+ library = get_concept_library()+ markdown_stuff = StuffFactory.make_stuff_from_stuff_content_or_data(+ stuff_content_or_data=MarkdownContent(text=_REPORT), concept_provider=library, name="report", read_scope=None+ )+ text_stuff = StuffFactory.make_stuff_from_stuff_content_or_data(+ stuff_content_or_data=_REPORT, concept_provider=library, name="note", read_scope=None+ )+ assert markdown_stuff.concept.concept_ref == "native.Markdown"+ assert text_stuff.concept.concept_ref == "native.Text"+ assert type(text_stuff.content) is TextContentdiff --git a/tests/unit/pipelex/core/stuffs/markdown_content/test_data.py b/tests/unit/pipelex/core/stuffs/markdown_content/test_data.pynew file mode 100644index 000000000..19996ce67--- /dev/null+++ b/tests/unit/pipelex/core/stuffs/markdown_content/test_data.py@@ -0,0 +1,30 @@+from typing import Any, ClassVar+++class TestData:+ # A short report, as an LLM writes one+ SAMPLE_REPORT = "# Inspection report\n\nThe **roof** needs work.\n\n- cracked tiles\n- sagging gutter"+ EXPECTED_REPORT_HTML = (+ "<h1>Inspection report</h1>\n<p>The <strong>roof</strong> needs work.</p>\n<ul>\n<li>cracked tiles</li>\n<li>sagging gutter</li>\n</ul>\n"+ )+ EXPECTED_SMART_DUMP: ClassVar[dict[str, Any]] = {"text": SAMPLE_REPORT}+ EXPECTED_SHORT_DESC = f"some markdown ({len(SAMPLE_REPORT)} chars)"+ EXPECTED_FIELD_DESCRIPTION = "The text, written in Markdown"++ # (source, expected HTML): what the conversion links, leaves as text and escapes+ HTML_CONVERSION_CASES: ClassVar[list[tuple[str, str]]] = [+ # A URL with a scheme is linked+ (+ "See https://example.com/report for details.",+ '<p>See <a href="https://example.com/report">https://example.com/report</a> for details.</p>\n',+ ),+ # A file name whose suffix is a country domain stays text: fuzzy linking is off+ ("Read README.md first.", "<p>Read README.md first.</p>\n"),+ # Raw HTML in the source is escaped, never inserted+ ("<script>alert(1)</script>", "<p><script>alert(1)</script></p>\n"),+ # A javascript: link target is refused, so the link stays text+ ("[click](javascript:alert(1))", "<p>[click](javascript:alert(1))</p>\n"),+ ]++ # Source that looks like HTML: a Text would pretty-print it as HTML, a Markdown never does+ SAMPLE_HTML_LOOKING_MARKDOWN = "<b>not html</b>\n\n# A heading"diff --git a/tests/unit/pipelex/core/stuffs/markdown_content/test_markdown_content_renders.py b/tests/unit/pipelex/core/stuffs/markdown_content/test_markdown_content_renders.pynew file mode 100644index 000000000..5b413b4b5--- /dev/null+++ b/tests/unit/pipelex/core/stuffs/markdown_content/test_markdown_content_renders.py@@ -0,0 +1,58 @@+import json++import pytest+from markupsafe import escape+from rich.markdown import Markdown++from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.tools.templating.text_format import TextFormat+from tests.unit.pipelex.core.stuffs.markdown_content.test_data import TestData+++class TestMarkdownContentRenders:+ def test_is_a_text_content_holding_the_source(self):+ """A Markdown value is a text: TextContent's one field, holding the Markdown source."""+ content = MarkdownContent(text=TestData.SAMPLE_REPORT)+ assert isinstance(content, TextContent)+ assert content.text == TestData.SAMPLE_REPORT+ assert content.smart_dump() == TestData.EXPECTED_SMART_DUMP+ assert content.short_desc == TestData.EXPECTED_SHORT_DESC+ assert MarkdownContent.model_fields["text"].description == TestData.EXPECTED_FIELD_DESCRIPTION++ def test_source_renderings_are_the_markdown_as_it_is(self):+ """Plain, Markdown (the saved `main_stuff.md`) and prompt renderings are the source, verbatim."""+ content = MarkdownContent(text=TestData.SAMPLE_REPORT)+ assert content.rendered_plain() == TestData.SAMPLE_REPORT+ assert content.rendered_markdown() == TestData.SAMPLE_REPORT+ assert content.rendered_for_prompt() == TestData.SAMPLE_REPORT+ assert content.rendered_for_prompt(text_format=TextFormat.MARKDOWN) == TestData.SAMPLE_REPORT+ assert str(content) == TestData.SAMPLE_REPORT+ assert json.loads(content.rendered_json()) == {"text": TestData.SAMPLE_REPORT}++ @pytest.mark.asyncio+ async def test_saved_markdown_is_the_source(self):+ """`pipelex run --save-main-stuff` writes `main_stuff.md` from the async Markdown rendering."""+ content = MarkdownContent(text=TestData.SAMPLE_REPORT)+ assert await content.rendered_markdown_async() == TestData.SAMPLE_REPORT++ def test_html_rendering_converts(self):+ """The HTML view converts the Markdown rather than escaping it, and markupsafe inserts that conversion."""+ content = MarkdownContent(text=TestData.SAMPLE_REPORT)+ assert content.rendered_html() == TestData.EXPECTED_REPORT_HTML+ assert content.rendered_for_prompt(text_format=TextFormat.HTML) == TestData.EXPECTED_REPORT_HTML+ # markupsafe reads `__html__`: escaping the content yields the conversion, not the escaped source.+ assert str(escape(content)) == TestData.EXPECTED_REPORT_HTML+ # A Text holding the same source keeps being escaped.+ assert TextContent(text="<b>x</b>").rendered_html() == "<b>x</b>"++ @pytest.mark.parametrize(("source", "expected_html"), TestData.HTML_CONVERSION_CASES)+ def test_html_conversion_links_schemes_and_escapes_raw_html(self, source: str, expected_html: str):+ """URLs with a scheme are linked, file names stay text, raw HTML and javascript: targets are neutralized."""+ assert MarkdownContent(text=source).rendered_html() == expected_html++ def test_pretty_rendering_is_always_markdown(self):+ """The pretty view renders the source as Markdown, even when it starts like HTML."""+ pretty = MarkdownContent(text=TestData.SAMPLE_HTML_LOOKING_MARKDOWN).rendered_pretty()+ assert isinstance(pretty, Markdown)+ assert pretty.markup == TestData.SAMPLE_HTML_LOOKING_MARKDOWNdiff --git a/tests/unit/pipelex/core/stuffs/test_stuff_artefact_markdown_html.py b/tests/unit/pipelex/core/stuffs/test_stuff_artefact_markdown_html.pynew file mode 100644index 000000000..2272c12f9--- /dev/null+++ b/tests/unit/pipelex/core/stuffs/test_stuff_artefact_markdown_html.py@@ -0,0 +1,164 @@+import pytest+from markupsafe import escape++from pipelex.cogt.templating.template_preprocessor import preprocess_template+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.markdown_content import MarkdownContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff import Stuff+from pipelex.core.stuffs.stuff_artefact import StuffArtefact+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.kernel.compose_ops import run_compose_template+from pipelex.tools.jinja2.exceptions import Jinja2TemplateSecurityError+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_async+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.templating.templating_style import TagStyle, TemplatingStyle+from pipelex.tools.templating.text_format import TextFormat++_REPORT_SOURCE = "# Title\n\nFish & chips"+_REPORT_HTML = "<h1>Title</h1>\n<p>Fish & chips</p>\n"+_REPORT_SOURCE_ESCAPED = "# Title\n\nFish & chips"+_NOTE_SOURCE = "<b>Fish</b> & chips"+_NOTE_ESCAPED = "<b>Fish</b> & chips"+_TEMPLATING_STYLE = TemplatingStyle(tag_style=TagStyle.XML, text_format=TextFormat.PLAIN)+++class InspectionDigest(StructuredContent):+ title: str+ summary: MarkdownContent+++def _make_stuff(*, content: StuffContent, name: str, concept_code: NativeConceptCode) -> Stuff:+ return Stuff(+ stuff_code=f"{name}_code",+ stuff_name=name,+ concept=ConceptFactory.make_native_concept(native_concept_code=concept_code),+ content=content,+ )+++def _make_report_artefact() -> StuffArtefact:+ return StuffArtefact(_make_stuff(content=MarkdownContent(text=_REPORT_SOURCE), name="report", concept_code=NativeConceptCode.MARKDOWN))+++def _make_note_artefact() -> StuffArtefact:+ return StuffArtefact(_make_stuff(content=TextContent(text=_NOTE_SOURCE), name="note", concept_code=NativeConceptCode.TEXT))++[email protected](loop_scope="class")+class TestStuffArtefactMarkdownHtml:+ async def test_compose_html_template_converts_markdown_and_escapes_text(self):+ """In a PipeCompose HTML template, `{{ report }}` is the converted Markdown, `{{ note }}` the escaped Text."""+ memory = WorkingMemoryFactory.make_from_multiple_stuffs(+ stuff_list=[+ _make_stuff(content=MarkdownContent(text=_REPORT_SOURCE), name="report", concept_code=NativeConceptCode.MARKDOWN),+ _make_stuff(content=TextContent(text=_NOTE_SOURCE), name="note", concept_code=NativeConceptCode.TEXT),+ ]+ )+ compose_result = await run_compose_template(+ memory=memory,+ template="<section>{{ report }}</section><aside>{{ note }}</aside>",+ category=TemplateCategory.HTML,+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.HTML),+ output_class=HtmlContent,+ templating_style=_TEMPLATING_STYLE,+ result_name="page",+ )+ assert compose_result.rendered_text == f"<section>{_REPORT_HTML}</section><aside>{_NOTE_ESCAPED}</aside>"++ @pytest.mark.parametrize(+ ("template_source", "expected"),+ [+ # The field is a plain string: escaped like any string.+ ("{{ report.text }}", _REPORT_SOURCE_ESCAPED),+ # markupsafe's escape and Markup both honour `__html__`.+ ("{{ report|e }}", _REPORT_HTML),+ ("{{ report|safe }}", _REPORT_HTML),+ # A Text stuff marked safe is inserted raw, as it always was: no `__html__` answers for it.+ ("{{ note|safe }}", _NOTE_SOURCE),+ ],+ )+ async def test_html_template_forms(self, template_source: str, expected: str):+ """What each spelling prints in an HTML template, for a Markdown and a Text stuff."""+ rendered = await render_jinja2_async(+ template_source=template_source,+ template_category=TemplateCategory.HTML,+ templating_context={"report": _make_report_artefact(), "note": _make_note_artefact()},+ )+ assert rendered == expected++ @pytest.mark.parametrize(+ ("template_source", "expected"),+ [+ ("$report", _REPORT_HTML),+ ("{{ report|format }}", _REPORT_HTML),+ ("{{ report|format('html') }}", _REPORT_HTML),+ # A format the template names wins: plain is the source, which the template escapes.+ ("{{ report|format('plain') }}", _REPORT_SOURCE_ESCAPED),+ ("$note", _NOTE_ESCAPED),+ ],+ ids=["sigil", "format_filter", "format_filter_naming_html", "format_filter_naming_plain", "text_sigil"],+ )+ async def test_the_format_filter_prints_markdown_as_html(self, template_source: str, expected: str):+ """In an HTML template, `$report` and `{{ report|format }}` print what `{{ report }}` prints, and a Text stays escaped."""+ rendered = await render_jinja2_async(+ template_source=preprocess_template(template_source),+ template_category=TemplateCategory.HTML,+ templating_context={"report": _make_report_artefact(), "note": _make_note_artefact()},+ templating_style=_TEMPLATING_STYLE,+ )+ assert rendered == expected++ async def test_markdown_field_of_a_structure_converts(self):+ """A Markdown value one field down, `{{ digest.summary }}`, converts too: the content itself answers `__html__`."""+ digest = InspectionDigest(title="Roof & gutters", summary=MarkdownContent(text=_REPORT_SOURCE))+ artefact = StuffArtefact(+ Stuff(+ stuff_code="digest_code",+ stuff_name="digest",+ concept=ConceptFactory.make(+ concept_code="InspectionDigest",+ domain_code="inspections",+ description="A digest",+ structure_class_name="InspectionDigest",+ ),+ content=digest,+ )+ )+ rendered = await render_jinja2_async(+ template_source="<h2>{{ digest.title }}</h2>{{ digest.summary }}",+ template_category=TemplateCategory.HTML,+ templating_context={"digest": artefact},+ )+ assert rendered == f"<h2>Roof & gutters</h2>{_REPORT_HTML}"++ @pytest.mark.parametrize("template_category", [TemplateCategory.MARKDOWN, TemplateCategory.LLM_PROMPT, TemplateCategory.BASIC])+ async def test_without_autoescape_markdown_prints_its_source(self, template_category: TemplateCategory):+ """Where nothing is escaped (a prompt, a Markdown template), a Markdown stuff prints its source as it is."""+ rendered = await render_jinja2_async(+ template_source="{{ report }}",+ template_category=template_category,+ templating_context={"report": _make_report_artefact()},+ )+ assert rendered == _REPORT_SOURCE++ async def test_template_cannot_call_html_dunder(self):+ """The sandbox refuses `__html__` from a template: only markupsafe calls it, when it escapes."""+ with pytest.raises(Jinja2TemplateSecurityError):+ await render_jinja2_async(+ template_source="{{ report.__html__() }}",+ template_category=TemplateCategory.HTML,+ templating_context={"report": _make_report_artefact()},+ )++ async def test_only_a_content_that_knows_its_html_answers_html_dunder(self):+ """The artefact answers `__html__` for a Markdown content only, so every other stuff escapes as before."""+ report_artefact = _make_report_artefact()+ assert hasattr(report_artefact, "__html__")+ assert str(escape(report_artefact)) == _REPORT_HTML+ assert str(escape(_make_note_artefact())) == _NOTE_ESCAPED+ assert not hasattr(_make_note_artefact(), "__html__")diff --git a/tests/unit/pipelex/kit/test_deck_variants.py b/tests/unit/pipelex/kit/test_deck_variants.pyindex 41b9c8126..b6a84d99c 100644--- a/tests/unit/pipelex/kit/test_deck_variants.py+++ b/tests/unit/pipelex/kit/test_deck_variants.py@@ -18,8 +18,13 @@ from pathlib import Pathimport pytest-from pipelex.cogt.models.deck_manifest import kit_deck_dir, list_managed_kit_files+from pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoice, DocGenSetting+from pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSetting+from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSetting+from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting+from pipelex.cogt.models.deck_manifest import KitManagedArea, kit_deck_dir, list_managed_kit_filesfrom pipelex.cogt.models.model_deck import (+ DocGenDeckBlueprint,ExtractDeckBlueprint,ImgGenDeckBlueprint,LLMDeckBlueprint,@@ -28,11 +33,12 @@ from pipelex.cogt.models.model_deck import ()from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprintfrom pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind+from pipelex.cogt.search.search_setting import SearchModelChoice, SearchSettingfrom pipelex.kit.paths import get_kit_configs_dir, get_kit_deck_variants_dirfrom pipelex.tools.misc.toml_utils import load_toml_from_path# A vocabulary coordinate: the model family, then the kind of name within it.-DeckFamilyBlueprint = LLMDeckBlueprint | ExtractDeckBlueprint | ImgGenDeckBlueprint | SearchDeckBlueprint+DeckFamilyBlueprint = LLMDeckBlueprint | ExtractDeckBlueprint | ImgGenDeckBlueprint | SearchDeckBlueprint | DocGenDeckBlueprintVocabularyKey = tuple[str, str]Vocabulary = dict[VocabularyKey, set[str]]PermittedDrops = Mapping[VocabularyKey, frozenset[str]]@@ -61,6 +67,7 @@ def extract_vocabulary(blueprint: ModelDeckBlueprint) -> Vocabulary:"extract": blueprint.extract,"img_gen": blueprint.img_gen,"search": blueprint.search,+ "doc_gen": blueprint.doc_gen,}vocabulary: Vocabulary = {}for family, family_blueprint in family_blueprints.items():@@ -106,20 +113,44 @@ def list_declared_backend_handles() -> set[str]:return handles+def list_choice_default_references(blueprint: ModelDeckBlueprint) -> list[str]:+ """The reference each family's default choice names: a setting's model, or the reference the deck wrote."""+ choices: list[LLMModelChoice | ExtractModelChoice | ImgGenModelChoice | SearchModelChoice | DocGenModelChoice] = [+ blueprint.llm.choice_defaults.for_text,+ blueprint.llm.choice_defaults.for_object,+ blueprint.extract.choice_default,+ blueprint.img_gen.choice_default,+ blueprint.search.choice_default,+ *blueprint.doc_gen.choice_defaults.values(),+ ]+ references: list[str] = []+ for choice in choices:+ match choice:+ case LLMSetting() | ExtractSetting() | ImgGenSetting() | SearchSetting() | DocGenSetting():+ references.append(choice.model)+ case ModelReference():+ references.append(choice.raw)+ case str():+ references.append(choice)+ return references++def extract_model_handles(blueprint: ModelDeckBlueprint) -> set[str]:- """Every concrete handle a deck names, from its alias targets, its waterfall entries and its presets' models.+ """Every concrete handle a deck names, from its alias targets, its waterfall entries, its presets' models and its default choices.A reference naming an alias, a preset or a waterfall carries no handle of its own: it resolves- through one of the three collections this function reads directly. A waterfall's own entries do- carry handles, which is why they are read here and not only through whatever names the waterfall.+ through one of the collections this function reads directly. A waterfall's own entries do+ carry handles, which is why they are read here and not only through whatever names the waterfall,+ and so does a default choice that names a model directly."""- family_blueprints: list[DeckFamilyBlueprint] = [blueprint.llm, blueprint.extract, blueprint.img_gen, blueprint.search]+ family_blueprints: list[DeckFamilyBlueprint] = [blueprint.llm, blueprint.extract, blueprint.img_gen, blueprint.search, blueprint.doc_gen]references: list[str] = []for family_blueprint in family_blueprints:references.extend(family_blueprint.aliases.values())for waterfall_entries in family_blueprint.waterfalls.values():references.extend(waterfall_entries)references.extend(setting.model for setting in family_blueprint.presets.values())+ references.extend(list_choice_default_references(blueprint))handles: set[str] = set()for reference in references:parsed = ModelReference.parse(reference)@@ -144,13 +175,13 @@ class TestDeckVariants:@pytest.mark.parametrize("variant_dir", list_variant_dirs(), ids=lambda path: path.name)def test_variant_ships_the_same_deck_files(self, variant_dir: Path):variant_filenames = {entry.name for entry in variant_dir.iterdir() if entry.is_file() and entry.suffix == ".toml"}- assert variant_filenames == set(list_managed_kit_files()), (+ assert variant_filenames == set(list_managed_kit_files(area=KitManagedArea.DECK)), (f"Variant '{variant_dir.name}' does not hold the same numbered deck files as the kit's shipped deck")@pytest.mark.parametrize("variant_dir", list_variant_dirs(), ids=lambda path: path.name)def test_variant_matches_the_shipped_vocabulary(self, variant_dir: Path):- managed_filenames = list(list_managed_kit_files())+ managed_filenames = list(list_managed_kit_files(area=KitManagedArea.DECK))shipped_blueprint = load_deck_from_dir(kit_deck_dir(), filenames=managed_filenames)variant_blueprint = load_deck_from_dir(variant_dir, filenames=managed_filenames)@@ -164,7 +195,7 @@ class TestDeckVariants:@pytest.mark.parametrize("variant_dir", list_variant_dirs(), ids=lambda path: path.name)def test_variant_only_handles_are_still_declared_by_a_backend(self, variant_dir: Path):"""Name parity says nothing about a handle that was retired from the backends underneath it."""- managed_filenames = list(list_managed_kit_files())+ managed_filenames = list(list_managed_kit_files(area=KitManagedArea.DECK))shipped_handles = extract_model_handles(load_deck_from_dir(kit_deck_dir(), filenames=managed_filenames))variant_handles = extract_model_handles(load_deck_from_dir(variant_dir, filenames=managed_filenames))@@ -181,13 +212,22 @@ class TestDeckVariants:No deck declares a waterfall today, so nothing else in this module would notice thecollector skipping them, and the guard would go quietly blind the moment one does."""- blueprint = load_deck_from_dir(kit_deck_dir(), filenames=list(list_managed_kit_files()))+ blueprint = load_deck_from_dir(kit_deck_dir(), filenames=list(list_managed_kit_files(area=KitManagedArea.DECK)))probe_handle = "handle-named-only-by-a-waterfall"assert probe_handle not in extract_model_handles(blueprint)blueprint.llm.waterfalls["waterfall-parity-probe"] = [probe_handle]assert probe_handle in extract_model_handles(blueprint), "A handle named inside a waterfall escaped the handle collection"+ def test_handle_collection_reads_default_choices(self):+ """A handle a deck names only as a family's default choice must still reach the retirement check above."""+ blueprint = load_deck_from_dir(kit_deck_dir(), filenames=list(list_managed_kit_files(area=KitManagedArea.DECK)))+ probe_handle = "handle-named-only-by-a-default"+ assert probe_handle not in extract_model_handles(blueprint)++ blueprint.doc_gen.choice_defaults["xlsx.layout"] = ModelReference.parse(probe_handle)+ assert probe_handle in extract_model_handles(blueprint), "A handle named as a default choice escaped the handle collection"+def test_comparator_reports_a_preset_the_variant_is_missing(self):differences = compare_vocabularies(shipped=make_vocabulary(aliases={"default-general"}, presets={"writing-factual", "vision"}),diff --git a/tests/unit/pipelex/language/test_mthds_schema.py b/tests/unit/pipelex/language/test_mthds_schema.pyindex 9517dfcbd..4f40d8648 100644--- a/tests/unit/pipelex/language/test_mthds_schema.py+++ b/tests/unit/pipelex/language/test_mthds_schema.py@@ -25,6 +25,7 @@ _PIPE_KIND_EXTRA_FIELDS: dict[str, dict[str, Any]] = {"PipeExtract": {},"PipeSearch": {"prompt": "find it"},"PipeStructure": {},+ "PipeDocGen": {"format": "pdf"},"PipeBatch": {"branch_pipe_code": "sub_pipe", "input_list_name": "items", "input_item_name": "item"},"PipeCondition": {"default_outcome": "fallback_pipe", "outcomes": {"yes": "yes_pipe"}},"PipeParallel": {"branches": [{"pipe": "sub_pipe"}]},diff --git a/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.py b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.pynew file mode 100644index 000000000..a9578113a--- /dev/null+++ b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.py@@ -0,0 +1,48 @@+from typing import Any++import pytest+from pydantic import ValidationError++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat+from pipelex.pipe_operators.doc_gen.pipe_doc_gen_blueprint import PipeDocGenBlueprint+++def _blueprint(**overrides: Any) -> PipeDocGenBlueprint:+ kwargs: dict[str, Any] = {"description": "Print the invoice", "inputs": {"invoice": "Invoice"}, "output": "Document", "format": "pdf"}+ kwargs.update(overrides)+ return PipeDocGenBlueprint.model_validate(kwargs)+++class TestPipeDocGenBlueprint:+ def test_a_pdf_without_a_template_is_the_auto_layout(self) -> None:+ blueprint = _blueprint(filename="invoice-{{ invoice.number }}")+ assert blueprint.format == DocGenFormat.PDF+ assert blueprint.template is None+ assert blueprint.template_file is None++ @pytest.mark.parametrize(+ ("topic", "overrides", "expected_message"),+ [+ ("both_templates", {"template": "<p>{{ invoice.number }}</p>", "template_file": "invoice.html"}, "both 'template' and 'template_file'"),+ ("inline_template_for_xlsx", {"format": "xlsx", "template": "<p>{{ invoice.number }}</p>"}, "for the 'pdf' format only"),+ ("wrong_template_suffix", {"format": "docx", "template_file": "invoice.xlsx"}, "must be a .docx file"),+ ("pptx_without_template", {"format": "pptx"}, "has no auto-layout"),+ ("unknown_format", {"format": "odt"}, "format"),+ ("template_reads_an_undeclared_input", {"template": "<p>{{ customer.name }}</p>"}, "Variable 'customer'"),+ ("filename_reads_an_undeclared_input", {"filename": "invoice-{{ order.number }}"}, "Variable 'order'"),+ ("list_output", {"output": "Document[]"}, "produces one file"),+ ("broken_template", {"template": "{% for %}"}, "Could not parse"),+ ],+ )+ def test_refused_blueprints(+ self,+ topic: str, # ruff: ignore[unused-method-argument]+ overrides: dict[str, Any],+ expected_message: str,+ ) -> None:+ with pytest.raises(ValidationError, match=expected_message):+ _blueprint(**overrides)++ def test_a_sigil_in_the_template_reads_its_input(self) -> None:+ blueprint = _blueprint(template="<h1>$invoice.number</h1>")+ assert blueprint.template == "<h1>$invoice.number</h1>"diff --git a/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_safe_filename.py b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_safe_filename.pynew file mode 100644index 000000000..9780e53b1--- /dev/null+++ b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_safe_filename.py@@ -0,0 +1,19 @@+import pytest++from pipelex.pipe_operators.doc_gen.pipe_doc_gen import safe_filename+++class TestSafeFilename:+ @pytest.mark.parametrize(+ ("rendered", "expected"),+ [+ ("invoice-INV-2026-0142", "invoice-INV-2026-0142.pdf"),+ ("invoice INV/2026:0142", "invoice-INV-2026-0142.pdf"),+ ("report.pdf", "report.pdf"),+ (" ../../etc/passwd ", "etc-passwd.pdf"),+ ("", "render_invoice.pdf"),+ ("...", "render_invoice.pdf"),+ ],+ )+ def test_the_rendered_name_becomes_one_safe_segment(self, rendered: str, expected: str) -> None:+ assert safe_filename(rendered=rendered, fallback="render_invoice", suffix="pdf") == expecteddiff --git a/tests/unit/pipelex/pipeline/test_markdown_input_form_and_hints.py b/tests/unit/pipelex/pipeline/test_markdown_input_form_and_hints.pynew file mode 100644index 000000000..86ec32703--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_markdown_input_form_and_hints.py@@ -0,0 +1,43 @@+import pytest++from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.domains.domain_blueprint import DomainBlueprint+from pipelex.libraries.crate_qualification import qualify_crate+from pipelex.libraries.library_crate import LibraryCrate+from pipelex.pipeline.hint_warnings import build_hint_warnings+from pipelex.pipeline.input_form import FieldKind, InputFormDeriver++_CONCEPTS: dict[str, ConceptBlueprint | str] = {+ # A report refining native Markdown, which refines Text: text-valued, so `label` applies.+ "docs.Report": ConceptBlueprint(description="a report", refines="native.Markdown", hints={"intent": "label"}),+ # Declaring the Markdown content class outright maps by identity to the native Markdown row.+ "docs.Minutes": ConceptBlueprint(description="meeting minutes", structure="MarkdownContent", hints={"intent": "prose"}),+}+++class TestMarkdownInputFormAndHints:+ def test_native_markdown_is_prose(self):+ """A Markdown input is typed as its source, like a Text one: a `prose` node."""+ node = InputFormDeriver(concepts={}).derive_concept(name="report", concept_ref="native.Markdown")+ assert node.kind is FieldKind.PROSE+ assert node.concept_ref == "native.Markdown"++ @pytest.mark.parametrize(+ ("concept_ref", "expected_kind"),+ [+ ("docs.Report", FieldKind.TEXT),+ ("docs.Minutes", FieldKind.PROSE),+ ],+ )+ def test_markdown_chains_are_text_valued_sites(self, concept_ref: str, expected_kind: FieldKind):+ """An intent word applies to a Markdown-backed concept, in the descriptor and in the lint alike."""+ node = InputFormDeriver(concepts=dict(_CONCEPTS)).derive_concept(name="field", concept_ref=concept_ref)+ assert node.kind is expected_kind++ crate = LibraryCrate(+ concepts=dict(_CONCEPTS),+ pipes={},+ domains={"docs": DomainBlueprint(code="docs", description="Docs domain")},+ source_map={},+ )+ assert build_hint_warnings(qualify_crate(crate)) == []diff --git a/tests/unit/pipelex/plugins/test_import_light_boot.py b/tests/unit/pipelex/plugins/test_import_light_boot.pyindex 03268bd49..f955fec4a 100644--- a/tests/unit/pipelex/plugins/test_import_light_boot.py+++ b/tests/unit/pipelex/plugins/test_import_light_boot.py@@ -37,6 +37,9 @@ _GUARD_SCRIPT = textwrap.dedent(# its factory, so registering the sinks must not load them."opentelemetry.sdk","opentelemetry.exporter",+ # Nor ReportLab: the built-in ``reportlab`` document engine defers it into its factory, so a+ # process that never prints a document never imports it.+ "reportlab",)class _Blocker(importlib.abc.MetaPathFinder):diff --git a/tests/unit/pipelex/plugins/test_plugin_model_declarations.py b/tests/unit/pipelex/plugins/test_plugin_model_declarations.pynew file mode 100644index 000000000..1cd8a2148--- /dev/null+++ b/tests/unit/pipelex/plugins/test_plugin_model_declarations.py@@ -0,0 +1,139 @@+"""The model declarations seam: a plugin declares internal models and model deck defaults, and the registrar freezes them.++Pins the registrar half independent of the model manager: each entry stores its data and records a contribution for+`pipelex plugins list`, a second declaration of the same name or the same format and source fails loud naming both+plugins, nothing is validated at registration, and `make_model_declarations` hands the model manager copies it can+merge, with the plugin that declared each one.+"""++from __future__ import annotations++from types import SimpleNamespace+from typing import TYPE_CHECKING, Any, cast++import pytest++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.exceptions import DuplicateDocGenDefaultError, DuplicateInternalModelError+from pipelex.plugins.plugin_group import PluginGroup+from pipelex.plugins.plugin_model_declarations import PluginModelDeclarations+from pipelex.plugins.registrar import PluginOrigin, PluginRegistrar++if TYPE_CHECKING:+ from pipelex.system.configuration.configs import PipelexConfig++PIPELEX_XLSX_SPEC: dict[str, Any] = {+ "model_type": "doc_gen",+ "sdk": "openpyxl",+ "model_id": "write-xlsx",+ "inputs": ["layout", "template_file"],+ "outputs": ["xlsx"],+ "costs": {},+}+++def _make_registrar() -> PluginRegistrar:+ config = cast("PipelexConfig", SimpleNamespace(runtime=SimpleNamespace(plugins=SimpleNamespace(disabled=[]))))+ return PluginRegistrar(config=config)+++class TestPluginModelDeclarations:+ def test_an_internal_model_is_stored_and_recorded_as_a_contribution(self) -> None:+ registrar = _make_registrar()+ discovery = registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)++ registrar.add_internal_model(name="pipelex-xlsx", spec=PIPELEX_XLSX_SPEC)++ assert registrar.internal_models == {"pipelex-xlsx": PIPELEX_XLSX_SPEC}+ assert "internal model pipelex-xlsx" in discovery.contributions++ def test_a_doc_gen_default_is_stored_and_recorded_as_a_contribution(self) -> None:+ registrar = _make_registrar()+ discovery = registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)++ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT, model="pipelex-xlsx")++ assert registrar.doc_gen_defaults == {(DocGenFormat.XLSX, DocGenSource.LAYOUT): "pipelex-xlsx"}+ assert "doc_gen default xlsx.layout = pipelex-xlsx" in discovery.contributions++ def test_an_internal_model_declared_twice_names_both_plugins(self) -> None:+ registrar = _make_registrar()+ registrar.begin_plugin(name="alpha", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)+ registrar.add_internal_model(name="pipelex-xlsx", spec=PIPELEX_XLSX_SPEC)+ registrar.begin_plugin(name="beta", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)++ with pytest.raises(DuplicateInternalModelError) as exc_info:+ registrar.add_internal_model(name="pipelex-xlsx", spec=PIPELEX_XLSX_SPEC)++ assert exc_info.value.first_plugin == "alpha"+ assert exc_info.value.second_plugin == "beta"+ assert "'alpha'" in str(exc_info.value)+ assert "'beta'" in str(exc_info.value)++ def test_a_doc_gen_default_declared_twice_names_both_plugins(self) -> None:+ registrar = _make_registrar()+ registrar.begin_plugin(name="alpha", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.HTML, model="pipelex-pdf")+ registrar.begin_plugin(name="beta", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)++ with pytest.raises(DuplicateDocGenDefaultError) as exc_info:+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.PDF, source=DocGenSource.HTML, model="other-pdf")++ assert exc_info.value.choice_key == "pdf.html"+ assert exc_info.value.first_plugin == "alpha"+ assert exc_info.value.second_plugin == "beta"++ def test_a_kernel_group_plugin_may_declare_both(self) -> None:+ """Plain data at the kernel tier: a kernel-group plugin reaches both entries without a layer violation."""+ registrar = _make_registrar()+ registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)++ registrar.add_internal_model(name="pipelex-xlsx", spec=PIPELEX_XLSX_SPEC)+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT, model="pipelex-xlsx")++ def test_nothing_is_validated_at_registration(self) -> None:+ """A table that is no model spec and a pair no step composes are stored: the model manager refuses them at boot."""+ registrar = _make_registrar()+ registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)++ registrar.add_internal_model(name="broken", spec={"not_a_field": True})+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.PPTX, source=DocGenSource.LAYOUT, model="pipelex-pptx")++ assert "broken" in registrar.internal_models+ assert (DocGenFormat.PPTX, DocGenSource.LAYOUT) in registrar.doc_gen_defaults++ def test_the_declarations_carry_each_entry_and_the_plugin_that_declared_it(self) -> None:+ registrar = _make_registrar()+ registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)+ registrar.add_internal_model(name="pipelex-xlsx", spec=PIPELEX_XLSX_SPEC)+ registrar.add_doc_gen_default(doc_gen_format=DocGenFormat.XLSX, source=DocGenSource.LAYOUT, model="pipelex-xlsx")++ declarations = registrar.make_model_declarations()++ assert declarations.internal_models["pipelex-xlsx"].spec == PIPELEX_XLSX_SPEC+ assert declarations.internal_models["pipelex-xlsx"].plugin == "doc-gen"+ (doc_gen_default,) = declarations.doc_gen_defaults+ assert doc_gen_default.choice_key == "xlsx.layout"+ assert doc_gen_default.model == "pipelex-xlsx"+ assert doc_gen_default.plugin == "doc-gen"+ assert declarations.make_deck_base() == {"doc_gen": {"choice_defaults": {"xlsx.layout": "pipelex-xlsx"}}}++ def test_the_declarations_do_not_follow_a_table_the_plugin_changes_afterwards(self) -> None:+ registrar = _make_registrar()+ registrar.begin_plugin(name="doc-gen", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=PluginGroup.KERNEL)+ spec = dict(PIPELEX_XLSX_SPEC, inputs=["layout", "template_file"])+ registrar.add_internal_model(name="pipelex-xlsx", spec=spec)+ declarations = registrar.make_model_declarations()++ spec["sdk"] = "changed"+ spec["inputs"].append("html")+ declarations.internal_models["pipelex-xlsx"].spec["inputs"].append("html")++ recorded_spec = registrar.make_model_declarations().internal_models["pipelex-xlsx"].spec+ assert recorded_spec["sdk"] == "openpyxl"+ assert recorded_spec["inputs"] == ["layout", "template_file"]++ def test_no_declarations_make_no_deck_base(self) -> None:+ assert PluginModelDeclarations.make_empty().make_deck_base() == {}+ assert _make_registrar().make_model_declarations() == PluginModelDeclarations.make_empty()diff --git a/tests/unit/pipelex/providers/reportlab/conftest.py b/tests/unit/pipelex/providers/reportlab/conftest.pynew file mode 100644index 000000000..d8b98ea44--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/conftest.py@@ -0,0 +1,9 @@+import pytest++from pipelex.providers.reportlab.pdf_elements import register_bundled_fonts++[email protected](autouse=True, scope="session")+def bundled_fonts_registered() -> None:+ """Paragraphs and text measures need the bundled fonts registered, which the engine does only when it is built."""+ register_bundled_fonts()diff --git a/tests/unit/pipelex/providers/reportlab/reportlab_test_helpers.py b/tests/unit/pipelex/providers/reportlab/reportlab_test_helpers.pynew file mode 100644index 000000000..2e910ec9e--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/reportlab_test_helpers.py@@ -0,0 +1,135 @@+import base64+import io+import re+from functools import cache+from typing import NamedTuple++import pypdfium2 as pdfium+import pypdfium2.raw as pdfium_c+from PIL import Image as PilImage+from typing_extensions import override++from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.layout_tree import LayoutBlock, LayoutDocument, MarkdownBlock+from pipelex.cogt.doc_gen.render_job import RenderJob, RenderResources+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.providers.reportlab.reportlab_pdf_renderer import ReportlabPdfWorker++TEST_FILENAME = "test-document.pdf"+TEST_TITLE = "Test document"+++class ResourceRead(NamedTuple):+ uri: str+ position: str+++class StubRenderResources(RenderResources):+ """Decodes `data:` URLs, or answers every read with `answer` when set, and records each read."""++ def __init__(self, *, answer: bytes | None = None):+ self.reads: list[ResourceRead] = []+ self._answer = answer++ @override+ def load(self, *, uri: str, position: str) -> bytes:+ self.reads.append(ResourceRead(uri=uri, position=position))+ if self._answer is not None:+ return self._answer+ header, _, payload = uri.partition(",")+ assert header.startswith("data:"), f"the stub reads data URLs only, not {uri[:40]}"+ assert header.endswith(";base64"), f"the stub reads base64 data URLs only, not {uri[:40]}"+ return base64.b64decode(payload)+++def reportlab_pdf_model() -> InferenceModelSpec:+ """The `reportlab-pdf` model as the kit's `internal.toml` declares it."""+ return InferenceModelSpec(+ backend_name="internal",+ name="reportlab-pdf",+ sdk="reportlab",+ model_type=ModelType.DOC_GEN,+ model_id="print-pdf",+ inputs=["layout"],+ outputs=["pdf"],+ costs={},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ )+++@cache+def get_test_renderer() -> ReportlabPdfWorker:+ return ReportlabPdfWorker(inference_model=reportlab_pdf_model())+++def render_layout(*, blocks: list[LayoutBlock], title: str = TEST_TITLE, resources: RenderResources | None = None) -> bytes:+ layout = LayoutDocument(title=title, blocks=blocks)+ job = RenderJob(format=DocGenFormat.PDF, source=DocGenSource.LAYOUT, filename=TEST_FILENAME, title=title, layout=layout)+ rendered = get_test_renderer().render(job=job, resources=resources or StubRenderResources())+ return rendered.data+++def render_markdown(*, markdown_text: str) -> bytes:+ return render_layout(blocks=[MarkdownBlock(markdown=markdown_text)])+++def page_texts(*, pdf_data: bytes) -> list[str]:+ """The text of every page, its lines separated by line feeds."""+ pdf = pdfium.PdfDocument(pdf_data)+ try:+ texts: list[str] = []+ for page_index in range(len(pdf)):+ page_text: str = pdf[page_index].get_textpage().get_text_bounded() # pyright: ignore[reportUnknownMemberType]+ texts.append(page_text.replace("\r\n", "\n"))+ return texts+ finally:+ pdf.close()+++def document_text(*, pdf_data: bytes) -> str:+ return "\n".join(page_texts(pdf_data=pdf_data))+++def pdf_title(*, pdf_data: bytes) -> str:+ pdf = pdfium.PdfDocument(pdf_data)+ try:+ return str(pdf.get_metadata_dict()["Title"])+ finally:+ pdf.close()+++def image_count(*, pdf_data: bytes) -> int:+ """How many images the pages draw: ReportLab stores identical pixels once, so the file's image objects may be fewer."""+ pdf = pdfium.PdfDocument(pdf_data)+ try:+ count = 0+ for page_index in range(len(pdf)):+ page_images = pdf[page_index].get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,)) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]+ count += len(list(page_images)) # pyright: ignore[reportUnknownArgumentType]+ return count+ finally:+ pdf.close()+++def link_targets(*, pdf_data: bytes) -> list[str]:+ """The target of every link annotation: ReportLab writes them uncompressed, as `/URI (target)`."""+ return [target.decode("latin-1") for target in re.findall(rb"/URI \(([^)]*)\)", pdf_data)]+++def embedded_font_names(*, pdf_data: bytes) -> set[str]:+ """The base font names, without their subset prefix, such as 'OpenSans-Bold'."""+ return {name.decode("latin-1").split("+")[-1] for name in re.findall(rb"/BaseFont /([A-Za-z0-9+\-]+)", pdf_data)}+++def png_bytes(*, width: int, height: int, color: tuple[int, int, int] = (40, 90, 200)) -> bytes:+ buffer = io.BytesIO()+ PilImage.new("RGB", (width, height), color).save(buffer, "PNG")+ return buffer.getvalue()+++def png_data_url(*, width: int, height: int, color: tuple[int, int, int] = (40, 90, 200)) -> str:+ return "data:image/png;base64," + base64.b64encode(png_bytes(width=width, height=height, color=color)).decode("ascii")diff --git a/tests/unit/pipelex/providers/reportlab/test_data.py b/tests/unit/pipelex/providers/reportlab/test_data.pynew file mode 100644index 000000000..a4290fc62--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/test_data.py@@ -0,0 +1,105 @@+import datetime+from typing import ClassVar++from pipelex.cogt.doc_gen.layout_tree import (+ FieldGridBlock,+ ImageBlock,+ LayoutBlock,+ LayoutColumn,+ LayoutField,+ LayoutScalar,+ ParagraphsBlock,+ SectionBlock,+ TableBlock,+)+from tests.unit.pipelex.providers.reportlab.reportlab_test_helpers import png_data_url++MARKUP_VALUE = "<b>Tom & Jerry</b>"+++def line_items_table(*, row_count: int) -> TableBlock:+ return TableBlock(+ title="Line items",+ path="invoice.line_items",+ columns=[+ LayoutColumn(key="description", label="Description"),+ LayoutColumn(key="quantity", label="Quantity"),+ LayoutColumn(key="amount", label="Amount"),+ ],+ rows=[+ {"description": f"Consulting day {index} of the project", "quantity": index % 5 + 1, "amount": 800.5 + index}+ for index in range(1, row_count + 1)+ ],+ )+++class ReportlabRendererTestData:+ """Each markup case: (topic, blocks holding MARKUP_VALUE somewhere). Each script case: (language, sample)."""++ MARKUP_CASES: ClassVar[list[tuple[str, list[LayoutBlock]]]] = [+ ("field grid value", [FieldGridBlock(fields=[LayoutField(label="Customer", value=MARKUP_VALUE)])]),+ ("field grid label", [FieldGridBlock(fields=[LayoutField(label=MARKUP_VALUE, value="x")])]),+ ("paragraph", [ParagraphsBlock(paragraphs=[f"Dear {MARKUP_VALUE}, welcome."])]),+ ("section title", [SectionBlock(title=MARKUP_VALUE, level=1, blocks=[])]),+ ("short table cell", [TableBlock(title="T", path="t", columns=[LayoutColumn(key="name", label="Name")], rows=[{"name": MARKUP_VALUE}])]),+ (+ "wrapping table cell",+ [+ TableBlock(+ title="T",+ path="t",+ columns=[LayoutColumn(key="name", label="Name"), LayoutColumn(key="notes", label="Notes")],+ rows=[{"name": MARKUP_VALUE, "notes": "word " * 200}],+ )+ ],+ ),+ ("image caption", [ImageBlock(url=png_data_url(width=20, height=10), caption=MARKUP_VALUE)]),+ ]++ SCRIPT_CASES: ClassVar[list[tuple[str, str]]] = [+ ("Polish", "Zażółć gęślą jaźń"),+ ("Czech", "Příliš žluťoučký kůň"),+ ("Greek", "Καλημέρα κόσμε"),+ ("Cyrillic", "Съешь же ещё этих мягких французских булок"),+ ("Vietnamese", "Tiếng Việt có dấu"),+ ]++ SCALAR_CASES: ClassVar[list[tuple[LayoutScalar, str]]] = [+ (None, ""),+ (True, "Yes"),+ (False, "No"),+ (42, "42"),+ (2.0, "2"),+ (2.5, "2.5"),+ (0.1 + 0.2, "0.3"),+ (datetime.date(2026, 9, 29), "2026-09-29"),+ (datetime.datetime(2026, 9, 29, 14, 5, 59), "2026-09-29 14:05"),+ (datetime.time(9, 7, 30), "09:07"),+ (datetime.datetime(2026, 9, 29, 14, 5, tzinfo=datetime.UTC), "2026-09-29 14:05 UTC"),+ (datetime.datetime(2026, 9, 29, 14, 5, tzinfo=datetime.timezone(datetime.timedelta(hours=2))), "2026-09-29 14:05 +02:00"),+ (datetime.time(9, 7, tzinfo=datetime.timezone(datetime.timedelta(hours=-5, minutes=-30))), "09:07 -05:30"),+ ("As written", "As written"),+ ]+++class MarkdownFlowablesTestData:+ """Each link case: (href, whether it becomes a link)."""++ NESTED_LISTS = "1. alpha\n2. beta\n 1. gamma\n 2. delta\n - epsilon\n3. zeta\n4. eta\n"++ # CommonMark lets only a list starting at 1 interrupt a paragraph, so a nested list with its own start follows a blank line.+ STARTED_LISTS = "7. seven\n8. eight\n\n 3. three\n 4. four\n\n9. nine\n"++ LINK_CASES: ClassVar[list[tuple[str, bool]]] = [+ ("https://pipelex.com", True),+ ("HTTP://PIPELEX.COM", True),+ ("mailto:[email protected]", True),+ ("javascript:alert(1)", False),+ (" JaVaScRiPt:alert(1)", False),+ ("ftp://example.com/file", False),+ ("file:///etc/passwd", False),+ ("data:text/html,<b>x</b>", False),+ ("relative/page.md", False),+ ("#anchor", False),+ ("http://[invalid", False),+ ]diff --git a/tests/unit/pipelex/providers/reportlab/test_markdown_flowables.py b/tests/unit/pipelex/providers/reportlab/test_markdown_flowables.pynew file mode 100644index 000000000..d2f2a6157--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/test_markdown_flowables.py@@ -0,0 +1,127 @@+import re++import pytest+from markdown_it.token import Token+from markdown_it.tree import SyntaxTreeNode+from reportlab.platypus import Paragraph # type: ignore[import-untyped]++from pipelex.cogt.doc_gen.layout_tree import MarkdownBlock+from pipelex.providers.reportlab.markdown_flowables import inline_markup, is_linked_href, markdown_nodes_to_flowables, plain_text+from pipelex.providers.reportlab.pdf_elements import MONO_FONT, build_pdf_styles+from pipelex.tools.markdown.markdown_parser import get_markdown_parser+from tests.unit.pipelex.providers.reportlab.reportlab_test_helpers import (+ StubRenderResources,+ document_text,+ embedded_font_names,+ link_targets,+ page_texts,+ render_layout,+ render_markdown,+)+from tests.unit.pipelex.providers.reportlab.test_data import MarkdownFlowablesTestData+++def _list_lines(*, text: str) -> list[str]:+ """The lines of a text that are list items: a number and a dot, or a bullet, then the item's text."""+ return [line for line in text.splitlines() if re.match(r"^(\d+\.|•|–) ", line)]+++def _inline_nodes(*, markdown_text: str) -> list[SyntaxTreeNode]:+ paragraph = SyntaxTreeNode(get_markdown_parser().parse(markdown_text)).children[0]+ return paragraph.children+++class TestMarkdownFlowables:+ def test_a_nested_list_numbers_itself_and_keeps_the_outer_numbering(self) -> None:+ text = document_text(pdf_data=render_markdown(markdown_text=MarkdownFlowablesTestData.NESTED_LISTS))+ assert _list_lines(text=text) == ["1. alpha", "2. beta", "1. gamma", "2. delta", "• epsilon", "3. zeta", "4. eta"]++ def test_an_ordered_list_starts_at_its_own_start(self) -> None:+ text = document_text(pdf_data=render_markdown(markdown_text=MarkdownFlowablesTestData.STARTED_LISTS))+ assert _list_lines(text=text) == ["7. seven", "8. eight", "3. three", "4. four", "9. nine"]++ def test_nested_bullets_alternate(self) -> None:+ text = document_text(pdf_data=render_markdown(markdown_text="- one\n - two\n - three\n"))+ assert _list_lines(text=text) == ["• one", "– two", "• three"]++ def test_headings_paragraphs_quotes_and_rules_print_their_text_in_order(self) -> None:+ markdown_text = "# Report\n\n## Findings\n\n###### Small print\n\nPlain paragraph.\n\n> Quoted *words*\n\n---\n\nAfter the rule."+ text = document_text(pdf_data=render_markdown(markdown_text=markdown_text))+ assert text.endswith("Report\nFindings\nSmall print\nPlain paragraph.\nQuoted words\nAfter the rule.")++ def test_a_table_prints_its_header_and_cells(self) -> None:+ markdown_text = "| Quarter | Revenue |\n| --- | --: |\n| Q1 | **$1.2M** |\n| Q2 |\n"+ assert "Quarter Revenue\nQ1 $1.2M\nQ2" in document_text(pdf_data=render_markdown(markdown_text=markdown_text))++ def test_a_long_table_repeats_its_header_on_the_next_page(self) -> None:+ rows = "".join(f"| Row {index} | {index * 10} |\n" for index in range(1, 91))+ pages = page_texts(pdf_data=render_markdown(markdown_text=f"| Label | Value |\n| --- | --- |\n{rows}"))+ assert len(pages) >= 2+ for page_text in pages:+ assert "Label Value" in page_text++ def test_a_code_block_prints_as_written_in_the_monospace_face(self) -> None:+ code_line = 'if a < b && c > d: print("<b>**not bold**</b>") # Zażółć Καλημέρα'+ pdf_data = render_markdown(markdown_text=f"```python\n{code_line}\n```\n\n indented <i>code</i>\n")+ assert document_text(pdf_data=pdf_data).endswith(f"{code_line}\nindented <i>code</i>")+ assert "RobotoMono-Regular" in embedded_font_names(pdf_data=pdf_data)++ def test_a_long_code_line_wraps_inside_the_page(self) -> None:+ text = document_text(pdf_data=render_markdown(markdown_text="```\n" + "x" * 400 + "\n```\n"))+ assert text.count("x") == 400+ assert max(len(line) for line in text.splitlines()) < 400++ def test_bold_italics_strikethrough_and_code_pick_their_faces(self) -> None:+ pdf_data = render_markdown(markdown_text="Some **bold**, some *italic*, some ***both***, ~~struck~~ and `code`.")+ assert "Some bold, some italic, some both, struck and code." in document_text(pdf_data=pdf_data)+ expected_fonts = {"OpenSans-Regular", "OpenSans-Bold", "OpenSans-Italic", "OpenSans-BoldItalic", "RobotoMono-Regular"}+ assert expected_fonts <= embedded_font_names(pdf_data=pdf_data)++ def test_markup_in_markdown_text_prints_as_written(self) -> None:+ text = document_text(pdf_data=render_markdown(markdown_text="Tom & Jerry say <b>hello</b> & <br/> bye"))+ assert "Tom & Jerry say <b>hello</b> & <br/> bye" in text++ def test_only_web_and_mail_links_become_links(self) -> None:+ markdown_text = (+ "See [the site](https://pipelex.com), [write](mailto:[email protected]), [evil](javascript:alert(1)), "+ "[files](ftp://example.com/file) and https://example.com/auto."+ )+ pdf_data = render_markdown(markdown_text=markdown_text)+ assert sorted(link_targets(pdf_data=pdf_data)) == ["https://example.com/auto", "https://pipelex.com", "mailto:[email protected]"]+ assert "See the site, write, [evil](javascript:alert(1)), files and https://example.com/auto." in document_text(pdf_data=pdf_data)++ @pytest.mark.parametrize(("href", "is_linked"), MarkdownFlowablesTestData.LINK_CASES)+ def test_a_link_target_is_linked_only_for_web_and_mail(self, href: str, is_linked: bool) -> None:+ assert is_linked_href(href=href) is is_linked++ def test_a_link_target_is_escaped_in_the_markup(self) -> None:+ tokens = [Token(type="link_open", tag="a", nesting=1, attrs={"href": 'https://x.com/"><b>'}), Token(type="link_close", tag="a", nesting=-1)]+ markup = inline_markup(nodes=SyntaxTreeNode(tokens).children)+ assert markup == '<a href="https://x.com/"><b>" color="#1d4ed8"></a>'++ def test_a_markdown_image_prints_its_alt_text_and_is_never_read(self) -> None:+ resources = StubRenderResources()+ markdown_block = MarkdownBlock(markdown="Before  after")+ pdf_data = render_layout(blocks=[markdown_block], resources=resources)+ assert "Before [image: a chart of sales] after" in document_text(pdf_data=pdf_data)+ assert resources.reads == []++ def test_inline_markup_escapes_text_and_tags_emphasis(self) -> None:+ markup = inline_markup(nodes=_inline_nodes(markdown_text="**a < b** and *c & d* and `<e>`"))+ assert markup == f'<b>a < b</b> and <i>c & d</i> and <font face="{MONO_FONT}"><e></font>'++ def test_unknown_nodes_print_their_text(self) -> None:+ tokens = [+ Token(type="custom_block", tag="div", nesting=0, content="Custom <text>"),+ Token(type="weird_open", tag="x", nesting=1),+ Token(+ type="inline", tag="", nesting=0, content="inner", children=[Token(type="mystery_inline", tag="", nesting=0, content="Inner & text")]+ ),+ Token(type="weird_close", tag="x", nesting=-1),+ ]+ nodes = SyntaxTreeNode(tokens).children+ assert plain_text(nodes=nodes) == "Custom <text>Inner & text"+ flowables = markdown_nodes_to_flowables(nodes=nodes, styles=build_pdf_styles(), available_width=400)+ paragraphs = [flowable for flowable in flowables if isinstance(flowable, Paragraph)]+ assert len(paragraphs) == len(flowables)+ assert [paragraph.getPlainText() for paragraph in paragraphs] == ["Custom <text>", "Inner & text"]diff --git a/tests/unit/pipelex/providers/reportlab/test_pdf_elements.py b/tests/unit/pipelex/providers/reportlab/test_pdf_elements.pynew file mode 100644index 000000000..790f73ede--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/test_pdf_elements.py@@ -0,0 +1,84 @@+import pytest+from reportlab.lib.fonts import tt2ps # type: ignore[import-untyped]+from reportlab.pdfbase import pdfmetrics # type: ignore[import-untyped]+from reportlab.platypus import Paragraph # type: ignore[import-untyped]++from pipelex.providers.reportlab.pdf_elements import (+ BUNDLED_FACES,+ FONTS_DIR,+ MONO_FONT,+ SANS_BOLD_FONT,+ SANS_BOLD_ITALIC_FONT,+ SANS_FONT,+ SANS_ITALIC_FONT,+ ColumnExtent,+ build_pdf_styles,+ escape_attribute,+ escape_text,+ fit_column_widths,+ register_bundled_fonts,+ text_cell,+)+++class TestPdfElements:+ def test_every_bundled_face_ships_with_its_licence_and_the_fonts_stay_small(self) -> None:+ for face in BUNDLED_FACES:+ assert face.file_path.is_file(), face.file_path+ assert (face.file_path.parent / "OFL.txt").is_file(), face.file_path.parent+ total_size = sum(font_file.stat().st_size for font_file in FONTS_DIR.rglob("*") if font_file.is_file())+ assert total_size < 1_000_000++ def test_the_fonts_are_registered_once_and_their_families_map_bold_and_italic(self) -> None:+ register_bundled_fonts()+ sans_font = pdfmetrics.getFont(SANS_FONT)+ register_bundled_fonts()+ assert pdfmetrics.getFont(SANS_FONT) is sans_font+ assert tt2ps(SANS_FONT, 1, 0) == SANS_BOLD_FONT+ assert tt2ps(SANS_FONT, 0, 1) == SANS_ITALIC_FONT+ assert tt2ps(SANS_FONT, 1, 1) == SANS_BOLD_ITALIC_FONT+ assert tt2ps(MONO_FONT, 1, 1) == MONO_FONT++ @pytest.mark.parametrize(+ ("text", "expected"),+ [+ ("<b>Tom & Jerry</b>", "<b>Tom & Jerry</b>"),+ ("& ", "&amp; &nbsp;"),+ ("first\nsecond\r\nthird\rfourth", "first<br/>second<br/>third<br/>fourth"),+ ("bell\x07 null\x00 tab\t", "bell null tab\t"),+ ],+ )+ def test_escaped_text_prints_as_written(self, text: str, expected: str) -> None:+ assert escape_text(text=text) == expected++ def test_an_attribute_escapes_its_quotes(self) -> None:+ assert escape_attribute(value='https://x.com/?a="1"&b=<2>') == "https://x.com/?a="1"&b=<2>"++ def test_columns_that_fit_keep_their_preferred_widths(self) -> None:+ extents = [ColumnExtent(preferred=100, minimum=40), ColumnExtent(preferred=50, minimum=20)]+ assert fit_column_widths(extents=extents, available_width=400) == [100, 50]++ def test_a_long_text_column_wraps_and_the_short_columns_beside_it_do_not(self) -> None:+ extents = [ColumnExtent(preferred=90, minimum=50), ColumnExtent(preferred=4000, minimum=30), ColumnExtent(preferred=40, minimum=40)]+ widths = fit_column_widths(extents=extents, available_width=500)+ assert widths == [90, 370, 40]++ def test_columns_that_all_want_more_share_the_width_evenly_above_their_minimums(self) -> None:+ extents = [ColumnExtent(preferred=1000, minimum=100), ColumnExtent(preferred=3000, minimum=20)]+ widths = fit_column_widths(extents=extents, available_width=400)+ assert widths == [240, 160]++ def test_minimums_wider_than_the_page_shrink_in_proportion(self) -> None:+ extents = [ColumnExtent(preferred=600, minimum=300), ColumnExtent(preferred=400, minimum=200)]+ assert fit_column_widths(extents=extents, available_width=250) == [150, 100]++ def test_a_text_that_fits_its_column_is_drawn_as_it_is_and_one_that_does_not_wraps(self) -> None:+ cell_style = build_pdf_styles().cell+ assert text_cell(text="<b>short</b>", width=200, style=cell_style) == "<b>short</b>"+ assert isinstance(text_cell(text="word " * 60, width=200, style=cell_style), Paragraph)+ assert isinstance(text_cell(text="two\nlines", width=200, style=cell_style), Paragraph)++ @pytest.mark.parametrize(("level", "expected_index"), [(-3, 0), (0, 0), (1, 0), (3, 2), (6, 5), (12, 5)])+ def test_a_heading_level_is_clamped_to_the_styles_there_are(self, level: int, expected_index: int) -> None:+ styles = build_pdf_styles()+ assert styles.heading(level=level) is styles.headings[expected_index]diff --git a/tests/unit/pipelex/providers/reportlab/test_reportlab_pdf_renderer.py b/tests/unit/pipelex/providers/reportlab/test_reportlab_pdf_renderer.pynew file mode 100644index 000000000..079b67b4b--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/test_reportlab_pdf_renderer.py@@ -0,0 +1,197 @@+from __future__ import annotations++import datetime+import sys+from concurrent.futures import ThreadPoolExecutor+from typing import TYPE_CHECKING, Any++import pytest+from reportlab.platypus import Table # type: ignore[import-untyped]+from reportlab.platypus.doctemplate import BaseDocTemplate, LayoutError # type: ignore[import-untyped]++from pipelex.base_exceptions import ErrorDomain+from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormat, DocGenSource+from pipelex.cogt.doc_gen.exceptions import DocGenRenderError+from pipelex.cogt.doc_gen.layout_tree import (+ FieldGridBlock,+ ImageBlock,+ LayoutBlock,+ LayoutColumn,+ LayoutDocument,+ LayoutField,+ LayoutScalar,+ MarkdownBlock,+ ParagraphsBlock,+ SectionBlock,+ TableBlock,+)+from pipelex.cogt.doc_gen.render_job import RenderJob+from pipelex.providers.reportlab.reportlab_pdf_renderer import display_scalar+from tests.unit.pipelex.providers.reportlab.reportlab_test_helpers import (+ TEST_FILENAME,+ TEST_TITLE,+ ResourceRead,+ StubRenderResources,+ document_text,+ embedded_font_names,+ get_test_renderer,+ image_count,+ page_texts,+ pdf_title,+ png_data_url,+ render_layout,+)+from tests.unit.pipelex.providers.reportlab.test_data import MARKUP_VALUE, ReportlabRendererTestData, line_items_table++if TYPE_CHECKING:+ from pytest_mock import MockerFixture+++class TestReportlabPdfRenderer:+ def test_a_long_table_repeats_its_header_on_the_next_page(self) -> None:+ pages = page_texts(pdf_data=render_layout(blocks=[line_items_table(row_count=90)]))+ assert len(pages) >= 2+ for page_text in pages:+ assert "Description Quantity Amount" in page_text+ assert "Consulting day 1 of the project" in pages[0]+ assert "Consulting day 1 of the project" not in pages[1]+ assert "Consulting day 90 of the project" in pages[-1]++ def test_a_long_table_starts_right_under_its_heading(self) -> None:+ """A table too long for the page is not pushed onto the next page to stay with its heading."""+ pdf_data = render_layout(blocks=[FieldGridBlock(fields=[LayoutField(label="Invoice", value="INV-1")]), line_items_table(row_count=90)])+ first_page = page_texts(pdf_data=pdf_data)[0]+ assert "Line items\nDescription Quantity Amount\nConsulting day 1 of the project" in first_page++ def test_every_page_carries_the_running_header_and_its_number_out_of_the_count(self) -> None:+ pages = page_texts(pdf_data=render_layout(blocks=[line_items_table(row_count=50)], title="Invoice INV-2026-0142"))+ assert len(pages) == 2+ for page_number, page_text in enumerate(pages, start=1):+ assert page_text.startswith(f"Invoice INV-2026-0142\nPage {page_number} of 2\n")++ def test_the_title_prints_first_and_is_the_pdf_title(self) -> None:+ pdf_data = render_layout(blocks=[ParagraphsBlock(paragraphs=["Body"])], title="Quarterly review")+ assert pdf_title(pdf_data=pdf_data) == "Quarterly review"+ assert document_text(pdf_data=pdf_data) == "Quarterly review\nPage 1 of 1\nQuarterly review\nBody"++ @pytest.mark.parametrize(("topic", "blocks"), ReportlabRendererTestData.MARKUP_CASES)+ def test_markup_characters_print_as_written(self, topic: str, blocks: list[LayoutBlock]) -> None:+ assert MARKUP_VALUE in document_text(pdf_data=render_layout(blocks=blocks)), topic++ def test_the_title_prints_its_markup_characters_as_written(self) -> None:+ assert document_text(pdf_data=render_layout(blocks=[], title=MARKUP_VALUE)).count(MARKUP_VALUE) == 2++ @pytest.mark.parametrize(("language", "sample"), ReportlabRendererTestData.SCRIPT_CASES)+ def test_text_beyond_western_european_prints_in_the_bundled_font(self, language: str, sample: str) -> None:+ pdf_data = render_layout(blocks=[ParagraphsBlock(paragraphs=[sample]), FieldGridBlock(fields=[LayoutField(label=language, value=sample)])])+ assert document_text(pdf_data=pdf_data).count(sample) == 2+ assert embedded_font_names(pdf_data=pdf_data) >= {"OpenSans-Regular", "OpenSans-Bold"}++ def test_line_breaks_in_a_paragraph_are_kept(self) -> None:+ text = document_text(pdf_data=render_layout(blocks=[ParagraphsBlock(paragraphs=["1 rue de la Paix\n75002 Paris"])]))+ assert "1 rue de la Paix\n75002 Paris" in text++ def test_deep_sections_print_every_heading(self) -> None:+ innermost: LayoutBlock = ParagraphsBlock(paragraphs=["Deepest text"])+ for level in range(9, 0, -1):+ innermost = SectionBlock(title=f"Section level {level}", level=level, blocks=[innermost])+ text = document_text(pdf_data=render_layout(blocks=[innermost]))+ assert text.endswith("\n".join([*(f"Section level {level}" for level in range(1, 10)), "Deepest text"]))++ def test_an_image_is_read_once_through_the_resources_with_its_position(self) -> None:+ first_url = png_data_url(width=64, height=32)+ second_url = png_data_url(width=32, height=64, color=(200, 40, 40))+ resources = StubRenderResources()+ pdf_data = render_layout(+ blocks=[+ ImageBlock(url=first_url, caption="The first figure"),+ SectionBlock(title="Appendix", level=1, blocks=[ImageBlock(url=second_url, caption=None)]),+ ],+ resources=resources,+ )+ assert resources.reads == [+ ResourceRead(uri=first_url, position="image 1 of the document"),+ ResourceRead(uri=second_url, position="image 2 of the document"),+ ]+ assert image_count(pdf_data=pdf_data) == 2+ assert "The first figure\nAppendix" in document_text(pdf_data=pdf_data)++ def test_a_large_image_is_scaled_into_one_page(self) -> None:+ pdf_data = render_layout(blocks=[ImageBlock(url=png_data_url(width=4000, height=6000), caption="Huge")])+ assert len(page_texts(pdf_data=pdf_data)) == 1+ assert image_count(pdf_data=pdf_data) == 1++ def test_a_file_that_is_not_an_image_is_refused_with_its_position(self) -> None:+ resources = StubRenderResources(answer=b"<svg xmlns='http://www.w3.org/2000/svg'/>")+ with pytest.raises(DocGenRenderError, match="image 1 of the document") as exc_info:+ render_layout(blocks=[ImageBlock(url="pipelex-storage://run/chart.svg", caption=None)], resources=resources)+ assert exc_info.value.error_domain == ErrorDomain.INPUT++ def test_a_job_that_is_not_a_pdf_from_the_layout_is_refused(self) -> None:+ job = RenderJob(+ format=DocGenFormat.XLSX,+ source=DocGenSource.LAYOUT,+ filename="book.xlsx",+ title=TEST_TITLE,+ layout=LayoutDocument(title=TEST_TITLE, blocks=[]),+ )+ with pytest.raises(DocGenRenderError, match="built-in PDF engine prints a pdf from the auto-layout"):+ get_test_renderer().render(job=job, resources=StubRenderResources())++ def test_a_layout_error_names_the_part_of_the_document_that_did_not_fit(self, mocker: MockerFixture) -> None:+ original_handle_flowable = BaseDocTemplate.handle_flowable++ def refuse_tables(doc_template: Any, flowables: list[Any]) -> None:+ if flowables and isinstance(flowables[0], Table):+ msg = "Flowable too large on page 1"+ raise LayoutError(msg)+ original_handle_flowable(doc_template, flowables)++ mocker.patch.object(BaseDocTemplate, "handle_flowable", autospec=True, side_effect=refuse_tables)+ with pytest.raises(DocGenRenderError) as exc_info:+ render_layout(blocks=[line_items_table(row_count=3)])+ assert str(exc_info.value) == (+ f"The PDF engine cannot print '{TEST_FILENAME}': the table 'Line items' does not fit on an A4 page, "+ "as part of it is taller than a page and cannot be split across pages."+ )+ assert exc_info.value.error_domain == ErrorDomain.INPUT++ @pytest.mark.parametrize(("value", "expected"), ReportlabRendererTestData.SCALAR_CASES)+ def test_a_scalar_prints_plainly(self, value: LayoutScalar, expected: str) -> None:+ assert display_scalar(value=value) == expected++ def test_table_cells_print_their_scalars(self) -> None:+ table = TableBlock(+ title="Mixed",+ path="mixed",+ columns=[LayoutColumn(key="flag", label="Flag"), LayoutColumn(key="day", label="Day"), LayoutColumn(key="count", label="Count")],+ rows=[{"flag": True, "day": datetime.date(2026, 1, 2), "count": 3.0}, {"flag": None, "day": None, "count": None}],+ )+ assert "Flag Day Count\nYes 2026-01-02 3" in document_text(pdf_data=render_layout(blocks=[table]))++ def test_a_table_with_no_rows_says_so(self) -> None:+ table = TableBlock(title="Empty", path="empty", columns=[LayoutColumn(key="name", label="Name")], rows=[])+ assert "Empty\nName\nNo rows." in document_text(pdf_data=render_layout(blocks=[table]))++ def test_renders_from_several_threads_at_once_print_what_one_render_prints(self) -> None:+ """ReportLab subsets fonts through state the registered fonts share, so the engine builds one document at a time."""+ documents: list[list[LayoutBlock]] = [+ [line_items_table(row_count=60), ParagraphsBlock(paragraphs=["Zażółć gęślą jaźń, Καλημέρα κόσμε"])],+ [MarkdownBlock(markdown="# Report\n\n1. **one**\n2. *two*\n\n```\ncode Съешь\n```\n" * 10)],+ ]+ expected_texts = [document_text(pdf_data=render_layout(blocks=blocks)) for blocks in documents]++ def render_document(index: int) -> bytes:+ return render_layout(blocks=documents[index % 2])++ # A tiny switch interval makes the threads interleave inside ReportLab, where a race would show.+ switch_interval = sys.getswitchinterval()+ sys.setswitchinterval(1e-6)+ try:+ with ThreadPoolExecutor(max_workers=8) as pool:+ rendered = list(pool.map(render_document, range(16)))+ finally:+ sys.setswitchinterval(switch_interval)+ # pdfium is not thread-safe, so the texts are read here, one after the other.+ for index, pdf_data in enumerate(rendered):+ assert document_text(pdf_data=pdf_data) == expected_texts[index % 2]diff --git a/tests/unit/pipelex/providers/reportlab/test_reportlab_plugin.py b/tests/unit/pipelex/providers/reportlab/test_reportlab_plugin.pynew file mode 100644index 000000000..1df70d2f4--- /dev/null+++ b/tests/unit/pipelex/providers/reportlab/test_reportlab_plugin.py@@ -0,0 +1,84 @@+import subprocess # ruff: ignore[suspicious-subprocess-import]+import sys+import textwrap+from types import SimpleNamespace+from typing import TYPE_CHECKING, cast++from pytest_mock import MockerFixture++from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.inference_backend_registry import InferenceFamily+from pipelex.plugins.registrar import PluginRegistrar+from pipelex.providers.builtins import KERNEL_BUILTIN_PLUGINS+from pipelex.providers.reportlab.reportlab_pdf_renderer import ReportlabPdfWorker+from pipelex.providers.reportlab.reportlab_plugin import ReportlabDocGenPlugin+from tests.unit.pipelex.providers.reportlab.reportlab_test_helpers import reportlab_pdf_model++if TYPE_CHECKING:+ from pipelex.system.configuration.configs import PipelexConfig++# Imports the plugin and registers it in a fresh interpreter whose import system refuses ReportLab, so a module-level+# import of ReportLab anywhere on that path fails loudly, naming the importer.+_LAZY_IMPORT_SCRIPT = textwrap.dedent(+ """+ import importlib.abc+ import sys++ class _ReportlabBlocker(importlib.abc.MetaPathFinder):+ def find_spec(self, fullname, path, target=None):+ if fullname == "reportlab" or fullname.startswith("reportlab."):+ raise ImportError(f"the import-light guard blocked '{fullname}'")+ return None++ sys.meta_path.insert(0, _ReportlabBlocker())++ from types import SimpleNamespace++ from pipelex.plugins.registrar import PluginRegistrar+ from pipelex.providers.builtins import KERNEL_BUILTIN_PLUGINS+ from pipelex.providers.reportlab.reportlab_plugin import ReportlabDocGenPlugin++ registrar = PluginRegistrar(config=SimpleNamespace(runtime=SimpleNamespace(plugins=SimpleNamespace(disabled=[]))))+ ReportlabDocGenPlugin().register(registrar)+ assert list(registrar.inference_backends) == [("doc_gen", "reportlab")]+ assert not [name for name in sys.modules if name == "reportlab" or name.startswith("reportlab.")]+ print("import-light OK")+ """+)+++def _registrar() -> PluginRegistrar:+ return PluginRegistrar(config=cast("PipelexConfig", SimpleNamespace(runtime=SimpleNamespace(plugins=SimpleNamespace(disabled=[])))))+++class TestReportlabPlugin:+ def test_the_plugin_registers_the_reportlab_sdk_of_the_doc_gen_family(self) -> None:+ plugin = ReportlabDocGenPlugin()+ registrar = _registrar()+ plugin.register(registrar)+ assert plugin.name == "reportlab"+ assert plugin.targets_api == PLUGIN_API_VERSION+ assert list(registrar.inference_backends) == [(InferenceFamily.DOC_GEN, "reportlab")]++ def test_the_factory_builds_the_reportlab_worker(self, mocker: MockerFixture) -> None:+ registrar = _registrar()+ ReportlabDocGenPlugin().register(registrar)+ (make_worker,) = registrar.inference_backends.values()+ worker = make_worker(+ inference_model=reportlab_pdf_model(), backend=mocker.MagicMock(), sdk_clients=mocker.MagicMock(), reporting_delegate=None+ )+ assert isinstance(worker, ReportlabPdfWorker)+ assert worker.check_template(request=mocker.MagicMock()) == []++ def test_the_plugin_is_a_kernel_builtin(self) -> None:+ assert [plugin.name for plugin in KERNEL_BUILTIN_PLUGINS].count("reportlab") == 1++ def test_registering_the_plugin_imports_no_reportlab(self) -> None:+ result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]+ [sys.executable, "-c", _LAZY_IMPORT_SCRIPT],+ capture_output=True,+ text=True,+ check=False,+ )+ assert result.returncode == 0, f"import-light guard failed:\nstdout={result.stdout}\nstderr={result.stderr}"+ assert "import-light OK" in result.stdoutdiff --git a/tests/unit/pipelex/tools/markdown/test_markdown_parser.py b/tests/unit/pipelex/tools/markdown/test_markdown_parser.pynew file mode 100644index 000000000..d6632309a--- /dev/null+++ b/tests/unit/pipelex/tools/markdown/test_markdown_parser.py@@ -0,0 +1,14 @@+from pipelex.tools.markdown.markdown_parser import render_markdown_as_html+++class TestMarkdownParser:+ def test_only_a_url_with_a_scheme_becomes_a_link(self) -> None:+ html = render_markdown_as_html("See https://example.com/docs, README.md, www.example.com and [email protected].")+ assert '<a href="https://example.com/docs">' in html+ assert "README.md" in html+ assert html.count("<a ") == 1++ def test_tables_and_strikethrough_are_formatted(self) -> None:+ html = render_markdown_as_html("| a | b |\n| - | - |\n| 1 | 2 |\n\n~~gone~~")+ assert "<table>" in html+ assert "<s>gone</s>" in htmldiff --git a/tests/unit/pipelex/tools/test_jinja2_field_paths.py b/tests/unit/pipelex/tools/test_jinja2_field_paths.pynew file mode 100644index 000000000..0eeb1ac5b--- /dev/null+++ b/tests/unit/pipelex/tools/test_jinja2_field_paths.py@@ -0,0 +1,50 @@+import pytest++from pipelex.tools.jinja2.exceptions import Jinja2DetectVariablesError+from pipelex.tools.jinja2.jinja2_field_paths import LIST_ITEM_SEGMENT, detect_template_field_paths+from pipelex.tools.jinja2.template_category import TemplateCategory+++def _paths(template_source: str) -> list[tuple[str, ...]]:+ found = detect_template_field_paths(template_source=template_source, template_category=TemplateCategory.HTML)+ return [(field_path.root, *field_path.segments) for field_path in found]+++class TestDetectTemplateFieldPaths:+ def test_an_attribute_chain_reads_its_path(self) -> None:+ assert _paths("{{ invoice.customer.name }}") == [("invoice", "customer", "name")]++ def test_a_loop_variable_reads_an_item_of_the_iterated_list(self) -> None:+ paths = _paths("{% for item in invoice.line_items %}{{ item.amount }}{% endfor %}")+ assert paths == [("invoice", "line_items"), ("invoice", "line_items", LIST_ITEM_SEGMENT, "amount")]++ def test_nested_loops_follow_each_level(self) -> None:+ paths = _paths("{% for section in report.sections %}{% for row in section.rows %}{{ row.label }}{% endfor %}{% endfor %}")+ assert ("report", "sections", LIST_ITEM_SEGMENT, "rows", LIST_ITEM_SEGMENT, "label") in paths++ def test_the_written_chain_is_kept_for_messages(self) -> None:+ found = detect_template_field_paths(+ template_source="{% for item in invoice.line_items %}{{ item.amount }}{% endfor %}", template_category=TemplateCategory.HTML+ )+ assert found[-1].written == "item.amount"++ @pytest.mark.parametrize(+ ("topic", "template_source"),+ [+ ("set_alias", "{% set customer = invoice.customer %}{{ customer.nosuch }}"),+ ("with_alias", "{% with customer = invoice.customer %}{{ customer.nosuch }}{% endwith %}"),+ ("macro_argument", "{% macro show(customer) %}{{ customer.nosuch }}{% endmacro %}"),+ ("tuple_loop_target", "{% for key, value in invoice.pairs %}{{ value.nosuch }}{% endfor %}"),+ ("loop_helper", "{% for item in invoice.line_items %}{{ loop.index }}{% endfor %}"),+ ],+ )+ def test_what_cannot_be_followed_gives_no_path_through_it(self, topic: str, template_source: str) -> None:+ paths = _paths(template_source)+ assert not any("nosuch" in path or "index" in path for path in paths), f"{topic}: {paths}"++ def test_a_subscript_is_not_followed(self) -> None:+ assert _paths("{{ invoice['customer'].name }}") == [("invoice",)]++ def test_a_syntax_error_is_reported(self) -> None:+ with pytest.raises(Jinja2DetectVariablesError):+ detect_template_field_paths(template_source="{% for item in %}", template_category=TemplateCategory.HTML)diff --git a/tests/unit/pipelex/tools/test_jinja2_markdown_filter.py b/tests/unit/pipelex/tools/test_jinja2_markdown_filter.pynew file mode 100644index 000000000..b27bbc98e--- /dev/null+++ b/tests/unit/pipelex/tools/test_jinja2_markdown_filter.py@@ -0,0 +1,36 @@+import pytest++from pipelex.cogt.templating.template_rendering import render_template+from pipelex.tools.jinja2.exceptions import Jinja2TemplateRenderError+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_async+from pipelex.tools.jinja2.template_category import TemplateCategory+++class TestMarkdownFilter:+ async def test_the_filter_turns_markdown_into_html_in_an_html_template(self) -> None:+ rendered = await render_template("<div>{{ notes | markdown }}</div>", category=TemplateCategory.HTML, context={"notes": "**Paid** in full."})+ assert rendered == "<div><p><strong>Paid</strong> in full.</p>\n</div>"++ async def test_raw_html_in_the_source_is_shown_as_text(self) -> None:+ rendered = await render_template("{{ notes | markdown }}", category=TemplateCategory.HTML, context={"notes": "<script>alert(1)</script>"})+ assert "<script>" not in rendered+ assert "<script>" in rendered++ async def test_a_missing_value_prints_nothing_in_a_lenient_template(self) -> None:+ rendered = await render_jinja2_async(+ template_source="[{{ notes | markdown }}][{{ absent | markdown }}]",+ template_category=TemplateCategory.HTML,+ templating_context={"notes": None},+ )+ assert rendered == "[][]"++ async def test_a_missing_field_fails_in_a_strict_template(self) -> None:+ """A misspelled field reached through an alias, which no load-time check can follow, still fails at render."""+ with pytest.raises(Jinja2TemplateRenderError):+ await render_jinja2_async(+ template_source="{% set invoice = inv %}{{ invoice.notez | markdown }}",+ template_category=TemplateCategory.HTML,+ templating_context={"inv": {"notes": "Paid."}},+ is_undefined_strict=True,+ )diff --git a/tests/unit/pipelex/tools/test_jinja2_strict_undefined.py b/tests/unit/pipelex/tools/test_jinja2_strict_undefined.pynew file mode 100644index 000000000..dbad74b7f--- /dev/null+++ b/tests/unit/pipelex/tools/test_jinja2_strict_undefined.py@@ -0,0 +1,30 @@+import pytest++from pipelex.cogt.templating.template_rendering import render_template+from pipelex.tools.jinja2.exceptions import Jinja2TemplateRenderError+from pipelex.tools.jinja2.template_category import TemplateCategory+++class TestStrictUndefined:+ async def test_a_missing_field_fails_the_render(self) -> None:+ with pytest.raises(Jinja2TemplateRenderError):+ await render_template(+ "{{ invoice.nosuch }}", category=TemplateCategory.BASIC, context={"invoice": {"number": "INV-1"}}, is_undefined_strict=True+ )++ async def test_the_default_undefined_still_prints_nothing(self) -> None:+ rendered = await render_template("[{{ invoice.nosuch }}]", category=TemplateCategory.BASIC, context={"invoice": {"number": "INV-1"}})+ assert rendered == "[]"++ @pytest.mark.parametrize(+ "template",+ [+ "{% if notes %}{{ notes }}{% else %}none{% endif %}",+ "{% if notes is defined %}{{ notes }}{% else %}none{% endif %}",+ "{{ notes | default('none') }}",+ ],+ )+ async def test_an_absent_optional_input_can_be_tested(self, template: str) -> None:+ rendered = await render_template(template, category=TemplateCategory.BASIC, context={}, is_undefined_strict=True)+ assert rendered == "none"diff --git a/tests/unit/pipelex/tools/test_jinja2_style_context_strictness.py b/tests/unit/pipelex/tools/test_jinja2_style_context_strictness.pyindex 6d5e118ad..63a15eabe 100644--- a/tests/unit/pipelex/tools/test_jinja2_style_context_strictness.py+++ b/tests/unit/pipelex/tools/test_jinja2_style_context_strictness.py@@ -24,6 +24,8 @@ class TestStyleContextStrictness:def _make_context(self, mocker: MockerFixture, *, context_dict: dict[str, Any]) -> Any:context = mocker.MagicMock(spec=Context)context.get = lambda key, default=None: context_dict.get(key, default) # pyright: ignore[reportUnknownLambdaType, reportUnknownArgumentType]+ # Jinja2 sets `eval_ctx` on the instance, so the class spec lacks it; a real render context always has one.+ context.eval_ctx = mocker.MagicMock(autoescape=False)return contextdef test_tag_style_missing_raises(self, mocker: MockerFixture) -> None:diff --git a/uv.lock b/uv.lockindex 3159f8a03..58443358f 100644--- a/uv.lock+++ b/uv.lock@@ -2123,6 +2123,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/51/78/a0705a67cacd81e5fa01a5035b3adbdfbb43a7b8d4bd27e2b282ae61baf2/librt-0.10.0-cp314-cp314t-win_arm64.whl", hash = "sha256:1ee63f30abf18ed4830fdbaf87b2b6f4bba1e198d46085c314edde4045e56715", size = 58247, upload-time = "2026-05-05T16:31:03.191Z" },]+[[package]]+name = "linkify-it-py"+version = "2.2.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/45/98/7a1a5f31fd5c7ba93e963b168e244b8e3dd705b3d2a718e3c3307583bf57/linkify_it_py-2.2.0.tar.gz", hash = "sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee", size = 32939, upload-time = "2026-08-29T07:07:08.305Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/13/d4/1152d1c7ab42d8b908be64fd200ddc870dc9d4925e951198702084aa1a7d/linkify_it_py-2.2.0-py3-none-any.whl", hash = "sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443", size = 21971, upload-time = "2026-08-29T07:07:07.164Z" },+]+[[package]]name = "linkup-sdk"version = "0.13.0"@@ -2268,6 +2277,11 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },]+[package.optional-dependencies]+linkify = [+ { name = "linkify-it-py" },+]+[[package]]name = "marko"version = "2.2.2"@@ -3456,7 +3470,7 @@ dependencies = [{ name = "jinja2" },{ name = "json2html" },{ name = "kajson" },- { name = "markdown" },+ { name = "markdown-it-py", extra = ["linkify"] },{ name = "mthds" },{ name = "networkx" },{ name = "openai" },@@ -3518,7 +3532,6 @@ dev = [{ name = "types-aiobotocore", extra = ["bedrock", "bedrock-runtime"] },{ name = "types-aiofiles" },{ name = "types-jsonschema" },- { name = "types-markdown" },{ name = "types-networkx" },{ name = "types-pyyaml" },]@@ -3608,7 +3621,7 @@ requires-dist = [{ name = "kajson", specifier = "==0.7.1" },{ name = "linkup-sdk", marker = "extra == 'linkup'", specifier = ">=0.12.0" },{ name = "lxml", marker = "extra == 'docling'", specifier = ">=6.1.0" },- { name = "markdown", specifier = ">=3.6" },+ { name = "markdown-it-py", extras = ["linkify"], specifier = ">=3.0.0" },{ name = "mike", marker = "extra == 'docs'", specifier = ">=2.1.3" },{ name = "mistralai", marker = "extra == 'mistralai'", specifier = ">=2.10.0" },{ name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6.1" },@@ -3661,7 +3674,6 @@ requires-dist = [{ name = "types-aiobotocore", extras = ["bedrock", "bedrock-runtime"], marker = "extra == 'dev'", specifier = ">=3.0.0" },{ name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=24.1.0.20240626" },{ name = "types-jsonschema", marker = "extra == 'dev'", specifier = ">=4.20.0" },- { name = "types-markdown", marker = "extra == 'dev'", specifier = ">=3.6.0.20240316" },{ name = "types-networkx", marker = "extra == 'dev'", specifier = ">=3.3.0.20241020" },{ name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12.20250326" },{ name = "typing-extensions", specifier = ">=4.13.2" },@@ -5657,15 +5669,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/07/d5/134f8a147dcecda10db7f60cfc6af0578a25a5c53c87b3907a64385e0184/types_jsonschema-4.26.0.20260518-py3-none-any.whl", hash = "sha256:30b30a518c7fe335df85c919fcbcc631b69c03d4a4b5b632fa916bea03065307", size = 16072, upload-time = "2026-05-18T06:06:43.264Z" },]-[[package]]-name = "types-markdown"-version = "3.10.2.20260408"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/dd/0e/a690840934c459aa50e0470e7550d7f151632eafa4a8e3c21d18009ad15c/types_markdown-3.10.2.20260408.tar.gz", hash = "sha256:d5cba15ed65a1420e80e31c17e3d4a2ad7208a3f3a4da97fd2c5f093caf523cd", size = 19784, upload-time = "2026-04-08T04:33:07.644Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/75/7e/265a8df257c8dced6ea89295f793a19f0a49ccbfeae1ed562368b2caf7a3/types_markdown-3.10.2.20260408-py3-none-any.whl", hash = "sha256:b0bbe8b7a8174db732067b86e391262898f5f536589ea81efec6d35ceb829331", size = 25857, upload-time = "2026-04-08T04:33:06.769Z" },-]-[[package]]name = "types-networkx"version = "3.6.1.20260408"