Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
fix/File-input-formats · L-260924-577415 (#1320)
Louis Choquelee054df93861
diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.tomlindex bce5e373f..29f07a347 100644--- a/.drift/acks/pipelex-kernel-docs.toml+++ b/.drift/acks/pipelex-kernel-docs.toml@@ -1,8 +1,8 @@contract = "pipelex-kernel-docs"-digest = "sha256:3ef90940ed68b1c1c7f772cf97bcda3fc3d3d81bad30805cdc93ed5966350b2a"+digest = "sha256:3fde64b7f78e5a662bebb1f86757336c3dc437bee2e48dfa9d25b4d74865df8c"reviewed_by = "Louis Choquel"-reviewed_at = "2026-09-29T01:36:33Z"-rationale = "PipelexKernel.make and shape_inputs gained a required read_scope; the kernel doc now passes it in every example, documents the read scope beside storage_scope, and lists it among the run-constant fields of RunMetadata."+reviewed_at = "2026-10-03T09:31:38Z"+rationale = "clean-pass: assemble_llm_prompt now passes each registry image's mime_type into its prompt image, the way documents already carried theirs; the kernel doc describes the assemble functions' responsibilities and signatures, which are unchanged, so no doc change is needed."[trigger_files]"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"@@ -17,7 +17,7 @@ rationale = "PipelexKernel.make and shape_inputs gained a required read_scope; t"pipelex/kernel/img_gen_prompt.py" = "blob:e4f72040561929aa0d96ad5f00b1c4af652d3f77""pipelex/kernel/img_gen_results.py" = "blob:46b4ca11c70d906ae6d58ce66461f9b3414f1bfa""pipelex/kernel/llm_ops.py" = "blob:64a6d3d77a165f4f70614ced69876f56d873a8c7"-"pipelex/kernel/llm_prompt_content.py" = "blob:82c34283dacd2bfd9a2c263915e744736d399875"+"pipelex/kernel/llm_prompt_content.py" = "blob:3fade4a660226e63ad4f916952ee37d34e7d2bbd""pipelex/kernel/llm_results.py" = "blob:6e3b8a2878b5bb284ece5fac10262fb59ac2029d""pipelex/kernel/memory_ops.py" = "blob:7fbbfaaec99f9712d8e215520cb02e26aad3fb40""pipelex/kernel/pipelex_kernel.py" = "blob:5dec4e68d517d5ab1d7e9cbb91d836f37b10e713"diff --git a/.pipelex/inference/backends/internal.toml b/.pipelex/inference/backends/internal.tomlindex 1503a8457..e5444f72b 100644--- a/.pipelex/inference/backends/internal.toml+++ b/.pipelex/inference/backends/internal.toml@@ -43,7 +43,19 @@ costs = {}model_type = "text_extractor"sdk = "docling_sdk"model_id = "extract-text"-inputs = ["pdf", "image"]+inputs = [+ "pdf",+ "docx",+ "pptx",+ "xlsx",+ "html",+ "md",+ "csv",+ "txt",+ "vtt",+ "eml",+ "image",+]outputs = ["pages"]costs = {}diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0f4eef591..dba2ae784 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,6 +15,7 @@### Fixed+- **File inputs are checked against the models that read them (Breaking)**: run setup now identifies every image and document input from its first bytes, refuses an `Image` input holding something other than an image with `PipelineInputNotAnImageError`, and refuses a document certain to reach a `PipeExtract` or `PipeLLM` step whose model does not read its format with `PipelineInputFormatUnsupportedError`, both 422 input errors naming the input, the step, the model and the formats it reads; a step reached only through a condition refuses the file itself with `ExtractInputFormatError` or `PromptDocumentFormatError`. A `PipeLLM` document its model does not read now raises `PromptDocumentFormatError` where it raised `LLMCapabilityError`, so a consumer branching on `error_type` must follow. `docling-extract-text` now declares the Word, PowerPoint, Excel, HTML, Markdown, CSV, plain-text, WebVTT and email files it already extracted, which `pipelex update` installs.- **Markdown with unclosed emphasis is converted in memory proportional to its length**: the Markdown parser behind the `markdown` filter, the `Markdown` concept's HTML and the built-in PDF engine held memory growing with the square of the text when emphasis or strikethrough markers were left unmatched, about 1.5 GB for 64,000 characters of `**a `; it now joins the text they leave behind in one pass, and renders the same HTML.- **Routing profiles no longer rewrite their own routes**: matching a model merged the enabled `optional_routes` into the profile's declared `routes` in place, so after the first match a route declared optional had become an unconditional one in the loaded profile. Only the enablement checks each match repeats kept that from changing a routing decision.- **The logging documentation no longer promises that a server refuses the `console` sink**: the log-sink, logging, logging-configuration and Rich-imports pages said a server installed without the `cli` extra stops at boot when it selects the `console` sink or the `rich` pretty-print mode. The core dependencies `typer` and `instructor` install Rich anyway, so such a server boots on the Rich surface; the pages now say a server selects `json` and a Rich-free mode in its own configuration, since nothing at boot makes that choice for it.diff --git a/api/.pipelex/inference/backends/internal.toml b/api/.pipelex/inference/backends/internal.tomlindex 1503a8457..e5444f72b 100644--- a/api/.pipelex/inference/backends/internal.toml+++ b/api/.pipelex/inference/backends/internal.toml@@ -43,7 +43,19 @@ costs = {}model_type = "text_extractor"sdk = "docling_sdk"model_id = "extract-text"-inputs = ["pdf", "image"]+inputs = [+ "pdf",+ "docx",+ "pptx",+ "xlsx",+ "html",+ "md",+ "csv",+ "txt",+ "vtt",+ "eml",+ "image",+]outputs = ["pages"]costs = {}diff --git a/docs/building-methods/concepts/native-concepts.md b/docs/building-methods/concepts/native-concepts.mdindex de3e3df62..8a022bd6a 100644--- a/docs/building-methods/concepts/native-concepts.md+++ b/docs/building-methods/concepts/native-concepts.md@@ -27,8 +27,8 @@ Here are all the native concepts you can use out of the box:|-----------------|-------------|---------------------|| `Text` | A text | `TextContent` || `Markdown` | A text written in Markdown, which refines `Text` | `MarkdownContent` |-| `Image` | An image | `ImageContent` |-| `Document` | A document (PDF, DOCX, PPTX, web page) | `DocumentContent` |+| `Image` | An image file | `ImageContent` |+| `Document` | A document: PDF, Office (Word, PowerPoint, Excel), web page, or a text file such as Markdown or CSV | `DocumentContent` || `TextAndImages` | Text with its associated images | `TextAndImagesContent` || `Number` | A number | `NumberContent` || `YesNo` | The answer to a yes/no question | `YesNoContent` |@@ -155,15 +155,17 @@ class ImageContent(StuffContent):- `public_url`: A URL a viewer can open, which the runtime fills for every image input it normalizes, passed alone, in a list or in a structured field: the storage provider's link for a stored file (a `pipelex-storage://` reference, a `data:` URL or an uploaded local file), and the URL itself for an `http(s)` one unless the input names another. A stored file's link is signed when signed URLs are configured, so it expires: a template writing `{{ image.public_url }}` into HTML produces a report that stops showing the image once the link has expired.- `source_prompt` / `source_negative_prompt`: The prompts used to generate the image (if applicable)- `caption`: Descriptive text for the image-- `mime_type`: Optional MIME type of the image+- `mime_type`: The MIME type of the image. For every image input, the runtime sets it at the start of the run from the file's own bytes, which win over a declared type (see [File formats are checked before the run](#file-formats-are-checked-before-the-run))- `width` / `height`: Pixel dimensions — present together or not at all- `filename`: Optional original filename**Use for:** Photos, generated images, diagrams, screenshots.+An `Image` must hold an image file, such as PNG, JPEG or WebP. A run whose `Image` input, or an input of a concept refining `Image`, holds a file whose bytes say it is something else, a PDF for one, is refused before it starts. A scanned letter that arrives as a PDF belongs in a `Document` input.+### DocumentContent-Represents a document (PDF, DOCX, PPTX, etc.):+Represents a document: a PDF, an Office file (Word, PowerPoint, Excel), a web page, or a text file such as Markdown, CSV, plain text, WebVTT captions or an email message.```pythonclass DocumentContent(StuffContent):@@ -179,13 +181,30 @@ class DocumentContent(StuffContent):- `url`: Location of the document file, storage URL, or web page URL- `public_url`: A URL a viewer can open, filled by the runtime for every document input it normalizes, exactly as for `ImageContent`: the storage provider's link for a stored file, which expires when signed URLs are configured, and the URL itself for an `http(s)` one-- `mime_type`: Optional MIME type of the document (e.g., "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")+- `mime_type`: The MIME type of the document (e.g., "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"). For every document input, the runtime sets it at the start of the run from the file's own bytes, and keeps the declared type only when the bytes do not identify the file, as for plain text, Markdown or HTML- `filename`: Optional filename of the document- `title`: Optional title of the document or source- `snippet`: Optional text snippet or excerpt from the document**Use for:** Contracts, invoices, reports, presentations, web pages, search source citations, any document file.+Which document formats a method can take depends on the models that read them: a `PipeExtract` step reads the formats its extract model declares, and a `PipeLLM` step the document formats its model declares. A Word document works with an extract model that reads Word files, and fails with one that reads only PDF.++### File formats are checked before the run++At the start of every run, before any pipe runs, the runtime establishes the format of each image and document input, including the items of a list and the fields of a structured input:++- A file given as a `data:` URL, a local path or a `pipelex-storage://` reference is identified from its first bytes. The identified type replaces any declared one, so a `data:image/png` URL holding a PDF is a PDF. An Office file whose bytes show only that it is a zip archive is known by its declared type or by its file name's extension (`.docx`, `.pptx`, `.xlsx`), and otherwise its format is unknown.+- A file the bytes do not identify, such as plain text, Markdown, CSV or HTML, keeps the type it was declared with.+- An `http(s)` URL is not fetched before the run, so its format is the type it was declared with, and only that type is checked. An `http(s)` URL with no declared type is not checked: the provider receives it as it is.++The run is then refused, with an input error naming the input, when:++- an `Image` input holds a file that is not an image, whatever reads it;+- a document input is certain to reach a step whose model does not read its format: for example, Word transcripts batched into a `PipeExtract` whose extract model reads only PDF. The error names each input, its format, the step, the model and the formats that model reads.++A step reached only through a `PipeCondition`, or a step that may be skipped because an optional input is absent, may never run, so the run is not refused on its account. If it does run, it refuses the file itself, with the same kind of input error. The same holds for files a run produces along the way.+### NumberContentRepresents numeric values:diff --git a/docs/configuration/config-technical/inference-backend-config.md b/docs/configuration/config-technical/inference-backend-config.mdindex e0b121bd2..f0110a263 100644--- a/docs/configuration/config-technical/inference-backend-config.md+++ b/docs/configuration/config-technical/inference-backend-config.md@@ -336,6 +336,26 @@ costs = { input = 0.04, output = 0.0 }The `[defaults]` table applies to every model of the file, and a model table overrides any key of it.+#### Input formats++`inputs` lists what a model reads. For files, its entries are format keys, the same keys the runtime derives from a file's MIME type to check, before a run starts, that every file reaches a model able to read it: every image type is the `image` family, and any other type is its extension.++- **LLMs** declare `images` to read images (vision), and the document formats they read among `pdf`, `docx`, `pptx`, `xlsx` and `html`.+- **Extract models** declare the file formats they read among `pdf`, `docx`, `pptx`, `xlsx`, `html`, `md`, `csv`, `txt`, `vtt`, `eml` and `image`, and `web_page` when they fetch a web page from its URL themselves.++```toml+# internal.toml+[docling-extract-text]+model_type = "text_extractor"+sdk = "docling_sdk"+model_id = "extract-text"+inputs = ["pdf", "docx", "pptx", "xlsx", "html", "md", "csv", "txt", "vtt", "eml", "image"]+outputs = ["pages"]+costs = {}+```++A file whose format the consuming model does not declare is refused with an input error that names the input, the model and the formats it reads. A file whose format is unknown, with no type or the generic `application/octet-stream`, is left to the provider. Declare a format only once the model is known to read it.+#### Structure methods`structure_method` says how a model is asked for structured output. A structure method names a provider, but the SDK decides how the request is sent. Each method stands for one of `instructor`'s core modes: every `*_tools` method is tool calling, and so is `instructor/openai_structured_outputs`, which sends OpenAI a non-strict tool schema; `instructor/mistral_structured_outputs` or `instructor/openrouter_structured_outputs` is a JSON-schema response format. So a method named after another provider still works through the Gateway's OpenAI-compatible SDKs.diff --git a/docs/errors/execution-and-runtime.md b/docs/errors/execution-and-runtime.mdindex 9e6804845..8ed82dc00 100644--- a/docs/errors/execution-and-runtime.md+++ b/docs/errors/execution-and-runtime.md@@ -32,6 +32,9 @@ own page. Classes are grouped by subsystem.- [`PipeStackOverflowError`](pipe-stack-overflow-error.md) — Pipe stack overflow- [`PipelineExecutionError`](pipeline-execution-error.md) — Pipeline execution- [`PipelineInputContentError`](pipeline-input-content-error.md) — Pipeline input content+- [`PipelineInputFormatError`](pipeline-input-format-error.md) — Pipeline input format+- [`PipelineInputFormatUnsupportedError`](pipeline-input-format-unsupported-error.md) — Pipeline input format unsupported+- [`PipelineInputNotAnImageError`](pipeline-input-not-an-image-error.md) — Pipeline input not an image- [`PipelineInputUrlInvalidError`](pipeline-input-url-invalid-error.md) — Pipeline input url invalid- [`PipelineInputUrlMissingError`](pipeline-input-url-missing-error.md) — Pipeline input url missing- [`PipelineManagerAlreadyExistsError`](pipeline-manager-already-exists-error.md) — Pipeline manager already existsdiff --git a/docs/errors/extract-input-format-error.md b/docs/errors/extract-input-format-error.mdnew file mode 100644index 000000000..2a6f3aa5f--- /dev/null+++ b/docs/errors/extract-input-format-error.md@@ -0,0 +1,21 @@+---+title: "Extract input format"+description: "Reference for the `ExtractInputFormatError` Pipelex error class."+---++<!-- pipelex:generated -->++# Extract input format++The file given to an extraction has a known format that the extract model does not read.++| Field | Value |+|---|---|+| `error_type` | `ExtractInputFormatError` |+| `title` | Extract input format |+| `type_uri` | `https://docs.pipelex.com/latest/errors/extract-input-format-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`CogtError`](cogt-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/inference-and-providers.md b/docs/errors/inference-and-providers.mdindex 20d21eec7..fdad57de7 100644--- a/docs/errors/inference-and-providers.md+++ b/docs/errors/inference-and-providers.md@@ -23,6 +23,7 @@ own page. Classes are grouped by subsystem.- [`ExtractCapabilityError`](extract-capability-error.md) — Extract capability- [`ExtractHandleNotFoundError`](extract-handle-not-found-error.md) — Extract handle not found- [`ExtractInputError`](extract-input-error.md) — Extract input+- [`ExtractInputFormatError`](extract-input-format-error.md) — Extract input format- [`ExtractJobFailureError`](extract-job-failure-error.md) — Extract job failure- [`ExtractModelNotFoundError`](extract-model-not-found-error.md) — Extract model not found- [`ExtractOutputError`](extract-output-error.md) — Extract output@@ -69,6 +70,7 @@ own page. Classes are grouped by subsystem.- [`OutputStructureSchemaError`](output-structure-schema-error.md) — Output structure schema- [`PluginModelDeclarationError`](plugin-model-declaration-error.md) — Plugin model declaration- [`PromptDocumentFactoryError`](prompt-document-factory-error.md) — Prompt document factory+- [`PromptDocumentFormatError`](prompt-document-format-error.md) — Prompt document format- [`PromptImageFactoryError`](prompt-image-factory-error.md) — Prompt image factory- [`PromptImageFormatError`](prompt-image-format-error.md) — Prompt image format- [`ReportingManagerError`](reporting-manager-error.md) — Reporting managerdiff --git a/docs/errors/pipeline-input-format-error.md b/docs/errors/pipeline-input-format-error.mdnew file mode 100644index 000000000..0886ec115--- /dev/null+++ b/docs/errors/pipeline-input-format-error.md@@ -0,0 +1,22 @@+---+title: "Pipeline input format"+description: "Reference for the `PipelineInputFormatError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipeline input format++A file input's format is one its input slot, or the pipe that consumes it, cannot take.++| Field | Value |+|---|---|+| `error_type` | `PipelineInputFormatError` |+| `title` | Pipeline input format |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipeline-input-format-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.pipeline.exceptions` |+| Parent class | [`PipelineInputContentError`](pipeline-input-content-error.md) |+| `user_action` | `change_input` — Give the input a file in a format the method can read. |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipeline-input-format-unsupported-error.md b/docs/errors/pipeline-input-format-unsupported-error.mdnew file mode 100644index 000000000..966b6eb1e--- /dev/null+++ b/docs/errors/pipeline-input-format-unsupported-error.md@@ -0,0 +1,22 @@+---+title: "Pipeline input format unsupported"+description: "Reference for the `PipelineInputFormatUnsupportedError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipeline input format unsupported++A file input is certain to reach a pipe whose model cannot read its format.++| Field | Value |+|---|---|+| `error_type` | `PipelineInputFormatUnsupportedError` |+| `title` | Pipeline input format unsupported |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipeline-input-format-unsupported-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.pipeline.exceptions` |+| Parent class | [`PipelineInputFormatError`](pipeline-input-format-error.md) |+| `user_action` | `change_input` — Give each file input in a format the model of the pipe that consumes it reads. |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipeline-input-not-an-image-error.md b/docs/errors/pipeline-input-not-an-image-error.mdnew file mode 100644index 000000000..e4a1861a5--- /dev/null+++ b/docs/errors/pipeline-input-not-an-image-error.md@@ -0,0 +1,22 @@+---+title: "Pipeline input not an image"+description: "Reference for the `PipelineInputNotAnImageError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipeline input not an image++An Image input holds a file whose bytes identify it as something other than an image.++| Field | Value |+|---|---|+| `error_type` | `PipelineInputNotAnImageError` |+| `title` | Pipeline input not an image |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipeline-input-not-an-image-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.pipeline.exceptions` |+| Parent class | [`PipelineInputFormatError`](pipeline-input-format-error.md) |+| `user_action` | `change_input` — Give an image file, such as PNG, JPEG or WebP, to every Image input. |++[Back to Error Reference](index.md)diff --git a/docs/errors/prompt-document-format-error.md b/docs/errors/prompt-document-format-error.mdnew file mode 100644index 000000000..9b5b0225e--- /dev/null+++ b/docs/errors/prompt-document-format-error.md@@ -0,0 +1,21 @@+---+title: "Prompt document format"+description: "Reference for the `PromptDocumentFormatError` Pipelex error class."+---++<!-- pipelex:generated -->++# Prompt document format++A prompt document's known format is one the LLM does not read.++| Field | Value |+|---|---|+| `error_type` | `PromptDocumentFormatError` |+| `title` | Prompt document format |+| `type_uri` | `https://docs.pipelex.com/latest/errors/prompt-document-format-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.cogt.exceptions` |+| Parent class | [`CogtError`](cogt-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/features/document-extraction.md b/docs/features/document-extraction.mdindex aa34ae4ce..0d7929f5c 100644--- a/docs/features/document-extraction.md+++ b/docs/features/document-extraction.md@@ -24,6 +24,21 @@ Unlike LLM APIs (partly standardized around OpenAI's completions API), the OCR l| **Deepseek-OCR** | Gateway | Open-source model optimized for markdown extraction from images || **Linkup Fetch** | Cloud API | Web page content extraction — fetches and extracts text from web URLs |+## Formats Each Extractor Reads++Each extract model declares the file formats it reads, and a `PipeExtract` step can only extract a file in one of those formats.++| Extract model | Reads |+|---------------|-------|+| `pypdfium2-extract-pdf` | PDF |+| `docling-extract-text` | PDF, Word (`.docx`), PowerPoint (`.pptx`), Excel (`.xlsx`), HTML, Markdown (`.md`), CSV, plain text (`.txt`), WebVTT captions (`.vtt`), email messages (`.eml`) and images |+| Mistral OCR (`mistral-ocr` and its versions) | PDF and images |+| `azure-document-intelligence` (Pipelex Gateway) | PDF and images |+| `deepseek-ocr` (Pipelex Gateway) | Images |+| `linkup-fetch` (Pipelex Gateway) | Web pages, fetched from their URL |++A file in a format the step's model does not read is refused before the run starts, with an input error that names the input, its format, the step, the model and the formats that model reads. Give the file in a format the model reads, or choose a model that reads it. When the step can only be reached through a condition, the run starts, and the step refuses the file the same way when it gets to it. See [File formats are checked before the run](../building-methods/concepts/native-concepts.md#file-formats-are-checked-before-the-run).+## Key Capabilities- **Page view generation** — High-fidelity image rendering of extracted pages via pypdfium2diff --git a/pipelex/cogt/document/prompt_document.py b/pipelex/cogt/document/prompt_document.pyindex 27bd22540..3ad3beca2 100644--- a/pipelex/cogt/document/prompt_document.py+++ b/pipelex/cogt/document/prompt_document.py@@ -5,6 +5,7 @@ for document understanding. Follows the same pattern as PromptImage."""import base64+import binasciifrom functools import cached_propertyfrom typing import Annotated, Literal, Union@@ -13,10 +14,12 @@ from typing_extensions import overridefrom pipelex.tools.misc.attribute_utils import AttributePolisherfrom pipelex.tools.misc.filetype_utils import (+ FILE_HEAD_NB_BYTES,UNKNOWN_FILE_TYPE,FileType,detect_file_type_from_base64,detect_file_type_from_bytes,+ identify_mime_type,mime_type_to_extension,)from pipelex.tools.misc.hash_utils import hash_sha256@@ -68,6 +71,10 @@ class PromptDocumentUri(BaseModel):return mime_type_to_extension(self.mime_type)return UNKNOWN_FILE_TYPE+ def known_mime_type(self) -> str | None:+ """The MIME type this document is known to have, without loading it: the one run setup stamped, if any."""+ return self.mime_type+def get_content_hash(self, *, length: int | None = None) -> str:"""Return a hash of the document content."""return hash_sha256(self.uri, length=length)@@ -78,6 +85,8 @@ class PromptDocumentBase64(BaseModel):kind: Literal["base64"] = "base64"base64_data: str+ mime_type: str | None = None+ """The type run setup established or the data URL declared, which refines what the bytes alone identify."""def get_file_type(self) -> FileType:return detect_file_type_from_base64(self.base64_data)@@ -88,6 +97,18 @@ class PromptDocumentBase64(BaseModel):def get_decoded_bytes(self) -> bytes:return base64.b64decode(self.base64_data)+ def known_mime_type(self) -> str | None:+ """The MIME type this document is known to have, identified the way run setup identifies it.++ The bytes win when they identify a type, the declared type stands when they identify none or+ do not decode, and a declared zip-based type refines bytes that show only a zip.+ """+ try:+ raw_bytes = self.get_decoded_bytes()+ except binascii.Error:+ return self.mime_type+ return identify_mime_type(head=raw_bytes[:FILE_HEAD_NB_BYTES], declared_mime_type=self.mime_type, file_name=None)+@overridedef __str__(self) -> str:truncated_base64 = AttributePolisher.get_truncated_value(value=self.base64_data)@@ -119,6 +140,8 @@ class PromptDocumentBinary(BaseModel):kind: Literal["binary"] = "binary"raw_bytes: bytes+ mime_type: str | None = None+ """The type the caller declared, which refines what the bytes alone identify."""def get_file_type(self) -> FileType:return detect_file_type_from_bytes(self.raw_bytes)@@ -126,6 +149,10 @@ class PromptDocumentBinary(BaseModel):def get_mime_type(self) -> str:return self.get_file_type().mime+ def known_mime_type(self) -> str | None:+ """The MIME type this document is known to have, identified the way run setup identifies it."""+ return identify_mime_type(head=self.raw_bytes[:FILE_HEAD_NB_BYTES], declared_mime_type=self.mime_type, file_name=None)+@overridedef __str__(self) -> str:return "PromptDocumentBinary(raw_bytes=...)"diff --git a/pipelex/cogt/document/prompt_document_factory.py b/pipelex/cogt/document/prompt_document_factory.pyindex 0afe8cb83..d619c49a1 100644--- a/pipelex/cogt/document/prompt_document_factory.py+++ b/pipelex/cogt/document/prompt_document_factory.py@@ -24,7 +24,9 @@ class PromptDocumentFactory:uri: A URI string (file path, HTTP URL, pipelex-storage://, or data: URL)base64_data: Base64 string (with or without data: prefix)raw_bytes: Raw binary document data- mime_type: Optional MIME type for the document (only used with uri)+ mime_type: The document's MIME type, as run setup established it or the caller declared it.+ For inline bytes it refines what the bytes alone identify; a data URL's own type is+ the fallback.Returns:A PromptDocument instance (PromptDocumentUri, PromptDocumentBase64, or PromptDocumentBinary)@@ -33,16 +35,16 @@ class PromptDocumentFactory:PromptDocumentFactoryError: If no valid input is provided"""if raw_bytes:- return PromptDocumentBinary(raw_bytes=raw_bytes)+ return PromptDocumentBinary(raw_bytes=raw_bytes, mime_type=mime_type)if base64_data:stripped_base64_data = strip_base64_str_if_needed(base64_data)- return PromptDocumentBase64(base64_data=stripped_base64_data)+ return PromptDocumentBase64(base64_data=stripped_base64_data, mime_type=mime_type)if uri:# Check if it's a data URL and extract base64 data to avoid URL_MAX_LENGTH validationextracted = extract_base64_str_from_base64_url_if_possible(uri)if extracted is not None:- extracted_base64_data, _mime_type = extracted- return PromptDocumentBase64(base64_data=extracted_base64_data)+ extracted_base64_data, data_url_mime_type = extracted+ return PromptDocumentBase64(base64_data=extracted_base64_data, mime_type=mime_type or data_url_mime_type)return PromptDocumentUri(uri=uri, mime_type=mime_type)msg = "PromptDocumentFactory requires one of: uri, base64_data, or raw_bytes"raise PromptDocumentFactoryError(msg)diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 7dc113913..a2905df86 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -411,6 +411,17 @@ class PromptDocumentFactoryError(CogtError):error_category = InferenceErrorCategory.CONTENT+class PromptDocumentFormatError(CogtError):+ """A prompt document's known format is one the LLM does not read.++ A content error, and so in the input domain: the model is fixed by the method and reads the+ formats it declares, while the file changes from run to run. A model that reads no documents at+ all is the author's choice of model, which stays an `LLMCapabilityError`.+ """++ error_category = InferenceErrorCategory.CONTENT++class ImgGenModelNotFoundError(ModelNotFoundError):pass@@ -435,6 +446,17 @@ class ExtractCapabilityError(CogtError):error_category = InferenceErrorCategory.CONFIGURATION+class ExtractInputFormatError(CogtError):+ """The file given to an extraction has a known format that the extract model does not read.++ A content error, and so in the input domain: the model is fixed by the method and reads the+ formats it declares, while the file changes from run to run, and the person who can act is the+ one supplying it.+ """++ error_category = InferenceErrorCategory.CONTENT++class ExtractJobFailureError(CogtError):passdiff --git a/pipelex/cogt/extract/extract_input.py b/pipelex/cogt/extract/extract_input.pyindex ecc6a1299..aa4f4c7b9 100644--- a/pipelex/cogt/extract/extract_input.py+++ b/pipelex/cogt/extract/extract_input.py@@ -8,8 +8,17 @@ from pipelex.tools.uri.uri_read_scope import UriReferenceclass ExtractInput(BaseModel):+ """The file an extraction reads: an image or a document, with what is known of it.++ `mime_type` is the file's identified type, which run setup stamps on every file input, and+ `input_name` the input it came from, as a caller writes it (`transcripts[2]`), for messages.+ Both are `None` when unknown, and the worker then leaves the format to the provider.+ """+image_uri: str | None = Nonedocument_uri: str | None = None+ mime_type: str | None = None+ input_name: str | None = None@model_validator(mode="after")def validate_at_exactly_one_input(self) -> Self:diff --git a/pipelex/cogt/extract/extract_worker_abstract.py b/pipelex/cogt/extract/extract_worker_abstract.pyindex 3df1c983e..5a02af972 100644--- a/pipelex/cogt/extract/extract_worker_abstract.py+++ b/pipelex/cogt/extract/extract_worker_abstract.py@@ -4,7 +4,8 @@ from typing import Anyfrom typing_extensions import overridefrom pipelex import log-from pipelex.cogt.exceptions import CogtError, ExtractCapabilityError+from pipelex.cogt.exceptions import CogtError, ExtractCapabilityError, ExtractInputFormatError+from pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job import ExtractJobfrom pipelex.cogt.extract.extract_output import ExtractOutputfrom pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract@@ -12,6 +13,12 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategoryfrom pipelex.reporting.reporting_protocol import ReportingProtocolfrom pipelex.system.job_metadata import UnitJobId+from pipelex.tools.misc.filetype_utils import IMAGE_FORMAT_KEY, describe_file_format, describe_format_keys, format_key_from_mime_type+from pipelex.tools.uri.resolved_uri import ResolvedHttpUrl+from pipelex.tools.uri.uri_resolver import resolve_uri++# How a web-page model's declaration is named among the formats it reads, in a refusal.+_WEB_PAGE_INPUT: str = "web_page"class ExtractWorkerAbstract(InferenceWorkerAbstract):@@ -65,6 +72,42 @@ class ExtractWorkerAbstract(InferenceWorkerAbstract):if not self.inference_model.is_caption_supported_for_extract:msg = f"Extract engine '{self.inference_model.tag}' does not support image captioning."raise ExtractCapabilityError(msg)+ self._check_input_format(extract_input=extract_input)++ def _check_input_format(self, *, extract_input: ExtractInput) -> None:+ """Refuse a file whose known format the model does not read, naming the input and what the model reads.++ An unknown format is left to the provider. A web-page model given an http(s) URL fetches the+ page itself, so its format is not checked. A file given as an image must be an image,+ whatever else the model reads.++ Raises:+ ExtractInputFormatError: If the file's format is known and the model does not read it.+ """+ format_key = format_key_from_mime_type(mime_type=extract_input.mime_type)+ if format_key is None:+ return+ if (+ extract_input.document_uri is not None+ and self.inference_model.is_web_page_supported_for_extract+ and isinstance(resolve_uri(extract_input.document_uri), ResolvedHttpUrl)+ ):+ return+ readable_formats = self.inference_model.readable_formats_for_extract+ if extract_input.image_uri is not None:+ readable_formats &= {IMAGE_FORMAT_KEY}+ if format_key in readable_formats:+ return+ listed_formats = set(readable_formats)+ if extract_input.document_uri is not None and self.inference_model.is_web_page_supported_for_extract:+ listed_formats.add(_WEB_PAGE_INPUT)+ subject = f"Input '{extract_input.input_name}' is" if extract_input.input_name else "The file to extract is"+ msg = (+ f"{subject} {describe_file_format(format_key=format_key, mime_type=extract_input.mime_type)}, "+ f"which model '{self.inference_model.name}' cannot extract: it reads {describe_format_keys(format_keys=listed_formats)}. "+ "Give a file in one of those formats, or use an extract model that reads this one."+ )+ raise ExtractInputFormatError(msg)async def extract_pages(self,diff --git a/pipelex/cogt/image/prompt_image.py b/pipelex/cogt/image/prompt_image.pyindex 2c4638da7..7f13ff412 100644--- a/pipelex/cogt/image/prompt_image.py+++ b/pipelex/cogt/image/prompt_image.py@@ -1,4 +1,5 @@import base64+import binasciifrom enum import StrEnumfrom functools import cached_propertyfrom typing import Annotated, Literal, Union@@ -12,6 +13,7 @@ from pipelex.tools.misc.filetype_utils import (FileType,detect_file_type_from_base64,detect_file_type_from_bytes,+ guess_file_type_from_bytes,mime_type_to_extension,)from pipelex.tools.misc.http_utils import URL_MAX_LENGTH@@ -78,6 +80,10 @@ class PromptImageUri(BaseModel):return mime_type_to_extension(self.mime_type)return UNKNOWN_FILE_TYPE+ def known_mime_type(self) -> str | None:+ """The MIME type this image is known to have, without loading it: the one run setup stamped, if any."""+ return self.mime_type+class PromptImageBase64(BaseModel):"""A prompt image as base64-encoded string."""@@ -94,6 +100,15 @@ class PromptImageBase64(BaseModel):def get_decoded_bytes(self) -> bytes:return base64.b64decode(self.base64_data)+ def known_mime_type(self) -> str | None:+ """The MIME type the bytes identify, or `None` when they identify none or do not decode."""+ try:+ raw_bytes = self.get_decoded_bytes()+ except binascii.Error:+ return None+ file_type = guess_file_type_from_bytes(raw_bytes=raw_bytes)+ return file_type.mime if file_type else None+@overridedef __str__(self) -> str:truncated_base64 = AttributePolisher.get_truncated_value(value=self.base64_data)@@ -124,6 +139,11 @@ class PromptImageBinary(BaseModel):def get_mime_type(self) -> str:return self.get_file_type().mime+ def known_mime_type(self) -> str | None:+ """The MIME type the bytes identify, or `None` when they identify none."""+ file_type = guess_file_type_from_bytes(raw_bytes=self.raw_bytes)+ return file_type.mime if file_type else None+@overridedef __str__(self) -> str:return "PromptImageBinary(raw_bytes=...)"diff --git a/pipelex/cogt/image/prompt_image_factory.py b/pipelex/cogt/image/prompt_image_factory.pyindex d8dc85bdc..9eec1e279 100644--- a/pipelex/cogt/image/prompt_image_factory.py+++ b/pipelex/cogt/image/prompt_image_factory.py@@ -15,13 +15,17 @@ from pipelex.tools.misc.base64_utils import (class PromptImageFactory:@classmethod- def make_prompt_image(cls, *, uri: str | None = None, base64_data: str | None = None, raw_bytes: bytes | None = None) -> PromptImage:+ def make_prompt_image(+ cls, *, uri: str | None = None, base64_data: str | None = None, raw_bytes: bytes | None = None, mime_type: str | None = None+ ) -> PromptImage:"""Create a PromptImage from the provided input.Args:uri: A URI string (file path, HTTP URL, pipelex-storage://, or data: URL)base64_data: Base64 string (with or without data: prefix)raw_bytes: Raw binary image data+ mime_type: The image's MIME type when it is known (only used with a uri other than a data URL,+ whose bytes carry their own type)Returns:A PromptImage instance (PromptImageUri, PromptImageBase64, or PromptImageBinary)@@ -40,6 +44,6 @@ class PromptImageFactory:if extracted is not None:extracted_base64_data, _mime_type = extractedreturn PromptImageBase64(base64_data=extracted_base64_data)- return PromptImageUri(uri=uri)+ return PromptImageUri(uri=uri, mime_type=mime_type)msg = "PromptImageFactory requires one of: uri, base64_data, or raw_bytes"raise PromptImageFactoryError(msg)diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.pyindex 6a6477297..53aead7cd 100644--- a/pipelex/cogt/llm/llm_worker_abstract.py+++ b/pipelex/cogt/llm/llm_worker_abstract.py@@ -9,7 +9,7 @@ from opentelemetry.trace import NonRecordingSpan, Span, SpanContext, SpanKind, Sfrom typing_extensions import overridefrom pipelex import log-from pipelex.cogt.exceptions import CogtError, LLMCapabilityError+from pipelex.cogt.exceptions import CogtError, LLMCapabilityError, PromptDocumentFormatError, PromptImageFormatErrorfrom pipelex.cogt.inference.inference_constants import InferenceOutputTypefrom pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstractfrom pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedConstraint@@ -28,7 +28,12 @@ from pipelex.system.telemetry.otel_constants import (from pipelex.system.telemetry.otel_factory import OtelFactoryfrom pipelex.system.telemetry.telemetry_identity import make_run_identity_span_attributesfrom pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract-from pipelex.tools.misc.filetype_utils import UNKNOWN_FILE_TYPE+from pipelex.tools.misc.filetype_utils import (+ IMAGE_FORMAT_KEY,+ describe_file_format,+ describe_format_keys,+ format_key_from_mime_type,+)from pipelex.tools.misc.package_utils import get_package_versionif TYPE_CHECKING:@@ -434,6 +439,20 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):msg = f"LLM Engine '{self.inference_model.tag}' does not accept that many images: {nb_images}."raise LLMCapabilityError(msg)+ # Run setup refuses a non-image given to an Image input, but an image the run produced+ # itself never went through setup: this is the line that catches it before the provider.+ # Only a known format is refused; an unknown one is left to the provider.+ for image_index, prompt_image in enumerate(llm_job.llm_prompt.user_images, start=1):+ mime_type = prompt_image.known_mime_type()+ format_key = format_key_from_mime_type(mime_type=mime_type)+ if format_key is None or format_key == IMAGE_FORMAT_KEY:+ continue+ msg = (+ f"Prompt image {image_index} given to model '{self.inference_model.name}' is "+ f"{describe_file_format(format_key=format_key, mime_type=mime_type)}, not an image."+ )+ raise PromptImageFormatError(msg)+def _check_document_support(self, llm_job: LLMJob):if not llm_job.llm_prompt.user_documents:return@@ -442,16 +461,21 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):msg = f"LLM Engine '{self.inference_model.tag}' does not support documents."raise LLMCapabilityError(msg)- # Check each document's type is supported+ # Each document's known format must be one the model reads. The file is the caller's and+ # changes from run to run, so a format the model does not read is an input error; an unknown+ # format is left to the provider.supported = self.inference_model.supported_document_types- for doc in llm_job.llm_prompt.user_documents:- doc_type = doc.get_document_type()- # Skip validation for unknown types - let the provider handle it- if doc_type == UNKNOWN_FILE_TYPE:+ for document_index, prompt_document in enumerate(llm_job.llm_prompt.user_documents, start=1):+ mime_type = prompt_document.known_mime_type()+ format_key = format_key_from_mime_type(mime_type=mime_type)+ if format_key is None or format_key in supported:continue- if doc_type not in supported:- msg = f"LLM Engine '{self.inference_model.tag}' does not support {doc_type} documents."- raise LLMCapabilityError(msg)+ msg = (+ f"Prompt document {document_index} given to model '{self.inference_model.name}' is "+ f"{describe_file_format(format_key=format_key, mime_type=mime_type)}, which it does not read: "+ f"it reads {describe_format_keys(format_keys=supported)}."+ )+ raise PromptDocumentFormatError(msg)async def gen_text(self,diff --git a/pipelex/cogt/model_backends/model_spec.py b/pipelex/cogt/model_backends/model_spec.pyindex 966bbfb13..fc062a491 100644--- a/pipelex/cogt/model_backends/model_spec.py+++ b/pipelex/cogt/model_backends/model_spec.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Finalfrom pydantic import Field@@ -9,12 +9,19 @@ from pipelex.cogt.model_backends.constraints import ListedConstraint, ValuedConsfrom pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.usage.cost_category import CostsByCategoryDictfrom pipelex.system.configuration.config_model import ConfigModel+from pipelex.tools.misc.filetype_utils import IMAGE_FORMAT_KEYfrom pipelex.tools.typing.pydantic_utils import empty_dict_factory_of, empty_list_factory_ofif TYPE_CHECKING:from instructor import Mode as InstructorMode+# The file formats a model spec's `inputs` may declare, as the format keys every format check compares+# (see `format_key_from_mime_type`). An LLM's vision flag stays spelled `images`, outside this vocabulary.+_EXTRACT_FILE_FORMATS: Final[frozenset[str]] = frozenset({"pdf", "docx", "pptx", "xlsx", "html", "md", "csv", "txt", "vtt", "eml", IMAGE_FORMAT_KEY})+_LLM_DOCUMENT_FORMATS: Final[frozenset[str]] = frozenset({"pdf", "docx", "pptx", "xlsx", "html"})++class InferenceModelSpec(ConfigModel):backend_name: strname: str@@ -67,11 +74,19 @@ class InferenceModelSpec(ConfigModel):def is_caption_supported_for_extract(self) -> bool:return "captions" in self.outputs+ @property+ def readable_formats_for_extract(self) -> set[str]:+ """The file formats this extractor reads, as format keys: its declared pdf, docx, pptx, xlsx, html, md, csv, txt, vtt, eml and image.++ `web_page` is not among them: a web-page model fetches its page itself, from a URL, so it is+ not a format a file is checked against.+ """+ return set(self.inputs) & _EXTRACT_FILE_FORMATS+@propertydef supported_document_types(self) -> set[str]:- """Return set of supported document types (pdf, docx, pptx) from inputs."""- document_types = {"pdf", "docx", "pptx"}- return document_types & set(self.inputs)+ """The document formats this LLM reads, as format keys: its declared pdf, docx, pptx, xlsx and html."""+ return set(self.inputs) & _LLM_DOCUMENT_FORMATS@propertydef is_document_supported(self) -> bool:diff --git a/pipelex/kernel/llm_prompt_content.py b/pipelex/kernel/llm_prompt_content.pyindex 82c34283d..3fade4a66 100644--- a/pipelex/kernel/llm_prompt_content.py+++ b/pipelex/kernel/llm_prompt_content.py@@ -229,7 +229,11 @@ async def assemble_llm_prompt(############################################################# The registry contains all images (direct + nested) in the correct order,# already deduplicated by URL. This ensures [Image N] tokens match positions.- all_images: list[PromptImage] = [PromptImageFactory.make_prompt_image(uri=registry_image.url) for registry_image in image_registry.images]+ # Each keeps the format its preparation established, the way a document does, so the worker checks+ # the format instead of guessing it again.+ all_images: list[PromptImage] = [+ PromptImageFactory.make_prompt_image(uri=registry_image.url, mime_type=registry_image.mime_type) for registry_image in image_registry.images+ ]############################################################# Collect all documentsdiff --git a/pipelex/kit/configs/inference/backends/internal.toml b/pipelex/kit/configs/inference/backends/internal.tomlindex 1503a8457..e5444f72b 100644--- a/pipelex/kit/configs/inference/backends/internal.toml+++ b/pipelex/kit/configs/inference/backends/internal.toml@@ -43,7 +43,19 @@ costs = {}model_type = "text_extractor"sdk = "docling_sdk"model_id = "extract-text"-inputs = ["pdf", "image"]+inputs = [+ "pdf",+ "docx",+ "pptx",+ "xlsx",+ "html",+ "md",+ "csv",+ "txt",+ "vtt",+ "eml",+ "image",+]outputs = ["pages"]costs = {}diff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.pyindex 2583e3bb1..bc07b4b8c 100644--- a/pipelex/pipe_operators/extract/pipe_extract.py+++ b/pipelex/pipe_operators/extract/pipe_extract.py@@ -8,7 +8,7 @@ from pipelex.cogt.extract.extract_setting import ExtractModelChoicefrom pipelex.cogt.models.model_deck_check import check_extract_choice_with_deckfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipes.exceptions import PipeValidationError+from pipelex.core.pipes.exceptions import PipeRunError, PipeValidationErrorfrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.interpreter_hub import get_concept_library, get_native_concept@@ -110,14 +110,18 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeExtractOutput:- image_uri: str | None = None- pdf_uri: str | None = None+ # The format run setup established for the file rides along, with the input's name, so the+ # worker checks it against its model and names the input when it refuses.+ extract_input: ExtractInputif self.image_stuff_name:image_stuff = working_memory.get_stuff_as_image(name=self.image_stuff_name)- image_uri = image_stuff.url+ extract_input = ExtractInput(image_uri=image_stuff.url, mime_type=image_stuff.mime_type, input_name=self.image_stuff_name)elif self.document_stuff_name:document_stuff = working_memory.get_stuff_as_document(name=self.document_stuff_name)- pdf_uri = document_stuff.url+ extract_input = ExtractInput(document_uri=document_stuff.url, mime_type=document_stuff.mime_type, input_name=self.document_stuff_name)+ else:+ msg = f"PipeExtract '{self.code}' has neither an image nor a document input"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)# The deck chain and the job-params composition are kernel semantics; the setting is resolved# per run into a local and never cached onto `self`, for the reason `pipe_llm.py` states about@@ -134,7 +138,7 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):)extract_result = await run_extract(memory=working_memory,- extract_input=ExtractInput(image_uri=image_uri, document_uri=pdf_uri),+ extract_input=extract_input,extract_setting=extract_setting,extract_job_params=extract_job_params,concept=self.output.concept,diff --git a/pipelex/pipeline/exceptions.py b/pipelex/pipeline/exceptions.pyindex 6054932af..bfd4890a9 100644--- a/pipelex/pipeline/exceptions.py+++ b/pipelex/pipeline/exceptions.py@@ -413,6 +413,53 @@ class PipelineInputUrlInvalidError(PipelineInputContentError):)+class PipelineInputFormatError(PipelineInputContentError):+ """A file input's format is one its input slot, or the pipe that consumes it, cannot take.++ Raised at run setup, after the setup established each file input's format from its bytes, so the+ run is refused before any pipe spends anything on it. Caller-facing on purpose, and the flag spans+ every child and every raise site: a message in this family carries only facts the caller supplied+ (the input's path in the inputs, the file's identified type) or the method declares (a pipe, a+ model, the formats that model reads), never a url, a storage key or a server-side reason.+ """++ _authors_caller_facing_message = True+ user_action = UserAction(+ kind=UserActionKind.CHANGE_INPUT,+ detail="Give the input a file in a format the method can read.",+ )+++class PipelineInputNotAnImageError(PipelineInputFormatError):+ """An Image input holds a file whose bytes identify it as something other than an image.++ Refused whatever will consume it: an Image holding a PDF is wrong for every consumer, so the check+ needs no model. It fires only on a positive identification, so an image the sniffer cannot+ identify, an SVG for one, is left to its consumer.+ """++ user_action = UserAction(+ kind=UserActionKind.CHANGE_INPUT,+ detail="Give an image file, such as PNG, JPEG or WebP, to every Image input.",+ )+++class PipelineInputFormatUnsupportedError(PipelineInputFormatError):+ """A file input is certain to reach a pipe whose model cannot read its format.++ Raised at run setup by the pre-run consumer check, which walks the entry pipe to find the+ operators each input reaches. It refuses only a consumer that is certain to run, reached through+ no condition and no liftable step, and resolved to a model that does not read the file's known+ format. One error lists every violation, each naming the input, the format, the pipe, the model+ and the formats that model reads.+ """++ user_action = UserAction(+ kind=UserActionKind.CHANGE_INPUT,+ detail="Give each file input in a format the model of the pipe that consumes it reads.",+ )++class PipelineInputUrlMissingError(PipelineInputContentError):"""An Image/Document input carries a blank url.diff --git a/pipelex/pipeline/execution_seams.py b/pipelex/pipeline/execution_seams.pyindex bbe14b1e2..1dc7f9a9f 100644--- a/pipelex/pipeline/execution_seams.py+++ b/pipelex/pipeline/execution_seams.py@@ -13,7 +13,7 @@ Two pure-ish building blocks composed by both the single-run wrapper``main_pipe`` (when an ``mthds_contents`` bundle declares one), leaving piperesolution to the caller.- :func:`prepare_pipe_job` — build a :class:`PipeJob` against an already-open- library: working memory (user inputs, mock inputs, data-url normalization),+ library: working memory (user inputs, mock inputs, file-input preparation),run params, job metadata, and the library crate. **Pure**: no pipeline-managerregistration, no report-registry open, no telemetry, no graph-tracer open, nolibrary mutation.@@ -46,7 +46,8 @@ from pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import VariableMultiplicityfrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.pipeline.blueprint_selection import select_primary_blueprint-from pipelex.pipeline.input_normalizer import normalize_data_urls_to_storage+from pipelex.pipeline.file_input_consumers import check_file_inputs_against_consumers+from pipelex.pipeline.input_normalizer import collect_file_inputs, prepare_file_inputsfrom pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_error, withholding_host_library_filesfrom pipelex.system.configuration.configs import PipelineExecutionConfigfrom pipelex.system.job_metadata import JobMetadata, OtelContext, RunMetadata@@ -213,7 +214,7 @@ async def prepare_pipe_job("""Build a :class:`PipeJob` for ``pipe`` against an already-open library.Pure: assembles working memory (user inputs, mock inputs when- ``execution_config.is_mock_inputs``, optional data-url normalization), run+ ``execution_config.is_mock_inputs``, file-input identification and optional relocation), runparams, job metadata, and the library crate. Performs no pipeline-managerregistration, no report-registry open, no telemetry, no graph-tracer open,and no library mutation. ``trace_context`` / ``otel_context`` are created by@@ -231,7 +232,7 @@ async def prepare_pipe_job(``read_scope`` bounds what the run may read (see :mod:`pipelex.tools.uri.uri_read_scope`). It isrequired, ``None`` for an unscoped run, and it rides the job metadata to every leaf. Here it alsogates the two reads the input seam makes itself: a CSV input read while shaping, and a local file- uploaded by the normalization, which on a scoped run is refused instead, naming the input.+ read or uploaded by the file-input preparation, which on a scoped run is refused instead, naming the input."""# Validate the scope HERE, before anything composes a storage key from it.#@@ -329,9 +330,19 @@ async def prepare_pipe_job())- # Normalize data URLs to pipelex-storage:// URIs, and give every image and document input a public_url, if configured.- if working_memory and execution_config.is_normalize_data_urls_to_storage and not execution_config.is_mock_inputs:- working_memory = await normalize_data_urls_to_storage(working_memory, storage_scope=storage_scope, read_scope=read_scope)+ # Establish the format of every image and document input, which runs whether or not relocation is+ # configured, and relocate them to storage with a public_url when it is. Mock inputs are skipped:+ # they are placeholders, not files.+ if working_memory and not execution_config.is_mock_inputs:+ working_memory = await prepare_file_inputs(+ working_memory,+ storage_scope=storage_scope,+ read_scope=read_scope,+ is_relocation_enabled=execution_config.is_normalize_data_urls_to_storage,+ )+ # Refuse the run now, rather than at the step that would fail mid-run, when a file input is+ # certain to reach a pipe whose model cannot read its format.+ check_file_inputs_against_consumers(entry_pipe=pipe, file_inputs=collect_file_inputs(working_memory))job_metadata = JobMetadata(run_metadata=RunMetadata(diff --git a/pipelex/pipeline/file_input_consumers.py b/pipelex/pipeline/file_input_consumers.pynew file mode 100644index 000000000..fcda2d876--- /dev/null+++ b/pipelex/pipeline/file_input_consumers.py@@ -0,0 +1,573 @@+"""Which operators will consume each input slot's files, found by walking the entry pipe statically.++Run setup uses it to refuse, before the run starts, a file input that is certain to reach a consumer+that cannot read its format. The walk starts from the entry pipe's input slots and follows them by+name through the controllers, the way the absence-taint analysis does (`PipeSequence.analyze_taint`),+resolving sub-pipes through the hub:++- **Sequence.** Steps are visited in order, and a slot a step overwrites stops being followed after+ that step. A step overwrites its result's name, and also whatever a nested sequence or condition+ outcome writes, since those run on the caller's memory; after a step that may write a name the+ walk cannot know, nothing more is followed. A step's batch parameters map the list slot to its+ item slot.+- **Parallel.** Every branch is visited.+- **Batch.** The list slot maps to the item slot, and the branch pipe is visited.+- **Condition.** Every outcome is visited, and whatever is found below it is conditional.+- **Liftable steps.** A step the absence-taint analysis marks as liftable may be skipped, so what it+ consumes is conditional too. The analysis's own verdicts are reused, never re-derived.+- **Opaque pipes.** A `PipeFunc`, a `PipeCompose`, a search pipe, any other operator, and an+ unresolved cross-package reference consume nothing as far as the walk knows: it never guesses.+- **Consumers.** A `PipeExtract` consumes its document input, and reads the formats of the model its+ extract choice resolves to. A `PipeLLM` consumes the documents its prompt references by variable+ path, and reads its resolved model's document types. A waterfall reads a format when any of its+ members reads it.++The walk depends only on the library and the model deck, never on the input values.+"""++from enum import StrEnum+from typing import Final, NamedTuple++from pydantic import BaseModel, ConfigDict++from pipelex.cogt.exceptions import ModelChoiceNotFoundError+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.exceptions import ModelReferenceParseError+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.interpreter_hub import get_concept_library, get_native_concept, get_optional_pipe+from pipelex.kernel.extract_ops import resolve_extract_setting+from pipelex.kernel.llm_ops import resolve_llm_setting_for_object, resolve_llm_setting_for_text+from pipelex.pipe_controllers.batch.pipe_batch import PipeBatch+from pipelex.pipe_controllers.condition.pipe_condition import PipeCondition+from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel+from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequence+from pipelex.pipe_controllers.sub_pipe import SubPipe+from pipelex.pipe_machinery.pipe_abstract import PipeAbstract+from pipelex.pipe_operators.extract.pipe_extract import PipeExtract+from pipelex.pipe_operators.llm.pipe_llm import PipeLLM+from pipelex.pipe_run.pipe_run_params import BatchParams+from pipelex.pipeline.exceptions import PipelineInputFormatUnsupportedError+from pipelex.pipeline.input_normalizer import IdentifiedFileInput+from pipelex.runtime_hub import get_model_deck+from pipelex.tools.misc.filetype_utils import describe_file_format, describe_format_keys+from pipelex.tools.uri.resolved_uri import UriKind++# The segment of a consumed path that stands for any item of a list: `("transcripts", "[]")`+# is every item of `transcripts`. A field name never contains brackets, so it cannot collide.+LIST_ITEM_SEGMENT: Final[str] = "[]"++# A model reference resolves through presets, aliases and waterfalls in a handful of hops; this+# only stops a malformed deck that points a reference back at itself.+_MAX_PRESET_HOPS: Final[int] = 8++# How a refusal names a web-page model's declaration among the formats it reads.+_WEB_PAGE_INPUT: Final[str] = "web_page"++# What a variable of the current frame stands for: the path, from an entry input slot, of the+# value it holds. `{"transcript": ("transcripts", "[]")}` inside a batch over `transcripts`.+_Frame = dict[str, tuple[str, ...]]+++class FileConsumerKind(StrEnum):+ """How an operator consumes a file."""++ EXTRACT = "extract"+ LLM_DOCUMENT = "llm_document"++ @property+ def model_type(self) -> ModelType:+ match self:+ case FileConsumerKind.EXTRACT:+ return ModelType.TEXT_EXTRACTOR+ case FileConsumerKind.LLM_DOCUMENT:+ return ModelType.LLM+++class FileInputConsumer(BaseModel):+ """One operator that consumes the files of an entry input slot, with what its model reads."""++ model_config = ConfigDict(frozen=True)++ slot_name: str+ """The entry pipe's input slot the files come from."""++ consumed_path: tuple[str, ...]+ """The path, from the slot, of what the operator consumes: `("transcripts", "[]")` is every item."""++ pipe_ref: str+ pipe_code: str+ kind: FileConsumerKind++ model: str | None+ """The model the consumer resolves to: a concrete handle, or the waterfall it names. `None` when unresolved."""++ readable_formats: frozenset[str] | None+ """The format keys the model reads; `None` when the model could not be resolved, which makes the consumer opaque."""++ reads_web_pages: bool+ """Whether the model fetches a web page itself from an http(s) URL, which is then not format-checked."""++ is_conditional: bool+ """Whether the consumer is reached through a condition or a liftable step, so it may not run at all."""++ @property+ def model_type(self) -> ModelType:+ return self.kind.model_type++ @property+ def display_consumed_path(self) -> str:+ """The consumed path as a caller reads it: `transcripts[]`, `case.attachment`."""+ rendered = ""+ for segment in self.consumed_path:+ if segment == LIST_ITEM_SEGMENT:+ rendered += LIST_ITEM_SEGMENT+ elif rendered:+ rendered += f".{segment}"+ else:+ rendered = segment+ return rendered++ @property+ def listed_formats(self) -> set[str]:+ """What a refusal says the model reads: its file formats, and web pages when it fetches them."""+ listed_formats = set(self.readable_formats or ())+ if self.reads_web_pages:+ listed_formats.add(_WEB_PAGE_INPUT)+ return listed_formats++ @property+ def verb_phrase(self) -> str:+ """How a refusal says what the pipe does with the file."""+ match self.kind:+ case FileConsumerKind.EXTRACT:+ return "extracts it with"+ case FileConsumerKind.LLM_DOCUMENT:+ return "gives it to"++ def covers(self, *, file_input_path: tuple[str | int, ...]) -> bool:+ """Whether a file at this path in the inputs is among what the consumer consumes.++ A list index matches the item segment, and a consumed path covers everything below it, so+ `transcripts` covers `transcripts[2]` and `case` covers `case.attachment`.+ """+ normalized_path = tuple(LIST_ITEM_SEGMENT if isinstance(segment, int) else segment for segment in file_input_path)+ return normalized_path[: len(self.consumed_path)] == self.consumed_path+++class _ResolvedModel(NamedTuple):+ """What a consumer's model reference resolved to."""++ model: str+ specs: list[InferenceModelSpec]+++def collect_file_input_consumers(entry_pipe: PipeAbstract) -> dict[str, list[FileInputConsumer]]:+ """For each input slot of the entry pipe, the operators that will consume its files.++ Must run while the pipe's library is loaded: sub-pipes are resolved through the hub, and models+ through the model deck. A slot that reaches no consumer has no entry.++ Args:+ entry_pipe: The pipe a run starts from.++ Returns:+ The consumers, by entry input slot name, in the order the walk found them.+ """+ frame: _Frame = {input_name: (input_name,) for input_name in entry_pipe.inputs.root}+ consumers: list[FileInputConsumer] = []+ _visit(pipe=entry_pipe, frame=frame, is_conditional=False, visiting=frozenset(), consumers=consumers)+ consumers_by_slot: dict[str, list[FileInputConsumer]] = {}+ for consumer in consumers:+ consumers_by_slot.setdefault(consumer.slot_name, []).append(consumer)+ return consumers_by_slot+++def check_file_inputs_against_consumers(*, entry_pipe: PipeAbstract, file_inputs: list[IdentifiedFileInput]) -> None:+ """Refuse a run whose file inputs are certain to reach a consumer that cannot read their format.++ Each file input with a known format is compared with the certain consumers that cover it: a+ consumer reached through a condition or a liftable step may never run, and one whose model could+ not be resolved is opaque, so both are left to the operator's own check. A web-page model given+ an http(s) URL fetches the page itself, so it is not compared. Every violation is collected and+ reported together.++ Args:+ entry_pipe: The pipe the run starts from. Its library must be loaded.+ file_inputs: The run's file inputs with the formats run setup established+ (`collect_file_inputs`).++ Raises:+ PipelineInputFormatUnsupportedError: If any file input is certain to reach a consumer whose+ model does not read its format.+ """+ known_file_inputs = [file_input for file_input in file_inputs if file_input.format_key is not None]+ if not known_file_inputs:+ return+ consumers_by_slot = collect_file_input_consumers(entry_pipe)+ violations: list[str] = []+ for file_input in known_file_inputs:+ format_key = file_input.format_key+ if format_key is None:+ continue+ slot_name = str(file_input.path[0])+ for consumer in consumers_by_slot.get(slot_name, []):+ if consumer.is_conditional or consumer.readable_formats is None or not consumer.covers(file_input_path=file_input.path):+ continue+ if consumer.reads_web_pages and _is_http_url(uri_kind=file_input.uri_kind):+ continue+ if format_key in consumer.readable_formats:+ continue+ violations.append(+ f"Input '{file_input.display_path}' is {describe_file_format(format_key=format_key, mime_type=file_input.mime_type)}: "+ f"pipe '{consumer.pipe_code}' {consumer.verb_phrase} model '{consumer.model}', "+ f"which reads {describe_format_keys(format_keys=consumer.listed_formats)}."+ )+ if violations:+ msg = " ".join([*violations, "Give a file in a format its model reads, or use a model that reads this format."])+ raise PipelineInputFormatUnsupportedError(msg)+++def _is_http_url(*, uri_kind: UriKind) -> bool:+ match uri_kind:+ case UriKind.HTTP_URL:+ return True+ case UriKind.LOCAL_PATH | UriKind.PIPELEX_STORAGE | UriKind.BASE64_DATA_URL:+ return False+++def resolve_model_specs(*, model_reference: str, model_type: ModelType) -> list[InferenceModelSpec]:+ """The model specs a model reference can be served by: one for a handle or an alias, every available member for a waterfall.++ A preset is not resolved here: the caller resolves a step's choice to a setting first. A member+ the deck does not serve (its backend is disabled) is skipped, since no run could use it either.+ An unresolvable reference gives no spec.+ """+ return _resolve_model_specs(model_reference=model_reference, model_type=model_type, visited=frozenset())+++def _resolve_model_specs(*, model_reference: str, model_type: ModelType, visited: frozenset[str]) -> list[InferenceModelSpec]:+ if model_reference in visited:+ return []+ visited |= {model_reference}+ try:+ reference = ModelReference.parse(model_reference)+ except ModelReferenceParseError:+ return []+ model_deck = get_model_deck()+ aliases, waterfalls = model_deck.get_aliases_and_waterfalls_for_type(model_type)+ alias_target: str | None = None+ waterfall_members: list[str] | None = None+ match reference.kind:+ case ModelReferenceKind.PRESET:+ return []+ case ModelReferenceKind.ALIAS:+ alias_target = aliases.get(reference.name)+ case ModelReferenceKind.WATERFALL:+ waterfall_members = waterfalls.get(reference.name)+ case ModelReferenceKind.HANDLE:+ if model_spec := model_deck.inference_models.get(reference.name):+ return [model_spec] if model_spec.model_type == model_type else []+ alias_target = aliases.get(reference.name)+ waterfall_members = None if alias_target else waterfalls.get(reference.name)+ if alias_target:+ return _resolve_model_specs(model_reference=alias_target, model_type=model_type, visited=visited)+ model_specs: list[InferenceModelSpec] = []+ for member in waterfall_members or []:+ model_specs.extend(_resolve_model_specs(model_reference=member, model_type=model_type, visited=visited))+ return model_specs+++def _visit(*, pipe: PipeAbstract, frame: _Frame, is_conditional: bool, visiting: frozenset[str], consumers: list[FileInputConsumer]) -> None:+ """Visit one pipe with the frame of tracked variables it sees, recording the consumers it holds."""+ if pipe.pipe_ref in visiting or not frame:+ return+ visiting |= {pipe.pipe_ref}+ if isinstance(pipe, PipeSequence):+ _visit_sequence(sequence=pipe, frame=frame, is_conditional=is_conditional, visiting=visiting, consumers=consumers)+ elif isinstance(pipe, PipeParallel):+ liftable_refs = {liftable_step.pipe_ref for liftable_step in pipe.analyze_branch_taint().liftable_steps}+ for sub_pipe in pipe.parallel_sub_pipes:+ _visit_sub_pipe(+ sub_pipe=sub_pipe, frame=frame, is_conditional=is_conditional, liftable_refs=liftable_refs, visiting=visiting, consumers=consumers+ )+ elif isinstance(pipe, PipeBatch):+ if branch_pipe := get_optional_pipe(pipe_code=pipe.branch_pipe_code):+ _visit(+ pipe=branch_pipe,+ frame=_batch_frame(frame=frame, batch_params=pipe.batch_params),+ is_conditional=is_conditional,+ visiting=visiting,+ consumers=consumers,+ )+ elif isinstance(pipe, PipeCondition):+ for outcome_pipe_code in sorted(pipe.pipe_dependencies()):+ if outcome_pipe := get_optional_pipe(pipe_code=outcome_pipe_code):+ _visit(pipe=outcome_pipe, frame=frame, is_conditional=True, visiting=visiting, consumers=consumers)+ elif isinstance(pipe, PipeExtract):+ _record_extract_consumer(pipe_extract=pipe, frame=frame, is_conditional=is_conditional, consumers=consumers)+ elif isinstance(pipe, PipeLLM):+ _record_llm_document_consumers(pipe_llm=pipe, frame=frame, is_conditional=is_conditional, consumers=consumers)+++def _visit_sequence(+ *, sequence: PipeSequence, frame: _Frame, is_conditional: bool, visiting: frozenset[str], consumers: list[FileInputConsumer]+) -> None:+ liftable_refs = {liftable_step.pipe_ref for liftable_step in sequence.analyze_taint().liftable_steps}+ step_frame = dict(frame)+ for sub_pipe in sequence.sequential_sub_pipes:+ step_pipe = _visit_sub_pipe(+ sub_pipe=sub_pipe, frame=step_frame, is_conditional=is_conditional, liftable_refs=liftable_refs, visiting=visiting, consumers=consumers+ )+ # Whatever the step writes into the flow overwrites the slot of that name: later steps read+ # the step's result there, not the input, so it stops being followed.+ step_writes = _sub_pipe_writes(sub_pipe=sub_pipe, step_pipe=step_pipe, visiting=frozenset())+ if step_writes.may_write_any:+ return+ for written_name in step_writes.names:+ step_frame.pop(written_name, None)+++class _Writes(NamedTuple):+ """The names a step may write into the working memory of the sequence that runs it."""++ names: frozenset[str]+ may_write_any: bool+ """Whether it may write a name the walk cannot know statically, after which nothing is followed."""+++_NO_WRITES: Final[_Writes] = _Writes(names=frozenset(), may_write_any=False)+_ANY_WRITES: Final[_Writes] = _Writes(names=frozenset(), may_write_any=True)+++def _sub_pipe_writes(*, sub_pipe: SubPipe, step_pipe: PipeAbstract | None, visiting: frozenset[str]) -> _Writes:+ """What a controller's step writes into the memory it runs on: its result, and whatever its pipe writes there.++ A batched step runs every item on a copy of the memory, and only its result comes back.+ """+ result_names = frozenset({sub_pipe.output_name}) if sub_pipe.output_name else frozenset[str]()+ if sub_pipe.batch_params:+ return _Writes(names=result_names, may_write_any=False)+ if step_pipe is None:+ return _ANY_WRITES+ pipe_writes = _pipe_writes(pipe=step_pipe, visiting=visiting)+ return _Writes(names=result_names | pipe_writes.names, may_write_any=pipe_writes.may_write_any)+++def _pipe_writes(*, pipe: PipeAbstract, visiting: frozenset[str]) -> _Writes:+ """What a pipe writes into the memory it runs on, besides its own result, which its caller names.++ A sequence runs its steps on that memory and a condition runs its outcome on it, so what they+ write lands in the caller's memory too. A parallel runs its branches on copies and writes back+ only its branch results, when it adds each of them. A batch runs on copies. An operator writes+ only its result.+ """+ if pipe.pipe_ref in visiting:+ return _NO_WRITES+ visiting |= {pipe.pipe_ref}+ if isinstance(pipe, PipeSequence):+ return _merge_writes(+ writes=[+ _sub_pipe_writes(sub_pipe=sub_pipe, step_pipe=get_optional_pipe(pipe_code=sub_pipe.pipe_code), visiting=visiting)+ for sub_pipe in pipe.sequential_sub_pipes+ ]+ )+ if isinstance(pipe, PipeCondition):+ # The alias a condition adds is named after the value its expression renders to.+ if pipe.add_alias_from_expression_to:+ return _ANY_WRITES+ outcome_writes: list[_Writes] = []+ for outcome_pipe_code in pipe.pipe_dependencies():+ outcome_pipe = get_optional_pipe(pipe_code=outcome_pipe_code)+ outcome_writes.append(_pipe_writes(pipe=outcome_pipe, visiting=visiting) if outcome_pipe else _ANY_WRITES)+ return _merge_writes(writes=outcome_writes)+ if isinstance(pipe, PipeParallel) and pipe.add_each_output:+ return _Writes(names=frozenset(branch.output_name for branch in pipe.parallel_sub_pipes if branch.output_name), may_write_any=False)+ return _NO_WRITES+++def _merge_writes(*, writes: list[_Writes]) -> _Writes:+ return _Writes(+ names=frozenset[str]().union(*(step_writes.names for step_writes in writes)),+ may_write_any=any(step_writes.may_write_any for step_writes in writes),+ )+++def _visit_sub_pipe(+ *,+ sub_pipe: SubPipe,+ frame: _Frame,+ is_conditional: bool,+ liftable_refs: set[str],+ visiting: frozenset[str],+ consumers: list[FileInputConsumer],+) -> PipeAbstract | None:+ """Visit a controller's step or branch, returning the pipe it resolved to, `None` when unresolved."""+ step_pipe = get_optional_pipe(pipe_code=sub_pipe.pipe_code)+ if step_pipe is None:+ return None+ step_frame = _batch_frame(frame=frame, batch_params=sub_pipe.batch_params) if sub_pipe.batch_params else frame+ _visit(+ pipe=step_pipe,+ frame=step_frame,+ is_conditional=is_conditional or step_pipe.pipe_ref in liftable_refs,+ visiting=visiting,+ consumers=consumers,+ )+ return step_pipe+++def _batch_frame(*, frame: _Frame, batch_params: BatchParams) -> _Frame:+ """The frame a batch's branch sees: the item slot stands for an item of the list slot, when that is tracked."""+ batch_frame = dict(frame)+ batch_frame.pop(batch_params.input_item_stuff_name, None)+ if list_path := _tracked_path(frame=frame, variable_path=batch_params.input_list_stuff_name):+ batch_frame[batch_params.input_item_stuff_name] = (*list_path, LIST_ITEM_SEGMENT)+ return batch_frame+++def _tracked_path(*, frame: _Frame, variable_path: str) -> tuple[str, ...] | None:+ """The entry path a dotted variable path stands for, when its root is tracked: `case.attachment` → `("case", "attachment")`."""+ root_name, *field_names = variable_path.split(".")+ if root_path := frame.get(root_name):+ return (*root_path, *field_names)+ return None+++def _record_extract_consumer(*, pipe_extract: PipeExtract, frame: _Frame, is_conditional: bool, consumers: list[FileInputConsumer]) -> None:+ # An Image input is an image by the setup check, and an extractor that reads no images is the+ # method author's choice of model, not the caller's input: only a document input is followed.+ if not pipe_extract.document_stuff_name:+ return+ consumed_path = _tracked_path(frame=frame, variable_path=pipe_extract.document_stuff_name)+ if consumed_path is None:+ return+ resolved_model = _resolve_extract_model(pipe_extract=pipe_extract)+ readable_formats: frozenset[str] | None = None+ reads_web_pages = False+ if resolved_model is not None:+ readable_formats = frozenset[str]().union(*(model_spec.readable_formats_for_extract for model_spec in resolved_model.specs))+ reads_web_pages = any(model_spec.is_web_page_supported_for_extract for model_spec in resolved_model.specs)+ consumers.append(+ FileInputConsumer(+ slot_name=consumed_path[0],+ consumed_path=consumed_path,+ pipe_ref=pipe_extract.pipe_ref,+ pipe_code=pipe_extract.code,+ kind=FileConsumerKind.EXTRACT,+ model=resolved_model.model if resolved_model else None,+ readable_formats=readable_formats,+ reads_web_pages=reads_web_pages,+ is_conditional=is_conditional,+ )+ )+++def _record_llm_document_consumers(*, pipe_llm: PipeLLM, frame: _Frame, is_conditional: bool, consumers: list[FileInputConsumer]) -> None:+ prompt_spec = pipe_llm.llm_prompt_spec+ document_references = [*(prompt_spec.user_document_references or []), *(prompt_spec.system_document_references or [])]+ if not document_references:+ return+ resolved_model = _resolve_llm_model(pipe_llm=pipe_llm)+ readable_formats: frozenset[str] | None = None+ if resolved_model is not None:+ readable_formats = frozenset[str]().union(*(model_spec.supported_document_types for model_spec in resolved_model.specs))+ # A model that reads no documents at all is the method author's choice of model, which the+ # worker reports as a capability error: there is no caller's format to refuse.+ if not readable_formats:+ return+ for document_reference in document_references:+ consumed_path = _tracked_path(frame=frame, variable_path=document_reference.variable_path)+ if consumed_path is None:+ continue+ consumers.append(+ FileInputConsumer(+ slot_name=consumed_path[0],+ consumed_path=consumed_path,+ pipe_ref=pipe_llm.pipe_ref,+ pipe_code=pipe_llm.code,+ kind=FileConsumerKind.LLM_DOCUMENT,+ model=resolved_model.model if resolved_model else None,+ readable_formats=readable_formats,+ reads_web_pages=False,+ is_conditional=is_conditional,+ )+ )+++def _resolve_extract_model(*, pipe_extract: PipeExtract) -> _ResolvedModel | None:+ """The model a PipeExtract's choice resolves to, through the deck chain and any preset, `None` when unresolvable."""+ model_deck = get_model_deck()+ try:+ model_reference = resolve_extract_setting(extract_choice=pipe_extract.extract_choice).model+ for _hop in range(_MAX_PRESET_HOPS):+ if not _is_preset(model_reference=model_reference):+ break+ model_reference = model_deck.get_extract_setting(extract_choice=model_reference).model+ except (ModelChoiceNotFoundError, ModelReferenceParseError):+ return None+ return _resolved_model(model_reference=model_reference, model_type=ModelType.TEXT_EXTRACTOR)+++def _resolve_llm_model(*, pipe_llm: PipeLLM) -> _ResolvedModel | None:+ """The model a PipeLLM's documents reach, `None` when unresolvable.++ Which of its two settings serves the prompt depends on its output: a single text goes to the+ text setting, anything else to the object setting. A text output can still be generated as a+ list when a step asks for several, so a pipe whose output is text reaches both, and reads a+ format when either model reads it.+ """+ llm_for_text_choice = pipe_llm.llm_choices.for_text if pipe_llm.llm_choices else None+ llm_for_object_choice = pipe_llm.llm_choices.for_object if pipe_llm.llm_choices else None+ try:+ object_model = resolve_llm_setting_for_object(llm_choice=llm_for_object_choice, llm_choice_for_text=llm_for_text_choice).model+ model_references = [object_model]+ if get_concept_library().is_compatible(+ tested_concept=pipe_llm.output.concept,+ wanted_concept=get_native_concept(native_concept=NativeConceptCode.TEXT),+ strict=True,+ ):+ text_model = resolve_llm_setting_for_text(llm_choice=llm_for_text_choice).model+ model_references = [text_model] if text_model == object_model else [text_model, object_model]+ model_references = [_resolve_llm_presets(model_reference=model_reference) for model_reference in model_references]+ except (ModelChoiceNotFoundError, ModelReferenceParseError):+ return None+ model_specs: list[InferenceModelSpec] = []+ for model_reference in model_references:+ model_specs.extend(resolve_model_specs(model_reference=model_reference, model_type=ModelType.LLM))+ if not model_specs:+ return None+ return _ResolvedModel(model=_display_model(model_references=model_references, model_specs=model_specs), specs=model_specs)+++def _resolve_llm_presets(*, model_reference: str) -> str:+ model_deck = get_model_deck()+ for _hop in range(_MAX_PRESET_HOPS):+ if not _is_preset(model_reference=model_reference):+ break+ model_reference = model_deck.get_llm_setting(llm_choice=model_reference).model+ return model_reference+++def _is_preset(*, model_reference: str) -> bool:+ match ModelReference.parse(model_reference).kind:+ case ModelReferenceKind.PRESET:+ return True+ case ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL | ModelReferenceKind.HANDLE:+ return False+++def _resolved_model(*, model_reference: str, model_type: ModelType) -> _ResolvedModel | None:+ model_specs = resolve_model_specs(model_reference=model_reference, model_type=model_type)+ if not model_specs:+ return None+ return _ResolvedModel(model=_display_model(model_references=[model_reference], model_specs=model_specs), specs=model_specs)+++def _display_model(*, model_references: list[str], model_specs: list[InferenceModelSpec]) -> str:+ """How a message names the model: its concrete handle when there is one, else the references as written."""+ spec_names = sorted({model_spec.name for model_spec in model_specs})+ if len(spec_names) == 1:+ return spec_names[0]+ return " or ".join(model_references)diff --git a/pipelex/pipeline/input_normalizer.py b/pipelex/pipeline/input_normalizer.pyindex 15b00c513..30ed1e840 100644--- a/pipelex/pipeline/input_normalizer.py+++ b/pipelex/pipeline/input_normalizer.py@@ -1,16 +1,35 @@-"""Normalize pipeline inputs by converting data URLs and local file paths to pipelex-storage:// URIs.--This module provides functions to scan WorkingMemory and convert any ImageContent-or DocumentContent with data URLs (data:...;base64,...) or local file paths to-pipelex-storage:// URIs for more efficient pipeline processing, and to give every-such input a public_url a template can render, whichever form its url took.+"""Prepare a run's file inputs: establish each one's format, and relocate it to storage when configured.++This module walks the WorkingMemory a run starts from and visits every ImageContent and+DocumentContent, including list items and the fields nested in structured content. It does two+things to each:++- **Identification**, always (mock inputs aside, which the caller skips): the content's `mime_type`+ is stamped with the type that describes its bytes, so every later check and operator reads the+ format instead of guessing it again. The bytes are the truth: a sniffed type replaces a declared+ one. A stored file is identified by a head read of its first bytes, a data URL and a local file by+ the bytes already in hand. An http(s) input is never fetched, and keeps whatever type the caller+ declared. An Image input whose bytes identify something other than an image is refused here,+ before it is relocated, whatever will consume it.+- **Relocation**, when the run is configured for it: a data URL, or a local file when local uploads+ are enabled, is stored and its url becomes a `pipelex-storage://` reference, and every input gets+ a `public_url` a template can render.++`collect_file_inputs` then reads the stamped formats back, each with its path in the inputs+(`transcripts[2]`, `case.attachment`), for the checks that refuse a run before it starts."""+import asyncioimport base64-from pathlib import Path-from typing import Any, cast+import binascii+from collections.abc import Coroutine, Sequence+from enum import StrEnum+from pathlib import Path, PurePosixPath+from typing import Any, Final, NamedTuple, TypeAlias, TypeVar, cast+import aiofilesimport shortuuid+from pydantic import BaseModel, ConfigDictfrom pipelex.config import get_configfrom pipelex.core.memory.working_memory import WorkingMemory@@ -18,367 +37,558 @@ from pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.structured_content import StructuredContent-from pipelex.pipeline.exceptions import PipelineInputContentError, PipelineInputUrlInvalidError, PipelineInputUrlMissingError+from pipelex.pipeline.exceptions import (+ PipelineInputContentError,+ PipelineInputNotAnImageError,+ PipelineInputUrlInvalidError,+ PipelineInputUrlMissingError,+)from pipelex.runtime_hub import get_storage_providerfrom pipelex.tools.misc.file_utils import load_binary_async-from pipelex.tools.misc.filetype_utils import detect_file_type_from_bytes+from pipelex.tools.misc.filetype_utils import (+ FILE_HEAD_NB_BYTES,+ IMAGE_FORMAT_KEY,+ UNKNOWN_FILE_TYPE,+ describe_file_format,+ format_key_from_mime_type,+ guess_file_type_from_bytes,+ identify_mime_type,+ mime_type_to_extension,+)from pipelex.tools.misc.http_utils import validate_http_url_syntax-from pipelex.tools.storage.exceptions import StorageInvalidUriError+from pipelex.tools.storage.exceptions import StorageFileNotFoundError, StorageInvalidUriErrorfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract-from pipelex.tools.uri.resolved_uri import ResolvedBase64DataUrl, ResolvedHttpUrl, ResolvedPipelexStorage+from pipelex.tools.uri.resolved_uri import ResolvedBase64DataUrl, ResolvedHttpUrl, ResolvedLocalPath, ResolvedPipelexStorage, UriKindfrom pipelex.tools.uri.uri_read_scope import authorize_uri_readfrom pipelex.tools.uri.uri_resolver import resolve_uri# Type alias for content types that can have their URLs normalizedNormalizableContent = ImageContent | DocumentContent+# Where a file sits in a run's inputs: the input's name, then field names and list indexes.+# `("transcripts", 2)` is `transcripts[2]`, `("case", "attachment")` is `case.attachment`.+FileInputPath: TypeAlias = tuple[str | int, ...]-async def normalize_data_urls_to_storage(working_memory: WorkingMemory, *, storage_scope: str, read_scope: str | None) -> WorkingMemory:- """Convert data URLs and local files in ImageContent and DocumentContent to pipelex-storage:// URIs, and fill their public_url.+# How many file inputs are prepared at once. A list of stored documents is identified by one head+# read each, and these overlap rather than costing a round-trip each in sequence.+MAX_CONCURRENT_FILE_IDENTIFICATIONS: Final[int] = 8- Scans all stuffs in working memory and for any ImageContent or DocumentContent with- a data:...;base64,... URL, or a local path when local uploads are enabled, stores the- data and replaces the URL with a pipelex-storage:// URI. Every such content also comes- out with a public_url: a link signed through the storage provider for a stored file,- the URL itself for an http(s) one. See `_normalize_url_content` for each form.+_ResultT = TypeVar("_ResultT")- This handles:- - Direct ImageContent and DocumentContent- - ListContent containing ImageContent or DocumentContent items- - StructuredContent with nested ImageContent or DocumentContent fields (recursive)+class FileInputKind(StrEnum):+ """Which content class holds a file input: what the caller's input slot promises it is."""- The images inside a TextAndImagesContent are not reached, and keep their url and public_url as given.+ IMAGE = "image"+ DOCUMENT = "document"- Args:- working_memory: The working memory to normalize.- storage_scope: The run's opaque storage prefix. Normalized bytes land- under `{storage_scope}/assets/`, inside the run's own namespace —- they used to go to a flat top-level `normalized/` prefix shared by- every run of every tenant.- read_scope: The run's read scope. On a scoped run every url is authorized- before it is read or linked: a local path is refused instead of- being uploaded or kept for a later reader, and a storage reference- outside the scope is refused instead of being signed, since a signed- link is a read. The refusal names the input. ``None`` for an unscoped- run. See :mod:`pipelex.tools.uri.uri_read_scope`.- Returns:- The same WorkingMemory instance with normalized URLs.+class IdentifiedFileInput(BaseModel):+ """One file input of a run, with the format its preparation established."""- Raises:- UriReadRefusedError: an input's url is a local path, or a storage reference outside the- read scope, on a run with a read scope.- """- storage = get_storage_provider()+ model_config = ConfigDict(frozen=True)- for input_name, stuff in working_memory.root.items():- content = stuff.content- normalized_content, changed = await _normalize_value(- value=content, storage=storage, storage_scope=storage_scope, read_scope=read_scope, input_name=input_name- )- if changed:- stuff.content = normalized_content+ path: FileInputPath+ kind: FileInputKind+ uri_kind: UriKind+ mime_type: str | None- return working_memory+ @property+ def display_path(self) -> str:+ """The path as a caller reads it: `transcripts[2]`, `case.attachment`."""+ return format_file_input_path(self.path)+ @property+ def format_key(self) -> str | None:+ """The key every format check compares, `None` when the format is unknown."""+ return format_key_from_mime_type(mime_type=self.mime_type)-async def _normalize_value(- value: Any,++class _PreparationContext(NamedTuple):+ """What every visited file input is prepared with, shared by the whole walk."""++ storage: StorageProviderAbstract+ storage_scope: str+ read_scope: str | None+ is_relocation_enabled: bool+ semaphore: asyncio.Semaphore+++def format_file_input_path(path: FileInputPath) -> str:+ """Render a file input's path the way a caller writes it: `transcripts[2]`, `case.attachments[0]`."""+ rendered = ""+ for segment in path:+ if isinstance(segment, int):+ rendered += f"[{segment}]"+ elif rendered:+ rendered += f".{segment}"+ else:+ rendered = segment+ return rendered+++async def prepare_file_inputs(+ working_memory: WorkingMemory,*,- storage: StorageProviderAbstract,storage_scope: str,read_scope: str | None,- input_name: str,-) -> tuple[Any, bool]:- """Recursively normalize a value, converting data URLs in ImageContent/DocumentContent to storage URIs.+ is_relocation_enabled: bool,+) -> WorkingMemory:+ """Establish the format of every file input, and relocate it to storage when configured.++ Visits every ImageContent and DocumentContent in the working memory, including the items of a+ list and the fields of a structured content, recursively. The images inside a+ TextAndImagesContent are not reached, and keep their url, public_url and mime_type as given.+ File inputs are prepared concurrently, at most `MAX_CONCURRENT_FILE_IDENTIFICATIONS` at a time.+ See `_prepare_url_content` for what happens to each form of url.Args:- value: The value to normalize (can be ImageContent, DocumentContent, StructuredContent, list, or any other type).- storage: The storage provider to use.- storage_scope: The run's opaque storage prefix; normalized bytes land- under `{storage_scope}/assets/`.- read_scope: The run's read scope, which every url must satisfy before it is read or linked.- input_name: The input the value belongs to, named in a refusal.+ working_memory: The working memory to prepare. Its stuffs are updated in place.+ storage_scope: The run's opaque storage prefix. Relocated bytes land under+ `{storage_scope}/assets/`, inside the run's own namespace.+ read_scope: The run's read scope. On a scoped run every url is authorized before it is+ read or linked: a local path is refused instead of being read, uploaded or kept for a+ later reader, and a storage reference outside the scope is refused instead of being+ read or signed, since a signed link is a read. The refusal names the input. ``None``+ for an unscoped run. See :mod:`pipelex.tools.uri.uri_read_scope`.+ is_relocation_enabled: Whether data URLs and local files are stored and every input gets a+ `public_url`. Identification runs either way.Returns:- A tuple of (normalized_value, has_changed).+ The same WorkingMemory instance, its file inputs stamped with their `mime_type`.++ Raises:+ PipelineInputUrlMissingError: An input's url is blank.+ PipelineInputNotAnImageError: An Image input's bytes identify something other than an image.+ PipelineInputUrlInvalidError: An http(s) url does not parse as one (when relocating).+ PipelineInputContentError: A data URL is not valid base64, a local path cannot be read+ for an upload, or a storage reference names no stored file or is refused as a key.+ UriReadRefusedError: An input's url is a local path, or a storage reference outside the+ read scope, on a run with a read scope.+ """+ context = _PreparationContext(+ storage=get_storage_provider(),+ storage_scope=storage_scope,+ read_scope=read_scope,+ is_relocation_enabled=is_relocation_enabled,+ semaphore=asyncio.Semaphore(MAX_CONCURRENT_FILE_IDENTIFICATIONS),+ )+ named_stuffs = list(working_memory.root.items())+ results = await _gather_cancelling(+ coroutines=[_prepare_value(value=stuff.content, path=(input_name,), context=context) for input_name, stuff in named_stuffs]+ )+ for (_input_name, stuff), (prepared_content, changed) in zip(named_stuffs, results, strict=True):+ if changed:+ stuff.content = prepared_content+ return working_memory+++def collect_file_inputs(working_memory: WorkingMemory) -> list[IdentifiedFileInput]:+ """Every file input of the working memory, with its path and the `mime_type` it carries.++ Pure: it reads what `prepare_file_inputs` stamped and does no IO. The walk is the same, so the+ images inside a TextAndImagesContent are not reached either."""- # Handle ImageContent and DocumentContent+ file_inputs: list[IdentifiedFileInput] = []+ for input_name, stuff in working_memory.root.items():+ _collect_value(value=stuff.content, path=(input_name,), file_inputs=file_inputs)+ return file_inputs+++def _collect_value(*, value: Any, path: FileInputPath, file_inputs: list[IdentifiedFileInput]) -> None:if isinstance(value, (ImageContent, DocumentContent)):- normalized = await _normalize_url_content(- content=value, storage=storage, storage_scope=storage_scope, read_scope=read_scope, input_name=input_name+ file_inputs.append(+ IdentifiedFileInput(+ path=path,+ kind=FileInputKind.IMAGE if isinstance(value, ImageContent) else FileInputKind.DOCUMENT,+ uri_kind=resolve_uri(value.url).kind,+ mime_type=value.mime_type,+ ))- return normalized, normalized is not value+ elif isinstance(value, StructuredContent):+ for field_name, field_value in value:+ _collect_value(value=field_value, path=(*path, field_name), file_inputs=file_inputs)+ elif isinstance(value, ListContent):+ list_items = cast("list[Any]", value.items) # pyright: ignore[reportUnknownMemberType]+ for item_index, item in enumerate(list_items):+ _collect_value(value=item, path=(*path, item_index), file_inputs=file_inputs)+ elif isinstance(value, list):+ for item_index, item in enumerate(cast("list[Any]", value)):+ _collect_value(value=item, path=(*path, item_index), file_inputs=file_inputs)+++async def _gather_cancelling(*, coroutines: Sequence[Coroutine[Any, Any, _ResultT]]) -> list[_ResultT]:+ """Run the coroutines concurrently and return their results in order.++ On the first failure the others are cancelled and awaited before the failure propagates, as+ itself and not wrapped in a group, so a caller's error mapping sees the input error it raised.+ """+ tasks = [asyncio.ensure_future(coroutine) for coroutine in coroutines]+ try:+ return list(await asyncio.gather(*tasks))+ except BaseException:+ for task in tasks:+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)+ raise+++async def _prepare_value(*, value: Any, path: FileInputPath, context: _PreparationContext) -> tuple[Any, bool]:+ """Recursively prepare a value, returning the prepared value and whether it changed."""+ if isinstance(value, (ImageContent, DocumentContent)):+ prepared = await _prepare_url_content(content=value, path=path, context=context)+ return prepared, prepared is not value- # Handle StructuredContent (recursively process all fields)if isinstance(value, StructuredContent):- return await _normalize_structured_content(- structured_content=value, storage=storage, storage_scope=storage_scope, read_scope=read_scope, input_name=input_name- )+ return await _prepare_structured_content(structured_content=value, path=path, context=context)- # Handle ListContentif isinstance(value, ListContent):- return await _normalize_list_content(+ return await _prepare_list_content(list_content=value, # pyright: ignore[reportUnknownArgumentType]- storage=storage,- storage_scope=storage_scope,- read_scope=read_scope,- input_name=input_name,+ path=path,+ context=context,)- # Handle plain lists (might contain ImageContent, DocumentContent, or StructuredContent)+ # Plain lists, which might contain ImageContent, DocumentContent, or StructuredContentif isinstance(value, list):- return await _normalize_list(+ return await _prepare_list(items=value, # pyright: ignore[reportUnknownArgumentType]- storage=storage,- storage_scope=storage_scope,- read_scope=read_scope,- input_name=input_name,+ path=path,+ context=context,)- # Other types don't need normalization+ # Other types don't need preparationreturn value, False-async def _normalize_structured_content(- structured_content: StructuredContent,+async def _prepare_structured_content(*,- storage: StorageProviderAbstract,- storage_scope: str,- read_scope: str | None,- input_name: str,+ structured_content: StructuredContent,+ path: FileInputPath,+ context: _PreparationContext,) -> tuple[StructuredContent, bool]:- """Normalize a StructuredContent by recursively processing all its fields.-- Args:- structured_content: The structured content to normalize.- storage: The storage provider to use.- storage_scope: The run's opaque storage prefix; normalized bytes land- under `{storage_scope}/assets/`.- read_scope: The run's read scope, which every url must satisfy before it is read or linked.- input_name: The input the value belongs to, named in a refusal.-- Returns:- A tuple of (normalized_content, has_changed).- """+ """Prepare a StructuredContent by preparing all its fields."""+ fields = list(structured_content)+ results = await _gather_cancelling(+ coroutines=[_prepare_value(value=field_value, path=(*path, field_name), context=context) for field_name, field_value in fields]+ )updates: dict[str, Any] = {}- has_changes = False-- for field_name, field_value in structured_content:- normalized_value, changed = await _normalize_value(- value=field_value, storage=storage, storage_scope=storage_scope, read_scope=read_scope, input_name=input_name- )+ for (field_name, _field_value), (prepared_value, changed) in zip(fields, results, strict=True):if changed:- updates[field_name] = normalized_value- has_changes = True+ updates[field_name] = prepared_value- if not has_changes:+ if not updates:return structured_content, False- # Create a new instance with updated fields- # Use model_copy with update to preserve all other fields+ # model_copy with update preserves all other fieldsreturn structured_content.model_copy(update=updates), True-async def _normalize_list_content(- list_content: ListContent[Any],+async def _prepare_list_content(*,- storage: StorageProviderAbstract,- storage_scope: str,- read_scope: str | None,- input_name: str,+ list_content: ListContent[Any],+ path: FileInputPath,+ context: _PreparationContext,) -> tuple[ListContent[Any], bool]:- """Normalize a ListContent by processing all its items.-- Args:- list_content: The list content to normalize.- storage: The storage provider to use.- storage_scope: The run's opaque storage prefix; normalized bytes land- under `{storage_scope}/assets/`.- read_scope: The run's read scope, which every url must satisfy before it is read or linked.- input_name: The input the value belongs to, named in a refusal.-- Returns:- A tuple of (normalized_list_content, has_changed).- """+ """Prepare a ListContent by preparing all its items."""raw_items = list_content.items # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]if not raw_items:return list_content, False- normalized_items, has_changes = await _normalize_list(- items=raw_items, storage=storage, storage_scope=storage_scope, read_scope=read_scope, input_name=input_name- ) # pyright: ignore[reportUnknownArgumentType]+ prepared_items, has_changes = await _prepare_list(items=raw_items, path=path, context=context) # pyright: ignore[reportUnknownArgumentType]if not has_changes:return list_content, False# Check the type of the first item to determine the ListContent type- first_item = normalized_items[0]+ first_item = prepared_items[0]if isinstance(first_item, ImageContent):- return ListContent[ImageContent](items=cast("list[ImageContent]", normalized_items)), True+ return ListContent[ImageContent](items=cast("list[ImageContent]", prepared_items)), Trueif isinstance(first_item, DocumentContent):- return ListContent[DocumentContent](items=cast("list[DocumentContent]", normalized_items)), True+ return ListContent[DocumentContent](items=cast("list[DocumentContent]", prepared_items)), True# For other types (e.g., StructuredContent subclasses), use generic ListContent- return ListContent(items=normalized_items), True+ return ListContent(items=prepared_items), True-async def _normalize_list(- items: list[Any],+async def _prepare_list(*,- storage: StorageProviderAbstract,- storage_scope: str,- read_scope: str | None,- input_name: str,+ items: list[Any],+ path: FileInputPath,+ context: _PreparationContext,) -> tuple[list[Any], bool]:- """Normalize a list by processing all its items.-- Args:- items: The list items to normalize.- storage: The storage provider to use.- storage_scope: The run's opaque storage prefix; normalized bytes land- under `{storage_scope}/assets/`.- read_scope: The run's read scope, which every url must satisfy before it is read or linked.- input_name: The input the value belongs to, named in a refusal.-- Returns:- A tuple of (normalized_items, has_changed).- """- normalized_items: list[Any] = []- has_changes = False-- for item in items:- normalized_item, changed = await _normalize_value(- value=item, storage=storage, storage_scope=storage_scope, read_scope=read_scope, input_name=input_name- )- normalized_items.append(normalized_item)- if changed:- has_changes = True-- return normalized_items, has_changes+ """Prepare a list by preparing all its items, each at its index."""+ results = await _gather_cancelling(+ coroutines=[_prepare_value(value=item, path=(*path, item_index), context=context) for item_index, item in enumerate(items)]+ )+ prepared_items = [prepared_item for prepared_item, _changed in results]+ has_changes = any(changed for _prepared_item, changed in results)+ return prepared_items, has_changes-async def _normalize_url_content(- content: NormalizableContent,+async def _prepare_url_content(*,- storage: StorageProviderAbstract,- storage_scope: str,- read_scope: str | None,- input_name: str,+ content: NormalizableContent,+ path: FileInputPath,+ context: _PreparationContext,) -> NormalizableContent:- """Normalize ImageContent or DocumentContent to a storage reference and give it a public_url.+ """Identify one ImageContent or DocumentContent, and relocate it when configured.- Each form of url comes out with a `public_url` a template can render:+ Every form comes out with its `mime_type` established, and, when relocating, with a+ `public_url` a template can render:- - A `data:` URL is stored, and its url becomes the storage reference, signed as its `public_url`.- - A local path is stored the same way when local uploads are enabled, and kept unchanged otherwise.- - A `pipelex-storage://` reference is signed as its `public_url`, replacing any link it carried,- which may have expired; a provider that cannot link one leaves the carried link.- - An `http(s)` URL is already public: it becomes its own `public_url` unless the input names one.-- Args:- content: The image or document content to normalize.- storage: The storage provider to use.- storage_scope: The run's opaque storage prefix; normalized bytes land- under `{storage_scope}/assets/`.- read_scope: The run's read scope, which every url must satisfy before it is read or linked.- input_name: The input the value belongs to, named in a refusal.-- Returns:- The original content if no normalization needed, or a new instance- with the normalized url and its public_url.+ - A `data:` URL is decoded and identified from its bytes. When relocating, it is stored, its+ url becomes the storage reference, signed as its `public_url`.+ - A local path is identified from its bytes. When relocating with local uploads enabled, it is+ stored the same way; otherwise it is kept, and identified from a head read of the file.+ - A `pipelex-storage://` reference is identified from a head read of the stored object. When+ relocating, it is signed as its `public_url`, replacing any link it carried, which may have+ expired; a provider that cannot link one leaves the carried link.+ - An `http(s)` URL is never fetched and keeps its declared type. When relocating, its syntax is+ checked and it becomes its own `public_url` unless the input names one.Raises:PipelineInputUrlMissingError: If the url is blank.- PipelineInputUrlInvalidError: If an http(s) url does not parse as one.- PipelineInputContentError: If a local path cannot be read, or the storage- provider refuses a pipelex-storage:// reference as a key.- UriReadRefusedError: If the url is a local path, or a storage reference outside- the read scope, on a run with a read scope.+ PipelineInputNotAnImageError: If the content is an ImageContent and its bytes identify+ something other than an image. It is raised before anything is relocated.+ PipelineInputUrlInvalidError: If an http(s) url does not parse as one (when relocating).+ PipelineInputContentError: If a data URL is not valid base64, a local path cannot be read+ for an upload, or a pipelex-storage:// reference names no stored file or is refused+ by the storage provider as a key.+ UriReadRefusedError: If the url is a local path, or a storage reference outside the+ read scope, on a run with a read scope."""+ input_name = format_file_input_path(path)if not content.url.strip():msg = f"{type(content).__name__} input has a blank url — provide https://, data:, pipelex-storage://, or a local file path."raise PipelineInputUrlMissingError(msg)# Before anything reads or links the url. A signed link is a read by whoever holds it, so a- # foreign storage reference is refused here rather than signed; a local path is refused rather- # than uploaded, or kept for the prompt preparation to read later, which is what happens to it- # when local uploads are off.- authorize_uri_read(uri=content.url, read_scope=read_scope, position=f"the file given for input '{input_name}'")+ # foreign storage reference is refused here rather than read or signed; a local path is refused+ # rather than read, uploaded, or kept for the prompt preparation to read later.+ authorize_uri_read(uri=content.url, read_scope=context.read_scope, position=f"the file given for input '{input_name}'")resolved_uri = resolve_uri(content.url)+ async with context.semaphore:+ match resolved_uri:+ case ResolvedHttpUrl():+ return _prepare_http_url(content=content, resolved_uri=resolved_uri, context=context)+ case ResolvedPipelexStorage():+ return await _prepare_storage_reference(content=content, resolved_uri=resolved_uri, input_name=input_name, context=context)+ case ResolvedBase64DataUrl():+ return await _prepare_data_url(content=content, resolved_uri=resolved_uri, input_name=input_name, context=context)+ case ResolvedLocalPath():+ return await _prepare_local_path(content=content, resolved_uri=resolved_uri, input_name=input_name, context=context)+++def _prepare_http_url(*, content: NormalizableContent, resolved_uri: ResolvedHttpUrl, context: _PreparationContext) -> NormalizableContent:+ # Never fetched: setup makes no outbound request to a host the caller chose. The operator that+ # consumes the input fetches it and reports a real failure. Its format is only what the caller+ # declared: the checks compare a declared type, and leave a URL that declares none to the provider.+ if not context.is_relocation_enabled:+ return content+ # Syntax only, no network: the operator's fetch is the one honest test of a remote resource.+ try:+ validate_http_url_syntax(url=resolved_uri.url)+ except ValueError as exc:+ msg = f"{type(content).__name__} input: {exc}"+ raise PipelineInputUrlInvalidError(msg) from exc+ if content.public_url is not None:+ return content+ return content.model_copy(update={"public_url": content.url})- if isinstance(resolved_uri, ResolvedHttpUrl):- # Syntax only, no network: the operator that consumes the input fetches it and- # reports a real failure, which is the one honest test of a remote resource.- try:- validate_http_url_syntax(url=resolved_uri.url)- except ValueError as exc:- msg = f"{type(content).__name__} input: {exc}"- raise PipelineInputUrlInvalidError(msg) from exc- if content.public_url is not None:- return content- return content.model_copy(update={"public_url": content.url})- if isinstance(resolved_uri, ResolvedPipelexStorage):+async def _prepare_storage_reference(+ *,+ content: NormalizableContent,+ resolved_uri: ResolvedPipelexStorage,+ input_name: str,+ context: _PreparationContext,+) -> NormalizableContent:+ try:+ head = await context.storage.load_head(uri=resolved_uri.storage_uri, nb_bytes=FILE_HEAD_NB_BYTES)+ except StorageFileNotFoundError as exc:+ msg = f"{type(content).__name__} input '{input_name}': the storage reference '{content.url}' names no stored file"+ raise PipelineInputContentError(msg) from exc+ except StorageInvalidUriError as exc:+ msg = f"{type(content).__name__} input '{input_name}': the storage reference '{content.url}' cannot be read ({exc})"+ raise PipelineInputContentError(msg) from exc+ updates: dict[str, Any] = {}+ mime_type = _identify_file_input(+ content=content,+ head=head,+ declared_mime_type=content.mime_type,+ file_name=content.filename or PurePosixPath(resolved_uri.storage_uri).name,+ input_name=input_name,+ )+ if mime_type != content.mime_type:+ updates["mime_type"] = mime_type++ if context.is_relocation_enabled:# The reference is the durable fact and the link a derivative with an expiry, so the link is# always signed afresh: an input carrying both is usually a previous run's output passed back in.try:- public_url = await storage.public_url(uri=resolved_uri.storage_uri)+ public_url = await context.storage.public_url(uri=resolved_uri.storage_uri)except StorageInvalidUriError as exc:msg = f"{type(content).__name__} input: the storage reference '{content.url}' cannot be linked ({exc})"raise PipelineInputContentError(msg) from exc- if public_url is None:- return content- return content.model_copy(update={"public_url": public_url})+ if public_url is not None:+ updates["public_url"] = public_url++ if not updates:+ return content+ return content.model_copy(update=updates)+- if isinstance(resolved_uri, ResolvedBase64DataUrl):- # Decode base64 data and store+async def _prepare_data_url(+ *,+ content: NormalizableContent,+ resolved_uri: ResolvedBase64DataUrl,+ input_name: str,+ context: _PreparationContext,+) -> NormalizableContent:+ # Lenient on purpose, as it always was: characters outside the alphabet, such as the line breaks+ # of wrapped base64, are skipped. Only a payload that cannot decode at all is refused.+ try:raw_bytes = base64.b64decode(resolved_uri.base64_data)- file_type = detect_file_type_from_bytes(raw_bytes)- mime_type = resolved_uri.mime_type or content.mime_type- # `{scope}/assets/`, NOT the old flat `normalized/`. That top-level- # prefix was shared by every run of every tenant: the bytes of an input- # someone pasted as a data: URL landed beside everyone else's, keyed- # only by a random id. Under the run's own scope they are inside the- # tenant's namespace and go away with the run.- key = f"{storage_scope}/assets/{shortuuid.uuid()}.{file_type.extension}"- storage_uri = await storage.store(data=raw_bytes, key=key, content_type=mime_type)- public_url = await storage.public_url(uri=storage_uri)-- # Use model_copy to preserve all type-specific fields- return content.model_copy(- update={- "url": storage_uri,- "public_url": public_url,- "mime_type": mime_type,- }- )+ except binascii.Error as exc:+ msg = f"{type(content).__name__} input '{input_name}': the data URL does not hold valid base64 ({exc})"+ raise PipelineInputContentError(msg) from exc+ mime_type = _identify_file_input(+ content=content,+ head=raw_bytes[:FILE_HEAD_NB_BYTES],+ declared_mime_type=resolved_uri.mime_type or content.mime_type,+ file_name=content.filename,+ input_name=input_name,+ )- # The one form left is a local path- if not get_config().runtime.storage.is_upload_local_content_enabled:- return content+ if not context.is_relocation_enabled:+ if mime_type == content.mime_type:+ return content+ return content.model_copy(update={"mime_type": mime_type})++ # `{scope}/assets/`, NOT the old flat `normalized/`. That top-level+ # prefix was shared by every run of every tenant: the bytes of an input+ # someone pasted as a data: URL landed beside everyone else's, keyed+ # only by a random id. Under the run's own scope they are inside the+ # tenant's namespace and go away with the run.+ extension = _storage_key_extension(raw_bytes=raw_bytes, mime_type=mime_type, fallback_suffix=None)+ key = f"{context.storage_scope}/assets/{shortuuid.uuid()}.{extension}"+ storage_uri = await context.storage.store(data=raw_bytes, key=key, content_type=mime_type)+ public_url = await context.storage.public_url(uri=storage_uri)++ # model_copy preserves all type-specific fields+ return content.model_copy(+ update={+ "url": storage_uri,+ "public_url": public_url,+ "mime_type": mime_type,+ }+ )+++async def _prepare_local_path(+ *,+ content: NormalizableContent,+ resolved_uri: ResolvedLocalPath,+ input_name: str,+ context: _PreparationContext,+) -> NormalizableContent:+ local_path = Path(resolved_uri.path)+ if not (context.is_relocation_enabled and get_config().runtime.storage.is_upload_local_content_enabled):+ # Kept for its consumer to read. A path that cannot be read here is left unidentified: whoever+ # reads it later reports that failure, as it did before identification existed.+ try:+ head = await _load_local_head(local_path)+ except (OSError, ValueError):+ return content+ mime_type = _identify_file_input(+ content=content, head=head, declared_mime_type=content.mime_type, file_name=content.filename or local_path.name, input_name=input_name+ )+ if mime_type == content.mime_type:+ return content+ return content.model_copy(update={"mime_type": mime_type})- # Read local file, detect type, upload to storage. OSError covers+ # Read the local file, identify it, upload it to storage. OSError covers# the caller-controllable failure surface (FileNotFoundError,# IsADirectoryError, PermissionError, name-too-long, ...) and ValueError# the one left, a NUL byte in the path — all of them mean the supplied# path is unusable, an INPUT fault.try:- raw_bytes = await load_binary_async(Path(resolved_uri.path))+ raw_bytes = await load_binary_async(local_path)except (OSError, ValueError) as exc:msg = f"Input file cannot be read: '{resolved_uri.path}' ({type(exc).__name__})"raise PipelineInputContentError(msg) from exc- file_type = detect_file_type_from_bytes(raw_bytes)- key = f"{storage_scope}/assets/{shortuuid.uuid()}.{file_type.extension}"- storage_uri = await storage.store(data=raw_bytes, key=key, content_type=file_type.mime)- public_url = await storage.public_url(uri=storage_uri)+ mime_type = _identify_file_input(+ content=content,+ head=raw_bytes[:FILE_HEAD_NB_BYTES],+ declared_mime_type=content.mime_type,+ file_name=content.filename or local_path.name,+ input_name=input_name,+ )+ extension = _storage_key_extension(raw_bytes=raw_bytes, mime_type=mime_type, fallback_suffix=local_path.suffix.removeprefix(".") or None)+ key = f"{context.storage_scope}/assets/{shortuuid.uuid()}.{extension}"+ storage_uri = await context.storage.store(data=raw_bytes, key=key, content_type=mime_type)+ public_url = await context.storage.public_url(uri=storage_uri)return content.model_copy(update={"url": storage_uri,"public_url": public_url,- "mime_type": file_type.mime,+ "mime_type": mime_type,})+++def _identify_file_input(+ *, content: NormalizableContent, head: bytes | None, declared_mime_type: str | None, file_name: str | None, input_name: str+) -> str | None:+ """The MIME type that describes a file input, refusing an Image input whose bytes are not an image.++ The type comes from `identify_mime_type`: the sniffed type wins, the declared one stands when the+ sniff fails, and a bare zip is narrowed by the declared type or the file's name. The refusal fires+ only when the bytes were identified, so an image the sniffer cannot identify, an SVG for one, is+ left to its consumer, and so is a type the caller merely declared. A bare zip is not an image+ whatever it holds, so it is refused even when its precise format stays unknown.++ Raises:+ PipelineInputNotAnImageError: If the content is an ImageContent and its bytes identify a+ format other than an image.+ """+ mime_type = identify_mime_type(head=head, declared_mime_type=declared_mime_type, file_name=file_name)+ if not isinstance(content, ImageContent) or not head:+ return mime_type+ sniffed = guess_file_type_from_bytes(raw_bytes=head)+ if sniffed is None:+ return mime_type+ mime_type_to_describe = mime_type or sniffed.mime+ format_key = format_key_from_mime_type(mime_type=mime_type_to_describe)+ if format_key is None or format_key == IMAGE_FORMAT_KEY:+ return mime_type+ msg = (+ f"Input '{input_name}' expects an image, but the file is {describe_file_format(format_key=format_key, mime_type=mime_type_to_describe)}. "+ "Give an image file such as PNG, JPEG or WebP."+ )+ raise PipelineInputNotAnImageError(msg)+++async def _load_local_head(local_path: Path) -> bytes:+ """The first bytes of a local file, which is all identifying it needs."""+ async with aiofiles.open(local_path, "rb") as file_handle: # pyright: ignore[reportUnknownMemberType]+ return await file_handle.read(FILE_HEAD_NB_BYTES)+++def _storage_key_extension(*, raw_bytes: bytes, mime_type: str | None, fallback_suffix: str | None) -> str:+ """The extension a relocated file's storage key ends with.++ The sniffed extension when the bytes were identified, else the extension of the declared type,+ else the local file's own suffix, else `bin`.+ """+ sniffed = guess_file_type_from_bytes(raw_bytes=raw_bytes)+ if sniffed is not None and sniffed.mime == mime_type:+ return sniffed.extension+ if mime_type:+ extension = mime_type_to_extension(mime_type)+ if extension != UNKNOWN_FILE_TYPE:+ return extension+ return fallback_suffix or "bin"diff --git a/pipelex/tools/misc/filetype_utils.py b/pipelex/tools/misc/filetype_utils.pyindex feee863e6..85fe16c7c 100644--- a/pipelex/tools/misc/filetype_utils.py+++ b/pipelex/tools/misc/filetype_utils.py@@ -1,6 +1,7 @@import base64import binasciiimport mimetypes+from collections.abc import Collectionfrom pathlib import Pathfrom typing import Final@@ -156,3 +157,180 @@ def mime_type_to_extension(mime_type: str) -> str:# Remove leading dotreturn ext.removeprefix(".")+++#####################################################################################################+# Identifying a file input's format, and the format keys the format checks compare+#####################################################################################################++# How many leading bytes identify a file: exactly what `filetype` reads, so a head read of this+# size identifies a stored file as well as loading all of it would.+FILE_HEAD_NB_BYTES: Final[int] = 8192++# A declaration that says nothing: a file with this type, or with none, has no known format.+_GENERIC_MIME_TYPES: Final[frozenset[str]] = frozenset({"application/octet-stream"})++# A zip container the sniffer recognised without recognising what it contains, and the types that+# name a more precise zip-based format. An Office Open XML or OpenDocument file is a zip, and+# `filetype` names it only when its identifying entry sits among the first few, so a bare zip says+# only that the file is some zip: a declared zip-based type, or the file's name, says which.+_ZIP_MIME_TYPE: Final[str] = "application/zip"+_ZIP_BASED_DOCUMENT_MIME_PREFIXES: Final[tuple[str, ...]] = (+ "application/vnd.openxmlformats-officedocument.",+ "application/vnd.oasis.opendocument.",+)++# The family every image type belongs to, which is how models declare that they read images.+IMAGE_FORMAT_KEY: Final[str] = "image"++_FORMAT_KEY_DESCRIPTIONS: Final[dict[str, str]] = {+ "pdf": "a PDF document",+ "docx": "a Word document",+ "doc": "a Word 97-2003 document",+ "pptx": "a PowerPoint presentation",+ "ppt": "a PowerPoint 97-2003 presentation",+ "xlsx": "an Excel workbook",+ "xls": "an Excel 97-2003 workbook",+ "html": "an HTML page",+ "md": "a Markdown file",+ "csv": "a CSV file",+ "txt": "a plain-text file",+ "vtt": "a WebVTT caption file",+ "eml": "an email message",+ IMAGE_FORMAT_KEY: "an image",+}+++# How a list of readable formats names each key, in the order it lists them: `PDF and images`.+_FORMAT_KEY_LIST_NAMES: Final[dict[str, str]] = {+ "pdf": "PDF",+ "docx": "Word (.docx)",+ "doc": "Word 97-2003 (.doc)",+ "pptx": "PowerPoint (.pptx)",+ "ppt": "PowerPoint 97-2003 (.ppt)",+ "xlsx": "Excel (.xlsx)",+ "xls": "Excel 97-2003 (.xls)",+ "html": "HTML",+ "md": "Markdown (.md)",+ "csv": "CSV (.csv)",+ "txt": "plain text (.txt)",+ "vtt": "WebVTT (.vtt)",+ "eml": "email (.eml)",+ IMAGE_FORMAT_KEY: "images",+ "web_page": "web pages",+}+++def _base_mime_type(*, mime_type: str) -> str:+ """The MIME type without its parameters, lowercased: `Text/Plain; charset=utf-8` → `text/plain`."""+ return mime_type.split(";", 1)[0].strip().lower()+++def format_key_from_mime_type(*, mime_type: str | None) -> str | None:+ """The format key a file of this MIME type has, which is what every format check compares.++ Every `image/*` type is the `image` family, the key models declare to say they read images. Any+ other identified type is its extension (`pdf`, `docx`, `pptx`, `xlsx`, `doc`, `html`, `md`…). No+ type, the generic `application/octet-stream`, and a type the MIME database does not know have no+ key: the format is unknown, and a check leaves an unknown format to the provider.+ """+ if mime_type is None:+ return None+ base = _base_mime_type(mime_type=mime_type)+ if not base or base in _GENERIC_MIME_TYPES:+ return None+ if base.startswith("image/"):+ return IMAGE_FORMAT_KEY+ extension = mime_type_to_extension(base)+ if extension == UNKNOWN_FILE_TYPE:+ return None+ return extension+++def describe_format_key(*, format_key: str) -> str:+ """A format key in plain words, for a message: `docx` → `a Word document`, `json` → `a .json file`.++ The words carry no type: a message adds the extension or the MIME type that tells two keys apart.+ """+ if description := _FORMAT_KEY_DESCRIPTIONS.get(format_key):+ return description+ return f"a .{format_key} file"+++def describe_file_format(*, format_key: str, mime_type: str | None) -> str:+ """A file's format in plain words with the type that tells it apart: `a Word document (.docx)`.++ An image is told apart by its MIME type, since every image shares the `image` key:+ `an image (image/png)`.+ """+ description = _FORMAT_KEY_DESCRIPTIONS.get(format_key)+ if description is None:+ # `a .json file` already names its extension.+ return describe_format_key(format_key=format_key)+ if format_key == IMAGE_FORMAT_KEY:+ return f"{description} ({mime_type})" if mime_type else description+ return f"{description} (.{format_key})"+++def describe_format_keys(*, format_keys: Collection[str]) -> str:+ """A set of readable formats in plain words, in a stable order: `PDF, Word (.docx) and images`."""+ known_keys = [format_key for format_key in _FORMAT_KEY_LIST_NAMES if format_key in format_keys]+ other_keys = sorted(set(format_keys) - set(_FORMAT_KEY_LIST_NAMES))+ names = [_FORMAT_KEY_LIST_NAMES[format_key] for format_key in known_keys] + [f".{format_key}" for format_key in other_keys]+ if not names:+ return "no file format"+ if len(names) == 1:+ return names[0]+ return f"{', '.join(names[:-1])} and {names[-1]}"+++def guess_file_type_from_bytes(*, raw_bytes: bytes) -> FileType | None:+ """The type the leading bytes identify, or `None` when they identify none.++ Unlike `detect_file_type_from_bytes`, which raises, this is the form for a caller that has a+ fallback: text formats (plain text, Markdown, CSV, JSON, HTML) carry no signature to sniff.+ """+ if not raw_bytes:+ return None+ kind = filetype.guess(raw_bytes[:FILE_HEAD_NB_BYTES]) # pyright: ignore[reportUnknownMemberType]+ if kind is None:+ return None+ return FileType(extension=f"{kind.extension}", mime=f"{kind.mime}")+++def identify_mime_type(*, head: bytes | None, declared_mime_type: str | None, file_name: str | None) -> str | None:+ """The MIME type that describes a file, from its leading bytes, what the caller declared and its name.++ The bytes are the truth: when they identify a type, it replaces any declared one, so a+ `data:image/png` URL holding a PDF is a PDF. When they identify none, which is the case for text+ formats, the declared type stands. No head (a file that was not read) keeps the declared type.++ A sniffed bare zip is the one identification that does not settle the format: it may be an+ Office file whose identifying entry came late. An Office or OpenDocument type says which, whether+ it is declared or given by the file's name, and it wins over a bare zip from either source, so a+ `report.docx` declared `application/zip` is a Word file. A bare zip, declared or named, is a zip.+ Otherwise the format is unknown rather than a zip, so no check refuses a valid Office file for+ the order of its entries.++ Args:+ head: The file's leading bytes, `None` when it was not read.+ declared_mime_type: The type the caller declared, if any.+ file_name: The file's name or the last segment of its path, if it has one.++ Returns:+ The MIME type, `None` when the format is unknown.+ """+ sniffed = guess_file_type_from_bytes(raw_bytes=head) if head else None+ if sniffed is None:+ return declared_mime_type+ if sniffed.mime != _ZIP_MIME_TYPE:+ return sniffed.mime+ named_mime_type = _MIME_DB.guess_type(file_name, strict=True)[0] if file_name else None+ candidate_mime_types = [mime_type for mime_type in (declared_mime_type, named_mime_type) if mime_type is not None]+ for candidate_mime_type in candidate_mime_types:+ if _base_mime_type(mime_type=candidate_mime_type).startswith(_ZIP_BASED_DOCUMENT_MIME_PREFIXES):+ return candidate_mime_type+ for candidate_mime_type in candidate_mime_types:+ if _base_mime_type(mime_type=candidate_mime_type) == _ZIP_MIME_TYPE:+ return candidate_mime_type+ return Nonediff --git a/pipelex/tools/storage/gcp_storage_provider.py b/pipelex/tools/storage/gcp_storage_provider.pyindex 8d085a20f..161e3767d 100644--- a/pipelex/tools/storage/gcp_storage_provider.py+++ b/pipelex/tools/storage/gcp_storage_provider.py@@ -132,6 +132,60 @@ class GcpStorageProvider(StorageProviderAbstract):"""return await asyncio.to_thread(self._load_with_metadata_sync, key)+ def _load_head_sync(self, key: str, *, nb_bytes: int) -> bytes:+ """Synchronous implementation of the head read for use with to_thread.++ Args:+ key: Storage key (without scheme prefix).+ nb_bytes: How many leading bytes to read.++ Returns:+ At most `nb_bytes` leading bytes of the object.++ Raises:+ StorageFileNotFoundError: If the object does not exist.+ StorageGcpError: If the GCS operation fails.+ """+ bucket = self._get_bucket()++ from google.api_core.exceptions import ( # type: ignore[import-untyped] # ruff: ignore[import-outside-top-level] - optional dependency, lazy import+ GoogleAPIError,+ NotFound,+ RequestRangeNotSatisfiable,+ )++ try:+ blob = bucket.blob(key)+ # The end offset is inclusive.+ data: bytes = blob.download_as_bytes(start=0, end=nb_bytes - 1)+ return data[:nb_bytes]+ except NotFound as exc: # pyright: ignore[reportUnknownVariableType]+ msg = f"Object not found in GCS: '{key}'"+ raise StorageFileNotFoundError(msg) from exc+ except RequestRangeNotSatisfiable: # pyright: ignore[reportUnknownVariableType]+ # GCS refuses any range over an empty object: its head is empty.+ return b""+ except GoogleAPIError as exc: # pyright: ignore[reportUnknownVariableType]+ msg = f"Failed to load object from GCS: '{key}': {exc}"+ raise StorageGcpError(msg) from exc++ @override+ async def _load_head(self, key: str, *, nb_bytes: int) -> bytes:+ """Load the first bytes of a GCS object with a ranged download, instead of the whole object.++ Args:+ key: Storage key (without scheme prefix).+ nb_bytes: How many leading bytes to read. An object shorter than this is read whole.++ Returns:+ At most `nb_bytes` leading bytes of the object.++ Raises:+ StorageFileNotFoundError: If the object does not exist.+ StorageGcpError: If the GCS operation fails.+ """+ return await asyncio.to_thread(self._load_head_sync, key, nb_bytes=nb_bytes)+def _store_sync(self, data: bytes, *, key: str, content_type: str | None) -> None:"""Synchronous implementation of store for use with to_thread.diff --git a/pipelex/tools/storage/s3_storage_provider.py b/pipelex/tools/storage/s3_storage_provider.pyindex e07c24b24..394cbe454 100644--- a/pipelex/tools/storage/s3_storage_provider.py+++ b/pipelex/tools/storage/s3_storage_provider.py@@ -136,6 +136,56 @@ class S3StorageProvider(StorageProviderAbstract):msg = f"S3 backend error for key '{key}': {type(exc).__name__}"raise StorageS3Error(msg) from exc+ @override+ async def _load_head(self, key: str, *, nb_bytes: int) -> bytes:+ """Load the first bytes of an S3 object with a ranged GET, instead of the whole object.++ Args:+ key: Storage key (without scheme prefix).+ nb_bytes: How many leading bytes to read. An object shorter than this is read whole.++ Returns:+ At most `nb_bytes` leading bytes of the object.++ Raises:+ StorageFileNotFoundError: If the object does not exist.+ StorageS3Error: If the S3 operation fails (any other ClientError or BotoCoreError).+ """+ from botocore.exceptions import ( # ruff: ignore[import-outside-top-level] - optional dependency, lazy import+ BotoCoreError,+ ClientError,+ )++ session = self._get_session()+ client_config = self._get_client_config()++ async with session.create_client(**client_config) as client:+ try:+ # The Range header's end offset is inclusive.+ response = await client.get_object(Bucket=self._bucket_name, Key=key, Range=f"bytes=0-{nb_bytes - 1}")+ async with response["Body"] as stream:+ data: bytes = await stream.read()+ return data[:nb_bytes]+ except client.exceptions.NoSuchKey as exc:+ msg = f"Object not found in S3: '{key}'"+ raise StorageFileNotFoundError(msg) from exc+ except client.exceptions.NoSuchBucket as exc:+ msg = f"Bucket not found in S3: '{self._bucket_name}'"+ raise StorageS3Error(msg) from exc+ except ClientError as exc:+ error_code = (exc.response.get("Error") or {}).get("Code", "Unknown")+ if error_code == "InvalidRange":+ # S3 refuses any range over an empty object: its head is empty.+ return b""+ if error_code == "NoSuchKey":+ msg = f"Object not found in S3: '{key}'"+ raise StorageFileNotFoundError(msg) from exc+ msg = f"S3 ClientError ({error_code}) for key '{key}'"+ raise StorageS3Error(msg) from exc+ except BotoCoreError as exc:+ msg = f"S3 backend error for key '{key}': {type(exc).__name__}"+ raise StorageS3Error(msg) from exc+@overrideasync def _store(self, data: bytes, *, key: str, content_type: str | None) -> None:"""Store bytes to an S3 object.diff --git a/pipelex/tools/storage/storage_provider_abstract.py b/pipelex/tools/storage/storage_provider_abstract.pyindex b858c5778..469da08de 100644--- a/pipelex/tools/storage/storage_provider_abstract.py+++ b/pipelex/tools/storage/storage_provider_abstract.py@@ -94,6 +94,36 @@ class StorageProviderAbstract(ABC):StoredData containing bytes and optional MIME type."""+ async def load_head(self, uri: str, *, nb_bytes: int) -> bytes:+ """Load the first bytes of a stored object, which is all identifying its format needs.++ Args:+ uri: Full URI including PIPELEX_STORAGE_SCHEME prefix.+ nb_bytes: How many leading bytes to read. An object shorter than this is read whole.++ Returns:+ At most `nb_bytes` leading bytes of the object.+ """+ key = self._strip_scheme(uri)+ return await self._load_head(key, nb_bytes=nb_bytes)++ async def _load_head(self, key: str, *, nb_bytes: int) -> bytes:+ """Load the first bytes of an object by key.++ This default loads the whole object and slices it, so a provider that cannot read a byte+ range, an external storage plugin included, keeps working unchanged. A provider that can+ read a range overrides it.++ Args:+ key: Storage key (without scheme prefix).+ nb_bytes: How many leading bytes to read.++ Returns:+ At most `nb_bytes` leading bytes of the object.+ """+ stored_data = await self._load_with_metadata(key)+ return stored_data.data[:nb_bytes]+async def store(self, data: bytes, *, key: str, content_type: str | None = None) -> str:"""Store data and return full URI with scheme.diff --git a/subject_grants.toml b/subject_grants.tomlindex f3e96fc9f..79a9d5731 100644--- a/subject_grants.toml+++ b/subject_grants.toml@@ -4608,6 +4608,10 @@ rationale = "Verb-object: loads the given library dirs; options stay keyword."param = "pipe"rationale = "Verb-object: prepares the job for the pipe; job fields stay keyword."+["pipelex/pipeline/file_input_consumers.py::collect_file_input_consumers"]+param = "entry_pipe"+rationale = "Collects the file input consumers of the entry pipe, its single operand."+["pipelex/pipeline/fixes/applicability.py::is_safe_fix_for_load_scope"]param = "fix"rationale = "Predicate on the fix; is_single_file is context."@@ -4776,29 +4780,21 @@ rationale = "Reads as a verb-object sentence over the loaded pipes, mirroring buparam = "pipes"rationale = "The output twin of build_input_form; the two are called side by side and must read alike."-["pipelex/pipeline/input_normalizer.py::_normalize_list"]-param = "items"-rationale = "Verb-object: normalizes the items; context stays keyword."--["pipelex/pipeline/input_normalizer.py::_normalize_list_content"]-param = "list_content"-rationale = "Verb-object: normalizes the list content; context stays keyword."--["pipelex/pipeline/input_normalizer.py::_normalize_structured_content"]-param = "structured_content"-rationale = "Verb-object: normalizes the structured content; context stays keyword."+["pipelex/pipeline/input_normalizer.py::_load_local_head"]+param = "local_path"+rationale = "Reads the head of the local file, its single operand."-["pipelex/pipeline/input_normalizer.py::_normalize_url_content"]-param = "content"-rationale = "Verb-object: normalizes the content; context stays keyword."+["pipelex/pipeline/input_normalizer.py::collect_file_inputs"]+param = "working_memory"+rationale = "Reads the file inputs out of the working memory, its single operand."-["pipelex/pipeline/input_normalizer.py::_normalize_value"]-param = "value"-rationale = "Verb-object: normalizes the value; context stays keyword."+["pipelex/pipeline/input_normalizer.py::format_file_input_path"]+param = "path"+rationale = "Renders the file input path, its single operand."-["pipelex/pipeline/input_normalizer.py::normalize_data_urls_to_storage"]+["pipelex/pipeline/input_normalizer.py::prepare_file_inputs"]param = "working_memory"-rationale = "Verb-object: rewrites data URLs within the working memory; single operand."+rationale = "Verb-object: prepares the file inputs within the working memory; single operand."["pipelex/pipeline/liftable_pipes.py::build_liftable_pipes"]param = "taint_analyses"@@ -7052,6 +7048,10 @@ rationale = "Positional-Callable re.sub repl: match is the single operand re pasparam = "key"rationale = "Keyed accessor by the key; single operand."+["pipelex/tools/storage/gcp_storage_provider.py::GcpStorageProvider._load_head_sync"]+param = "key"+rationale = "Keyed accessor by the storage key, called positionally through asyncio.to_thread."+["pipelex/tools/storage/gcp_storage_provider.py::GcpStorageProvider._load_with_metadata_sync"]param = "key"rationale = "Parses/reads the key; single operand."@@ -7076,6 +7076,10 @@ rationale = "Factory from source: builds the result from the key; single operandparam = "key"rationale = "Keyed accessor by the key; single operand."+["pipelex/tools/storage/storage_provider_abstract.py::StorageProviderAbstract._load_head"]+param = "key"+rationale = "Keyed accessor by the storage key; single operand."+["pipelex/tools/storage/storage_provider_abstract.py::StorageProviderAbstract._load_with_metadata"]param = "key"rationale = "Parses/reads the key; single operand."@@ -7092,6 +7096,10 @@ rationale = "Keyed accessor by the uri; single operand."param = "uri"rationale = "Keyed accessor by the uri; single operand."+["pipelex/tools/storage/storage_provider_abstract.py::StorageProviderAbstract.load_head"]+param = "uri"+rationale = "Keyed accessor by the uri; single operand."+["pipelex/tools/storage/storage_provider_abstract.py::StorageProviderAbstract.load_with_metadata"]param = "uri"rationale = "Parses/reads the uri; single operand."diff --git a/tests/cases/documents.py b/tests/cases/documents.pyindex 73a1df31d..0cc95e9be 100644--- a/tests/cases/documents.py+++ b/tests/cases/documents.py@@ -22,12 +22,38 @@ class DocumentTestCases:PDF_FILE_PATH_4,]DOCX_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/CV-ELIAS-THORNE.docx"- DOCUMENT_FILE_PATHS: ClassVar[list[str]] = [- PDF_FILE_PATH_1,- PDF_FILE_PATH_2,- PDF_FILE_PATH_3,- PDF_FILE_PATH_4,- DOCX_FILE_PATH_1,+ PPTX_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Quarterly-Review.pptx"+ XLSX_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Office-Budget.xlsx"+ HTML_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Solar-Article.html"+ MD_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Reading-List.md"+ CSV_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Team-Scores.csv"+ TXT_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Field-Notes.txt"+ VTT_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Interview-Captions.vtt"+ EML_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/Meeting-Invite.eml"++ DOCX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+ PPTX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.presentationml.presentation"+ XLSX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"+ HTML_MIME_TYPE = "text/html"+ MD_MIME_TYPE = "text/markdown"+ CSV_MIME_TYPE = "text/csv"+ TXT_MIME_TYPE = "text/plain"+ VTT_MIME_TYPE = "text/vtt"+ EML_MIME_TYPE = "message/rfc822"++ # Documents in a format other than PDF, each with the MIME type run setup would stamp on it and a+ # phrase its extraction must contain. An extract model reads one only if it declares its format,+ # and is refused with ExtractInputFormatError otherwise.+ NON_PDF_DOCUMENT_CASES: ClassVar[list[tuple[str, str, str]]] = [ # file_path, mime_type, expected_phrase+ (DOCX_FILE_PATH_1, DOCX_MIME_TYPE, "ELIAS THORNE"),+ (PPTX_FILE_PATH_1, PPTX_MIME_TYPE, "Revenue grew twelve percent"),+ (XLSX_FILE_PATH_1, XLSX_MIME_TYPE, "Salaries"),+ (HTML_FILE_PATH_1, HTML_MIME_TYPE, "eight planets"),+ (MD_FILE_PATH_1, MD_MIME_TYPE, "Books the team recommends"),+ (CSV_FILE_PATH_1, CSV_MIME_TYPE, "Copper Kettle"),+ (TXT_FILE_PATH_1, TXT_MIME_TYPE, "tide pools"),+ (VTT_FILE_PATH_1, VTT_MIME_TYPE, "project you led"),+ (EML_FILE_PATH_1, EML_MIME_TYPE, "analytical engine budget"),]# Remote URLsdiff --git a/tests/data/documents/Field-Notes.txt b/tests/data/documents/Field-Notes.txtnew file mode 100644index 000000000..064672dfa--- /dev/null+++ b/tests/data/documents/Field-Notes.txt@@ -0,0 +1,3 @@+Field notes from the coastal survey.++The tide pools held more starfish than last year.diff --git a/tests/data/documents/Interview-Captions.vtt b/tests/data/documents/Interview-Captions.vttnew file mode 100644index 000000000..913e1d7fd--- /dev/null+++ b/tests/data/documents/Interview-Captions.vtt@@ -0,0 +1,7 @@+WEBVTT++00:00:01.000 --> 00:00:04.000+Thank you for joining the interview today.++00:00:04.500 --> 00:00:08.000+Tell us about the project you led last year.diff --git a/tests/data/documents/Meeting-Invite.eml b/tests/data/documents/Meeting-Invite.emlnew file mode 100644index 000000000..fe4249c4c--- /dev/null+++ b/tests/data/documents/Meeting-Invite.eml@@ -0,0 +1,11 @@+From: Ada Lovelace <[email protected]>+To: Charles Babbage <[email protected]>+Subject: Planning meeting on Thursday+Date: Thu, 01 Oct 2026 09:00:00 +0000+Content-Type: text/plain; charset=utf-8++Hello Charles,++Shall we review the analytical engine budget on Thursday at ten?++Adadiff --git a/tests/data/documents/Office-Budget.xlsx b/tests/data/documents/Office-Budget.xlsxnew file mode 100644index 000000000..00d6dfc08Binary files /dev/null and b/tests/data/documents/Office-Budget.xlsx differdiff --git a/tests/data/documents/Quarterly-Review.pptx b/tests/data/documents/Quarterly-Review.pptxnew file mode 100644index 000000000..c9a6f4464Binary files /dev/null and b/tests/data/documents/Quarterly-Review.pptx differdiff --git a/tests/data/documents/Reading-List.md b/tests/data/documents/Reading-List.mdnew file mode 100644index 000000000..4fbe66f06--- /dev/null+++ b/tests/data/documents/Reading-List.md@@ -0,0 +1,8 @@+# Reading List++Books the team recommends for the autumn.++- *The Pragmatic Programmer*, for anyone starting out+- *Designing Data-Intensive Applications*, for the backend crew++The list is reviewed every quarter.diff --git a/tests/data/documents/Solar-Article.html b/tests/data/documents/Solar-Article.htmlnew file mode 100644index 000000000..2efd96084--- /dev/null+++ b/tests/data/documents/Solar-Article.html@@ -0,0 +1,10 @@+<!DOCTYPE html>+<html lang="en">+<head><meta charset="utf-8"><title>The Solar System</title></head>+<body>+<h1>The Solar System</h1>+<p>The Solar System has eight planets orbiting the Sun.</p>+<h2>Inner planets</h2>+<p>Mercury, Venus, Earth and Mars are rocky worlds.</p>+</body>+</html>diff --git a/tests/data/documents/Team-Scores.csv b/tests/data/documents/Team-Scores.csvnew file mode 100644index 000000000..f436baa7e--- /dev/null+++ b/tests/data/documents/Team-Scores.csv@@ -0,0 +1,4 @@+team,quarter,score+Harbour Lights,Q3,87+Copper Kettle,Q3,92+Northwind,Q3,78diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex ce6893101..e0e6a5959 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -92,6 +92,7 @@ ExplicitConceptIncompatibleError | Explicit concept incompatible | https://docs.ExtractCapabilityError | Extract capability | https://docs.pipelex.com/latest/errors/extract-capability-error/ExtractHandleNotFoundError | Extract handle not found | https://docs.pipelex.com/latest/errors/extract-handle-not-found-error/ExtractInputError | Extract input | https://docs.pipelex.com/latest/errors/extract-input-error/+ExtractInputFormatError | Extract input format | https://docs.pipelex.com/latest/errors/extract-input-format-error/ExtractJobFailureError | Extract job failure | https://docs.pipelex.com/latest/errors/extract-job-failure-error/ExtractModelNotFoundError | Extract model not found | https://docs.pipelex.com/latest/errors/extract-model-not-found-error/ExtractOutputError | Extract output | https://docs.pipelex.com/latest/errors/extract-output-error/@@ -274,6 +275,9 @@ PipelexSetupError | Pipelex setup | https://docs.pipelex.com/latest/errors/pipelPipelexUnexpectedError | Unexpected internal error | https://docs.pipelex.com/latest/errors/pipelex-unexpected-error/PipelineExecutionError | Pipeline execution | https://docs.pipelex.com/latest/errors/pipeline-execution-error/PipelineInputContentError | Pipeline input content | https://docs.pipelex.com/latest/errors/pipeline-input-content-error/+PipelineInputFormatError | Pipeline input format | https://docs.pipelex.com/latest/errors/pipeline-input-format-error/+PipelineInputFormatUnsupportedError | Pipeline input format unsupported | https://docs.pipelex.com/latest/errors/pipeline-input-format-unsupported-error/+PipelineInputNotAnImageError | Pipeline input not an image | https://docs.pipelex.com/latest/errors/pipeline-input-not-an-image-error/PipelineInputUrlInvalidError | Pipeline input url invalid | https://docs.pipelex.com/latest/errors/pipeline-input-url-invalid-error/PipelineInputUrlMissingError | Pipeline input url missing | https://docs.pipelex.com/latest/errors/pipeline-input-url-missing-error/PipelineManagerAlreadyExistsError | Pipeline manager already exists | https://docs.pipelex.com/latest/errors/pipeline-manager-already-exists-error/@@ -287,6 +291,7 @@ PortkeyCredentialsError | Portkey credentials | https://docs.pipelex.com/latest/PortkeyError | Portkey | https://docs.pipelex.com/latest/errors/portkey-error/PortkeyFactoryError | Portkey factory | https://docs.pipelex.com/latest/errors/portkey-factory-error/PromptDocumentFactoryError | Prompt document factory | https://docs.pipelex.com/latest/errors/prompt-document-factory-error/+PromptDocumentFormatError | Prompt document format | https://docs.pipelex.com/latest/errors/prompt-document-format-error/PromptImageFactoryError | Prompt image factory | https://docs.pipelex.com/latest/errors/prompt-image-factory-error/PromptImageFormatError | Prompt image format | https://docs.pipelex.com/latest/errors/prompt-image-format-error/PyPdfium2RendererError | Py pdfium 2 renderer | https://docs.pipelex.com/latest/errors/py-pdfium2-renderer-error/diff --git a/tests/integration/pipelex/cogt/image/test_data_url_image_flow.py b/tests/integration/pipelex/cogt/image/test_data_url_image_flow.pyindex c36ce7d11..e9244b80e 100644--- a/tests/integration/pipelex/cogt/image/test_data_url_image_flow.py+++ b/tests/integration/pipelex/cogt/image/test_data_url_image_flow.py@@ -15,7 +15,7 @@ from pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.structured_content import StructuredContentfrom pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.pipeline.input_normalizer import normalize_data_urls_to_storage+from pipelex.pipeline.input_normalizer import prepare_file_inputsfrom pipelex.tools.storage.in_memory_storage_provider import InMemoryStorageProviderfrom pipelex.tools.uri.prepared_file import PreparedFileBase64from pipelex.urls import URLs@@ -85,7 +85,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify the URL was converted to pipelex-storage://normalized_stuff = normalized_memory.get_stuff("test_image")@@ -122,7 +122,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize to storage- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)normalized_stuff = normalized_memory.get_stuff("test_image")assert isinstance(normalized_stuff.content, ImageContent)storage_uri = normalized_stuff.content.url@@ -163,7 +163,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify all images were convertednormalized_stuff = normalized_memory.get_stuff("image_list")@@ -194,7 +194,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify URL was NOT changednormalized_stuff = normalized_memory.get_stuff("test_image")@@ -224,7 +224,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify the embedded image URL was converted to pipelex-storage://normalized_stuff = normalized_memory.get_stuff("test_article")@@ -264,7 +264,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify all image URLs were convertednormalized_stuff = normalized_memory.get_stuff("nested_article")@@ -314,7 +314,7 @@ class TestDataUrlImageFlow:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify all images in the list were convertednormalized_stuff = normalized_memory.get_stuff("gallery")diff --git a/tests/integration/pipelex/cogt/test_extract.py b/tests/integration/pipelex/cogt/test_extract.pyindex 0a84de183..dffa9429c 100644--- a/tests/integration/pipelex/cogt/test_extract.py+++ b/tests/integration/pipelex/cogt/test_extract.py@@ -2,11 +2,13 @@ import pytestfrom pipelex import pretty_printfrom pipelex.cogt.content_generation.generated_content_factory import GeneratedContentFactory+from pipelex.cogt.exceptions import ExtractInputFormatErrorfrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobParamsfrom pipelex.cogt.extract.extract_job_factory import ExtractJobFactoryfrom pipelex.runtime_hub import get_extract_workerfrom pipelex.system.job_metadata import JobMetadata+from pipelex.tools.misc.filetype_utils import format_key_from_mime_typefrom tests.cases import DocumentTestCases, ImageTestCasesfrom tests.integration.pipelex.fixtures.model_combo import ModelCombo@@ -16,7 +18,7 @@ from tests.integration.pipelex.fixtures.model_combo import ModelCombo@pytest.mark.asyncio(loop_scope="class")@pytest.mark.filterwarnings("ignore:Accessing the 'model_fields' attribute on the instance is deprecated:DeprecationWarning")class TestExtract:- @pytest.mark.parametrize("file_path", DocumentTestCases.DOCUMENT_FILE_PATHS)+ @pytest.mark.parametrize("file_path", DocumentTestCases.PDF_FILE_PATHS)async def test_extract_pdf_path(self,generated_content_factory: GeneratedContentFactory,@@ -48,6 +50,38 @@ class TestExtract:for page_index, page_content in enumerate(page_contents):pretty_print(page_content, title=f"Page {page_index}")+ @pytest.mark.parametrize(("file_path", "mime_type", "expected_phrase"), DocumentTestCases.NON_PDF_DOCUMENT_CASES)+ async def test_extract_non_pdf_document(+ self,+ job_metadata: JobMetadata,+ extract_combo: ModelCombo,+ extract_job_params: ExtractJobParams,+ file_path: str,+ mime_type: str,+ expected_phrase: str,+ ):+ """A model declaring the document's format extracts it; any other refuses it before the provider call."""+ extract_worker = get_extract_worker(extract_handle=extract_combo.handle)+ if extract_job_params.should_caption_images and not extract_worker.is_caption_supported:+ msg = f"Image captioning is not supported for this extract worker: '{extract_worker.desc}'"+ pytest.skip(msg)+ if not (extract_worker.is_pdf_supported or extract_worker.is_web_page_supported):+ msg = f"Document extraction is not supported for this extract worker: '{extract_worker.desc}'"+ pytest.skip(msg)+ extract_job = ExtractJobFactory.make_extract_job(+ extract_input=ExtractInput(document_uri=file_path, mime_type=mime_type, input_name="document"),+ extract_job_params=extract_job_params,+ job_metadata=job_metadata,+ )+ format_key = format_key_from_mime_type(mime_type=mime_type)+ if format_key not in extract_worker.inference_model.readable_formats_for_extract:+ with pytest.raises(ExtractInputFormatError, match="Input 'document' is"):+ await extract_worker.extract_pages(extract_job=extract_job)+ return+ extract_output = await extract_worker.extract_pages(extract_job=extract_job)+ extracted_text = "\n".join(page.text or "" for page in extract_output.pages.values())+ assert expected_phrase in extracted_text+@pytest.mark.parametrize("url", DocumentTestCases.DOCUMENT_URLS)async def test_extract_pdf_url(self,@@ -163,7 +197,7 @@ class TestExtract:for page_index, page_content in enumerate(page_contents):pretty_print(page_content, title=f"Page {page_index}")- @pytest.mark.parametrize("file_path", DocumentTestCases.DOCUMENT_FILE_PATHS)+ @pytest.mark.parametrize("file_path", DocumentTestCases.PDF_FILE_PATHS)async def test_extract_image_save(self,job_metadata: JobMetadata,diff --git a/tests/integration/pipelex/pipeline/test_execution_seams.py b/tests/integration/pipelex/pipeline/test_execution_seams.pyindex 640b74dd0..78508db8f 100644--- a/tests/integration/pipelex/pipeline/test_execution_seams.py+++ b/tests/integration/pipelex/pipeline/test_execution_seams.py@@ -189,7 +189,7 @@ class TestExecutionSeams:try:assert qualified_main_pipe is not Nonepipe = get_required_entry_pipe(pipe_code=qualified_main_pipe)- normalize_spy = mocker.spy(execution_seams_module, "normalize_data_urls_to_storage")+ normalize_spy = mocker.spy(execution_seams_module, "prepare_file_inputs")normalize_config = (get_config().interpreter.pipeline_execution.with_execution_overrides(@@ -236,7 +236,7 @@ class TestExecutionSeams:try:assert qualified_main_pipe is not Nonepipe = get_required_entry_pipe(pipe_code=qualified_main_pipe)- normalize_spy = mocker.spy(execution_seams_module, "normalize_data_urls_to_storage")+ normalize_spy = mocker.spy(execution_seams_module, "prepare_file_inputs")normalize_config = (get_config().interpreter.pipeline_execution.with_execution_overrides(diff --git a/tests/integration/pipelex/pipeline/test_file_input_consumers.py b/tests/integration/pipelex/pipeline/test_file_input_consumers.pynew file mode 100644index 000000000..b085ba1a0--- /dev/null+++ b/tests/integration/pipelex/pipeline/test_file_input_consumers.py@@ -0,0 +1,391 @@+"""The static walk that finds, for each input slot of an entry pipe, the operators that will consume its files.++It follows the slots by name through the controllers, the way the absence-taint analysis does: a+sequence visits its steps in order and stops following a slot a step overwrites, a batch maps the+list slot to its item slot, a parallel visits every branch, and a condition or a liftable step makes+whatever lies below it conditional. A `PipeFunc`, a `PipeCompose` or an unresolved pipe consumes+nothing as far as the walk knows. A `PipeExtract` consumes its document input with the formats its+resolved model reads, and a `PipeLLM` the documents its prompt references, by dotted path.+"""++from collections.abc import Callable++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.interpreter_hub import get_library_manager, get_required_pipe+from pipelex.kernel.llm_ops import resolve_llm_setting_for_object, resolve_llm_setting_for_text+from pipelex.mthds_parsing.parser import MthdsParser+from pipelex.pipeline import file_input_consumers as file_input_consumers_module+from pipelex.pipeline.file_input_consumers import FileConsumerKind, FileInputConsumer, collect_file_input_consumers+from pipelex.runtime_hub import get_model_deck+from pipelex.system.registries.func_registry import func_registry++PDF_ONLY_MODEL = "pypdfium2-extract-pdf"+DOCLING_MODEL = "docling-extract-text"+DOCLING_FORMATS = frozenset({"pdf", "docx", "pptx", "xlsx", "html", "md", "csv", "txt", "vtt", "eml", "image"})++_EXTRACTORS_MTHDS = f"""+[pipe.extract_pdf]+type = "PipeExtract"+description = "Extract a transcript with a model reading PDF only"+inputs = {{ transcript = "Document" }}+output = "Page[]"+model = "{PDF_ONLY_MODEL}"++[pipe.extract_any]+type = "PipeExtract"+description = "Extract a transcript with a model reading Office files too"+inputs = {{ transcript = "Document" }}+output = "Page[]"+model = "{DOCLING_MODEL}"+"""+++def file_consumers_replace_transcript(working_memory: WorkingMemory) -> DocumentContent:+ return DocumentContent(url=working_memory.get_stuff_as_document(name="transcript").url)+++def _bundle(*, domain: str, pipes: str) -> str:+ return f'domain = "{domain}"\ndescription = "File input consumers test"\n{pipes}\n{_EXTRACTORS_MTHDS}'+++def _consumers(*, load_empty_library: Callable[[], str], domain: str, pipes: str) -> dict[str, list[FileInputConsumer]]:+ """Load the bundle into a fresh library and walk its `main` pipe."""+ library_id = load_empty_library()+ blueprint = MthdsParser.make_pipelex_bundle_blueprint(mthds_content=_bundle(domain=domain, pipes=pipes))+ get_library_manager().load_from_blueprints(library_id=library_id, blueprints=[blueprint])+ return collect_file_input_consumers(get_required_pipe(pipe_code=f"{domain}.main"))+++def _summary(consumers: list[FileInputConsumer]) -> list[tuple[str, str, bool]]:+ return sorted((consumer.pipe_code, consumer.display_consumed_path, consumer.is_conditional) for consumer in consumers)+++class TestFileInputConsumers:+ @pytest.fixture(autouse=True)+ def register_funcs(self):+ func_registry.register_function(file_consumers_replace_transcript)+ yield+ if func_registry.has_function(file_consumers_replace_transcript.__name__):+ func_registry.unregister_function_by_name(file_consumers_replace_transcript.__name__)++ def test_a_sequence_step_consumes_the_input(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Extract the transcript"+inputs = { transcript = "Document" }+output = "Page[]"+steps = [{ pipe = "extract_pdf", result = "pages" }]+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_seq", pipes=pipes)++ (consumer,) = consumers["transcript"]+ assert consumer.pipe_code == "extract_pdf"+ assert consumer.consumed_path == ("transcript",)+ assert consumer.model == PDF_ONLY_MODEL+ assert consumer.readable_formats == frozenset({"pdf"})+ assert consumer.reads_web_pages is False+ assert consumer.is_conditional is False++ def test_a_slot_overwritten_before_the_consumer_is_not_followed(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Replace the transcript, then extract it"+inputs = { transcript = "Document" }+output = "Page[]"+steps = [+ { pipe = "replace_transcript", result = "transcript" },+ { pipe = "extract_pdf", result = "pages" },+]++[pipe.replace_transcript]+type = "PipeFunc"+description = "Replace the transcript with another document"+inputs = { transcript = "Document" }+output = "Document"+function_name = "file_consumers_replace_transcript"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_overwrite", pipes=pipes)++ assert consumers.get("transcript", []) == []++ @pytest.mark.parametrize(+ ("domain", "main_inputs", "converter_pipe"),+ [+ (+ "fic_nested_overwrite",+ 'transcript = "Document"',+ """+[pipe.convert_transcript]+type = "PipeSequence"+description = "Convert the transcript in place"+inputs = { transcript = "Document" }+output = "Document"+steps = [{ pipe = "replace_transcript", result = "transcript" }]+""",+ ),+ (+ "fic_condition_overwrite",+ 'transcript = "Document", mode = "Text"',+ """+[pipe.convert_transcript]+type = "PipeCondition"+description = "Convert the transcript in place when asked to"+inputs = { transcript = "Document", mode = "Text" }+output = "Document"+expression = "mode"+default_outcome = "fail"++[pipe.convert_transcript.outcomes]+convert = "convert_in_place"++[pipe.convert_in_place]+type = "PipeSequence"+description = "Convert the transcript in place"+inputs = { transcript = "Document" }+output = "Document"+steps = [{ pipe = "replace_transcript", result = "transcript" }]+""",+ ),+ ],+ )+ def test_a_slot_a_nested_controller_overwrites_is_not_followed(+ self, load_empty_library: Callable[[], str], domain: str, main_inputs: str, converter_pipe: str+ ):+ """A nested sequence or a condition outcome runs on the caller's working memory, so what its steps write overwrites the caller's slots."""+ pipes = f"""+[pipe.main]+type = "PipeSequence"+description = "Convert the transcript, then extract it"+inputs = {{ {main_inputs} }}+output = "Page[]"+steps = [+ {{ pipe = "convert_transcript", result = "converted" }},+ {{ pipe = "extract_pdf", result = "pages" }},+]+{converter_pipe}+[pipe.replace_transcript]+type = "PipeFunc"+description = "Replace the transcript with another document"+inputs = {{ transcript = "Document" }}+output = "Document"+function_name = "file_consumers_replace_transcript"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain=domain, pipes=pipes)++ assert consumers.get("transcript", []) == []++ def test_a_batch_step_maps_the_list_slot_to_its_item_slot(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Extract every transcript"+inputs = { transcripts = "Document[]" }+output = "Page[]"+steps = [{ pipe = "extract_pdf", batch_over = "transcripts", batch_as = "transcript", result = "pages" }]+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_batch_step", pipes=pipes)++ assert _summary(consumers["transcripts"]) == [("extract_pdf", "transcripts[]", False)]+ assert "transcript" not in consumers++ def test_a_standalone_pipe_batch_maps_the_list_slot_to_its_item_slot(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeBatch"+description = "Extract every transcript"+inputs = { transcripts = "Document[]" }+output = "Page[]"+branch_pipe_code = "extract_pdf"+input_list_name = "transcripts"+input_item_name = "transcript"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_batch", pipes=pipes)++ assert _summary(consumers["transcripts"]) == [("extract_pdf", "transcripts[]", False)]++ def test_every_parallel_branch_is_visited(self, load_empty_library: Callable[[], str]):+ pipes = """+[concept.BothExtractions]+description = "The transcript extracted by two models"++[concept.BothExtractions.structure]+pdf_pages = { type = "list", item_type = "concept", item_concept_ref = "native.Page", description = "Pages read as PDF", required = true }+any_pages = { type = "list", item_type = "concept", item_concept_ref = "native.Page", description = "Pages read by any format", required = true }++[pipe.main]+type = "PipeParallel"+description = "Extract the transcript twice"+inputs = { transcript = "Document" }+output = "BothExtractions"+branches = [+ { pipe = "extract_pdf", result = "pdf_pages" },+ { pipe = "extract_any", result = "any_pages" },+]+add_each_output = true+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_parallel", pipes=pipes)++ assert _summary(consumers["transcript"]) == [("extract_any", "transcript", False), ("extract_pdf", "transcript", False)]++ def test_a_consumer_below_a_condition_is_conditional(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeCondition"+description = "Extract the transcript in one mode only"+inputs = { mode = "Text", transcript = "Document" }+output = "Page[]"+expression = "mode"+default_outcome = "extract_any"++[pipe.main.outcomes]+pdf = "extract_pdf"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_condition", pipes=pipes)++ assert _summary(consumers["transcript"]) == [("extract_any", "transcript", True), ("extract_pdf", "transcript", True)]++ def test_a_liftable_step_is_conditional(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Extract the transcript when there is one"+inputs = { transcript = "Document?" }+output = "Page[]"+steps = [{ pipe = "extract_pdf", result = "pages" }]+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_liftable", pipes=pipes)++ assert _summary(consumers["transcript"]) == [("extract_pdf", "transcript", True)]++ def test_a_nested_sequence_is_followed(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Delegate to an inner sequence"+inputs = { transcript = "Document" }+output = "Page[]"+steps = [{ pipe = "inner", result = "pages" }]++[pipe.inner]+type = "PipeSequence"+description = "Extract the transcript"+inputs = { transcript = "Document" }+output = "Page[]"+steps = [{ pipe = "extract_pdf", result = "inner_pages" }]+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_nested", pipes=pipes)++ assert _summary(consumers["transcript"]) == [("extract_pdf", "transcript", False)]++ def test_pipe_func_and_pipe_compose_are_opaque(self, load_empty_library: Callable[[], str]):+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Hand the transcript to opaque pipes only"+inputs = { transcript = "Document" }+output = "Text"+steps = [+ { pipe = "replace_transcript", result = "other_transcript" },+ { pipe = "compose_note", result = "note" },+]++[pipe.replace_transcript]+type = "PipeFunc"+description = "Replace the transcript with another document"+inputs = { transcript = "Document" }+output = "Document"+function_name = "file_consumers_replace_transcript"++[pipe.compose_note]+type = "PipeCompose"+description = "Compose a note about the transcript"+inputs = { transcript = "Document" }+output = "Text"+template = "A note about $transcript"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_opaque", pipes=pipes)++ assert consumers.get("transcript", []) == []++ def test_an_unresolved_pipe_is_opaque(self, load_empty_library: Callable[[], str], mocker: MockerFixture):+ """A cross-package reference whose package is not loaded resolves to nothing, and the walk never guesses past it."""+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Extract the transcript"+inputs = { transcript = "Document" }+output = "Page[]"+steps = [{ pipe = "extract_pdf", result = "pages" }]+"""+ mthds_content = _bundle(domain="fic_unresolved", pipes=pipes)+ library_id = load_empty_library()+ get_library_manager().load_from_blueprints(+ library_id=library_id, blueprints=[MthdsParser.make_pipelex_bundle_blueprint(mthds_content=mthds_content)]+ )+ entry_pipe = get_required_pipe(pipe_code="fic_unresolved.main")+ mocker.patch.object(file_input_consumers_module, "get_optional_pipe", return_value=None)++ assert collect_file_input_consumers(entry_pipe) == {}++ def test_a_pipe_llm_consumes_the_documents_its_prompt_references(self, load_empty_library: Callable[[], str]):+ """An item reference reached through a batch, and a list reference: the prompt names each by its variable path."""+ pipes = """+[pipe.main]+type = "PipeSequence"+description = "Summarize every transcript, then compare them all"+inputs = { transcripts = "Document[]" }+output = "Text"+steps = [+ { pipe = "summarize_transcript", batch_over = "transcripts", batch_as = "transcript", result = "summaries" },+ { pipe = "compare_transcripts", result = "comparison" },+]++[pipe.summarize_transcript]+type = "PipeLLM"+description = "Summarize one transcript"+inputs = { transcript = "Document" }+output = "Text"+prompt = "Summarize this transcript: $transcript"++[pipe.compare_transcripts]+type = "PipeLLM"+description = "Compare the transcripts"+inputs = { transcripts = "Document[]", summaries = "Text[]" }+output = "Text"+prompt = "Compare these transcripts: $transcripts, given their summaries: $summaries"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_llm", pipes=pipes)++ assert _summary(consumers["transcripts"]) == [("compare_transcripts", "transcripts", False), ("summarize_transcript", "transcripts[]", False)]+ expected_formats: set[str] = set()+ for llm_setting in (resolve_llm_setting_for_text(), resolve_llm_setting_for_object()):+ for model_spec in file_input_consumers_module.resolve_model_specs(model_reference=llm_setting.model, model_type=ModelType.LLM):+ expected_formats |= model_spec.supported_document_types+ for consumer in consumers["transcripts"]:+ assert consumer.kind == FileConsumerKind.LLM_DOCUMENT+ assert consumer.readable_formats == frozenset(expected_formats)+ assert consumer.covers(file_input_path=("transcripts", 1))+ assert not consumer.covers(file_input_path=("notes", 1))++ def test_a_waterfall_reads_a_format_when_any_member_reads_it(self, load_empty_library: Callable[[], str], mocker: MockerFixture):+ mocker.patch.dict(get_model_deck().extract_waterfalls, {"fic-mixed-extractors": [PDF_ONLY_MODEL, DOCLING_MODEL]})+ pipes = """+[pipe.main]+type = "PipeExtract"+description = "Extract the transcript with a waterfall"+inputs = { transcript = "Document" }+output = "Page[]"+model = "~fic-mixed-extractors"+"""+ consumers = _consumers(load_empty_library=load_empty_library, domain="fic_waterfall", pipes=pipes)++ (consumer,) = consumers["transcript"]+ assert consumer.model == "fic-mixed-extractors"+ assert consumer.readable_formats == DOCLING_FORMATSdiff --git a/tests/integration/pipelex/pipeline/test_pipeline_run_setup_consumer_check.py b/tests/integration/pipelex/pipeline/test_pipeline_run_setup_consumer_check.pynew file mode 100644index 000000000..cdca530de--- /dev/null+++ b/tests/integration/pipelex/pipeline/test_pipeline_run_setup_consumer_check.py@@ -0,0 +1,271 @@+"""Run setup refuses a run whose file inputs are certain to reach a consumer that cannot read their format.++The first proof-lab scenario: Word transcripts batched into a `PipeExtract` whose model reads PDF+only. They used to fail the run at the extraction, after it started; run setup now refuses the run+before any pipe runs, with one input error naming every transcript. A consumer reached only through+a condition is not certain, so the run starts, and the operator refuses the file when it gets there.+"""++import base64+import re+import zipfile+from pathlib import Path++import pytest++from pipelex.base_exceptions import DisclosureMode, ErrorDomain+from pipelex.cogt.exceptions import ExtractInputFormatError+from pipelex.config import get_config+from pipelex.core.stuffs.text_content import TextContent+from pipelex.interpreter_hub import get_pipe_router+from pipelex.pipe_run.exceptions import PipeRouterError+from pipelex.pipeline.exceptions import PipelineInputFormatUnsupportedError+from pipelex.pipeline.pipeline_run_setup import pipeline_run_setup+from tests.cases import DocumentTestCases++DOCX_BYTES = Path(DocumentTestCases.DOCX_FILE_PATH_1).read_bytes()+PDF_BYTES = Path(DocumentTestCases.PDF_FILE_PATH_3).read_bytes()++_TRANSCRIPTS_MTHDS = """+domain = "consumer_check_transcripts"+description = "Extract interview transcripts"++[pipe.extract_transcripts]+type = "PipeSequence"+description = "Extract every transcript"+inputs = { transcripts = "Document[]" }+output = "Page[]"+steps = [{ pipe = "extract_transcript", batch_over = "transcripts", batch_as = "transcript", result = "pages" }]++[pipe.extract_transcript]+type = "PipeExtract"+description = "Extract one transcript"+inputs = { transcript = "Document" }+output = "Page[]"+model = "pypdfium2-extract-pdf"+"""++_SINGLE_TRANSCRIPT_MTHDS = """+domain = "consumer_check_single"+description = "Extract one transcript with either model"++[pipe.extract_with_pdf_model]+type = "PipeExtract"+description = "Extract the transcript with a model reading PDF only"+inputs = { transcript = "Document" }+output = "Page[]"+model = "pypdfium2-extract-pdf"++[pipe.extract_with_word_model]+type = "PipeExtract"+description = "Extract the transcript with a model reading Word files"+inputs = { transcript = "Document" }+output = "Page[]"+model = "docling-extract-text"+"""++_CONDITIONAL_MTHDS = """+domain = "consumer_check_conditional"+description = "Extract a transcript in one mode only"++[pipe.maybe_extract]+type = "PipeCondition"+description = "Extract the transcript when asked to"+inputs = { mode = "Text", transcript = "Document" }+output = "Page[]"+expression = "mode"+default_outcome = "fail"++[pipe.maybe_extract.outcomes]+extract = "extract_transcript"++[pipe.extract_transcript]+type = "PipeExtract"+description = "Extract one transcript"+inputs = { transcript = "Document" }+output = "Page[]"+model = "pypdfium2-extract-pdf"+"""+++def _data_url(*, mime_type: str, raw_bytes: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(raw_bytes).decode('ascii')}"+++def _write_late_entry_docx(*, path: Path) -> Path:+ """Rewrite the Word fixture with a large custom part first, so the sniffer sees only a zip container."""+ with zipfile.ZipFile(DocumentTestCases.DOCX_FILE_PATH_1) as source, zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED) as target:+ entry_names = source.namelist()+ leading = [entry_name for entry_name in entry_names if entry_name in {"[Content_Types].xml", "_rels/.rels"}]+ for entry_name in leading:+ target.writestr(entry_name, source.read(entry_name))+ target.writestr("customXml/item1.xml", "<data>" + "x" * 12_000 + "</data>")+ for entry_name in entry_names:+ if entry_name not in leading:+ target.writestr(entry_name, source.read(entry_name))+ return path+++def _find_in_cause_chain(error: BaseException, *, error_class: type[BaseException]) -> BaseException | None:+ current: BaseException | None = error+ while current is not None:+ if isinstance(current, error_class):+ return current+ current = current.__cause__ or current.__context__+ return None++[email protected](loop_scope="class")+class TestPipelineRunSetupConsumerCheck:+ async def test_word_transcripts_batched_into_a_pdf_extractor_are_refused_before_the_run(self):+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)+ docx_url = _data_url(mime_type="application/octet-stream", raw_bytes=DOCX_BYTES)++ with pytest.raises(PipelineInputFormatUnsupportedError) as exc_info:+ await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_TRANSCRIPTS_MTHDS],+ pipe_code="extract_transcripts",+ inputs={"transcripts": {"concept": "native.Document", "content": [{"url": docx_url}, {"url": docx_url}, {"url": docx_url}]}},+ )++ report = exc_info.value.to_error_report()+ assert report.error_domain == ErrorDomain.INPUT+ assert report.http_status == 422+ message = exc_info.value.to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT)["message"]+ for index in range(3):+ assert (+ f"Input 'transcripts[{index}]' is a Word document (.docx): pipe 'extract_transcript' extracts it "+ "with model 'pypdfium2-extract-pdf', which reads PDF." in message+ )+ assert message.endswith("Give a file in a format its model reads, or use a model that reads this format.")++ async def test_a_pdf_among_the_transcripts_is_not_named(self):+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)++ with pytest.raises(PipelineInputFormatUnsupportedError) as exc_info:+ await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_TRANSCRIPTS_MTHDS],+ pipe_code="extract_transcripts",+ inputs={+ "transcripts": {+ "concept": "native.Document",+ "content": [+ {"url": _data_url(mime_type="application/pdf", raw_bytes=PDF_BYTES)},+ {"url": _data_url(mime_type="application/octet-stream", raw_bytes=DOCX_BYTES)},+ ],+ }+ },+ )++ assert "transcripts[1]" in str(exc_info.value)+ assert "transcripts[0]" not in str(exc_info.value)++ async def test_pdf_transcripts_pass_setup(self):+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)+ pdf_url = _data_url(mime_type="application/pdf", raw_bytes=PDF_BYTES)++ pipe_job, _pipeline_run_id, _ = await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_TRANSCRIPTS_MTHDS],+ pipe_code="extract_transcripts",+ inputs={"transcripts": {"concept": "native.Document", "content": [{"url": pdf_url}]}},+ )++ assert pipe_job.pipe.code == "extract_transcripts"++ @pytest.mark.parametrize(+ ("pipe_code", "declared_mime_type", "expected_refusal"),+ [+ ("extract_with_word_model", None, None),+ ("extract_with_word_model", "application/zip", None),+ (+ "extract_with_pdf_model",+ None,+ "Input 'transcript' is a Word document (.docx): pipe 'extract_with_pdf_model' extracts it with model 'pypdfium2-extract-pdf'",+ ),+ ],+ )+ async def test_a_word_file_the_sniffer_sees_only_as_a_zip_is_known_by_its_name(+ self, tmp_path: Path, pipe_code: str, declared_mime_type: str | None, expected_refusal: str | None+ ):+ """The file's `.docx` name says what the zip holds, even over a declared bare zip, so the model reading Word files takes it."""+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)+ docx_path = _write_late_entry_docx(path=tmp_path / "interview.docx")+ content: dict[str, str] = {"url": str(docx_path)}+ if declared_mime_type is not None:+ content["mime_type"] = declared_mime_type+ setup = pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_SINGLE_TRANSCRIPT_MTHDS],+ pipe_code=pipe_code,+ inputs={"transcript": {"concept": "native.Document", "content": content}},+ )++ if expected_refusal is None:+ pipe_job, _pipeline_run_id, _ = await setup+ assert pipe_job.pipe.code == pipe_code+ else:+ with pytest.raises(PipelineInputFormatUnsupportedError, match=re.escape(expected_refusal)):+ await setup++ @pytest.mark.parametrize(+ ("mime_type", "raw_bytes"),+ [+ ("text/markdown", b"# Reading List\n\nBooks the team recommends.\n"),+ ("text/csv", b"team,score\nNorthwind,78\n"),+ ("message/rfc822", b"From: [email protected]\nSubject: Hi\n\nBody.\n"),+ ],+ )+ async def test_a_declared_text_document_passes_setup_into_docling(self, mime_type: str, raw_bytes: bytes):+ """Docling reads Markdown, CSV and email files, so a document declared as one passes setup into it."""+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)++ pipe_job, _pipeline_run_id, _ = await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_SINGLE_TRANSCRIPT_MTHDS],+ pipe_code="extract_with_word_model",+ inputs={"transcript": _data_url(mime_type=mime_type, raw_bytes=raw_bytes)},+ )++ assert pipe_job.pipe.code == "extract_with_word_model"++ async def test_a_consumer_behind_a_condition_lets_the_run_start_and_the_operator_refuses(self):+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)++ pipe_job, _pipeline_run_id, _ = await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_CONDITIONAL_MTHDS],+ pipe_code="maybe_extract",+ inputs={+ "mode": TextContent(text="extract"),+ "transcript": _data_url(mime_type="application/octet-stream", raw_bytes=DOCX_BYTES),+ },+ )++ # The operator's refusal arrives wrapped by the router that ran the pipe.+ with pytest.raises(PipeRouterError) as exc_info:+ await get_pipe_router().run(pipe_job=pipe_job)++ extract_refusal = _find_in_cause_chain(exc_info.value, error_class=ExtractInputFormatError)+ assert extract_refusal is not None, f"expected an ExtractInputFormatError in the cause chain, got {exc_info.value!r}"+ assert "Input 'transcript' is a Word document (.docx)" in str(extract_refusal)diff --git a/tests/integration/pipelex/pipeline/test_pipeline_run_setup_image_format.py b/tests/integration/pipelex/pipeline/test_pipeline_run_setup_image_format.pynew file mode 100644index 000000000..bb89ccac7--- /dev/null+++ b/tests/integration/pipelex/pipeline/test_pipeline_run_setup_image_format.py@@ -0,0 +1,94 @@+"""A PDF given to an Image input is refused by run setup, before any pipe runs, as an input error an API answers with 422.++The proof-lab scenario: a referral letter scanned as a PDF, given to an input whose concept refines+`Image`. It used to pass preparation and fail at the first model call, in the provider's words and+with no error domain. Run setup is where an API's `/start` runs, so a refusal here is the answer+the caller receives synchronously.+"""++import base64+from pathlib import Path++import pytest++from pipelex.base_exceptions import ErrorDomain, error_domain_to_http_status+from pipelex.config import get_config+from pipelex.pipeline.exceptions import PipelineInputNotAnImageError+from pipelex.pipeline.pipeline_run_setup import pipeline_run_setup++PDF_BYTES = Path("tests/data/documents/solar_system.pdf").read_bytes()+PNG_BYTES = Path("tests/data/images/logo-tiny.png").read_bytes()++_REFERRAL_MTHDS = """+domain = "referral_image_format_test"+description = "A method reading a referral letter given as an image"++[concept.ReferralLetter]+description = "A scanned referral letter"+refines = "Image"++[pipe.read_referral]+type = "PipeLLM"+description = "Read the referral letter"+inputs = { referral_letter = "ReferralLetter", photo = "Image" }+output = "Text"+prompt = "Summarize the referral letter: $referral_letter. Describe the photo: $photo."+"""+++def _data_url(*, mime_type: str, raw_bytes: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(raw_bytes).decode('ascii')}"++[email protected](loop_scope="class")+class TestPipelineRunSetupImageFormat:+ @pytest.mark.parametrize(+ ("referral_bytes", "photo_bytes", "refused_input"),+ [+ (PDF_BYTES, PNG_BYTES, "referral_letter"),+ (PNG_BYTES, PDF_BYTES, "photo"),+ ],+ )+ async def test_a_pdf_in_an_image_input_is_refused_at_setup_with_422(self, referral_bytes: bytes, photo_bytes: bytes, refused_input: str):+ """Both a concept refining Image and the native Image itself are checked."""+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)++ with pytest.raises(PipelineInputNotAnImageError) as exc_info:+ await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_REFERRAL_MTHDS],+ pipe_code="read_referral",+ inputs={+ "referral_letter": _data_url(mime_type="image/png", raw_bytes=referral_bytes),+ "photo": _data_url(mime_type="image/png", raw_bytes=photo_bytes),+ },+ )++ report = exc_info.value.to_error_report()+ assert report.error_domain == ErrorDomain.INPUT+ assert error_domain_to_http_status(report.error_domain) == 422+ assert report.http_status == 422+ assert f"Input '{refused_input}' expects an image, but the file is a PDF document (.pdf)" in report.message++ async def test_images_pass_setup(self):+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)++ pipe_job, _pipeline_run_id, _ = await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_REFERRAL_MTHDS],+ pipe_code="read_referral",+ inputs={+ "referral_letter": _data_url(mime_type="image/png", raw_bytes=PNG_BYTES),+ "photo": _data_url(mime_type="image/png", raw_bytes=PNG_BYTES),+ },+ )++ assert pipe_job.working_memory is not None+ referral_letter = pipe_job.working_memory.get_stuff_as_image(name="referral_letter")+ assert referral_letter.mime_type == "image/png"diff --git a/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract_input_format.py b/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract_input_format.pynew file mode 100644index 000000000..b685133c4--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract_input_format.py@@ -0,0 +1,102 @@+"""PipeExtract hands its extract worker the file's identified format and the name of the input it came from.++Run setup stamps every file input with the MIME type of its bytes; the operator passes it on in the+`ExtractInput`, beside the input's name, so the worker checks the format against its model instead+of guessing it again, and names the input when it refuses.+"""++import base64+from pathlib import Path++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.extract.extract_input import ExtractInput+from pipelex.config import get_config+from pipelex.pipe_operators.extract import pipe_extract as pipe_extract_module+from pipelex.pipe_operators.extract.pipe_extract import PipeExtract+from pipelex.pipeline.pipeline_run_setup import pipeline_run_setup++DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+DOCX_BYTES = Path("tests/data/documents/CV-ELIAS-THORNE.docx").read_bytes()+PNG_BYTES = Path("tests/data/images/logo-tiny.png").read_bytes()++_EXTRACT_MTHDS = """+domain = "extract_input_format_test"+description = "Methods extracting a document and an image"++[pipe.extract_transcript]+type = "PipeExtract"+description = "Extract the transcript"+inputs = { transcript = "Document" }+output = "Page[]"+model = "docling-extract-text"++[pipe.extract_scan]+type = "PipeExtract"+description = "Extract the scan"+inputs = { scan = "Image" }+output = "Page[]"+"""+++def _data_url(*, mime_type: str, raw_bytes: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(raw_bytes).decode('ascii')}"+++class _ExtractInterruptedError(Exception):+ """Raised by the patched `run_extract` once it has captured its input, so no extraction runs."""++[email protected](loop_scope="class")+class TestPipeExtractInputFormat:+ @pytest.mark.parametrize(+ ("pipe_code", "input_name", "input_value", "expected_mime_type", "is_image"),+ [+ ("extract_transcript", "transcript", _data_url(mime_type="application/octet-stream", raw_bytes=DOCX_BYTES), DOCX_MIME, False),+ ("extract_scan", "scan", _data_url(mime_type="image/png", raw_bytes=PNG_BYTES), "image/png", True),+ ],+ )+ async def test_the_extract_input_carries_the_format_and_the_input_name(+ self,+ mocker: MockerFixture,+ pipe_code: str,+ input_name: str,+ input_value: str,+ expected_mime_type: str,+ is_image: bool,+ ):+ run_extract = mocker.patch.object(pipe_extract_module, "run_extract", side_effect=_ExtractInterruptedError)+ execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)+ pipe_job, _pipeline_run_id, _ = await pipeline_run_setup(+ storage_scope="test/scope",+ read_scope=None,+ user_id="test-user",+ execution_config=execution_config,+ mthds_contents=[_EXTRACT_MTHDS],+ pipe_code=pipe_code,+ inputs={input_name: input_value},+ )+ pipe = pipe_job.pipe+ assert isinstance(pipe, PipeExtract)+ assert pipe_job.working_memory is not None++ with pytest.raises(_ExtractInterruptedError):+ await pipe._live_run_operator_pipe( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]+ job_metadata=pipe_job.job_metadata,+ working_memory=pipe_job.working_memory,+ pipe_run_params=pipe_job.pipe_run_params,+ )++ run_extract.assert_awaited_once()+ assert run_extract.await_args is not None+ extract_input = run_extract.await_args.kwargs["extract_input"]+ assert isinstance(extract_input, ExtractInput)+ assert extract_input.mime_type == expected_mime_type+ assert extract_input.input_name == input_name+ if is_image:+ assert extract_input.image_uri is not None+ assert extract_input.document_uri is None+ else:+ assert extract_input.document_uri is not None+ assert extract_input.image_uri is Nonediff --git a/tests/integration/pipelex/providers/test_docling_document_formats.py b/tests/integration/pipelex/providers/test_docling_document_formats.pynew file mode 100644index 000000000..529ce987c--- /dev/null+++ b/tests/integration/pipelex/providers/test_docling_document_formats.py@@ -0,0 +1,45 @@+"""Docling, as the internal backend declares it, extracts every format it declares beside PDF and images.++`docling-extract-text` declares Word, PowerPoint, Excel, HTML, Markdown, CSV, plain text, WebVTT and+email files beside `pdf` and `image`, so a file of those formats passes the extract worker's format+check, and this proves Docling then reads it. Docling converts these formats with its declarative+backends, which need no model download and run offline, so this runs with the rest of the suite+rather than behind the inference markers.+"""++import pytest++from pipelex.cogt.extract.extract_input import ExtractInput+from pipelex.cogt.extract.extract_job_components import ExtractJobParams+from pipelex.cogt.extract.extract_job_factory import ExtractJobFactory+from pipelex.runtime_hub import get_extract_worker+from pipelex.system.job_metadata import JobMetadata+from pipelex.tools.misc.filetype_utils import format_key_from_mime_type+from tests.cases import DocumentTestCases++DOCLING_EXTRACT_HANDLE = "docling-extract-text"++[email protected](loop_scope="class")+class TestDoclingDocumentFormats:+ @pytest.mark.parametrize(("file_path", "mime_type", "expected_phrase"), DocumentTestCases.NON_PDF_DOCUMENT_CASES)+ async def test_docling_extracts_the_formats_it_declares(self, job_metadata: JobMetadata, file_path: str, mime_type: str, expected_phrase: str):+ pytest.importorskip("docling")+ extract_worker = get_extract_worker(extract_handle=DOCLING_EXTRACT_HANDLE)+ assert format_key_from_mime_type(mime_type=mime_type) in extract_worker.inference_model.readable_formats_for_extract+ extract_job = ExtractJobFactory.make_extract_job(+ extract_input=ExtractInput(document_uri=file_path, mime_type=mime_type, input_name="document"),+ extract_job_params=ExtractJobParams(+ max_nb_images=None,+ should_caption_images=False,+ should_include_page_views=False,+ page_views_dpi=None,+ image_min_size=None,+ ),+ job_metadata=job_metadata,+ )++ extract_output = await extract_worker.extract_pages(extract_job=extract_job)++ extracted_text = "\n".join(page.text or "" for page in extract_output.pages.values())+ assert expected_phrase in extracted_textdiff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.pyindex 7bfef6520..ffd4571ee 100644--- a/tests/integration/pipelex/test_data.py+++ b/tests/integration/pipelex/test_data.py@@ -159,7 +159,7 @@ class PipeExtractTestCases:ImageTestCases.IMAGE_FILE_PATH_PNG_2,ImageTestCases.IMAGE_URL_PNG,]- PIPE_OCR_PDF_TEST_CASES: ClassVar[list[str]] = DocumentTestCases.DOCUMENT_FILE_PATHS + DocumentTestCases.DOCUMENT_URLS+ PIPE_OCR_PDF_TEST_CASES: ClassVar[list[str]] = DocumentTestCases.PDF_FILE_PATHS + DocumentTestCases.DOCUMENT_URLSclass ImageGenTestCases:diff --git a/tests/integration/pipelex/tools/storage/test_user_provided_image_storage.py b/tests/integration/pipelex/tools/storage/test_user_provided_image_storage.pyindex cc54a1bd7..d1aabb266 100644--- a/tests/integration/pipelex/tools/storage/test_user_provided_image_storage.py+++ b/tests/integration/pipelex/tools/storage/test_user_provided_image_storage.py@@ -1,6 +1,6 @@"""Integration tests for user-provided images with storage providers.-These tests verify the flow: User Input -> normalize_data_urls_to_storage -> Storage -> Retrieval+These tests verify the flow: User Input -> prepare_file_inputs -> Storage -> RetrievalFuture storage implementations (S3, GCP) must pass these tests to ensurecompatibility with user-provided image handling.@@ -18,7 +18,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.pipeline.input_normalizer import normalize_data_urls_to_storage+from pipelex.pipeline.input_normalizer import prepare_file_inputsfrom pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEMEfrom pipelex.tools.uri.prepared_file import PreparedFileBase64from pipelex.urls import URLs@@ -30,7 +30,7 @@ from tests.cases import ImageTestCasesclass TestUserProvidedImageStorage:"""Integration tests for user-provided images with storage providers.- These tests verify the flow: User Input -> normalize_data_urls_to_storage -> Storage -> Retrieval+ These tests verify the flow: User Input -> prepare_file_inputs -> Storage -> RetrievalFuture storage implementations (S3, GCP) must pass these tests to ensurecompatibility with user-provided image handling.@@ -41,7 +41,7 @@ class TestUserProvidedImageStorage:Flow:1. User provides ImageContent with data:// URL- 2. normalize_data_urls_to_storage converts to pipelex-storage:// URI+ 2. prepare_file_inputs converts to pipelex-storage:// URI3. Image can be retrieved via PromptImageFactory -> PreparedImage"""# Create ImageContent with data URL (simulating user input)@@ -54,7 +54,7 @@ class TestUserProvidedImageStorage:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize to storage- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify the URL was converted to pipelex-storage://normalized_stuff = normalized_memory.get_stuff("user_image")@@ -116,7 +116,7 @@ class TestUserProvidedImageStorage:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize (should not change HTTP URLs when fetch is disabled)- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify URL was NOT changednormalized_stuff = normalized_memory.get_stuff("remote_image")@@ -141,7 +141,7 @@ class TestUserProvidedImageStorage:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize - should upload the local file to storage- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify the URL was converted to pipelex-storage://normalized_stuff = normalized_memory.get_stuff("local_image")@@ -180,7 +180,7 @@ class TestUserProvidedImageStorage:working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)# Normalize (should not change local paths when upload is disabled)- normalized_memory = await normalize_data_urls_to_storage(working_memory, storage_scope="test/scope", read_scope=None)+ normalized_memory = await prepare_file_inputs(working_memory, storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)# Verify URL was NOT changednormalized_stuff = normalized_memory.get_stuff("local_image")diff --git a/tests/unit/pipelex/cogt/extract/test_extract_input_format_check.py b/tests/unit/pipelex/cogt/extract/test_extract_input_format_check.pynew file mode 100644index 000000000..586c23b78--- /dev/null+++ b/tests/unit/pipelex/cogt/extract/test_extract_input_format_check.py@@ -0,0 +1,129 @@+"""The extract worker refuses a file whose known format its model does not read, before the provider sees it.++The model is fixed by the method and reads the formats it declares; the file changes from run to run,+so a format the model does not read is the caller's input fault, an `ExtractInputFormatError` in the+input domain, and its message names the input, the format, the model and what that model reads. An+unknown format is left to the provider. A web-page model given an http(s) URL fetches the page itself,+so its format is not checked.+"""++import pytest+from typing_extensions import override++from pipelex.base_exceptions import ErrorDomain+from pipelex.cogt.exceptions import ExtractCapabilityError, ExtractInputFormatError+from pipelex.cogt.extract.extract_input import ExtractInput+from pipelex.cogt.extract.extract_job import ExtractJob+from pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParams, ExtractJobReport+from pipelex.cogt.extract.extract_output import ExtractOutput, Page+from pipelex.cogt.extract.extract_worker_abstract import ExtractWorkerAbstract+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.job_metadata import JobMetadata, RunMetadata++DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+STORED_DOCX = "pipelex-storage://org/uploads/transcript.docx"+++class _CallCountingExtractWorker(ExtractWorkerAbstract):+ """An extract worker whose provider half only counts the calls that reach it."""++ def __init__(self, *, inference_model: InferenceModelSpec) -> None:+ super().__init__(extra_config={}, inference_model=inference_model)+ self.nb_provider_calls = 0++ @override+ async def _extract_pages(self, extract_job: ExtractJob) -> ExtractOutput:+ self.nb_provider_calls += 1+ return ExtractOutput(pages={0: Page(text="page text")})+++def _make_worker(*, inputs: list[str]) -> _CallCountingExtractWorker:+ inference_model = InferenceModelSpec(+ backend_name="test",+ name="azure-document-intelligence",+ sdk="test_extract",+ model_type=ModelType.TEXT_EXTRACTOR,+ model_id="prebuilt-layout",+ inputs=inputs,+ outputs=["text"],+ costs={CostCategory.INPUT: 1.0, CostCategory.OUTPUT: 1.0},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ )+ return _CallCountingExtractWorker(inference_model=inference_model)+++def _make_job(*, extract_input: ExtractInput) -> ExtractJob:+ return ExtractJob(+ extract_input=extract_input,+ job_params=ExtractJobParams.make_default_extract_job_params(),+ job_config=ExtractJobConfig(),+ job_report=ExtractJobReport(),+ job_metadata=JobMetadata(+ run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="test-user", pipeline_run_id="test-run")+ ),+ )++[email protected](loop_scope="class")+class TestExtractInputFormatCheck:+ async def test_a_docx_given_to_a_model_reading_pdf_and_images_is_refused(self):+ worker = _make_worker(inputs=["pdf", "image"])+ job = _make_job(extract_input=ExtractInput(document_uri=STORED_DOCX, mime_type=DOCX_MIME, input_name="transcripts[0]"))++ with pytest.raises(ExtractInputFormatError) as exc_info:+ await worker.extract_pages(extract_job=job)++ assert str(exc_info.value) == (+ "Input 'transcripts[0]' is a Word document (.docx), which model 'azure-document-intelligence' cannot extract: "+ "it reads PDF and images. Give a file in one of those formats, or use an extract model that reads this one."+ )+ assert exc_info.value.to_error_report().error_domain == ErrorDomain.INPUT+ assert worker.nb_provider_calls == 0++ async def test_a_file_with_no_input_name_is_named_as_the_file_to_extract(self):+ worker = _make_worker(inputs=["pdf"])+ job = _make_job(extract_input=ExtractInput(document_uri=STORED_DOCX, mime_type=DOCX_MIME))++ with pytest.raises(ExtractInputFormatError, match=r"^The file to extract is a Word document \(\.docx\), which model"):+ await worker.extract_pages(extract_job=job)++ @pytest.mark.parametrize(+ ("inputs", "extract_input"),+ [+ (["pdf", "docx", "image"], ExtractInput(document_uri=STORED_DOCX, mime_type=DOCX_MIME, input_name="transcript")),+ (["pdf"], ExtractInput(document_uri=STORED_DOCX, input_name="transcript")),+ (["pdf"], ExtractInput(document_uri=STORED_DOCX, mime_type="application/octet-stream", input_name="transcript")),+ (["web_page"], ExtractInput(document_uri="https://example.com/report", mime_type="application/pdf", input_name="page")),+ (["pdf", "image"], ExtractInput(image_uri="pipelex-storage://org/uploads/scan.png", mime_type="image/png", input_name="scan")),+ (["pdf", "image"], ExtractInput(document_uri="pipelex-storage://org/uploads/scan.png", mime_type="image/png", input_name="scan")),+ ],+ ids=["declared-format", "unknown-format", "generic-format", "web-page-url", "image-as-image", "image-as-document"],+ )+ async def test_a_readable_or_unknown_format_reaches_the_provider(self, inputs: list[str], extract_input: ExtractInput):+ worker = _make_worker(inputs=inputs)++ output = await worker.extract_pages(extract_job=_make_job(extract_input=extract_input))++ assert output.pages[0].text == "page text"+ assert worker.nb_provider_calls == 1++ async def test_a_web_page_model_given_a_stored_file_is_format_checked(self):+ """Only an http(s) URL is the web-page model's to fetch; a stored file of a format it does not read is refused."""+ worker = _make_worker(inputs=["web_page"])+ job = _make_job(extract_input=ExtractInput(document_uri=STORED_DOCX, mime_type=DOCX_MIME, input_name="page"))++ with pytest.raises(ExtractInputFormatError, match=r"it reads web pages\."):+ await worker.extract_pages(extract_job=job)++ async def test_a_model_reading_no_images_given_an_image_input_is_still_a_capability_error(self):+ """Choosing a model that extracts no images for an Image input is the method author's choice, not the caller's input."""+ worker = _make_worker(inputs=["pdf"])+ job = _make_job(extract_input=ExtractInput(image_uri="pipelex-storage://org/uploads/scan.png", mime_type="image/png", input_name="scan"))++ with pytest.raises(ExtractCapabilityError):+ await worker.extract_pages(extract_job=job)diff --git a/tests/unit/pipelex/cogt/llm/test_llm_worker_document_format.py b/tests/unit/pipelex/cogt/llm/test_llm_worker_document_format.pynew file mode 100644index 000000000..900d28108--- /dev/null+++ b/tests/unit/pipelex/cogt/llm/test_llm_worker_document_format.py@@ -0,0 +1,160 @@+"""The LLM worker refuses a prompt document whose known format its model does not read, as an input error.++A document format the model does not read used to raise `LLMCapabilityError`, a configuration error+that answers 500, although the file is the caller's and changes from run to run. It is now a+`PromptDocumentFormatError`, a content error in the input domain. `LLMCapabilityError` stays for a+model that reads no documents at all, which is the method author's choice of model. An unknown+format is left to the provider, and a document whose bytes identify nothing no longer raises.+"""++import base64+import io+import zipfile++import pytest+from typing_extensions import override++from pipelex.base_exceptions import ErrorDomain+from pipelex.cogt.document.prompt_document import PromptDocument, PromptDocumentBase64, PromptDocumentBinary, PromptDocumentUri+from pipelex.cogt.document.prompt_document_factory import PromptDocumentFactory+from pipelex.cogt.exceptions import LLMCapabilityError, PromptDocumentFormatError+from pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams+from pipelex.cogt.llm.llm_prompt import LLMPrompt+from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.job_metadata import JobMetadata, RunMetadata+from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar++DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+STORED_DOCX = "pipelex-storage://org/uploads/brief.docx"+STORED_PDF = "pipelex-storage://org/uploads/brief.pdf"+MARKDOWN_BYTES = b"# Brief\n\nNothing a sniffer can recognise."+++def _late_entry_docx_bytes() -> bytes:+ """A Word file whose `word/` entry comes after the entries the sniffer looks at, so its bytes show only a zip."""+ buffer = io.BytesIO()+ with zipfile.ZipFile(buffer, "w") as archive:+ for entry_name in (+ "[Content_Types].xml",+ "_rels/.rels",+ "docProps/app.xml",+ "docProps/core.xml",+ "docProps/custom.xml",+ "customXml/item1.xml",+ ):+ archive.writestr(entry_name, "<xml/>")+ archive.writestr("word/document.xml", "<w:document/>")+ return buffer.getvalue()+++LATE_ENTRY_DOCX_BASE64 = base64.b64encode(_late_entry_docx_bytes()).decode("ascii")+++class _CallCountingLLMWorker(LLMWorkerAbstract):+ """A worker whose provider half only counts the calls that reach it."""++ def __init__(self, *, inference_model: InferenceModelSpec) -> None:+ LLMWorkerAbstract.__init__(self, inference_model=inference_model, reporting_delegate=None)+ self.nb_provider_calls = 0++ @override+ async def _gen_text(self, llm_job: LLMJob) -> str:+ self.nb_provider_calls += 1+ return "answer"++ @override+ async def _gen_object(self, llm_job: LLMJob, *, schema: type[BaseModelTypeVar]) -> BaseModelTypeVar:+ self.nb_provider_calls += 1+ return schema.model_validate({})+++def _make_worker(*, inputs: list[str]) -> _CallCountingLLMWorker:+ inference_model = InferenceModelSpec(+ backend_name="openai",+ name="gpt-docs-test",+ sdk="openai",+ model_type=ModelType.LLM,+ model_id="gpt-docs-test-id",+ inputs=inputs,+ outputs=["text"],+ costs={CostCategory.INPUT: 1, CostCategory.OUTPUT: 2},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ )+ return _CallCountingLLMWorker(inference_model=inference_model)+++def _make_llm_job(*, user_documents: list[PromptDocument]) -> LLMJob:+ return LLMJob(+ job_metadata=JobMetadata(+ run_metadata=RunMetadata(user_id="pytest", pipeline_run_id="plr-document-format", storage_scope="test/scope", read_scope=None),+ pipe_code="summarize_brief",+ ),+ llm_prompt=LLMPrompt(user_text="Summarize the brief.", user_documents=user_documents),+ job_params=LLMJobParams(temperature=0.5),+ job_config=LLMJobConfig(schema_reask_max_attempts=1),+ )++[email protected](loop_scope="class")+class TestLLMWorkerDocumentFormat:+ async def test_a_document_format_the_model_does_not_read_is_an_input_error(self):+ worker = _make_worker(inputs=["text", "pdf"])+ documents: list[PromptDocument] = [+ PromptDocumentUri(uri=STORED_PDF, mime_type="application/pdf"),+ PromptDocumentUri(uri=STORED_DOCX, mime_type=DOCX_MIME),+ ]++ with pytest.raises(PromptDocumentFormatError) as exc_info:+ await worker.gen_text(llm_job=_make_llm_job(user_documents=documents))++ assert str(exc_info.value) == (+ "Prompt document 2 given to model 'gpt-docs-test' is a Word document (.docx), which it does not read: it reads PDF."+ )+ assert exc_info.value.to_error_report().error_domain == ErrorDomain.INPUT+ assert worker.nb_provider_calls == 0++ async def test_a_model_reading_no_documents_is_still_a_capability_error(self):+ worker = _make_worker(inputs=["text", "images"])++ with pytest.raises(LLMCapabilityError, match="does not support documents"):+ await worker.gen_text(llm_job=_make_llm_job(user_documents=[PromptDocumentUri(uri=STORED_PDF, mime_type="application/pdf")]))++ @pytest.mark.parametrize(+ "document",+ [+ PromptDocumentUri(uri=STORED_DOCX, mime_type=DOCX_MIME),+ PromptDocumentUri(uri=STORED_DOCX),+ PromptDocumentBase64(base64_data=base64.b64encode(MARKDOWN_BYTES).decode("ascii")),+ PromptDocumentBinary(raw_bytes=MARKDOWN_BYTES),+ ],+ ids=["declared-format", "unknown-uri", "unidentifiable-base64", "unidentifiable-binary"],+ )+ async def test_a_readable_or_unknown_format_reaches_the_provider(self, document: PromptDocument):+ worker = _make_worker(inputs=["text", "pdf", "docx"])++ assert await worker.gen_text(llm_job=_make_llm_job(user_documents=[document])) == "answer"+ assert worker.nb_provider_calls == 1++ @pytest.mark.parametrize(+ ("uri", "mime_type"),+ [+ (f"data:application/octet-stream;base64,{LATE_ENTRY_DOCX_BASE64}", DOCX_MIME),+ (f"data:{DOCX_MIME};base64,{LATE_ENTRY_DOCX_BASE64}", None),+ ],+ ids=["type-stamped-at-setup", "type-declared-by-the-data-url"],+ )+ async def test_an_inline_word_file_seen_only_as_a_zip_keeps_its_declared_type(self, uri: str, mime_type: str | None):+ """Run setup knew this file as Word from its declared type; the worker must not re-identify it as a zip and refuse it mid-run."""+ worker = _make_worker(inputs=["text", "pdf", "docx"])+ document = PromptDocumentFactory.make_prompt_document(uri=uri, mime_type=mime_type)++ assert document.known_mime_type() == DOCX_MIME+ assert await worker.gen_text(llm_job=_make_llm_job(user_documents=[document])) == "answer"+ assert worker.nb_provider_calls == 1diff --git a/tests/unit/pipelex/cogt/llm/test_llm_worker_prompt_image_format.py b/tests/unit/pipelex/cogt/llm/test_llm_worker_prompt_image_format.pynew file mode 100644index 000000000..da0a819c2--- /dev/null+++ b/tests/unit/pipelex/cogt/llm/test_llm_worker_prompt_image_format.py@@ -0,0 +1,117 @@+"""The LLM worker refuses a prompt image whose known format is not an image, before the provider sees it.++Run setup refuses a non-image given to an Image input, but a run can produce its own images mid-run,+and those never pass through setup. The worker is the second line: a prompt image whose format is+known and is not the image family raises `PromptImageFormatError`, which is a content error and so+in the input domain. An image whose format is unknown is left to the provider, as before.+"""++import base64+from pathlib import Path++import pytest+from typing_extensions import override++from pipelex.base_exceptions import ErrorDomain+from pipelex.cogt.exceptions import PromptImageFormatError+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImageBinary, PromptImageUri+from pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams+from pipelex.cogt.llm.llm_prompt import LLMPrompt+from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract+from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.usage.cost_category import CostCategory+from pipelex.system.job_metadata import JobMetadata, RunMetadata+from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar++PDF_BYTES = Path("tests/data/documents/solar_system.pdf").read_bytes()+PNG_BYTES = Path("tests/data/images/logo-tiny.png").read_bytes()+SVG_BYTES = b'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10"/>'+STORED_URI = "pipelex-storage://org/run/assets/figure.png"+++class _CallCountingLLMWorker(LLMWorkerAbstract):+ """A worker whose provider half only counts the calls that reach it."""++ def __init__(self, *, inference_model: InferenceModelSpec) -> None:+ LLMWorkerAbstract.__init__(self, inference_model=inference_model, reporting_delegate=None)+ self.nb_provider_calls = 0++ @override+ async def _gen_text(self, llm_job: LLMJob) -> str:+ self.nb_provider_calls += 1+ return "answer"++ @override+ async def _gen_object(self, llm_job: LLMJob, *, schema: type[BaseModelTypeVar]) -> BaseModelTypeVar:+ self.nb_provider_calls += 1+ return schema.model_validate({})+++def _make_worker() -> _CallCountingLLMWorker:+ inference_model = InferenceModelSpec(+ backend_name="openai",+ name="gpt-vision-test",+ sdk="openai",+ model_type=ModelType.LLM,+ model_id="gpt-vision-test-id",+ inputs=["text", "images"],+ outputs=["text"],+ costs={CostCategory.INPUT: 1, CostCategory.OUTPUT: 2},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ )+ return _CallCountingLLMWorker(inference_model=inference_model)+++def _make_llm_job(*, user_images: list[PromptImage]) -> LLMJob:+ return LLMJob(+ job_metadata=JobMetadata(+ run_metadata=RunMetadata(user_id="pytest", pipeline_run_id="plr-image-format", storage_scope="test/scope", read_scope=None),+ pipe_code="describe_figure",+ ),+ llm_prompt=LLMPrompt(user_text="Describe the figure.", user_images=user_images),+ job_params=LLMJobParams(temperature=0.5),+ job_config=LLMJobConfig(schema_reask_max_attempts=1),+ )++[email protected](loop_scope="class")+class TestLLMWorkerPromptImageFormat:+ @pytest.mark.parametrize(+ "pdf_image",+ [+ PromptImageBase64(base64_data=base64.b64encode(PDF_BYTES).decode("ascii")),+ PromptImageBinary(raw_bytes=PDF_BYTES),+ PromptImageUri(uri=STORED_URI, mime_type="application/pdf"),+ ],+ )+ async def test_an_image_produced_mid_run_whose_bytes_are_a_pdf_is_refused(self, pdf_image: PromptImage):+ worker = _make_worker()+ png_image = PromptImageBinary(raw_bytes=PNG_BYTES)++ with pytest.raises(PromptImageFormatError) as exc_info:+ await worker.gen_text(llm_job=_make_llm_job(user_images=[png_image, pdf_image]))++ assert str(exc_info.value) == ("Prompt image 2 given to model 'gpt-vision-test' is a PDF document (.pdf), not an image.")+ assert exc_info.value.to_error_report().error_domain == ErrorDomain.INPUT+ assert worker.nb_provider_calls == 0++ @pytest.mark.parametrize(+ "image",+ [+ PromptImageBinary(raw_bytes=PNG_BYTES),+ PromptImageBase64(base64_data=base64.b64encode(PNG_BYTES).decode("ascii")),+ PromptImageUri(uri=STORED_URI, mime_type="image/png"),+ PromptImageUri(uri=STORED_URI),+ PromptImageBinary(raw_bytes=SVG_BYTES),+ ],+ )+ async def test_an_image_or_an_unknown_format_reaches_the_provider(self, image: PromptImage):+ worker = _make_worker()++ assert await worker.gen_text(llm_job=_make_llm_job(user_images=[image])) == "answer"+ assert worker.nb_provider_calls == 1diff --git a/tests/unit/pipelex/cogt/model_backends/test_model_spec_readable_formats.py b/tests/unit/pipelex/cogt/model_backends/test_model_spec_readable_formats.pynew file mode 100644index 000000000..d04229c0b--- /dev/null+++ b/tests/unit/pipelex/cogt/model_backends/test_model_spec_readable_formats.py@@ -0,0 +1,61 @@+"""A model spec's `inputs` names the file formats the model reads, with the same keys every format check compares.++An extractor's readable formats are its declared `pdf`, `docx`, `pptx`, `xlsx`, `html`, `md`, `csv`,+`txt`, `vtt`, `eml` and `image`; `web_page` is not a file format, since a web-page model fetches its page itself. An LLM's readable+document types are its declared `pdf`, `docx`, `pptx`, `xlsx` and `html`; its vision flag stays `images`.+"""++import pytest++from pipelex.cogt.llm.thinking_mode import ThinkingMode+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.usage.cost_category import CostCategory+++def _make_spec(*, model_type: ModelType, inputs: list[str]) -> InferenceModelSpec:+ return InferenceModelSpec(+ backend_name="test",+ name="test-model",+ sdk="test_sdk",+ model_type=model_type,+ model_id="test-model-id",+ inputs=inputs,+ outputs=["text"],+ costs={CostCategory.INPUT: 1.0, CostCategory.OUTPUT: 1.0},+ thinking_mode=ThinkingMode.NONE,+ max_tokens=None,+ max_prompt_images=None,+ )+++class TestModelSpecReadableFormats:+ @pytest.mark.parametrize(+ ("inputs", "expected"),+ [+ (["pdf", "image"], {"pdf", "image"}),+ (["pdf", "docx", "pptx", "xlsx", "html", "image"], {"pdf", "docx", "pptx", "xlsx", "html", "image"}),+ (["md", "csv", "txt", "vtt", "eml", "text"], {"md", "csv", "txt", "vtt", "eml"}),+ (["web_page"], set[str]()),+ (["pdf", "web_page", "captions"], {"pdf"}),+ ([], set[str]()),+ ],+ )+ def test_an_extractor_reads_the_file_formats_it_declares(self, inputs: list[str], expected: set[str]):+ spec = _make_spec(model_type=ModelType.TEXT_EXTRACTOR, inputs=inputs)++ assert spec.readable_formats_for_extract == expected++ @pytest.mark.parametrize(+ ("inputs", "expected"),+ [+ (["text", "images", "pdf"], {"pdf"}),+ (["text", "pdf", "docx", "pptx", "xlsx", "html"], {"pdf", "docx", "pptx", "xlsx", "html"}),+ (["text", "images"], set[str]()),+ ],+ )+ def test_an_llm_reads_the_document_types_it_declares(self, inputs: list[str], expected: set[str]):+ spec = _make_spec(model_type=ModelType.LLM, inputs=inputs)++ assert spec.supported_document_types == expected+ assert spec.is_document_supported == bool(expected)diff --git a/tests/unit/pipelex/cogt/test_worker_error_enrichment.py b/tests/unit/pipelex/cogt/test_worker_error_enrichment.pyindex 4f9caa5bd..61299b4fd 100644--- a/tests/unit/pipelex/cogt/test_worker_error_enrichment.py+++ b/tests/unit/pipelex/cogt/test_worker_error_enrichment.py@@ -124,6 +124,7 @@ def _make_extract_job(mocker: MockerFixture) -> Any:job = mocker.MagicMock()job.extract_input.image_uri = Nonejob.extract_input.document_uri = None+ job.extract_input.mime_type = Nonejob.job_params.should_caption_images = Falsereturn jobdiff --git a/tests/unit/pipelex/kernel/test_llm_prompt_image_mime_type.py b/tests/unit/pipelex/kernel/test_llm_prompt_image_mime_type.pynew file mode 100644index 000000000..8c197761a--- /dev/null+++ b/tests/unit/pipelex/kernel/test_llm_prompt_image_mime_type.py@@ -0,0 +1,58 @@+"""An image's established format rides into the prompt, the way a document's already does.++Run setup stamps every image input with the `mime_type` of its bytes. Assembling the prompt used to+build each prompt image from its url alone and drop that stamp, so the LLM worker had to guess the+format again, from a provider-side sniff, and could not refuse a non-image before the provider did.+"""++import pytest++from pipelex.cogt.image.prompt_image import PromptImageUri+from pipelex.cogt.image.prompt_image_factory import PromptImageFactory+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.kernel.llm_prompt_content import LlmPromptContent, assemble_llm_prompt+from pipelex.kernel.prompt_references import ImageReference, ImageReferenceKind+from pipelex.tools.jinja2.template_category import TemplateCategory+from pipelex.tools.templating.templating_style import TagStyle, TemplatingStyle+from pipelex.tools.templating.text_format import TextFormat++STORED_IMAGE_URI = "pipelex-storage://org/uploads/photo.png"+++class TestLlmPromptImageMimeType:+ def test_the_factory_keeps_a_given_mime_type(self):+ prompt_image = PromptImageFactory.make_prompt_image(uri=STORED_IMAGE_URI, mime_type="image/png")++ assert isinstance(prompt_image, PromptImageUri)+ assert prompt_image.mime_type == "image/png"++ @pytest.mark.asyncio(loop_scope="class")+ async def test_an_assembled_prompt_image_carries_the_content_mime_type(self):+ memory = WorkingMemoryFactory.make_from_single_stuff(+ stuff=StuffFactory.make_stuff(+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),+ content=ImageContent(url=STORED_IMAGE_URI, mime_type="application/pdf"),+ name="photo",+ ),+ )+ prompt_content = LlmPromptContent(+ user_template=TemplateBlueprint(template="Describe the photo.", category=TemplateCategory.LLM_PROMPT),+ user_image_references=[ImageReference(variable_path="photo", kind=ImageReferenceKind.DIRECT)],+ )++ llm_prompt = await assemble_llm_prompt(+ prompt_content=prompt_content,+ context_provider=memory,+ templating_style=TemplatingStyle(tag_style=TagStyle.NO_TAG, text_format=TextFormat.PLAIN),+ )++ assert len(llm_prompt.user_images) == 1+ prompt_image = llm_prompt.user_images[0]+ assert isinstance(prompt_image, PromptImageUri)+ assert prompt_image.uri == STORED_IMAGE_URI+ assert prompt_image.mime_type == "application/pdf"diff --git a/tests/unit/pipelex/pipeline/test_input_format_errors.py b/tests/unit/pipelex/pipeline/test_input_format_errors.pynew file mode 100644index 000000000..43d24de37--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_input_format_errors.py@@ -0,0 +1,38 @@+"""The input-format error family: a file input whose format its slot or its consumer cannot take.++Its messages carry only what the caller supplied (the input's path, the file's type) and what the+method declares, so they are caller-facing and survive STRICT disclosure, where a placeholder would+leave the caller with nothing to act on. The family sits in the input domain, so an API answers 422.+"""++import pytest++from pipelex.base_exceptions import INTERNAL_ERROR_PLACEHOLDER, DisclosureMode, ErrorDomain, error_domain_to_http_status+from pipelex.cogt.inference.error_classification import UserActionKind+from pipelex.pipeline.exceptions import PipelineInputContentError, PipelineInputFormatError, PipelineInputNotAnImageError++NOT_AN_IMAGE_MESSAGE = (+ "Input 'referral_letter' expects an image, but the file is a PDF document (.pdf). Give an image file such as PNG, JPEG or WebP."+)+++class TestPipelineInputFormatErrors:+ @pytest.mark.parametrize("error_class", [PipelineInputFormatError, PipelineInputNotAnImageError])+ def test_the_family_is_an_input_content_error_the_caller_can_fix(self, error_class: type[PipelineInputFormatError]):+ error = error_class(NOT_AN_IMAGE_MESSAGE)++ assert isinstance(error, PipelineInputContentError)+ assert error.error_domain == ErrorDomain.INPUT+ assert error_domain_to_http_status(error.error_domain) == 422+ assert error.user_action is not None+ assert error.user_action.kind == UserActionKind.CHANGE_INPUT++ def test_not_an_image_is_a_format_error(self):+ assert issubclass(PipelineInputNotAnImageError, PipelineInputFormatError)++ @pytest.mark.parametrize("error_class", [PipelineInputFormatError, PipelineInputNotAnImageError])+ def test_the_message_survives_strict_disclosure(self, error_class: type[PipelineInputFormatError]):+ strict = error_class(NOT_AN_IMAGE_MESSAGE).to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT)++ assert strict["message"] == NOT_AN_IMAGE_MESSAGE+ assert strict["message"] != INTERNAL_ERROR_PLACEHOLDERdiff --git a/tests/unit/pipelex/pipeline/test_input_normalizer.py b/tests/unit/pipelex/pipeline/test_input_normalizer.pyindex f36697b6a..420dda6f3 100644--- a/tests/unit/pipelex/pipeline/test_input_normalizer.py+++ b/tests/unit/pipelex/pipeline/test_input_normalizer.py@@ -25,7 +25,7 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.pipeline.exceptions import PipelineInputContentError, PipelineInputUrlInvalidError, PipelineInputUrlMissingError-from pipelex.pipeline.input_normalizer import normalize_data_urls_to_storage+from pipelex.pipeline.input_normalizer import prepare_file_inputsfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract@@ -56,7 +56,7 @@ class TestInputNormalizerUrlGuards:_patch_storage_and_config(mocker)with pytest.raises(PipelineInputUrlMissingError, match="blank url"):- await normalize_data_urls_to_storage(_memory_with_document(blank_url), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(_memory_with_document(blank_url), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)@pytest.mark.parametrize("bad_url", ["https://", "https://exa mple.com/file.pdf"])async def test_malformed_http_url_raises_caller_facing_input_error(self, mocker: MockerFixture, bad_url: str) -> None:@@ -64,7 +64,7 @@ class TestInputNormalizerUrlGuards:_patch_storage_and_config(mocker)with pytest.raises(PipelineInputUrlInvalidError) as exc_info:- await normalize_data_urls_to_storage(_memory_with_document(bad_url), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(_memory_with_document(bad_url), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)strict = exc_info.value.to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT)assert "not a valid http(s) URL" in strict["message"]@@ -74,8 +74,8 @@ class TestInputNormalizerUrlGuards:_patch_storage_and_config(mocker)mock_get = mocker.patch("httpx.AsyncClient.get")- memory = await normalize_data_urls_to_storage(- _memory_with_document("https://example.com/file.pdf"), storage_scope="test/scope", read_scope=None+ memory = await prepare_file_inputs(+ _memory_with_document("https://example.com/file.pdf"), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)content = memory.get_stuff("document").content@@ -93,7 +93,7 @@ class TestInputNormalizerUrlGuards:_patch_storage_and_config(mocker)with pytest.raises(PipelineInputUrlMissingError) as exc_info:- await normalize_data_urls_to_storage(_memory_with_document(""), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(_memory_with_document(""), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)strict = exc_info.value.to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT)assert "blank url" in strict["message"]@@ -111,7 +111,9 @@ class TestInputNormalizerUrlGuards:missing_path = tmp_path / "nope.pdf"with pytest.raises(PipelineInputContentError) as exc_info:- await normalize_data_urls_to_storage(_memory_with_document(str(missing_path)), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(+ _memory_with_document(str(missing_path)), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True+ )report = exc_info.value.to_error_report()assert str(missing_path) in report.message, "precondition: the raw message carries the path"@@ -127,17 +129,21 @@ class TestInputNormalizerUrlGuards:_patch_storage_and_config(mocker)with pytest.raises(PipelineInputContentError, match="cannot be read"):- await normalize_data_urls_to_storage(_memory_with_document(str(tmp_path)), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(_memory_with_document(str(tmp_path)), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)async def test_path_with_null_byte_raises_input_error(self, mocker: MockerFixture, tmp_path: Path) -> None:"""A NUL byte makes the path unusable: the read raises ValueError, not OSError, and it must still be an input error."""_patch_storage_and_config(mocker)with pytest.raises(PipelineInputContentError, match="cannot be read"):- await normalize_data_urls_to_storage(_memory_with_document(f"{tmp_path}/a\x00b.pdf"), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(+ _memory_with_document(f"{tmp_path}/a\x00b.pdf"), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True+ )async def test_missing_file_raises_input_error(self, mocker: MockerFixture, tmp_path: Path) -> None:_patch_storage_and_config(mocker)with pytest.raises(PipelineInputContentError, match="cannot be read"):- await normalize_data_urls_to_storage(_memory_with_document(str(tmp_path / "nope.pdf")), storage_scope="test/scope", read_scope=None)+ await prepare_file_inputs(+ _memory_with_document(str(tmp_path / "nope.pdf")), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True+ )diff --git a/tests/unit/pipelex/pipeline/test_input_normalizer_identification.py b/tests/unit/pipelex/pipeline/test_input_normalizer_identification.pynew file mode 100644index 000000000..f214d3146--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_input_normalizer_identification.py@@ -0,0 +1,244 @@+"""Run setup establishes the format of every file input once, and stamps it as the content's `mime_type`.++The bytes are the truth: a sniffed type replaces any declared one. A stored file is identified by a+head read of its first bytes, a data URL and a local file by the bytes already in hand. When the+bytes identify nothing (plain text, Markdown, HTML), the declared type stands. An http(s) input is+never fetched. Identification runs whether or not relocation to storage is enabled, and every file+input comes back with its path in the inputs (`transcripts[2]`, `case.attachment`) for messages.+"""++import asyncio+import base64+from pathlib import Path+from typing import Any++import pytest+from pydantic import Field+from pytest_mock import MockerFixture++from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.pipeline.exceptions import PipelineInputContentError+from pipelex.pipeline.input_normalizer import (+ MAX_CONCURRENT_FILE_IDENTIFICATIONS,+ FileInputKind,+ collect_file_inputs,+ prepare_file_inputs,+)+from pipelex.tools.misc.filetype_utils import FILE_HEAD_NB_BYTES+from pipelex.tools.storage.local_storage_provider import LocalStorageProvider+from pipelex.tools.uri.resolved_uri import UriKind++DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+DOCX_BYTES = Path("tests/data/documents/CV-ELIAS-THORNE.docx").read_bytes()+PDF_BYTES = Path("tests/data/documents/solar_system.pdf").read_bytes()+PNG_BYTES = Path("tests/data/images/logo-tiny.png").read_bytes()+STORAGE_SCOPE = "test/scope"+++class CaseFile(StructuredContent):+ """A structure holding a document among other fields, for the nested walk."""++ title: str = Field(description="The case title")+ attachment: DocumentContent = Field(description="The case attachment")+++def _concept(code: NativeConceptCode) -> Any:+ return ConceptFactory.make_native_concept(native_concept_code=code)+++def _memory(**contents: StuffContent) -> WorkingMemory:+ memory = WorkingMemoryFactory.make_empty()+ for name, content in contents.items():+ concept_code = NativeConceptCode.IMAGE if isinstance(content, ImageContent) else NativeConceptCode.DOCUMENT+ memory.add_new_stuff(name=name, stuff=StuffFactory.make_stuff(concept=_concept(concept_code), content=content, name=name))+ return memory+++def _data_url(*, mime_type: str, raw_bytes: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(raw_bytes).decode('ascii')}"+++def _storage_root(tmp_path: Path) -> Path:+ return tmp_path / "storage"+++def _stored_bytes(*, tmp_path: Path, uri: str) -> bytes:+ return (_storage_root(tmp_path) / uri.removeprefix("pipelex-storage://")).read_bytes()+++def _stored_keys(*, tmp_path: Path) -> set[str]:+ root = _storage_root(tmp_path)+ return {path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file()}+++def storage(mocker: MockerFixture, tmp_path: Path) -> LocalStorageProvider:+ """A real local storage provider, installed where the normalizer looks it up, with local uploads enabled."""+ provider = LocalStorageProvider(root_path=_storage_root(tmp_path))+ mocker.patch("pipelex.pipeline.input_normalizer.get_storage_provider", return_value=provider)+ mocker.patch(+ "pipelex.pipeline.input_normalizer.get_config",+ return_value=mocker.Mock(runtime=mocker.Mock(storage=mocker.Mock(is_upload_local_content_enabled=True))),+ )+ return provider++[email protected](loop_scope="class")+class TestFileInputIdentification:+ async def test_a_stored_docx_without_mime_type_gets_stamped_from_a_head_read(self, storage: LocalStorageProvider, mocker: MockerFixture):+ uri = await storage.store(data=DOCX_BYTES, key="org/uploads/transcript.docx")+ head_spy = mocker.spy(storage, "load_head")+ full_load_spy = mocker.spy(storage, "load")+ memory = _memory(transcript=DocumentContent(url=uri))++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ content = memory.get_stuff("transcript").content+ assert isinstance(content, DocumentContent)+ assert content.url == uri+ assert content.mime_type == DOCX_MIME+ head_spy.assert_awaited_once_with(uri=uri, nb_bytes=FILE_HEAD_NB_BYTES)+ full_load_spy.assert_not_called()++ async def test_a_png_data_url_holding_a_pdf_is_corrected_to_pdf(self, storage: LocalStorageProvider, tmp_path: Path): # ruff: ignore[unused-method-argument]+ """In a Document slot the corrected type is stamped; in an Image slot the same file is refused (see the image check)."""+ memory = _memory(referral_letter=DocumentContent(url=_data_url(mime_type="image/png", raw_bytes=PDF_BYTES)))++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ content = memory.get_stuff("referral_letter").content+ assert isinstance(content, DocumentContent)+ assert content.mime_type == "application/pdf"+ assert content.url.startswith("pipelex-storage://test/scope/assets/")+ assert content.url.endswith(".pdf")+ assert _stored_bytes(tmp_path=tmp_path, uri=content.url) == PDF_BYTES++ async def test_an_unidentifiable_data_url_keeps_its_declared_type(self, storage: LocalStorageProvider, tmp_path: Path): # ruff: ignore[unused-method-argument]+ """Plain text carries no signature: the declared type stands, where the sniff used to raise FileTypeError (a 500)."""+ markdown = b"# Notes\n\nNothing a sniffer can recognise."+ memory = _memory(notes=DocumentContent(url=_data_url(mime_type="text/markdown", raw_bytes=markdown)))++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ content = memory.get_stuff("notes").content+ assert isinstance(content, DocumentContent)+ assert content.mime_type == "text/markdown"+ assert content.url.endswith(".md")+ assert _stored_bytes(tmp_path=tmp_path, uri=content.url) == markdown++ async def test_an_http_url_is_never_fetched_and_keeps_its_declared_type(self, storage: LocalStorageProvider, mocker: MockerFixture):+ http_get = mocker.patch("httpx.AsyncClient.get")+ head_spy = mocker.spy(storage, "load_head")+ memory = _memory(+ page=DocumentContent(url="https://example.com/report", mime_type="text/html"), other=DocumentContent(url="https://example.com/x")+ )++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ page = memory.get_stuff("page").content+ other = memory.get_stuff("other").content+ assert isinstance(page, DocumentContent)+ assert isinstance(other, DocumentContent)+ assert page.mime_type == "text/html"+ assert other.mime_type is None+ http_get.assert_not_called()+ head_spy.assert_not_called()++ async def test_list_items_and_nested_fields_are_identified_with_their_paths(self, storage: LocalStorageProvider):+ docx_uri = await storage.store(data=DOCX_BYTES, key="org/uploads/a.docx")+ pdf_uri = await storage.store(data=PDF_BYTES, key="org/uploads/b.pdf")+ memory = _memory(+ transcripts=ListContent[DocumentContent](items=[DocumentContent(url=pdf_uri), DocumentContent(url=docx_uri)]),+ )+ memory.add_new_stuff(+ name="case",+ stuff=StuffFactory.make_stuff(+ concept=_concept(NativeConceptCode.TEXT),+ content=CaseFile(title="A case", attachment=DocumentContent(url=docx_uri)),+ name="case",+ ),+ )++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ file_inputs = {file_input.display_path: file_input for file_input in collect_file_inputs(memory)}+ assert set(file_inputs) == {"transcripts[0]", "transcripts[1]", "case.attachment"}+ assert file_inputs["transcripts[0]"].mime_type == "application/pdf"+ assert file_inputs["transcripts[0]"].format_key == "pdf"+ assert file_inputs["transcripts[1]"].format_key == "docx"+ assert file_inputs["transcripts[1]"].path == ("transcripts", 1)+ assert file_inputs["case.attachment"].format_key == "docx"+ assert file_inputs["case.attachment"].path == ("case", "attachment")+ assert file_inputs["case.attachment"].kind == FileInputKind.DOCUMENT+ assert file_inputs["case.attachment"].uri_kind == UriKind.PIPELEX_STORAGE+ case_content = memory.get_stuff("case").content+ assert isinstance(case_content, CaseFile)+ assert case_content.title == "A case"++ async def test_identification_runs_when_relocation_is_disabled(self, storage: LocalStorageProvider, tmp_path: Path):+ stored_uri = await storage.store(data=DOCX_BYTES, key="org/uploads/c.docx")+ local_pdf = tmp_path / "letter.pdf"+ local_pdf.write_bytes(PDF_BYTES)+ data_url = _data_url(mime_type="image/png", raw_bytes=PNG_BYTES)+ memory = _memory(+ stored=DocumentContent(url=stored_uri),+ local=DocumentContent(url=str(local_pdf)),+ inline=ImageContent(url=data_url),+ )++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=False)++ stored = memory.get_stuff("stored").content+ local = memory.get_stuff("local").content+ inline = memory.get_stuff("inline").content+ assert isinstance(stored, DocumentContent)+ assert isinstance(local, DocumentContent)+ assert isinstance(inline, ImageContent)+ assert stored.mime_type == DOCX_MIME+ assert local.mime_type == "application/pdf"+ assert inline.mime_type == "image/png"+ # Nothing was relocated: every url is the one given, and no public link was made.+ assert stored.url == stored_uri+ assert local.url == str(local_pdf)+ assert inline.url == data_url+ assert stored.public_url is None+ assert _stored_keys(tmp_path=tmp_path) == {"org/uploads/c.docx"}++ async def test_a_missing_stored_object_is_an_input_error(self, storage: LocalStorageProvider): # ruff: ignore[unused-method-argument]+ memory = _memory(transcript=DocumentContent(url="pipelex-storage://org/uploads/never-uploaded.docx"))++ with pytest.raises(PipelineInputContentError, match="transcript"):+ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ async def test_stored_files_are_identified_concurrently_within_a_bound(self, storage: LocalStorageProvider, mocker: MockerFixture):+ nb_files = MAX_CONCURRENT_FILE_IDENTIFICATIONS * 3+ uris = [await storage.store(data=PDF_BYTES, key=f"org/uploads/{index}.pdf") for index in range(nb_files)]+ in_flight = 0+ peak_in_flight = 0+ original_load_head = storage.load_head++ async def _slow_load_head(uri: str, *, nb_bytes: int) -> bytes:+ nonlocal in_flight, peak_in_flight+ in_flight += 1+ peak_in_flight = max(peak_in_flight, in_flight)+ await asyncio.sleep(0.01)+ in_flight -= 1+ return await original_load_head(uri, nb_bytes=nb_bytes)++ mocker.patch.object(storage, "load_head", side_effect=_slow_load_head)+ memory = _memory(transcripts=ListContent[DocumentContent](items=[DocumentContent(url=uri) for uri in uris]))++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ assert peak_in_flight > 1, "the head reads overlap"+ assert peak_in_flight <= MAX_CONCURRENT_FILE_IDENTIFICATIONS, "and never more of them than the bound"+ assert all(file_input.format_key == "pdf" for file_input in collect_file_inputs(memory))diff --git a/tests/unit/pipelex/pipeline/test_input_normalizer_image_check.py b/tests/unit/pipelex/pipeline/test_input_normalizer_image_check.pynew file mode 100644index 000000000..194892089--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_input_normalizer_image_check.py@@ -0,0 +1,181 @@+"""Run setup refuses an Image input whose file is not an image, whatever will consume it.++The check needs no model: an Image holding a PDF is wrong for every consumer. It rides the setup's+identification walk, so it covers every listed and nested image and names the input by its path. It+fires only on a positive identification: an image the sniffer cannot identify, an SVG for one, is+left to its consumer. A refused file is refused before it is relocated, so nothing is stored for it.+"""++import base64+import io+import re+import zipfile+from pathlib import Path+from typing import Any++import pytest+from pydantic import Field+from pytest_mock import MockerFixture++from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.core.stuffs.stuff_factory import StuffFactory+from pipelex.pipeline.exceptions import PipelineInputNotAnImageError+from pipelex.pipeline.input_normalizer import prepare_file_inputs+from pipelex.tools.storage.local_storage_provider import LocalStorageProvider++DOCX_BYTES = Path("tests/data/documents/CV-ELIAS-THORNE.docx").read_bytes()+PDF_BYTES = Path("tests/data/documents/solar_system.pdf").read_bytes()+PNG_BYTES = Path("tests/data/images/logo-tiny.png").read_bytes()+JPEG_BYTES = Path("tests/data/images/eiffel_tower.jpg").read_bytes()+SVG_BYTES = b'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10"><rect width="10" height="10"/></svg>'+STORAGE_SCOPE = "test/scope"+++class Patient(StructuredContent):+ """A structure holding an image among other fields, for the nested walk."""++ name: str = Field(description="The patient's name")+ scan: ImageContent = Field(description="The patient's scan")+++def _memory(**contents: StuffContent) -> WorkingMemory:+ memory = WorkingMemoryFactory.make_empty()+ for name, content in contents.items():+ concept_code = NativeConceptCode.TEXT if isinstance(content, Patient) else NativeConceptCode.IMAGE+ memory.add_new_stuff(+ name=name,+ stuff=StuffFactory.make_stuff(concept=ConceptFactory.make_native_concept(native_concept_code=concept_code), content=content, name=name),+ )+ return memory+++def _data_url(*, mime_type: str, raw_bytes: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(raw_bytes).decode('ascii')}"+++def _stored_keys(*, root: Path) -> set[str]:+ return {path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file()}+++def storage(mocker: MockerFixture, tmp_path: Path) -> LocalStorageProvider:+ """A real local storage provider, installed where the normalizer looks it up, with local uploads enabled."""+ provider = LocalStorageProvider(root_path=tmp_path / "storage")+ mocker.patch("pipelex.pipeline.input_normalizer.get_storage_provider", return_value=provider)+ mocker.patch(+ "pipelex.pipeline.input_normalizer.get_config",+ return_value=mocker.Mock(runtime=mocker.Mock(storage=mocker.Mock(is_upload_local_content_enabled=True))),+ )+ return provider++[email protected](loop_scope="class")+class TestInputNormalizerImageCheck:+ @pytest.mark.parametrize("is_relocation_enabled", [True, False])+ async def test_a_pdf_given_to_an_image_input_is_refused_naming_the_input(+ self,+ storage: LocalStorageProvider, # ruff: ignore[unused-method-argument]+ tmp_path: Path,+ is_relocation_enabled: bool,+ ):+ """The data URL claims a PNG; its bytes are a PDF. Nothing is stored for the refused file."""+ memory = _memory(referral_letter=ImageContent(url=_data_url(mime_type="image/png", raw_bytes=PDF_BYTES)))++ with pytest.raises(PipelineInputNotAnImageError) as exc_info:+ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=is_relocation_enabled)++ assert str(exc_info.value) == (+ "Input 'referral_letter' expects an image, but the file is a PDF document (.pdf). Give an image file such as PNG, JPEG or WebP."+ )+ assert not (tmp_path / "storage").exists() or _stored_keys(root=tmp_path / "storage") == set()++ async def test_an_archive_given_to_an_image_input_is_refused(self, storage: LocalStorageProvider): # ruff: ignore[unused-method-argument]+ """A bare zip has no format of its own to report, but it is not an image whatever it holds."""+ buffer = io.BytesIO()+ with zipfile.ZipFile(buffer, "w") as archive:+ archive.writestr("photos/readme.txt", "not a photo")+ memory = _memory(photo=ImageContent(url=_data_url(mime_type="image/png", raw_bytes=buffer.getvalue())))++ with pytest.raises(+ PipelineInputNotAnImageError,+ match=re.escape("Input 'photo' expects an image, but the file is a .zip file."),+ ):+ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=False)++ async def test_a_stored_word_document_given_to_an_image_input_is_refused(self, storage: LocalStorageProvider):+ uri = await storage.store(data=DOCX_BYTES, key="org/uploads/photo.png")++ with pytest.raises(+ PipelineInputNotAnImageError,+ match=re.escape("Input 'photo' expects an image, but the file is a Word document (.docx)."),+ ):+ await prepare_file_inputs(_memory(photo=ImageContent(url=uri)), storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ async def test_a_pdf_as_the_third_item_of_an_image_list_is_named_by_its_index(self, storage: LocalStorageProvider):+ png_uri = await storage.store(data=PNG_BYTES, key="org/uploads/a.png")+ jpeg_uri = await storage.store(data=JPEG_BYTES, key="org/uploads/b.jpg")+ pdf_uri = await storage.store(data=PDF_BYTES, key="org/uploads/c.png")+ photos = ListContent[ImageContent](items=[ImageContent(url=png_uri), ImageContent(url=jpeg_uri), ImageContent(url=pdf_uri)])++ with pytest.raises(PipelineInputNotAnImageError, match=r"Input 'photos\[2\]' expects an image, but the file is a PDF document"):+ await prepare_file_inputs(_memory(photos=photos), storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ async def test_an_image_nested_in_a_structure_is_checked_by_its_path(self, storage: LocalStorageProvider):+ pdf_uri = await storage.store(data=PDF_BYTES, key="org/uploads/scan.png")+ memory = _memory(patient=Patient(name="Ada", scan=ImageContent(url=pdf_uri)))++ with pytest.raises(PipelineInputNotAnImageError, match=r"Input 'patient\.scan' expects an image"):+ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ async def test_real_images_pass(self, storage: LocalStorageProvider):+ png_uri = await storage.store(data=PNG_BYTES, key="org/uploads/a.png")+ memory = _memory(+ stored=ImageContent(url=png_uri),+ inline=ImageContent(url=_data_url(mime_type="image/jpeg", raw_bytes=JPEG_BYTES)),+ )++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ stored = memory.get_stuff("stored").content+ inline = memory.get_stuff("inline").content+ assert isinstance(stored, ImageContent)+ assert isinstance(inline, ImageContent)+ assert stored.mime_type == "image/png"+ assert inline.mime_type == "image/jpeg"++ @pytest.mark.parametrize(+ ("declared_mime_type", "expected_mime_type"),+ [+ ("image/svg+xml", "image/svg+xml"),+ (None, None),+ ],+ )+ async def test_an_unidentifiable_image_is_left_to_its_consumer(+ self, storage: LocalStorageProvider, declared_mime_type: str | None, expected_mime_type: str | None+ ):+ """An SVG carries no signature the sniffer reads: whatever was declared stands, and nothing is refused."""+ svg_uri = await storage.store(data=SVG_BYTES, key="org/uploads/drawing.svg")+ memory = _memory(drawing=ImageContent(url=svg_uri, mime_type=declared_mime_type))++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ drawing: Any = memory.get_stuff("drawing").content+ assert isinstance(drawing, ImageContent)+ assert drawing.mime_type == expected_mime_type++ async def test_an_http_image_is_not_checked(self, storage: LocalStorageProvider): # ruff: ignore[unused-method-argument]+ """An http(s) input is never fetched, so its declared type is all setup knows, and setup does not refuse on a declaration alone."""+ memory = _memory(photo=ImageContent(url="https://example.com/letter.pdf", mime_type="application/pdf"))++ await prepare_file_inputs(memory, storage_scope=STORAGE_SCOPE, read_scope=None, is_relocation_enabled=True)++ photo = memory.get_stuff("photo").content+ assert isinstance(photo, ImageContent)+ assert photo.mime_type == "application/pdf"diff --git a/tests/unit/pipelex/pipeline/test_input_normalizer_public_url.py b/tests/unit/pipelex/pipeline/test_input_normalizer_public_url.pyindex 7cb531112..fc4b47af7 100644--- a/tests/unit/pipelex/pipeline/test_input_normalizer_public_url.py+++ b/tests/unit/pipelex/pipeline/test_input_normalizer_public_url.py@@ -11,7 +11,7 @@ from pipelex.core.stuffs.document_content import DocumentContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.pipeline.exceptions import PipelineInputContentError-from pipelex.pipeline.input_normalizer import NormalizableContent, normalize_data_urls_to_storage+from pipelex.pipeline.input_normalizer import NormalizableContent, prepare_file_inputsfrom pipelex.tools.storage.exceptions import StorageInvalidUriErrorfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract@@ -20,6 +20,8 @@ FRESH_SIGNED_URL = "https://pipelex-app-dev.s3.us-west-2.amazonaws.com/org/uploaSTALE_SIGNED_URL = "https://pipelex-app-dev.s3.us-west-2.amazonaws.com/org/uploads/moodboard.png?X-Amz-Signature=expired"HTTP_INPUT_URL = "https://example.com/moodboard.png"CALLER_PUBLIC_URL = "https://cdn.example.com/moodboard.png"+# The leading bytes of a PNG file, which is what the stored moodboard's head read returns.+STORED_INPUT_HEAD = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"CONTENT_CLASSES = [ImageContent, DocumentContent]@@ -35,7 +37,7 @@ def _memory_with_content(content: NormalizableContent) -> WorkingMemory:async def _normalized_content(content: NormalizableContent) -> Any:- memory = await normalize_data_urls_to_storage(_memory_with_content(content), storage_scope="test/scope", read_scope=None)+ memory = await prepare_file_inputs(_memory_with_content(content), storage_scope="test/scope", read_scope=None, is_relocation_enabled=True)return memory.get_stuff("visual").content@@ -45,6 +47,7 @@ class TestInputNormalizerPublicUrl:def mock_storage(self, mocker: MockerFixture) -> Any:"""A storage provider whose signing returns a fresh link, installed where the normalizer looks it up."""storage = mocker.Mock(spec=StorageProviderAbstract)+ storage.load_head = mocker.AsyncMock(return_value=STORED_INPUT_HEAD)storage.public_url = mocker.AsyncMock(return_value=FRESH_SIGNED_URL)mocker.patch("pipelex.pipeline.input_normalizer.get_storage_provider", return_value=storage)return storagediff --git a/tests/unit/pipelex/pipeline/test_input_normalizer_read_scope.py b/tests/unit/pipelex/pipeline/test_input_normalizer_read_scope.pyindex a1e119bc6..7cebbfb4b 100644--- a/tests/unit/pipelex/pipeline/test_input_normalizer_read_scope.py+++ b/tests/unit/pipelex/pipeline/test_input_normalizer_read_scope.py@@ -19,7 +19,7 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.pipeline.input_normalizer import normalize_data_urls_to_storage+from pipelex.pipeline.input_normalizer import prepare_file_inputsfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstractfrom pipelex.tools.uri.exceptions import UriReadRefusalReason, UriReadRefusedError@@ -47,6 +47,8 @@ def _memory_with_image_list(urls: list[str]) -> WorkingMemory:def _patch_storage_and_config(mocker: MockerFixture, *, is_upload_local_content_enabled: bool) -> Any:storage = mocker.Mock(spec=StorageProviderAbstract)+ # The leading bytes of a PNG file: every stored reference is identified by a head read.+ storage.load_head = mocker.AsyncMock(return_value=b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")storage.public_url = mocker.AsyncMock(return_value="https://signed.example/link")storage.store = mocker.AsyncMock(return_value="pipelex-storage://org_abc/mt_1/run_1/assets/x.png")mocker.patch("pipelex.pipeline.input_normalizer.get_storage_provider", return_value=storage)@@ -68,7 +70,9 @@ class TestInputNormalizerReadScope:local_file.write_bytes(b"\x89PNG\r\n\x1a\n")with pytest.raises(UriReadRefusedError) as exc_info:- await normalize_data_urls_to_storage(_memory_with_image(str(local_file)), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE)+ await prepare_file_inputs(+ _memory_with_image(str(local_file)), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE, is_relocation_enabled=True+ )assert exc_info.value.reason == UriReadRefusalReason.LOCAL_PATHassert "input 'photo'" in str(exc_info.value)@@ -78,14 +82,19 @@ class TestInputNormalizerReadScope:async def test_a_scoped_run_refuses_a_file_uri(self, mocker: MockerFixture) -> None:_patch_storage_and_config(mocker, is_upload_local_content_enabled=True)with pytest.raises(UriReadRefusedError):- await normalize_data_urls_to_storage(_memory_with_image("file:///etc/passwd"), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE)+ await prepare_file_inputs(+ _memory_with_image("file:///etc/passwd"), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE, is_relocation_enabled=True+ )async def test_a_scoped_run_refuses_to_sign_a_foreign_storage_reference(self, mocker: MockerFixture) -> None:storage = _patch_storage_and_config(mocker, is_upload_local_content_enabled=True)with pytest.raises(UriReadRefusedError) as exc_info:- await normalize_data_urls_to_storage(- _memory_with_image("pipelex-storage://org_other/assets/secret.png"), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE+ await prepare_file_inputs(+ _memory_with_image("pipelex-storage://org_other/assets/secret.png"),+ storage_scope=STORAGE_SCOPE,+ read_scope=READ_SCOPE,+ is_relocation_enabled=True,)assert exc_info.value.reason == UriReadRefusalReason.FOREIGN_STORAGE_KEY@@ -94,18 +103,22 @@ class TestInputNormalizerReadScope:async def test_a_foreign_reference_inside_a_list_is_refused_too(self, mocker: MockerFixture) -> None:_patch_storage_and_config(mocker, is_upload_local_content_enabled=True)with pytest.raises(UriReadRefusedError) as exc_info:- await normalize_data_urls_to_storage(+ await prepare_file_inputs(_memory_with_image_list(["pipelex-storage://org_abc/assets/ok.png", "pipelex-storage://org_other/assets/secret.png"]),storage_scope=STORAGE_SCOPE,read_scope=READ_SCOPE,+ is_relocation_enabled=True,)- assert "input 'album'" in str(exc_info.value)+ assert "input 'album[1]'" in str(exc_info.value)async def test_a_scoped_run_signs_an_in_scope_reference(self, mocker: MockerFixture) -> None:storage = _patch_storage_and_config(mocker, is_upload_local_content_enabled=True)- working_memory = await normalize_data_urls_to_storage(- _memory_with_image("pipelex-storage://org_abc/assets/photo.png"), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE+ working_memory = await prepare_file_inputs(+ _memory_with_image("pipelex-storage://org_abc/assets/photo.png"),+ storage_scope=STORAGE_SCOPE,+ read_scope=READ_SCOPE,+ is_relocation_enabled=True,)storage.public_url.assert_awaited_once_with(uri="pipelex-storage://org_abc/assets/photo.png")@@ -114,16 +127,18 @@ class TestInputNormalizerReadScope:async def test_a_scoped_run_keeps_a_data_url_and_an_https_url(self, mocker: MockerFixture) -> None:_patch_storage_and_config(mocker, is_upload_local_content_enabled=True)- await normalize_data_urls_to_storage(- _memory_with_image("data:image/png;base64,iVBORw0KGgo="), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE+ await prepare_file_inputs(+ _memory_with_image("data:image/png;base64,iVBORw0KGgo="), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE, is_relocation_enabled=True+ )+ await prepare_file_inputs(+ _memory_with_image("https://example.com/photo.png"), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE, is_relocation_enabled=True)- await normalize_data_urls_to_storage(_memory_with_image("https://example.com/photo.png"), storage_scope=STORAGE_SCOPE, read_scope=READ_SCOPE)async def test_an_unscoped_run_uploads_a_local_path_as_before(self, mocker: MockerFixture, tmp_path: Path) -> None:storage = _patch_storage_and_config(mocker, is_upload_local_content_enabled=True)local_file = tmp_path / "photo.png"local_file.write_bytes(b"\x89PNG\r\n\x1a\n")- await normalize_data_urls_to_storage(_memory_with_image(str(local_file)), storage_scope="run_1", read_scope=None)+ await prepare_file_inputs(_memory_with_image(str(local_file)), storage_scope="run_1", read_scope=None, is_relocation_enabled=True)storage.store.assert_awaited_once()diff --git a/tests/unit/pipelex/tools/misc/test_file_format_keys.py b/tests/unit/pipelex/tools/misc/test_file_format_keys.pynew file mode 100644index 000000000..e19e8f2f6--- /dev/null+++ b/tests/unit/pipelex/tools/misc/test_file_format_keys.py@@ -0,0 +1,198 @@+"""The format-key vocabulary and the non-raising identification helpers in `filetype_utils`.++A format key is what every format check compares: a file's key, derived from its MIME type, against+the keys a model declares it reads. Every image type is the `image` family; any other identified+type is its extension; no type, or the generic octet-stream, has no key.+"""++import base64+import io+import zipfile+from pathlib import Path++import pytest++from pipelex.tools.misc.filetype_utils import (+ FILE_HEAD_NB_BYTES,+ describe_file_format,+ describe_format_key,+ describe_format_keys,+ format_key_from_mime_type,+ guess_file_type_from_bytes,+ identify_mime_type,+)++DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+PPTX_MIME = "application/vnd.openxmlformats-officedocument.presentationml.presentation"+XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"++DOCUMENTS_DIR = Path("tests/data/documents")+IMAGES_DIR = Path("tests/data/images")+++def _late_office_entry_zip_head() -> bytes:+ """The head of a Word file whose `word/` entry comes after the entries the sniffer looks at, so it sees only a zip."""+ buffer = io.BytesIO()+ with zipfile.ZipFile(buffer, "w") as archive:+ for entry_name in (+ "[Content_Types].xml",+ "_rels/.rels",+ "docProps/app.xml",+ "docProps/core.xml",+ "docProps/custom.xml",+ "customXml/item1.xml",+ ):+ archive.writestr(entry_name, "<xml/>")+ archive.writestr("word/document.xml", "<w:document/>")+ return buffer.getvalue()[:FILE_HEAD_NB_BYTES]+++class TestFileFormatKeys:+ @pytest.mark.parametrize(+ ("mime_type", "expected_key"),+ [+ ("application/pdf", "pdf"),+ (DOCX_MIME, "docx"),+ (PPTX_MIME, "pptx"),+ (XLSX_MIME, "xlsx"),+ ("application/msword", "doc"),+ ("application/vnd.ms-powerpoint", "ppt"),+ ("application/vnd.ms-excel", "xls"),+ ("image/png", "image"),+ ("image/jpeg", "image"),+ ("image/webp", "image"),+ ("image/svg+xml", "image"),+ ("text/html", "html"),+ ("text/markdown", "md"),+ ("text/plain; charset=utf-8", "txt"),+ ("APPLICATION/PDF", "pdf"),+ ],+ )+ def test_identified_types_have_a_key(self, mime_type: str, expected_key: str):+ assert format_key_from_mime_type(mime_type=mime_type) == expected_key++ @pytest.mark.parametrize("mime_type", [None, "", " ", "application/octet-stream", "application/x-unheard-of"])+ def test_unknown_types_have_no_key(self, mime_type: str | None):+ assert format_key_from_mime_type(mime_type=mime_type) is None++ def test_identifies_a_pdf(self):+ raw_bytes = (DOCUMENTS_DIR / "solar_system.pdf").read_bytes()+ file_type = guess_file_type_from_bytes(raw_bytes=raw_bytes)+ assert file_type is not None+ assert file_type.mime == "application/pdf"++ def test_identifies_a_docx_from_its_head_alone(self):+ head = (DOCUMENTS_DIR / "CV-ELIAS-THORNE.docx").read_bytes()[:FILE_HEAD_NB_BYTES]+ file_type = guess_file_type_from_bytes(raw_bytes=head)+ assert file_type is not None+ assert file_type.mime == DOCX_MIME++ def test_returns_none_for_plain_text_instead_of_raising(self):+ assert guess_file_type_from_bytes(raw_bytes=b"# A Markdown title\n\nSome text.") is None++ def test_returns_none_for_no_bytes(self):+ assert guess_file_type_from_bytes(raw_bytes=b"") is None++ def test_the_sniffed_type_wins_over_the_declared_one(self):+ pdf_head = (DOCUMENTS_DIR / "solar_system.pdf").read_bytes()[:FILE_HEAD_NB_BYTES]+ assert identify_mime_type(head=pdf_head, declared_mime_type="image/png", file_name=None) == "application/pdf"++ def test_the_declared_type_stands_when_the_sniff_fails(self):+ assert identify_mime_type(head=b"plain words", declared_mime_type="text/markdown", file_name=None) == "text/markdown"++ def test_no_head_keeps_the_declared_type(self):+ assert identify_mime_type(head=None, declared_mime_type="application/pdf", file_name=None) == "application/pdf"++ def test_nothing_known_gives_none(self):+ assert identify_mime_type(head=b"plain words", declared_mime_type=None, file_name=None) is None++ def test_a_declared_office_type_refines_a_sniffed_zip(self):+ """An Office Open XML file is a zip: when the sniffer sees only the container, the declared Office type is the more precise truth."""+ zip_head = b"PK\x03\x04" + b"\x00" * 26 + b"unrelated/entry.bin" + b"\x00" * 64+ sniffed = guess_file_type_from_bytes(raw_bytes=zip_head)+ assert sniffed is not None, "precondition: the sniffer sees a zip container"+ assert sniffed.mime == "application/zip"+ assert identify_mime_type(head=zip_head, declared_mime_type=DOCX_MIME, file_name=None) == DOCX_MIME+ assert identify_mime_type(head=zip_head, declared_mime_type="application/zip", file_name=None) == "application/zip"++ @pytest.mark.parametrize(+ ("declared_mime_type", "file_name", "expected"),+ [+ (None, None, None),+ ("application/octet-stream", None, None),+ ("image/png", None, None),+ (None, "report.docx", DOCX_MIME),+ (None, "deck.pptx", PPTX_MIME),+ ("application/octet-stream", "budget.xlsx", XLSX_MIME),+ (None, "notes.txt", None),+ (PPTX_MIME, "report.docx", PPTX_MIME),+ ("application/zip", "report.docx", DOCX_MIME),+ ("application/zip; charset=binary", "deck.pptx", PPTX_MIME),+ ("application/zip", "archive.zip", "application/zip"),+ (None, "archive.zip", "application/zip"),+ ],+ )+ def test_a_bare_zip_is_unknown_unless_its_declared_type_or_its_name_says_what_it_holds(+ self, declared_mime_type: str | None, file_name: str | None, expected: str | None+ ):+ """A sniffed bare zip may be an Office file whose identifying entry came late, so it is never a format of its own to refuse."""+ zip_head = _late_office_entry_zip_head()+ sniffed = guess_file_type_from_bytes(raw_bytes=zip_head)+ assert sniffed is not None, "precondition: the sniffer sees a zip container"+ assert sniffed.mime == "application/zip", "precondition: the sniffer does not see the Word entry"+ assert identify_mime_type(head=zip_head, declared_mime_type=declared_mime_type, file_name=file_name) == expected++ def test_a_png_data_url_holding_a_pdf_is_a_pdf(self):+ pdf_bytes = (DOCUMENTS_DIR / "solar_system.pdf").read_bytes()+ decoded = base64.b64decode(base64.b64encode(pdf_bytes))+ assert identify_mime_type(head=decoded[:FILE_HEAD_NB_BYTES], declared_mime_type="image/png", file_name=None) == "application/pdf"++ def test_an_image_is_identified_as_one(self):+ png_head = (IMAGES_DIR / "logo-tiny.png").read_bytes()[:FILE_HEAD_NB_BYTES]+ assert identify_mime_type(head=png_head, declared_mime_type=None, file_name=None) == "image/png"++ @pytest.mark.parametrize(+ ("format_key", "expected"),+ [+ ("pdf", "a PDF document"),+ ("docx", "a Word document"),+ ("pptx", "a PowerPoint presentation"),+ ("xlsx", "an Excel workbook"),+ ("doc", "a Word 97-2003 document"),+ ("html", "an HTML page"),+ ("image", "an image"),+ ("zip", "a .zip file"),+ ],+ )+ def test_describes_a_key_in_plain_words(self, format_key: str, expected: str):+ assert describe_format_key(format_key=format_key) == expected++ @pytest.mark.parametrize(+ ("format_key", "mime_type", "expected"),+ [+ ("pdf", "application/pdf", "a PDF document (.pdf)"),+ ("docx", DOCX_MIME, "a Word document (.docx)"),+ ("image", "image/png", "an image (image/png)"),+ ("md", "text/markdown", "a Markdown file (.md)"),+ ("eml", "message/rfc822", "an email message (.eml)"),+ ("json", "application/json", "a .json file"),+ ("zip", "application/zip", "a .zip file"),+ ],+ )+ def test_describes_a_file_with_the_type_that_tells_it_apart(self, format_key: str, mime_type: str, expected: str):+ assert describe_file_format(format_key=format_key, mime_type=mime_type) == expected++ @pytest.mark.parametrize(+ ("format_keys", "expected"),+ [+ ({"pdf"}, "PDF"),+ ({"image", "pdf"}, "PDF and images"),+ ({"html", "xlsx", "pptx", "docx", "pdf"}, "PDF, Word (.docx), PowerPoint (.pptx), Excel (.xlsx) and HTML"),+ ({"web_page"}, "web pages"),+ ({"md", "pdf"}, "PDF and Markdown (.md)"),+ ({"json", "pdf"}, "PDF and .json"),+ (set[str](), "no file format"),+ ],+ )+ def test_describes_a_set_of_readable_formats_in_a_stable_order(self, format_keys: set[str], expected: str):+ assert describe_format_keys(format_keys=format_keys) == expecteddiff --git a/tests/unit/pipelex/tools/storage/test_gcp_storage_provider.py b/tests/unit/pipelex/tools/storage/test_gcp_storage_provider.pyindex 187771213..3a6759bd3 100644--- a/tests/unit/pipelex/tools/storage/test_gcp_storage_provider.py+++ b/tests/unit/pipelex/tools/storage/test_gcp_storage_provider.py@@ -279,3 +279,43 @@ class TestGcpStorageProvider:# Client should only be created once (lazy initialization)mock_gcp_storage["from_service_account_json"].assert_called_once()++ async def test_load_head_downloads_a_byte_range(+ self,+ gcp_provider_no_signed_urls: GcpStorageProvider,+ mock_gcp_storage: dict[str, Any],+ ) -> None:+ """A head read asks GCS for the first bytes only, through an inclusive end offset."""+ mock_gcp_storage["blob"].download_as_bytes.return_value = b"%PDF-1.7"++ head = await gcp_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}test/file.pdf", nb_bytes=8192)++ assert head == b"%PDF-1.7"+ mock_gcp_storage["bucket"].blob.assert_called_with("test/file.pdf")+ mock_gcp_storage["blob"].download_as_bytes.assert_called_once_with(start=0, end=8191)++ async def test_load_head_of_an_empty_object_is_empty(+ self,+ gcp_provider_no_signed_urls: GcpStorageProvider,+ mock_gcp_storage: dict[str, Any],+ ) -> None:+ """GCS answers a range over an empty object with 416, which is an empty head and not a failure."""+ from google.api_core.exceptions import RequestRangeNotSatisfiable # type: ignore[import-untyped] # ruff: ignore[import-outside-top-level]++ mock_gcp_storage["blob"].download_as_bytes.side_effect = RequestRangeNotSatisfiable("Requested range not satisfiable")++ head = await gcp_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}test/empty.bin", nb_bytes=8192)++ assert head == b""++ async def test_load_head_of_a_missing_object_raises_not_found(+ self,+ gcp_provider_no_signed_urls: GcpStorageProvider,+ mock_gcp_storage: dict[str, Any],+ ) -> None:+ from google.api_core.exceptions import NotFound # type: ignore[import-untyped] # ruff: ignore[import-outside-top-level]++ mock_gcp_storage["blob"].download_as_bytes.side_effect = NotFound("Object not found")++ with pytest.raises(StorageFileNotFoundError, match=r"missing/file\.pdf"):+ await gcp_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}missing/file.pdf", nb_bytes=8192)diff --git a/tests/unit/pipelex/tools/storage/test_s3_storage_provider.py b/tests/unit/pipelex/tools/storage/test_s3_storage_provider.pyindex 37b982fcd..221602fee 100644--- a/tests/unit/pipelex/tools/storage/test_s3_storage_provider.py+++ b/tests/unit/pipelex/tools/storage/test_s3_storage_provider.py@@ -446,3 +446,54 @@ class TestS3StorageProvider:assert first_call_count == 1assert second_call_count == 1 # Session should only be created once++ async def test_load_head_sends_a_ranged_get(+ self,+ mocker: MockerFixture,+ s3_provider_no_signed_urls: S3StorageProvider,+ mock_aiobotocore: dict[str, Any],+ ) -> None:+ """A head read asks S3 for the first bytes only, through an inclusive Range header."""+ mock_aiobotocore["stream"].read = mocker.AsyncMock(return_value=b"%PDF-1.7")++ head = await s3_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}test/file.pdf", nb_bytes=8192)++ assert head == b"%PDF-1.7"+ mock_aiobotocore["client"].get_object.assert_called_once_with(Bucket=S3_TEST_BUCKET, Key="test/file.pdf", Range="bytes=0-8191")++ async def test_load_head_of_an_empty_object_is_empty(+ self,+ mocker: MockerFixture,+ s3_provider_no_signed_urls: S3StorageProvider,+ mock_aiobotocore: dict[str, Any],+ ) -> None:+ """S3 answers a range over an empty object with InvalidRange, which is an empty head and not a failure."""+ invalid_range = ClientError({"Error": {"Code": "InvalidRange", "Message": "The requested range is not satisfiable"}}, "GetObject")+ mock_aiobotocore["client"].get_object = mocker.AsyncMock(side_effect=invalid_range)++ head = await s3_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}test/empty.bin", nb_bytes=8192)++ assert head == b""++ async def test_load_head_of_a_missing_object_raises_not_found(+ self,+ mocker: MockerFixture,+ s3_provider_no_signed_urls: S3StorageProvider,+ mock_aiobotocore: dict[str, Any],+ ) -> None:+ mock_aiobotocore["client"].get_object = mocker.AsyncMock(side_effect=mock_aiobotocore["exceptions"].NoSuchKey("Key not found"))++ with pytest.raises(StorageFileNotFoundError, match=r"missing/file\.pdf"):+ await s3_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}missing/file.pdf", nb_bytes=8192)++ async def test_load_head_maps_other_client_errors(+ self,+ mocker: MockerFixture,+ s3_provider_no_signed_urls: S3StorageProvider,+ mock_aiobotocore: dict[str, Any],+ ) -> None:+ access_denied = ClientError({"Error": {"Code": "AccessDenied", "Message": "Access Denied"}}, "GetObject")+ mock_aiobotocore["client"].get_object = mocker.AsyncMock(side_effect=access_denied)++ with pytest.raises(StorageS3Error, match="AccessDenied"):+ await s3_provider_no_signed_urls.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}test/file.pdf", nb_bytes=8192)diff --git a/tests/unit/pipelex/tools/storage/test_storage_load_head.py b/tests/unit/pipelex/tools/storage/test_storage_load_head.pynew file mode 100644index 000000000..0397435f0--- /dev/null+++ b/tests/unit/pipelex/tools/storage/test_storage_load_head.py@@ -0,0 +1,47 @@+"""The default head read every storage provider inherits: the whole object, sliced.++A provider that can read a byte range (S3, GCS) overrides it; one that cannot, including an external+storage plugin written before the head read existed, keeps working through this default.+"""++from pathlib import Path++import pytest++from pipelex.tools.storage.exceptions import StorageFileNotFoundError+from pipelex.tools.storage.in_memory_storage_provider import InMemoryStorageProvider+from pipelex.tools.storage.local_storage_provider import LocalStorageProvider+from pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEME++PAYLOAD = b"%PDF-1.7\n" + b"x" * 20_000++[email protected](loop_scope="class")+class TestDefaultLoadHead:+ async def test_in_memory_head_is_the_first_bytes(self) -> None:+ provider = InMemoryStorageProvider()+ uri = await provider.store(data=PAYLOAD, key="head/file.pdf")++ head = await provider.load_head(uri=uri, nb_bytes=8192)++ assert head == PAYLOAD[:8192]++ async def test_local_head_is_the_first_bytes(self, tmp_path: Path) -> None:+ provider = LocalStorageProvider(root_path=tmp_path)+ uri = await provider.store(data=PAYLOAD, key="head/file.pdf")++ head = await provider.load_head(uri=uri, nb_bytes=8192)++ assert head == PAYLOAD[:8192]++ async def test_a_head_longer_than_the_object_is_the_whole_object(self) -> None:+ provider = InMemoryStorageProvider()+ uri = await provider.store(data=b"short", key="head/short.txt")++ assert await provider.load_head(uri=uri, nb_bytes=8192) == b"short"++ async def test_a_missing_object_raises_not_found(self, tmp_path: Path) -> None:+ provider = LocalStorageProvider(root_path=tmp_path)++ with pytest.raises(StorageFileNotFoundError):+ await provider.load_head(uri=f"{PIPELEX_STORAGE_SCHEME}missing/file.pdf", nb_bytes=8192)