Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
fix/Fetch-ssrf-guard · L-260823-6e13cf (#1302)
Louis Choquel4438efe6e7f2
diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.tomlindex cb4c4bf4a..d994c1c75 100644--- a/.drift/acks/config-docs.toml+++ b/.drift/acks/config-docs.toml@@ -1,21 +1,27 @@contract = "config-docs"-digest = "sha256:f7050e3e65c768937a8f9796327e1d85501bc6d3d55849955ebc0610e1dfd64e"+digest = "sha256:7884ebc7893389b5d54058576d36b731e56da1b3707986aac9b4000547cb040a"reviewed_by = "Louis Choquel"-reviewed_at = "2026-09-25T14:56:51Z"-rationale = "Merge of dev, which now carries the redaction and gcp members, into the Rich extraction branch. The trigger files combine what each side acked on its own: pipelex.toml and log_config.py are dev's, with both the redaction and gcp sections already documented in logging-config.md, and config_check.py only defers its Rich Panel import into the warning branch that prints it, because Rich is now the cli extra. No config model, default or validator changes in the merge itself, so no configuration doc and no migration ledger entry is owed."+reviewed_at = "2026-09-29T11:07:53Z"+rationale = "Widened the triggers to every config submodel that configs.py imports from its own package: network_config.py, new on this branch, plus storage_config.py, secrets_config.py, graph_config.py, mthds_config.py and methods_config.py, which were already missing. None of the five pre-existing files changes content here, so no doc is stale because of them. network_config.py still matches docs/configuration/config-technical/network-config.md: one bool, is_fetch_ssrf_guard_enabled, with its default in pipelex/pipelex.toml. No validator narrows a domain and no migration ledger entry is owed."[trigger_files]"pipelex/cogt/config_cogt.py" = "blob:02f71cb4a6b01aebda59c2e31f262ef9c865e17b""pipelex/cogt/img_gen/img_gen_job_components.py" = "blob:084985080413fd8d6c3b1b9b0747020cb0161858"-"pipelex/pipelex.toml" = "blob:8d5b60c54ea342992546918b718563a02629d31f"+"pipelex/graph/graph_config.py" = "blob:dc0939a6d2730c0a10b88aa1a6e0b2dd583462a6"+"pipelex/language/mthds_config.py" = "blob:353009ad8c59dc5266bdd9dfa42b1ad18b67dca7"+"pipelex/methods/methods_config.py" = "blob:76ca9a0e64f8c10ef6a355aafd0d79d36b09f06a"+"pipelex/pipelex.toml" = "blob:ca10eca8a5c04a73601ed1fc1f5fae0221ffadb0""pipelex/system/configuration/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc""pipelex/system/configuration/config_check.py" = "blob:2d5de29aba3d2ce205d3d82b4623f37b53108332""pipelex/system/configuration/config_loader.py" = "blob:ac384ce993933656682999c26afffb4c9f9b1975""pipelex/system/configuration/config_model.py" = "blob:83b29dc6e93393c9b98c4353636caf0135c80339""pipelex/system/configuration/config_root.py" = "blob:c7d43851270912d17413cfc6a511247d3c7cc160""pipelex/system/configuration/config_surface.py" = "blob:96f685656a3f4355e8a073e8bf6184e020fea8f5"-"pipelex/system/configuration/configs.py" = "blob:cd7dee01503888351d71e58fd1b1cc88420a1def"+"pipelex/system/configuration/configs.py" = "blob:3cb65cf9c3d34aef0baf92b62d8bd63e5c69d4eb""pipelex/system/configuration/pipe_func_config.py" = "blob:dc8890e914fbcdb611147e42eaa0cd87b32288a5""pipelex/system/telemetry/telemetry_config.py" = "blob:7070951a6d3e4c10cb7d59822ce168c6590c53be""pipelex/tools/aws/aws_config.py" = "blob:c1aeafeae08d3921464db33795f11a29ff3892ba""pipelex/tools/log/log_config.py" = "blob:5b98b3ebe7090fa306cd5253c96812605f0d3dbc"+"pipelex/tools/network/network_config.py" = "blob:e12d1d0e4a40d04859f10a77addeaf8fe2b00414"+"pipelex/tools/secrets/secrets_config.py" = "blob:5f23e754ed4c5e76e62ea3a8f88fc885a4062551"+"pipelex/tools/storage/storage_config.py" = "blob:3ba0fee5282937e2639d2707ee8ab61aad7bf206"diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.tomlindex 8100c079a..5524343c1 100644--- a/.pipelex/pipelex.toml+++ b/.pipelex/pipelex.toml@@ -67,6 +67,17 @@ bucket_name = ""project_id = ""signed_urls_lifespan_seconds = 3600 # Set to "disabled" for public URLs+####################################################################################################+# Network Config+####################################################################################################++[runtime.network]+# Whether fetching a document or image URL refuses private, loopback, link-local and cloud-metadata+# destinations, on the first request and on every redirect hop. Keep it on unless your documents live+# on an intranet host or your only way out is an HTTP proxy: with the guard on, fetches connect+# directly and ignore HTTP_PROXY / HTTPS_PROXY. Webhook delivery is guarded whatever this says.+is_fetch_ssrf_guard_enabled = true+##################################################################################################### Secrets Config####################################################################################################diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e5d9e635f..331cc5c17 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -19,6 +19,7 @@- **A template reads a `Composite`'s parts by name**: `{{ combo.summary }}`, `{{ combo['summary'] }}` and `combo.get('summary')` resolve the named parts of a `Composite` output, and `iter_keys` lists them; they used to render empty, and only the raw `_content`, now refused, reached them. Like any private name, a part whose name starts with an underscore is refused when a template reads it on an input.- **A template's arithmetic error is a render error**: a division by zero, or a `range` longer than the sandbox's cap, raises `Jinja2TemplateRenderError` where the raw Python exception escaped, which validation reported as an internal error rather than as the method's.- **A sandboxed PipeFunc's imports resolve to the same file in every process**: the sandbox searches a method's root directory first and then its subdirectories in sorted order, where the order followed the hash of their paths, which changes from one process to the next. A module name used in two directories of a method, such as a `structures.py` shipped in a subdirectory beside the `structures` module the sandbox generates at the root, could resolve to either file. A temporary directory reached through a symlink, as on macOS, no longer lets a file's own directory jump ahead of that order or stay on `sys.path` after the run.+- **Docling extracts HTML, Markdown and CSV**: a document Docling reads without pages (HTML, Markdown, CSV, AsciiDoc) comes back as one page holding the whole document, where it came back with no pages and no text, and one given as a storage key or a `data:` URL reaches Docling, where its bytes were refused as an unknown file type.### Security@@ -26,6 +27,7 @@- **Dotted paths refuse underscore segments**: a PipeCompose construct `from` path such as `note.__class__.__init__.__globals__.sys.modules.os.environ` walked into Python internals with plain `getattr` and could copy the process environment into a method's output. A `from` path or `list_to_dict_keyed_by` name with a segment starting with an underscore is now refused at validation, and every dotted path the runtime walks, batch lists and prompt image references included, refuses one.- **A scoped run reads only under its read scope, and never the local disk**: on a run with a read scope, a `pipelex-storage://` key is read only when it lies under that scope, compared segment by segment, and a bare path or a `file://` URI is never read, whichever value carries it — an input, a `PipeCompose` construct that assembles a URL from text, a model's structured output. Every content-generation leaf checks the URLs it will read before the dry-run branch, and the input seam checks a CSV input and every file it would upload or sign; a refused read raises `UriReadRefusedError`, answered as a 422, which names where the URL sat without quoting it. A run whose read scope is `None` reads as before.- **A sandbox-hosted load imports no method Python, and refuses structure classes (Breaking)**: on a deployment whose PipeFunc execution mode is not `direct`, loading a library directory captures every `.py` file as source for the sandbox and imports none of them, where a file declaring a `StructuredContent` subclass was imported into the loading process and its module-level code ran. A load whose Python declares such a class, however the method arrived, now raises `MethodStructuresRefusedError` naming each file and class by its relative path, the error a fetched package already got: declare the types as MTHDS concepts with inline structures, which a PipeFunc imports in the sandbox from the generated `structures` module. The module `pipelex build structures` writes is accepted while it is left as generated, by this refusal and by the fetched-package one, and is not sent to the sandbox, which generates its own from the same concepts. Both refusals, and the local warnings that predict them, now give that same route. A `direct` load is unchanged.+- **Fetching a URL refuses private destinations (Breaking)**: every download of a URL a value carries — a prompt image or document for a provider that does not take URLs, a document to extract or render, a generated image a provider returned as a link — now refuses a host that is `localhost`, a cloud metadata alias or resolves to any private, loopback, link-local or metadata address, on the first request and on every redirect hop, raising `SsrfBlockedError` and failing the pipe, where it followed redirects anywhere and returned the body. The guarded fetch connects directly and ignores `HTTP_PROXY` / `HTTPS_PROXY`; a self-hosted deployment that reads documents from an intranet host or reaches the internet through a proxy sets the new `[runtime.network] is_fetch_ssrf_guard_enabled = false`. `make_base64_url_from_any_uri` and `make_base64_url_from_http_url` move to `pipelex.tools.uri.uri_base64`.## [v0.69.0] - 2026-09-28diff --git a/docs/configuration/config-technical/network-config.md b/docs/configuration/config-technical/network-config.mdnew file mode 100644index 000000000..2b426dbac--- /dev/null+++ b/docs/configuration/config-technical/network-config.md@@ -0,0 +1,48 @@+---+description: "Configure how Pipelex fetches the documents and images a method points at — the SSRF guard that refuses private destinations, and when a self-hosted deployment turns it off."+---++# Network Configuration++Configuration section: `[runtime.network]`++## Overview++A method's values can carry `http://` and `https://` URLs: an image input, a PDF to extract, a document a model reads, a file an image-generation provider hands back. When a model or an extractor needs the bytes rather than the link, Pipelex downloads them itself, following redirects. Since a method, a model's structured output or a function can produce any URL, the download is guarded: Pipelex refuses to connect to any destination that is not a public internet address.++```toml+[runtime.network]+is_fetch_ssrf_guard_enabled = true+```++The value above is the default.++## What the guard refuses++With `is_fetch_ssrf_guard_enabled = true`, every download of a value's URL is checked when the connection opens, on the first request and on every redirect hop to a new host. The host is resolved, and the connection is refused when the name is `localhost` or a cloud metadata alias, or when any address it resolves to is not globally routable: private networks (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`), loopback, link-local (including the `169.254.169.254` metadata endpoint), carrier-grade NAT, multicast, reserved and unspecified addresses, in IPv4 and IPv6. Pipelex then connects to the address it checked, so a DNS record that changes between the check and the connection cannot slip through.++A refusal raises [`SsrfBlockedError`](../../errors/ssrf-blocked-error.md) and fails the pipe that needed the bytes. It is not treated as a failed download: where Pipelex would otherwise keep a generated image's remote URL after a download error, a refused URL fails the pipe instead. The error message names the host that was requested and never the address it resolved to.++A URL passed to a provider as a URL, for a model that fetches images itself, is not downloaded by Pipelex and is not checked by this guard.++## When to turn it off++Keep the guard on unless your deployment needs one of these:++- **Documents on a private network.** A self-hosted deployment inside a company network may exist precisely to read documents from an intranet host, which resolves to a private address.+- **An image-generation server on your own machine or network.** A self-hosted model server that answers with a link to the image on `localhost` or a private address has that link refused when Pipelex downloads the image, which it does while `[runtime.storage] is_fetch_remote_content_enabled` is on.+- **Egress through an HTTP proxy.** The guard connects directly and ignores `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`: through a proxy, the proxy makes the connection, and the guard would only ever see the proxy's own address. If a proxy is your only way out, remote downloads fail until the guard is off.++Turning it off restores a plain HTTP client, which honours the proxy variables and follows redirects without checking where they lead:++```toml+[runtime.network]+is_fetch_ssrf_guard_enabled = false+```++Do not turn it off on a server that runs methods from people you do not trust with your network.++## What the switch does not cover++- **Webhook delivery** is always guarded, whatever this switch says: a run's callback URL on a private address is refused.+- **Local paths and storage keys** (`pipelex-storage://`) are not network fetches; what a run may read from them is governed by the run's read scope, described in [Distributed content generation](../../under-the-hood/distributed-content-generation.md#what-a-leaf-may-read-the-read-scope).diff --git a/docs/configuration/index.md b/docs/configuration/index.mdindex 3075c9603..ad3a7681f 100644--- a/docs/configuration/index.md+++ b/docs/configuration/index.md@@ -51,7 +51,7 @@ In addition to the base `pipelex.toml`, Pipelex applies override files from **inThe configuration is organized into four main sections, which mirror the layers of the runtime:-1. `[runtime]` - process-scoped infrastructure: storage, secrets, logging, cloud credentials, reporting, tracing, observation, and the plugin denylist+1. `[runtime]` - process-scoped infrastructure: storage, secrets, logging, cloud credentials, outbound network posture, reporting, tracing, observation, and the plugin denylist2. `[inference]` - the model-calling seam: the model deck, LLM, image generation, extraction, the default templating style, and dry-run mocks3. `[interpreter]` - library-scoped method machinery: MTHDS parsing, pipe runs, pipe functions, pipeline execution, source scanning, and the builder4. `[kit]` - settings for the `pipelex-dev` kit toolingdiff --git a/docs/features/cloud-storage.md b/docs/features/cloud-storage.mdindex 4308ff3f8..46cd892eb 100644--- a/docs/features/cloud-storage.md+++ b/docs/features/cloud-storage.md@@ -36,3 +36,5 @@ A deployment whose egress rules allow S3 by hostname must allow `*.s3.<region>.aStorage is configured in `pipelex.toml` under `[runtime.storage]`. Set `method` to `local`, `in_memory`, `s3`, or `gcp`, then provide provider-specific settings such as `bucket_name`, `region`, or `project_id` in the matching subsection.For AWS configuration, see [AWS Configuration](../configuration/config-technical/aws-config.md).++When an image-generation provider returns a URL rather than bytes and `is_fetch_remote_content_enabled` is on, Pipelex downloads the image into storage. That download, like every download of a URL a value carries, goes through the SSRF guard, which refuses private and metadata addresses; see [Network Configuration](../configuration/config-technical/network-config.md).diff --git a/docs/under-the-hood/distributed-content-generation.md b/docs/under-the-hood/distributed-content-generation.mdindex 3b8313e56..96d8d9f87 100644--- a/docs/under-the-hood/distributed-content-generation.md+++ b/docs/under-the-hood/distributed-content-generation.md@@ -177,6 +177,8 @@ On a run whose read scope is set, a storage key is read only when it lies underThe check runs in the leaves, because both orchestration modes call the same leaves and every read of a value's URL happens while one of them handles an assignment. Each assignment declares the URLs it will read through `referenced_uris()` — the prompt's images and documents given by URI, an image generation's input images, the image or document to extract, the document to render as page views — and the leaf authorizes them as its first statement, before the dry-run branch and before any worker is built. A dry run therefore refuses the same method a live run would, and the check performs no IO. The input seam applies the same rule to what it reads itself: a table given as a CSV input, and a file the normalization would upload or link. A refused read raises `UriReadRefusedError`, a caller-facing input error answered as a 422, which names where the URL sat and never quotes it. A unit test walks every assignment model's fields for URL-shaped ones, so a new assignment or a new field that carries a URL fails until its `referenced_uris()` declares it.+The read scope leaves `https://` URLs alone because they are not the process's to scope: they are fetched over the network, and the network fetch has its own guard. Every leaf that needs the bytes behind a URL — prompt images and documents for a provider that does not take URLs, the document an extractor reads, the PDF rendered as page views, a generated image a provider returned as a link — goes through one fetch helper, and that helper connects through `SsrfGuardedTransport` unless `[runtime.network] is_fetch_ssrf_guard_enabled` is off. The transport resolves the host when each connection opens, on the first request and on every redirect hop to a new origin, refuses a name that resolves to any private, loopback, link-local or metadata address, and dials the address it vetted. A refusal raises `SsrfBlockedError`, a security error that no leaf's download fallback absorbs, so the pipe fails. The guard connects directly and ignores `HTTP(S)_PROXY`; a deployment whose egress goes through a proxy, or that reads documents from an intranet host, turns the switch off, as the [network configuration](../configuration/config-technical/network-config.md) explains. Checking in the fetch helper rather than in input normalization is deliberate: a construct, a structured output or a function result can produce a URL as easily as an input can.+---## Kajson class resolutiondiff --git a/drift.toml b/drift.tomlindex cf25fa5c4..211737f4e 100644--- a/drift.toml+++ b/drift.toml@@ -28,8 +28,14 @@ triggers = ["pipelex/system/telemetry/telemetry_config.py","pipelex/cogt/config_cogt.py","pipelex/cogt/img_gen/img_gen_job_components.py",+ "pipelex/graph/graph_config.py",+ "pipelex/language/mthds_config.py",+ "pipelex/methods/methods_config.py","pipelex/tools/aws/aws_config.py","pipelex/tools/log/log_config.py",+ "pipelex/tools/network/network_config.py",+ "pipelex/tools/secrets/secrets_config.py",+ "pipelex/tools/storage/storage_config.py","pipelex/pipelex.toml",]# Three kinds of target, not one. The docs are the user-facing prose. The ledgers are where adiff --git a/mkdocs.yml b/mkdocs.ymlindex 648eeb327..071d473f7 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -315,6 +315,7 @@ plugins:- configuration/config-technical/inference-config.md: "Inference Configuration"- configuration/config-technical/inference-backend-config.md: "LLM Providers & Models Configuration"- configuration/config-technical/library-config.md: "Library Configuration"+ - configuration/config-technical/network-config.md: "Network Configuration"- tools/plxt.md: "plxt (Formatter & Linter)"- tools/logging.md: "Logging"- analytics/data-extraction.md: "Observer Data Extraction"@@ -555,6 +556,7 @@ nav:- Inference: configuration/config-technical/inference-config.md- LLM Providers & Models: configuration/config-technical/inference-backend-config.md- Library: configuration/config-technical/library-config.md+ - Network: configuration/config-technical/network-config.md- Advanced:- Advanced Customizations:- Overview: advanced/index.mddiff --git a/pipelex/cogt/content_generation/generated_content_factory.py b/pipelex/cogt/content_generation/generated_content_factory.pyindex 7c55916e5..6ba2d4b92 100644--- a/pipelex/cogt/content_generation/generated_content_factory.py+++ b/pipelex/cogt/content_generation/generated_content_factory.py@@ -192,10 +192,13 @@ class GeneratedContentFactory:public_url: str | Noneif is_remote_url and get_config().runtime.storage.is_fetch_remote_content_enabled:try:- # `RemoteFileFetchError` is the ONLY exception this can raise: the fetch- # helper converts every httpx failure into it. Catching the httpx classes- # here, as this used to, caught nothing — a 404 or a timeout on a remote- # image escaped and failed the whole pipe instead of degrading to the URL.+ # The fetch helper converts every httpx failure into `RemoteFileFetchError`,+ # so a 404 or a timeout on a remote image degrades to keeping the URL.+ # `SsrfBlockedError` is left to fail the pipe on purpose: a provider URL that+ # resolves to a private address is not a flaky download, and degrading to+ # keeping the URL would report a refused destination as a success. With+ # `is_fetch_remote_content_enabled` off, nothing here connects to the URL,+ # so there is nothing to refuse and the URL is kept as it came.actual_bytes, fetched_mime_type = await self._fetch_remote_content(url=url)except RemoteFileFetchError as exc:log.warning(f"Failed to fetch a remote image: {exc}")diff --git a/pipelex/config.py b/pipelex/config.pyindex 4debc3eee..001904104 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -57,6 +57,18 @@ def is_method_fetch_on_miss_enabled() -> bool:return optional_config.interpreter.methods.fetch_on_miss+def is_fetch_ssrf_guard_enabled() -> bool:+ """Whether fetching a URL's bytes goes through the SSRF guard in this process (non-raising).++ The config's ``runtime.network.is_fetch_ssrf_guard_enabled``, defaulting to enabled when no+ config is set: a process that fetches before, or without, loading the config fails closed.+ """+ optional_config = get_optional_config()+ if not isinstance(optional_config, PipelexConfig):+ return True+ return optional_config.runtime.network.is_fetch_ssrf_guard_enabled++def is_pipe_func_sandbox_hosted() -> bool:"""Whether PipeFunc runs out-of-process in this process (non-raising; defaults to local).diff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.tomlindex 8100c079a..5524343c1 100644--- a/pipelex/kit/configs/pipelex.toml+++ b/pipelex/kit/configs/pipelex.toml@@ -67,6 +67,17 @@ bucket_name = ""project_id = ""signed_urls_lifespan_seconds = 3600 # Set to "disabled" for public URLs+####################################################################################################+# Network Config+####################################################################################################++[runtime.network]+# Whether fetching a document or image URL refuses private, loopback, link-local and cloud-metadata+# destinations, on the first request and on every redirect hop. Keep it on unless your documents live+# on an intranet host or your only way out is an HTTP proxy: with the guard on, fetches connect+# directly and ignore HTTP_PROXY / HTTPS_PROXY. Webhook delivery is guarded whatever this says.+is_fetch_ssrf_guard_enabled = true+##################################################################################################### Secrets Config####################################################################################################diff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index 8d5b60c54..ca10eca8a 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -28,6 +28,9 @@ bucket_name = ""project_id = ""signed_urls_lifespan_seconds = 3600+[runtime.network]+is_fetch_ssrf_guard_enabled = true+[runtime.secrets]method = "env" # env (built-in); an external plugin may add e.g. vaultdiff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index 94a2f350f..fb7c00c17 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -1249,6 +1249,17 @@"default": "console","open_node": false},+ "runtime.network": {+ "value_type": "table",+ "required": true,+ "open_node": false+ },+ "runtime.network.is_fetch_ssrf_guard_enabled": {+ "value_type": "bool",+ "required": true,+ "default": true,+ "open_node": false+ },"runtime.observer": {"value_type": "table","required": true,diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 8d5b60c54..ca10eca8a 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -28,6 +28,9 @@ bucket_name = ""project_id = ""signed_urls_lifespan_seconds = 3600+[runtime.network]+is_fetch_ssrf_guard_enabled = true+[runtime.secrets]method = "env" # env (built-in); an external plugin may add e.g. vaultdiff --git a/pipelex/providers/docling/docling_extract_worker.py b/pipelex/providers/docling/docling_extract_worker.pyindex 0fae387ef..e5d85f475 100644--- a/pipelex/providers/docling/docling_extract_worker.py+++ b/pipelex/providers/docling/docling_extract_worker.py@@ -1,8 +1,10 @@import asyncioimport base64+import reimport tempfile-from pathlib import Path+from pathlib import Path, PurePosixPathfrom typing import Any+from urllib.parse import urlparseimport aiofilesfrom typing_extensions import override@@ -12,7 +14,6 @@ from pipelex.cogt.extract.exceptions import ExtractInputErrorfrom pipelex.cogt.extract.extract_job import ExtractJobfrom pipelex.cogt.extract.extract_output import ExtractOutputfrom pipelex.cogt.extract.extract_worker_abstract import ExtractWorkerAbstract-from pipelex.cogt.file.file_preparation_utils import prepare_file_from_urifrom pipelex.cogt.inference.error_classification import extract_local_extract_metadatafrom pipelex.cogt.inference.error_classify import classify_inference_errorfrom pipelex.cogt.inference.error_render import InferenceErrorFamily, render_inference_error@@ -21,7 +22,21 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.providers.docling.docling_factory import DoclingFactoryfrom pipelex.providers.docling.docling_sdk import DoclingSdkfrom pipelex.reporting.reporting_protocol import ReportingProtocol-from pipelex.tools.uri.prepared_file import PreparedFileBase64, PreparedFileHttpUrl, PreparedFileLocalPath+from pipelex.runtime_hub import get_storage_provider+from pipelex.tools.misc.file_fetch_utils import fetch_file_and_content_type_from_url_httpx+from pipelex.tools.misc.filetype_utils import mime_type_to_extension+from pipelex.tools.uri.resolved_uri import ResolvedBase64DataUrl, ResolvedHttpUrl, ResolvedLocalPath, ResolvedPipelexStorage+from pipelex.tools.uri.uri_resolver import resolve_uri++# A Google Drive, Docs, Sheets or Slides link names a viewer page, not the file. Docling+# rewrote one to its export URL when it downloaded URLs itself, and the download here does too.+_GOOGLE_LINK_PATTERN = re.compile(r"google\.com/(file|document|spreadsheets|presentation)/d/([\w-]+)")+_GOOGLE_EXPORT_URL_TEMPLATES = {+ "file": "https://drive.google.com/uc?export=download&id={doc_id}",+ "document": "https://docs.google.com/document/d/{doc_id}/export?format=docx",+ "spreadsheets": "https://docs.google.com/spreadsheets/d/{doc_id}/export?format=xlsx",+ "presentation": "https://docs.google.com/presentation/d/{doc_id}/export?format=pptx",+}class DoclingExtractWorker(ExtractWorkerAbstract):@@ -61,34 +76,46 @@ class DoclingExtractWorker(ExtractWorkerAbstract):return await self._extract_from_source(source_uri=source_uri)async def _extract_from_source(self, source_uri: str) -> ExtractOutput:- """Extract text from any supported URI type (file path, http(s) URL, pipelex-storage://, or base64 data URL)."""- prepared = await prepare_file_from_uri(- uri=source_uri,- keep_http_url=True,- keep_local_path=True,- )+ """Extract text from any supported URI type (file path, http(s) URL, pipelex-storage://, or base64 data URL).++ Docling reads a path, so any source that is not a local file is written to a temp file+ first, named with an extension Docling can read the format from.++ An http(s) URL is downloaded by the runtime's fetch helper, never handed to Docling.+ Docling's own client checks each host with a single DNS lookup before connecting, which+ a record that changes in between defeats, and it does not follow the runtime's+ `is_fetch_ssrf_guard_enabled` switch. The helper vets the address it actually connects+ to, on every redirect hop.+ """+ raw_bytes: bytes+ media_type: str | None+ named_after: str | None+ resolved_uri = resolve_uri(source_uri)+ match resolved_uri:+ case ResolvedLocalPath():+ return await self._convert(docling_source=resolved_uri.path)+ case ResolvedHttpUrl():+ download_url = self._export_url_for_google_link(url=resolved_uri.url)+ raw_bytes, media_type = await fetch_file_and_content_type_from_url_httpx(download_url)+ named_after = download_url+ case ResolvedPipelexStorage():+ raw_bytes, media_type = await get_storage_provider().load_with_metadata(uri=resolved_uri.storage_uri)+ named_after = resolved_uri.storage_uri+ case ResolvedBase64DataUrl():+ raw_bytes = base64.b64decode(resolved_uri.base64_data)+ media_type = resolved_uri.mime_type+ named_after = None- docling_source: str- temp_path: Path | None = None-- match prepared:- case PreparedFileHttpUrl():- docling_source = prepared.url- case PreparedFileLocalPath():- docling_source = prepared.path- case PreparedFileBase64():- # Docling needs a file path, so write base64 data to temp file- suffix = f".{prepared.file_type.extension}" if prepared.file_type.extension else ".pdf"- with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file:- temp_path = Path(temp_file.name)- try:- async with aiofiles.open(temp_path, "wb") as file:- await file.write(base64.b64decode(prepared.base64_data))- docling_source = str(temp_path)- except BaseException:- temp_path.unlink(missing_ok=True)- raise+ temp_path = await self._write_temp_file(+ data=raw_bytes,+ suffix=self._suffix_for_docling(named_after=named_after, media_type=media_type),+ )+ try:+ return await self._convert(docling_source=str(temp_path))+ finally:+ temp_path.unlink(missing_ok=True)+ async def _convert(self, *, docling_source: str) -> ExtractOutput:try:# Run synchronous Docling conversion in a thread pool to avoid blockingconversion_result = await asyncio.to_thread(self.docling_sdk.document_converter.convert, docling_source)@@ -107,6 +134,56 @@ class DoclingExtractWorker(ExtractWorkerAbstract):model_desc=self.inference_model.desc,model_handle=self.inference_model.name,) from sdk_exc- finally:- if temp_path:- temp_path.unlink(missing_ok=True)++ @classmethod+ def _export_url_for_google_link(cls, *, url: str) -> str:+ """Return the export URL of a Google Drive, Docs, Sheets or Slides link, and any other URL unchanged."""+ google_link = _GOOGLE_LINK_PATTERN.search(url)+ if not google_link:+ return url+ doc_type, doc_id = google_link.groups()+ return _GOOGLE_EXPORT_URL_TEMPLATES[doc_type].format(doc_id=doc_id)++ @classmethod+ def _suffix_for_docling(cls, *, named_after: str | None, media_type: str | None) -> str:+ """Pick the extension a temp file is written under, so Docling can tell its format.++ Docling reads a file's format from its bytes first, which settles every binary format.+ A text format (HTML, Markdown, CSV) says nothing in its bytes, so Docling falls back on+ the extension, and refuses one it does not know. So the extension is one Docling knows:+ the one the source's URL or storage key ends with, else the one the declared media type+ implies. With neither, the file gets none, and Docling recognizes HTML and CSV from the+ content and refuses anything else. When Docling fetched URLs itself it also read a+ `Content-Disposition` filename, which the fetch helper does not return.+ """+ docling_extensions = cls._docling_extensions()+ if named_after:+ suffix = PurePosixPath(urlparse(named_after).path).suffix+ if suffix.removeprefix(".").lower() in docling_extensions:+ return suffix+ if media_type:+ extension = mime_type_to_extension(media_type)+ if extension.lower() in docling_extensions:+ return f".{extension}"+ return ""++ @classmethod+ def _docling_extensions(cls) -> frozenset[str]:+ """The file extensions Docling maps to a format, lowercased."""+ # Deferred import: avoid pulling heavy SDK at module-load time+ from docling.datamodel.base_models import FormatToExtensions # ruff: ignore[import-outside-top-level]++ return frozenset(extension.lower() for extensions in FormatToExtensions.values() for extension in extensions)++ @classmethod+ async def _write_temp_file(cls, *, data: bytes, suffix: str) -> Path:+ """Write `data` to a new temp file named with `suffix`, which the caller deletes."""+ with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file:+ temp_path = Path(temp_file.name)+ try:+ async with aiofiles.open(temp_path, "wb") as file:+ await file.write(data)+ except BaseException:+ temp_path.unlink(missing_ok=True)+ raise+ return temp_pathdiff --git a/pipelex/providers/docling/docling_factory.py b/pipelex/providers/docling/docling_factory.pyindex 3917d7244..d7c66c656 100644--- a/pipelex/providers/docling/docling_factory.py+++ b/pipelex/providers/docling/docling_factory.py@@ -20,6 +20,10 @@ class DoclingFactory:) -> ExtractOutput:"""Convert a Docling document to ExtractOutput with markdown text per page.+ A document with no pages, which is every format that is not laid out on pages (HTML,+ Markdown, CSV, AsciiDoc), becomes a single page holding the whole document: serialized+ page by page, it would come back empty.+Args:doc: The Docling document from conversion result.@@ -31,6 +35,10 @@ class DoclingFactory:pages: dict[int, Page] = {}+ if not doc.pages:+ pages[0] = Page(text=MarkdownDocSerializer(doc=doc).serialize().text)+ return ExtractOutput(pages=pages)+# doc.pages is a dictionary where keys are page numbers (1-based in Docling)for page_no in sorted(doc.pages.keys()):# Create a serializer that targets only the specific pagediff --git a/pipelex/providers/gateway/gateway_extract_worker.py b/pipelex/providers/gateway/gateway_extract_worker.pyindex 060010dd0..0445b3858 100644--- a/pipelex/providers/gateway/gateway_extract_worker.py+++ b/pipelex/providers/gateway/gateway_extract_worker.py@@ -25,7 +25,7 @@ from pipelex.providers.gateway.gateway_protocols import GatewayExtractProtocolfrom pipelex.providers.gateway.gateway_search_schemas import GatewayFetchRequestParamsfrom pipelex.reporting.reporting_protocol import ReportingProtocolfrom pipelex.runtime_hub import get_storage_provider-from pipelex.tools.uri.uri_resolver import make_base64_url_from_any_uri+from pipelex.tools.uri.uri_base64 import make_base64_url_from_any_uriclass GatewayExtractWorker(ExtractWorkerAbstract):diff --git a/pipelex/providers/manifold/manifold_extract_worker.py b/pipelex/providers/manifold/manifold_extract_worker.pyindex c96e10486..a73b872e6 100644--- a/pipelex/providers/manifold/manifold_extract_worker.py+++ b/pipelex/providers/manifold/manifold_extract_worker.py@@ -39,7 +39,7 @@ from pipelex.providers.manifold.manifold_native_client import ManifoldNativeCliefrom pipelex.providers.manifold.manifold_schemas import ManifoldExtractInput, ManifoldExtractParams, ManifoldExtractRequestfrom pipelex.runtime_hub import get_storage_providerfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error-from pipelex.tools.uri.uri_resolver import make_base64_url_from_any_uri+from pipelex.tools.uri.uri_base64 import make_base64_url_from_any_uriif TYPE_CHECKING:from pipelex.cogt.extract.extract_job import ExtractJobdiff --git a/pipelex/system/configuration/configs.py b/pipelex/system/configuration/configs.pyindex cd7dee015..3cb65cf9c 100644--- a/pipelex/system/configuration/configs.py+++ b/pipelex/system/configuration/configs.py@@ -12,6 +12,7 @@ from pipelex.system.configuration.config_root import ConfigRootfrom pipelex.system.configuration.pipe_func_config import PipeFuncConfigfrom pipelex.tools.aws.aws_config import AwsConfigfrom pipelex.tools.log.log_config import LogConfig+from pipelex.tools.network.network_config import NetworkConfigfrom pipelex.tools.secrets.secrets_config import SecretsProviderConfigfrom pipelex.tools.storage.storage_config import StorageConfig@@ -179,10 +180,11 @@ class RuntimeConfig(ConfigModel):The machinery present at execution time whatever is loaded, per``docs/contribute/hub-layering.md``: storage, secrets, logging, cloud credentials,- reporting, tracing, observation, and the plugin system's own denylist.+ reporting, tracing, observation, outbound network posture, and the plugin system's own denylist."""storage: StorageConfig+ network: NetworkConfigsecrets: SecretsProviderConfiglog: LogConfigaws: AwsConfigdiff --git a/pipelex/tools/misc/base64_utils.py b/pipelex/tools/misc/base64_utils.pyindex d990e301f..6879bf665 100644--- a/pipelex/tools/misc/base64_utils.py+++ b/pipelex/tools/misc/base64_utils.py@@ -1,7 +1,6 @@import base64from pathlib import Path-from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpxfrom pipelex.tools.misc.file_utils import load_binary_asyncfrom pipelex.tools.misc.filetype_utils import FileType, detect_file_type_from_bytes@@ -20,14 +19,6 @@ async def make_base64_url_from_path(path: Path) -> str:return make_base64_url(base64_data=base64_data, file_type=file_type)-async def make_base64_url_from_http_url(url: str) -> str:- """Fetch a URL and create a data: URL from its contents."""- raw_bytes = await fetch_file_from_url_httpx(url=url)- base64_data = base64.b64encode(raw_bytes).decode("ascii")- file_type = detect_file_type_from_bytes(raw_bytes=raw_bytes)- return make_base64_url(base64_data=base64_data, file_type=file_type)--def make_base64_url_from_bytes(raw_bytes: bytes) -> str:"""Create a data: URL from raw bytes, detecting file type automatically."""base64_data = base64.b64encode(raw_bytes).decode("ascii")diff --git a/pipelex/tools/misc/file_fetch_utils.py b/pipelex/tools/misc/file_fetch_utils.pyindex a367ee834..f5782e738 100644--- a/pipelex/tools/misc/file_fetch_utils.py+++ b/pipelex/tools/misc/file_fetch_utils.py@@ -1,8 +1,10 @@import httpxfrom httpx import USE_CLIENT_DEFAULT, Response, Timeout+from pipelex.config import is_fetch_ssrf_guard_enabledfrom pipelex.tools.misc.exceptions import RemoteFileFetchErrorfrom pipelex.tools.misc.http_utils import get_user_agent+from pipelex.tools.network.ssrf_guard import SsrfGuardedTransportasync def fetch_file_and_content_type_from_url_httpx(@@ -18,13 +20,32 @@ async def fetch_file_and_content_type_from_url_httpx(answer about what a remote URL actually serves, so a caller that stores those bytesunder a media type has to ask for it rather than guess from what it requested.+ The URL is whatever a value carried, so a method can aim it anywhere. Unless the+ ``runtime.network.is_fetch_ssrf_guard_enabled`` switch is off, the request goes through+ :class:`SsrfGuardedTransport`, which refuses a private, loopback, link-local or+ metadata destination at connect time, on the first request and on every redirect hop.+ The guard dials directly, so ``HTTP_PROXY`` / ``HTTPS_PROXY`` are honoured only with the+ switch off.++ Args:+ url: The http(s) URL to fetch.+ request_timeout: Seconds allowed for each phase of the request, or ``None`` for+ httpx's default.+ transport: A test seam, for a ``MockTransport``. Production code never passes it:+ passing one replaces the guard.+Returns:The response body, and the declared media type or ``None``.Raises:RemoteFileFetchError: The server answered with an error status, refused theconnection, or did not answer in time.+ SsrfBlockedError: The destination, or a redirect hop's, is not a globally routable+ address. It is a security error and deliberately not a ``RemoteFileFetchError``,+ so no caller's fallback for a failed download absorbs it."""+ if transport is None and is_fetch_ssrf_guard_enabled():+ transport = SsrfGuardedTransport()user_agent = get_user_agent()# httpx reads an explicit `timeout=None` as NO timeout, not as "use the client's# default" — that is what its `USE_CLIENT_DEFAULT` sentinel is for. Passing the@@ -67,9 +88,14 @@ async def fetch_file_from_url_httpx() -> bytes:"""Fetch the bytes at ``url``, or raise :class:`RemoteFileFetchError` saying why not.+ Guarded against private destinations like+ :func:`fetch_file_and_content_type_from_url_httpx`, whose ``transport`` test seam it forwards.+Raises:RemoteFileFetchError: The server answered with an error status, refused theconnection, or did not answer in time.+ SsrfBlockedError: The destination, or a redirect hop's, is not a globally routable+ address."""raw_bytes, _ = await fetch_file_and_content_type_from_url_httpx(url,diff --git a/pipelex/tools/misc/filetype_utils.py b/pipelex/tools/misc/filetype_utils.pyindex 7442a73b2..feee863e6 100644--- a/pipelex/tools/misc/filetype_utils.py+++ b/pipelex/tools/misc/filetype_utils.py@@ -18,6 +18,8 @@ _MIME_OVERRIDES: Final[dict[str, str]] = {"image/jpeg": "jpeg","image/jpg": "jpeg","text/plain": "txt",+ # Python 3.11's default table has no entry for Markdown; 3.12 added `.md`+ "text/markdown": "md","application/json": "json",}diff --git a/pipelex/tools/network/exceptions.py b/pipelex/tools/network/exceptions.pyindex 79a3b1cbc..d6794922f 100644--- a/pipelex/tools/network/exceptions.py+++ b/pipelex/tools/network/exceptions.py@@ -5,23 +5,29 @@ class SsrfBlockedError(SecurityError):"""Raised when an outbound request is refused because the destination hostresolved to a disallowed (private / loopback / link-local / metadata) address.- Closes the DNS-rebinding gap that a request-time literal-IP check cannot:- a callback URL like ``https://attacker.example/cb`` passes a literal-host- check, yet its DNS record can resolve to ``169.254.169.254`` / ``127.0.0.0/8``- / ``10.0.0.0/8`` by the time the worker actually fires the webhook. The guard- re-resolves at connect time and refuses the socket.+ Raised for any request to a URL the runtime did not choose: a webhook's+ callback URL, and the document, image or generated file a value points at,+ on the first request and on every redirect hop. It closes the DNS-rebinding+ gap that a request-time literal-IP check cannot: a URL like+ ``https://attacker.example/x`` passes a literal-host check, yet its DNS record+ can resolve to ``169.254.169.254`` / ``127.0.0.0/8`` / ``10.0.0.0/8`` by the+ time the request is made. The guard re-resolves at connect time and refuses+ the socket.- A :class:`SecurityError` (not a ``WebhookDeliveryError``) on purpose: a- blocked SSRF attempt is a security signal that must not be swallowed by the- domain-level ``except httpx.RequestError`` handlers around webhook delivery —- it surfaces and aborts the delivery.+ A :class:`SecurityError` (not a ``WebhookDeliveryError`` or a+ ``RemoteFileFetchError``) on purpose: a blocked SSRF attempt is a security+ signal that must not be swallowed by the domain-level handlers around webhook+ delivery or a download's fallback — it surfaces and fails the delivery or the+ pipe.- ``error_domain = INPUT`` because the destination was caller-supplied (a- callback URL on the originating request); the caller fixes it by providing a- public URL. ``_authors_caller_facing_message`` lets the message survive STRICT- disclosure — it deliberately names only the caller's own hostname, never the- resolved private IP, so it cannot confirm internal network topology to a- probing client.+ ``error_domain = INPUT`` because the destination was supplied from outside the+ runtime (a callback URL on the originating request, or a URL a method's value+ carried); the caller fixes it by providing a public URL.+ ``_authors_caller_facing_message`` lets the message survive STRICT+ disclosure — it deliberately names only the requested hostname, never the+ resolved private IP, so it never reveals an internal address to a probing+ client. That a name resolves to a private address at all stays observable,+ since the refusal is told apart from a host that cannot be reached."""error_domain = ErrorDomain.INPUTdiff --git a/pipelex/tools/network/network_config.py b/pipelex/tools/network/network_config.pynew file mode 100644index 000000000..e12d1d0e4--- /dev/null+++ b/pipelex/tools/network/network_config.py@@ -0,0 +1,14 @@+from pipelex.system.configuration.config_model import ConfigModel+++class NetworkConfig(ConfigModel):+ """Outbound network posture of the runtime.++ ``is_fetch_ssrf_guard_enabled`` decides whether fetching the bytes behind a URL a value carries+ (a document, an image, a provider's generated file) goes through the SSRF guard, which refuses+ every destination and redirect hop that is not a globally routable address. It is on by default;+ a self-hosted deployment that reads documents from an intranet host, or whose only way out is an+ HTTP proxy, turns it off. Webhook delivery is guarded regardless of this switch.+ """++ is_fetch_ssrf_guard_enabled: booldiff --git a/pipelex/tools/network/ssrf_guard.py b/pipelex/tools/network/ssrf_guard.pyindex cdabc7e77..b61839d54 100644--- a/pipelex/tools/network/ssrf_guard.py+++ b/pipelex/tools/network/ssrf_guard.py@@ -1,4 +1,4 @@-"""DNS-rebinding-safe httpx transport for caller-supplied outbound URLs.+"""DNS-rebinding-safe httpx transport for outbound URLs the runtime did not choose.A request-time literal-IP check (``is_disallowed_host``) cannot stop SSRF via DNSrebinding: ``https://attacker.example/cb`` passes validation, but its A record can@@ -89,7 +89,8 @@ class SsrfGuardedBackend(httpcore.AsyncNetworkBackend):"""Wraps another network backend, vetting the destination IP before connecting.Only ``connect_tcp`` carries the guard — it is the sole DNS-resolving entry- point for the http(s) origins webhooks use. ``connect_unix_socket`` / ``sleep``+ point for http(s) origins, and httpcore calls it for every new connection, so+ each redirect hop to a new origin is vetted too. ``connect_unix_socket`` / ``sleep``delegate unchanged."""@@ -151,9 +152,15 @@ class SsrfGuardedBackend(httpcore.AsyncNetworkBackend):class SsrfGuardedTransport(httpx.AsyncHTTPTransport):"""An :class:`httpx.AsyncHTTPTransport` that vets every destination IP at connect.- Use for any outbound request to a caller-supplied URL (webhook delivery). The+ Use for any outbound request to a URL the runtime did not choose: webhook+ delivery, and the fetch helper that reads the bytes behind a value's URL. Thetransport re-resolves the host at connect time and refuses private/metadata- destinations, raising :class:`SsrfBlockedError`.+ destinations, raising :class:`SsrfBlockedError`. Because the check runs per+ connection, a client that follows redirects has every new-origin hop vetted.++ Passing a transport to an httpx client turns off its ``HTTP(S)_PROXY``+ handling. The guard needs that: through a proxy it would only ever see the+ proxy's own address."""def __init__(self) -> None:diff --git a/pipelex/tools/uri/uri_base64.py b/pipelex/tools/uri/uri_base64.pynew file mode 100644index 000000000..63cebcebe--- /dev/null+++ b/pipelex/tools/uri/uri_base64.py@@ -0,0 +1,75 @@+"""Turn any URI a value carries into a base64 data URL, fetching remote content as needed.++This lives apart from ``uri_resolver`` and ``base64_utils`` on purpose: resolving a URI and+encoding bytes are pure functions the kernel layer needs everywhere, while these two reach the+network through the fetch helper. Keeping them apart keeps the fetch helper, and the+configuration it reads, out of the kernel's import closure.+"""++from pathlib import Path++from pipelex.tools.misc.base64_utils import make_base64_url_from_bytes, make_base64_url_from_path+from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract+from pipelex.tools.uri.resolved_uri import (+ ResolvedBase64DataUrl,+ ResolvedHttpUrl,+ ResolvedLocalPath,+ ResolvedPipelexStorage,+)+from pipelex.tools.uri.uri_resolver import resolve_uri+++async def make_base64_url_from_http_url(url: str) -> str:+ """Fetch a URL and create a data: URL from its contents.++ Raises:+ RemoteFileFetchError: The server answered with an error status, refused the+ connection, or did not answer in time.+ SsrfBlockedError: The destination, or a redirect hop's, is a private, loopback,+ link-local or metadata address.+ """+ raw_bytes = await fetch_file_from_url_httpx(url=url)+ return make_base64_url_from_bytes(raw_bytes=raw_bytes)+++async def make_base64_url_from_any_uri(+ uri: str,+ *,+ storage_provider: StorageProviderAbstract | None = None,+) -> str:+ """Convert a URI to a base64 data URL.++ Resolves the URI and fetches/converts content to base64 format.+ If the URI is already a data URL, returns it as-is.++ Args:+ uri: A URI string (http://, local path, data: URL, or pipelex-storage://)+ storage_provider: Optional storage provider for resolving pipelex-storage:// URIs.+ Required when the URI is a pipelex-storage:// URI.++ Returns:+ A base64 data URL string containing the base64-encoded data, whichever way we got it.++ Raises:+ ValueError: If the URI is a pipelex-storage:// URI and no storage provider is given.+ SsrfBlockedError: If the URI is an http(s) URL whose destination, or a redirect hop's,+ is a private, loopback, link-local or metadata address.+ """+ base64_url: str+ resolved_uri = resolve_uri(uri)+ match resolved_uri:+ case ResolvedBase64DataUrl():+ # Already a data URL, return as-is+ base64_url = resolved_uri.original+ case ResolvedHttpUrl():+ base64_url = await make_base64_url_from_http_url(url=resolved_uri.url)+ case ResolvedLocalPath():+ base64_url = await make_base64_url_from_path(path=Path(resolved_uri.path))+ case ResolvedPipelexStorage():+ if storage_provider is None:+ msg = f"Cannot convert pipelex-storage:// URI to base64 without a storage provider: {uri}"+ raise ValueError(msg)+ raw_bytes = await storage_provider.load(uri=resolved_uri.storage_uri)+ base64_url = make_base64_url_from_bytes(raw_bytes=raw_bytes)+ return base64_urldiff --git a/pipelex/tools/uri/uri_resolver.py b/pipelex/tools/uri/uri_resolver.pyindex 86163e983..fc3107ac2 100644--- a/pipelex/tools/uri/uri_resolver.py+++ b/pipelex/tools/uri/uri_resolver.py@@ -1,12 +1,7 @@import urllib.parsefrom pathlib import Path-from pipelex.tools.misc.base64_utils import (- make_base64_url_from_bytes,- make_base64_url_from_http_url,- make_base64_url_from_path,-)-from pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEME, StorageProviderAbstract+from pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEMEfrom pipelex.tools.uri.resolved_uri import (ResolvedBase64DataUrl,ResolvedHttpUrl,@@ -205,43 +200,3 @@ def _resolve_base64_data_url(uri: str) -> ResolvedBase64DataUrl:mime_type=mime_type,base64_data=base64_data,)---async def make_base64_url_from_any_uri(- uri: str,- *,- storage_provider: StorageProviderAbstract | None = None,-) -> str:- """Convert a URI to a base64 data URL.-- Resolves the URI and fetches/converts content to base64 format.- If the URI is already a data URL, returns it as-is.-- Args:- uri: A URI string (http://, local path, data: URL, or pipelex-storage://)- storage_provider: Optional storage provider for resolving pipelex-storage:// URIs.- Required when the URI is a pipelex-storage:// URI.-- Returns:- A base64 data URL string containing the base64-encoded data, whichever way we got it.-- Raises:- ValueError: If the URI is a pipelex-storage:// URI and no storage provider is given.- """- base64_url: str- resolved_uri = resolve_uri(uri)- match resolved_uri:- case ResolvedBase64DataUrl():- # Already a data URL, return as-is- base64_url = resolved_uri.original- case ResolvedHttpUrl():- base64_url = await make_base64_url_from_http_url(url=resolved_uri.url)- case ResolvedLocalPath():- base64_url = await make_base64_url_from_path(path=Path(resolved_uri.path))- case ResolvedPipelexStorage():- if storage_provider is None:- msg = f"Cannot convert pipelex-storage:// URI to base64 without a storage provider: {uri}"- raise ValueError(msg)- raw_bytes = await storage_provider.load(uri=resolved_uri.storage_uri)- base64_url = make_base64_url_from_bytes(raw_bytes=raw_bytes)- return base64_urldiff --git a/subject_grants.toml b/subject_grants.tomlindex 740e9d544..a87cc8209 100644--- a/subject_grants.toml+++ b/subject_grants.toml@@ -6224,10 +6224,6 @@ rationale = "Factory from source: builds the result from the base64_data; singleparam = "raw_bytes"rationale = "Factory from source: builds the result from the raw_bytes; single operand."-["pipelex/tools/misc/base64_utils.py::make_base64_url_from_http_url"]-param = "url"-rationale = "Factory from source: builds the result from the url; single operand."-["pipelex/tools/misc/base64_utils.py::make_base64_url_from_path"]param = "path"rationale = "Factory from source: builds the result from the path; single operand."@@ -7024,6 +7020,14 @@ rationale = "Predicate on the obj; single operand."param = "obj"rationale = "Predicate on the obj; single operand."+["pipelex/tools/uri/uri_base64.py::make_base64_url_from_any_uri"]+param = "uri"+rationale = "Factory from source: builds the result from the uri; single operand."++["pipelex/tools/uri/uri_base64.py::make_base64_url_from_http_url"]+param = "url"+rationale = "Factory from source: builds the result from the url; single operand."+["pipelex/tools/uri/uri_resolver.py::_resolve_base64_data_url"]param = "uri"rationale = "Parses/reads the uri; single operand."@@ -7044,10 +7048,6 @@ rationale = "Extracts/derives from the uri; single operand."param = "uri"rationale = "Predicate on the uri; single operand."-["pipelex/tools/uri/uri_resolver.py::make_base64_url_from_any_uri"]-param = "uri"-rationale = "Factory from source: builds the result from the uri; single operand."-["pipelex/tools/uri/uri_resolver.py::resolve_local_path_reference"]param = "uri"rationale = "Verb-object: resolves the uri; base_dir is context."diff --git a/tests/integration/pipelex/tools/uri/test_uri_resolver_integration.py b/tests/integration/pipelex/tools/uri/test_uri_resolver_integration.pyindex af3887078..d633495d8 100644--- a/tests/integration/pipelex/tools/uri/test_uri_resolver_integration.py+++ b/tests/integration/pipelex/tools/uri/test_uri_resolver_integration.py@@ -2,9 +2,9 @@ import base64import pytest-from pipelex.tools.misc.base64_utils import make_base64_url_from_http_urlfrom pipelex.tools.uri.resolved_uri import ResolvedHttpUrl, ResolvedLocalPath, UriKind-from pipelex.tools.uri.uri_resolver import make_base64_url_from_any_uri, resolve_uri+from pipelex.tools.uri.uri_base64 import make_base64_url_from_any_uri, make_base64_url_from_http_url+from pipelex.tools.uri.uri_resolver import resolve_urifrom tests.cases import ImageTestCases, TestURLsdiff --git a/tests/unit/pipelex/providers/docling/test_docling_worker_error_handling.py b/tests/unit/pipelex/providers/docling/test_docling_worker_error_handling.pyindex 400b8b8f0..d5245afcf 100644--- a/tests/unit/pipelex/providers/docling/test_docling_worker_error_handling.py+++ b/tests/unit/pipelex/providers/docling/test_docling_worker_error_handling.py@@ -13,7 +13,6 @@ if TYPE_CHECKING:from pipelex.cogt.exceptions import ExtractJobFailureError, InferenceErrorCategoryfrom pipelex.providers.docling.docling_extract_worker import DoclingExtractWorker-from pipelex.tools.uri.prepared_file import PreparedFileLocalPathfrom tests.unit.pipelex.providers.docling.test_data import DoclingErrorHandlingTestData@@ -59,13 +58,6 @@ class TestDoclingWorkerErrorHandling:"pipelex.providers.docling.docling_extract_worker.asyncio.to_thread",side_effect=sdk_exc,)- # Mock prepare_file_from_uri to return a local path that the match statement can handle- mock_prepared_file = PreparedFileLocalPath(path="/tmp/test.pdf") # ruff: ignore[hardcoded-temp-file]- mocker.patch(- "pipelex.providers.docling.docling_extract_worker.prepare_file_from_uri",- new_callable=mocker.AsyncMock,- return_value=mock_prepared_file,- )with pytest.raises(ExtractJobFailureError) as exc_info:await worker._extract_from_source(source_uri="/tmp/test.pdf") # ruff: ignore[private-member-access, hardcoded-temp-file] # pyright: ignore[reportPrivateUsage]@@ -84,12 +76,6 @@ class TestDoclingWorkerErrorHandling:"pipelex.providers.docling.docling_extract_worker.asyncio.to_thread",side_effect=sdk_exc,)- mock_prepared_file = PreparedFileLocalPath(path="/tmp/test.pdf") # ruff: ignore[hardcoded-temp-file]- mocker.patch(- "pipelex.providers.docling.docling_extract_worker.prepare_file_from_uri",- new_callable=mocker.AsyncMock,- return_value=mock_prepared_file,- )with pytest.raises(ExtractJobFailureError) as exc_info:await worker._extract_from_source(source_uri="/tmp/test.pdf") # ruff: ignore[private-member-access, hardcoded-temp-file] # pyright: ignore[reportPrivateUsage]@@ -108,12 +94,6 @@ class TestDoclingWorkerErrorHandling:"pipelex.providers.docling.docling_extract_worker.asyncio.to_thread",side_effect=sdk_exc,)- mock_prepared_file = PreparedFileLocalPath(path="/tmp/test.pdf") # ruff: ignore[hardcoded-temp-file]- mocker.patch(- "pipelex.providers.docling.docling_extract_worker.prepare_file_from_uri",- new_callable=mocker.AsyncMock,- return_value=mock_prepared_file,- )with pytest.raises(ExtractJobFailureError) as exc_info:await worker._extract_from_source(source_uri="/tmp/test.pdf") # ruff: ignore[private-member-access, hardcoded-temp-file] # pyright: ignore[reportPrivateUsage]diff --git a/tests/unit/pipelex/providers/docling/test_docling_worker_semantic.py b/tests/unit/pipelex/providers/docling/test_docling_worker_semantic.pyindex 8a7728596..502479e50 100644--- a/tests/unit/pipelex/providers/docling/test_docling_worker_semantic.py+++ b/tests/unit/pipelex/providers/docling/test_docling_worker_semantic.py@@ -58,13 +58,6 @@ class TestDoclingWorkerSemantic:worker = _make_worker(mocker)sdk_exc = exception_class(exception_message)- from pipelex.tools.uri.prepared_file import PreparedFileLocalPath # ruff: ignore[import-outside-top-level]-- mocker.patch(- "pipelex.providers.docling.docling_extract_worker.prepare_file_from_uri",- new_callable=mocker.AsyncMock,- return_value=PreparedFileLocalPath(path="/tmp/test.pdf"), # ruff: ignore[hardcoded-temp-file]- )mocker.patch("pipelex.providers.docling.docling_extract_worker.asyncio.to_thread",new_callable=mocker.AsyncMock,diff --git a/tests/unit/pipelex/providers/docling/test_docling_worker_url_fetch.py b/tests/unit/pipelex/providers/docling/test_docling_worker_url_fetch.pynew file mode 100644index 000000000..7d3ca3fa3--- /dev/null+++ b/tests/unit/pipelex/providers/docling/test_docling_worker_url_fetch.py@@ -0,0 +1,161 @@+"""Docling reads every source that is not a local file from a temp file the runtime wrote.++An http(s) URL is downloaded by the runtime's guarded fetch helper, never by Docling, and the+temp file is named with an extension Docling can read the format from.+"""++from __future__ import annotations++import base64+from pathlib import Path+from typing import TYPE_CHECKING++import pytest++from pipelex.cogt.exceptions import ExtractJobFailureError+from pipelex.providers.docling.docling_extract_worker import DoclingExtractWorker+from pipelex.providers.docling.docling_sdk import DoclingSdk+from pipelex.tools.network.exceptions import SsrfBlockedError+from pipelex.tools.storage.storage_provider_abstract import StoredData++if TYPE_CHECKING:+ from pytest_mock import MockerFixture++WORKER_MODULE = "pipelex.providers.docling.docling_extract_worker"+FETCH_TARGET = f"{WORKER_MODULE}.fetch_file_and_content_type_from_url_httpx"+PDF_BYTES = b"%PDF-1.4\n%fake\n"+HTML_BYTES = b"<html><body><h1>Quarterly report</h1><p>Revenue grew.</p></body></html>"+CSV_BYTES = b"name,qty\nbolt,3\n"+MARKDOWN_BYTES = b"# Title\n\nSome *markdown* text.\n"+++def _make_worker(mocker: MockerFixture, *, docling_sdk: DoclingSdk | None = None) -> DoclingExtractWorker:+ worker = object.__new__(DoclingExtractWorker)+ mock_model = mocker.MagicMock()+ mock_model.desc = "test-docling"+ mock_model.name = "docling"+ worker.inference_model = mock_model+ worker.docling_sdk = docling_sdk or mocker.MagicMock()+ return worker+++def _data_url(*, mime_type: str, body: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(body).decode('ascii')}"++[email protected](scope="module")+def real_docling_sdk() -> DoclingSdk:+ return DoclingSdk()++[email protected](loop_scope="class")+class TestDoclingWorkerUrlFetch:+ async def test_an_http_url_reaches_docling_as_a_downloaded_file(self, mocker: MockerFixture) -> None:+ worker = _make_worker(mocker)+ fetch_mock = mocker.patch(FETCH_TARGET, new_callable=mocker.AsyncMock, return_value=(PDF_BYTES, "application/pdf"))+ docling_sources: list[str] = []++ def record_source(_convert: object, docling_source: str) -> object:+ docling_sources.append(docling_source)+ assert Path(docling_source).read_bytes() == PDF_BYTES+ stop_msg = "stop after recording the source"+ raise ValueError(stop_msg)++ mocker.patch(f"{WORKER_MODULE}.asyncio.to_thread", new_callable=mocker.AsyncMock, side_effect=record_source)++ with pytest.raises(ExtractJobFailureError, match="stop after recording the source"):+ await worker._extract_from_source(source_uri="https://example.com/doc.pdf") # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]++ fetch_mock.assert_awaited_once_with("https://example.com/doc.pdf")+ assert len(docling_sources) == 1+ assert docling_sources[0] != "https://example.com/doc.pdf"+ assert docling_sources[0].endswith(".pdf")+ assert not Path(docling_sources[0]).exists()++ @pytest.mark.parametrize(+ ("url", "expected_download_url"),+ [+ ("https://docs.google.com/document/d/abc_-1/edit", "https://docs.google.com/document/d/abc_-1/export?format=docx"),+ ("https://docs.google.com/spreadsheets/d/abc/edit#gid=0", "https://docs.google.com/spreadsheets/d/abc/export?format=xlsx"),+ ("https://docs.google.com/presentation/d/abc/edit", "https://docs.google.com/presentation/d/abc/export?format=pptx"),+ ("https://drive.google.com/file/d/abc/view?usp=sharing", "https://drive.google.com/uc?export=download&id=abc"),+ ("https://example.com/doc.pdf", "https://example.com/doc.pdf"),+ ],+ )+ async def test_a_google_link_is_downloaded_from_its_export_url(self, mocker: MockerFixture, url: str, expected_download_url: str) -> None:+ worker = _make_worker(mocker)+ fetch_mock = mocker.patch(FETCH_TARGET, new_callable=mocker.AsyncMock, return_value=(PDF_BYTES, "application/pdf"))+ mocker.patch(f"{WORKER_MODULE}.asyncio.to_thread", new_callable=mocker.AsyncMock, side_effect=ValueError("stop"))++ with pytest.raises(ExtractJobFailureError):+ await worker._extract_from_source(source_uri=url) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]++ fetch_mock.assert_awaited_once_with(expected_download_url)++ @pytest.mark.parametrize(+ ("named_after", "media_type", "expected_suffix"),+ [+ # The URL's or storage key's own extension wins when Docling knows it+ ("https://example.com/data.csv", "text/plain", ".csv"),+ ("https://example.com/notes/readme.md?raw=1", "text/plain; charset=utf-8", ".md"),+ ("pipelex-storage://scope/assets/abc123.html", None, ".html"),+ # Otherwise the declared media type names the format+ ("https://example.com/page", "text/html", ".html"),+ ("https://example.com/docs/v1.2", "text/markdown", ".md"),+ ("https://example.com/report?format=x.pdf", "text/csv", ".csv"),+ (None, "text/markdown", ".md"),+ # With neither, the file gets no extension and Docling sniffs the content+ ("https://example.com/download", "application/octet-stream", ""),+ ("https://example.com/download", None, ""),+ (None, None, ""),+ ],+ )+ async def test_the_temp_file_is_named_so_docling_can_tell_its_format(+ self,+ named_after: str | None,+ media_type: str | None,+ expected_suffix: str,+ ) -> None:+ suffix = DoclingExtractWorker._suffix_for_docling(named_after=named_after, media_type=media_type) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]+ assert suffix == expected_suffix++ @pytest.mark.parametrize(+ ("source_uri", "media_type", "body", "expected_text"),+ [+ ("https://example.com/page", "text/html", HTML_BYTES, "Revenue grew."),+ ("https://example.com/stock.csv", "text/plain", CSV_BYTES, "bolt"),+ ("https://example.com/readme.md", "text/plain", MARKDOWN_BYTES, "markdown"),+ ("pipelex-storage://scope/assets/abc123.md", None, MARKDOWN_BYTES, "markdown"),+ ("pipelex-storage://scope/assets/abc123", "text/markdown", MARKDOWN_BYTES, "markdown"),+ (_data_url(mime_type="text/csv", body=CSV_BYTES), None, CSV_BYTES, "bolt"),+ (_data_url(mime_type="text/markdown", body=MARKDOWN_BYTES), None, MARKDOWN_BYTES, "markdown"),+ ],+ )+ async def test_a_text_format_source_is_extracted_by_docling(+ self,+ mocker: MockerFixture,+ real_docling_sdk: DoclingSdk,+ source_uri: str,+ media_type: str | None,+ body: bytes,+ expected_text: str,+ ) -> None:+ worker = _make_worker(mocker, docling_sdk=real_docling_sdk)+ mocker.patch(FETCH_TARGET, new_callable=mocker.AsyncMock, return_value=(body, media_type))+ storage_provider = mocker.MagicMock()+ storage_provider.load_with_metadata = mocker.AsyncMock(return_value=StoredData(data=body, mime_type=media_type))+ mocker.patch(f"{WORKER_MODULE}.get_storage_provider", return_value=storage_provider)++ extract_output = await worker._extract_from_source(source_uri=source_uri) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]++ assert list(extract_output.pages) == [0]+ assert expected_text in (extract_output.pages[0].text or "")++ async def test_a_private_url_is_refused_before_docling_runs(self, mocker: MockerFixture) -> None:+ worker = _make_worker(mocker)+ to_thread_mock = mocker.patch(f"{WORKER_MODULE}.asyncio.to_thread", new_callable=mocker.AsyncMock)++ with pytest.raises(SsrfBlockedError):+ await worker._extract_from_source(source_uri="http://127.0.0.1:9/doc.pdf") # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]++ to_thread_mock.assert_not_awaited()diff --git a/tests/unit/pipelex/test_config_pre_boot.py b/tests/unit/pipelex/test_config_pre_boot.pyindex 62aad4104..6d433e598 100644--- a/tests/unit/pipelex/test_config_pre_boot.py+++ b/tests/unit/pipelex/test_config_pre_boot.py@@ -1,6 +1,6 @@from pytest_mock import MockerFixture-from pipelex.config import get_pipe_func_execution_mode, is_pipe_func_sandbox_hosted+from pipelex.config import get_pipe_func_execution_mode, is_fetch_ssrf_guard_enabled, is_pipe_func_sandbox_hostedfrom pipelex.plugins.pipe_func_executor_registry import DIRECT_PIPE_FUNC_EXECUTION_MODEfrom pipelex.runtime_hub import RuntimeHub, get_optional_config@@ -18,3 +18,8 @@ class TestConfigPreBoot:mocker.patch.object(RuntimeHub, "_instance", None)assert get_pipe_func_execution_mode() == DIRECT_PIPE_FUNC_EXECUTION_MODEassert is_pipe_func_sandbox_hosted() is False++ def test_fetch_ssrf_guard_is_enabled_without_hub(self, mocker: MockerFixture):+ """A process that fetches before, or without, loading the config fails closed: the guard is on."""+ mocker.patch.object(RuntimeHub, "_instance", None)+ assert is_fetch_ssrf_guard_enabled() is Truediff --git a/tests/unit/pipelex/tools/misc/base64/test_make_base64_url_from_http_url_async.py b/tests/unit/pipelex/tools/misc/base64/test_make_base64_url_from_http_url_async.pyindex e5ecce94e..8f1c12ece 100644--- a/tests/unit/pipelex/tools/misc/base64/test_make_base64_url_from_http_url_async.py+++ b/tests/unit/pipelex/tools/misc/base64/test_make_base64_url_from_http_url_async.py@@ -4,7 +4,7 @@ import httpximport pytestfrom pytest_mock import MockerFixture-from pipelex.tools.misc.base64_utils import make_base64_url_from_http_url+from pipelex.tools.uri.uri_base64 import make_base64_url_from_http_urlfrom pipelex.urls import URLsfrom tests.cases import ImageTestCases@@ -21,7 +21,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:# Mock the HTTP fetch to return PNG bytesmocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",return_value=png_bytes,)@@ -41,7 +41,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:# Mock the HTTP fetch to return JPEG bytesmocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",return_value=jpeg_bytes,)@@ -56,7 +56,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:async def test_propagates_http_error_on_404(self, mocker: MockerFixture) -> None:"""Test that HTTP 404 errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message="404 Not Found",request=httpx.Request("GET", URLs.png_example_1),@@ -72,7 +72,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:async def test_propagates_http_error_on_500(self, mocker: MockerFixture) -> None:"""Test that HTTP 500 errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message="500 Internal Server Error",request=httpx.Request("GET", URLs.png_example_1),@@ -88,7 +88,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:async def test_propagates_connection_error(self, mocker: MockerFixture) -> None:"""Test that connection errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.ConnectError("Connection refused"),)@@ -98,7 +98,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:async def test_propagates_timeout_error(self, mocker: MockerFixture) -> None:"""Test that timeout errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.TimeoutException("Request timed out"),)@@ -125,7 +125,7 @@ class TestMakeBase64UrlFromHttpUrlAsync:# Mock with a misleading URL extensionmocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",return_value=file_bytes,)diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/conftest.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/conftest.pynew file mode 100644index 000000000..43332f275--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/conftest.py@@ -0,0 +1,18 @@+import threading+from collections.abc import Iterator++import pytest++from tests.unit.pipelex.tools.misc.fetch_ssrf_guard.loopback import LoopbackHandler, LoopbackServer+++def loopback_server() -> Iterator[LoopbackServer]:+ server = LoopbackServer(("127.0.0.1", 0), LoopbackHandler)+ server.hits = []+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ yield server+ server.shutdown()+ server.server_close()+ thread.join()diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/loopback.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/loopback.pynew file mode 100644index 000000000..e0b8d0f6f--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/loopback.py@@ -0,0 +1,67 @@+"""A real HTTP server on ``127.0.0.1``, which the fetch-guard tests aim the fetch helper at.++The tests reach it with no mock between the helper and the socket, because the property under+test is what the default path does when nobody passes anything. A ``MockTransport`` would+replace the guard and prove nothing about it.+"""++from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from typing import cast++from pytest_mock import MockerFixture+from typing_extensions import override++SECRET_BODY = b"internal-only-secret"+SECRET_PATH = "/secret"+REDIRECT_TO_LOCALHOST_PATH = "/redirect-to-localhost"++PROXY_ENV_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy")+++class LoopbackServer(ThreadingHTTPServer):+ """A server on 127.0.0.1 that records the path of every request it answers."""++ hits: list[str]+++class LoopbackHandler(BaseHTTPRequestHandler):+ def do_GET(self) -> None:+ server = cast("LoopbackServer", self.server)+ server.hits.append(self.path)+ if self.path == SECRET_PATH:+ self.send_response(200)+ self.send_header("Content-Type", "text/plain")+ self.send_header("Content-Length", str(len(SECRET_BODY)))+ self.end_headers()+ self.wfile.write(SECRET_BODY)+ elif self.path == REDIRECT_TO_LOCALHOST_PATH:+ self.send_response(302)+ self.send_header("Location", f"http://localhost:{server.server_port}{SECRET_PATH}")+ self.send_header("Content-Length", "0")+ self.end_headers()+ else:+ self.send_response(404)+ self.send_header("Content-Length", "0")+ self.end_headers()++ @override+ def log_message(self, format: str, *args: object) -> None:+ pass+++def loopback_url(server: LoopbackServer, *, path: str) -> str:+ return f"http://127.0.0.1:{server.server_port}{path}"+++def treat_only_the_localhost_name_as_private(mocker: MockerFixture) -> None:+ """Let the first hop to 127.0.0.1 through, so that a refusal can only come from a later hop.++ The rules are patched where the guard reads them: no literal IP counts as private, and the+ only disallowed host is the name ``localhost``, which the server's redirect points at.+ """++ def is_the_localhost_name(host: str) -> bool:+ return host.rstrip(".").lower() == "localhost"++ mocker.patch("pipelex.tools.network.ssrf_guard.is_disallowed_host", side_effect=is_the_localhost_name)+ mocker.patch("pipelex.tools.network.ssrf_guard.is_disallowed_ip", return_value=False)diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_callers_surface_the_refusal.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_callers_surface_the_refusal.pynew file mode 100644index 000000000..091e9aaa7--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_callers_surface_the_refusal.py@@ -0,0 +1,65 @@+"""Every production entry point that fetches a value's URL raises the refusal rather than absorbing it."""++from collections.abc import Awaitable, Callable++import pytest+from pytest_mock import MockerFixture++from pipelex.cogt.content_generation.generated_content_factory import GeneratedContentFactory+from pipelex.cogt.document.prompt_document import PromptDocumentUri+from pipelex.cogt.document.prompt_document_utils import prepare_prompt_document, prepare_prompt_document_as_base64+from pipelex.cogt.file.file_preparation_utils import prepare_file_from_uri+from pipelex.cogt.image.generated_image import GeneratedImageRawDetails+from pipelex.cogt.image.prompt_image import PromptImageUri+from pipelex.cogt.image.prompt_image_utils import prepare_prompt_image, prepare_prompt_image_as_base64+from pipelex.config import get_config+from pipelex.tools.network.exceptions import SsrfBlockedError+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract+from pipelex.tools.uri.uri_base64 import make_base64_url_from_any_uri, make_base64_url_from_http_url+from tests.unit.pipelex.tools.misc.fetch_ssrf_guard.loopback import SECRET_PATH, LoopbackServer, loopback_url++#: Every production entry point that fetches a value's URL, called the way its callers call it.+FETCHING_CALLERS: list[tuple[str, Callable[[str], Awaitable[object]]]] = [+ ("prepare_prompt_image", lambda url: prepare_prompt_image(PromptImageUri(uri=url), is_http_url_enabled=False)),+ ("prepare_prompt_image_as_base64", lambda url: prepare_prompt_image_as_base64(PromptImageUri(uri=url))),+ ("prepare_prompt_document", lambda url: prepare_prompt_document(PromptDocumentUri(uri=url), is_http_url_enabled=False)),+ ("prepare_prompt_document_as_base64", lambda url: prepare_prompt_document_as_base64(PromptDocumentUri(uri=url))),+ ("prepare_file_from_uri", lambda url: prepare_file_from_uri(url, keep_http_url=False, keep_local_path=False)),+ ("make_base64_url_from_any_uri", make_base64_url_from_any_uri),+ ("make_base64_url_from_http_url", make_base64_url_from_http_url),+]++[email protected](loop_scope="class")+class TestCallersSurfaceTheRefusal:+ """No caller absorbs the refusal into a fallback: it reaches the pipe as `SsrfBlockedError`."""++ @pytest.mark.parametrize(+ "call",+ [pytest.param(call, id=caller_name) for caller_name, call in FETCHING_CALLERS],+ )+ async def test_caller_raises_the_refusal(self, loopback_server: LoopbackServer, call: Callable[[str], Awaitable[object]]) -> None:+ with pytest.raises(SsrfBlockedError) as exc_info:+ await call(loopback_url(loopback_server, path=SECRET_PATH))++ assert exc_info.type is SsrfBlockedError+ assert loopback_server.hits == []++ async def test_a_generated_image_url_on_a_private_address_fails_instead_of_degrading(+ self,+ mocker: MockerFixture,+ loopback_server: LoopbackServer,+ ) -> None:+ mocker.patch.object(get_config().runtime.storage, "is_fetch_remote_content_enabled", True)+ storage_provider = mocker.MagicMock(spec=StorageProviderAbstract)+ storage_provider.store = mocker.AsyncMock(return_value="pipelex-storage://stored-uri")+ factory = GeneratedContentFactory(storage_provider=storage_provider)++ with pytest.raises(SsrfBlockedError):+ await factory.make_image_content(+ storage_scope="test/scope",+ raw_details=GeneratedImageRawDetails(size=None, actual_url=loopback_url(loopback_server, path=SECRET_PATH)),+ )++ assert loopback_server.hits == []+ storage_provider.store.assert_not_awaited()diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_every_redirect_hop_is_vetted.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_every_redirect_hop_is_vetted.pynew file mode 100644index 000000000..002333628--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_every_redirect_hop_is_vetted.py@@ -0,0 +1,34 @@+"""The guard vets every redirect hop, not only the first request."""++import pytest+from pytest_mock import MockerFixture++from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx+from pipelex.tools.network.exceptions import SsrfBlockedError+from tests.unit.pipelex.tools.misc.fetch_ssrf_guard.loopback import (+ REDIRECT_TO_LOCALHOST_PATH,+ SECRET_BODY,+ SECRET_PATH,+ LoopbackServer,+ loopback_url,+ treat_only_the_localhost_name_as_private,+)++[email protected](loop_scope="class")+class TestEveryRedirectHopIsVetted:+ async def test_a_first_hop_the_rules_allow_is_fetched(self, mocker: MockerFixture, loopback_server: LoopbackServer) -> None:+ """The control for the next test: under the patched rules, the loopback server itself is reachable."""+ treat_only_the_localhost_name_as_private(mocker)++ raw_bytes = await fetch_file_from_url_httpx(url=loopback_url(loopback_server, path=SECRET_PATH))++ assert raw_bytes == SECRET_BODY++ async def test_a_redirect_to_a_private_destination_is_refused(self, mocker: MockerFixture, loopback_server: LoopbackServer) -> None:+ treat_only_the_localhost_name_as_private(mocker)++ with pytest.raises(SsrfBlockedError, match="localhost"):+ await fetch_file_from_url_httpx(url=loopback_url(loopback_server, path=REDIRECT_TO_LOCALHOST_PATH))++ assert loopback_server.hits == [REDIRECT_TO_LOCALHOST_PATH]diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_fetch_is_guarded_by_default.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_fetch_is_guarded_by_default.pynew file mode 100644index 000000000..839692049--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_fetch_is_guarded_by_default.py@@ -0,0 +1,33 @@+"""The fetch helper refuses a private destination when nobody passes anything."""++import pytest+from pytest_mock import MockerFixture++from pipelex.runtime_hub import RuntimeHub+from pipelex.tools.misc.file_fetch_utils import fetch_file_and_content_type_from_url_httpx, fetch_file_from_url_httpx+from pipelex.tools.network.exceptions import SsrfBlockedError+from tests.unit.pipelex.tools.misc.fetch_ssrf_guard.loopback import SECRET_PATH, LoopbackServer, loopback_url++[email protected](loop_scope="class")+class TestFetchIsGuardedByDefault:+ async def test_a_loopback_url_is_refused_before_any_request(self, loopback_server: LoopbackServer) -> None:+ with pytest.raises(SsrfBlockedError) as exc_info:+ await fetch_file_from_url_httpx(url=loopback_url(loopback_server, path=SECRET_PATH))++ assert exc_info.type is SsrfBlockedError+ assert loopback_server.hits == []++ async def test_the_content_type_variant_is_guarded_too(self, loopback_server: LoopbackServer) -> None:+ with pytest.raises(SsrfBlockedError):+ await fetch_file_and_content_type_from_url_httpx(loopback_url(loopback_server, path=SECRET_PATH))++ assert loopback_server.hits == []++ async def test_a_process_with_no_config_loaded_is_guarded(self, mocker: MockerFixture, loopback_server: LoopbackServer) -> None:+ mocker.patch.object(RuntimeHub, "_instance", None)++ with pytest.raises(SsrfBlockedError):+ await fetch_file_from_url_httpx(url=loopback_url(loopback_server, path=SECRET_PATH))++ assert loopback_server.hits == []diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_the_kernel_closure_holds_no_fetch.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_the_kernel_closure_holds_no_fetch.pynew file mode 100644index 000000000..de64511da--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_the_kernel_closure_holds_no_fetch.py@@ -0,0 +1,30 @@+"""The runtime hub's import closure never loads the fetch helper, which reads the config back."""++import subprocess # ruff: ignore[suspicious-subprocess-import]+import sys++#: Wall-clock bound on the import-closure subprocess, so a deadlock fails instead of hanging the suite.+SUBPROCESS_TIMEOUT_SECONDS = 300+++class TestTheKernelClosureHoldsNoFetch:+ def test_importing_the_runtime_hub_does_not_load_the_fetch_helper(self) -> None:+ """The helper reads the config, which imports the runtime hub: the hub must not import the helper back.++ Run in a subprocess so the closure is exactly what the import pulls in.+ """+ script = (+ "import sys\n"+ "import pipelex.runtime_hub\n"+ "loaded = [name for name in ('pipelex.tools.misc.file_fetch_utils', 'pipelex.tools.uri.uri_base64') if name in sys.modules]\n"+ "print(','.join(loaded))\n"+ )+ completed = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true]+ [sys.executable, "-c", script],+ capture_output=True,+ text=True,+ check=True,+ timeout=SUBPROCESS_TIMEOUT_SECONDS,+ )++ assert completed.stdout.strip() == ""diff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_the_switch_turns_the_guard_off.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_the_switch_turns_the_guard_off.pynew file mode 100644index 000000000..b15587857--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_the_switch_turns_the_guard_off.py@@ -0,0 +1,46 @@+"""`is_fetch_ssrf_guard_enabled = false` restores a plain client that follows redirects unvetted."""++import pytest+from pytest_mock import MockerFixture++from pipelex.config import get_config+from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx+from tests.unit.pipelex.tools.misc.fetch_ssrf_guard.loopback import (+ PROXY_ENV_VARS,+ REDIRECT_TO_LOCALHOST_PATH,+ SECRET_BODY,+ SECRET_PATH,+ LoopbackServer,+ loopback_url,+)++[email protected](loop_scope="class")+class TestTheSwitchTurnsTheGuardOff:+ async def test_switch_off_fetches_aloopback_url(+ self,+ mocker: MockerFixture,+ monkeypatch: pytest.MonkeyPatch,+ loopback_server: LoopbackServer,+ ) -> None:+ mocker.patch.object(get_config().runtime.network, "is_fetch_ssrf_guard_enabled", False)+ for env_var in PROXY_ENV_VARS:+ monkeypatch.delenv(env_var, raising=False)++ raw_bytes = await fetch_file_from_url_httpx(url=loopback_url(loopback_server, path=SECRET_PATH))++ assert raw_bytes == SECRET_BODY++ async def test_switch_off_follows_redirects_unvetted(+ self,+ mocker: MockerFixture,+ monkeypatch: pytest.MonkeyPatch,+ loopback_server: LoopbackServer,+ ) -> None:+ mocker.patch.object(get_config().runtime.network, "is_fetch_ssrf_guard_enabled", False)+ for env_var in PROXY_ENV_VARS:+ monkeypatch.delenv(env_var, raising=False)++ raw_bytes = await fetch_file_from_url_httpx(url=loopback_url(loopback_server, path=REDIRECT_TO_LOCALHOST_PATH))++ assert raw_bytes == SECRET_BODYdiff --git a/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_transport_is_a_test_seam_only.py b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_transport_is_a_test_seam_only.pynew file mode 100644index 000000000..14164afd8--- /dev/null+++ b/tests/unit/pipelex/tools/misc/fetch_ssrf_guard/test_transport_is_a_test_seam_only.py@@ -0,0 +1,46 @@+"""No production module passes a transport to the fetch helpers, since a transport replaces the guard."""++import ast+from pathlib import Path++FETCH_FUNCTION_NAMES = frozenset({"fetch_file_from_url_httpx", "fetch_file_and_content_type_from_url_httpx"})+PIPELEX_SOURCE_ROOT = Path(__file__).resolve().parents[6] / "pipelex"+FETCH_HELPER_SOURCE = PIPELEX_SOURCE_ROOT / "tools" / "misc" / "file_fetch_utils.py"+++def _fetch_calls_passing_transport(source_path: Path) -> tuple[int, list[str]]:+ """Count the calls to the fetch helpers in ``source_path``, and list those passing ``transport``."""+ tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path))+ nb_fetch_calls = 0+ offenders: list[str] = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ func_name = func.id if isinstance(func, ast.Name) else func.attr if isinstance(func, ast.Attribute) else None+ if func_name not in FETCH_FUNCTION_NAMES:+ continue+ nb_fetch_calls += 1+ if any(keyword.arg in {"transport", None} for keyword in node.keywords):+ offenders.append(f"{source_path.relative_to(PIPELEX_SOURCE_ROOT.parent)}:{node.lineno}")+ return nb_fetch_calls, offenders+++class TestTransportIsATestSeamOnly:+ def test_no_production_module_passes_a_transport_to_the_fetch_helpers(self) -> None:+ """Passing a transport replaces the guard, so only tests may do it.++ The helper's own forwarding from one fetch function to the other is the one exception.+ Calls with ``**kwargs`` count as passing one, since they could.+ """+ nb_fetch_calls = 0+ offenders: list[str] = []+ for source_path in sorted(PIPELEX_SOURCE_ROOT.rglob("*.py")):+ if source_path == FETCH_HELPER_SOURCE:+ continue+ nb_calls_in_file, offenders_in_file = _fetch_calls_passing_transport(source_path)+ nb_fetch_calls += nb_calls_in_file+ offenders.extend(offenders_in_file)++ assert nb_fetch_calls > 0, "found no call to the fetch helpers at all: the scan is looking in the wrong place"+ assert offenders == []diff --git a/tests/unit/pipelex/tools/misc/test_mime_type_to_extension.py b/tests/unit/pipelex/tools/misc/test_mime_type_to_extension.pynew file mode 100644index 000000000..52e1c5602--- /dev/null+++ b/tests/unit/pipelex/tools/misc/test_mime_type_to_extension.py@@ -0,0 +1,23 @@+import pytest++from pipelex.tools.misc.filetype_utils import UNKNOWN_FILE_TYPE, mime_type_to_extension+++class TestMimeTypeToExtension:+ @pytest.mark.parametrize(+ ("mime_type", "expected_extension"),+ [+ # Every supported Python answers these the same way, whatever its own MIME table holds+ ("text/markdown", "md"),+ ("text/markdown; charset=utf-8", "md"),+ ("TEXT/Markdown", "md"),+ ("text/plain", "txt"),+ ("text/csv", "csv"),+ ("text/html", "html"),+ ("application/pdf", "pdf"),+ ("image/jpeg", "jpeg"),+ ("", UNKNOWN_FILE_TYPE),+ ],+ )+ def test_extension_for_mime_type(self, mime_type: str, expected_extension: str) -> None:+ assert mime_type_to_extension(mime_type) == expected_extensiondiff --git a/tests/unit/pipelex/tools/uri/test_make_base64_url_from_uri_async.py b/tests/unit/pipelex/tools/uri/test_make_base64_url_from_uri_async.pyindex 9ea58f381..3a7337d0a 100644--- a/tests/unit/pipelex/tools/uri/test_make_base64_url_from_uri_async.py+++ b/tests/unit/pipelex/tools/uri/test_make_base64_url_from_uri_async.py@@ -6,7 +6,7 @@ import pytestfrom pytest_mock import MockerFixturefrom pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEME-from pipelex.tools.uri.uri_resolver import make_base64_url_from_any_uri+from pipelex.tools.uri.uri_base64 import make_base64_url_from_any_urifrom tests.cases import ImageTestCases@@ -94,7 +94,7 @@ class TestMakeBase64UrlFromUriAsync:# Mock the HTTP fetch to return PNG bytesmocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",return_value=png_bytes,)@@ -113,7 +113,7 @@ class TestMakeBase64UrlFromUriAsync:# Mock the HTTP fetch to return JPEG bytesmocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",return_value=jpeg_bytes,)@@ -127,7 +127,7 @@ class TestMakeBase64UrlFromUriAsync:async def test_http_url_propagates_network_errors(self, mocker: MockerFixture) -> None:"""Test that HTTP network errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message="404 Not Found",request=httpx.Request("GET", "https://example.com/not-found.png"),diff --git a/tests/unit/pipelex/tools/uri/test_uri_resolver_error_handling.py b/tests/unit/pipelex/tools/uri/test_uri_resolver_error_handling.pyindex aaad61dd5..b5d0cab7d 100644--- a/tests/unit/pipelex/tools/uri/test_uri_resolver_error_handling.py+++ b/tests/unit/pipelex/tools/uri/test_uri_resolver_error_handling.py@@ -3,7 +3,7 @@ import pytestfrom pytest_mock import MockerFixturefrom pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEME-from pipelex.tools.uri.uri_resolver import make_base64_url_from_any_uri+from pipelex.tools.uri.uri_base64 import make_base64_url_from_any_uri@pytest.mark.asyncio(loop_scope="class")@@ -13,7 +13,7 @@ class TestUriResolverErrorHandling:async def test_http_404_error_propagates(self, mocker: MockerFixture) -> None:"""Test that HTTP 404 errors are propagated through make_base64_url_from_any_uri."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message="404 Not Found",request=httpx.Request("GET", "https://example.com/not-found.png"),@@ -29,7 +29,7 @@ class TestUriResolverErrorHandling:async def test_http_500_error_propagates(self, mocker: MockerFixture) -> None:"""Test that HTTP 500 errors are propagated through make_base64_url_from_any_uri."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message="500 Internal Server Error",request=httpx.Request("GET", "https://example.com/error"),@@ -45,7 +45,7 @@ class TestUriResolverErrorHandling:async def test_http_403_forbidden_error_propagates(self, mocker: MockerFixture) -> None:"""Test that HTTP 403 Forbidden errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message="403 Forbidden",request=httpx.Request("GET", "https://example.com/forbidden.png"),@@ -61,7 +61,7 @@ class TestUriResolverErrorHandling:async def test_connection_error_propagates(self, mocker: MockerFixture) -> None:"""Test that connection errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.ConnectError("Connection refused"),)@@ -71,7 +71,7 @@ class TestUriResolverErrorHandling:async def test_timeout_error_propagates(self, mocker: MockerFixture) -> None:"""Test that timeout errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.TimeoutException("Request timed out"),)@@ -81,7 +81,7 @@ class TestUriResolverErrorHandling:async def test_dns_error_propagates(self, mocker: MockerFixture) -> None:"""Test that DNS resolution errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.ConnectError("DNS resolution failed"),)@@ -120,7 +120,7 @@ class TestUriResolverErrorHandling:async def test_ssl_certificate_error_propagates(self, mocker: MockerFixture) -> None:"""Test that SSL certificate errors are propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.ConnectError("SSL certificate verification failed"),)@@ -150,7 +150,7 @@ class TestUriResolverErrorHandling:) -> None:"""Test that various HTTP status codes are properly propagated."""mocker.patch(- "pipelex.tools.misc.base64_utils.fetch_file_from_url_httpx",+ "pipelex.tools.uri.uri_base64.fetch_file_from_url_httpx",side_effect=httpx.HTTPStatusError(message=f"{status_code} {topic}",request=httpx.Request("GET", f"https://example.com/{topic}"),