Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Test and error handling for Pipe Extract web (#854)

Louis Choquel7df226d61ba7
diff --git a/.pipelex-dev/test_profiles.toml b/.pipelex-dev/test_profiles.toml
index cca3a7789..37f9c9de3 100644
--- a/.pipelex-dev/test_profiles.toml
+++ b/.pipelex-dev/test_profiles.toml
@@ -285,9 +285,14 @@ search_models = []
[profiles.dev]
description = "Quick dev testing with fast and cheap models from key providers"
backends = ["pipelex_gateway", "internal"]
-llm_models = ["claude-4.5-haiku", "gpt-4o-mini", "gemini-2.5-flash-lite"]
+llm_models = [
+ "claude-4.5-haiku",
+ "gpt-4o-mini",
+ "linkup",
+ "gemini-2.5-flash-lite",
+]
img_gen_models = ["gpt-image-1-mini"]
-extract_models = ["@from_pdf"]
+extract_models = ["@from_pdf", "linkup-fetch"]
search_models = ["linkup-standard"]
################################################################################
diff --git a/pipelex/plugins/gateway/gateway_extract_worker.py b/pipelex/plugins/gateway/gateway_extract_worker.py
index c40ef0cf5..8c64b120c 100644
--- a/pipelex/plugins/gateway/gateway_extract_worker.py
+++ b/pipelex/plugins/gateway/gateway_extract_worker.py
@@ -151,7 +151,10 @@ class GatewayExtractWorker(ExtractWorkerAbstract):
)
except portkey_exceptions.APIError as exc:
error_summary = GatewayFactory.make_error_summary_from_portkey_error(exc)
- msg = f"Web fetch service error for model '{self.inference_model.tag}' after {attempt_number} attempt(s): {error_summary}"
+ msg = (
+ f"Web fetch service error for URL '{document_uri}' via model '{self.inference_model.tag}' "
+ f"after {attempt_number} attempt(s): {error_summary}"
+ )
raise ExtractJobFailureError(msg) from exc
if response is None:
@@ -231,7 +234,9 @@ class GatewayExtractWorker(ExtractWorkerAbstract):
if isinstance(exc, portkey_exceptions.NotFoundError):
msg = str(exc).lower()
return "specified deployment could not be found" in msg
- return False
+ # Transient upstream gateway failures (e.g. openrouter 500s during batch fetches)
+ # should be retried rather than killing the whole batch on the first hiccup.
+ return isinstance(exc, (portkey_exceptions.InternalServerError, portkey_exceptions.APITimeoutError, portkey_exceptions.APIConnectionError))
def _log_retry(self, retry_state: RetryCallState) -> None:
"""Called before sleeping between retries."""
diff --git a/pipelex/tools/misc/http_utils.py b/pipelex/tools/misc/http_utils.py
index 591c5e418..4377ffbc4 100644
--- a/pipelex/tools/misc/http_utils.py
+++ b/pipelex/tools/misc/http_utils.py
@@ -1,5 +1,6 @@
import httpx
+from pipelex import log
from pipelex.tools.misc.file_utils import path_exists
from pipelex.tools.misc.package_utils import get_package_version
from pipelex.urls import URLs
@@ -22,12 +23,16 @@ def validate_url_resource_exists(url: str) -> None:
By the time a URL reaches DocumentContent/ImageContent, it should already
be resolved (absolute path or fully qualified URL).
- For HTTP/HTTPS URLs: performs a streaming GET request to check reachability.
+ For HTTP/HTTPS URLs: performs a HEAD request (falling back to a streaming GET
+ on 405) as a best-effort reachability probe. Failures are logged as warnings
+ but do NOT raise — the downstream extractor is the source of truth, and many
+ sites bot-block HEAD/unknown User-Agents with 403/401/429 while still serving
+ the actual content fine.
For local file paths: checks that the file exists on disk.
Skips validation for internal URIs (base64 data URLs, pipelex-storage://).
Raises:
- ValueError: If the resource does not exist or is unreachable.
+ ValueError: If a local file path does not exist. HTTP failures only log a warning.
"""
if url.startswith(_SKIP_VALIDATION_PREFIXES):
return
@@ -50,17 +55,22 @@ def _validate_http_url(url: str) -> None:
else:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
- msg = f"URL '{url}' returned HTTP {exc.response.status_code}"
- raise ValueError(msg) from exc
- except httpx.ConnectError as exc:
- msg = f"URL '{url}' could not be reached (connection failed)"
- raise ValueError(msg) from exc
- except httpx.TimeoutException as exc:
- msg = f"URL '{url}' timed out"
- raise ValueError(msg) from exc
- except httpx.HTTPError as exc:
- msg = f"URL '{url}' could not be fetched"
- raise ValueError(msg) from exc
+ status_code = exc.response.status_code
+ msg = f"Pre-flight URL check: URL '{url}' returned HTTP {status_code} (continuing — downstream extractor will decide)"
+ # 401/403/429 are typical bot-block codes when servers reject HEAD/unknown User-Agents while still serving real content — keep these quiet.
+ if status_code in {401, 403, 429}:
+ log.debug(msg)
+ else:
+ log.warning(msg)
+ except httpx.ConnectError:
+ msg = f"Pre-flight URL check: URL '{url}' could not be reached (connection failed) (continuing — downstream extractor will decide)"
+ log.warning(msg)
+ except httpx.TimeoutException:
+ msg = f"Pre-flight URL check: URL '{url}' timed out (continuing — downstream extractor will decide)"
+ log.warning(msg)
+ except httpx.HTTPError:
+ msg = f"Pre-flight URL check: URL '{url}' could not be fetched (continuing — downstream extractor will decide)"
+ log.warning(msg)
def _validate_local_path(url: str) -> None:
diff --git a/tests/cases/documents.py b/tests/cases/documents.py
index aaad5a295..73a1df31d 100644
--- a/tests/cases/documents.py
+++ b/tests/cases/documents.py
@@ -42,8 +42,10 @@ class DocumentTestCases:
# Web URLs
WEB_URL_1 = "https://books.toscrape.com/catalogue/cravings-recipes-for-what-you-want-to-eat_589/index.html"
WEB_URL_2 = "https://www.scrapethissite.com/pages/"
+ WEB_URL_3 = "https://www.allrecipes.com/recipe/91192/french-onion-soup-gratinee/"
WEB_URLS: ClassVar[list[str]] = [
- WEB_URL_1,
- WEB_URL_2,
+ # WEB_URL_1,
+ # WEB_URL_2,
+ WEB_URL_3,
]
diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/pipe_extract.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/pipe_extract.mthds
new file mode 100644
index 000000000..3a0f787a9
--- /dev/null
+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/pipe_extract.mthds
@@ -0,0 +1,9 @@
+domain = "pipe_extract_e2e"
+description = "Test PipeExtract for web page fetching"
+
+[pipe.extract_web_page_e2e]
+type = "PipeExtract"
+description = "Fetch full page content from a web URL"
+inputs = { document = "Document" }
+output = "Page[]"
+model = "@default-extract-web-page"
diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_data.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_data.py
new file mode 100644
index 000000000..c0f90ec51
--- /dev/null
+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_data.py
@@ -0,0 +1,19 @@
+"""Test data for PipeExtract E2E tests."""
+
+from typing import ClassVar
+
+from tests.cases.documents import DocumentTestCases
+
+
+class PipeExtractTestCases:
+ """Test cases for PipeExtract E2E tests."""
+
+ # (variant, url)
+ # WEB_URL_3 (AllRecipes) is a known bot-blocked site that returns 403 to HEAD requests
+ # but serves the actual content fine through linkup-fetch. Included here to guard
+ # against regressions where the pre-flight URL check aborts the pipeline.
+ WEB_URL_CASES: ClassVar[list[tuple[str, str]]] = [
+ ("books_toscrape", DocumentTestCases.WEB_URL_1),
+ ("scrapethissite", DocumentTestCases.WEB_URL_2),
+ ("allrecipes_bot_blocked", DocumentTestCases.WEB_URL_3),
+ ]
diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_pipe_extract.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_pipe_extract.py
new file mode 100644
index 000000000..bec00b235
--- /dev/null
+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_pipe_extract.py
@@ -0,0 +1,59 @@
+"""E2E tests for PipeExtract operator."""
+
+import pytest
+
+from pipelex import pretty_print
+from pipelex.core.stuffs.document_content import DocumentContent
+from pipelex.core.stuffs.page_content import PageContent
+from pipelex.pipe_run.pipe_run_mode import PipeRunMode
+from pipelex.pipeline.runner import PipelexRunner
+from tests.e2e.pipelex.pipes.pipe_operators.pipe_extract.test_data import PipeExtractTestCases
+
+LIBRARY_DIRS = ["tests/e2e/pipelex/pipes/pipe_operators/pipe_extract"]
+
+
[email protected](loop_scope="class")
+class TestPipeExtract:
+ """E2E tests for PipeExtract operator."""
+
+ @pytest.mark.parametrize(("variant", "url"), PipeExtractTestCases.WEB_URL_CASES)
+ async def test_extract_web_page(
+ self,
+ pipe_run_mode: PipeRunMode,
+ variant: str,
+ url: str,
+ ) -> None:
+ """Extract a web page via PipeExtract + linkup-fetch.
+
+ Regression test for pre-flight URL validation: some sites (e.g. AllRecipes)
+ bot-block HEAD requests with 403, and the pre-flight check must NOT abort
+ the pipeline — the downstream extractor is the source of truth.
+ """
+ runner = PipelexRunner(
+ library_dirs=LIBRARY_DIRS,
+ pipe_run_mode=pipe_run_mode,
+ )
+ pipeline_response = await runner.execute_pipeline(
+ pipe_code="extract_web_page_e2e",
+ inputs={
+ "document": DocumentContent(url=url),
+ },
+ )
+
+ pipe_output = pipeline_response.pipe_output
+ assert pipe_output is not None
+ assert pipe_output.working_memory is not None
+ assert pipe_output.main_stuff is not None
+
+ if pipe_run_mode.is_live:
+ pages = pipe_output.main_stuff_as_items(item_type=PageContent)
+ assert len(pages) > 0, f"Expected at least one page for {variant}"
+ first_page = pages[0]
+ page_text_content = first_page.text_and_images.text
+ assert page_text_content is not None, f"Expected markdown text for {variant}"
+ page_text = page_text_content.text
+ assert len(page_text.strip()) > 0, f"Expected non-empty markdown text for {variant}"
+ pretty_print(page_text[:500], title=f"Extracted markdown ({variant})")
diff --git a/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py b/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py
index 7d43eec1c..50b5dfca5 100644
--- a/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py
+++ b/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py
@@ -1,5 +1,7 @@
"""Unit tests for URL validation in DocumentContent."""
+import logging
+
import pytest
from pipelex.core.stuffs.document_content import DocumentContent
@@ -13,11 +15,12 @@ class TestDocumentContentUrlValidation:
doc = DocumentContent(url="https://this-domain-cannot-exist.invalid/file.pdf")
assert doc.url == "https://this-domain-cannot-exist.invalid/file.pdf"
- def test_validate_resources_unreachable_remote_url(self) -> None:
- """validate_resources() raises ValueError for an unreachable HTTP URL."""
+ def test_validate_resources_unreachable_remote_url(self, caplog: pytest.LogCaptureFixture) -> None:
+ """validate_resources() does NOT raise for an unreachable HTTP URL: it logs a warning and lets the downstream extractor decide."""
doc = DocumentContent(url="https://this-domain-cannot-exist.invalid/file.pdf")
- with pytest.raises(ValueError, match="could not be reached"):
+ with caplog.at_level(logging.WARNING, logger="pipelex"):
doc.validate_resources()
+ assert any("could not be reached" in record.message for record in caplog.records)
def test_validate_resources_nonexistent_local_file(self) -> None:
"""validate_resources() raises ValueError for a non-existent local file."""
diff --git a/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py b/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py
index 96d63b8cc..58e8ca90e 100644
--- a/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py
+++ b/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py
@@ -1,5 +1,7 @@
"""Unit tests for URL validation in ImageContent."""
+import logging
+
import pytest
from pipelex.core.stuffs.image_content import ImageContent
@@ -13,11 +15,12 @@ class TestImageContentUrlValidation:
img = ImageContent(url="https://this-domain-cannot-exist.invalid/image.png")
assert img.url == "https://this-domain-cannot-exist.invalid/image.png"
- def test_validate_resources_unreachable_remote_url(self) -> None:
- """validate_resources() raises ValueError for an unreachable HTTP URL."""
+ def test_validate_resources_unreachable_remote_url(self, caplog: pytest.LogCaptureFixture) -> None:
+ """validate_resources() does NOT raise for an unreachable HTTP URL: it logs a warning and lets the downstream extractor decide."""
img = ImageContent(url="https://this-domain-cannot-exist.invalid/image.png")
- with pytest.raises(ValueError, match="could not be reached"):
+ with caplog.at_level(logging.WARNING, logger="pipelex"):
img.validate_resources()
+ assert any("could not be reached" in record.message for record in caplog.records)
def test_validate_resources_nonexistent_local_file(self) -> None:
"""validate_resources() raises ValueError for a non-existent local file."""
diff --git a/tests/unit/pipelex/tools/misc/test_http_utils.py b/tests/unit/pipelex/tools/misc/test_http_utils.py
new file mode 100644
index 000000000..169bcb8d0
--- /dev/null
+++ b/tests/unit/pipelex/tools/misc/test_http_utils.py
@@ -0,0 +1,55 @@
+import logging
+from collections.abc import Callable
+from typing import NoReturn
+
+import httpx
+import pytest
+from pytest_mock import MockerFixture
+
+from pipelex.tools.misc.http_utils import validate_url_resource_exists
+
+
+def _make_status_error(status_code: int) -> httpx.HTTPStatusError:
+ request = httpx.Request("HEAD", "https://example.test/resource")
+ response = httpx.Response(status_code=status_code, request=request)
+ return httpx.HTTPStatusError(message=f"HTTP {status_code}", request=request, response=response)
+
+
+def _raise_status_error(status_code: int) -> Callable[..., NoReturn]:
+ def _stub(*_args: object, **_kwargs: object) -> NoReturn:
+ raise _make_status_error(status_code)
+
+ return _stub
+
+
+class TestValidateHttpUrl:
+ @pytest.mark.parametrize("status_code", [401, 403, 429])
+ def test_bot_block_status_codes_are_debug_only(
+ self,
+ status_code: int,
+ mocker: MockerFixture,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """401/403/429 are typical bot-block codes and must NOT log at WARNING."""
+ mocker.patch("pipelex.tools.misc.http_utils.httpx.head", side_effect=_raise_status_error(status_code))
+ url = "https://example.test/resource"
+ with caplog.at_level(logging.DEBUG, logger="pipelex"):
+ validate_url_resource_exists(url)
+ warning_records = [record for record in caplog.records if record.levelno >= logging.WARNING and url in record.message]
+ assert not warning_records, f"Expected no WARNING-level log for status {status_code}, got: {[record.message for record in warning_records]}"
+
+ @pytest.mark.parametrize("status_code", [404, 500, 503])
+ def test_other_status_codes_emit_warning(
+ self,
+ status_code: int,
+ mocker: MockerFixture,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """4xx (other than 401/403/429) and 5xx codes must surface as WARNING per the warn-only contract."""
+ mocker.patch("pipelex.tools.misc.http_utils.httpx.head", side_effect=_raise_status_error(status_code))
+ url = "https://example.test/resource"
+ with caplog.at_level(logging.DEBUG, logger="pipelex"):
+ validate_url_resource_exists(url)
+ warning_records = [record for record in caplog.records if record.levelno == logging.WARNING and url in record.message]
+ assert warning_records, f"Expected a WARNING-level log mentioning the URL for status {status_code}"
+ assert str(status_code) in warning_records[0].message