Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Test and error handling for Pipe Extract web (#854)
Louis Choquel7df226d61ba7
diff --git a/.pipelex-dev/test_profiles.toml b/.pipelex-dev/test_profiles.tomlindex cca3a7789..37f9c9de3 100644--- a/.pipelex-dev/test_profiles.toml+++ b/.pipelex-dev/test_profiles.toml@@ -285,9 +285,14 @@ search_models = [][profiles.dev]description = "Quick dev testing with fast and cheap models from key providers"backends = ["pipelex_gateway", "internal"]-llm_models = ["claude-4.5-haiku", "gpt-4o-mini", "gemini-2.5-flash-lite"]+llm_models = [+ "claude-4.5-haiku",+ "gpt-4o-mini",+ "linkup",+ "gemini-2.5-flash-lite",+]img_gen_models = ["gpt-image-1-mini"]-extract_models = ["@from_pdf"]+extract_models = ["@from_pdf", "linkup-fetch"]search_models = ["linkup-standard"]################################################################################diff --git a/pipelex/plugins/gateway/gateway_extract_worker.py b/pipelex/plugins/gateway/gateway_extract_worker.pyindex c40ef0cf5..8c64b120c 100644--- a/pipelex/plugins/gateway/gateway_extract_worker.py+++ b/pipelex/plugins/gateway/gateway_extract_worker.py@@ -151,7 +151,10 @@ class GatewayExtractWorker(ExtractWorkerAbstract):)except portkey_exceptions.APIError as exc:error_summary = GatewayFactory.make_error_summary_from_portkey_error(exc)- msg = f"Web fetch service error for model '{self.inference_model.tag}' after {attempt_number} attempt(s): {error_summary}"+ msg = (+ f"Web fetch service error for URL '{document_uri}' via model '{self.inference_model.tag}' "+ f"after {attempt_number} attempt(s): {error_summary}"+ )raise ExtractJobFailureError(msg) from excif response is None:@@ -231,7 +234,9 @@ class GatewayExtractWorker(ExtractWorkerAbstract):if isinstance(exc, portkey_exceptions.NotFoundError):msg = str(exc).lower()return "specified deployment could not be found" in msg- return False+ # Transient upstream gateway failures (e.g. openrouter 500s during batch fetches)+ # should be retried rather than killing the whole batch on the first hiccup.+ return isinstance(exc, (portkey_exceptions.InternalServerError, portkey_exceptions.APITimeoutError, portkey_exceptions.APIConnectionError))def _log_retry(self, retry_state: RetryCallState) -> None:"""Called before sleeping between retries."""diff --git a/pipelex/tools/misc/http_utils.py b/pipelex/tools/misc/http_utils.pyindex 591c5e418..4377ffbc4 100644--- a/pipelex/tools/misc/http_utils.py+++ b/pipelex/tools/misc/http_utils.py@@ -1,5 +1,6 @@import httpx+from pipelex import logfrom pipelex.tools.misc.file_utils import path_existsfrom pipelex.tools.misc.package_utils import get_package_versionfrom pipelex.urls import URLs@@ -22,12 +23,16 @@ def validate_url_resource_exists(url: str) -> None:By the time a URL reaches DocumentContent/ImageContent, it should alreadybe resolved (absolute path or fully qualified URL).- For HTTP/HTTPS URLs: performs a streaming GET request to check reachability.+ For HTTP/HTTPS URLs: performs a HEAD request (falling back to a streaming GET+ on 405) as a best-effort reachability probe. Failures are logged as warnings+ but do NOT raise — the downstream extractor is the source of truth, and many+ sites bot-block HEAD/unknown User-Agents with 403/401/429 while still serving+ the actual content fine.For local file paths: checks that the file exists on disk.Skips validation for internal URIs (base64 data URLs, pipelex-storage://).Raises:- ValueError: If the resource does not exist or is unreachable.+ ValueError: If a local file path does not exist. HTTP failures only log a warning."""if url.startswith(_SKIP_VALIDATION_PREFIXES):return@@ -50,17 +55,22 @@ def _validate_http_url(url: str) -> None:else:response.raise_for_status()except httpx.HTTPStatusError as exc:- msg = f"URL '{url}' returned HTTP {exc.response.status_code}"- raise ValueError(msg) from exc- except httpx.ConnectError as exc:- msg = f"URL '{url}' could not be reached (connection failed)"- raise ValueError(msg) from exc- except httpx.TimeoutException as exc:- msg = f"URL '{url}' timed out"- raise ValueError(msg) from exc- except httpx.HTTPError as exc:- msg = f"URL '{url}' could not be fetched"- raise ValueError(msg) from exc+ status_code = exc.response.status_code+ msg = f"Pre-flight URL check: URL '{url}' returned HTTP {status_code} (continuing — downstream extractor will decide)"+ # 401/403/429 are typical bot-block codes when servers reject HEAD/unknown User-Agents while still serving real content — keep these quiet.+ if status_code in {401, 403, 429}:+ log.debug(msg)+ else:+ log.warning(msg)+ except httpx.ConnectError:+ msg = f"Pre-flight URL check: URL '{url}' could not be reached (connection failed) (continuing — downstream extractor will decide)"+ log.warning(msg)+ except httpx.TimeoutException:+ msg = f"Pre-flight URL check: URL '{url}' timed out (continuing — downstream extractor will decide)"+ log.warning(msg)+ except httpx.HTTPError:+ msg = f"Pre-flight URL check: URL '{url}' could not be fetched (continuing — downstream extractor will decide)"+ log.warning(msg)def _validate_local_path(url: str) -> None:diff --git a/tests/cases/documents.py b/tests/cases/documents.pyindex aaad5a295..73a1df31d 100644--- a/tests/cases/documents.py+++ b/tests/cases/documents.py@@ -42,8 +42,10 @@ class DocumentTestCases:# Web URLsWEB_URL_1 = "https://books.toscrape.com/catalogue/cravings-recipes-for-what-you-want-to-eat_589/index.html"WEB_URL_2 = "https://www.scrapethissite.com/pages/"+ WEB_URL_3 = "https://www.allrecipes.com/recipe/91192/french-onion-soup-gratinee/"WEB_URLS: ClassVar[list[str]] = [- WEB_URL_1,- WEB_URL_2,+ # WEB_URL_1,+ # WEB_URL_2,+ WEB_URL_3,]diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/pipe_extract.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/pipe_extract.mthdsnew file mode 100644index 000000000..3a0f787a9--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/pipe_extract.mthds@@ -0,0 +1,9 @@+domain = "pipe_extract_e2e"+description = "Test PipeExtract for web page fetching"++[pipe.extract_web_page_e2e]+type = "PipeExtract"+description = "Fetch full page content from a web URL"+inputs = { document = "Document" }+output = "Page[]"+model = "@default-extract-web-page"diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_data.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_data.pynew file mode 100644index 000000000..c0f90ec51--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_data.py@@ -0,0 +1,19 @@+"""Test data for PipeExtract E2E tests."""++from typing import ClassVar++from tests.cases.documents import DocumentTestCases+++class PipeExtractTestCases:+ """Test cases for PipeExtract E2E tests."""++ # (variant, url)+ # WEB_URL_3 (AllRecipes) is a known bot-blocked site that returns 403 to HEAD requests+ # but serves the actual content fine through linkup-fetch. Included here to guard+ # against regressions where the pre-flight URL check aborts the pipeline.+ WEB_URL_CASES: ClassVar[list[tuple[str, str]]] = [+ ("books_toscrape", DocumentTestCases.WEB_URL_1),+ ("scrapethissite", DocumentTestCases.WEB_URL_2),+ ("allrecipes_bot_blocked", DocumentTestCases.WEB_URL_3),+ ]diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_pipe_extract.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_pipe_extract.pynew file mode 100644index 000000000..bec00b235--- /dev/null+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_extract/test_pipe_extract.py@@ -0,0 +1,59 @@+"""E2E tests for PipeExtract operator."""++import pytest++from pipelex import pretty_print+from pipelex.core.stuffs.document_content import DocumentContent+from pipelex.core.stuffs.page_content import PageContent+from pipelex.pipe_run.pipe_run_mode import PipeRunMode+from pipelex.pipeline.runner import PipelexRunner+from tests.e2e.pipelex.pipes.pipe_operators.pipe_extract.test_data import PipeExtractTestCases++LIBRARY_DIRS = ["tests/e2e/pipelex/pipes/pipe_operators/pipe_extract"]++[email protected]_runnable[email protected](loop_scope="class")+class TestPipeExtract:+ """E2E tests for PipeExtract operator."""++ @pytest.mark.parametrize(("variant", "url"), PipeExtractTestCases.WEB_URL_CASES)+ async def test_extract_web_page(+ self,+ pipe_run_mode: PipeRunMode,+ variant: str,+ url: str,+ ) -> None:+ """Extract a web page via PipeExtract + linkup-fetch.++ Regression test for pre-flight URL validation: some sites (e.g. AllRecipes)+ bot-block HEAD requests with 403, and the pre-flight check must NOT abort+ the pipeline — the downstream extractor is the source of truth.+ """+ runner = PipelexRunner(+ library_dirs=LIBRARY_DIRS,+ pipe_run_mode=pipe_run_mode,+ )+ pipeline_response = await runner.execute_pipeline(+ pipe_code="extract_web_page_e2e",+ inputs={+ "document": DocumentContent(url=url),+ },+ )++ pipe_output = pipeline_response.pipe_output+ assert pipe_output is not None+ assert pipe_output.working_memory is not None+ assert pipe_output.main_stuff is not None++ if pipe_run_mode.is_live:+ pages = pipe_output.main_stuff_as_items(item_type=PageContent)+ assert len(pages) > 0, f"Expected at least one page for {variant}"+ first_page = pages[0]+ page_text_content = first_page.text_and_images.text+ assert page_text_content is not None, f"Expected markdown text for {variant}"+ page_text = page_text_content.text+ assert len(page_text.strip()) > 0, f"Expected non-empty markdown text for {variant}"+ pretty_print(page_text[:500], title=f"Extracted markdown ({variant})")diff --git a/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py b/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.pyindex 7d43eec1c..50b5dfca5 100644--- a/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py+++ b/tests/unit/pipelex/core/stuffs/test_document_content_url_validation.py@@ -1,5 +1,7 @@"""Unit tests for URL validation in DocumentContent."""+import logging+import pytestfrom pipelex.core.stuffs.document_content import DocumentContent@@ -13,11 +15,12 @@ class TestDocumentContentUrlValidation:doc = DocumentContent(url="https://this-domain-cannot-exist.invalid/file.pdf")assert doc.url == "https://this-domain-cannot-exist.invalid/file.pdf"- def test_validate_resources_unreachable_remote_url(self) -> None:- """validate_resources() raises ValueError for an unreachable HTTP URL."""+ def test_validate_resources_unreachable_remote_url(self, caplog: pytest.LogCaptureFixture) -> None:+ """validate_resources() does NOT raise for an unreachable HTTP URL: it logs a warning and lets the downstream extractor decide."""doc = DocumentContent(url="https://this-domain-cannot-exist.invalid/file.pdf")- with pytest.raises(ValueError, match="could not be reached"):+ with caplog.at_level(logging.WARNING, logger="pipelex"):doc.validate_resources()+ assert any("could not be reached" in record.message for record in caplog.records)def test_validate_resources_nonexistent_local_file(self) -> None:"""validate_resources() raises ValueError for a non-existent local file."""diff --git a/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py b/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.pyindex 96d63b8cc..58e8ca90e 100644--- a/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py+++ b/tests/unit/pipelex/core/stuffs/test_image_content_url_validation.py@@ -1,5 +1,7 @@"""Unit tests for URL validation in ImageContent."""+import logging+import pytestfrom pipelex.core.stuffs.image_content import ImageContent@@ -13,11 +15,12 @@ class TestImageContentUrlValidation:img = ImageContent(url="https://this-domain-cannot-exist.invalid/image.png")assert img.url == "https://this-domain-cannot-exist.invalid/image.png"- def test_validate_resources_unreachable_remote_url(self) -> None:- """validate_resources() raises ValueError for an unreachable HTTP URL."""+ def test_validate_resources_unreachable_remote_url(self, caplog: pytest.LogCaptureFixture) -> None:+ """validate_resources() does NOT raise for an unreachable HTTP URL: it logs a warning and lets the downstream extractor decide."""img = ImageContent(url="https://this-domain-cannot-exist.invalid/image.png")- with pytest.raises(ValueError, match="could not be reached"):+ with caplog.at_level(logging.WARNING, logger="pipelex"):img.validate_resources()+ assert any("could not be reached" in record.message for record in caplog.records)def test_validate_resources_nonexistent_local_file(self) -> None:"""validate_resources() raises ValueError for a non-existent local file."""diff --git a/tests/unit/pipelex/tools/misc/test_http_utils.py b/tests/unit/pipelex/tools/misc/test_http_utils.pynew file mode 100644index 000000000..169bcb8d0--- /dev/null+++ b/tests/unit/pipelex/tools/misc/test_http_utils.py@@ -0,0 +1,55 @@+import logging+from collections.abc import Callable+from typing import NoReturn++import httpx+import pytest+from pytest_mock import MockerFixture++from pipelex.tools.misc.http_utils import validate_url_resource_exists+++def _make_status_error(status_code: int) -> httpx.HTTPStatusError:+ request = httpx.Request("HEAD", "https://example.test/resource")+ response = httpx.Response(status_code=status_code, request=request)+ return httpx.HTTPStatusError(message=f"HTTP {status_code}", request=request, response=response)+++def _raise_status_error(status_code: int) -> Callable[..., NoReturn]:+ def _stub(*_args: object, **_kwargs: object) -> NoReturn:+ raise _make_status_error(status_code)++ return _stub+++class TestValidateHttpUrl:+ @pytest.mark.parametrize("status_code", [401, 403, 429])+ def test_bot_block_status_codes_are_debug_only(+ self,+ status_code: int,+ mocker: MockerFixture,+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """401/403/429 are typical bot-block codes and must NOT log at WARNING."""+ mocker.patch("pipelex.tools.misc.http_utils.httpx.head", side_effect=_raise_status_error(status_code))+ url = "https://example.test/resource"+ with caplog.at_level(logging.DEBUG, logger="pipelex"):+ validate_url_resource_exists(url)+ warning_records = [record for record in caplog.records if record.levelno >= logging.WARNING and url in record.message]+ assert not warning_records, f"Expected no WARNING-level log for status {status_code}, got: {[record.message for record in warning_records]}"++ @pytest.mark.parametrize("status_code", [404, 500, 503])+ def test_other_status_codes_emit_warning(+ self,+ status_code: int,+ mocker: MockerFixture,+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """4xx (other than 401/403/429) and 5xx codes must surface as WARNING per the warn-only contract."""+ mocker.patch("pipelex.tools.misc.http_utils.httpx.head", side_effect=_raise_status_error(status_code))+ url = "https://example.test/resource"+ with caplog.at_level(logging.DEBUG, logger="pipelex"):+ validate_url_resource_exists(url)+ warning_records = [record for record in caplog.records if record.levelno == logging.WARNING and url in record.message]+ assert warning_records, f"Expected a WARNING-level log mentioning the URL for status {status_code}"+ assert str(status_code) in warning_records[0].message