Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(ingest): guard the optional beautifulsoup4 import; skip the psutil tests (#2041)

Shubham Srivastava48ebee3c6eda
diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.py
index 47d5b6b2..e4b624b8 100644
--- a/semantica/ingest/__init__.py
+++ b/semantica/ingest/__init__.py
@@ -417,6 +417,27 @@ def __getattr__(name: str) -> Any:
if message:
raise ImportError(message)
+ if module_name == ".web_ingestor" and name in {
+ "WebIngestor",
+ "ContentExtractor",
+ }:
+ if getattr(module, "BeautifulSoup", None) is None:
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
+ if module_name == ".feed_ingestor" and name in {"FeedIngestor"}:
+ if getattr(module, "BeautifulSoup", None) is None:
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
+ if module_name == ".email_ingestor" and name in {"EmailIngestor"}:
+ if getattr(module, "BeautifulSoup", None) is None:
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
if module_name == ".xml_ingestor" and name in {"XMLIngestor"}:
if getattr(module, "etree", None) is None:
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
diff --git a/semantica/ingest/email_ingestor.py b/semantica/ingest/email_ingestor.py
index 81c2af20..76dbc1e3 100644
--- a/semantica/ingest/email_ingestor.py
+++ b/semantica/ingest/email_ingestor.py
@@ -40,12 +40,19 @@ from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
-from bs4 import BeautifulSoup
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+# beautifulsoup4 ships in the optional `documents` extra, so guard it the way
+# the sibling ingestors guard theirs (`git` in repo_ingestor, `lxml` in
+# xml_ingestor): importing this module must not require the extra.
+try:
+ from bs4 import BeautifulSoup
+except (ImportError, OSError):
+ BeautifulSoup = None
+
@dataclass
class EmailData:
@@ -382,6 +389,14 @@ class EmailParser:
# Extract from HTML if available
if email_content.strip().startswith("<"):
+ # Checked before the try: the `except Exception: pass` below would
+ # suppress this and fall back to the regex scan, which misses
+ # relative URLs and href-only links, with no hint to the caller.
+ if BeautifulSoup is None:
+ raise ProcessingError(
+ "HTML parsing requires 'beautifulsoup4'. "
+ "Install it with: pip install 'semantica[documents]'"
+ )
try:
soup = BeautifulSoup(email_content, "html.parser")
for link_tag in soup.find_all("a", href=True):
diff --git a/semantica/ingest/feed_ingestor.py b/semantica/ingest/feed_ingestor.py
index 7a1e50f7..3dd5d8a6 100644
--- a/semantica/ingest/feed_ingestor.py
+++ b/semantica/ingest/feed_ingestor.py
@@ -37,11 +37,18 @@ from typing import Any, Dict, List, Optional
from urllib.parse import urljoin, urlparse
import requests
-from bs4 import BeautifulSoup
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+
+# beautifulsoup4 ships in the optional `documents` extra, so guard it the way
+# the sibling ingestors guard theirs (`git` in repo_ingestor, `lxml` in
+# xml_ingestor): importing this module must not require the extra.
+try:
+ from bs4 import BeautifulSoup
+except (ImportError, OSError):
+ BeautifulSoup = None
from .ssrf import parse_bool, request_with_ssrf_guard
@@ -708,6 +715,14 @@ class FeedIngestor:
"""
feed_urls = []
+ # Checked before the try: the `except Exception` below returns [] on
+ # failure, which would report a missing dependency as "no feeds found".
+ if BeautifulSoup is None:
+ raise ProcessingError(
+ "HTML parsing requires 'beautifulsoup4'. "
+ "Install it with: pip install 'semantica[documents]'"
+ )
+
try:
# Fetch website content
response = request_with_ssrf_guard(
diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py
index 7bd5684d..82186e98 100644
--- a/semantica/ingest/methods.py
+++ b/semantica/ingest/methods.py
@@ -567,6 +567,7 @@ def ingest_web(
try:
try:
+ from . import web_ingestor as _web_ingestor
from .web_ingestor import WebIngestor
except ModuleNotFoundError as exc:
if _is_missing_dependency(exc, "bs4"):
@@ -576,6 +577,13 @@ def ingest_web(
) from exc
raise
+ # The module now imports with BeautifulSoup = None rather than raising,
+ # so the translation above no longer fires. Keep this entry point
+ # reporting the same ConfigurationError, before any validation or
+ # network work, instead of a late ProcessingError from the parser.
+ if getattr(_web_ingestor, "BeautifulSoup", None) is None:
+ raise _missing_optional_dependency("Web ingestion", "beautifulsoup4")
+
# Get config
config = ingest_config.get_method_config("web")
config.update(kwargs)
@@ -754,6 +762,7 @@ def ingest_feed(
try:
try:
+ from . import feed_ingestor as _feed_ingestor
from .feed_ingestor import FeedIngestor
except ModuleNotFoundError as exc:
if _is_missing_dependency(exc, "bs4"):
@@ -763,6 +772,13 @@ def ingest_feed(
) from exc
raise
+ # The module now imports with BeautifulSoup = None rather than raising,
+ # so the translation above no longer fires. Keep this entry point
+ # reporting the same ConfigurationError, before any validation or
+ # network work, instead of a late ProcessingError from the parser.
+ if getattr(_feed_ingestor, "BeautifulSoup", None) is None:
+ raise _missing_optional_dependency("Feed ingestion", "beautifulsoup4")
+
# Get config
config = ingest_config.get_method_config("feed")
config.update(kwargs)
@@ -985,6 +1001,7 @@ def ingest_email(
try:
try:
+ from . import email_ingestor as _email_ingestor
from .email_ingestor import EmailIngestor
except ModuleNotFoundError as exc:
if _is_missing_dependency(exc, "bs4"):
@@ -994,6 +1011,13 @@ def ingest_email(
) from exc
raise
+ # The module now imports with BeautifulSoup = None rather than raising,
+ # so the translation above no longer fires. Keep this entry point
+ # reporting the same ConfigurationError, before any validation or
+ # network work, instead of a late ProcessingError from the parser.
+ if getattr(_email_ingestor, "BeautifulSoup", None) is None:
+ raise _missing_optional_dependency("Email ingestion", "beautifulsoup4")
+
# Get config
config = ingest_config.get_method_config("email")
config.update(kwargs)
diff --git a/semantica/ingest/web_ingestor.py b/semantica/ingest/web_ingestor.py
index 5dd77f31..aa31bf99 100644
--- a/semantica/ingest/web_ingestor.py
+++ b/semantica/ingest/web_ingestor.py
@@ -41,15 +41,36 @@ from urllib.parse import urljoin, urlparse
from urllib.robotparser import RobotFileParser
import requests
-from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+
from .ssrf import parse_bool, request_with_ssrf_guard, validate_url_for_request
+# beautifulsoup4 ships in the optional `documents` extra, so guard it the way
+# the sibling ingestors guard theirs (`git` in repo_ingestor, `lxml` in
+# xml_ingestor). The public `from semantica.ingest import WebIngestor` path
+# already turns a missing bs4 into an actionable ImportError; importing this
+# module directly raised a bare ModuleNotFoundError, which aborted collection
+# for every test module that imports it.
+try:
+ from bs4 import BeautifulSoup
+except (ImportError, OSError):
+ BeautifulSoup = None
+
+
+def _parse_html(html_content: str) -> Any:
+ """Parse HTML, or fail with the install hint if beautifulsoup4 is absent."""
+ if BeautifulSoup is None:
+ raise ProcessingError(
+ "HTML parsing requires 'beautifulsoup4'. "
+ "Install it with: pip install 'semantica[documents]'"
+ )
+ return BeautifulSoup(html_content, "html.parser")
+
@dataclass
class WebContent:
@@ -210,7 +231,7 @@ class ContentExtractor:
Returns:
str: Cleaned text content extracted from HTML
"""
- soup = BeautifulSoup(html_content, "html.parser")
+ soup = _parse_html(html_content)
# Remove script and style elements (not useful for text extraction)
for element in soup(["script", "style"]):
@@ -253,7 +274,7 @@ class ContentExtractor:
"""
metadata = {"url": url} if url else {}
- soup = BeautifulSoup(html_content, "html.parser")
+ soup = _parse_html(html_content)
# Extract page title
title_tag = soup.find("title")
@@ -298,7 +319,7 @@ class ContentExtractor:
"""
links = []
- soup = BeautifulSoup(html_content, "html.parser")
+ soup = _parse_html(html_content)
# Extract all anchor tags with href attributes
for link_tag in soup.find_all("a", href=True):
diff --git a/tests/change_management/test_performance.py b/tests/change_management/test_performance.py
index d0bccf2f..d6426a9e 100644
--- a/tests/change_management/test_performance.py
+++ b/tests/change_management/test_performance.py
@@ -9,12 +9,18 @@ import time
import tempfile
import os
import threading
-import psutil
import statistics
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import List, Dict, Any, Tuple
import pytest
+
+# psutil is not declared in any extra and is not in base-deps.txt.
+# Every test class's setup_method builds a PerformanceTestSuite, which
+# calls psutil.Process() to measure RSS, so the whole module needs it.
+pytest.importorskip("psutil")
+import psutil # noqa: E402
+
from semantica.change_management import (
TemporalVersionManager,
OntologyVersionManager,
diff --git a/tests/ingest/test_feed_ingestor.py b/tests/ingest/test_feed_ingestor.py
index 6dea96e8..319d4243 100644
--- a/tests/ingest/test_feed_ingestor.py
+++ b/tests/ingest/test_feed_ingestor.py
@@ -202,6 +202,7 @@ def test_validate_feed() -> None:
def test_discover_feeds_empty() -> None:
"""Test discovery finding nothing."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
ingestor = FeedIngestor()
html = "<html><body>No feeds here</body></html>"
@@ -228,6 +229,7 @@ def test_discover_feeds_empty() -> None:
def test_discover_feeds_found() -> None:
"""Test discovering feeds in HTML content."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
ingestor = FeedIngestor()
html = """
<html>
diff --git a/tests/ingest/test_notebook_02.py b/tests/ingest/test_notebook_02.py
index 5a2d4ada..6463f26f 100644
--- a/tests/ingest/test_notebook_02.py
+++ b/tests/ingest/test_notebook_02.py
@@ -1,12 +1,17 @@
import os
import tempfile
-import pytest
import sqlite3
import time
from pathlib import Path
from unittest.mock import MagicMock, patch
-from semantica.ingest import (
+import pytest
+
+# WebIngestor/FeedIngestor need beautifulsoup4 (the optional documents
+# extra); semantica.ingest raises ImportError for them without it.
+pytest.importorskip("bs4")
+
+from semantica.ingest import ( # noqa: E402
ingest,
FileIngestor, FileTypeDetector, CloudStorageIngestor,
WebIngestor, ContentExtractor, SitemapCrawler, RobotsChecker,
diff --git a/tests/ingest/test_notebook_06.py b/tests/ingest/test_notebook_06.py
index ec857e64..29d8ad2f 100644
--- a/tests/ingest/test_notebook_06.py
+++ b/tests/ingest/test_notebook_06.py
@@ -1,12 +1,17 @@
import os
import tempfile
-import pytest
from unittest.mock import MagicMock, patch
-from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor
-from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker
-from semantica.provenance import InMemoryStorage, ProvenanceManager
-from semantica.conflicts import ConflictDetector
+import pytest
+
+# WebIngestor/FeedIngestor need beautifulsoup4 (the optional documents
+# extra); semantica.ingest raises ImportError for them without it.
+pytest.importorskip("bs4")
+
+from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor # noqa: E402
+from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker # noqa: E402
+from semantica.provenance import InMemoryStorage, ProvenanceManager # noqa: E402
+from semantica.conflicts import ConflictDetector # noqa: E402
pytestmark = pytest.mark.integration
diff --git a/tests/ingest/test_optional_imports.py b/tests/ingest/test_optional_imports.py
index 1b4816c6..0ace706c 100644
--- a/tests/ingest/test_optional_imports.py
+++ b/tests/ingest/test_optional_imports.py
@@ -429,3 +429,188 @@ else:
assert result.returncode == 0, result.stderr
assert "ImportError" in result.stdout
assert "ingest-looker" in result.stdout
+
+
+def test_web_ingestion_reports_missing_beautifulsoup_at_import() -> None:
+ """The probe import must fail fast, not defer to call time.
+
+ `semantica.ingest.__getattr__` raises for a backend whose module imported
+ with its dependency set to None, so `try: from semantica.ingest import X`
+ stays a usable availability check.
+ """
+ result = _run_python_with_blocked_modules(
+ """
+try:
+ from semantica.ingest import WebIngestor
+except ImportError as exc:
+ print("ImportError", exc)
+else:
+ raise SystemExit("expected WebIngestor to fail without beautifulsoup4")
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ImportError" in result.stdout
+ assert "Web ingestion" in result.stdout
+ assert "beautifulsoup4" in result.stdout
+
+
+def test_feed_ingestion_reports_missing_beautifulsoup_at_import() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+try:
+ from semantica.ingest import FeedIngestor
+except ImportError as exc:
+ print("ImportError", exc)
+else:
+ raise SystemExit("expected FeedIngestor to fail without beautifulsoup4")
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ImportError" in result.stdout
+ assert "Feed ingestion" in result.stdout
+
+
+def test_web_ingestor_module_imports_without_beautifulsoup() -> None:
+ """Importing the module must not require the extra.
+
+ Six test modules import `semantica.ingest.web_ingestor` directly; a bare
+ `from bs4 import BeautifulSoup` there aborted collection for all of them.
+ """
+ result = _run_python_with_blocked_modules(
+ """
+import semantica.ingest.web_ingestor as web
+
+print("imported", web.BeautifulSoup is None)
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "imported True" in result.stdout
+
+
+def test_html_parsing_reports_missing_beautifulsoup_when_used() -> None:
+ """The guarded branch raises with the install hint rather than crashing."""
+ result = _run_python_with_blocked_modules(
+ """
+from semantica.ingest.web_ingestor import _parse_html
+
+try:
+ _parse_html("<p>hi</p>")
+except Exception as exc:
+ print(type(exc).__name__, exc)
+else:
+ raise SystemExit("expected HTML parsing to fail without beautifulsoup4")
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ProcessingError" in result.stdout
+ assert "beautifulsoup4" in result.stdout
+
+
+def test_feed_discovery_does_not_report_no_feeds_without_beautifulsoup() -> None:
+ """discover_feeds must raise, not return [].
+
+ Its `except Exception` returns an empty list, so a dependency checked
+ inside the try would read as "this site publishes no feeds".
+ """
+ result = _run_python_with_blocked_modules(
+ """
+import logging
+from semantica.ingest.feed_ingestor import FeedIngestor
+
+ingestor = FeedIngestor.__new__(FeedIngestor)
+ingestor.allow_private_ips = False
+ingestor.logger = logging.getLogger("probe")
+
+try:
+ result = ingestor.discover_feeds("https://example.com")
+except Exception as exc:
+ print(type(exc).__name__, exc)
+else:
+ raise SystemExit(f"expected discover_feeds to raise, got {result!r}")
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ProcessingError" in result.stdout
+ assert "beautifulsoup4" in result.stdout
+
+
+def test_ingest_web_reports_configuration_error_without_beautifulsoup() -> None:
+ """The public entry point keeps its ConfigurationError contract.
+
+ `ingest_web` translated an import-time ModuleNotFoundError into
+ ConfigurationError. Guarding the module import made that translation stop
+ firing, so the call reached validation and fetching and then failed with a
+ ProcessingError from the parser — a different type, later.
+ """
+ result = _run_python_with_blocked_modules(
+ """
+from semantica.ingest import ingest_web
+
+try:
+ ingest_web("https://example.com")
+except Exception as exc:
+ print(type(exc).__name__, exc)
+else:
+ raise SystemExit("expected ingest_web to fail without beautifulsoup4")
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ConfigurationError" in result.stdout
+ assert "Web ingestion" in result.stdout
+ assert "beautifulsoup4" in result.stdout
+
+
+def test_ingest_feed_reports_configuration_error_without_beautifulsoup() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+from semantica.ingest import ingest_feed
+
+try:
+ ingest_feed("https://example.com/feed.xml")
+except Exception as exc:
+ print(type(exc).__name__, exc)
+else:
+ raise SystemExit("expected ingest_feed to fail without beautifulsoup4")
+""",
+ ("bs4",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ConfigurationError" in result.stdout
+ assert "Feed ingestion" in result.stdout
+
+
+def test_email_link_extraction_does_not_silently_degrade(monkeypatch) -> None:
+ """extract_links must not swallow the missing dependency.
+
+ Its `except Exception: pass` would drop the error and fall back to a regex
+ scan that misses relative URLs and href-only links, leaving the caller with
+ quietly worse results and no install hint.
+
+ Tested in-process rather than through _run_python_with_blocked_modules:
+ importing `semantica.ingest.email_ingestor` directly raises
+ `AttributeError: module 'email' has no attribute 'message'` on main as well,
+ independently of beautifulsoup4, so the subprocess form cannot reach the
+ class. Patching the guarded name reproduces the same branch.
+ """
+ pytest.importorskip("bs4")
+ from semantica.ingest import email_ingestor
+ from semantica.utils.exceptions import ProcessingError
+
+ monkeypatch.setattr(email_ingestor, "BeautifulSoup", None)
+ parser = email_ingestor.EmailParser.__new__(email_ingestor.EmailParser)
+
+ with pytest.raises(ProcessingError, match="beautifulsoup4"):
+ parser.extract_links("<html><body><a href='/a'>x</a></body></html>")
diff --git a/tests/ingest/test_web_ingestor.py b/tests/ingest/test_web_ingestor.py
index 890159ff..7e1c7497 100644
--- a/tests/ingest/test_web_ingestor.py
+++ b/tests/ingest/test_web_ingestor.py
@@ -91,6 +91,7 @@ def test_sitemap_invalid_xml() -> None:
# --- Content Extraction Tests ---
def test_extract_links_schemes() -> None:
"""Ensure we ignore mailto and javascript links."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
extractor = ContentExtractor()
html = """
@@ -107,6 +108,7 @@ def test_extract_links_schemes() -> None:
def test_extract_metadata_empty() -> None:
"""Test extraction with missing meta tags."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
extractor = ContentExtractor()
html = "<html><body>No Head</body></html>"
@@ -120,6 +122,7 @@ def test_extract_metadata_empty() -> None:
@patch("semantica.ingest.web_ingestor.request_with_ssrf_guard")
def test_ingest_url_happy(mock_get: MagicMock, sample_html: str) -> None:
"""Test successful URL ingestion."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
mock_get.return_value = MagicMock(status_code=200, text=sample_html)
ingestor = WebIngestor(respect_robots=False)
@@ -211,6 +214,7 @@ def test_url_filters() -> None:
def test_extract_text_cleaning() -> None:
"""Test stripping scripts and styles from text."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
extractor = ContentExtractor()
html = """
@@ -270,6 +274,7 @@ def test_web_ingestor_crawl_sitemap_integration() -> None:
def test_metadata_priority() -> None:
"""Test that OpenGraph tags take precedence over standard meta tags."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
extractor = ContentExtractor()
html = """
@@ -397,6 +402,7 @@ def test_crawl_domain_max_pages() -> None:
def test_metadata_opengraph_priority() -> None:
"""Test that OpenGraph tags are captured correctly."""
+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extra
extractor = ContentExtractor()
# HTML with both standard meta and OG tags
html = """
diff --git a/tests/vector_store/test_performance_benchmarks.py b/tests/vector_store/test_performance_benchmarks.py
index 85fb6386..808644d8 100644
--- a/tests/vector_store/test_performance_benchmarks.py
+++ b/tests/vector_store/test_performance_benchmarks.py
@@ -17,17 +17,22 @@ import pytest
import numpy as np
import time
import threading
-import psutil
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Dict, List, Any
+
from semantica.vector_store import VectorStore, DecisionEmbeddingPipeline, HybridSimilarityCalculator
from semantica.context import DecisionContext, ContextRetriever
from semantica.vector_store.decision_vector_methods import (
quick_decision, find_precedents, batch_decisions
)
+# psutil is not declared in any extra and is not in base-deps.txt.
+# This module measures RSS through psutil in its benchmark helpers.
+pytest.importorskip("psutil")
+import psutil # noqa: E402
+
class TestPerformanceBenchmarks:
"""Performance benchmarks for enhanced vector store."""