Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(ingest): guard the optional beautifulsoup4 import; skip the psutil tests (#2041)
Shubham Srivastava48ebee3c6eda
diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.pyindex 47d5b6b2..e4b624b8 100644--- a/semantica/ingest/__init__.py+++ b/semantica/ingest/__init__.py@@ -417,6 +417,27 @@ def __getattr__(name: str) -> Any:if message:raise ImportError(message)+ if module_name == ".web_ingestor" and name in {+ "WebIngestor",+ "ContentExtractor",+ }:+ if getattr(module, "BeautifulSoup", None) is None:+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)++ if module_name == ".feed_ingestor" and name in {"FeedIngestor"}:+ if getattr(module, "BeautifulSoup", None) is None:+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)++ if module_name == ".email_ingestor" and name in {"EmailIngestor"}:+ if getattr(module, "BeautifulSoup", None) is None:+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)+if module_name == ".xml_ingestor" and name in {"XMLIngestor"}:if getattr(module, "etree", None) is None:message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)diff --git a/semantica/ingest/email_ingestor.py b/semantica/ingest/email_ingestor.pyindex 81c2af20..76dbc1e3 100644--- a/semantica/ingest/email_ingestor.py+++ b/semantica/ingest/email_ingestor.py@@ -40,12 +40,19 @@ from pathlib import Pathfrom typing import Any, Dict, List, Optionalfrom urllib.parse import urlparse-from bs4 import BeautifulSoupfrom ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+# beautifulsoup4 ships in the optional `documents` extra, so guard it the way+# the sibling ingestors guard theirs (`git` in repo_ingestor, `lxml` in+# xml_ingestor): importing this module must not require the extra.+try:+ from bs4 import BeautifulSoup+except (ImportError, OSError):+ BeautifulSoup = None+@dataclassclass EmailData:@@ -382,6 +389,14 @@ class EmailParser:# Extract from HTML if availableif email_content.strip().startswith("<"):+ # Checked before the try: the `except Exception: pass` below would+ # suppress this and fall back to the regex scan, which misses+ # relative URLs and href-only links, with no hint to the caller.+ if BeautifulSoup is None:+ raise ProcessingError(+ "HTML parsing requires 'beautifulsoup4'. "+ "Install it with: pip install 'semantica[documents]'"+ )try:soup = BeautifulSoup(email_content, "html.parser")for link_tag in soup.find_all("a", href=True):diff --git a/semantica/ingest/feed_ingestor.py b/semantica/ingest/feed_ingestor.pyindex 7a1e50f7..3dd5d8a6 100644--- a/semantica/ingest/feed_ingestor.py+++ b/semantica/ingest/feed_ingestor.py@@ -37,11 +37,18 @@ from typing import Any, Dict, List, Optionalfrom urllib.parse import urljoin, urlparseimport requests-from bs4 import BeautifulSoupfrom ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker++# beautifulsoup4 ships in the optional `documents` extra, so guard it the way+# the sibling ingestors guard theirs (`git` in repo_ingestor, `lxml` in+# xml_ingestor): importing this module must not require the extra.+try:+ from bs4 import BeautifulSoup+except (ImportError, OSError):+ BeautifulSoup = Nonefrom .ssrf import parse_bool, request_with_ssrf_guard@@ -708,6 +715,14 @@ class FeedIngestor:"""feed_urls = []+ # Checked before the try: the `except Exception` below returns [] on+ # failure, which would report a missing dependency as "no feeds found".+ if BeautifulSoup is None:+ raise ProcessingError(+ "HTML parsing requires 'beautifulsoup4'. "+ "Install it with: pip install 'semantica[documents]'"+ )+try:# Fetch website contentresponse = request_with_ssrf_guard(diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.pyindex 7bd5684d..82186e98 100644--- a/semantica/ingest/methods.py+++ b/semantica/ingest/methods.py@@ -567,6 +567,7 @@ def ingest_web(try:try:+ from . import web_ingestor as _web_ingestorfrom .web_ingestor import WebIngestorexcept ModuleNotFoundError as exc:if _is_missing_dependency(exc, "bs4"):@@ -576,6 +577,13 @@ def ingest_web() from excraise+ # The module now imports with BeautifulSoup = None rather than raising,+ # so the translation above no longer fires. Keep this entry point+ # reporting the same ConfigurationError, before any validation or+ # network work, instead of a late ProcessingError from the parser.+ if getattr(_web_ingestor, "BeautifulSoup", None) is None:+ raise _missing_optional_dependency("Web ingestion", "beautifulsoup4")+# Get configconfig = ingest_config.get_method_config("web")config.update(kwargs)@@ -754,6 +762,7 @@ def ingest_feed(try:try:+ from . import feed_ingestor as _feed_ingestorfrom .feed_ingestor import FeedIngestorexcept ModuleNotFoundError as exc:if _is_missing_dependency(exc, "bs4"):@@ -763,6 +772,13 @@ def ingest_feed() from excraise+ # The module now imports with BeautifulSoup = None rather than raising,+ # so the translation above no longer fires. Keep this entry point+ # reporting the same ConfigurationError, before any validation or+ # network work, instead of a late ProcessingError from the parser.+ if getattr(_feed_ingestor, "BeautifulSoup", None) is None:+ raise _missing_optional_dependency("Feed ingestion", "beautifulsoup4")+# Get configconfig = ingest_config.get_method_config("feed")config.update(kwargs)@@ -985,6 +1001,7 @@ def ingest_email(try:try:+ from . import email_ingestor as _email_ingestorfrom .email_ingestor import EmailIngestorexcept ModuleNotFoundError as exc:if _is_missing_dependency(exc, "bs4"):@@ -994,6 +1011,13 @@ def ingest_email() from excraise+ # The module now imports with BeautifulSoup = None rather than raising,+ # so the translation above no longer fires. Keep this entry point+ # reporting the same ConfigurationError, before any validation or+ # network work, instead of a late ProcessingError from the parser.+ if getattr(_email_ingestor, "BeautifulSoup", None) is None:+ raise _missing_optional_dependency("Email ingestion", "beautifulsoup4")+# Get configconfig = ingest_config.get_method_config("email")config.update(kwargs)diff --git a/semantica/ingest/web_ingestor.py b/semantica/ingest/web_ingestor.pyindex 5dd77f31..aa31bf99 100644--- a/semantica/ingest/web_ingestor.py+++ b/semantica/ingest/web_ingestor.py@@ -41,15 +41,36 @@ from urllib.parse import urljoin, urlparsefrom urllib.robotparser import RobotFileParserimport requests-from bs4 import BeautifulSoupfrom requests.adapters import HTTPAdapterfrom urllib3.util.retry import Retryfrom ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+from .ssrf import parse_bool, request_with_ssrf_guard, validate_url_for_request+# beautifulsoup4 ships in the optional `documents` extra, so guard it the way+# the sibling ingestors guard theirs (`git` in repo_ingestor, `lxml` in+# xml_ingestor). The public `from semantica.ingest import WebIngestor` path+# already turns a missing bs4 into an actionable ImportError; importing this+# module directly raised a bare ModuleNotFoundError, which aborted collection+# for every test module that imports it.+try:+ from bs4 import BeautifulSoup+except (ImportError, OSError):+ BeautifulSoup = None+++def _parse_html(html_content: str) -> Any:+ """Parse HTML, or fail with the install hint if beautifulsoup4 is absent."""+ if BeautifulSoup is None:+ raise ProcessingError(+ "HTML parsing requires 'beautifulsoup4'. "+ "Install it with: pip install 'semantica[documents]'"+ )+ return BeautifulSoup(html_content, "html.parser")+@dataclassclass WebContent:@@ -210,7 +231,7 @@ class ContentExtractor:Returns:str: Cleaned text content extracted from HTML"""- soup = BeautifulSoup(html_content, "html.parser")+ soup = _parse_html(html_content)# Remove script and style elements (not useful for text extraction)for element in soup(["script", "style"]):@@ -253,7 +274,7 @@ class ContentExtractor:"""metadata = {"url": url} if url else {}- soup = BeautifulSoup(html_content, "html.parser")+ soup = _parse_html(html_content)# Extract page titletitle_tag = soup.find("title")@@ -298,7 +319,7 @@ class ContentExtractor:"""links = []- soup = BeautifulSoup(html_content, "html.parser")+ soup = _parse_html(html_content)# Extract all anchor tags with href attributesfor link_tag in soup.find_all("a", href=True):diff --git a/tests/change_management/test_performance.py b/tests/change_management/test_performance.pyindex d0bccf2f..d6426a9e 100644--- a/tests/change_management/test_performance.py+++ b/tests/change_management/test_performance.py@@ -9,12 +9,18 @@ import timeimport tempfileimport osimport threading-import psutilimport statisticsfrom concurrent.futures import ThreadPoolExecutor, as_completedfrom typing import List, Dict, Any, Tupleimport pytest++# psutil is not declared in any extra and is not in base-deps.txt.+# Every test class's setup_method builds a PerformanceTestSuite, which+# calls psutil.Process() to measure RSS, so the whole module needs it.+pytest.importorskip("psutil")+import psutil # noqa: E402+from semantica.change_management import (TemporalVersionManager,OntologyVersionManager,diff --git a/tests/ingest/test_feed_ingestor.py b/tests/ingest/test_feed_ingestor.pyindex 6dea96e8..319d4243 100644--- a/tests/ingest/test_feed_ingestor.py+++ b/tests/ingest/test_feed_ingestor.py@@ -202,6 +202,7 @@ def test_validate_feed() -> None:def test_discover_feeds_empty() -> None:"""Test discovery finding nothing."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraingestor = FeedIngestor()html = "<html><body>No feeds here</body></html>"@@ -228,6 +229,7 @@ def test_discover_feeds_empty() -> None:def test_discover_feeds_found() -> None:"""Test discovering feeds in HTML content."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraingestor = FeedIngestor()html = """<html>diff --git a/tests/ingest/test_notebook_02.py b/tests/ingest/test_notebook_02.pyindex 5a2d4ada..6463f26f 100644--- a/tests/ingest/test_notebook_02.py+++ b/tests/ingest/test_notebook_02.py@@ -1,12 +1,17 @@import osimport tempfile-import pytestimport sqlite3import timefrom pathlib import Pathfrom unittest.mock import MagicMock, patch-from semantica.ingest import (+import pytest++# WebIngestor/FeedIngestor need beautifulsoup4 (the optional documents+# extra); semantica.ingest raises ImportError for them without it.+pytest.importorskip("bs4")++from semantica.ingest import ( # noqa: E402ingest,FileIngestor, FileTypeDetector, CloudStorageIngestor,WebIngestor, ContentExtractor, SitemapCrawler, RobotsChecker,diff --git a/tests/ingest/test_notebook_06.py b/tests/ingest/test_notebook_06.pyindex ec857e64..29d8ad2f 100644--- a/tests/ingest/test_notebook_06.py+++ b/tests/ingest/test_notebook_06.py@@ -1,12 +1,17 @@import osimport tempfile-import pytestfrom unittest.mock import MagicMock, patch-from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor-from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker-from semantica.provenance import InMemoryStorage, ProvenanceManager-from semantica.conflicts import ConflictDetector+import pytest++# WebIngestor/FeedIngestor need beautifulsoup4 (the optional documents+# extra); semantica.ingest raises ImportError for them without it.+pytest.importorskip("bs4")++from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor # noqa: E402+from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker # noqa: E402+from semantica.provenance import InMemoryStorage, ProvenanceManager # noqa: E402+from semantica.conflicts import ConflictDetector # noqa: E402pytestmark = pytest.mark.integrationdiff --git a/tests/ingest/test_optional_imports.py b/tests/ingest/test_optional_imports.pyindex 1b4816c6..0ace706c 100644--- a/tests/ingest/test_optional_imports.py+++ b/tests/ingest/test_optional_imports.py@@ -429,3 +429,188 @@ else:assert result.returncode == 0, result.stderrassert "ImportError" in result.stdoutassert "ingest-looker" in result.stdout+++def test_web_ingestion_reports_missing_beautifulsoup_at_import() -> None:+ """The probe import must fail fast, not defer to call time.++ `semantica.ingest.__getattr__` raises for a backend whose module imported+ with its dependency set to None, so `try: from semantica.ingest import X`+ stays a usable availability check.+ """+ result = _run_python_with_blocked_modules(+ """+try:+ from semantica.ingest import WebIngestor+except ImportError as exc:+ print("ImportError", exc)+else:+ raise SystemExit("expected WebIngestor to fail without beautifulsoup4")+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "ImportError" in result.stdout+ assert "Web ingestion" in result.stdout+ assert "beautifulsoup4" in result.stdout+++def test_feed_ingestion_reports_missing_beautifulsoup_at_import() -> None:+ result = _run_python_with_blocked_modules(+ """+try:+ from semantica.ingest import FeedIngestor+except ImportError as exc:+ print("ImportError", exc)+else:+ raise SystemExit("expected FeedIngestor to fail without beautifulsoup4")+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "ImportError" in result.stdout+ assert "Feed ingestion" in result.stdout+++def test_web_ingestor_module_imports_without_beautifulsoup() -> None:+ """Importing the module must not require the extra.++ Six test modules import `semantica.ingest.web_ingestor` directly; a bare+ `from bs4 import BeautifulSoup` there aborted collection for all of them.+ """+ result = _run_python_with_blocked_modules(+ """+import semantica.ingest.web_ingestor as web++print("imported", web.BeautifulSoup is None)+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "imported True" in result.stdout+++def test_html_parsing_reports_missing_beautifulsoup_when_used() -> None:+ """The guarded branch raises with the install hint rather than crashing."""+ result = _run_python_with_blocked_modules(+ """+from semantica.ingest.web_ingestor import _parse_html++try:+ _parse_html("<p>hi</p>")+except Exception as exc:+ print(type(exc).__name__, exc)+else:+ raise SystemExit("expected HTML parsing to fail without beautifulsoup4")+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "ProcessingError" in result.stdout+ assert "beautifulsoup4" in result.stdout+++def test_feed_discovery_does_not_report_no_feeds_without_beautifulsoup() -> None:+ """discover_feeds must raise, not return [].++ Its `except Exception` returns an empty list, so a dependency checked+ inside the try would read as "this site publishes no feeds".+ """+ result = _run_python_with_blocked_modules(+ """+import logging+from semantica.ingest.feed_ingestor import FeedIngestor++ingestor = FeedIngestor.__new__(FeedIngestor)+ingestor.allow_private_ips = False+ingestor.logger = logging.getLogger("probe")++try:+ result = ingestor.discover_feeds("https://example.com")+except Exception as exc:+ print(type(exc).__name__, exc)+else:+ raise SystemExit(f"expected discover_feeds to raise, got {result!r}")+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "ProcessingError" in result.stdout+ assert "beautifulsoup4" in result.stdout+++def test_ingest_web_reports_configuration_error_without_beautifulsoup() -> None:+ """The public entry point keeps its ConfigurationError contract.++ `ingest_web` translated an import-time ModuleNotFoundError into+ ConfigurationError. Guarding the module import made that translation stop+ firing, so the call reached validation and fetching and then failed with a+ ProcessingError from the parser — a different type, later.+ """+ result = _run_python_with_blocked_modules(+ """+from semantica.ingest import ingest_web++try:+ ingest_web("https://example.com")+except Exception as exc:+ print(type(exc).__name__, exc)+else:+ raise SystemExit("expected ingest_web to fail without beautifulsoup4")+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "ConfigurationError" in result.stdout+ assert "Web ingestion" in result.stdout+ assert "beautifulsoup4" in result.stdout+++def test_ingest_feed_reports_configuration_error_without_beautifulsoup() -> None:+ result = _run_python_with_blocked_modules(+ """+from semantica.ingest import ingest_feed++try:+ ingest_feed("https://example.com/feed.xml")+except Exception as exc:+ print(type(exc).__name__, exc)+else:+ raise SystemExit("expected ingest_feed to fail without beautifulsoup4")+""",+ ("bs4",),+ )++ assert result.returncode == 0, result.stderr+ assert "ConfigurationError" in result.stdout+ assert "Feed ingestion" in result.stdout+++def test_email_link_extraction_does_not_silently_degrade(monkeypatch) -> None:+ """extract_links must not swallow the missing dependency.++ Its `except Exception: pass` would drop the error and fall back to a regex+ scan that misses relative URLs and href-only links, leaving the caller with+ quietly worse results and no install hint.++ Tested in-process rather than through _run_python_with_blocked_modules:+ importing `semantica.ingest.email_ingestor` directly raises+ `AttributeError: module 'email' has no attribute 'message'` on main as well,+ independently of beautifulsoup4, so the subprocess form cannot reach the+ class. Patching the guarded name reproduces the same branch.+ """+ pytest.importorskip("bs4")+ from semantica.ingest import email_ingestor+ from semantica.utils.exceptions import ProcessingError++ monkeypatch.setattr(email_ingestor, "BeautifulSoup", None)+ parser = email_ingestor.EmailParser.__new__(email_ingestor.EmailParser)++ with pytest.raises(ProcessingError, match="beautifulsoup4"):+ parser.extract_links("<html><body><a href='/a'>x</a></body></html>")diff --git a/tests/ingest/test_web_ingestor.py b/tests/ingest/test_web_ingestor.pyindex 890159ff..7e1c7497 100644--- a/tests/ingest/test_web_ingestor.py+++ b/tests/ingest/test_web_ingestor.py@@ -91,6 +91,7 @@ def test_sitemap_invalid_xml() -> None:# --- Content Extraction Tests ---def test_extract_links_schemes() -> None:"""Ensure we ignore mailto and javascript links."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraextractor = ContentExtractor()html = """@@ -107,6 +108,7 @@ def test_extract_links_schemes() -> None:def test_extract_metadata_empty() -> None:"""Test extraction with missing meta tags."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraextractor = ContentExtractor()html = "<html><body>No Head</body></html>"@@ -120,6 +122,7 @@ def test_extract_metadata_empty() -> None:@patch("semantica.ingest.web_ingestor.request_with_ssrf_guard")def test_ingest_url_happy(mock_get: MagicMock, sample_html: str) -> None:"""Test successful URL ingestion."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extramock_get.return_value = MagicMock(status_code=200, text=sample_html)ingestor = WebIngestor(respect_robots=False)@@ -211,6 +214,7 @@ def test_url_filters() -> None:def test_extract_text_cleaning() -> None:"""Test stripping scripts and styles from text."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraextractor = ContentExtractor()html = """@@ -270,6 +274,7 @@ def test_web_ingestor_crawl_sitemap_integration() -> None:def test_metadata_priority() -> None:"""Test that OpenGraph tags take precedence over standard meta tags."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraextractor = ContentExtractor()html = """@@ -397,6 +402,7 @@ def test_crawl_domain_max_pages() -> None:def test_metadata_opengraph_priority() -> None:"""Test that OpenGraph tags are captured correctly."""+ pytest.importorskip("bs4") # HTML parsing needs the optional documents extraextractor = ContentExtractor()# HTML with both standard meta and OG tagshtml = """diff --git a/tests/vector_store/test_performance_benchmarks.py b/tests/vector_store/test_performance_benchmarks.pyindex 85fb6386..808644d8 100644--- a/tests/vector_store/test_performance_benchmarks.py+++ b/tests/vector_store/test_performance_benchmarks.py@@ -17,17 +17,22 @@ import pytestimport numpy as npimport timeimport threading-import psutilimport osfrom concurrent.futures import ThreadPoolExecutor, as_completedfrom typing import Dict, List, Any+from semantica.vector_store import VectorStore, DecisionEmbeddingPipeline, HybridSimilarityCalculatorfrom semantica.context import DecisionContext, ContextRetrieverfrom semantica.vector_store.decision_vector_methods import (quick_decision, find_precedents, batch_decisions)+# psutil is not declared in any extra and is not in base-deps.txt.+# This module measures RSS through psutil in its benchmark helpers.+pytest.importorskip("psutil")+import psutil # noqa: E402+class TestPerformanceBenchmarks:"""Performance benchmarks for enhanced vector store."""