Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(deps): address review feedback on import exceptions and probes (#1513)
Zohaib Hassnain1ff890abdd7b
diff --git a/.gitignore b/.gitignoreindex 79197d52..bbfe449f 100644Binary files a/.gitignore and b/.gitignore differdiff --git a/integrations/google_adk/session_service.py b/integrations/google_adk/session_service.pyindex f5a0a02d..1c97e099 100644--- a/integrations/google_adk/session_service.py+++ b/integrations/google_adk/session_service.py@@ -21,6 +21,7 @@ from typing import Any, List, Optionaltry:from google.adk.events import Eventfrom google.adk.sessions import BaseSessionService, Session+try:from google.adk.sessions import ListSessionsResponseexcept ImportError:@@ -33,7 +34,7 @@ try:except ImportError:from google.adk.sessions.base_session_service import GetSessionConfigADK_AVAILABLE = True-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):ADK_AVAILABLE = FalseBaseSessionService = objectSession = Any@@ -162,10 +163,10 @@ class SemanticaSessionService(BaseSessionService):return {}def _find_session_node(- self,- app_name: str,- user_id: str,- session_id: str,+ self,+ app_name: str,+ user_id: str,+ session_id: str,) -> Optional[Any]:"""Find a session node by its logical ADK session ID."""expected_node_id = self._node_id(app_name, user_id, session_id)@@ -182,18 +183,18 @@ class SemanticaSessionService(BaseSessionService):metadata = node.get("metadata")if (- isinstance(metadata, dict)- and str(metadata.get("session_id")) == str(session_id)- and str(metadata.get("app_name")) == str(app_name)- and str(metadata.get("user_id")) == str(user_id)+ isinstance(metadata, dict)+ and str(metadata.get("session_id")) == str(session_id)+ and str(metadata.get("app_name")) == str(app_name)+ and str(metadata.get("user_id")) == str(user_id)):return nodereturn Nonedef _find_node_by_id(- self,- node_id: str,+ self,+ node_id: str,) -> Optional[Any]:"""Find a ContextGraph node by graph node ID."""for node in self.graph.find_nodes() or []:@@ -212,13 +213,13 @@ class SemanticaSessionService(BaseSessionService):data = SemanticaSessionService._safe_dict(event)for field in (- "id",- "invocation_id",- "author",- "timestamp",- "partial",- "turn_complete",- "branch",+ "id",+ "invocation_id",+ "author",+ "timestamp",+ "partial",+ "turn_complete",+ "branch",):if field not in data and hasattr(event, field):value = getattr(event, field)@@ -231,10 +232,10 @@ class SemanticaSessionService(BaseSessionService):return datadef _event_nodes(- self,- app_name: str,- user_id: str,- session_id: str,+ self,+ app_name: str,+ user_id: str,+ session_id: str,) -> List[Any]:"""Return all event nodes connected to a session."""session_node_id = self._node_id(app_name, user_id, session_id)@@ -275,8 +276,8 @@ class SemanticaSessionService(BaseSessionService):return str(timestamp)def _event_from_node(- self,- node: Any,+ self,+ node: Any,) -> Any:"""Reconstruct an ADK Event from its stored metadata.@@ -289,7 +290,7 @@ class SemanticaSessionService(BaseSessionService):if not event_id and graph_node_id:graph_node_id = str(graph_node_id)if graph_node_id.startswith("adk-event:"):- event_id = graph_node_id[len("adk-event:"):]+ event_id = graph_node_id[len("adk-event:") :]if event_id:properties["id"] = event_id@@ -310,11 +311,11 @@ class SemanticaSessionService(BaseSessionService):@staticmethoddef _session_kwargs(- app_name: str,- user_id: str,- session_id: str,- state: Optional[dict],- events: Optional[List[Any]],+ app_name: str,+ user_id: str,+ session_id: str,+ state: Optional[dict],+ events: Optional[List[Any]],) -> dict:"""Build kwargs for the ADK Session model."""return {@@ -326,8 +327,8 @@ class SemanticaSessionService(BaseSessionService):}def _session_from_node(- self,- node: Any,+ self,+ node: Any,) -> Session:"""Reconstruct an ADK Session from a ContextGraph node."""properties = self._node_properties(node)@@ -347,14 +348,14 @@ class SemanticaSessionService(BaseSessionService):# splitting on ':' after the prefix always yields# exactly 3 parts regardless of what characters the# original app_name/user_id/session_id contained.- parts = graph_node_id[len("adk-session:"):].split(":")+ parts = graph_node_id[len("adk-session:") :].split(":")if len(parts) == 3:decoded = [urllib.parse.unquote(part) for part in parts]app_name = app_name or decoded[0]user_id = user_id or decoded[1]session_id = decoded[2]else:- session_id = graph_node_id[len("adk-session:"):]+ session_id = graph_node_id[len("adk-session:") :]else:session_id = graph_node_id@@ -383,12 +384,12 @@ class SemanticaSessionService(BaseSessionService):# ------------------------------------------------------------------async def create_session(- self,- *,- app_name: str,- user_id: str,- state: Optional[dict[str, Any]] = None,- session_id: Optional[str] = None,+ self,+ *,+ app_name: str,+ user_id: str,+ state: Optional[dict[str, Any]] = None,+ session_id: Optional[str] = None,) -> Session:"""Create and persist an ADK session."""return await asyncio.to_thread(@@ -396,11 +397,11 @@ class SemanticaSessionService(BaseSessionService):)def _create_session_sync(- self,- app_name: str,- user_id: str,- state: Optional[dict[str, Any]],- session_id: Optional[str],+ self,+ app_name: str,+ user_id: str,+ state: Optional[dict[str, Any]],+ session_id: Optional[str],) -> Session:with self._lock:session_id = session_id or str(uuid.uuid4())@@ -430,12 +431,12 @@ class SemanticaSessionService(BaseSessionService):)async def get_session(- self,- *,- app_name: str,- user_id: str,- session_id: str,- config: Optional[GetSessionConfig] = None,+ self,+ *,+ app_name: str,+ user_id: str,+ session_id: str,+ config: Optional[GetSessionConfig] = None,) -> Optional[Session]:"""Retrieve an ADK session from ContextGraph."""return await asyncio.to_thread(@@ -443,11 +444,11 @@ class SemanticaSessionService(BaseSessionService):)def _get_session_sync(- self,- app_name: str,- user_id: str,- session_id: str,- config: Optional[GetSessionConfig],+ self,+ app_name: str,+ user_id: str,+ session_id: str,+ config: Optional[GetSessionConfig],) -> Optional[Session]:with self._lock:node = self._find_session_node(app_name, user_id, session_id)@@ -468,7 +469,7 @@ class SemanticaSessionService(BaseSessionService):# trims the already-built Session object.if config:if config.num_recent_events:- session.events = session.events[-config.num_recent_events:]+ session.events = session.events[-config.num_recent_events :]if config.after_timestamp:i = len(session.events) - 1while i >= 0:@@ -476,14 +477,14 @@ class SemanticaSessionService(BaseSessionService):breaki -= 1if i >= 0:- session.events = session.events[i + 1:]+ session.events = session.events[i + 1 :]return sessionasync def append_event(- self,- session: Session,- event: Event,+ self,+ session: Session,+ event: Event,) -> Event:"""Persist an ADK event and associate it with a session."""# ADK's own base implementation is a no-op for partial/streaming@@ -510,8 +511,13 @@ class SemanticaSessionService(BaseSessionService):# Verify cross-tenant securityproperties = self._node_properties(session_node)- if properties.get("app_name") != app_name or properties.get("user_id") != user_id:- raise ValueError("Cross-tenant session write denied: app_name or user_id mismatch.")+ if (+ properties.get("app_name") != app_name+ or properties.get("user_id") != user_id+ ):+ raise ValueError(+ "Cross-tenant session write denied: app_name or user_id mismatch."+ )# Apply ADK in-memory event and state delta semantics. This# runs inside asyncio.to_thread's worker thread, which has no@@ -553,11 +559,11 @@ class SemanticaSessionService(BaseSessionService):)async def delete_session(- self,- *,- app_name: str,- user_id: str,- session_id: str,+ self,+ *,+ app_name: str,+ user_id: str,+ session_id: str,) -> None:"""Delete a session and all of its graph-backed events."""await asyncio.to_thread(@@ -565,10 +571,10 @@ class SemanticaSessionService(BaseSessionService):)def _delete_session_sync(- self,- app_name: str,- user_id: str,- session_id: str,+ self,+ app_name: str,+ user_id: str,+ session_id: str,) -> None:with self._lock:session_node = self._find_session_node(app_name, user_id, session_id)@@ -590,9 +596,9 @@ class SemanticaSessionService(BaseSessionService):continueif (- edge.get("source") == session_node_id- and edge.get("type") == "HAS_EVENT"- and edge.get("target")+ edge.get("source") == session_node_id+ and edge.get("type") == "HAS_EVENT"+ and edge.get("target")):event_node_ids.append(str(edge["target"]))@@ -602,18 +608,18 @@ class SemanticaSessionService(BaseSessionService):self.graph.purge_node(session_node_id)async def list_sessions(- self,- *,- app_name: str,- user_id: Optional[str] = None,+ self,+ *,+ app_name: str,+ user_id: Optional[str] = None,) -> ListSessionsResponse:"""List sessions for an app, optionally scoped to one user."""return await asyncio.to_thread(self._list_sessions_sync, app_name, user_id)def _list_sessions_sync(- self,- app_name: str,- user_id: Optional[str],+ self,+ app_name: str,+ user_id: Optional[str],) -> ListSessionsResponse:with self._lock:sessions: List[Session] = []@@ -643,4 +649,4 @@ class SemanticaSessionService(BaseSessionService):__all__ = ["ADK_AVAILABLE","SemanticaSessionService",-]\ No newline at end of file+]diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.pyindex e7859604..bba55644 100644--- a/semantica/ingest/__init__.py+++ b/semantica/ingest/__init__.py@@ -133,7 +133,11 @@ import importlibfrom typing import TYPE_CHECKING, Any, Dict, Tupleif TYPE_CHECKING:- from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor+ from .salesforce_ingestor import (+ SalesforceConnector,+ SalesforceData,+ SalesforceIngestor,+ )from .config import IngestConfig, ingest_configfrom .file_ingestor import (@@ -296,13 +300,55 @@ def __getattr__(name: str) -> Any:module_name, attr_name = _LAZY_EXPORTS[name]try:module = importlib.import_module(module_name, __name__)- except ModuleNotFoundError as exc:+ except (ImportError, OSError) as exc:message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)missing_name = getattr(exc, "name", None)- if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce", "lxml"}:+ if message and (+ missing_name is None+ or any(+ pkg in missing_name+ for pkg in ("git", "bs4", "pyarrow", "simple_salesforce", "lxml")+ )+ ):raise ImportError(message) from excraise+ # Guard against backends whose modules imported cleanly with dependencies+ # set to None; ensure probe imports (e.g. try: from semantica.ingest import ...)+ # fail at import time rather than postponing failure to construction time.+ if module_name == ".repo_ingestor" and name in {"RepoIngestor"}:+ if getattr(module, "git", None) is None:+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)++ if module_name == ".xml_ingestor" and name in {"XMLIngestor"}:+ if getattr(module, "etree", None) is None:+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)++ if module_name == ".parquet_ingestor" and name in {"ParquetIngestor"}:+ if not getattr(module, "PARQUET_AVAILABLE", True):+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)++ if module_name == ".arrow_ingestor" and name in {"ArrowIngestor"}:+ if not getattr(module, "ARROW_AVAILABLE", True):+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)++ if module_name == ".salesforce_ingestor" and name in {+ "SalesforceIngestor",+ "SalesforceConnector",+ }:+ if not getattr(module, "SALESFORCE_AVAILABLE", True):+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)+ if message:+ raise ImportError(message)+value = getattr(module, attr_name)globals()[name] = valuereturn valuediff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.pyindex 3736288b..7bd5684d 100644--- a/semantica/ingest/methods.py+++ b/semantica/ingest/methods.py@@ -193,6 +193,7 @@ def _is_scp_like_repo_source(source: str) -> bool:"""Return True for scp-like SSH remotes (``user@host:path``)."""return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))+if TYPE_CHECKING:from .api_ingestor import APIDatafrom .arrow_ingestor import ArrowData@@ -252,7 +253,12 @@ def ingest_file(if custom_method and custom_method != ingest_file:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -318,21 +324,18 @@ def ingest_parquet(if custom_method and custom_method != ingest_parquet:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return resulttry:- try:- from .parquet_ingestor import ParquetIngestor- except ModuleNotFoundError as exc:- if _is_missing_dependency(exc, "pyarrow"):- raise _missing_optional_dependency(- "Parquet ingestion",- "pyarrow",- ) from exc- raise+ from .parquet_ingestor import ParquetIngestorconfig = ingest_config.get_method_config("parquet")config.update(kwargs)@@ -397,21 +400,18 @@ def ingest_arrow(if custom_method and custom_method != ingest_arrow:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return resulttry:- try:- from .arrow_ingestor import ArrowIngestor- except ModuleNotFoundError as exc:- if _is_missing_dependency(exc, "pyarrow"):- raise _missing_optional_dependency(- "Arrow ingestion",- "pyarrow",- ) from exc- raise+ from .arrow_ingestor import ArrowIngestorconfig = ingest_config.get_method_config("arrow")config.update(kwargs)@@ -481,7 +481,12 @@ def ingest_xml(if custom_method and custom_method != ingest_xml:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -491,7 +496,10 @@ def ingest_xml(config = ingest_config.get_method_config("xml")config.update(kwargs)- ingestor = XMLIngestor(**config)+ try:+ ingestor = XMLIngestor(**config)+ except ImportError as exc:+ raise _missing_optional_dependency("XML ingestion", "lxml") from excdef _run_single(path: Union[str, Path],@@ -511,6 +519,8 @@ def ingest_xml(return _run_single(source_path)+ except ConfigurationError:+ raiseexcept Exception as e:logger.error(f"Failed to ingest XML: {e}")raise@@ -545,7 +555,12 @@ def ingest_web(if custom_method and custom_method != ingest_web:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -635,7 +650,12 @@ def ingest_public_api(if custom_method and custom_method != ingest_public_api:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -722,7 +742,12 @@ def ingest_feed(if custom_method and custom_method != ingest_feed:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -791,7 +816,12 @@ def ingest_stream(if custom_method and custom_method != ingest_stream:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -868,20 +898,18 @@ def ingest_repository(if custom_method and custom_method != ingest_repository:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return resulttry:- try:- from .repo_ingestor import RepoIngestor- except ModuleNotFoundError as exc:- if _is_missing_dependency(exc, "git"):- raise _missing_optional_dependency(- "Repository ingestion", "GitPython"- ) from exc- raise+ from .repo_ingestor import RepoIngestor# Get configconfig = ingest_config.get_method_config("repo")@@ -945,7 +973,12 @@ def ingest_email(if custom_method and custom_method != ingest_email:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -1024,7 +1057,12 @@ def ingest_ontology(if custom_method and custom_method != ingest_ontology:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -1090,7 +1128,12 @@ def ingest_database(if custom_method and custom_method != ingest_database:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -1238,105 +1281,122 @@ def ingest_salesforce(if custom_method and custom_method != ingest_salesforce:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source,- fallback_on_custom_error=fallback, **kwargs,+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return resulttry:from .salesforce_ingestor import SalesforceIngestor- except ModuleNotFoundError as exc:- if _is_missing_dependency(exc, "simple_salesforce"):++ # Unpack credential dict (if given); everything else stays in kwargs.+ creds: Dict[str, Any] = {}+ if source is not None:+ if not isinstance(source, dict):+ raise ProcessingError(+ "ingest_salesforce() source must be a credential dict or None. "+ "Pass sobject_name / soql as keyword arguments."+ )+ creds = dict(source)++ # Merge any ingest_config method config under "salesforce".+ # get_method_config() now returns a copy, so this dict is safe to mutate.+ # We build the final connector config in order of increasing priority:+ # 1. base method config (lowest — global defaults set by operator)+ # 2. per-call credential dict supplied via `source`+ # 3. per-call connector params supplied as kwargs+ # Credentials are extracted from kwargs and removed so they don't also+ # flow into the ingest method call (which doesn't understand them).+ _CONNECTOR_PARAMS = frozenset(+ {+ "username",+ "password",+ "security_token",+ "domain",+ "instance_url",+ "session_id",+ "api_version",+ }+ )+ connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}+ for k in _CONNECTOR_PARAMS:+ kwargs.pop(k, None)++ # Build a fresh per-call config dict — never mutate the global store.+ config: Dict[str, Any] = {+ **ingest_config.get_method_config("salesforce"), # base (already a copy)+ **creds, # source dict credentials+ **connector_kwargs, # kwarg credentials+ }++ try:+ ingestor = SalesforceIngestor(**config)+ except ImportError as exc:raise _missing_optional_dependency("Salesforce ingestion", "simple-salesforce") from exc- raise- # Unpack credential dict (if given); everything else stays in kwargs.- creds: Dict[str, Any] = {}- if source is not None:- if not isinstance(source, dict):- raise ProcessingError(- "ingest_salesforce() source must be a credential dict or None. "- "Pass sobject_name / soql as keyword arguments."- )- creds = dict(source)-- # Merge any ingest_config method config under "salesforce".- # get_method_config() now returns a copy, so this dict is safe to mutate.- # We build the final connector config in order of increasing priority:- # 1. base method config (lowest — global defaults set by operator)- # 2. per-call credential dict supplied via `source`- # 3. per-call connector params supplied as kwargs- # Credentials are extracted from kwargs and removed so they don't also- # flow into the ingest method call (which doesn't understand them).- _CONNECTOR_PARAMS = frozenset({- "username", "password", "security_token", "domain",- "instance_url", "session_id", "api_version",- })- connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}- for k in _CONNECTOR_PARAMS:- kwargs.pop(k, None)-- # Build a fresh per-call config dict — never mutate the global store.- config: Dict[str, Any] = {- **ingest_config.get_method_config("salesforce"), # base (already a copy)- **creds, # source dict credentials- **connector_kwargs, # kwarg credentials- }-- ingestor = SalesforceIngestor(**config)-- if method == "sobject":- sobject_name = kwargs.pop("sobject_name", None)- if not sobject_name:- raise ProcessingError(- "ingest_salesforce() with method='sobject' requires "- "sobject_name keyword argument."- )- return ingestor.ingest_sobject(sobject_name, **kwargs)+ if method == "sobject":+ sobject_name = kwargs.pop("sobject_name", None)+ if not sobject_name:+ raise ProcessingError(+ "ingest_salesforce() with method='sobject' requires "+ "sobject_name keyword argument."+ )+ return ingestor.ingest_sobject(sobject_name, **kwargs)- elif method == "query":- soql = kwargs.pop("soql", None)- if not soql:- raise ProcessingError(- "ingest_salesforce() with method='query' requires "- "soql keyword argument."- )- return ingestor.ingest_query(soql, **kwargs)+ elif method == "query":+ soql = kwargs.pop("soql", None)+ if not soql:+ raise ProcessingError(+ "ingest_salesforce() with method='query' requires "+ "soql keyword argument."+ )+ return ingestor.ingest_query(soql, **kwargs)- elif method == "list_sobjects":- return ingestor.list_sobjects()+ elif method == "list_sobjects":+ return ingestor.list_sobjects()- elif method == "schema":- sobject_name = kwargs.pop("sobject_name", None)- if not sobject_name:- raise ProcessingError(- "ingest_salesforce() with method='schema' requires "- "sobject_name keyword argument."+ elif method == "schema":+ sobject_name = kwargs.pop("sobject_name", None)+ if not sobject_name:+ raise ProcessingError(+ "ingest_salesforce() with method='schema' requires "+ "sobject_name keyword argument."+ )+ return ingestor.get_sobject_schema(sobject_name)++ elif method == "documents":+ sobject_name = kwargs.pop("sobject_name", None)+ if not sobject_name:+ raise ProcessingError(+ "ingest_salesforce() with method='documents' requires "+ "sobject_name keyword argument."+ )+ id_field = kwargs.pop("id_field", "Id")+ text_fields = kwargs.pop("text_fields", None)+ data = ingestor.ingest_sobject(sobject_name, **kwargs)+ return ingestor.export_as_documents(+ data, id_field=id_field, text_fields=text_fields)- return ingestor.get_sobject_schema(sobject_name)- elif method == "documents":- sobject_name = kwargs.pop("sobject_name", None)- if not sobject_name:+ else:raise ProcessingError(- "ingest_salesforce() with method='documents' requires "- "sobject_name keyword argument."+ f"Unknown ingest_salesforce method: {method!r}. "+ "Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "+ "'documents'.")- id_field = kwargs.pop("id_field", "Id")- text_fields = kwargs.pop("text_fields", None)- data = ingestor.ingest_sobject(sobject_name, **kwargs)- return ingestor.export_as_documents(data, id_field=id_field,- text_fields=text_fields)- else:- raise ProcessingError(- f"Unknown ingest_salesforce method: {method!r}. "- "Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "- "'documents'."- )+ except ConfigurationError:+ raise+ except Exception as e:+ logger.error(f"Failed to ingest salesforce: {e}")+ raisedef ingest_mcp(@@ -1404,7 +1464,12 @@ def ingest_mcp(if custom_method and custom_method != ingest_mcp:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ source,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -1643,8 +1708,9 @@ def ingest(elif source_type == "mcp":return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}elif source_type == "salesforce":- return {"data": ingest_salesforce(sources,- method=method or "sobject", **kwargs)}+ return {+ "data": ingest_salesforce(sources, method=method or "sobject", **kwargs)+ }else:raise ProcessingError(f"Unknown source type: {source_type}")diff --git a/semantica/ingest/public_api_ingestor.py b/semantica/ingest/public_api_ingestor.pyindex 7df07be4..001c132e 100644--- a/semantica/ingest/public_api_ingestor.py+++ b/semantica/ingest/public_api_ingestor.py@@ -31,15 +31,18 @@ import requeststry:from lxml import etree as lxml_etree+_SAFE_XML_PARSER = lxml_etree.XMLParser(resolve_entities=False,no_network=True,recover=False,huge_tree=False,load_dtd=False,+ remove_comments=True,+ remove_pis=True,)_LXML_SYNTAX_ERRORS: Tuple[type, ...] = (lxml_etree.XMLSyntaxError,)-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):lxml_etree = None_SAFE_XML_PARSER = None_LXML_SYNTAX_ERRORS = ()@@ -47,7 +50,7 @@ except (ImportError, ModuleNotFoundError):try:from defusedxml import ElementTree as safe_xml_etreefrom defusedxml.common import DefusedXmlException-except ModuleNotFoundError: # pragma: no cover - fallback for minimal installs+except (ImportError, OSError): # pragma: no cover - fallback for minimal installssafe_xml_etree = Noneclass DefusedXmlException(Exception):@@ -449,7 +452,9 @@ class PublicAPIIngestor(RESTIngestor):APIData: Normalized public API response and metadata"""self._validate_endpoint(endpoint)- self._validate_no_auth_request(headers=headers, params=params, options=options, endpoint=endpoint)+ self._validate_no_auth_request(+ headers=headers, params=params, options=options, endpoint=endpoint+ )tracking_id = self.progress_tracker.start_tracking(file=endpoint,@@ -610,7 +615,9 @@ class PublicAPIIngestor(RESTIngestor):for endpoint in endpoints:try:results.append(- self.ingest_public_api(endpoint, method=method, **copy.deepcopy(options))+ self.ingest_public_api(+ endpoint, method=method, **copy.deepcopy(options)+ ))except Exception as exc:self.logger.warning(f"Failed to fetch public API {endpoint}: {exc}")@@ -781,6 +788,16 @@ class PublicAPIIngestor(RESTIngestor):xml_text.encode("utf-8"),parser=_SAFE_XML_PARSER,)+ for elem in root.iter():+ if (+ elem.tag is lxml_etree.Comment+ or elem.tag is lxml_etree.PI+ or getattr(elem.tag, "__name__", "")+ in ("Comment", "ProcessingInstruction", "PI")+ ):+ continue+ if callable(elem.tag) or not isinstance(elem.tag, str):+ raise ProcessingError("Failed to parse XML public API response")else:raise ProcessingError("XML parsing requires 'defusedxml' or 'lxml'. "@@ -789,7 +806,17 @@ class PublicAPIIngestor(RESTIngestor):return self._element_to_dict(root)def _element_to_dict(self, element: Any) -> Dict[str, Any]:- children = [self._element_to_dict(child) for child in list(element)]+ children = [+ self._element_to_dict(child)+ for child in list(element)+ if not (+ callable(child.tag)+ or (+ lxml_etree is not None+ and (child.tag is lxml_etree.Comment or child.tag is lxml_etree.PI)+ )+ )+ ]return {"tag": self._strip_namespace(element.tag),"attributes": {@@ -800,7 +827,9 @@ class PublicAPIIngestor(RESTIngestor):"children": children,}- def _strip_namespace(self, value: str) -> str:+ def _strip_namespace(self, value: Any) -> str:+ if not isinstance(value, str):+ return str(value)if value.startswith("{") and "}" in value:return value.split("}", 1)[1]return valuediff --git a/semantica/ingest/repo_ingestor.py b/semantica/ingest/repo_ingestor.pyindex 5996a9d7..1f8a906f 100644--- a/semantica/ingest/repo_ingestor.py+++ b/semantica/ingest/repo_ingestor.py@@ -28,6 +28,7 @@ Example Usage:Author: Semantica ContributorsLicense: MIT"""+from __future__ import annotationsimport ipaddress@@ -47,7 +48,7 @@ from urllib.parse import urlparsetry:import git-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):git = Nonefrom ..utils.exceptions import ProcessingError, ValidationError@@ -61,9 +62,7 @@ ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")-_ENV_VAR_TOKEN_RE = re.compile(- r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"-)+_ENV_VAR_TOKEN_RE = re.compile(r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)")# Short-lived DNS cache for host validation. This reduces repeated lookups but# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —# network egress controls remain recommended.@@ -599,10 +598,7 @@ class RepoIngestor:networks). Those addresses are not SSRF-sensitive."""return bool(- ip.is_private- or ip.is_loopback- or ip.is_link_local- or ip.is_unspecified+ ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_unspecified)@staticmethod@@ -629,9 +625,7 @@ class RepoIngestor:# or hanging lookup for one host cannot stall cache access for# concurrent lookups of other hosts.try:- addrinfos = socket.getaddrinfo(- host, None, type=socket.SOCK_STREAM- )+ addrinfos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)except socket.gaierror as exc:raise ValidationError(f"Cannot resolve repository host {host!r}: {exc}"@@ -804,9 +798,7 @@ class RepoIngestor:f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}")if not parsed.netloc or not host:- raise ValidationError(- f"Repository URL must include a host: {repo_url}"- )+ raise ValidationError(f"Repository URL must include a host: {repo_url}")RepoIngestor._validate_repo_host(host)@@ -821,9 +813,7 @@ class RepoIngestor:"include_extensions","max_depth",}- candidate = {- k: v for k, v in options.items() if k not in non_git_options- }+ candidate = {k: v for k, v in options.items() if k not in non_git_options}unsafe = set(candidate) - ALLOWED_CLONE_OPTIONSif unsafe:raise ValidationError(@@ -902,9 +892,7 @@ class RepoIngestor:if "include_extensions" in options:# Normalize extensions to include dot prefixexts = options["include_extensions"]- normalized_exts = [- e if e.startswith(".") else f".{e}" for e in exts- ]+ normalized_exts = [e if e.startswith(".") else f".{e}" for e in exts]file_filters["extensions"] = normalized_exts# Process code files@@ -1120,6 +1108,7 @@ class RepoIngestor:def cleanup(self):"""Cleanup temporary repository files."""if self.temp_dir and os.path.exists(self.temp_dir):+def onexc(func, path, exc_info):"""Error handler for shutil.rmtree.@@ -1132,6 +1121,7 @@ class RepoIngestor:Usage : shutil.rmtree(path, onerror=onexc)"""import stat+if not os.access(path, os.W_OK):# Is the error an access error ?os.chmod(path, stat.S_IWUSR)diff --git a/semantica/ingest/xml_ingestor.py b/semantica/ingest/xml_ingestor.pyindex 9f3be0ed..84f4067a 100644--- a/semantica/ingest/xml_ingestor.py+++ b/semantica/ingest/xml_ingestor.py@@ -26,7 +26,7 @@ from typing import Any, Dict, List, Optional, Tuple, Uniontry:from lxml import etree-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):etree = Nonefrom ..utils.constants import FILE_SIZE_LIMITS@@ -76,7 +76,7 @@ class XMLIngestor:"""self.logger = get_logger("xml_ingestor")if etree is None:- raise ProcessingError(+ raise ImportError("lxml is required for XMLIngestor. ""Install it with: pip install 'semantica[documents]'")diff --git a/semantica/parse/docx_parser.py b/semantica/parse/docx_parser.pyindex 390e8ba8..9a8f1d50 100644--- a/semantica/parse/docx_parser.py+++ b/semantica/parse/docx_parser.py@@ -39,7 +39,7 @@ try:from docx.oxml.text.paragraph import CT_Pfrom docx.table import Tablefrom docx.text.paragraph import Paragraph-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):Document = NoneDocxDocument = NoneCT_Tbl = None@@ -92,7 +92,9 @@ class DOCXParser:self.config = configself.progress_tracker = get_progress_tracker()- def parse(self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options) -> Dict[str, Any]:+ def parse(+ self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options+ ) -> Dict[str, Any]:"""Parse DOCX document.diff --git a/semantica/parse/excel_parser.py b/semantica/parse/excel_parser.pyindex c3ba3d91..c1b3bab5 100644--- a/semantica/parse/excel_parser.py+++ b/semantica/parse/excel_parser.py@@ -33,9 +33,10 @@ from pathlib import Pathfrom typing import Any, Dict, List, Optional, Unionimport pandas as pd+try:from openpyxl import load_workbook-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):load_workbook = Nonefrom ..utils.exceptions import ProcessingError, ValidationErrordiff --git a/semantica/parse/html_parser.py b/semantica/parse/html_parser.pyindex 8cf4007e..b642f381 100644--- a/semantica/parse/html_parser.py+++ b/semantica/parse/html_parser.py@@ -35,7 +35,7 @@ from urllib.parse import urljointry:from bs4 import BeautifulSoup-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):BeautifulSoup = Nonefrom ..utils.exceptions import ProcessingError, ValidationErrordiff --git a/semantica/parse/methods.py b/semantica/parse/methods.pyindex 8dd74824..9eb44ae7 100644--- a/semantica/parse/methods.py+++ b/semantica/parse/methods.py@@ -123,7 +123,6 @@ Example Usage:from pathlib import Pathfrom typing import Any, Callable, Dict, List, Optional, Union-from ..utils.exceptions import ConfigurationError, ProcessingErrorfrom ..utils.logging import get_loggerfrom ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_methodfrom .code_parser import CodeParser@@ -181,7 +180,13 @@ def parse_document(if custom_method and custom_method != parse_document:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, file_type, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ file_type,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -248,7 +253,8 @@ def parse_document_docling(# Register Docling methodtry:- from .docling_parser import DoclingParser+ from . import docling_parser # noqa: F401+method_registry.register("document", "docling", parse_document_docling)except (ImportError, OSError):# Docling not available, skip registration@@ -292,7 +298,14 @@ def parse_web_content(if custom_method and custom_method != parse_web_content:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, content, content_type, base_url, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ content,+ content_type,+ base_url,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -348,7 +361,13 @@ def parse_structured_data(if custom_method and custom_method != parse_structured_data:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, data, data_format, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ data,+ data_format,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -395,7 +414,12 @@ def parse_email(if custom_method and custom_method != parse_email:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, email_content, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ email_content,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -445,7 +469,13 @@ def parse_code(if custom_method and custom_method != parse_code:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, language, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ language,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -498,7 +528,13 @@ def parse_media(if custom_method and custom_method != parse_media:fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, media_type, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ media_type,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -544,7 +580,12 @@ def parse_pdf(if custom_method and custom_method not in (parse_pdf, parse_document):fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -588,7 +629,12 @@ def parse_docx(if custom_method and custom_method not in (parse_docx, parse_document):fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -631,7 +677,12 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) -if custom_method and custom_method not in (parse_json, parse_structured_data):fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -678,7 +729,13 @@ def parse_csv(if custom_method and custom_method not in (parse_csv, parse_structured_data):fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, delimiter, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ delimiter,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -717,7 +774,12 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) ->if custom_method and custom_method not in (parse_xml, parse_structured_data):fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return result@@ -765,7 +827,12 @@ def parse_image(if custom_method and custom_method not in (parse_image, parse_media):fallback = kwargs.pop("fallback_on_custom_error", False)result = call_custom_method(- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs+ logger,+ method,+ custom_method,+ file_path,+ fallback_on_custom_error=fallback,+ **kwargs,)if result is not CUSTOM_METHOD_FELL_BACK:return resultdiff --git a/semantica/parse/web_parser.py b/semantica/parse/web_parser.pyindex 0ea4f8e7..2a4cc3be 100644--- a/semantica/parse/web_parser.py+++ b/semantica/parse/web_parser.py@@ -35,7 +35,7 @@ from urllib.parse import urljoin, urlparsetry:from bs4 import BeautifulSoup-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):BeautifulSoup = Nonefrom ..utils.exceptions import ProcessingError, ValidationErrordiff --git a/semantica/parse/xml_parser.py b/semantica/parse/xml_parser.pyindex 5088577c..d12f430b 100644--- a/semantica/parse/xml_parser.py+++ b/semantica/parse/xml_parser.py@@ -36,7 +36,7 @@ from typing import Any, Dict, List, Optional, Uniontry:from lxml import etree-except (ImportError, ModuleNotFoundError):+except (ImportError, OSError):etree = Nonefrom ..utils.exceptions import ProcessingError, ValidationError@@ -156,7 +156,7 @@ class XMLParser:self, xml_string: str, source: str, options: Dict[str, Any]) -> XMLData:"""Parse XML using lxml."""- parser = etree.XMLParser(remove_blank_text=True)+ parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)root = etree.fromstring(xml_string.encode("utf-8"), parser)# Extract namespaces@@ -197,8 +197,10 @@ class XMLParser:metadata={"source": source, "engine": "etree"},)- def _element_to_xml_element(self, element) -> XMLElement:+ def _element_to_xml_element(self, element) -> Optional[XMLElement]:"""Convert lxml element to XMLElement."""+ if not hasattr(element, "tag") or not isinstance(element.tag, str):+ return Nonetag = element.tagif "}" in tag:namespace, tag = tag.split("}", 1)@@ -215,12 +217,18 @@ class XMLParser:# Process childrenfor child in element:- xml_elem.children.append(self._element_to_xml_element(child))+ child_elem = self._element_to_xml_element(child)+ if child_elem is not None:+ xml_elem.children.append(child_elem)return xml_elem- def _etree_element_to_xml_element(self, element: ET.Element) -> XMLElement:+ def _etree_element_to_xml_element(+ self, element: ET.Element+ ) -> Optional[XMLElement]:"""Convert ElementTree element to XMLElement."""+ if not hasattr(element, "tag") or not isinstance(element.tag, str):+ return Nonetag = element.tagif "}" in tag:namespace, tag = tag.split("}", 1)@@ -237,7 +245,9 @@ class XMLParser:# Process childrenfor child in element:- xml_elem.children.append(self._etree_element_to_xml_element(child))+ child_elem = self._etree_element_to_xml_element(child)+ if child_elem is not None:+ xml_elem.children.append(child_elem)return xml_elem@@ -267,15 +277,21 @@ class XMLParser:xml_string = (file_pathif isinstance(file_path, str) and not Path(file_path).exists()- else Path(file_path).read_text()+ else Path(file_path).read_text(encoding="utf-8"))- root = etree.fromstring(xml_string.encode("utf-8"))+ parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)+ root = etree.fromstring(xml_string.encode("utf-8"), parser=parser)# Register namespaces for XPathnamespaces = xml_data.namespaceselements = root.xpath(xpath, namespaces=namespaces)- return [self._element_to_xml_element(elem) for elem in elements]+ results = []+ for elem in elements:+ xml_elem = self._element_to_xml_element(elem)+ if xml_elem is not None:+ results.append(xml_elem)+ return resultsdef extract_by_tag(self, file_path: Union[str, Path], tag_name: str, **optionsdiff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.pyindex 5e76e8f6..64d9ebcf 100644--- a/semantica/semantic_extract/providers.py+++ b/semantica/semantic_extract/providers.py@@ -61,7 +61,7 @@ Example Usage:>>> from semantica.semantic_extract.providers import create_provider>>> provider = create_provider("openai", model="gpt-4")>>> response = provider.generate("Extract entities from: Apple Inc. was founded in 1976.")- >>>+ >>>>>> loader = HuggingFaceModelLoader(device="cuda")>>> ner_model = loader.load_ner_model("dslim/bert-base-NER")@@ -70,7 +70,6 @@ License: MIT"""import json-import osimport timefrom typing import Any, Dict, List, Optional, Union, Type@@ -121,7 +120,7 @@ class BaseProvider:"""Extract and parse JSON from text, supporting objects and lists."""if not text:raise ProcessingError("Empty response from LLM")-+# Clean up text - remove markdown code blocks if presentcleaned_text = text.strip()if "```json" in cleaned_text:@@ -131,8 +130,9 @@ class BaseProvider:blocks = cleaned_text.split("```")for block in blocks:block = block.strip()- if (block.startswith("{") and block.endswith("}")) or \- (block.startswith("[") and block.endswith("]")):+ if (block.startswith("{") and block.endswith("}")) or (+ block.startswith("[") and block.endswith("]")+ ):cleaned_text = blockbreak@@ -150,22 +150,22 @@ class BaseProvider:# Try to find JSON boundaries (outermost { } or [ ])start_obj = cleaned_text.find("{")start_list = cleaned_text.find("[")-+# Determine which one starts firststart = -1if start_obj >= 0 and (start_list < 0 or start_obj < start_list):start = start_objelif start_list >= 0:start = start_list-+if start >= 0:# Find the corresponding endend_obj = cleaned_text.rfind("}")end_list = cleaned_text.rfind("]")end = max(end_obj, end_list)-+if end > start:- candidate = cleaned_text[start:end+1]+ candidate = cleaned_text[start : end + 1]try:return json.loads(candidate)except json.JSONDecodeError:@@ -176,78 +176,88 @@ class BaseProvider:# Last resort: if it's truncated, try to close itif candidate.endswith("..."):candidate = candidate[:-3].strip()-+# Simple attempt to close unclosed structuresopen_braces = candidate.count("{") - candidate.count("}")open_brackets = candidate.count("[") - candidate.count("]")-+fixed_candidate = candidateif open_braces > 0:fixed_candidate += "}" * open_bracesif open_brackets > 0:fixed_candidate += "]" * open_brackets-+try:return json.loads(fix_json(fixed_candidate))except json.JSONDecodeError as e:- raise ProcessingError(f"Failed to parse JSON from LLM response after cleaning: {e}")-- raise ProcessingError(f"No valid JSON structure found in response. Preview: {text[:100]}...")+ raise ProcessingError(+ f"Failed to parse JSON from LLM response after cleaning: {e}"+ )++ raise ProcessingError(+ f"No valid JSON structure found in response. Preview: {text[:100]}..."+ )- def generate_structured(self, prompt: str, max_retries: int = 3, **kwargs) -> Union[dict, list]:+ def generate_structured(+ self, prompt: str, max_retries: int = 3, **kwargs+ ) -> Union[dict, list]:"""Generate structured output with retry logic."""last_error = Noneimport time-+for attempt in range(max_retries):try:# Add explicit JSON instruction if not presentstructured_prompt = promptif "JSON" not in prompt:- structured_prompt = f"{prompt}\n\nReturn the response as valid JSON only."-+ structured_prompt = (+ f"{prompt}\n\nReturn the response as valid JSON only."+ )+content = self.generate(structured_prompt, **kwargs)result = self._parse_json(content)-+# Basic validation: ensure it's not empty if we expect dataif not result and attempt < max_retries - 1:- self.logger.warning(f"Empty structured response (attempt {attempt + 1}/{max_retries}). Retrying...")+ self.logger.warning(+ f"Empty structured response (attempt {attempt + 1}/{max_retries}). Retrying..."+ )continue-+return result-+except (ProcessingError, Exception) as e:last_error = eif attempt < max_retries - 1:- wait_time = (attempt + 1) * 2 # Simple backoff- self.logger.warning(f"Extraction error (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s...")+ wait_time = (attempt + 1) * 2 # Simple backoff+ self.logger.warning(+ f"Extraction error (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s..."+ )time.sleep(wait_time)else:- self.logger.error(f"Structured generation failed after {max_retries} attempts: {e}")-+ self.logger.error(+ f"Structured generation failed after {max_retries} attempts: {e}"+ )+if last_error:raise ProcessingError(f"Failed to generate structured output: {last_error}")return []def generate_typed(- self,- prompt: str,- schema: Type[BaseModel],- max_retries: int = 3,- **kwargs+ self, prompt: str, schema: Type[BaseModel], max_retries: int = 3, **kwargs) -> BaseModel:"""Generate structured output validated against a Pydantic schema.Uses instructor if available and supported for the provider, otherwise falls back to a repair loop."""provider_name = self.__class__.__name__-+# Try using instructor first if availableif instructor:try:client = Nonemode = instructor.Mode.TOOLS # Default mode-+if provider_name == "OpenAIProvider" and self.client:custom_base_url = getattr(self, "base_url", None)if custom_base_url:@@ -255,7 +265,9 @@ class BaseProvider:# supported by third-party servers (Qwen, LLaMA gateways, etc.).# Mode.JSON asks the model to return plain JSON and is broadly# supported across all OpenAI-compatible APIs.- client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)+ client = instructor.from_openai(+ self.client, mode=instructor.Mode.JSON+ )elif hasattr(instructor, "from_provider"):try:client = instructor.from_provider(@@ -270,8 +282,8 @@ class BaseProvider:if hasattr(instructor, "from_provider"):try:client = instructor.from_provider(- provider=f"anthropic/{kwargs.get('model', self.model)}",- api_key=self.api_key+ provider=f"anthropic/{kwargs.get('model', self.model)}",+ api_key=self.api_key,)except Exception:client = instructor.from_anthropic(self.client)@@ -281,26 +293,24 @@ class BaseProvider:if hasattr(instructor, "from_provider"):try:client = instructor.from_provider(- provider=f"gemini/{kwargs.get('model', self.model)}",- api_key=self.api_key+ provider=f"gemini/{kwargs.get('model', self.model)}",+ api_key=self.api_key,)except Exception:client = instructor.from_gemini(- self.client,- mode=instructor.Mode.GEMINI_JSON+ self.client, mode=instructor.Mode.GEMINI_JSON)else:client = instructor.from_gemini(- self.client,- mode=instructor.Mode.GEMINI_JSON+ self.client, mode=instructor.Mode.GEMINI_JSON)elif provider_name == "GroqProvider" and self.client:# Try using from_provider which is recommended for Groq in latest instructorif hasattr(instructor, "from_provider"):try:client = instructor.from_provider(- provider=f"groq/{kwargs.get('model', self.model)}",- api_key=self.api_key+ provider=f"groq/{kwargs.get('model', self.model)}",+ api_key=self.api_key,)except Exception:client = None@@ -308,21 +318,28 @@ class BaseProvider:if not client:# Try using from_groq if available (newer instructor versions)if hasattr(instructor, "from_groq"):- client = instructor.from_groq(self.client, mode=instructor.Mode.JSON)+ client = instructor.from_groq(+ self.client, mode=instructor.Mode.JSON+ )else:# Fallback: Create OpenAI client pointing to Groq# This avoids the "Client should be an instance of openai.OpenAI" warningtry:from openai import OpenAI+# Fix: Use self.api_key instead of self.client.api_keygroq_client = OpenAI(base_url="https://api.groq.com/openai/v1",api_key=self.api_key,)- client = instructor.from_openai(groq_client, mode=instructor.Mode.JSON)+ client = instructor.from_openai(+ groq_client, mode=instructor.Mode.JSON+ )except Exception:# Last resort: try passing the groq client directly- client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)+ client = instructor.from_openai(+ self.client, mode=instructor.Mode.JSON+ )elif provider_name == "OllamaProvider":# Try from_provider for Ollama if availableif hasattr(instructor, "from_provider"):@@ -332,21 +349,26 @@ class BaseProvider:)except Exception:client = None-+if not client:# Create OpenAI-compatible client for Ollamatry:from openai import OpenAI+# Ollama typically runs on localhost:11434/v1- base_url = getattr(self, "base_url", "http://localhost:11434")+ base_url = getattr(+ self, "base_url", "http://localhost:11434"+ )if not base_url.endswith("/v1"):base_url = f"{base_url.rstrip('/')}/v1"-+ollama_client = OpenAI(base_url=base_url,- api_key="ollama", # required but unused+ api_key="ollama", # required but unused+ )+ client = instructor.from_openai(+ ollama_client, mode=instructor.Mode.JSON)- client = instructor.from_openai(ollama_client, mode=instructor.Mode.JSON)except ImportError:passelif provider_name == "DeepSeekProvider" and self.client:@@ -355,7 +377,7 @@ class BaseProvider:try:client = instructor.from_provider(provider=f"deepseek/{kwargs.get('model', self.model)}",- api_key=self.api_key+ api_key=self.api_key,)except Exception:client = None@@ -364,15 +386,20 @@ class BaseProvider:# DeepSeek is OpenAI compatibletry:from openai import OpenAI+if isinstance(self.client, OpenAI):- client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)+ client = instructor.from_openai(+ self.client, mode=instructor.Mode.JSON+ )else:- # Try creating fresh client- ds_client = OpenAI(- api_key=self.api_key,- base_url="https://api.deepseek.com"- )- client = instructor.from_openai(ds_client, mode=instructor.Mode.JSON)+ # Try creating fresh client+ ds_client = OpenAI(+ api_key=self.api_key,+ base_url="https://api.deepseek.com",+ )+ client = instructor.from_openai(+ ds_client, mode=instructor.Mode.JSON+ )except Exception:pass@@ -383,7 +410,9 @@ class BaseProvider:# Format for litellm in instructor is litellm/model_nameprovider_model = kwargs.get("model", self.model)litellm_provider = f"litellm/{provider_model}"- client = instructor.from_provider(litellm_provider, api_key=self.api_key)+ client = instructor.from_provider(+ litellm_provider, api_key=self.api_key+ )except Exception:pass@@ -395,10 +424,26 @@ class BaseProvider:"messages": [{"role": "user", "content": prompt}],"response_model": schema,"max_retries": max_retries,- "temperature": kwargs.get("temperature") if kwargs.get("temperature") is not None else 0.1,+ "temperature": (+ kwargs.get("temperature")+ if kwargs.get("temperature") is not None+ else 0.1+ ),}- self._add_if_set(create_kwargs, kwargs, "max_tokens", "max_completion_tokens",- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "logit_bias", "user", "top_k")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "max_tokens",+ "max_completion_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "logit_bias",+ "user",+ "top_k",+ )if provider_name == "GroqProvider":create_kwargs["response_format"] = {"type": "json_object"}@@ -423,20 +468,27 @@ class BaseProvider:primary_err,exc_info=True,)- json_client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)+ json_client = instructor.from_openai(+ self.client, mode=instructor.Mode.JSON+ )# Build a clean kwargs dict for the Mode.JSON retry: drop# response_format (Mode.JSON handles schema differently)# but keep response_model/max_retries so instructor still# validates the typed output.retry_kwargs = {- k: v for k, v in create_kwargs.items()+ k: v+ for k, v in create_kwargs.items()if k != "response_format"}- response = json_client.chat.completions.create(**retry_kwargs)+ response = json_client.chat.completions.create(+ **retry_kwargs+ )else:raise- verbose_mode = kwargs.get("verbose", False) or self.config.get("verbose", False)+ verbose_mode = kwargs.get("verbose", False) or self.config.get(+ "verbose", False+ )if verbose_mode:self.logger.debug("[BaseProvider.generate_typed] Typed response received via instructor (%s).",@@ -453,14 +505,16 @@ class BaseProvider:# Fallback: Manual repair looplast_error = Nonecurrent_prompt = prompt-+for attempt in range(max_retries):try:# 1. Generate JSON – try structured mode first, then fall back to# plain generate() + parse. Custom gateways that reject# response_format=json_object would otherwise loop forever here.try:- json_result = self.generate_structured(current_prompt, max_retries=1, **kwargs)+ json_result = self.generate_structured(+ current_prompt, max_retries=1, **kwargs+ )except Exception as struct_err:self.logger.warning("generate_structured failed (%s); retrying with plain generate() + JSON parse.",@@ -469,96 +523,133 @@ class BaseProvider:)raw_content = self.generate(current_prompt, **kwargs)json_result = self._parse_json(raw_content)-+# 2. Validate with Schema# If the result is a list and schema expects a wrapper, or vice versa, we might need adjustment# But we assume the prompt asks for the correct structure matching the schema.-+# Special handling if schema is a wrapper but result is a list- if isinstance(json_result, list) and hasattr(schema, "entities") and "entities" in schema.model_fields:- # Auto-wrap for entities- json_result = {"entities": json_result}+ if (+ isinstance(json_result, list)+ and hasattr(schema, "entities")+ and "entities" in schema.model_fields+ ):+ # Auto-wrap for entities+ json_result = {"entities": json_result}# Handle categorized dictionary input (e.g. {"PERSON": ["Name"], "ORG": ["Corp"]})- elif isinstance(json_result, dict) and hasattr(schema, "entities") and "entities" in schema.model_fields:- # Check if it's NOT already in the correct format (i.e., missing "entities" key)- if "entities" not in json_result:- # Check if values are lists, suggesting categorized output- is_categorized = any(isinstance(v, list) for v in json_result.values())- if is_categorized:- flat_entities = []- for label, items in json_result.items():- if isinstance(items, list):- for item in items:- if isinstance(item, str):- flat_entities.append({"text": item, "label": label})- elif isinstance(item, dict):- # If it's already a dict but nested under label- item["label"] = label- flat_entities.append(item)- json_result = {"entities": flat_entities}+ elif (+ isinstance(json_result, dict)+ and hasattr(schema, "entities")+ and "entities" in schema.model_fields+ ):+ # Check if it's NOT already in the correct format (i.e., missing "entities" key)+ if "entities" not in json_result:+ # Check if values are lists, suggesting categorized output+ is_categorized = any(+ isinstance(v, list) for v in json_result.values()+ )+ if is_categorized:+ flat_entities = []+ for label, items in json_result.items():+ if isinstance(items, list):+ for item in items:+ if isinstance(item, str):+ flat_entities.append(+ {"text": item, "label": label}+ )+ elif isinstance(item, dict):+ # If it's already a dict but nested under label+ item["label"] = label+ flat_entities.append(item)+ json_result = {"entities": flat_entities}# Handle categorized dictionary input for relations (e.g. {"founded_by": [{"subject":..., "object":...}]})- elif isinstance(json_result, dict) and hasattr(schema, "relations") and "relations" in schema.model_fields:- if "relations" not in json_result:- is_categorized = any(isinstance(v, list) for v in json_result.values())- if is_categorized:- flat_relations = []- for label, items in json_result.items():- if isinstance(items, list):- for item in items:- if isinstance(item, dict):- # If predicate is missing, use the key as predicate- if "predicate" not in item:- item["predicate"] = label- flat_relations.append(item)- json_result = {"relations": flat_relations}+ elif (+ isinstance(json_result, dict)+ and hasattr(schema, "relations")+ and "relations" in schema.model_fields+ ):+ if "relations" not in json_result:+ is_categorized = any(+ isinstance(v, list) for v in json_result.values()+ )+ if is_categorized:+ flat_relations = []+ for label, items in json_result.items():+ if isinstance(items, list):+ for item in items:+ if isinstance(item, dict):+ # If predicate is missing, use the key as predicate+ if "predicate" not in item:+ item["predicate"] = label+ flat_relations.append(item)+ json_result = {"relations": flat_relations}# Handle categorized dictionary input for triplets- elif isinstance(json_result, dict) and hasattr(schema, "triplets") and "triplets" in schema.model_fields:- if "triplets" not in json_result:- is_categorized = any(isinstance(v, list) for v in json_result.values())- if is_categorized:- flat_triplets = []- for label, items in json_result.items():- if isinstance(items, list):- for item in items:- if isinstance(item, dict):- flat_triplets.append(item)- json_result = {"triplets": flat_triplets}-- elif isinstance(json_result, list) and hasattr(schema, "relations") and "relations" in schema.model_fields:- json_result = {"relations": json_result}- elif isinstance(json_result, list) and hasattr(schema, "triplets") and "triplets" in schema.model_fields:- json_result = {"triplets": json_result}+ elif (+ isinstance(json_result, dict)+ and hasattr(schema, "triplets")+ and "triplets" in schema.model_fields+ ):+ if "triplets" not in json_result:+ is_categorized = any(+ isinstance(v, list) for v in json_result.values()+ )+ if is_categorized:+ flat_triplets = []+ for label, items in json_result.items():+ if isinstance(items, list):+ for item in items:+ if isinstance(item, dict):+ flat_triplets.append(item)+ json_result = {"triplets": flat_triplets}++ elif (+ isinstance(json_result, list)+ and hasattr(schema, "relations")+ and "relations" in schema.model_fields+ ):+ json_result = {"relations": json_result}+ elif (+ isinstance(json_result, list)+ and hasattr(schema, "triplets")+ and "triplets" in schema.model_fields+ ):+ json_result = {"triplets": json_result}validated = schema.model_validate(json_result)return validated-+except ValidationError as e:last_error = eerror_summary = str(e)# Simplify error summary for the LLM# (You could parse e.errors() for a better message)-+if attempt < max_retries - 1:wait_time = (attempt + 1) * 1- self.logger.warning(f"Schema validation failed (attempt {attempt + 1}): {e}. Retrying with error feedback...")-+ self.logger.warning(+ f"Schema validation failed (attempt {attempt + 1}): {e}. Retrying with error feedback..."+ )+# Update prompt with error infocurrent_prompt = f"{prompt}\n\nPrevious response was invalid JSON or didn't match schema:\n{error_summary}\n\nPlease fix the errors and return valid JSON matching the schema."time.sleep(wait_time)else:self.logger.error(f"Typed generation failed validation: {e}")-+except Exception as e:last_error = eif attempt < max_retries - 1:- time.sleep(1)+ time.sleep(1)else:self.logger.error(f"Typed generation failed: {e}")- raise ProcessingError(f"Failed to generate typed output after {max_retries} attempts: {last_error}")+ raise ProcessingError(+ f"Failed to generate typed output after {max_retries} attempts: {last_error}"+ )+class OpenAIProvider(BaseProvider):"""OpenAI provider implementation."""@@ -594,6 +685,7 @@ class OpenAIProvider(BaseProvider):# Reject non-HTTP schemes (file://, ftp://, etc.) to prevent SSRF# when base_url originates from configuration rather than hardcoded values.from urllib.parse import urlparse+scheme = urlparse(self.base_url).schemeif scheme not in ("http", "https"):raise ValueError(@@ -630,8 +722,20 @@ class OpenAIProvider(BaseProvider):"model": kwargs.get("model", self.model),"messages": [{"role": "user", "content": prompt}],}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "logit_bias", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_completion_tokens",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "logit_bias",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)return response.choices[0].message.content@@ -651,8 +755,20 @@ class OpenAIProvider(BaseProvider):if not self.base_url:create_kwargs["response_format"] = {"type": "json_object"}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "logit_bias", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_completion_tokens",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "logit_bias",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)try:@@ -679,6 +795,7 @@ class GeminiProvider(BaseProvider):def _init_client(self):try:from google import genai as new_genai+if self.api_key:self.client = new_genai.Client(api_key=self.api_key)self._use_new_genai = True@@ -687,13 +804,16 @@ class GeminiProvider(BaseProvider):passtry:import google.generativeai as old_genai+if self.api_key:old_genai.configure(api_key=self.api_key)self.client = old_genai.GenerativeModel(self.model)self._use_new_genai = Falseexcept Exception:self.client = None- self.logger.warning("Gemini SDK not installed. Install with: pip install semantica[llm-gemini]")+ self.logger.warning(+ "Gemini SDK not installed. Install with: pip install semantica[llm-gemini]"+ )def _legacy_client_for(self, requested_model: str):"""Return a legacy-SDK GenerativeModel bound to this instance's own@@ -711,6 +831,7 @@ class GeminiProvider(BaseProvider):"""try:import google.generativeai as old_genai+old_genai.configure(api_key=self.api_key)except Exception:# _init_client() already required this import to reach the@@ -747,18 +868,30 @@ class GeminiProvider(BaseProvider):)config = {}- self._add_if_set(config, kwargs, "temperature", "top_p", "top_k", "stop_sequences", "candidate_count")+ self._add_if_set(+ config,+ kwargs,+ "temperature",+ "top_p",+ "top_k",+ "stop_sequences",+ "candidate_count",+ )if "max_tokens" in kwargs:config["max_output_tokens"] = kwargs["max_tokens"]if self._use_new_genai:resp = self.client.models.generate_content(- model=kwargs.get("model", self.model), contents=prompt, config=config or None+ model=kwargs.get("model", self.model),+ contents=prompt,+ config=config or None,)return self._resp_text(resp)else:legacy_client = self._legacy_client_for(kwargs.get("model", self.model))- response = legacy_client.generate_content(prompt, generation_config=config or None)+ response = legacy_client.generate_content(+ prompt, generation_config=config or None+ )return self._resp_text(response)def generate_structured(self, prompt: str, **kwargs) -> dict:@@ -769,7 +902,15 @@ class GeminiProvider(BaseProvider):json_prompt = f"{prompt}\n\nReturn the response as valid JSON only."config = {}- self._add_if_set(config, kwargs, "temperature", "top_p", "top_k", "stop_sequences", "candidate_count")+ self._add_if_set(+ config,+ kwargs,+ "temperature",+ "top_p",+ "top_k",+ "stop_sequences",+ "candidate_count",+ )if "max_tokens" in kwargs:config["max_output_tokens"] = kwargs["max_tokens"]@@ -784,7 +925,9 @@ class GeminiProvider(BaseProvider):raise ProcessingError(f"Failed to parse JSON from Gemini response: {e}")else:legacy_client = self._legacy_client_for(kwargs.get("model", self.model))- response = legacy_client.generate_content(json_prompt, generation_config=config or None)+ response = legacy_client.generate_content(+ json_prompt, generation_config=config or None+ )try:return self._parse_json(self._resp_text(response))except Exception as e:@@ -795,7 +938,10 @@ class GroqProvider(BaseProvider):"""Groq provider implementation."""def __init__(- self, api_key: Optional[str] = None, model: str = "llama-3.3-70b-versatile", **kwargs+ self,+ api_key: Optional[str] = None,+ model: str = "llama-3.3-70b-versatile",+ **kwargs,):"""Initialize Groq provider."""super().__init__(**kwargs)@@ -815,11 +961,11 @@ class GroqProvider(BaseProvider):returnself.client = Groq(api_key=self.api_key)-+# Test connection with a minimal prompt# Only do this if we have a key and client# self._test_connection()- except ImportError:+ except (ImportError, OSError):self.client = Noneself.logger.warning("groq library not installed. Install with: pip install semantica[llm-groq]"@@ -837,7 +983,7 @@ class GroqProvider(BaseProvider):self.client.chat.completions.create(model=self.model,messages=[{"role": "user", "content": "ping"}],- max_tokens=1+ max_tokens=1,)self.logger.debug("Groq connection test successful")except Exception as e:@@ -858,8 +1004,19 @@ class GroqProvider(BaseProvider):"model": kwargs.get("model", self.model),"messages": [{"role": "user", "content": prompt}],}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_completion_tokens",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)return response.choices[0].message.content@@ -870,15 +1027,30 @@ class GroqProvider(BaseProvider):raise ProcessingError("Groq client not initialized.")# Groq requires 'json' in the prompt for json_object mode- json_prompt = prompt if "json" in prompt.lower() else f"{prompt}\n\nReturn the response as valid JSON only."+ json_prompt = (+ prompt+ if "json" in prompt.lower()+ else f"{prompt}\n\nReturn the response as valid JSON only."+ )create_kwargs = {"model": kwargs.get("model", self.model),"messages": [{"role": "user", "content": json_prompt}],"response_format": {"type": "json_object"},}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_completion_tokens",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)try:@@ -927,19 +1099,26 @@ class AnthropicProvider(BaseProvider):"Anthropic client not initialized. Set ANTHROPIC_API_KEY or pass api_key.")- # Anthropic requires max_tokens.+ # Anthropic requires max_tokens.# We rely on kwargs, but fallback to 8192 (safe max for newer models) if not provided.max_tokens = kwargs.get("max_tokens", 8192)-+# Prepare argumentscreate_kwargs = {"model": kwargs.get("model", self.model),"max_tokens": max_tokens,"messages": [{"role": "user", "content": prompt}],}-+# Pass through other common parameters- for param in ["temperature", "top_p", "top_k", "stop_sequences", "system", "metadata"]:+ for param in [+ "temperature",+ "top_p",+ "top_k",+ "stop_sequences",+ "system",+ "metadata",+ ]:if param in kwargs:create_kwargs[param] = kwargs[param]@@ -952,22 +1131,29 @@ class AnthropicProvider(BaseProvider):raise ProcessingError("Anthropic client not initialized.")json_prompt = f"{prompt}\n\nReturn the response as valid JSON only."-- # Anthropic requires max_tokens.++ # Anthropic requires max_tokens.max_tokens = kwargs.get("max_tokens", 8192)-+# Prepare argumentscreate_kwargs = {"model": kwargs.get("model", self.model),"max_tokens": max_tokens,"messages": [{"role": "user", "content": json_prompt}],}-+# Pass through other common parameters- for param in ["temperature", "top_p", "top_k", "stop_sequences", "system", "metadata"]:+ for param in [+ "temperature",+ "top_p",+ "top_k",+ "stop_sequences",+ "system",+ "metadata",+ ]:if param in kwargs:create_kwargs[param] = kwargs[param]-+response = self.client.messages.create(**create_kwargs)try:return self._parse_json(response.content[0].text)@@ -1017,7 +1203,9 @@ class OllamaProvider(BaseProvider):def _build_options(self, kwargs: dict) -> Optional[dict]:"""Build Ollama options dict from kwargs."""options = {}- self._add_if_set(options, kwargs, "temperature", "top_p", "top_k", "repeat_penalty", "seed")+ self._add_if_set(+ options, kwargs, "temperature", "top_p", "top_k", "repeat_penalty", "seed"+ )if "max_tokens" in kwargs:options["num_predict"] = kwargs["max_tokens"]if "num_ctx" in kwargs:@@ -1058,7 +1246,9 @@ class OllamaProvider(BaseProvider):class DeepSeekProvider(BaseProvider):- def __init__(self, api_key: Optional[str] = None, model: str = "deepseek-chat", **kwargs):+ def __init__(+ self, api_key: Optional[str] = None, model: str = "deepseek-chat", **kwargs+ ):super().__init__(**kwargs)self.api_key = api_key or config.get_api_key("deepseek")self.base_url = "https://api.deepseek.com/v1"@@ -1075,7 +1265,8 @@ class DeepSeekProvider(BaseProvider):except (ImportError, OSError):self.client = Noneself.logger.warning(- "openai library not installed. Install with: pip install semantica[llm-openai]"+ "openai library not installed. "+ "Install with: pip install semantica[llm-openai]")def is_available(self) -> bool:@@ -1083,13 +1274,26 @@ class DeepSeekProvider(BaseProvider):def generate(self, prompt: str, **kwargs) -> str:if not self.client:- raise ProcessingError("DeepSeek client not initialized. Set DEEPSEEK_API_KEY or pass api_key.")+ raise ProcessingError(+ "DeepSeek client not initialized. Set DEEPSEEK_API_KEY or pass api_key."+ )create_kwargs = {"model": kwargs.get("model", self.model),"messages": [{"role": "user", "content": prompt}],}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)return response.choices[0].message.content@@ -1104,7 +1308,18 @@ class DeepSeekProvider(BaseProvider):"messages": [{"role": "user", "content": prompt}],"response_format": {"type": "json_object"},}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)try:@@ -1116,7 +1331,12 @@ class DeepSeekProvider(BaseProvider):class NovitaProvider(BaseProvider):"""Novita AI provider implementation - OpenAI-compatible API."""- def __init__(self, api_key: Optional[str] = None, model: str = "deepseek/deepseek-v3.2", **kwargs):+ def __init__(+ self,+ api_key: Optional[str] = None,+ model: str = "deepseek/deepseek-v3.2",+ **kwargs,+ ):"""Initialize Novita provider."""super().__init__(**kwargs)self.api_key = api_key or config.get_api_key("novita")@@ -1134,7 +1354,8 @@ class NovitaProvider(BaseProvider):except (ImportError, OSError):self.client = Noneself.logger.warning(- "openai library not installed. Install with: pip install semantica[llm-openai]"+ "openai library not installed. "+ "Install with: pip install semantica[llm-openai]")def is_available(self) -> bool:@@ -1142,13 +1363,26 @@ class NovitaProvider(BaseProvider):def generate(self, prompt: str, **kwargs) -> str:if not self.client:- raise ProcessingError("Novita client not initialized. Set NOVITA_API_KEY or pass api_key.")+ raise ProcessingError(+ "Novita client not initialized. Set NOVITA_API_KEY or pass api_key."+ )create_kwargs = {"model": kwargs.get("model", self.model),"messages": [{"role": "user", "content": prompt}],}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)return response.choices[0].message.content@@ -1163,7 +1397,18 @@ class NovitaProvider(BaseProvider):"messages": [{"role": "user", "content": prompt}],"response_format": {"type": "json_object"},}- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")+ self._add_if_set(+ create_kwargs,+ kwargs,+ "temperature",+ "max_tokens",+ "top_p",+ "frequency_penalty",+ "presence_penalty",+ "seed",+ "stop",+ "user",+ )response = self.client.chat.completions.create(**create_kwargs)try:@@ -1171,6 +1416,7 @@ class NovitaProvider(BaseProvider):except Exception as e:raise ProcessingError(f"Failed to parse JSON from Novita response: {e}")+class HuggingFaceLLMProvider(BaseProvider):"""HuggingFace transformers for LLM tasks."""@@ -1186,7 +1432,7 @@ class HuggingFaceLLMProvider(BaseProvider):raise ImportError("torch is required for HuggingFaceLLMProvider. Install with: pip install 'semantica[models-huggingface]'")-+self.model_name = model_nameself.device = device or ("cuda" if torch.cuda.is_available() else "cpu")self.model = None@@ -1222,28 +1468,25 @@ class HuggingFaceLLMProvider(BaseProvider):raise ProcessingError("HuggingFace model not initialized.")inputs = self.tokenizer.encode(prompt, return_tensors="pt").to(self.device)-+# Use max_new_tokens if available, otherwise fallback to max_length with a safe defaultgenerate_kwargs = {"temperature": kwargs.get("temperature", 0.7),"do_sample": True,}-+if "max_new_tokens" in kwargs:generate_kwargs["max_new_tokens"] = kwargs["max_new_tokens"]elif "max_tokens" in kwargs:generate_kwargs["max_new_tokens"] = kwargs["max_tokens"]-+# Support legacy max_length if explicitly providedif "max_length" in kwargs:generate_kwargs["max_length"] = kwargs["max_length"]# Remove max_new_tokens if max_length is set to avoid conflictgenerate_kwargs.pop("max_new_tokens", None)- outputs = self.model.generate(- inputs,- **generate_kwargs- )+ outputs = self.model.generate(inputs, **generate_kwargs)generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)# Remove the original prompt from the responsereturn generated_text[len(prompt) :].strip()@@ -1271,9 +1514,10 @@ class HuggingFaceModelLoader:import torchexcept (ImportError, OSError):raise ImportError(- "torch is required for HuggingFaceModelLoader. Install with: pip install 'semantica[models-huggingface]'"+ "torch is required for HuggingFaceModelLoader. "+ "Install with: pip install 'semantica[models-huggingface]'")-+self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")self._cache: Dict[str, Any] = {}self.logger = get_logger("huggingface_loader")@@ -1282,7 +1526,7 @@ class HuggingFaceModelLoader:"""Load NER model."""# Import torch at method level to ensure it's availableimport torch-+# Include aggregation_strategy in cache keyagg_strategy = kwargs.get("aggregation_strategy", "simple")cache_key = f"{model_name}_ner_{agg_strategy}"@@ -1291,9 +1535,10 @@ class HuggingFaceModelLoader:try:from transformers import pipeline- except ImportError:+ except (ImportError, OSError):raise ImportError(- "transformers library not installed. Install with: pip install 'semantica[models-huggingface]'"+ "transformers library not installed. "+ "Install with: pip install 'semantica[models-huggingface]'")try:@@ -1302,13 +1547,16 @@ class HuggingFaceModelLoader:model=model_name,device=self.device if torch.cuda.is_available() else -1,aggregation_strategy=agg_strategy,- tokenizer=kwargs.get("tokenizer") # Allow custom tokenizer+ tokenizer=kwargs.get("tokenizer"), # Allow custom tokenizer)self._cache[cache_key] = nlpreturn nlpexcept OSError as e:self.logger.error(f"Failed to load NER model '{model_name}': {e}")- raise ValueError(f"Could not load HuggingFace model '{model_name}'. Check if model name is correct. Error: {e}")+ raise ValueError(+ f"Could not load HuggingFace model '{model_name}'. "+ f"Check if model name is correct. Error: {e}"+ )except Exception as e:self.logger.error(f"Failed to load NER model {model_name}: {e}")raise@@ -1317,16 +1565,17 @@ class HuggingFaceModelLoader:"""Load relation extraction model."""# Import torch at method level to ensure it's availableimport torch-+cache_key = f"{model_name}_relation"if cache_key in self._cache:return self._cache[cache_key]try:from transformers import pipeline, AutoTokenizer- except ImportError:+ except (ImportError, OSError):raise ImportError(- "transformers library not installed. Install with: pip install 'semantica[models-huggingface]'"+ "transformers library not installed. "+ "Install with: pip install 'semantica[models-huggingface]'")try:@@ -1347,7 +1596,10 @@ class HuggingFaceModelLoader:return nlpexcept OSError as e:self.logger.error(f"Failed to load relation model '{model_name}': {e}")- raise ValueError(f"Could not load HuggingFace model '{model_name}'. Check if model name is correct. Error: {e}")+ raise ValueError(+ f"Could not load HuggingFace model '{model_name}'. "+ f"Check if model name is correct. Error: {e}"+ )except Exception as e:self.logger.error(f"Failed to load relation model {model_name}: {e}")raise@@ -1359,10 +1611,11 @@ class HuggingFaceModelLoader:return self._cache[cache_key]try:- from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline- except ImportError:+ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer+ except (ImportError, OSError):raise ImportError(- "transformers library not installed. Install with: pip install 'semantica[models-huggingface]'"+ "transformers library not installed. "+ "Install with: pip install 'semantica[models-huggingface]'")try:@@ -1371,7 +1624,7 @@ class HuggingFaceModelLoader:if not tokenizer:tokenizer_name = kwargs.get("tokenizer_name", model_name)tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)-+model = AutoModelForSeq2SeqLM.from_pretrained(model_name)model.to(self.device)@@ -1380,7 +1633,10 @@ class HuggingFaceModelLoader:return nlpexcept OSError as e:self.logger.error(f"Failed to load triplet model '{model_name}': {e}")- raise ValueError(f"Could not load HuggingFace model '{model_name}'. Check if model name is correct. Error: {e}")+ raise ValueError(+ f"Could not load HuggingFace model '{model_name}'. "+ f"Check if model name is correct. Error: {e}"+ )except Exception as e:self.logger.error(f"Failed to load triplet model {model_name}: {e}")raise@@ -1389,83 +1645,103 @@ class HuggingFaceModelLoader:"""Extract entities using loaded model."""return model(text)- def extract_relations(self, model, text: str, entities: List, **kwargs) -> List[Dict]:+ def extract_relations(+ self, model, text: str, entities: List, **kwargs+ ) -> List[Dict]:"""Extract relations using loaded model.Iterates through entity pairs and classifies the relationship."""results = []-+# Sort entities by positionsorted_entities = sorted(entities, key=lambda e: e.start_char)-+# Marker configurationsubj_start = kwargs.get("subj_start_marker", "<subj>")subj_end = kwargs.get("subj_end_marker", "</subj>")obj_start = kwargs.get("obj_start_marker", "<obj>")obj_end = kwargs.get("obj_end_marker", "</obj>")-+# Iterate through all pairs- import itertoolsfor i, e1 in enumerate(sorted_entities):for e2 in sorted_entities:if e1 == e2:continue-+# Check distance (optional optimization)# if abs(e1.start_char - e2.start_char) > 200: continue-+# Format text with markers# Strategy: [CLS] text with <subj>...</subj> and <obj>...</obj> [SEP]# We need to insert markers into the original text-+# Create a copy of text with markers inserted- # We need to handle offsets correctly.+ # We need to handle offsets correctly.# Simplest way: reconstruct string pieces-+p1_start, p1_end = e1.start_char, e1.end_charp2_start, p2_end = e2.start_char, e2.end_char-+if p1_start < p2_start:formatted_text = (- text[:p1_start] +- f"{subj_start} " + text[p1_start:p1_end] + f" {subj_end}" +- text[p1_end:p2_start] +- f"{obj_start} " + text[p2_start:p2_end] + f" {obj_end}" +- text[p2_end:]+ text[:p1_start]+ + f"{subj_start} "+ + text[p1_start:p1_end]+ + f" {subj_end}"+ + text[p1_end:p2_start]+ + f"{obj_start} "+ + text[p2_start:p2_end]+ + f" {obj_end}"+ + text[p2_end:])else:- formatted_text = (- text[:p2_start] +- f"{obj_start} " + text[p2_start:p2_end] + f" {obj_end}" +- text[p2_end:p1_start] +- f"{subj_start} " + text[p1_start:p1_end] + f" {subj_end}" +- text[p1_end:]+ formatted_text = (+ text[:p2_start]+ + f"{obj_start} "+ + text[p2_start:p2_end]+ + f" {obj_end}"+ + text[p2_end:p1_start]+ + f"{subj_start} "+ + text[p1_start:p1_end]+ + f" {subj_end}"+ + text[p1_end:])-+# Predicttry:# Pipeline returns [{'label': 'LABEL', 'score': 0.99}]prediction = model(formatted_text, top_k=1)-+if prediction:- res = prediction[0] if isinstance(prediction, list) else prediction- if isinstance(res, list): res = res[0] # top_k=1 returns list of dicts-+ res = (+ prediction[0]+ if isinstance(prediction, list)+ else prediction+ )+ if isinstance(res, list):+ res = res[0] # top_k=1 returns list of dicts+label = res.get("label")score = res.get("score")-+# Filter "no_relation" or low confidence- if label != "no_relation" and score > kwargs.get("threshold", 0.5):- results.append({- "subject": e1,- "object": e2,- "relation": label,- "score": score- })+ if label != "no_relation" and score > kwargs.get(+ "threshold", 0.5+ ):+ results.append(+ {+ "subject": e1,+ "object": e2,+ "relation": label,+ "score": score,+ }+ )except Exception as e:- self.logger.warning(f"Relation prediction failed for pair {e1.text}-{e2.text}: {e}")-+ self.logger.warning(+ f"Relation prediction failed for pair {e1.text}-{e2.text}: {e}"+ )+return resultsdef extract_triplets(self, model, text: str, **kwargs) -> List[Dict]:@@ -1477,21 +1753,28 @@ class HuggingFaceModelLoader:# Use kwargs for max_length, default to 512 for input and 128 for output if not specifiedmax_input_length = kwargs.get("max_input_length", 512)max_length = kwargs.get("max_length", 128)-+generate_kwargs = {"max_length": max_length}if "max_new_tokens" in kwargs:generate_kwargs["max_new_tokens"] = kwargs["max_new_tokens"]-+# Pass other generation args including beams and penalties- for param in ["num_beams", "temperature", "top_p", "top_k", "do_sample",- "length_penalty", "repetition_penalty"]:+ for param in [+ "num_beams",+ "temperature",+ "top_p",+ "top_k",+ "do_sample",+ "length_penalty",+ "repetition_penalty",+ ]:if param in kwargs:generate_kwargs[param] = kwargs[param]inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=max_input_length).to(device)-+outputs = model_obj.generate(**inputs, **generate_kwargs)# Allow controlling skip_special_tokens (important for REBEL which uses special tokens for delimiters)skip_special_tokens = kwargs.get("skip_special_tokens", True)@@ -1502,7 +1785,7 @@ class HuggingFaceModelLoader:class ProviderPool:"""Pool for reusing provider instances."""-+def __init__(self):self._providers: Dict[str, BaseProvider] = {}self.logger = get_logger("provider_pool")@@ -1512,7 +1795,7 @@ class ProviderPool:# Create a cache key from name and kwargs# Filter out non-hashable items or volatile args if any# For now, we assume kwargs are configuration options that should match-+# Helper to make dict hashabledef make_hashable(value):if isinstance(value, dict):@@ -1523,20 +1806,20 @@ class ProviderPool:key_parts = [name]for k, v in sorted(kwargs.items()):- # Skip some keys if they shouldn't affect pooling?+ # Skip some keys if they shouldn't affect pooling?# For now, all init args matter for the instance identity.key_parts.append((k, make_hashable(v)))-+key = str(tuple(key_parts))-+if key in self._providers:return self._providers[key]-+self.logger.debug(f"Creating new provider instance for {name}")provider = self._create_provider(name, **kwargs)self._providers[key] = providerreturn provider-+def _create_provider(self, name: str, **kwargs) -> BaseProvider:"""Internal creation logic."""# Check registry first@@ -1553,7 +1836,7 @@ class ProviderPool:"ollama": OllamaProvider,"huggingface_llm": HuggingFaceLLMProvider,"deepseek": DeepSeekProvider,- "novita": NovitaProvider,+ "novita": NovitaProvider,}provider_class = builtin.get(name.lower())@@ -1576,7 +1859,7 @@ _provider_pool = ProviderPool()def create_provider(name: str, use_pool: bool = True, **kwargs) -> BaseProvider:"""Create provider - checks registry for custom providers.-+Args:name: Provider nameuse_pool: Whether to use the provider pool (default: True)@@ -1584,5 +1867,5 @@ def create_provider(name: str, use_pool: bool = True, **kwargs) -> BaseProvider:"""if use_pool:return _provider_pool.get(name, **kwargs)-+return _provider_pool._create_provider(name, **kwargs)diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.pyindex 48065855..feb3400c 100644--- a/semantica/utils/helpers.py+++ b/semantica/utils/helpers.py@@ -38,15 +38,15 @@ Example Usage:>>> from semantica.utils import clean_text, normalize_entities>>> cleaned = clean_text(" Hello World ")>>> entities = normalize_entities([{"id": "e1", "text": "John", "type": "PERSON"}])- >>>+ >>>>>> from semantica.utils import hash_data, safe_filename>>> data_hash = hash_data({"key": "value"})>>> safe_name = safe_filename("my file.txt")- >>>+ >>>>>> from semantica.utils import merge_dicts, get_nested_value>>> merged = merge_dicts({"a": 1}, {"b": 2}, deep=True)>>> value = get_nested_value(config, "database.host", default="localhost")- >>>+ >>>>>> from semantica.utils import retry_on_error>>> @retry_on_error(max_retries=3, delay=1.0)... def fetch_data():@@ -457,7 +457,9 @@ def chunk_list(items: List[Any], chunk_size: int) -> List[List[Any]]:Returns:List of chunks"""- return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]+ return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]++def flatten_dict(d: Dict[str, Any], parent_key: str = "", sep: str = ".") -> Dict[str, Any]:@@ -556,27 +558,27 @@ def safe_import() -> Tuple[Any, bool]:"""Safely import an optional module, handling both ImportError and OSError.-+This is useful for optional dependencies that may fail to import due to:- Missing package (ImportError)- DLL loading failures on Windows, e.g., PyTorch (OSError)-+Args:module_name: Name of the module to import (e.g., "spacy", "docling.document_converter")package: Optional package name for relative importsdefault: Default value to return if import failserror_message: Optional custom error message for logging-+Returns:Tuple of (module_or_default, success_flag):- If import succeeds: (imported_module, True)- If import fails: (default, False)-+Example:>>> spacy, available = safe_import("spacy")>>> if available:... doc = spacy.load("en_core_web_sm")- >>>+ >>>>>> converter, available = safe_import("docling.document_converter", default=None)>>> if available:... converter = converter()@@ -587,11 +589,13 @@ def safe_import(else:module = importlib.import_module(module_name)return module, True- except (ImportError, ModuleNotFoundError, OSError) as e:+ except (ImportError, OSError) as e:if error_message:import sys+if "logging" in sys.modules:from .logging import get_logger+logger = get_logger("utils.helpers")logger.debug(f"{error_message}: {e}")return default, False@@ -807,9 +811,13 @@ def _is_record(value: Any) -> bool:exporters rather than a ``ValidationError`` at the boundary where theproblem is visible."""- return isinstance(value, Mapping) or is_dataclass(value) or (- hasattr(value, "__dict__")- and not isinstance(value, (types.ModuleType, type))+ return (+ isinstance(value, Mapping)+ or is_dataclass(value)+ or (+ hasattr(value, "__dict__")+ and not isinstance(value, (types.ModuleType, type))+ ))diff --git a/semantica/visualization/embedding_visualizer.py b/semantica/visualization/embedding_visualizer.pyindex 648c9515..ba26ecea 100644--- a/semantica/visualization/embedding_visualizer.py+++ b/semantica/visualization/embedding_visualizer.py@@ -1,9 +1,10 @@"""Embedding Visualizer Module-This module provides comprehensive visualization capabilities for vector embeddings in the-Semantica framework, including 2D/3D dimensionality reduction projections, similarity heatmaps,-clustering visualizations, multi-modal comparisons, and quality metrics analysis.+This module provides comprehensive visualization capabilities for vector+embeddings in the Semantica framework, including 2D/3D dimensionality+reduction projections, similarity heatmaps, clustering visualizations,+multi-modal comparisons, and quality metrics analysis.Key Features:- 2D and 3D dimensionality reduction (UMAP, t-SNE, PCA)@@ -31,7 +32,7 @@ License: MIT"""from pathlib import Path-from typing import Any, Dict, List, Optional, Tuple, Union+from typing import Any, Dict, List, Optional, Unionimport numpy as np@@ -158,10 +159,12 @@ class EmbeddingVisualizer:try:self.logger.info(f"Visualizing 2D projection using {method}")-+# Step 2: Data Analysisn_samples, n_features = embeddings.shape- self.logger.info(f"Embedding Analysis: {n_samples} samples, {n_features} dimensions")+ self.logger.info(+ f"Embedding Analysis: {n_samples} samples, {n_features} dimensions"+ )if embeddings.shape[1] <= 2:# Already 2D or less, use directly@@ -183,20 +186,22 @@ class EmbeddingVisualizer:tracking_id, message="Generating visualization...")result = self._visualize_2d_plotly(- projected,- labels,- output,- file_path,+ projected,+ labels,+ output,+ file_path,color_by=color_by,size_by=size_by,hover_data=hover_data,- **options+ **options,)self.progress_tracker.stop_tracking(tracking_id,status="completed",- message=f"2D projection visualization generated: {len(projected)} points",+ message=(+ f"2D projection visualization generated: {len(projected)} points"+ ),)return resultexcept Exception as e:@@ -263,7 +268,9 @@ class EmbeddingVisualizer:self.progress_tracker.stop_tracking(tracking_id,status="completed",- message=f"3D projection visualization generated: {len(projected)} points",+ message=(+ f"3D projection visualization generated: {len(projected)} points"+ ),)return resultexcept Exception as e:@@ -354,7 +361,10 @@ class EmbeddingVisualizer:self.progress_tracker.stop_tracking(tracking_id,status="completed",- message=f"Similarity heatmap generated: {len(embeddings)}x{len(embeddings)} matrix",+ message=(+ f"Similarity heatmap generated: "+ f"{len(embeddings)}x{len(embeddings)} matrix"+ ),)return figelif file_path:@@ -459,7 +469,10 @@ class EmbeddingVisualizer:self.progress_tracker.stop_tracking(tracking_id,status="completed",- message=f"Clustering visualization generated: {num_clusters} clusters, {len(embeddings)} points",+ message=(+ f"Clustering visualization generated: "+ f"{num_clusters} clusters, {len(embeddings)} points"+ ),)return figelif file_path:@@ -558,7 +571,10 @@ class EmbeddingVisualizer:dim_options = dict(options)n_comp = dim_options.pop("n_components", 2)projected = self._reduce_dimensions(- combined_embeddings, method=method, n_components=n_comp, **dim_options+ combined_embeddings,+ method=method,+ n_components=n_comp,+ **dim_options,)# Color by type@@ -595,7 +611,10 @@ class EmbeddingVisualizer:self.progress_tracker.stop_tracking(tracking_id,status="completed",- message=f"Multi-modal comparison generated: {len(combined_embeddings)} embeddings",+ message=(+ f"Multi-modal comparison generated: "+ f"{len(combined_embeddings)} embeddings"+ ),)return figelif file_path:@@ -614,8 +633,6 @@ class EmbeddingVisualizer:)raise--def _reduce_dimensions(self,embeddings: np.ndarray,diff --git a/tests/ingest/test_optional_imports.py b/tests/ingest/test_optional_imports.pyindex 6c70efb8..9a9a6169 100644--- a/tests/ingest/test_optional_imports.py+++ b/tests/ingest/test_optional_imports.py@@ -141,3 +141,134 @@ else:assert "ConfigurationError" in result.stdoutassert "Parquet ingestion" in result.stdoutassert "pyarrow" in result.stdout+++def test_repo_ingestor_probe_fails_without_gitpython() -> None:+ result = _run_python_with_blocked_modules(+ """+try:+ from semantica.ingest import RepoIngestor+ has_git = True+except ImportError:+ has_git = False++assert not has_git, "Expected RepoIngestor import to fail without GitPython"+print("RepoIngestor probe passed")+""",+ ("git",),+ )++ assert result.returncode == 0, result.stderr+ assert "RepoIngestor probe passed" in result.stdout+++def test_xml_ingestor_probe_fails_without_lxml() -> None:+ result = _run_python_with_blocked_modules(+ """+try:+ from semantica.ingest import XMLIngestor+ has_lxml = True+except ImportError:+ has_lxml = False++assert not has_lxml, "Expected XMLIngestor import to fail without lxml"+print("XMLIngestor probe passed")+""",+ ("lxml",),+ )++ assert result.returncode == 0, result.stderr+ assert "XMLIngestor probe passed" in result.stdout+++def test_xml_ingestion_reports_missing_lxml_when_used() -> None:+ result = _run_python_with_blocked_modules(+ """+from semantica.ingest import ingest_xml++try:+ ingest_xml("catalog.xml")+except Exception as exc:+ print(type(exc).__name__, exc)+else:+ raise SystemExit("expected XML ingestion to fail without lxml")+""",+ ("lxml",),+ )++ assert result.returncode == 0, result.stderr+ assert "ConfigurationError" in result.stdout+ assert "XML ingestion" in result.stdout+ assert "lxml" in result.stdout+++def test_sibling_imports_succeed_without_optional_backends() -> None:+ result = _run_python_with_blocked_modules(+ """+from semantica.ingest import (+ CodeExtractor,+ CodeFile,+ CommitInfo,+ GitAnalyzer,+ XMLIngestionData,+ SalesforceData,+)+print(+ CodeExtractor.__name__,+ CodeFile.__name__,+ CommitInfo.__name__,+ GitAnalyzer.__name__,+ XMLIngestionData.__name__,+ SalesforceData.__name__,+)+""",+ ("git", "lxml", "simple_salesforce"),+ )++ assert result.returncode == 0, result.stderr+ assert (+ "CodeExtractor CodeFile CommitInfo GitAnalyzer XMLIngestionData SalesforceData"+ in result.stdout+ )+++def test_salesforce_ingestor_probe_fails_without_simple_salesforce() -> None:+ result = _run_python_with_blocked_modules(+ """+try:+ from semantica.ingest import SalesforceIngestor+ has_salesforce = True+except ImportError:+ has_salesforce = False++assert not has_salesforce, (+ "Expected SalesforceIngestor import to fail without simple-salesforce"+)+print("SalesforceIngestor probe passed")+""",+ ("simple_salesforce",),+ )++ assert result.returncode == 0, result.stderr+ assert "SalesforceIngestor probe passed" in result.stdout+++def test_salesforce_ingestion_reports_missing_dep_when_used() -> None:+ result = _run_python_with_blocked_modules(+ """+from semantica.ingest import ingest_salesforce++try:+ ingest_salesforce()+except Exception as exc:+ print(type(exc).__name__, exc)+else:+ raise SystemExit("expected Salesforce ingestion to fail without simple-salesforce")+""",+ ("simple_salesforce",),+ )++ assert result.returncode == 0, result.stderr+ assert "ConfigurationError" in result.stdout+ assert "Salesforce ingestion" in result.stdout+ assert "simple-salesforce" in result.stdoutdiff --git a/tests/semantic_extract/test_temporal_extraction.py b/tests/semantic_extract/test_temporal_extraction.pyindex 9aa7d46a..0dc2bc55 100644--- a/tests/semantic_extract/test_temporal_extraction.py+++ b/tests/semantic_extract/test_temporal_extraction.py@@ -18,7 +18,14 @@ from datetime import datetime, timezonefrom unittest.mock import MagicMock, patch# ── Mock optional heavyweight dependencies before any semantica import ──────-_MOCKED_MODULES = ["spacy", "instructor", "openai", "groq", "sentence_transformers", "transformers", "torch"]+_MOCKED_MODULES = [+ "spacy",+ "instructor",+ "openai",+ "groq",+ "sentence_transformers",+ "transformers",+]_original_modules = {k: sys.modules.get(k) for k in _MOCKED_MODULES}for k in _MOCKED_MODULES:@@ -26,14 +33,7 @@ for k in _MOCKED_MODULES:sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))-from semantica.semantic_extract.methods import extract_relations_llm-from semantica.semantic_extract.ner_extractor import Entity-from semantica.semantic_extract.schemas import (- RelationsResponse,- RelationsWithTemporalResponse,-)-from semantica.kg.temporal_normalizer import TemporalNormalizer-from semantica.utils.exceptions import TemporalAmbiguityWarning+from semantica.semantic_extract.methods import extract_relations_llm # noqa: E402for _key, _original in _original_modules.items():if _original is None:@@ -41,9 +41,17 @@ for _key, _original in _original_modules.items():else:sys.modules[_key] = _original+from semantica.semantic_extract.ner_extractor import Entity # noqa: E402+from semantica.semantic_extract.schemas import ( # noqa: E402+ RelationsResponse,+ RelationsWithTemporalResponse,+)+from semantica.kg.temporal_normalizer import TemporalNormalizer # noqa: E402+from semantica.utils.exceptions import TemporalAmbiguityWarning # noqa: E402# ── Helpers ─────────────────────────────────────────────────────────────────+def _make_entities():return [Entity(text="Apple", label="ORG", start_char=0, end_char=5),@@ -59,15 +67,17 @@ def _ref_date():# Part 1 – extract_relations_llm() temporal flag# ============================================================================+class TestTemporalExtractionFlag(unittest.TestCase):def setUp(self):from semantica.semantic_extract.methods import _result_cache+_result_cache.clear()@patch("semantica.semantic_extract.methods.create_provider")def test_extract_temporal_bounds_true_adds_four_fields(self, mock_create):- """With extract_temporal_bounds=True all four temporal keys appear in metadata."""+ """With extract_temporal_bounds=True all four temporal keys appear."""mock_prov = MagicMock()mock_prov.is_available.return_value = Truemock_prov.generate_typed.return_value = RelationsWithTemporalResponse(@@ -135,7 +145,9 @@ class TestTemporalExtractionFlag(unittest.TestCase):self.assertNotIn("temporal_source_text", meta)@patch("semantica.semantic_extract.methods.create_provider")- def test_no_temporal_signal_returns_zero_confidence_and_null_dates(self, mock_create):+ def test_no_temporal_signal_returns_zero_confidence_and_null_dates(+ self, mock_create+ ):"""When LLM returns no temporal signal, confidence=0.0 and dates are null."""mock_prov = MagicMock()mock_prov.is_available.return_value = True@@ -200,10 +212,12 @@ class TestTemporalExtractionFlag(unittest.TestCase):@patch("semantica.semantic_extract.methods.create_provider")def test_correct_schema_used_when_temporal_true(self, mock_create):- """generate_typed is called with RelationsWithTemporalResponse when flag=True."""+ """generate_typed is called with RelationsWithTemporalResponse."""mock_prov = MagicMock()mock_prov.is_available.return_value = True- mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(relations=[])+ mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(+ relations=[]+ )mock_create.return_value = mock_provextract_relations_llm(@@ -238,6 +252,7 @@ class TestTemporalExtractionFlag(unittest.TestCase):# Part 2 – TemporalNormalizer: relative dates# ============================================================================+class TestTemporalNormalizerRelativeDates(unittest.TestCase):def setUp(self):@@ -316,10 +331,13 @@ class TestTemporalNormalizerRelativeDates(unittest.TestCase):# Part 3 – TemporalNormalizer: partial / structured dates# ============================================================================+class TestTemporalNormalizerPartialDates(unittest.TestCase):def setUp(self):- self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))+ self.tn = TemporalNormalizer(+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)+ )def test_year_only(self):result = self.tn.normalize("2021")@@ -399,10 +417,13 @@ class TestTemporalNormalizerPartialDates(unittest.TestCase):# Part 4 – TemporalNormalizer: ambiguous formats# ============================================================================+class TestTemporalNormalizerAmbiguity(unittest.TestCase):def setUp(self):- self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))+ self.tn = TemporalNormalizer(+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)+ )def test_ambiguous_slash_date_raises_warning_and_returns_none(self):with warnings.catch_warnings(record=True) as w:@@ -435,14 +456,19 @@ class TestTemporalNormalizerAmbiguity(unittest.TestCase):# Part 5 – TemporalNormalizer: domain phrase map# ============================================================================+class TestTemporalNormalizerDomainPhrases(unittest.TestCase):def setUp(self):- self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))+ self.tn = TemporalNormalizer(+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)+ )def _assert_recognized(self, phrase):result = self.tn.normalize_phrase(phrase)- self.assertIsNotNone(result, f"Expected phrase {phrase!r} to be recognized but got None")+ self.assertIsNotNone(+ result, f"Expected phrase {phrase!r} to be recognized but got None"+ )return result# General / Policy@@ -525,6 +551,7 @@ class TestTemporalNormalizerDomainPhrases(unittest.TestCase):# Part 6 – TemporalNormalizer: custom phrase map# ============================================================================+class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):def setUp(self):@@ -568,10 +595,12 @@ class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):# Part 7 – Full pipeline: extract → normalize → BiTemporalFact# ============================================================================+class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):def setUp(self):from semantica.semantic_extract.methods import _result_cache+_result_cache.clear()@patch("semantica.semantic_extract.methods.create_provider")@@ -623,10 +652,12 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):self.assertEqual(vf[0].day, 1)# Feed into BiTemporalFact- fact = BiTemporalFact.from_relationship({- "valid_from": "2014-05-01T00:00:00Z",- "valid_until": None,- })+ fact = BiTemporalFact.from_relationship(+ {+ "valid_from": "2014-05-01T00:00:00Z",+ "valid_until": None,+ }+ )self.assertIsNotNone(fact.valid_from)self.assertEqual(fact.valid_from.year, 2014)self.assertEqual(fact.valid_from.month, 5)@@ -663,7 +694,9 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):extract_temporal_bounds=True,)meta = rels[0].metadata- tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))+ tn = TemporalNormalizer(+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)+ )vf = tn.normalize(meta["valid_from"])vu = tn.normalize(meta["valid_until"])diff --git a/tests/test_issue_1513_slim_core.py b/tests/test_issue_1513_slim_core.pyindex 039bccd9..f4de58a3 100644--- a/tests/test_issue_1513_slim_core.py+++ b/tests/test_issue_1513_slim_core.py@@ -14,13 +14,16 @@ def _load_toml(file_path: Path) -> dict:content = file_path.read_text(encoding="utf-8")try:import tomllib # Python 3.11+ standard library+return tomllib.loads(content)except ImportError:try:import tomli # Fast PEP 680 compatible parser for Python < 3.11+return tomli.loads(content)except ImportError:import toml # Fallback toml parser+return toml.loads(content)@@ -34,10 +37,28 @@ def test_core_dependencies_count():for d in deps}expected_22 = {- "numpy", "pandas", "scipy", "scikit-learn", "rdflib", "networkx",- "requests", "chardet", "protobuf", "grpcio", "pillow", "pydantic",- "click", "rich", "tqdm", "pyyaml", "toml", "python-dotenv",- "loguru", "structlog", "httpx", "pyarrow"+ "numpy",+ "pandas",+ "scipy",+ "scikit-learn",+ "rdflib",+ "networkx",+ "requests",+ "chardet",+ "protobuf",+ "grpcio",+ "pillow",+ "pydantic",+ "click",+ "rich",+ "tqdm",+ "pyyaml",+ "toml",+ "python-dotenv",+ "loguru",+ "structlog",+ "httpx",+ "pyarrow",}assert normalized_names == expected_22assert len(normalized_names) == 22@@ -49,14 +70,27 @@ def test_optional_extras_defined():data = _load_toml(repo_root / "pyproject.toml")extras = data["project"]["optional-dependencies"]for extra in [- "documents", "ingest-git", "embeddings-local", "nlp-spacy",- "viz", "media", "vectorstore-faiss", "graph-embeddings", "all"+ "documents",+ "ingest-git",+ "embeddings-local",+ "nlp-spacy",+ "viz",+ "media",+ "vectorstore-faiss",+ "graph-embeddings",+ "all",]:assert extra in extras, f"Missing extra {extra}"all_extra_str = str(extras["all"])for expected_ref in [- "documents", "ingest-git", "embeddings-local", "nlp-spacy",- "viz", "media", "graph-embeddings", "vectorstore-all"+ "documents",+ "ingest-git",+ "embeddings-local",+ "nlp-spacy",+ "viz",+ "media",+ "graph-embeddings",+ "vectorstore-all",]:assert expected_ref in all_extra_str, f"Missing {expected_ref} in all"# And vectorstore-faiss is in vectorstore-all@@ -81,6 +115,7 @@ def test_core_modules_importable():import semantica.visualizationimport semantica.semantic_extractimport semantica.pipeline+assert semantica.__version__ is not None@@ -129,20 +164,74 @@ def test_xml_parser_lxml_explicit_requires_documents_extra():def test_xml_ingestor_missing_hint():with patch("semantica.ingest.xml_ingestor.etree", None):from semantica.ingest.xml_ingestor import XMLIngestor- with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):++ with pytest.raises(ImportError, match=r"semantica\[documents\]"):XMLIngestor()+def test_xml_ingestor_package_import_missing_hint():+ import semantica.ingest as ingest_mod++ ingest_mod.__dict__.pop("XMLIngestor", None)+ with patch("semantica.ingest.xml_ingestor.etree", None):+ with pytest.raises(ImportError, match=r"semantica\[documents\]"):+ _ = ingest_mod.XMLIngestor++def test_repo_ingestor_missing_hint():with patch("semantica.ingest.repo_ingestor.git", None):from semantica.ingest.repo_ingestor import RepoIngestor+with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):RepoIngestor()+def test_repo_ingestor_package_import_missing_hint():+ import semantica.ingest as ingest_mod++ ingest_mod.__dict__.pop("RepoIngestor", None)+ with patch("semantica.ingest.repo_ingestor.git", None):+ with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):+ _ = ingest_mod.RepoIngestor+++def test_git_analyzer_package_import_succeeds_without_git():+ import semantica.ingest as ingest_mod++ ingest_mod.__dict__.pop("GitAnalyzer", None)+ with patch("semantica.ingest.repo_ingestor.git", None):+ analyzer_cls = ingest_mod.GitAnalyzer+ assert analyzer_cls is not None+ analyzer = analyzer_cls()+ assert analyzer is not None+++def test_salesforce_ingestor_package_import_missing_hint():+ import semantica.ingest as ingest_mod++ ingest_mod.__dict__.pop("SalesforceIngestor", None)+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):+ with pytest.raises(ImportError, match=r"semantica\[db-salesforce\]"):+ _ = ingest_mod.SalesforceIngestor+++def test_parse_methods_dynamic_default_resolution():+ from semantica.parse.methods import (+ get_parse_method,+ list_available_methods,+ parse_document,+ )++ assert get_parse_method("document", "default") == parse_document+ methods = list_available_methods()+ assert "default" in methods.get("document", [])+ assert "default" in methods.get("structured", [])++def test_node_embedder_gensim_missing_hint():with patch("semantica.kg.node_embeddings.GENSIM_AVAILABLE", False):from semantica.kg.node_embeddings import NodeEmbedder+with pytest.raises(ImportError, match=r"semantica\[graph-embeddings\]"):NodeEmbedder()@@ -150,6 +239,7 @@ def test_node_embedder_gensim_missing_hint():def test_faiss_store_missing_hint():with patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", False):from semantica.vector_store.faiss_store import FAISSIndexBuilder, FAISSStore+builder = FAISSIndexBuilder(128)with pytest.raises(ProcessingError, match=r"semantica\[vectorstore-faiss\]"):builder.build_index("flat")@@ -165,12 +255,16 @@ def test_visualization_missing_hint():# Plotly is checked via px and go in _check_dependencieswith patch("semantica.visualization.embedding_visualizer.px", None):visualizer = EmbeddingVisualizer()- with pytest.raises(ProcessingError, match=r"Plotly is required.*semantica\[viz\]"):+ with pytest.raises(+ ProcessingError, match=r"Plotly is required.*semantica\[viz\]"+ ):visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))with patch("semantica.visualization.embedding_visualizer.go", None):visualizer = EmbeddingVisualizer()- with pytest.raises(ProcessingError, match=r"Plotly is required.*semantica\[viz\]"):+ with pytest.raises(+ ProcessingError, match=r"Plotly is required.*semantica\[viz\]"+ ):visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))@@ -180,18 +274,25 @@ def test_visualization_umap_missing_hint():from semantica.visualization.embedding_visualizer import EmbeddingVisualizer# Stand in for Plotly so we reach dimensionality reduction- with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), \- patch("semantica.visualization.embedding_visualizer.go", MagicMock()), \- patch("semantica.visualization.embedding_visualizer.umap", None):+ with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(+ "semantica.visualization.embedding_visualizer.go", MagicMock()+ ), patch("semantica.visualization.embedding_visualizer.umap", None):visualizer = EmbeddingVisualizer()- # High-dimensional embeddings (>2D) trigger dimensionality reduction with method="umap"+ # High-dimensional embeddings (>2D) trigger dimensionality reduction+ # with method="umap"embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])- with pytest.raises(ProcessingError, match=r"UMAP is required.*semantica\[viz\]"):+ with pytest.raises(+ ProcessingError, match=r"UMAP is required.*semantica\[viz\]"+ ):visualizer.visualize_2d_projection(embeddings, method="umap")# Also verify 3D projection triggers the same actionable error on >3D embeddings- embeddings_4d = np.array([[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]])- with pytest.raises(ProcessingError, match=r"UMAP is required.*semantica\[viz\]"):+ embeddings_4d = np.array(+ [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]]+ )+ with pytest.raises(+ ProcessingError, match=r"UMAP is required.*semantica\[viz\]"+ ):visualizer.visualize_3d_projection(embeddings_4d, method="umap")@@ -201,8 +302,9 @@ def test_visualization_sklearn_missing_hint():from semantica.visualization.embedding_visualizer import EmbeddingVisualizer# Stand in for Plotly so we reach dimensionality reduction- with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), \- patch("semantica.visualization.embedding_visualizer.go", MagicMock()):+ with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(+ "semantica.visualization.embedding_visualizer.go", MagicMock()+ ):visualizer = EmbeddingVisualizer()embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])@@ -221,7 +323,7 @@ def test_visualization_sklearn_missing_hint():def test_visualization_options_collision_free():- """Options such as n_components, perplexity, n_neighbors must not cause keyword collisions."""+ """Options like n_components, perplexity must not cause keyword collisions."""import numpy as npfrom unittest.mock import MagicMockfrom semantica.visualization.embedding_visualizer import EmbeddingVisualizer@@ -232,26 +334,121 @@ def test_visualization_options_collision_free():mock_umap_module = MagicMock()mock_umap_module.UMAP = mock_umap_cls- with patch("semantica.visualization.embedding_visualizer.PCA", mock_pca), \- patch("semantica.visualization.embedding_visualizer.TSNE", mock_tsne), \- patch("semantica.visualization.embedding_visualizer.umap", mock_umap_module), \- patch("semantica.visualization.embedding_visualizer.px", MagicMock()), \- patch("semantica.visualization.embedding_visualizer.go", MagicMock()):+ with patch("semantica.visualization.embedding_visualizer.PCA", mock_pca), patch(+ "semantica.visualization.embedding_visualizer.TSNE", mock_tsne+ ), patch(+ "semantica.visualization.embedding_visualizer.umap", mock_umap_module+ ), patch(+ "semantica.visualization.embedding_visualizer.px", MagicMock()+ ), patch(+ "semantica.visualization.embedding_visualizer.go", MagicMock()+ ):visualizer = EmbeddingVisualizer()embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])# PCA with n_componentsvisualizer.visualize_2d_projection(embeddings, method="pca", n_components=2)# TSNE with perplexity and random_state- visualizer.visualize_2d_projection(embeddings, method="tsne", perplexity=1, random_state=42)+ visualizer.visualize_2d_projection(+ embeddings, method="tsne", perplexity=1, random_state=42+ )# UMAP with n_neighbors and min_dist- visualizer.visualize_2d_projection(embeddings, method="umap", n_neighbors=2, min_dist=0.1)+ visualizer.visualize_2d_projection(+ embeddings, method="umap", n_neighbors=2, min_dist=0.1+ )# 3D with n_componentsvisualizer.visualize_3d_projection(embeddings, method="pca", n_components=3)def test_spacy_load_missing_hint():from semantica.semantic_extract.methods import load_spacy_model+with patch("semantica.semantic_extract.methods.spacy", None):with pytest.raises(ImportError, match=r"semantica\[nlp-spacy\]"):load_spacy_model("en_core_web_sm")+++def test_xml_parser_handles_comments():+ xml_content = (+ "<root><!-- top comment --><item id='1'>Value</item>"+ "<!-- bottom comment --></root>"+ )+ # lxml engine+ p_lxml = XMLParser(engine="lxml")+ res_lxml = p_lxml.parse(xml_content)+ assert res_lxml.root.tag == "root"+ assert len(res_lxml.root.children) == 1+ assert res_lxml.root.children[0].tag == "item"+ assert res_lxml.root.children[0].text == "Value"++ # etree engine+ p_etree = XMLParser(engine="etree")+ res_etree = p_etree.parse(xml_content)+ assert res_etree.root.tag == "root"+ assert len(res_etree.root.children) == 1+ assert res_etree.root.children[0].tag == "item"+ assert res_etree.root.children[0].text == "Value"+++def test_public_api_ingestor_handles_xml_comments():+ from semantica.ingest.public_api_ingestor import PublicAPIIngestor++ xml_content = "<root><!-- comment --><item id='1'>Value</item></root>"+ ingestor = PublicAPIIngestor(rate_limit_delay=0)++ # 1. Default (defusedxml if available)+ parsed = ingestor._parse_xml(xml_content)+ assert parsed["tag"] == "root"+ assert len(parsed["children"]) == 1+ assert parsed["children"][0]["tag"] == "item"+ assert parsed["children"][0]["text"] == "Value"++ # 2. lxml fallback+ with patch("semantica.ingest.public_api_ingestor.safe_xml_etree", None):+ parsed_lxml = ingestor._parse_xml(xml_content)+ assert parsed_lxml["tag"] == "root"+ assert len(parsed_lxml["children"]) == 1+ assert parsed_lxml["children"][0]["tag"] == "item"+ assert parsed_lxml["children"][0]["text"] == "Value"+++def test_huggingface_model_loader_catches_oserror():+ import builtins+ from unittest.mock import MagicMock+ from semantica.semantic_extract.providers import HuggingFaceModelLoader++ mock_torch = MagicMock()+ mock_torch.Tensor = type("Tensor", (), {})+ with patch.dict("sys.modules", {"torch": mock_torch}):+ loader = HuggingFaceModelLoader()+ # 1. Test ModuleNotFoundError / ImportError+ with patch.dict("sys.modules", {"transformers": None}):+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):+ loader.load_ner_model("bert-base-cased")++ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):+ loader.load_relation_model("bert-base-cased")++ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):+ loader.load_triplet_model("t5-base")++ # 2. Test OSError (e.g. corrupt DLL / missing shared library)+ real_import = builtins.__import__++ def fake_import(name, *args, **kwargs):+ if name == "transformers":+ raise OSError("DLL load failed")+ return real_import(name, *args, **kwargs)++ try:+ builtins.__import__ = fake_import+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):+ loader.load_ner_model("bert-base-cased-oserror")++ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):+ loader.load_relation_model("bert-base-cased-oserror")++ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):+ loader.load_triplet_model("t5-base-oserror")+ finally:+ builtins.__import__ = real_import