Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(deps): address review feedback on import exceptions and probes (#1513)

Zohaib Hassnain1ff890abdd7b
diff --git a/.gitignore b/.gitignore
index 79197d52..bbfe449f 100644
Binary files a/.gitignore and b/.gitignore differ
diff --git a/integrations/google_adk/session_service.py b/integrations/google_adk/session_service.py
index f5a0a02d..1c97e099 100644
--- a/integrations/google_adk/session_service.py
+++ b/integrations/google_adk/session_service.py
@@ -21,6 +21,7 @@ from typing import Any, List, Optional
try:
from google.adk.events import Event
from google.adk.sessions import BaseSessionService, Session
+
try:
from google.adk.sessions import ListSessionsResponse
except ImportError:
@@ -33,7 +34,7 @@ try:
except ImportError:
from google.adk.sessions.base_session_service import GetSessionConfig
ADK_AVAILABLE = True
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
ADK_AVAILABLE = False
BaseSessionService = object
Session = Any
@@ -162,10 +163,10 @@ class SemanticaSessionService(BaseSessionService):
return {}
def _find_session_node(
- self,
- app_name: str,
- user_id: str,
- session_id: str,
+ self,
+ app_name: str,
+ user_id: str,
+ session_id: str,
) -> Optional[Any]:
"""Find a session node by its logical ADK session ID."""
expected_node_id = self._node_id(app_name, user_id, session_id)
@@ -182,18 +183,18 @@ class SemanticaSessionService(BaseSessionService):
metadata = node.get("metadata")
if (
- isinstance(metadata, dict)
- and str(metadata.get("session_id")) == str(session_id)
- and str(metadata.get("app_name")) == str(app_name)
- and str(metadata.get("user_id")) == str(user_id)
+ isinstance(metadata, dict)
+ and str(metadata.get("session_id")) == str(session_id)
+ and str(metadata.get("app_name")) == str(app_name)
+ and str(metadata.get("user_id")) == str(user_id)
):
return node
return None
def _find_node_by_id(
- self,
- node_id: str,
+ self,
+ node_id: str,
) -> Optional[Any]:
"""Find a ContextGraph node by graph node ID."""
for node in self.graph.find_nodes() or []:
@@ -212,13 +213,13 @@ class SemanticaSessionService(BaseSessionService):
data = SemanticaSessionService._safe_dict(event)
for field in (
- "id",
- "invocation_id",
- "author",
- "timestamp",
- "partial",
- "turn_complete",
- "branch",
+ "id",
+ "invocation_id",
+ "author",
+ "timestamp",
+ "partial",
+ "turn_complete",
+ "branch",
):
if field not in data and hasattr(event, field):
value = getattr(event, field)
@@ -231,10 +232,10 @@ class SemanticaSessionService(BaseSessionService):
return data
def _event_nodes(
- self,
- app_name: str,
- user_id: str,
- session_id: str,
+ self,
+ app_name: str,
+ user_id: str,
+ session_id: str,
) -> List[Any]:
"""Return all event nodes connected to a session."""
session_node_id = self._node_id(app_name, user_id, session_id)
@@ -275,8 +276,8 @@ class SemanticaSessionService(BaseSessionService):
return str(timestamp)
def _event_from_node(
- self,
- node: Any,
+ self,
+ node: Any,
) -> Any:
"""
Reconstruct an ADK Event from its stored metadata.
@@ -289,7 +290,7 @@ class SemanticaSessionService(BaseSessionService):
if not event_id and graph_node_id:
graph_node_id = str(graph_node_id)
if graph_node_id.startswith("adk-event:"):
- event_id = graph_node_id[len("adk-event:"):]
+ event_id = graph_node_id[len("adk-event:") :]
if event_id:
properties["id"] = event_id
@@ -310,11 +311,11 @@ class SemanticaSessionService(BaseSessionService):
@staticmethod
def _session_kwargs(
- app_name: str,
- user_id: str,
- session_id: str,
- state: Optional[dict],
- events: Optional[List[Any]],
+ app_name: str,
+ user_id: str,
+ session_id: str,
+ state: Optional[dict],
+ events: Optional[List[Any]],
) -> dict:
"""Build kwargs for the ADK Session model."""
return {
@@ -326,8 +327,8 @@ class SemanticaSessionService(BaseSessionService):
}
def _session_from_node(
- self,
- node: Any,
+ self,
+ node: Any,
) -> Session:
"""Reconstruct an ADK Session from a ContextGraph node."""
properties = self._node_properties(node)
@@ -347,14 +348,14 @@ class SemanticaSessionService(BaseSessionService):
# splitting on ':' after the prefix always yields
# exactly 3 parts regardless of what characters the
# original app_name/user_id/session_id contained.
- parts = graph_node_id[len("adk-session:"):].split(":")
+ parts = graph_node_id[len("adk-session:") :].split(":")
if len(parts) == 3:
decoded = [urllib.parse.unquote(part) for part in parts]
app_name = app_name or decoded[0]
user_id = user_id or decoded[1]
session_id = decoded[2]
else:
- session_id = graph_node_id[len("adk-session:"):]
+ session_id = graph_node_id[len("adk-session:") :]
else:
session_id = graph_node_id
@@ -383,12 +384,12 @@ class SemanticaSessionService(BaseSessionService):
# ------------------------------------------------------------------
async def create_session(
- self,
- *,
- app_name: str,
- user_id: str,
- state: Optional[dict[str, Any]] = None,
- session_id: Optional[str] = None,
+ self,
+ *,
+ app_name: str,
+ user_id: str,
+ state: Optional[dict[str, Any]] = None,
+ session_id: Optional[str] = None,
) -> Session:
"""Create and persist an ADK session."""
return await asyncio.to_thread(
@@ -396,11 +397,11 @@ class SemanticaSessionService(BaseSessionService):
)
def _create_session_sync(
- self,
- app_name: str,
- user_id: str,
- state: Optional[dict[str, Any]],
- session_id: Optional[str],
+ self,
+ app_name: str,
+ user_id: str,
+ state: Optional[dict[str, Any]],
+ session_id: Optional[str],
) -> Session:
with self._lock:
session_id = session_id or str(uuid.uuid4())
@@ -430,12 +431,12 @@ class SemanticaSessionService(BaseSessionService):
)
async def get_session(
- self,
- *,
- app_name: str,
- user_id: str,
- session_id: str,
- config: Optional[GetSessionConfig] = None,
+ self,
+ *,
+ app_name: str,
+ user_id: str,
+ session_id: str,
+ config: Optional[GetSessionConfig] = None,
) -> Optional[Session]:
"""Retrieve an ADK session from ContextGraph."""
return await asyncio.to_thread(
@@ -443,11 +444,11 @@ class SemanticaSessionService(BaseSessionService):
)
def _get_session_sync(
- self,
- app_name: str,
- user_id: str,
- session_id: str,
- config: Optional[GetSessionConfig],
+ self,
+ app_name: str,
+ user_id: str,
+ session_id: str,
+ config: Optional[GetSessionConfig],
) -> Optional[Session]:
with self._lock:
node = self._find_session_node(app_name, user_id, session_id)
@@ -468,7 +469,7 @@ class SemanticaSessionService(BaseSessionService):
# trims the already-built Session object.
if config:
if config.num_recent_events:
- session.events = session.events[-config.num_recent_events:]
+ session.events = session.events[-config.num_recent_events :]
if config.after_timestamp:
i = len(session.events) - 1
while i >= 0:
@@ -476,14 +477,14 @@ class SemanticaSessionService(BaseSessionService):
break
i -= 1
if i >= 0:
- session.events = session.events[i + 1:]
+ session.events = session.events[i + 1 :]
return session
async def append_event(
- self,
- session: Session,
- event: Event,
+ self,
+ session: Session,
+ event: Event,
) -> Event:
"""Persist an ADK event and associate it with a session."""
# ADK's own base implementation is a no-op for partial/streaming
@@ -510,8 +511,13 @@ class SemanticaSessionService(BaseSessionService):
# Verify cross-tenant security
properties = self._node_properties(session_node)
- if properties.get("app_name") != app_name or properties.get("user_id") != user_id:
- raise ValueError("Cross-tenant session write denied: app_name or user_id mismatch.")
+ if (
+ properties.get("app_name") != app_name
+ or properties.get("user_id") != user_id
+ ):
+ raise ValueError(
+ "Cross-tenant session write denied: app_name or user_id mismatch."
+ )
# Apply ADK in-memory event and state delta semantics. This
# runs inside asyncio.to_thread's worker thread, which has no
@@ -553,11 +559,11 @@ class SemanticaSessionService(BaseSessionService):
)
async def delete_session(
- self,
- *,
- app_name: str,
- user_id: str,
- session_id: str,
+ self,
+ *,
+ app_name: str,
+ user_id: str,
+ session_id: str,
) -> None:
"""Delete a session and all of its graph-backed events."""
await asyncio.to_thread(
@@ -565,10 +571,10 @@ class SemanticaSessionService(BaseSessionService):
)
def _delete_session_sync(
- self,
- app_name: str,
- user_id: str,
- session_id: str,
+ self,
+ app_name: str,
+ user_id: str,
+ session_id: str,
) -> None:
with self._lock:
session_node = self._find_session_node(app_name, user_id, session_id)
@@ -590,9 +596,9 @@ class SemanticaSessionService(BaseSessionService):
continue
if (
- edge.get("source") == session_node_id
- and edge.get("type") == "HAS_EVENT"
- and edge.get("target")
+ edge.get("source") == session_node_id
+ and edge.get("type") == "HAS_EVENT"
+ and edge.get("target")
):
event_node_ids.append(str(edge["target"]))
@@ -602,18 +608,18 @@ class SemanticaSessionService(BaseSessionService):
self.graph.purge_node(session_node_id)
async def list_sessions(
- self,
- *,
- app_name: str,
- user_id: Optional[str] = None,
+ self,
+ *,
+ app_name: str,
+ user_id: Optional[str] = None,
) -> ListSessionsResponse:
"""List sessions for an app, optionally scoped to one user."""
return await asyncio.to_thread(self._list_sessions_sync, app_name, user_id)
def _list_sessions_sync(
- self,
- app_name: str,
- user_id: Optional[str],
+ self,
+ app_name: str,
+ user_id: Optional[str],
) -> ListSessionsResponse:
with self._lock:
sessions: List[Session] = []
@@ -643,4 +649,4 @@ class SemanticaSessionService(BaseSessionService):
__all__ = [
"ADK_AVAILABLE",
"SemanticaSessionService",
-]
\ No newline at end of file
+]
diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.py
index e7859604..bba55644 100644
--- a/semantica/ingest/__init__.py
+++ b/semantica/ingest/__init__.py
@@ -133,7 +133,11 @@ import importlib
from typing import TYPE_CHECKING, Any, Dict, Tuple
if TYPE_CHECKING:
- from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor
+ from .salesforce_ingestor import (
+ SalesforceConnector,
+ SalesforceData,
+ SalesforceIngestor,
+ )
from .config import IngestConfig, ingest_config
from .file_ingestor import (
@@ -296,13 +300,55 @@ def __getattr__(name: str) -> Any:
module_name, attr_name = _LAZY_EXPORTS[name]
try:
module = importlib.import_module(module_name, __name__)
- except ModuleNotFoundError as exc:
+ except (ImportError, OSError) as exc:
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
missing_name = getattr(exc, "name", None)
- if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce", "lxml"}:
+ if message and (
+ missing_name is None
+ or any(
+ pkg in missing_name
+ for pkg in ("git", "bs4", "pyarrow", "simple_salesforce", "lxml")
+ )
+ ):
raise ImportError(message) from exc
raise
+ # Guard against backends whose modules imported cleanly with dependencies
+ # set to None; ensure probe imports (e.g. try: from semantica.ingest import ...)
+ # fail at import time rather than postponing failure to construction time.
+ if module_name == ".repo_ingestor" and name in {"RepoIngestor"}:
+ if getattr(module, "git", None) is None:
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
+ if module_name == ".xml_ingestor" and name in {"XMLIngestor"}:
+ if getattr(module, "etree", None) is None:
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
+ if module_name == ".parquet_ingestor" and name in {"ParquetIngestor"}:
+ if not getattr(module, "PARQUET_AVAILABLE", True):
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
+ if module_name == ".arrow_ingestor" and name in {"ArrowIngestor"}:
+ if not getattr(module, "ARROW_AVAILABLE", True):
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
+ if module_name == ".salesforce_ingestor" and name in {
+ "SalesforceIngestor",
+ "SalesforceConnector",
+ }:
+ if not getattr(module, "SALESFORCE_AVAILABLE", True):
+ message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
+ if message:
+ raise ImportError(message)
+
value = getattr(module, attr_name)
globals()[name] = value
return value
diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py
index 3736288b..7bd5684d 100644
--- a/semantica/ingest/methods.py
+++ b/semantica/ingest/methods.py
@@ -193,6 +193,7 @@ def _is_scp_like_repo_source(source: str) -> bool:
"""Return True for scp-like SSH remotes (``user@host:path``)."""
return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))
+
if TYPE_CHECKING:
from .api_ingestor import APIData
from .arrow_ingestor import ArrowData
@@ -252,7 +253,12 @@ def ingest_file(
if custom_method and custom_method != ingest_file:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -318,21 +324,18 @@ def ingest_parquet(
if custom_method and custom_method != ingest_parquet:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
- try:
- from .parquet_ingestor import ParquetIngestor
- except ModuleNotFoundError as exc:
- if _is_missing_dependency(exc, "pyarrow"):
- raise _missing_optional_dependency(
- "Parquet ingestion",
- "pyarrow",
- ) from exc
- raise
+ from .parquet_ingestor import ParquetIngestor
config = ingest_config.get_method_config("parquet")
config.update(kwargs)
@@ -397,21 +400,18 @@ def ingest_arrow(
if custom_method and custom_method != ingest_arrow:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
- try:
- from .arrow_ingestor import ArrowIngestor
- except ModuleNotFoundError as exc:
- if _is_missing_dependency(exc, "pyarrow"):
- raise _missing_optional_dependency(
- "Arrow ingestion",
- "pyarrow",
- ) from exc
- raise
+ from .arrow_ingestor import ArrowIngestor
config = ingest_config.get_method_config("arrow")
config.update(kwargs)
@@ -481,7 +481,12 @@ def ingest_xml(
if custom_method and custom_method != ingest_xml:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -491,7 +496,10 @@ def ingest_xml(
config = ingest_config.get_method_config("xml")
config.update(kwargs)
- ingestor = XMLIngestor(**config)
+ try:
+ ingestor = XMLIngestor(**config)
+ except ImportError as exc:
+ raise _missing_optional_dependency("XML ingestion", "lxml") from exc
def _run_single(
path: Union[str, Path],
@@ -511,6 +519,8 @@ def ingest_xml(
return _run_single(source_path)
+ except ConfigurationError:
+ raise
except Exception as e:
logger.error(f"Failed to ingest XML: {e}")
raise
@@ -545,7 +555,12 @@ def ingest_web(
if custom_method and custom_method != ingest_web:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -635,7 +650,12 @@ def ingest_public_api(
if custom_method and custom_method != ingest_public_api:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -722,7 +742,12 @@ def ingest_feed(
if custom_method and custom_method != ingest_feed:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -791,7 +816,12 @@ def ingest_stream(
if custom_method and custom_method != ingest_stream:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -868,20 +898,18 @@ def ingest_repository(
if custom_method and custom_method != ingest_repository:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
- try:
- from .repo_ingestor import RepoIngestor
- except ModuleNotFoundError as exc:
- if _is_missing_dependency(exc, "git"):
- raise _missing_optional_dependency(
- "Repository ingestion", "GitPython"
- ) from exc
- raise
+ from .repo_ingestor import RepoIngestor
# Get config
config = ingest_config.get_method_config("repo")
@@ -945,7 +973,12 @@ def ingest_email(
if custom_method and custom_method != ingest_email:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -1024,7 +1057,12 @@ def ingest_ontology(
if custom_method and custom_method != ingest_ontology:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -1090,7 +1128,12 @@ def ingest_database(
if custom_method and custom_method != ingest_database:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -1238,105 +1281,122 @@ def ingest_salesforce(
if custom_method and custom_method != ingest_salesforce:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source,
- fallback_on_custom_error=fallback, **kwargs,
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
try:
from .salesforce_ingestor import SalesforceIngestor
- except ModuleNotFoundError as exc:
- if _is_missing_dependency(exc, "simple_salesforce"):
+
+ # Unpack credential dict (if given); everything else stays in kwargs.
+ creds: Dict[str, Any] = {}
+ if source is not None:
+ if not isinstance(source, dict):
+ raise ProcessingError(
+ "ingest_salesforce() source must be a credential dict or None. "
+ "Pass sobject_name / soql as keyword arguments."
+ )
+ creds = dict(source)
+
+ # Merge any ingest_config method config under "salesforce".
+ # get_method_config() now returns a copy, so this dict is safe to mutate.
+ # We build the final connector config in order of increasing priority:
+ # 1. base method config (lowest — global defaults set by operator)
+ # 2. per-call credential dict supplied via `source`
+ # 3. per-call connector params supplied as kwargs
+ # Credentials are extracted from kwargs and removed so they don't also
+ # flow into the ingest method call (which doesn't understand them).
+ _CONNECTOR_PARAMS = frozenset(
+ {
+ "username",
+ "password",
+ "security_token",
+ "domain",
+ "instance_url",
+ "session_id",
+ "api_version",
+ }
+ )
+ connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
+ for k in _CONNECTOR_PARAMS:
+ kwargs.pop(k, None)
+
+ # Build a fresh per-call config dict — never mutate the global store.
+ config: Dict[str, Any] = {
+ **ingest_config.get_method_config("salesforce"), # base (already a copy)
+ **creds, # source dict credentials
+ **connector_kwargs, # kwarg credentials
+ }
+
+ try:
+ ingestor = SalesforceIngestor(**config)
+ except ImportError as exc:
raise _missing_optional_dependency(
"Salesforce ingestion", "simple-salesforce"
) from exc
- raise
- # Unpack credential dict (if given); everything else stays in kwargs.
- creds: Dict[str, Any] = {}
- if source is not None:
- if not isinstance(source, dict):
- raise ProcessingError(
- "ingest_salesforce() source must be a credential dict or None. "
- "Pass sobject_name / soql as keyword arguments."
- )
- creds = dict(source)
-
- # Merge any ingest_config method config under "salesforce".
- # get_method_config() now returns a copy, so this dict is safe to mutate.
- # We build the final connector config in order of increasing priority:
- # 1. base method config (lowest — global defaults set by operator)
- # 2. per-call credential dict supplied via `source`
- # 3. per-call connector params supplied as kwargs
- # Credentials are extracted from kwargs and removed so they don't also
- # flow into the ingest method call (which doesn't understand them).
- _CONNECTOR_PARAMS = frozenset({
- "username", "password", "security_token", "domain",
- "instance_url", "session_id", "api_version",
- })
- connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
- for k in _CONNECTOR_PARAMS:
- kwargs.pop(k, None)
-
- # Build a fresh per-call config dict — never mutate the global store.
- config: Dict[str, Any] = {
- **ingest_config.get_method_config("salesforce"), # base (already a copy)
- **creds, # source dict credentials
- **connector_kwargs, # kwarg credentials
- }
-
- ingestor = SalesforceIngestor(**config)
-
- if method == "sobject":
- sobject_name = kwargs.pop("sobject_name", None)
- if not sobject_name:
- raise ProcessingError(
- "ingest_salesforce() with method='sobject' requires "
- "sobject_name keyword argument."
- )
- return ingestor.ingest_sobject(sobject_name, **kwargs)
+ if method == "sobject":
+ sobject_name = kwargs.pop("sobject_name", None)
+ if not sobject_name:
+ raise ProcessingError(
+ "ingest_salesforce() with method='sobject' requires "
+ "sobject_name keyword argument."
+ )
+ return ingestor.ingest_sobject(sobject_name, **kwargs)
- elif method == "query":
- soql = kwargs.pop("soql", None)
- if not soql:
- raise ProcessingError(
- "ingest_salesforce() with method='query' requires "
- "soql keyword argument."
- )
- return ingestor.ingest_query(soql, **kwargs)
+ elif method == "query":
+ soql = kwargs.pop("soql", None)
+ if not soql:
+ raise ProcessingError(
+ "ingest_salesforce() with method='query' requires "
+ "soql keyword argument."
+ )
+ return ingestor.ingest_query(soql, **kwargs)
- elif method == "list_sobjects":
- return ingestor.list_sobjects()
+ elif method == "list_sobjects":
+ return ingestor.list_sobjects()
- elif method == "schema":
- sobject_name = kwargs.pop("sobject_name", None)
- if not sobject_name:
- raise ProcessingError(
- "ingest_salesforce() with method='schema' requires "
- "sobject_name keyword argument."
+ elif method == "schema":
+ sobject_name = kwargs.pop("sobject_name", None)
+ if not sobject_name:
+ raise ProcessingError(
+ "ingest_salesforce() with method='schema' requires "
+ "sobject_name keyword argument."
+ )
+ return ingestor.get_sobject_schema(sobject_name)
+
+ elif method == "documents":
+ sobject_name = kwargs.pop("sobject_name", None)
+ if not sobject_name:
+ raise ProcessingError(
+ "ingest_salesforce() with method='documents' requires "
+ "sobject_name keyword argument."
+ )
+ id_field = kwargs.pop("id_field", "Id")
+ text_fields = kwargs.pop("text_fields", None)
+ data = ingestor.ingest_sobject(sobject_name, **kwargs)
+ return ingestor.export_as_documents(
+ data, id_field=id_field, text_fields=text_fields
)
- return ingestor.get_sobject_schema(sobject_name)
- elif method == "documents":
- sobject_name = kwargs.pop("sobject_name", None)
- if not sobject_name:
+ else:
raise ProcessingError(
- "ingest_salesforce() with method='documents' requires "
- "sobject_name keyword argument."
+ f"Unknown ingest_salesforce method: {method!r}. "
+ "Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
+ "'documents'."
)
- id_field = kwargs.pop("id_field", "Id")
- text_fields = kwargs.pop("text_fields", None)
- data = ingestor.ingest_sobject(sobject_name, **kwargs)
- return ingestor.export_as_documents(data, id_field=id_field,
- text_fields=text_fields)
- else:
- raise ProcessingError(
- f"Unknown ingest_salesforce method: {method!r}. "
- "Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
- "'documents'."
- )
+ except ConfigurationError:
+ raise
+ except Exception as e:
+ logger.error(f"Failed to ingest salesforce: {e}")
+ raise
def ingest_mcp(
@@ -1404,7 +1464,12 @@ def ingest_mcp(
if custom_method and custom_method != ingest_mcp:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ source,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -1643,8 +1708,9 @@ def ingest(
elif source_type == "mcp":
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
elif source_type == "salesforce":
- return {"data": ingest_salesforce(sources,
- method=method or "sobject", **kwargs)}
+ return {
+ "data": ingest_salesforce(sources, method=method or "sobject", **kwargs)
+ }
else:
raise ProcessingError(f"Unknown source type: {source_type}")
diff --git a/semantica/ingest/public_api_ingestor.py b/semantica/ingest/public_api_ingestor.py
index 7df07be4..001c132e 100644
--- a/semantica/ingest/public_api_ingestor.py
+++ b/semantica/ingest/public_api_ingestor.py
@@ -31,15 +31,18 @@ import requests
try:
from lxml import etree as lxml_etree
+
_SAFE_XML_PARSER = lxml_etree.XMLParser(
resolve_entities=False,
no_network=True,
recover=False,
huge_tree=False,
load_dtd=False,
+ remove_comments=True,
+ remove_pis=True,
)
_LXML_SYNTAX_ERRORS: Tuple[type, ...] = (lxml_etree.XMLSyntaxError,)
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
lxml_etree = None
_SAFE_XML_PARSER = None
_LXML_SYNTAX_ERRORS = ()
@@ -47,7 +50,7 @@ except (ImportError, ModuleNotFoundError):
try:
from defusedxml import ElementTree as safe_xml_etree
from defusedxml.common import DefusedXmlException
-except ModuleNotFoundError: # pragma: no cover - fallback for minimal installs
+except (ImportError, OSError): # pragma: no cover - fallback for minimal installs
safe_xml_etree = None
class DefusedXmlException(Exception):
@@ -449,7 +452,9 @@ class PublicAPIIngestor(RESTIngestor):
APIData: Normalized public API response and metadata
"""
self._validate_endpoint(endpoint)
- self._validate_no_auth_request(headers=headers, params=params, options=options, endpoint=endpoint)
+ self._validate_no_auth_request(
+ headers=headers, params=params, options=options, endpoint=endpoint
+ )
tracking_id = self.progress_tracker.start_tracking(
file=endpoint,
@@ -610,7 +615,9 @@ class PublicAPIIngestor(RESTIngestor):
for endpoint in endpoints:
try:
results.append(
- self.ingest_public_api(endpoint, method=method, **copy.deepcopy(options))
+ self.ingest_public_api(
+ endpoint, method=method, **copy.deepcopy(options)
+ )
)
except Exception as exc:
self.logger.warning(f"Failed to fetch public API {endpoint}: {exc}")
@@ -781,6 +788,16 @@ class PublicAPIIngestor(RESTIngestor):
xml_text.encode("utf-8"),
parser=_SAFE_XML_PARSER,
)
+ for elem in root.iter():
+ if (
+ elem.tag is lxml_etree.Comment
+ or elem.tag is lxml_etree.PI
+ or getattr(elem.tag, "__name__", "")
+ in ("Comment", "ProcessingInstruction", "PI")
+ ):
+ continue
+ if callable(elem.tag) or not isinstance(elem.tag, str):
+ raise ProcessingError("Failed to parse XML public API response")
else:
raise ProcessingError(
"XML parsing requires 'defusedxml' or 'lxml'. "
@@ -789,7 +806,17 @@ class PublicAPIIngestor(RESTIngestor):
return self._element_to_dict(root)
def _element_to_dict(self, element: Any) -> Dict[str, Any]:
- children = [self._element_to_dict(child) for child in list(element)]
+ children = [
+ self._element_to_dict(child)
+ for child in list(element)
+ if not (
+ callable(child.tag)
+ or (
+ lxml_etree is not None
+ and (child.tag is lxml_etree.Comment or child.tag is lxml_etree.PI)
+ )
+ )
+ ]
return {
"tag": self._strip_namespace(element.tag),
"attributes": {
@@ -800,7 +827,9 @@ class PublicAPIIngestor(RESTIngestor):
"children": children,
}
- def _strip_namespace(self, value: str) -> str:
+ def _strip_namespace(self, value: Any) -> str:
+ if not isinstance(value, str):
+ return str(value)
if value.startswith("{") and "}" in value:
return value.split("}", 1)[1]
return value
diff --git a/semantica/ingest/repo_ingestor.py b/semantica/ingest/repo_ingestor.py
index 5996a9d7..1f8a906f 100644
--- a/semantica/ingest/repo_ingestor.py
+++ b/semantica/ingest/repo_ingestor.py
@@ -28,6 +28,7 @@ Example Usage:
Author: Semantica Contributors
License: MIT
"""
+
from __future__ import annotations
import ipaddress
@@ -47,7 +48,7 @@ from urllib.parse import urlparse
try:
import git
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
git = None
from ..utils.exceptions import ProcessingError, ValidationError
@@ -61,9 +62,7 @@ ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"
ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})
# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
-_ENV_VAR_TOKEN_RE = re.compile(
- r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"
-)
+_ENV_VAR_TOKEN_RE = re.compile(r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)")
# Short-lived DNS cache for host validation. This reduces repeated lookups but
# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —
# network egress controls remain recommended.
@@ -599,10 +598,7 @@ class RepoIngestor:
networks). Those addresses are not SSRF-sensitive.
"""
return bool(
- ip.is_private
- or ip.is_loopback
- or ip.is_link_local
- or ip.is_unspecified
+ ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_unspecified
)
@staticmethod
@@ -629,9 +625,7 @@ class RepoIngestor:
# or hanging lookup for one host cannot stall cache access for
# concurrent lookups of other hosts.
try:
- addrinfos = socket.getaddrinfo(
- host, None, type=socket.SOCK_STREAM
- )
+ addrinfos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
except socket.gaierror as exc:
raise ValidationError(
f"Cannot resolve repository host {host!r}: {exc}"
@@ -804,9 +798,7 @@ class RepoIngestor:
f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}"
)
if not parsed.netloc or not host:
- raise ValidationError(
- f"Repository URL must include a host: {repo_url}"
- )
+ raise ValidationError(f"Repository URL must include a host: {repo_url}")
RepoIngestor._validate_repo_host(host)
@@ -821,9 +813,7 @@ class RepoIngestor:
"include_extensions",
"max_depth",
}
- candidate = {
- k: v for k, v in options.items() if k not in non_git_options
- }
+ candidate = {k: v for k, v in options.items() if k not in non_git_options}
unsafe = set(candidate) - ALLOWED_CLONE_OPTIONS
if unsafe:
raise ValidationError(
@@ -902,9 +892,7 @@ class RepoIngestor:
if "include_extensions" in options:
# Normalize extensions to include dot prefix
exts = options["include_extensions"]
- normalized_exts = [
- e if e.startswith(".") else f".{e}" for e in exts
- ]
+ normalized_exts = [e if e.startswith(".") else f".{e}" for e in exts]
file_filters["extensions"] = normalized_exts
# Process code files
@@ -1120,6 +1108,7 @@ class RepoIngestor:
def cleanup(self):
"""Cleanup temporary repository files."""
if self.temp_dir and os.path.exists(self.temp_dir):
+
def onexc(func, path, exc_info):
"""
Error handler for shutil.rmtree.
@@ -1132,6 +1121,7 @@ class RepoIngestor:
Usage : shutil.rmtree(path, onerror=onexc)
"""
import stat
+
if not os.access(path, os.W_OK):
# Is the error an access error ?
os.chmod(path, stat.S_IWUSR)
diff --git a/semantica/ingest/xml_ingestor.py b/semantica/ingest/xml_ingestor.py
index 9f3be0ed..84f4067a 100644
--- a/semantica/ingest/xml_ingestor.py
+++ b/semantica/ingest/xml_ingestor.py
@@ -26,7 +26,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
try:
from lxml import etree
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
etree = None
from ..utils.constants import FILE_SIZE_LIMITS
@@ -76,7 +76,7 @@ class XMLIngestor:
"""
self.logger = get_logger("xml_ingestor")
if etree is None:
- raise ProcessingError(
+ raise ImportError(
"lxml is required for XMLIngestor. "
"Install it with: pip install 'semantica[documents]'"
)
diff --git a/semantica/parse/docx_parser.py b/semantica/parse/docx_parser.py
index 390e8ba8..9a8f1d50 100644
--- a/semantica/parse/docx_parser.py
+++ b/semantica/parse/docx_parser.py
@@ -39,7 +39,7 @@ try:
from docx.oxml.text.paragraph import CT_P
from docx.table import Table
from docx.text.paragraph import Paragraph
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
Document = None
DocxDocument = None
CT_Tbl = None
@@ -92,7 +92,9 @@ class DOCXParser:
self.config = config
self.progress_tracker = get_progress_tracker()
- def parse(self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options) -> Dict[str, Any]:
+ def parse(
+ self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options
+ ) -> Dict[str, Any]:
"""
Parse DOCX document.
diff --git a/semantica/parse/excel_parser.py b/semantica/parse/excel_parser.py
index c3ba3d91..c1b3bab5 100644
--- a/semantica/parse/excel_parser.py
+++ b/semantica/parse/excel_parser.py
@@ -33,9 +33,10 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Union
import pandas as pd
+
try:
from openpyxl import load_workbook
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
load_workbook = None
from ..utils.exceptions import ProcessingError, ValidationError
diff --git a/semantica/parse/html_parser.py b/semantica/parse/html_parser.py
index 8cf4007e..b642f381 100644
--- a/semantica/parse/html_parser.py
+++ b/semantica/parse/html_parser.py
@@ -35,7 +35,7 @@ from urllib.parse import urljoin
try:
from bs4 import BeautifulSoup
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
BeautifulSoup = None
from ..utils.exceptions import ProcessingError, ValidationError
diff --git a/semantica/parse/methods.py b/semantica/parse/methods.py
index 8dd74824..9eb44ae7 100644
--- a/semantica/parse/methods.py
+++ b/semantica/parse/methods.py
@@ -123,7 +123,6 @@ Example Usage:
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
-from ..utils.exceptions import ConfigurationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
from .code_parser import CodeParser
@@ -181,7 +180,13 @@ def parse_document(
if custom_method and custom_method != parse_document:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, file_type, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ file_type,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -248,7 +253,8 @@ def parse_document_docling(
# Register Docling method
try:
- from .docling_parser import DoclingParser
+ from . import docling_parser # noqa: F401
+
method_registry.register("document", "docling", parse_document_docling)
except (ImportError, OSError):
# Docling not available, skip registration
@@ -292,7 +298,14 @@ def parse_web_content(
if custom_method and custom_method != parse_web_content:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, content, content_type, base_url, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ content,
+ content_type,
+ base_url,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -348,7 +361,13 @@ def parse_structured_data(
if custom_method and custom_method != parse_structured_data:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, data, data_format, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ data,
+ data_format,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -395,7 +414,12 @@ def parse_email(
if custom_method and custom_method != parse_email:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, email_content, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ email_content,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -445,7 +469,13 @@ def parse_code(
if custom_method and custom_method != parse_code:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, language, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ language,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -498,7 +528,13 @@ def parse_media(
if custom_method and custom_method != parse_media:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, media_type, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ media_type,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -544,7 +580,12 @@ def parse_pdf(
if custom_method and custom_method not in (parse_pdf, parse_document):
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -588,7 +629,12 @@ def parse_docx(
if custom_method and custom_method not in (parse_docx, parse_document):
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -631,7 +677,12 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) -
if custom_method and custom_method not in (parse_json, parse_structured_data):
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -678,7 +729,13 @@ def parse_csv(
if custom_method and custom_method not in (parse_csv, parse_structured_data):
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, delimiter, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ delimiter,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -717,7 +774,12 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) ->
if custom_method and custom_method not in (parse_xml, parse_structured_data):
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
@@ -765,7 +827,12 @@ def parse_image(
if custom_method and custom_method not in (parse_image, parse_media):
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
- logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
+ logger,
+ method,
+ custom_method,
+ file_path,
+ fallback_on_custom_error=fallback,
+ **kwargs,
)
if result is not CUSTOM_METHOD_FELL_BACK:
return result
diff --git a/semantica/parse/web_parser.py b/semantica/parse/web_parser.py
index 0ea4f8e7..2a4cc3be 100644
--- a/semantica/parse/web_parser.py
+++ b/semantica/parse/web_parser.py
@@ -35,7 +35,7 @@ from urllib.parse import urljoin, urlparse
try:
from bs4 import BeautifulSoup
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
BeautifulSoup = None
from ..utils.exceptions import ProcessingError, ValidationError
diff --git a/semantica/parse/xml_parser.py b/semantica/parse/xml_parser.py
index 5088577c..d12f430b 100644
--- a/semantica/parse/xml_parser.py
+++ b/semantica/parse/xml_parser.py
@@ -36,7 +36,7 @@ from typing import Any, Dict, List, Optional, Union
try:
from lxml import etree
-except (ImportError, ModuleNotFoundError):
+except (ImportError, OSError):
etree = None
from ..utils.exceptions import ProcessingError, ValidationError
@@ -156,7 +156,7 @@ class XMLParser:
self, xml_string: str, source: str, options: Dict[str, Any]
) -> XMLData:
"""Parse XML using lxml."""
- parser = etree.XMLParser(remove_blank_text=True)
+ parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
root = etree.fromstring(xml_string.encode("utf-8"), parser)
# Extract namespaces
@@ -197,8 +197,10 @@ class XMLParser:
metadata={"source": source, "engine": "etree"},
)
- def _element_to_xml_element(self, element) -> XMLElement:
+ def _element_to_xml_element(self, element) -> Optional[XMLElement]:
"""Convert lxml element to XMLElement."""
+ if not hasattr(element, "tag") or not isinstance(element.tag, str):
+ return None
tag = element.tag
if "}" in tag:
namespace, tag = tag.split("}", 1)
@@ -215,12 +217,18 @@ class XMLParser:
# Process children
for child in element:
- xml_elem.children.append(self._element_to_xml_element(child))
+ child_elem = self._element_to_xml_element(child)
+ if child_elem is not None:
+ xml_elem.children.append(child_elem)
return xml_elem
- def _etree_element_to_xml_element(self, element: ET.Element) -> XMLElement:
+ def _etree_element_to_xml_element(
+ self, element: ET.Element
+ ) -> Optional[XMLElement]:
"""Convert ElementTree element to XMLElement."""
+ if not hasattr(element, "tag") or not isinstance(element.tag, str):
+ return None
tag = element.tag
if "}" in tag:
namespace, tag = tag.split("}", 1)
@@ -237,7 +245,9 @@ class XMLParser:
# Process children
for child in element:
- xml_elem.children.append(self._etree_element_to_xml_element(child))
+ child_elem = self._etree_element_to_xml_element(child)
+ if child_elem is not None:
+ xml_elem.children.append(child_elem)
return xml_elem
@@ -267,15 +277,21 @@ class XMLParser:
xml_string = (
file_path
if isinstance(file_path, str) and not Path(file_path).exists()
- else Path(file_path).read_text()
+ else Path(file_path).read_text(encoding="utf-8")
)
- root = etree.fromstring(xml_string.encode("utf-8"))
+ parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
+ root = etree.fromstring(xml_string.encode("utf-8"), parser=parser)
# Register namespaces for XPath
namespaces = xml_data.namespaces
elements = root.xpath(xpath, namespaces=namespaces)
- return [self._element_to_xml_element(elem) for elem in elements]
+ results = []
+ for elem in elements:
+ xml_elem = self._element_to_xml_element(elem)
+ if xml_elem is not None:
+ results.append(xml_elem)
+ return results
def extract_by_tag(
self, file_path: Union[str, Path], tag_name: str, **options
diff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.py
index 5e76e8f6..64d9ebcf 100644
--- a/semantica/semantic_extract/providers.py
+++ b/semantica/semantic_extract/providers.py
@@ -61,7 +61,7 @@ Example Usage:
>>> from semantica.semantic_extract.providers import create_provider
>>> provider = create_provider("openai", model="gpt-4")
>>> response = provider.generate("Extract entities from: Apple Inc. was founded in 1976.")
- >>>
+ >>>
>>> loader = HuggingFaceModelLoader(device="cuda")
>>> ner_model = loader.load_ner_model("dslim/bert-base-NER")
@@ -70,7 +70,6 @@ License: MIT
"""
import json
-import os
import time
from typing import Any, Dict, List, Optional, Union, Type
@@ -121,7 +120,7 @@ class BaseProvider:
"""Extract and parse JSON from text, supporting objects and lists."""
if not text:
raise ProcessingError("Empty response from LLM")
-
+
# Clean up text - remove markdown code blocks if present
cleaned_text = text.strip()
if "```json" in cleaned_text:
@@ -131,8 +130,9 @@ class BaseProvider:
blocks = cleaned_text.split("```")
for block in blocks:
block = block.strip()
- if (block.startswith("{") and block.endswith("}")) or \
- (block.startswith("[") and block.endswith("]")):
+ if (block.startswith("{") and block.endswith("}")) or (
+ block.startswith("[") and block.endswith("]")
+ ):
cleaned_text = block
break
@@ -150,22 +150,22 @@ class BaseProvider:
# Try to find JSON boundaries (outermost { } or [ ])
start_obj = cleaned_text.find("{")
start_list = cleaned_text.find("[")
-
+
# Determine which one starts first
start = -1
if start_obj >= 0 and (start_list < 0 or start_obj < start_list):
start = start_obj
elif start_list >= 0:
start = start_list
-
+
if start >= 0:
# Find the corresponding end
end_obj = cleaned_text.rfind("}")
end_list = cleaned_text.rfind("]")
end = max(end_obj, end_list)
-
+
if end > start:
- candidate = cleaned_text[start:end+1]
+ candidate = cleaned_text[start : end + 1]
try:
return json.loads(candidate)
except json.JSONDecodeError:
@@ -176,78 +176,88 @@ class BaseProvider:
# Last resort: if it's truncated, try to close it
if candidate.endswith("..."):
candidate = candidate[:-3].strip()
-
+
# Simple attempt to close unclosed structures
open_braces = candidate.count("{") - candidate.count("}")
open_brackets = candidate.count("[") - candidate.count("]")
-
+
fixed_candidate = candidate
if open_braces > 0:
fixed_candidate += "}" * open_braces
if open_brackets > 0:
fixed_candidate += "]" * open_brackets
-
+
try:
return json.loads(fix_json(fixed_candidate))
except json.JSONDecodeError as e:
- raise ProcessingError(f"Failed to parse JSON from LLM response after cleaning: {e}")
-
- raise ProcessingError(f"No valid JSON structure found in response. Preview: {text[:100]}...")
+ raise ProcessingError(
+ f"Failed to parse JSON from LLM response after cleaning: {e}"
+ )
+
+ raise ProcessingError(
+ f"No valid JSON structure found in response. Preview: {text[:100]}..."
+ )
- def generate_structured(self, prompt: str, max_retries: int = 3, **kwargs) -> Union[dict, list]:
+ def generate_structured(
+ self, prompt: str, max_retries: int = 3, **kwargs
+ ) -> Union[dict, list]:
"""Generate structured output with retry logic."""
last_error = None
import time
-
+
for attempt in range(max_retries):
try:
# Add explicit JSON instruction if not present
structured_prompt = prompt
if "JSON" not in prompt:
- structured_prompt = f"{prompt}\n\nReturn the response as valid JSON only."
-
+ structured_prompt = (
+ f"{prompt}\n\nReturn the response as valid JSON only."
+ )
+
content = self.generate(structured_prompt, **kwargs)
result = self._parse_json(content)
-
+
# Basic validation: ensure it's not empty if we expect data
if not result and attempt < max_retries - 1:
- self.logger.warning(f"Empty structured response (attempt {attempt + 1}/{max_retries}). Retrying...")
+ self.logger.warning(
+ f"Empty structured response (attempt {attempt + 1}/{max_retries}). Retrying..."
+ )
continue
-
+
return result
-
+
except (ProcessingError, Exception) as e:
last_error = e
if attempt < max_retries - 1:
- wait_time = (attempt + 1) * 2 # Simple backoff
- self.logger.warning(f"Extraction error (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s...")
+ wait_time = (attempt + 1) * 2 # Simple backoff
+ self.logger.warning(
+ f"Extraction error (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s..."
+ )
time.sleep(wait_time)
else:
- self.logger.error(f"Structured generation failed after {max_retries} attempts: {e}")
-
+ self.logger.error(
+ f"Structured generation failed after {max_retries} attempts: {e}"
+ )
+
if last_error:
raise ProcessingError(f"Failed to generate structured output: {last_error}")
return []
def generate_typed(
- self,
- prompt: str,
- schema: Type[BaseModel],
- max_retries: int = 3,
- **kwargs
+ self, prompt: str, schema: Type[BaseModel], max_retries: int = 3, **kwargs
) -> BaseModel:
"""
Generate structured output validated against a Pydantic schema.
Uses instructor if available and supported for the provider, otherwise falls back to a repair loop.
"""
provider_name = self.__class__.__name__
-
+
# Try using instructor first if available
if instructor:
try:
client = None
mode = instructor.Mode.TOOLS # Default mode
-
+
if provider_name == "OpenAIProvider" and self.client:
custom_base_url = getattr(self, "base_url", None)
if custom_base_url:
@@ -255,7 +265,9 @@ class BaseProvider:
# supported by third-party servers (Qwen, LLaMA gateways, etc.).
# Mode.JSON asks the model to return plain JSON and is broadly
# supported across all OpenAI-compatible APIs.
- client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)
+ client = instructor.from_openai(
+ self.client, mode=instructor.Mode.JSON
+ )
elif hasattr(instructor, "from_provider"):
try:
client = instructor.from_provider(
@@ -270,8 +282,8 @@ class BaseProvider:
if hasattr(instructor, "from_provider"):
try:
client = instructor.from_provider(
- provider=f"anthropic/{kwargs.get('model', self.model)}",
- api_key=self.api_key
+ provider=f"anthropic/{kwargs.get('model', self.model)}",
+ api_key=self.api_key,
)
except Exception:
client = instructor.from_anthropic(self.client)
@@ -281,26 +293,24 @@ class BaseProvider:
if hasattr(instructor, "from_provider"):
try:
client = instructor.from_provider(
- provider=f"gemini/{kwargs.get('model', self.model)}",
- api_key=self.api_key
+ provider=f"gemini/{kwargs.get('model', self.model)}",
+ api_key=self.api_key,
)
except Exception:
client = instructor.from_gemini(
- self.client,
- mode=instructor.Mode.GEMINI_JSON
+ self.client, mode=instructor.Mode.GEMINI_JSON
)
else:
client = instructor.from_gemini(
- self.client,
- mode=instructor.Mode.GEMINI_JSON
+ self.client, mode=instructor.Mode.GEMINI_JSON
)
elif provider_name == "GroqProvider" and self.client:
# Try using from_provider which is recommended for Groq in latest instructor
if hasattr(instructor, "from_provider"):
try:
client = instructor.from_provider(
- provider=f"groq/{kwargs.get('model', self.model)}",
- api_key=self.api_key
+ provider=f"groq/{kwargs.get('model', self.model)}",
+ api_key=self.api_key,
)
except Exception:
client = None
@@ -308,21 +318,28 @@ class BaseProvider:
if not client:
# Try using from_groq if available (newer instructor versions)
if hasattr(instructor, "from_groq"):
- client = instructor.from_groq(self.client, mode=instructor.Mode.JSON)
+ client = instructor.from_groq(
+ self.client, mode=instructor.Mode.JSON
+ )
else:
# Fallback: Create OpenAI client pointing to Groq
# This avoids the "Client should be an instance of openai.OpenAI" warning
try:
from openai import OpenAI
+
# Fix: Use self.api_key instead of self.client.api_key
groq_client = OpenAI(
base_url="https://api.groq.com/openai/v1",
api_key=self.api_key,
)
- client = instructor.from_openai(groq_client, mode=instructor.Mode.JSON)
+ client = instructor.from_openai(
+ groq_client, mode=instructor.Mode.JSON
+ )
except Exception:
# Last resort: try passing the groq client directly
- client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)
+ client = instructor.from_openai(
+ self.client, mode=instructor.Mode.JSON
+ )
elif provider_name == "OllamaProvider":
# Try from_provider for Ollama if available
if hasattr(instructor, "from_provider"):
@@ -332,21 +349,26 @@ class BaseProvider:
)
except Exception:
client = None
-
+
if not client:
# Create OpenAI-compatible client for Ollama
try:
from openai import OpenAI
+
# Ollama typically runs on localhost:11434/v1
- base_url = getattr(self, "base_url", "http://localhost:11434")
+ base_url = getattr(
+ self, "base_url", "http://localhost:11434"
+ )
if not base_url.endswith("/v1"):
base_url = f"{base_url.rstrip('/')}/v1"
-
+
ollama_client = OpenAI(
base_url=base_url,
- api_key="ollama", # required but unused
+ api_key="ollama", # required but unused
+ )
+ client = instructor.from_openai(
+ ollama_client, mode=instructor.Mode.JSON
)
- client = instructor.from_openai(ollama_client, mode=instructor.Mode.JSON)
except ImportError:
pass
elif provider_name == "DeepSeekProvider" and self.client:
@@ -355,7 +377,7 @@ class BaseProvider:
try:
client = instructor.from_provider(
provider=f"deepseek/{kwargs.get('model', self.model)}",
- api_key=self.api_key
+ api_key=self.api_key,
)
except Exception:
client = None
@@ -364,15 +386,20 @@ class BaseProvider:
# DeepSeek is OpenAI compatible
try:
from openai import OpenAI
+
if isinstance(self.client, OpenAI):
- client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)
+ client = instructor.from_openai(
+ self.client, mode=instructor.Mode.JSON
+ )
else:
- # Try creating fresh client
- ds_client = OpenAI(
- api_key=self.api_key,
- base_url="https://api.deepseek.com"
- )
- client = instructor.from_openai(ds_client, mode=instructor.Mode.JSON)
+ # Try creating fresh client
+ ds_client = OpenAI(
+ api_key=self.api_key,
+ base_url="https://api.deepseek.com",
+ )
+ client = instructor.from_openai(
+ ds_client, mode=instructor.Mode.JSON
+ )
except Exception:
pass
@@ -383,7 +410,9 @@ class BaseProvider:
# Format for litellm in instructor is litellm/model_name
provider_model = kwargs.get("model", self.model)
litellm_provider = f"litellm/{provider_model}"
- client = instructor.from_provider(litellm_provider, api_key=self.api_key)
+ client = instructor.from_provider(
+ litellm_provider, api_key=self.api_key
+ )
except Exception:
pass
@@ -395,10 +424,26 @@ class BaseProvider:
"messages": [{"role": "user", "content": prompt}],
"response_model": schema,
"max_retries": max_retries,
- "temperature": kwargs.get("temperature") if kwargs.get("temperature") is not None else 0.1,
+ "temperature": (
+ kwargs.get("temperature")
+ if kwargs.get("temperature") is not None
+ else 0.1
+ ),
}
- self._add_if_set(create_kwargs, kwargs, "max_tokens", "max_completion_tokens",
- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "logit_bias", "user", "top_k")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "max_tokens",
+ "max_completion_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "logit_bias",
+ "user",
+ "top_k",
+ )
if provider_name == "GroqProvider":
create_kwargs["response_format"] = {"type": "json_object"}
@@ -423,20 +468,27 @@ class BaseProvider:
primary_err,
exc_info=True,
)
- json_client = instructor.from_openai(self.client, mode=instructor.Mode.JSON)
+ json_client = instructor.from_openai(
+ self.client, mode=instructor.Mode.JSON
+ )
# Build a clean kwargs dict for the Mode.JSON retry: drop
# response_format (Mode.JSON handles schema differently)
# but keep response_model/max_retries so instructor still
# validates the typed output.
retry_kwargs = {
- k: v for k, v in create_kwargs.items()
+ k: v
+ for k, v in create_kwargs.items()
if k != "response_format"
}
- response = json_client.chat.completions.create(**retry_kwargs)
+ response = json_client.chat.completions.create(
+ **retry_kwargs
+ )
else:
raise
- verbose_mode = kwargs.get("verbose", False) or self.config.get("verbose", False)
+ verbose_mode = kwargs.get("verbose", False) or self.config.get(
+ "verbose", False
+ )
if verbose_mode:
self.logger.debug(
"[BaseProvider.generate_typed] Typed response received via instructor (%s).",
@@ -453,14 +505,16 @@ class BaseProvider:
# Fallback: Manual repair loop
last_error = None
current_prompt = prompt
-
+
for attempt in range(max_retries):
try:
# 1. Generate JSON – try structured mode first, then fall back to
# plain generate() + parse. Custom gateways that reject
# response_format=json_object would otherwise loop forever here.
try:
- json_result = self.generate_structured(current_prompt, max_retries=1, **kwargs)
+ json_result = self.generate_structured(
+ current_prompt, max_retries=1, **kwargs
+ )
except Exception as struct_err:
self.logger.warning(
"generate_structured failed (%s); retrying with plain generate() + JSON parse.",
@@ -469,96 +523,133 @@ class BaseProvider:
)
raw_content = self.generate(current_prompt, **kwargs)
json_result = self._parse_json(raw_content)
-
+
# 2. Validate with Schema
# If the result is a list and schema expects a wrapper, or vice versa, we might need adjustment
# But we assume the prompt asks for the correct structure matching the schema.
-
+
# Special handling if schema is a wrapper but result is a list
- if isinstance(json_result, list) and hasattr(schema, "entities") and "entities" in schema.model_fields:
- # Auto-wrap for entities
- json_result = {"entities": json_result}
+ if (
+ isinstance(json_result, list)
+ and hasattr(schema, "entities")
+ and "entities" in schema.model_fields
+ ):
+ # Auto-wrap for entities
+ json_result = {"entities": json_result}
# Handle categorized dictionary input (e.g. {"PERSON": ["Name"], "ORG": ["Corp"]})
- elif isinstance(json_result, dict) and hasattr(schema, "entities") and "entities" in schema.model_fields:
- # Check if it's NOT already in the correct format (i.e., missing "entities" key)
- if "entities" not in json_result:
- # Check if values are lists, suggesting categorized output
- is_categorized = any(isinstance(v, list) for v in json_result.values())
- if is_categorized:
- flat_entities = []
- for label, items in json_result.items():
- if isinstance(items, list):
- for item in items:
- if isinstance(item, str):
- flat_entities.append({"text": item, "label": label})
- elif isinstance(item, dict):
- # If it's already a dict but nested under label
- item["label"] = label
- flat_entities.append(item)
- json_result = {"entities": flat_entities}
+ elif (
+ isinstance(json_result, dict)
+ and hasattr(schema, "entities")
+ and "entities" in schema.model_fields
+ ):
+ # Check if it's NOT already in the correct format (i.e., missing "entities" key)
+ if "entities" not in json_result:
+ # Check if values are lists, suggesting categorized output
+ is_categorized = any(
+ isinstance(v, list) for v in json_result.values()
+ )
+ if is_categorized:
+ flat_entities = []
+ for label, items in json_result.items():
+ if isinstance(items, list):
+ for item in items:
+ if isinstance(item, str):
+ flat_entities.append(
+ {"text": item, "label": label}
+ )
+ elif isinstance(item, dict):
+ # If it's already a dict but nested under label
+ item["label"] = label
+ flat_entities.append(item)
+ json_result = {"entities": flat_entities}
# Handle categorized dictionary input for relations (e.g. {"founded_by": [{"subject":..., "object":...}]})
- elif isinstance(json_result, dict) and hasattr(schema, "relations") and "relations" in schema.model_fields:
- if "relations" not in json_result:
- is_categorized = any(isinstance(v, list) for v in json_result.values())
- if is_categorized:
- flat_relations = []
- for label, items in json_result.items():
- if isinstance(items, list):
- for item in items:
- if isinstance(item, dict):
- # If predicate is missing, use the key as predicate
- if "predicate" not in item:
- item["predicate"] = label
- flat_relations.append(item)
- json_result = {"relations": flat_relations}
+ elif (
+ isinstance(json_result, dict)
+ and hasattr(schema, "relations")
+ and "relations" in schema.model_fields
+ ):
+ if "relations" not in json_result:
+ is_categorized = any(
+ isinstance(v, list) for v in json_result.values()
+ )
+ if is_categorized:
+ flat_relations = []
+ for label, items in json_result.items():
+ if isinstance(items, list):
+ for item in items:
+ if isinstance(item, dict):
+ # If predicate is missing, use the key as predicate
+ if "predicate" not in item:
+ item["predicate"] = label
+ flat_relations.append(item)
+ json_result = {"relations": flat_relations}
# Handle categorized dictionary input for triplets
- elif isinstance(json_result, dict) and hasattr(schema, "triplets") and "triplets" in schema.model_fields:
- if "triplets" not in json_result:
- is_categorized = any(isinstance(v, list) for v in json_result.values())
- if is_categorized:
- flat_triplets = []
- for label, items in json_result.items():
- if isinstance(items, list):
- for item in items:
- if isinstance(item, dict):
- flat_triplets.append(item)
- json_result = {"triplets": flat_triplets}
-
- elif isinstance(json_result, list) and hasattr(schema, "relations") and "relations" in schema.model_fields:
- json_result = {"relations": json_result}
- elif isinstance(json_result, list) and hasattr(schema, "triplets") and "triplets" in schema.model_fields:
- json_result = {"triplets": json_result}
+ elif (
+ isinstance(json_result, dict)
+ and hasattr(schema, "triplets")
+ and "triplets" in schema.model_fields
+ ):
+ if "triplets" not in json_result:
+ is_categorized = any(
+ isinstance(v, list) for v in json_result.values()
+ )
+ if is_categorized:
+ flat_triplets = []
+ for label, items in json_result.items():
+ if isinstance(items, list):
+ for item in items:
+ if isinstance(item, dict):
+ flat_triplets.append(item)
+ json_result = {"triplets": flat_triplets}
+
+ elif (
+ isinstance(json_result, list)
+ and hasattr(schema, "relations")
+ and "relations" in schema.model_fields
+ ):
+ json_result = {"relations": json_result}
+ elif (
+ isinstance(json_result, list)
+ and hasattr(schema, "triplets")
+ and "triplets" in schema.model_fields
+ ):
+ json_result = {"triplets": json_result}
validated = schema.model_validate(json_result)
return validated
-
+
except ValidationError as e:
last_error = e
error_summary = str(e)
# Simplify error summary for the LLM
# (You could parse e.errors() for a better message)
-
+
if attempt < max_retries - 1:
wait_time = (attempt + 1) * 1
- self.logger.warning(f"Schema validation failed (attempt {attempt + 1}): {e}. Retrying with error feedback...")
-
+ self.logger.warning(
+ f"Schema validation failed (attempt {attempt + 1}): {e}. Retrying with error feedback..."
+ )
+
# Update prompt with error info
current_prompt = f"{prompt}\n\nPrevious response was invalid JSON or didn't match schema:\n{error_summary}\n\nPlease fix the errors and return valid JSON matching the schema."
time.sleep(wait_time)
else:
self.logger.error(f"Typed generation failed validation: {e}")
-
+
except Exception as e:
last_error = e
if attempt < max_retries - 1:
- time.sleep(1)
+ time.sleep(1)
else:
self.logger.error(f"Typed generation failed: {e}")
- raise ProcessingError(f"Failed to generate typed output after {max_retries} attempts: {last_error}")
+ raise ProcessingError(
+ f"Failed to generate typed output after {max_retries} attempts: {last_error}"
+ )
+
class OpenAIProvider(BaseProvider):
"""OpenAI provider implementation."""
@@ -594,6 +685,7 @@ class OpenAIProvider(BaseProvider):
# Reject non-HTTP schemes (file://, ftp://, etc.) to prevent SSRF
# when base_url originates from configuration rather than hardcoded values.
from urllib.parse import urlparse
+
scheme = urlparse(self.base_url).scheme
if scheme not in ("http", "https"):
raise ValueError(
@@ -630,8 +722,20 @@ class OpenAIProvider(BaseProvider):
"model": kwargs.get("model", self.model),
"messages": [{"role": "user", "content": prompt}],
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",
- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "logit_bias", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_completion_tokens",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "logit_bias",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
return response.choices[0].message.content
@@ -651,8 +755,20 @@ class OpenAIProvider(BaseProvider):
if not self.base_url:
create_kwargs["response_format"] = {"type": "json_object"}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",
- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "logit_bias", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_completion_tokens",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "logit_bias",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
try:
@@ -679,6 +795,7 @@ class GeminiProvider(BaseProvider):
def _init_client(self):
try:
from google import genai as new_genai
+
if self.api_key:
self.client = new_genai.Client(api_key=self.api_key)
self._use_new_genai = True
@@ -687,13 +804,16 @@ class GeminiProvider(BaseProvider):
pass
try:
import google.generativeai as old_genai
+
if self.api_key:
old_genai.configure(api_key=self.api_key)
self.client = old_genai.GenerativeModel(self.model)
self._use_new_genai = False
except Exception:
self.client = None
- self.logger.warning("Gemini SDK not installed. Install with: pip install semantica[llm-gemini]")
+ self.logger.warning(
+ "Gemini SDK not installed. Install with: pip install semantica[llm-gemini]"
+ )
def _legacy_client_for(self, requested_model: str):
"""Return a legacy-SDK GenerativeModel bound to this instance's own
@@ -711,6 +831,7 @@ class GeminiProvider(BaseProvider):
"""
try:
import google.generativeai as old_genai
+
old_genai.configure(api_key=self.api_key)
except Exception:
# _init_client() already required this import to reach the
@@ -747,18 +868,30 @@ class GeminiProvider(BaseProvider):
)
config = {}
- self._add_if_set(config, kwargs, "temperature", "top_p", "top_k", "stop_sequences", "candidate_count")
+ self._add_if_set(
+ config,
+ kwargs,
+ "temperature",
+ "top_p",
+ "top_k",
+ "stop_sequences",
+ "candidate_count",
+ )
if "max_tokens" in kwargs:
config["max_output_tokens"] = kwargs["max_tokens"]
if self._use_new_genai:
resp = self.client.models.generate_content(
- model=kwargs.get("model", self.model), contents=prompt, config=config or None
+ model=kwargs.get("model", self.model),
+ contents=prompt,
+ config=config or None,
)
return self._resp_text(resp)
else:
legacy_client = self._legacy_client_for(kwargs.get("model", self.model))
- response = legacy_client.generate_content(prompt, generation_config=config or None)
+ response = legacy_client.generate_content(
+ prompt, generation_config=config or None
+ )
return self._resp_text(response)
def generate_structured(self, prompt: str, **kwargs) -> dict:
@@ -769,7 +902,15 @@ class GeminiProvider(BaseProvider):
json_prompt = f"{prompt}\n\nReturn the response as valid JSON only."
config = {}
- self._add_if_set(config, kwargs, "temperature", "top_p", "top_k", "stop_sequences", "candidate_count")
+ self._add_if_set(
+ config,
+ kwargs,
+ "temperature",
+ "top_p",
+ "top_k",
+ "stop_sequences",
+ "candidate_count",
+ )
if "max_tokens" in kwargs:
config["max_output_tokens"] = kwargs["max_tokens"]
@@ -784,7 +925,9 @@ class GeminiProvider(BaseProvider):
raise ProcessingError(f"Failed to parse JSON from Gemini response: {e}")
else:
legacy_client = self._legacy_client_for(kwargs.get("model", self.model))
- response = legacy_client.generate_content(json_prompt, generation_config=config or None)
+ response = legacy_client.generate_content(
+ json_prompt, generation_config=config or None
+ )
try:
return self._parse_json(self._resp_text(response))
except Exception as e:
@@ -795,7 +938,10 @@ class GroqProvider(BaseProvider):
"""Groq provider implementation."""
def __init__(
- self, api_key: Optional[str] = None, model: str = "llama-3.3-70b-versatile", **kwargs
+ self,
+ api_key: Optional[str] = None,
+ model: str = "llama-3.3-70b-versatile",
+ **kwargs,
):
"""Initialize Groq provider."""
super().__init__(**kwargs)
@@ -815,11 +961,11 @@ class GroqProvider(BaseProvider):
return
self.client = Groq(api_key=self.api_key)
-
+
# Test connection with a minimal prompt
# Only do this if we have a key and client
# self._test_connection()
- except ImportError:
+ except (ImportError, OSError):
self.client = None
self.logger.warning(
"groq library not installed. Install with: pip install semantica[llm-groq]"
@@ -837,7 +983,7 @@ class GroqProvider(BaseProvider):
self.client.chat.completions.create(
model=self.model,
messages=[{"role": "user", "content": "ping"}],
- max_tokens=1
+ max_tokens=1,
)
self.logger.debug("Groq connection test successful")
except Exception as e:
@@ -858,8 +1004,19 @@ class GroqProvider(BaseProvider):
"model": kwargs.get("model", self.model),
"messages": [{"role": "user", "content": prompt}],
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",
- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_completion_tokens",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
return response.choices[0].message.content
@@ -870,15 +1027,30 @@ class GroqProvider(BaseProvider):
raise ProcessingError("Groq client not initialized.")
# Groq requires 'json' in the prompt for json_object mode
- json_prompt = prompt if "json" in prompt.lower() else f"{prompt}\n\nReturn the response as valid JSON only."
+ json_prompt = (
+ prompt
+ if "json" in prompt.lower()
+ else f"{prompt}\n\nReturn the response as valid JSON only."
+ )
create_kwargs = {
"model": kwargs.get("model", self.model),
"messages": [{"role": "user", "content": json_prompt}],
"response_format": {"type": "json_object"},
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_completion_tokens", "max_tokens",
- "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_completion_tokens",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
try:
@@ -927,19 +1099,26 @@ class AnthropicProvider(BaseProvider):
"Anthropic client not initialized. Set ANTHROPIC_API_KEY or pass api_key."
)
- # Anthropic requires max_tokens.
+ # Anthropic requires max_tokens.
# We rely on kwargs, but fallback to 8192 (safe max for newer models) if not provided.
max_tokens = kwargs.get("max_tokens", 8192)
-
+
# Prepare arguments
create_kwargs = {
"model": kwargs.get("model", self.model),
"max_tokens": max_tokens,
"messages": [{"role": "user", "content": prompt}],
}
-
+
# Pass through other common parameters
- for param in ["temperature", "top_p", "top_k", "stop_sequences", "system", "metadata"]:
+ for param in [
+ "temperature",
+ "top_p",
+ "top_k",
+ "stop_sequences",
+ "system",
+ "metadata",
+ ]:
if param in kwargs:
create_kwargs[param] = kwargs[param]
@@ -952,22 +1131,29 @@ class AnthropicProvider(BaseProvider):
raise ProcessingError("Anthropic client not initialized.")
json_prompt = f"{prompt}\n\nReturn the response as valid JSON only."
-
- # Anthropic requires max_tokens.
+
+ # Anthropic requires max_tokens.
max_tokens = kwargs.get("max_tokens", 8192)
-
+
# Prepare arguments
create_kwargs = {
"model": kwargs.get("model", self.model),
"max_tokens": max_tokens,
"messages": [{"role": "user", "content": json_prompt}],
}
-
+
# Pass through other common parameters
- for param in ["temperature", "top_p", "top_k", "stop_sequences", "system", "metadata"]:
+ for param in [
+ "temperature",
+ "top_p",
+ "top_k",
+ "stop_sequences",
+ "system",
+ "metadata",
+ ]:
if param in kwargs:
create_kwargs[param] = kwargs[param]
-
+
response = self.client.messages.create(**create_kwargs)
try:
return self._parse_json(response.content[0].text)
@@ -1017,7 +1203,9 @@ class OllamaProvider(BaseProvider):
def _build_options(self, kwargs: dict) -> Optional[dict]:
"""Build Ollama options dict from kwargs."""
options = {}
- self._add_if_set(options, kwargs, "temperature", "top_p", "top_k", "repeat_penalty", "seed")
+ self._add_if_set(
+ options, kwargs, "temperature", "top_p", "top_k", "repeat_penalty", "seed"
+ )
if "max_tokens" in kwargs:
options["num_predict"] = kwargs["max_tokens"]
if "num_ctx" in kwargs:
@@ -1058,7 +1246,9 @@ class OllamaProvider(BaseProvider):
class DeepSeekProvider(BaseProvider):
- def __init__(self, api_key: Optional[str] = None, model: str = "deepseek-chat", **kwargs):
+ def __init__(
+ self, api_key: Optional[str] = None, model: str = "deepseek-chat", **kwargs
+ ):
super().__init__(**kwargs)
self.api_key = api_key or config.get_api_key("deepseek")
self.base_url = "https://api.deepseek.com/v1"
@@ -1075,7 +1265,8 @@ class DeepSeekProvider(BaseProvider):
except (ImportError, OSError):
self.client = None
self.logger.warning(
- "openai library not installed. Install with: pip install semantica[llm-openai]"
+ "openai library not installed. "
+ "Install with: pip install semantica[llm-openai]"
)
def is_available(self) -> bool:
@@ -1083,13 +1274,26 @@ class DeepSeekProvider(BaseProvider):
def generate(self, prompt: str, **kwargs) -> str:
if not self.client:
- raise ProcessingError("DeepSeek client not initialized. Set DEEPSEEK_API_KEY or pass api_key.")
+ raise ProcessingError(
+ "DeepSeek client not initialized. Set DEEPSEEK_API_KEY or pass api_key."
+ )
create_kwargs = {
"model": kwargs.get("model", self.model),
"messages": [{"role": "user", "content": prompt}],
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
return response.choices[0].message.content
@@ -1104,7 +1308,18 @@ class DeepSeekProvider(BaseProvider):
"messages": [{"role": "user", "content": prompt}],
"response_format": {"type": "json_object"},
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
try:
@@ -1116,7 +1331,12 @@ class DeepSeekProvider(BaseProvider):
class NovitaProvider(BaseProvider):
"""Novita AI provider implementation - OpenAI-compatible API."""
- def __init__(self, api_key: Optional[str] = None, model: str = "deepseek/deepseek-v3.2", **kwargs):
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ model: str = "deepseek/deepseek-v3.2",
+ **kwargs,
+ ):
"""Initialize Novita provider."""
super().__init__(**kwargs)
self.api_key = api_key or config.get_api_key("novita")
@@ -1134,7 +1354,8 @@ class NovitaProvider(BaseProvider):
except (ImportError, OSError):
self.client = None
self.logger.warning(
- "openai library not installed. Install with: pip install semantica[llm-openai]"
+ "openai library not installed. "
+ "Install with: pip install semantica[llm-openai]"
)
def is_available(self) -> bool:
@@ -1142,13 +1363,26 @@ class NovitaProvider(BaseProvider):
def generate(self, prompt: str, **kwargs) -> str:
if not self.client:
- raise ProcessingError("Novita client not initialized. Set NOVITA_API_KEY or pass api_key.")
+ raise ProcessingError(
+ "Novita client not initialized. Set NOVITA_API_KEY or pass api_key."
+ )
create_kwargs = {
"model": kwargs.get("model", self.model),
"messages": [{"role": "user", "content": prompt}],
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
return response.choices[0].message.content
@@ -1163,7 +1397,18 @@ class NovitaProvider(BaseProvider):
"messages": [{"role": "user", "content": prompt}],
"response_format": {"type": "json_object"},
}
- self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "user")
+ self._add_if_set(
+ create_kwargs,
+ kwargs,
+ "temperature",
+ "max_tokens",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "user",
+ )
response = self.client.chat.completions.create(**create_kwargs)
try:
@@ -1171,6 +1416,7 @@ class NovitaProvider(BaseProvider):
except Exception as e:
raise ProcessingError(f"Failed to parse JSON from Novita response: {e}")
+
class HuggingFaceLLMProvider(BaseProvider):
"""HuggingFace transformers for LLM tasks."""
@@ -1186,7 +1432,7 @@ class HuggingFaceLLMProvider(BaseProvider):
raise ImportError(
"torch is required for HuggingFaceLLMProvider. Install with: pip install 'semantica[models-huggingface]'"
)
-
+
self.model_name = model_name
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
self.model = None
@@ -1222,28 +1468,25 @@ class HuggingFaceLLMProvider(BaseProvider):
raise ProcessingError("HuggingFace model not initialized.")
inputs = self.tokenizer.encode(prompt, return_tensors="pt").to(self.device)
-
+
# Use max_new_tokens if available, otherwise fallback to max_length with a safe default
generate_kwargs = {
"temperature": kwargs.get("temperature", 0.7),
"do_sample": True,
}
-
+
if "max_new_tokens" in kwargs:
generate_kwargs["max_new_tokens"] = kwargs["max_new_tokens"]
elif "max_tokens" in kwargs:
generate_kwargs["max_new_tokens"] = kwargs["max_tokens"]
-
+
# Support legacy max_length if explicitly provided
if "max_length" in kwargs:
generate_kwargs["max_length"] = kwargs["max_length"]
# Remove max_new_tokens if max_length is set to avoid conflict
generate_kwargs.pop("max_new_tokens", None)
- outputs = self.model.generate(
- inputs,
- **generate_kwargs
- )
+ outputs = self.model.generate(inputs, **generate_kwargs)
generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
# Remove the original prompt from the response
return generated_text[len(prompt) :].strip()
@@ -1271,9 +1514,10 @@ class HuggingFaceModelLoader:
import torch
except (ImportError, OSError):
raise ImportError(
- "torch is required for HuggingFaceModelLoader. Install with: pip install 'semantica[models-huggingface]'"
+ "torch is required for HuggingFaceModelLoader. "
+ "Install with: pip install 'semantica[models-huggingface]'"
)
-
+
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
self._cache: Dict[str, Any] = {}
self.logger = get_logger("huggingface_loader")
@@ -1282,7 +1526,7 @@ class HuggingFaceModelLoader:
"""Load NER model."""
# Import torch at method level to ensure it's available
import torch
-
+
# Include aggregation_strategy in cache key
agg_strategy = kwargs.get("aggregation_strategy", "simple")
cache_key = f"{model_name}_ner_{agg_strategy}"
@@ -1291,9 +1535,10 @@ class HuggingFaceModelLoader:
try:
from transformers import pipeline
- except ImportError:
+ except (ImportError, OSError):
raise ImportError(
- "transformers library not installed. Install with: pip install 'semantica[models-huggingface]'"
+ "transformers library not installed. "
+ "Install with: pip install 'semantica[models-huggingface]'"
)
try:
@@ -1302,13 +1547,16 @@ class HuggingFaceModelLoader:
model=model_name,
device=self.device if torch.cuda.is_available() else -1,
aggregation_strategy=agg_strategy,
- tokenizer=kwargs.get("tokenizer") # Allow custom tokenizer
+ tokenizer=kwargs.get("tokenizer"), # Allow custom tokenizer
)
self._cache[cache_key] = nlp
return nlp
except OSError as e:
self.logger.error(f"Failed to load NER model '{model_name}': {e}")
- raise ValueError(f"Could not load HuggingFace model '{model_name}'. Check if model name is correct. Error: {e}")
+ raise ValueError(
+ f"Could not load HuggingFace model '{model_name}'. "
+ f"Check if model name is correct. Error: {e}"
+ )
except Exception as e:
self.logger.error(f"Failed to load NER model {model_name}: {e}")
raise
@@ -1317,16 +1565,17 @@ class HuggingFaceModelLoader:
"""Load relation extraction model."""
# Import torch at method level to ensure it's available
import torch
-
+
cache_key = f"{model_name}_relation"
if cache_key in self._cache:
return self._cache[cache_key]
try:
from transformers import pipeline, AutoTokenizer
- except ImportError:
+ except (ImportError, OSError):
raise ImportError(
- "transformers library not installed. Install with: pip install 'semantica[models-huggingface]'"
+ "transformers library not installed. "
+ "Install with: pip install 'semantica[models-huggingface]'"
)
try:
@@ -1347,7 +1596,10 @@ class HuggingFaceModelLoader:
return nlp
except OSError as e:
self.logger.error(f"Failed to load relation model '{model_name}': {e}")
- raise ValueError(f"Could not load HuggingFace model '{model_name}'. Check if model name is correct. Error: {e}")
+ raise ValueError(
+ f"Could not load HuggingFace model '{model_name}'. "
+ f"Check if model name is correct. Error: {e}"
+ )
except Exception as e:
self.logger.error(f"Failed to load relation model {model_name}: {e}")
raise
@@ -1359,10 +1611,11 @@ class HuggingFaceModelLoader:
return self._cache[cache_key]
try:
- from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
- except ImportError:
+ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+ except (ImportError, OSError):
raise ImportError(
- "transformers library not installed. Install with: pip install 'semantica[models-huggingface]'"
+ "transformers library not installed. "
+ "Install with: pip install 'semantica[models-huggingface]'"
)
try:
@@ -1371,7 +1624,7 @@ class HuggingFaceModelLoader:
if not tokenizer:
tokenizer_name = kwargs.get("tokenizer_name", model_name)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
-
+
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
model.to(self.device)
@@ -1380,7 +1633,10 @@ class HuggingFaceModelLoader:
return nlp
except OSError as e:
self.logger.error(f"Failed to load triplet model '{model_name}': {e}")
- raise ValueError(f"Could not load HuggingFace model '{model_name}'. Check if model name is correct. Error: {e}")
+ raise ValueError(
+ f"Could not load HuggingFace model '{model_name}'. "
+ f"Check if model name is correct. Error: {e}"
+ )
except Exception as e:
self.logger.error(f"Failed to load triplet model {model_name}: {e}")
raise
@@ -1389,83 +1645,103 @@ class HuggingFaceModelLoader:
"""Extract entities using loaded model."""
return model(text)
- def extract_relations(self, model, text: str, entities: List, **kwargs) -> List[Dict]:
+ def extract_relations(
+ self, model, text: str, entities: List, **kwargs
+ ) -> List[Dict]:
"""
Extract relations using loaded model.
Iterates through entity pairs and classifies the relationship.
"""
results = []
-
+
# Sort entities by position
sorted_entities = sorted(entities, key=lambda e: e.start_char)
-
+
# Marker configuration
subj_start = kwargs.get("subj_start_marker", "<subj>")
subj_end = kwargs.get("subj_end_marker", "</subj>")
obj_start = kwargs.get("obj_start_marker", "<obj>")
obj_end = kwargs.get("obj_end_marker", "</obj>")
-
+
# Iterate through all pairs
- import itertools
for i, e1 in enumerate(sorted_entities):
for e2 in sorted_entities:
if e1 == e2:
continue
-
+
# Check distance (optional optimization)
# if abs(e1.start_char - e2.start_char) > 200: continue
-
+
# Format text with markers
# Strategy: [CLS] text with <subj>...</subj> and <obj>...</obj> [SEP]
# We need to insert markers into the original text
-
+
# Create a copy of text with markers inserted
- # We need to handle offsets correctly.
+ # We need to handle offsets correctly.
# Simplest way: reconstruct string pieces
-
+
p1_start, p1_end = e1.start_char, e1.end_char
p2_start, p2_end = e2.start_char, e2.end_char
-
+
if p1_start < p2_start:
formatted_text = (
- text[:p1_start] +
- f"{subj_start} " + text[p1_start:p1_end] + f" {subj_end}" +
- text[p1_end:p2_start] +
- f"{obj_start} " + text[p2_start:p2_end] + f" {obj_end}" +
- text[p2_end:]
+ text[:p1_start]
+ + f"{subj_start} "
+ + text[p1_start:p1_end]
+ + f" {subj_end}"
+ + text[p1_end:p2_start]
+ + f"{obj_start} "
+ + text[p2_start:p2_end]
+ + f" {obj_end}"
+ + text[p2_end:]
)
else:
- formatted_text = (
- text[:p2_start] +
- f"{obj_start} " + text[p2_start:p2_end] + f" {obj_end}" +
- text[p2_end:p1_start] +
- f"{subj_start} " + text[p1_start:p1_end] + f" {subj_end}" +
- text[p1_end:]
+ formatted_text = (
+ text[:p2_start]
+ + f"{obj_start} "
+ + text[p2_start:p2_end]
+ + f" {obj_end}"
+ + text[p2_end:p1_start]
+ + f"{subj_start} "
+ + text[p1_start:p1_end]
+ + f" {subj_end}"
+ + text[p1_end:]
)
-
+
# Predict
try:
# Pipeline returns [{'label': 'LABEL', 'score': 0.99}]
prediction = model(formatted_text, top_k=1)
-
+
if prediction:
- res = prediction[0] if isinstance(prediction, list) else prediction
- if isinstance(res, list): res = res[0] # top_k=1 returns list of dicts
-
+ res = (
+ prediction[0]
+ if isinstance(prediction, list)
+ else prediction
+ )
+ if isinstance(res, list):
+ res = res[0] # top_k=1 returns list of dicts
+
label = res.get("label")
score = res.get("score")
-
+
# Filter "no_relation" or low confidence
- if label != "no_relation" and score > kwargs.get("threshold", 0.5):
- results.append({
- "subject": e1,
- "object": e2,
- "relation": label,
- "score": score
- })
+ if label != "no_relation" and score > kwargs.get(
+ "threshold", 0.5
+ ):
+ results.append(
+ {
+ "subject": e1,
+ "object": e2,
+ "relation": label,
+ "score": score,
+ }
+ )
except Exception as e:
- self.logger.warning(f"Relation prediction failed for pair {e1.text}-{e2.text}: {e}")
-
+ self.logger.warning(
+ f"Relation prediction failed for pair {e1.text}-{e2.text}: {e}"
+ )
+
return results
def extract_triplets(self, model, text: str, **kwargs) -> List[Dict]:
@@ -1477,21 +1753,28 @@ class HuggingFaceModelLoader:
# Use kwargs for max_length, default to 512 for input and 128 for output if not specified
max_input_length = kwargs.get("max_input_length", 512)
max_length = kwargs.get("max_length", 128)
-
+
generate_kwargs = {"max_length": max_length}
if "max_new_tokens" in kwargs:
generate_kwargs["max_new_tokens"] = kwargs["max_new_tokens"]
-
+
# Pass other generation args including beams and penalties
- for param in ["num_beams", "temperature", "top_p", "top_k", "do_sample",
- "length_penalty", "repetition_penalty"]:
+ for param in [
+ "num_beams",
+ "temperature",
+ "top_p",
+ "top_k",
+ "do_sample",
+ "length_penalty",
+ "repetition_penalty",
+ ]:
if param in kwargs:
generate_kwargs[param] = kwargs[param]
inputs = tokenizer(
text, return_tensors="pt", truncation=True, max_length=max_input_length
).to(device)
-
+
outputs = model_obj.generate(**inputs, **generate_kwargs)
# Allow controlling skip_special_tokens (important for REBEL which uses special tokens for delimiters)
skip_special_tokens = kwargs.get("skip_special_tokens", True)
@@ -1502,7 +1785,7 @@ class HuggingFaceModelLoader:
class ProviderPool:
"""Pool for reusing provider instances."""
-
+
def __init__(self):
self._providers: Dict[str, BaseProvider] = {}
self.logger = get_logger("provider_pool")
@@ -1512,7 +1795,7 @@ class ProviderPool:
# Create a cache key from name and kwargs
# Filter out non-hashable items or volatile args if any
# For now, we assume kwargs are configuration options that should match
-
+
# Helper to make dict hashable
def make_hashable(value):
if isinstance(value, dict):
@@ -1523,20 +1806,20 @@ class ProviderPool:
key_parts = [name]
for k, v in sorted(kwargs.items()):
- # Skip some keys if they shouldn't affect pooling?
+ # Skip some keys if they shouldn't affect pooling?
# For now, all init args matter for the instance identity.
key_parts.append((k, make_hashable(v)))
-
+
key = str(tuple(key_parts))
-
+
if key in self._providers:
return self._providers[key]
-
+
self.logger.debug(f"Creating new provider instance for {name}")
provider = self._create_provider(name, **kwargs)
self._providers[key] = provider
return provider
-
+
def _create_provider(self, name: str, **kwargs) -> BaseProvider:
"""Internal creation logic."""
# Check registry first
@@ -1553,7 +1836,7 @@ class ProviderPool:
"ollama": OllamaProvider,
"huggingface_llm": HuggingFaceLLMProvider,
"deepseek": DeepSeekProvider,
- "novita": NovitaProvider,
+ "novita": NovitaProvider,
}
provider_class = builtin.get(name.lower())
@@ -1576,7 +1859,7 @@ _provider_pool = ProviderPool()
def create_provider(name: str, use_pool: bool = True, **kwargs) -> BaseProvider:
"""
Create provider - checks registry for custom providers.
-
+
Args:
name: Provider name
use_pool: Whether to use the provider pool (default: True)
@@ -1584,5 +1867,5 @@ def create_provider(name: str, use_pool: bool = True, **kwargs) -> BaseProvider:
"""
if use_pool:
return _provider_pool.get(name, **kwargs)
-
+
return _provider_pool._create_provider(name, **kwargs)
diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py
index 48065855..feb3400c 100644
--- a/semantica/utils/helpers.py
+++ b/semantica/utils/helpers.py
@@ -38,15 +38,15 @@ Example Usage:
>>> from semantica.utils import clean_text, normalize_entities
>>> cleaned = clean_text(" Hello World ")
>>> entities = normalize_entities([{"id": "e1", "text": "John", "type": "PERSON"}])
- >>>
+ >>>
>>> from semantica.utils import hash_data, safe_filename
>>> data_hash = hash_data({"key": "value"})
>>> safe_name = safe_filename("my file.txt")
- >>>
+ >>>
>>> from semantica.utils import merge_dicts, get_nested_value
>>> merged = merge_dicts({"a": 1}, {"b": 2}, deep=True)
>>> value = get_nested_value(config, "database.host", default="localhost")
- >>>
+ >>>
>>> from semantica.utils import retry_on_error
>>> @retry_on_error(max_retries=3, delay=1.0)
... def fetch_data():
@@ -457,7 +457,9 @@ def chunk_list(items: List[Any], chunk_size: int) -> List[List[Any]]:
Returns:
List of chunks
"""
- return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
+ return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
+
+
def flatten_dict(
d: Dict[str, Any], parent_key: str = "", sep: str = "."
) -> Dict[str, Any]:
@@ -556,27 +558,27 @@ def safe_import(
) -> Tuple[Any, bool]:
"""
Safely import an optional module, handling both ImportError and OSError.
-
+
This is useful for optional dependencies that may fail to import due to:
- Missing package (ImportError)
- DLL loading failures on Windows, e.g., PyTorch (OSError)
-
+
Args:
module_name: Name of the module to import (e.g., "spacy", "docling.document_converter")
package: Optional package name for relative imports
default: Default value to return if import fails
error_message: Optional custom error message for logging
-
+
Returns:
Tuple of (module_or_default, success_flag):
- If import succeeds: (imported_module, True)
- If import fails: (default, False)
-
+
Example:
>>> spacy, available = safe_import("spacy")
>>> if available:
... doc = spacy.load("en_core_web_sm")
- >>>
+ >>>
>>> converter, available = safe_import("docling.document_converter", default=None)
>>> if available:
... converter = converter()
@@ -587,11 +589,13 @@ def safe_import(
else:
module = importlib.import_module(module_name)
return module, True
- except (ImportError, ModuleNotFoundError, OSError) as e:
+ except (ImportError, OSError) as e:
if error_message:
import sys
+
if "logging" in sys.modules:
from .logging import get_logger
+
logger = get_logger("utils.helpers")
logger.debug(f"{error_message}: {e}")
return default, False
@@ -807,9 +811,13 @@ def _is_record(value: Any) -> bool:
exporters rather than a ``ValidationError`` at the boundary where the
problem is visible.
"""
- return isinstance(value, Mapping) or is_dataclass(value) or (
- hasattr(value, "__dict__")
- and not isinstance(value, (types.ModuleType, type))
+ return (
+ isinstance(value, Mapping)
+ or is_dataclass(value)
+ or (
+ hasattr(value, "__dict__")
+ and not isinstance(value, (types.ModuleType, type))
+ )
)
diff --git a/semantica/visualization/embedding_visualizer.py b/semantica/visualization/embedding_visualizer.py
index 648c9515..ba26ecea 100644
--- a/semantica/visualization/embedding_visualizer.py
+++ b/semantica/visualization/embedding_visualizer.py
@@ -1,9 +1,10 @@
"""
Embedding Visualizer Module
-This module provides comprehensive visualization capabilities for vector embeddings in the
-Semantica framework, including 2D/3D dimensionality reduction projections, similarity heatmaps,
-clustering visualizations, multi-modal comparisons, and quality metrics analysis.
+This module provides comprehensive visualization capabilities for vector
+embeddings in the Semantica framework, including 2D/3D dimensionality
+reduction projections, similarity heatmaps, clustering visualizations,
+multi-modal comparisons, and quality metrics analysis.
Key Features:
- 2D and 3D dimensionality reduction (UMAP, t-SNE, PCA)
@@ -31,7 +32,7 @@ License: MIT
"""
from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Union
import numpy as np
@@ -158,10 +159,12 @@ class EmbeddingVisualizer:
try:
self.logger.info(f"Visualizing 2D projection using {method}")
-
+
# Step 2: Data Analysis
n_samples, n_features = embeddings.shape
- self.logger.info(f"Embedding Analysis: {n_samples} samples, {n_features} dimensions")
+ self.logger.info(
+ f"Embedding Analysis: {n_samples} samples, {n_features} dimensions"
+ )
if embeddings.shape[1] <= 2:
# Already 2D or less, use directly
@@ -183,20 +186,22 @@ class EmbeddingVisualizer:
tracking_id, message="Generating visualization..."
)
result = self._visualize_2d_plotly(
- projected,
- labels,
- output,
- file_path,
+ projected,
+ labels,
+ output,
+ file_path,
color_by=color_by,
size_by=size_by,
hover_data=hover_data,
- **options
+ **options,
)
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
- message=f"2D projection visualization generated: {len(projected)} points",
+ message=(
+ f"2D projection visualization generated: {len(projected)} points"
+ ),
)
return result
except Exception as e:
@@ -263,7 +268,9 @@ class EmbeddingVisualizer:
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
- message=f"3D projection visualization generated: {len(projected)} points",
+ message=(
+ f"3D projection visualization generated: {len(projected)} points"
+ ),
)
return result
except Exception as e:
@@ -354,7 +361,10 @@ class EmbeddingVisualizer:
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
- message=f"Similarity heatmap generated: {len(embeddings)}x{len(embeddings)} matrix",
+ message=(
+ f"Similarity heatmap generated: "
+ f"{len(embeddings)}x{len(embeddings)} matrix"
+ ),
)
return fig
elif file_path:
@@ -459,7 +469,10 @@ class EmbeddingVisualizer:
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
- message=f"Clustering visualization generated: {num_clusters} clusters, {len(embeddings)} points",
+ message=(
+ f"Clustering visualization generated: "
+ f"{num_clusters} clusters, {len(embeddings)} points"
+ ),
)
return fig
elif file_path:
@@ -558,7 +571,10 @@ class EmbeddingVisualizer:
dim_options = dict(options)
n_comp = dim_options.pop("n_components", 2)
projected = self._reduce_dimensions(
- combined_embeddings, method=method, n_components=n_comp, **dim_options
+ combined_embeddings,
+ method=method,
+ n_components=n_comp,
+ **dim_options,
)
# Color by type
@@ -595,7 +611,10 @@ class EmbeddingVisualizer:
self.progress_tracker.stop_tracking(
tracking_id,
status="completed",
- message=f"Multi-modal comparison generated: {len(combined_embeddings)} embeddings",
+ message=(
+ f"Multi-modal comparison generated: "
+ f"{len(combined_embeddings)} embeddings"
+ ),
)
return fig
elif file_path:
@@ -614,8 +633,6 @@ class EmbeddingVisualizer:
)
raise
-
-
def _reduce_dimensions(
self,
embeddings: np.ndarray,
diff --git a/tests/ingest/test_optional_imports.py b/tests/ingest/test_optional_imports.py
index 6c70efb8..9a9a6169 100644
--- a/tests/ingest/test_optional_imports.py
+++ b/tests/ingest/test_optional_imports.py
@@ -141,3 +141,134 @@ else:
assert "ConfigurationError" in result.stdout
assert "Parquet ingestion" in result.stdout
assert "pyarrow" in result.stdout
+
+
+def test_repo_ingestor_probe_fails_without_gitpython() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+try:
+ from semantica.ingest import RepoIngestor
+ has_git = True
+except ImportError:
+ has_git = False
+
+assert not has_git, "Expected RepoIngestor import to fail without GitPython"
+print("RepoIngestor probe passed")
+""",
+ ("git",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "RepoIngestor probe passed" in result.stdout
+
+
+def test_xml_ingestor_probe_fails_without_lxml() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+try:
+ from semantica.ingest import XMLIngestor
+ has_lxml = True
+except ImportError:
+ has_lxml = False
+
+assert not has_lxml, "Expected XMLIngestor import to fail without lxml"
+print("XMLIngestor probe passed")
+""",
+ ("lxml",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "XMLIngestor probe passed" in result.stdout
+
+
+def test_xml_ingestion_reports_missing_lxml_when_used() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+from semantica.ingest import ingest_xml
+
+try:
+ ingest_xml("catalog.xml")
+except Exception as exc:
+ print(type(exc).__name__, exc)
+else:
+ raise SystemExit("expected XML ingestion to fail without lxml")
+""",
+ ("lxml",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ConfigurationError" in result.stdout
+ assert "XML ingestion" in result.stdout
+ assert "lxml" in result.stdout
+
+
+def test_sibling_imports_succeed_without_optional_backends() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+from semantica.ingest import (
+ CodeExtractor,
+ CodeFile,
+ CommitInfo,
+ GitAnalyzer,
+ XMLIngestionData,
+ SalesforceData,
+)
+print(
+ CodeExtractor.__name__,
+ CodeFile.__name__,
+ CommitInfo.__name__,
+ GitAnalyzer.__name__,
+ XMLIngestionData.__name__,
+ SalesforceData.__name__,
+)
+""",
+ ("git", "lxml", "simple_salesforce"),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert (
+ "CodeExtractor CodeFile CommitInfo GitAnalyzer XMLIngestionData SalesforceData"
+ in result.stdout
+ )
+
+
+def test_salesforce_ingestor_probe_fails_without_simple_salesforce() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+try:
+ from semantica.ingest import SalesforceIngestor
+ has_salesforce = True
+except ImportError:
+ has_salesforce = False
+
+assert not has_salesforce, (
+ "Expected SalesforceIngestor import to fail without simple-salesforce"
+)
+print("SalesforceIngestor probe passed")
+""",
+ ("simple_salesforce",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "SalesforceIngestor probe passed" in result.stdout
+
+
+def test_salesforce_ingestion_reports_missing_dep_when_used() -> None:
+ result = _run_python_with_blocked_modules(
+ """
+from semantica.ingest import ingest_salesforce
+
+try:
+ ingest_salesforce()
+except Exception as exc:
+ print(type(exc).__name__, exc)
+else:
+ raise SystemExit("expected Salesforce ingestion to fail without simple-salesforce")
+""",
+ ("simple_salesforce",),
+ )
+
+ assert result.returncode == 0, result.stderr
+ assert "ConfigurationError" in result.stdout
+ assert "Salesforce ingestion" in result.stdout
+ assert "simple-salesforce" in result.stdout
diff --git a/tests/semantic_extract/test_temporal_extraction.py b/tests/semantic_extract/test_temporal_extraction.py
index 9aa7d46a..0dc2bc55 100644
--- a/tests/semantic_extract/test_temporal_extraction.py
+++ b/tests/semantic_extract/test_temporal_extraction.py
@@ -18,7 +18,14 @@ from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
# ── Mock optional heavyweight dependencies before any semantica import ──────
-_MOCKED_MODULES = ["spacy", "instructor", "openai", "groq", "sentence_transformers", "transformers", "torch"]
+_MOCKED_MODULES = [
+ "spacy",
+ "instructor",
+ "openai",
+ "groq",
+ "sentence_transformers",
+ "transformers",
+]
_original_modules = {k: sys.modules.get(k) for k in _MOCKED_MODULES}
for k in _MOCKED_MODULES:
@@ -26,14 +33,7 @@ for k in _MOCKED_MODULES:
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
-from semantica.semantic_extract.methods import extract_relations_llm
-from semantica.semantic_extract.ner_extractor import Entity
-from semantica.semantic_extract.schemas import (
- RelationsResponse,
- RelationsWithTemporalResponse,
-)
-from semantica.kg.temporal_normalizer import TemporalNormalizer
-from semantica.utils.exceptions import TemporalAmbiguityWarning
+from semantica.semantic_extract.methods import extract_relations_llm # noqa: E402
for _key, _original in _original_modules.items():
if _original is None:
@@ -41,9 +41,17 @@ for _key, _original in _original_modules.items():
else:
sys.modules[_key] = _original
+from semantica.semantic_extract.ner_extractor import Entity # noqa: E402
+from semantica.semantic_extract.schemas import ( # noqa: E402
+ RelationsResponse,
+ RelationsWithTemporalResponse,
+)
+from semantica.kg.temporal_normalizer import TemporalNormalizer # noqa: E402
+from semantica.utils.exceptions import TemporalAmbiguityWarning # noqa: E402
# ── Helpers ─────────────────────────────────────────────────────────────────
+
def _make_entities():
return [
Entity(text="Apple", label="ORG", start_char=0, end_char=5),
@@ -59,15 +67,17 @@ def _ref_date():
# Part 1 – extract_relations_llm() temporal flag
# ============================================================================
+
class TestTemporalExtractionFlag(unittest.TestCase):
def setUp(self):
from semantica.semantic_extract.methods import _result_cache
+
_result_cache.clear()
@patch("semantica.semantic_extract.methods.create_provider")
def test_extract_temporal_bounds_true_adds_four_fields(self, mock_create):
- """With extract_temporal_bounds=True all four temporal keys appear in metadata."""
+ """With extract_temporal_bounds=True all four temporal keys appear."""
mock_prov = MagicMock()
mock_prov.is_available.return_value = True
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(
@@ -135,7 +145,9 @@ class TestTemporalExtractionFlag(unittest.TestCase):
self.assertNotIn("temporal_source_text", meta)
@patch("semantica.semantic_extract.methods.create_provider")
- def test_no_temporal_signal_returns_zero_confidence_and_null_dates(self, mock_create):
+ def test_no_temporal_signal_returns_zero_confidence_and_null_dates(
+ self, mock_create
+ ):
"""When LLM returns no temporal signal, confidence=0.0 and dates are null."""
mock_prov = MagicMock()
mock_prov.is_available.return_value = True
@@ -200,10 +212,12 @@ class TestTemporalExtractionFlag(unittest.TestCase):
@patch("semantica.semantic_extract.methods.create_provider")
def test_correct_schema_used_when_temporal_true(self, mock_create):
- """generate_typed is called with RelationsWithTemporalResponse when flag=True."""
+ """generate_typed is called with RelationsWithTemporalResponse."""
mock_prov = MagicMock()
mock_prov.is_available.return_value = True
- mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(relations=[])
+ mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(
+ relations=[]
+ )
mock_create.return_value = mock_prov
extract_relations_llm(
@@ -238,6 +252,7 @@ class TestTemporalExtractionFlag(unittest.TestCase):
# Part 2 – TemporalNormalizer: relative dates
# ============================================================================
+
class TestTemporalNormalizerRelativeDates(unittest.TestCase):
def setUp(self):
@@ -316,10 +331,13 @@ class TestTemporalNormalizerRelativeDates(unittest.TestCase):
# Part 3 – TemporalNormalizer: partial / structured dates
# ============================================================================
+
class TestTemporalNormalizerPartialDates(unittest.TestCase):
def setUp(self):
- self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
+ self.tn = TemporalNormalizer(
+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
+ )
def test_year_only(self):
result = self.tn.normalize("2021")
@@ -399,10 +417,13 @@ class TestTemporalNormalizerPartialDates(unittest.TestCase):
# Part 4 – TemporalNormalizer: ambiguous formats
# ============================================================================
+
class TestTemporalNormalizerAmbiguity(unittest.TestCase):
def setUp(self):
- self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
+ self.tn = TemporalNormalizer(
+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
+ )
def test_ambiguous_slash_date_raises_warning_and_returns_none(self):
with warnings.catch_warnings(record=True) as w:
@@ -435,14 +456,19 @@ class TestTemporalNormalizerAmbiguity(unittest.TestCase):
# Part 5 – TemporalNormalizer: domain phrase map
# ============================================================================
+
class TestTemporalNormalizerDomainPhrases(unittest.TestCase):
def setUp(self):
- self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
+ self.tn = TemporalNormalizer(
+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
+ )
def _assert_recognized(self, phrase):
result = self.tn.normalize_phrase(phrase)
- self.assertIsNotNone(result, f"Expected phrase {phrase!r} to be recognized but got None")
+ self.assertIsNotNone(
+ result, f"Expected phrase {phrase!r} to be recognized but got None"
+ )
return result
# General / Policy
@@ -525,6 +551,7 @@ class TestTemporalNormalizerDomainPhrases(unittest.TestCase):
# Part 6 – TemporalNormalizer: custom phrase map
# ============================================================================
+
class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):
def setUp(self):
@@ -568,10 +595,12 @@ class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):
# Part 7 – Full pipeline: extract → normalize → BiTemporalFact
# ============================================================================
+
class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
def setUp(self):
from semantica.semantic_extract.methods import _result_cache
+
_result_cache.clear()
@patch("semantica.semantic_extract.methods.create_provider")
@@ -623,10 +652,12 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
self.assertEqual(vf[0].day, 1)
# Feed into BiTemporalFact
- fact = BiTemporalFact.from_relationship({
- "valid_from": "2014-05-01T00:00:00Z",
- "valid_until": None,
- })
+ fact = BiTemporalFact.from_relationship(
+ {
+ "valid_from": "2014-05-01T00:00:00Z",
+ "valid_until": None,
+ }
+ )
self.assertIsNotNone(fact.valid_from)
self.assertEqual(fact.valid_from.year, 2014)
self.assertEqual(fact.valid_from.month, 5)
@@ -663,7 +694,9 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
extract_temporal_bounds=True,
)
meta = rels[0].metadata
- tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
+ tn = TemporalNormalizer(
+ reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
+ )
vf = tn.normalize(meta["valid_from"])
vu = tn.normalize(meta["valid_until"])
diff --git a/tests/test_issue_1513_slim_core.py b/tests/test_issue_1513_slim_core.py
index 039bccd9..f4de58a3 100644
--- a/tests/test_issue_1513_slim_core.py
+++ b/tests/test_issue_1513_slim_core.py
@@ -14,13 +14,16 @@ def _load_toml(file_path: Path) -> dict:
content = file_path.read_text(encoding="utf-8")
try:
import tomllib # Python 3.11+ standard library
+
return tomllib.loads(content)
except ImportError:
try:
import tomli # Fast PEP 680 compatible parser for Python < 3.11
+
return tomli.loads(content)
except ImportError:
import toml # Fallback toml parser
+
return toml.loads(content)
@@ -34,10 +37,28 @@ def test_core_dependencies_count():
for d in deps
}
expected_22 = {
- "numpy", "pandas", "scipy", "scikit-learn", "rdflib", "networkx",
- "requests", "chardet", "protobuf", "grpcio", "pillow", "pydantic",
- "click", "rich", "tqdm", "pyyaml", "toml", "python-dotenv",
- "loguru", "structlog", "httpx", "pyarrow"
+ "numpy",
+ "pandas",
+ "scipy",
+ "scikit-learn",
+ "rdflib",
+ "networkx",
+ "requests",
+ "chardet",
+ "protobuf",
+ "grpcio",
+ "pillow",
+ "pydantic",
+ "click",
+ "rich",
+ "tqdm",
+ "pyyaml",
+ "toml",
+ "python-dotenv",
+ "loguru",
+ "structlog",
+ "httpx",
+ "pyarrow",
}
assert normalized_names == expected_22
assert len(normalized_names) == 22
@@ -49,14 +70,27 @@ def test_optional_extras_defined():
data = _load_toml(repo_root / "pyproject.toml")
extras = data["project"]["optional-dependencies"]
for extra in [
- "documents", "ingest-git", "embeddings-local", "nlp-spacy",
- "viz", "media", "vectorstore-faiss", "graph-embeddings", "all"
+ "documents",
+ "ingest-git",
+ "embeddings-local",
+ "nlp-spacy",
+ "viz",
+ "media",
+ "vectorstore-faiss",
+ "graph-embeddings",
+ "all",
]:
assert extra in extras, f"Missing extra {extra}"
all_extra_str = str(extras["all"])
for expected_ref in [
- "documents", "ingest-git", "embeddings-local", "nlp-spacy",
- "viz", "media", "graph-embeddings", "vectorstore-all"
+ "documents",
+ "ingest-git",
+ "embeddings-local",
+ "nlp-spacy",
+ "viz",
+ "media",
+ "graph-embeddings",
+ "vectorstore-all",
]:
assert expected_ref in all_extra_str, f"Missing {expected_ref} in all"
# And vectorstore-faiss is in vectorstore-all
@@ -81,6 +115,7 @@ def test_core_modules_importable():
import semantica.visualization
import semantica.semantic_extract
import semantica.pipeline
+
assert semantica.__version__ is not None
@@ -129,20 +164,74 @@ def test_xml_parser_lxml_explicit_requires_documents_extra():
def test_xml_ingestor_missing_hint():
with patch("semantica.ingest.xml_ingestor.etree", None):
from semantica.ingest.xml_ingestor import XMLIngestor
- with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
+
+ with pytest.raises(ImportError, match=r"semantica\[documents\]"):
XMLIngestor()
+def test_xml_ingestor_package_import_missing_hint():
+ import semantica.ingest as ingest_mod
+
+ ingest_mod.__dict__.pop("XMLIngestor", None)
+ with patch("semantica.ingest.xml_ingestor.etree", None):
+ with pytest.raises(ImportError, match=r"semantica\[documents\]"):
+ _ = ingest_mod.XMLIngestor
+
+
def test_repo_ingestor_missing_hint():
with patch("semantica.ingest.repo_ingestor.git", None):
from semantica.ingest.repo_ingestor import RepoIngestor
+
with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):
RepoIngestor()
+def test_repo_ingestor_package_import_missing_hint():
+ import semantica.ingest as ingest_mod
+
+ ingest_mod.__dict__.pop("RepoIngestor", None)
+ with patch("semantica.ingest.repo_ingestor.git", None):
+ with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):
+ _ = ingest_mod.RepoIngestor
+
+
+def test_git_analyzer_package_import_succeeds_without_git():
+ import semantica.ingest as ingest_mod
+
+ ingest_mod.__dict__.pop("GitAnalyzer", None)
+ with patch("semantica.ingest.repo_ingestor.git", None):
+ analyzer_cls = ingest_mod.GitAnalyzer
+ assert analyzer_cls is not None
+ analyzer = analyzer_cls()
+ assert analyzer is not None
+
+
+def test_salesforce_ingestor_package_import_missing_hint():
+ import semantica.ingest as ingest_mod
+
+ ingest_mod.__dict__.pop("SalesforceIngestor", None)
+ with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):
+ with pytest.raises(ImportError, match=r"semantica\[db-salesforce\]"):
+ _ = ingest_mod.SalesforceIngestor
+
+
+def test_parse_methods_dynamic_default_resolution():
+ from semantica.parse.methods import (
+ get_parse_method,
+ list_available_methods,
+ parse_document,
+ )
+
+ assert get_parse_method("document", "default") == parse_document
+ methods = list_available_methods()
+ assert "default" in methods.get("document", [])
+ assert "default" in methods.get("structured", [])
+
+
def test_node_embedder_gensim_missing_hint():
with patch("semantica.kg.node_embeddings.GENSIM_AVAILABLE", False):
from semantica.kg.node_embeddings import NodeEmbedder
+
with pytest.raises(ImportError, match=r"semantica\[graph-embeddings\]"):
NodeEmbedder()
@@ -150,6 +239,7 @@ def test_node_embedder_gensim_missing_hint():
def test_faiss_store_missing_hint():
with patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", False):
from semantica.vector_store.faiss_store import FAISSIndexBuilder, FAISSStore
+
builder = FAISSIndexBuilder(128)
with pytest.raises(ProcessingError, match=r"semantica\[vectorstore-faiss\]"):
builder.build_index("flat")
@@ -165,12 +255,16 @@ def test_visualization_missing_hint():
# Plotly is checked via px and go in _check_dependencies
with patch("semantica.visualization.embedding_visualizer.px", None):
visualizer = EmbeddingVisualizer()
- with pytest.raises(ProcessingError, match=r"Plotly is required.*semantica\[viz\]"):
+ with pytest.raises(
+ ProcessingError, match=r"Plotly is required.*semantica\[viz\]"
+ ):
visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))
with patch("semantica.visualization.embedding_visualizer.go", None):
visualizer = EmbeddingVisualizer()
- with pytest.raises(ProcessingError, match=r"Plotly is required.*semantica\[viz\]"):
+ with pytest.raises(
+ ProcessingError, match=r"Plotly is required.*semantica\[viz\]"
+ ):
visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))
@@ -180,18 +274,25 @@ def test_visualization_umap_missing_hint():
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
# Stand in for Plotly so we reach dimensionality reduction
- with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), \
- patch("semantica.visualization.embedding_visualizer.go", MagicMock()), \
- patch("semantica.visualization.embedding_visualizer.umap", None):
+ with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(
+ "semantica.visualization.embedding_visualizer.go", MagicMock()
+ ), patch("semantica.visualization.embedding_visualizer.umap", None):
visualizer = EmbeddingVisualizer()
- # High-dimensional embeddings (>2D) trigger dimensionality reduction with method="umap"
+ # High-dimensional embeddings (>2D) trigger dimensionality reduction
+ # with method="umap"
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
- with pytest.raises(ProcessingError, match=r"UMAP is required.*semantica\[viz\]"):
+ with pytest.raises(
+ ProcessingError, match=r"UMAP is required.*semantica\[viz\]"
+ ):
visualizer.visualize_2d_projection(embeddings, method="umap")
# Also verify 3D projection triggers the same actionable error on >3D embeddings
- embeddings_4d = np.array([[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]])
- with pytest.raises(ProcessingError, match=r"UMAP is required.*semantica\[viz\]"):
+ embeddings_4d = np.array(
+ [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]]
+ )
+ with pytest.raises(
+ ProcessingError, match=r"UMAP is required.*semantica\[viz\]"
+ ):
visualizer.visualize_3d_projection(embeddings_4d, method="umap")
@@ -201,8 +302,9 @@ def test_visualization_sklearn_missing_hint():
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
# Stand in for Plotly so we reach dimensionality reduction
- with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), \
- patch("semantica.visualization.embedding_visualizer.go", MagicMock()):
+ with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(
+ "semantica.visualization.embedding_visualizer.go", MagicMock()
+ ):
visualizer = EmbeddingVisualizer()
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
@@ -221,7 +323,7 @@ def test_visualization_sklearn_missing_hint():
def test_visualization_options_collision_free():
- """Options such as n_components, perplexity, n_neighbors must not cause keyword collisions."""
+ """Options like n_components, perplexity must not cause keyword collisions."""
import numpy as np
from unittest.mock import MagicMock
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
@@ -232,26 +334,121 @@ def test_visualization_options_collision_free():
mock_umap_module = MagicMock()
mock_umap_module.UMAP = mock_umap_cls
- with patch("semantica.visualization.embedding_visualizer.PCA", mock_pca), \
- patch("semantica.visualization.embedding_visualizer.TSNE", mock_tsne), \
- patch("semantica.visualization.embedding_visualizer.umap", mock_umap_module), \
- patch("semantica.visualization.embedding_visualizer.px", MagicMock()), \
- patch("semantica.visualization.embedding_visualizer.go", MagicMock()):
+ with patch("semantica.visualization.embedding_visualizer.PCA", mock_pca), patch(
+ "semantica.visualization.embedding_visualizer.TSNE", mock_tsne
+ ), patch(
+ "semantica.visualization.embedding_visualizer.umap", mock_umap_module
+ ), patch(
+ "semantica.visualization.embedding_visualizer.px", MagicMock()
+ ), patch(
+ "semantica.visualization.embedding_visualizer.go", MagicMock()
+ ):
visualizer = EmbeddingVisualizer()
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
# PCA with n_components
visualizer.visualize_2d_projection(embeddings, method="pca", n_components=2)
# TSNE with perplexity and random_state
- visualizer.visualize_2d_projection(embeddings, method="tsne", perplexity=1, random_state=42)
+ visualizer.visualize_2d_projection(
+ embeddings, method="tsne", perplexity=1, random_state=42
+ )
# UMAP with n_neighbors and min_dist
- visualizer.visualize_2d_projection(embeddings, method="umap", n_neighbors=2, min_dist=0.1)
+ visualizer.visualize_2d_projection(
+ embeddings, method="umap", n_neighbors=2, min_dist=0.1
+ )
# 3D with n_components
visualizer.visualize_3d_projection(embeddings, method="pca", n_components=3)
def test_spacy_load_missing_hint():
from semantica.semantic_extract.methods import load_spacy_model
+
with patch("semantica.semantic_extract.methods.spacy", None):
with pytest.raises(ImportError, match=r"semantica\[nlp-spacy\]"):
load_spacy_model("en_core_web_sm")
+
+
+def test_xml_parser_handles_comments():
+ xml_content = (
+ "<root><!-- top comment --><item id='1'>Value</item>"
+ "<!-- bottom comment --></root>"
+ )
+ # lxml engine
+ p_lxml = XMLParser(engine="lxml")
+ res_lxml = p_lxml.parse(xml_content)
+ assert res_lxml.root.tag == "root"
+ assert len(res_lxml.root.children) == 1
+ assert res_lxml.root.children[0].tag == "item"
+ assert res_lxml.root.children[0].text == "Value"
+
+ # etree engine
+ p_etree = XMLParser(engine="etree")
+ res_etree = p_etree.parse(xml_content)
+ assert res_etree.root.tag == "root"
+ assert len(res_etree.root.children) == 1
+ assert res_etree.root.children[0].tag == "item"
+ assert res_etree.root.children[0].text == "Value"
+
+
+def test_public_api_ingestor_handles_xml_comments():
+ from semantica.ingest.public_api_ingestor import PublicAPIIngestor
+
+ xml_content = "<root><!-- comment --><item id='1'>Value</item></root>"
+ ingestor = PublicAPIIngestor(rate_limit_delay=0)
+
+ # 1. Default (defusedxml if available)
+ parsed = ingestor._parse_xml(xml_content)
+ assert parsed["tag"] == "root"
+ assert len(parsed["children"]) == 1
+ assert parsed["children"][0]["tag"] == "item"
+ assert parsed["children"][0]["text"] == "Value"
+
+ # 2. lxml fallback
+ with patch("semantica.ingest.public_api_ingestor.safe_xml_etree", None):
+ parsed_lxml = ingestor._parse_xml(xml_content)
+ assert parsed_lxml["tag"] == "root"
+ assert len(parsed_lxml["children"]) == 1
+ assert parsed_lxml["children"][0]["tag"] == "item"
+ assert parsed_lxml["children"][0]["text"] == "Value"
+
+
+def test_huggingface_model_loader_catches_oserror():
+ import builtins
+ from unittest.mock import MagicMock
+ from semantica.semantic_extract.providers import HuggingFaceModelLoader
+
+ mock_torch = MagicMock()
+ mock_torch.Tensor = type("Tensor", (), {})
+ with patch.dict("sys.modules", {"torch": mock_torch}):
+ loader = HuggingFaceModelLoader()
+ # 1. Test ModuleNotFoundError / ImportError
+ with patch.dict("sys.modules", {"transformers": None}):
+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
+ loader.load_ner_model("bert-base-cased")
+
+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
+ loader.load_relation_model("bert-base-cased")
+
+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
+ loader.load_triplet_model("t5-base")
+
+ # 2. Test OSError (e.g. corrupt DLL / missing shared library)
+ real_import = builtins.__import__
+
+ def fake_import(name, *args, **kwargs):
+ if name == "transformers":
+ raise OSError("DLL load failed")
+ return real_import(name, *args, **kwargs)
+
+ try:
+ builtins.__import__ = fake_import
+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
+ loader.load_ner_model("bert-base-cased-oserror")
+
+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
+ loader.load_relation_model("bert-base-cased-oserror")
+
+ with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
+ loader.load_triplet_model("t5-base-oserror")
+ finally:
+ builtins.__import__ = real_import