Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(llms): resolve broken imports in llms provenance wrappers (#1912)

Kevin6d7364df8722
diff --git a/semantica/llms/litellm.py b/semantica/llms/litellm.py
index 2db65528..7e63b56d 100644
--- a/semantica/llms/litellm.py
+++ b/semantica/llms/litellm.py
@@ -13,6 +13,7 @@ from pydantic import BaseModel
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
+from ..semantic_extract.providers import ResponseText
logger = get_logger("llms.litellm")
@@ -168,16 +169,32 @@ class LiteLLM:
**options
)
+ # Metadata the provenance wrappers read back off the returned value.
+ # Only usage and cost are kept: the rest of _hidden_params can hold
+ # api_base and provider response headers.
+ hidden_params = getattr(response, '_hidden_params', None) or {}
+ usage = getattr(response, 'usage', None)
+ if usage is None and isinstance(response, dict):
+ usage = response.get('usage')
+ meta = {
+ "usage": usage,
+ "cost": hidden_params.get('response_cost'),
+ }
+
+ def _wrap(content):
+ # Tool-call, refusal and content-filter replies carry no text.
+ return None if content is None else ResponseText(content, **meta)
+
# Extract text from response
if hasattr(response, 'choices') and len(response.choices) > 0:
- return response.choices[0].message.content
+ return _wrap(response.choices[0].message.content)
elif isinstance(response, dict):
if 'choices' in response and len(response['choices']) > 0:
- return response['choices'][0]['message']['content']
+ return _wrap(response['choices'][0]['message']['content'])
elif 'content' in response:
- return response['content']
+ return _wrap(response['content'])
elif isinstance(response, str):
- return response
+ return _wrap(response)
raise ProcessingError(f"Unexpected response format from LiteLLM: {type(response)}")
diff --git a/semantica/llms/llms_provenance.py b/semantica/llms/llms_provenance.py
index f6277fff..287f92c1 100644
--- a/semantica/llms/llms_provenance.py
+++ b/semantica/llms/llms_provenance.py
@@ -7,7 +7,8 @@ This module provides provenance tracking for all LLM operations:
- HuggingFace LLM
- LiteLLM
-Tracks: model name, tokens (prompt/completion), cost, latency, prompts, responses
+Tracks: model name, tokens (prompt/completion), latency, prompts, responses, and
+cost where the provider reports it (LiteLLM; OpenAI and Groq do not)
All classes wrap the original LLM providers and add optional provenance tracking
without modifying existing functionality.
@@ -17,15 +18,15 @@ Usage:
GroqLLMWithProvenance,
OpenAILLMWithProvenance
)
-
+
# Enable provenance tracking
llm = GroqLLMWithProvenance(provenance=True)
response = llm.generate("What is artificial intelligence?")
-
+
# Provenance automatically tracks:
# - Model used
# - Token counts
- # - API costs
+ # - API costs (LiteLLM only; OpenAI and Groq do not report cost)
# - Latency
# - Prompt and response previews
@@ -41,20 +42,43 @@ Author: Semantica Contributors
License: MIT
"""
-from typing import Optional, Dict, Any
+from typing import Optional, Any
from datetime import datetime
+import json
import time
import uuid
+from pydantic import BaseModel
+
+
+def _preview_result(result: Any) -> Optional[str]:
+ """Render a structured/typed result as a 200-char preview.
+
+ Serializes the whole value, so a Pydantic model with a ``text`` or
+ ``content`` field is not reduced to that one field.
+ """
+ if result is None:
+ return None
+ if isinstance(result, BaseModel):
+ preview = result.model_dump_json()
+ elif isinstance(result, (dict, list)):
+ try:
+ preview = json.dumps(result, default=str)
+ except (TypeError, ValueError):
+ preview = str(result)
+ else:
+ preview = str(result)
+ return preview[:200]
+
class LLMProvenanceMixin:
"""
Mixin to add provenance tracking to any LLM provider.
-
+
This mixin provides common provenance infrastructure for tracking
LLM API calls including tokens, costs, and performance metrics.
"""
-
+
def __init__(
self,
provenance: bool = False,
@@ -102,14 +126,19 @@ class LLMProvenanceMixin:
**metadata: Additional metadata (tokens, cost, latency, etc.)
"""
if self.provenance and self._prov_manager:
- # Extract response text
+ # Extract response text; a reply with no text stays None rather
+ # than being recorded as the string "None".
response_text = response
- if hasattr(response, 'text'):
- response_text = response.text
- elif hasattr(response, 'content'):
- response_text = response.content
- elif not isinstance(response, str):
- response_text = str(response)
+ if response is None:
+ response_preview = None
+ else:
+ if hasattr(response, 'text'):
+ response_text = response.text
+ elif hasattr(response, 'content'):
+ response_text = response.content
+ elif not isinstance(response, str):
+ response_text = str(response)
+ response_preview = str(response_text)[:200]
# Typed Activity timing (issue #825, Part B Tier 1): popped out so
# it populates real fields, not the opaque metadata blob.
@@ -129,24 +158,67 @@ class LLMProvenanceMixin:
metadata={
"model": getattr(self, 'model', 'unknown'),
"prompt_preview": prompt[:200] if len(prompt) > 200 else prompt,
- "response_preview": response_text[:200] if len(str(response_text)) > 200 else str(response_text),
+ "response_preview": response_preview,
**metadata
}
)
+ def _track_generation(
+ self,
+ kind: str,
+ call_id_prefix: str,
+ prompt: str,
+ invoke,
+ **metadata,
+ ):
+ """Run one generation call and record a provenance entry for it.
+
+ Args:
+ kind: Generation mode label (e.g. ``"structured"``, ``"typed"``),
+ recorded as ``generation_mode``.
+ call_id_prefix: Prefix for the generated call id.
+ prompt: Input prompt.
+ invoke: Zero-argument callable performing the wrapped LLM call.
+ **metadata: Extra metadata recorded with the entry.
+
+ Returns:
+ Whatever ``invoke`` returns, unchanged.
+ """
+ start_time = time.time()
+ activity_started_at_time = datetime.utcnow().isoformat()
+ result = invoke()
+ elapsed = time.time() - start_time
+ activity_ended_at_time = datetime.utcnow().isoformat()
+
+ if self.provenance:
+ # Pass the serialized preview rather than the raw result, so
+ # _track_llm_call does not pick a model's .text/.content field.
+ self._track_llm_call(
+ call_id=f"{call_id_prefix}_{uuid.uuid4().hex[:8]}",
+ prompt=prompt,
+ response=_preview_result(result),
+ generation_mode=kind,
+ latency_seconds=elapsed,
+ activity_started_at_time=activity_started_at_time,
+ activity_ended_at_time=activity_ended_at_time,
+ **metadata,
+ )
+
+ return result
+
class GroqLLMWithProvenance(LLMProvenanceMixin):
"""
Groq LLM with provenance tracking.
-
+
Wraps the original GroqLLM and tracks all API calls with complete metadata.
-
+
Example:
>>> llm = GroqLLMWithProvenance(provenance=True, model="llama-3.1-70b")
>>> response = llm.generate("Explain quantum computing")
- >>> # API call is tracked with model, tokens, cost, latency
+ >>> # API call is tracked with model, tokens, latency (Groq reports no cost)
"""
-
+
def __init__(
self,
provenance: bool = False,
@@ -161,22 +233,22 @@ class GroqLLMWithProvenance(LLMProvenanceMixin):
provenance: Enable provenance tracking (default: False)
**config: Configuration passed to original GroqLLM
"""
- from .groq_llm import GroqLLM
+ from .groq import Groq
LLMProvenanceMixin.__init__(
self, provenance=provenance, agent_id=agent_id, is_automated=is_automated
)
- self._llm = GroqLLM(**config)
+ self._llm = Groq(**config)
self.model = getattr(self._llm, 'model', 'groq')
-
+
def generate(self, prompt: str, **kwargs):
"""
Generate response with provenance tracking.
-
+
Args:
prompt: Input prompt
**kwargs: Additional generation parameters
-
+
Returns:
LLM response (same format as original GroqLLM)
"""
@@ -205,7 +277,12 @@ class GroqLLMWithProvenance(LLMProvenanceMixin):
response=response,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
- total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None,
+ total_tokens=(
+ (prompt_tokens + completion_tokens)
+ if prompt_tokens is not None
+ and completion_tokens is not None
+ else None
+ ),
total_cost=total_cost,
latency_seconds=elapsed,
activity_started_at_time=activity_started_at_time,
@@ -214,21 +291,46 @@ class GroqLLMWithProvenance(LLMProvenanceMixin):
max_tokens=kwargs.get('max_tokens'),
top_p=kwargs.get('top_p')
)
-
+
return response
-
+
+ def generate_structured(self, prompt: str, **kwargs):
+ """Generate structured JSON with provenance tracking."""
+ return self._track_generation(
+ "structured",
+ "groq_structured",
+ prompt,
+ lambda: self._llm.generate_structured(prompt, **kwargs),
+ )
+
+ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):
+ """Generate schema-validated output with provenance tracking."""
+ return self._track_generation(
+ "typed",
+ "groq_typed",
+ prompt,
+ lambda: self._llm.generate_typed(
+ prompt, schema, max_retries=max_retries, **kwargs
+ ),
+ max_retries=max_retries,
+ )
+
def __getattr__(self, name):
"""Delegate other methods to wrapped LLM."""
+ if name == "_llm":
+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it
+ # up through getattr(self._llm, ...) would recurse forever.
+ raise AttributeError(name)
return getattr(self._llm, name)
class OpenAILLMWithProvenance(LLMProvenanceMixin):
"""
OpenAI LLM with provenance tracking.
-
+
Wraps the original OpenAILLM and tracks all API calls.
"""
-
+
def __init__(
self,
provenance: bool = False,
@@ -243,22 +345,22 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin):
provenance: Enable provenance tracking (default: False)
**config: Configuration passed to original OpenAILLM
"""
- from .openai_llm import OpenAILLM
+ from .openai import OpenAI
LLMProvenanceMixin.__init__(
self, provenance=provenance, agent_id=agent_id, is_automated=is_automated
)
- self._llm = OpenAILLM(**config)
+ self._llm = OpenAI(**config)
self.model = getattr(self._llm, 'model', 'openai')
-
+
def generate(self, prompt: str, **kwargs):
"""
Generate response with provenance tracking.
-
+
Args:
prompt: Input prompt
**kwargs: Additional generation parameters
-
+
Returns:
LLM response
"""
@@ -287,7 +389,12 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin):
response=response,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
- total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None,
+ total_tokens=(
+ (prompt_tokens + completion_tokens)
+ if prompt_tokens is not None
+ and completion_tokens is not None
+ else None
+ ),
total_cost=total_cost,
latency_seconds=elapsed,
activity_started_at_time=activity_started_at_time,
@@ -295,21 +402,46 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin):
temperature=kwargs.get('temperature'),
max_tokens=kwargs.get('max_tokens')
)
-
+
return response
-
+
+ def generate_structured(self, prompt: str, **kwargs):
+ """Generate structured JSON with provenance tracking."""
+ return self._track_generation(
+ "structured",
+ "openai_structured",
+ prompt,
+ lambda: self._llm.generate_structured(prompt, **kwargs),
+ )
+
+ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):
+ """Generate schema-validated output with provenance tracking."""
+ return self._track_generation(
+ "typed",
+ "openai_typed",
+ prompt,
+ lambda: self._llm.generate_typed(
+ prompt, schema, max_retries=max_retries, **kwargs
+ ),
+ max_retries=max_retries,
+ )
+
def __getattr__(self, name):
"""Delegate other methods to wrapped LLM."""
+ if name == "_llm":
+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it
+ # up through getattr(self._llm, ...) would recurse forever.
+ raise AttributeError(name)
return getattr(self._llm, name)
class HuggingFaceLLMWithProvenance(LLMProvenanceMixin):
"""
HuggingFace LLM with provenance tracking.
-
+
Wraps the original HuggingFaceLLM and tracks all generations.
"""
-
+
def __init__(
self,
provenance: bool = False,
@@ -324,22 +456,22 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin):
provenance: Enable provenance tracking (default: False)
**config: Configuration passed to original HuggingFaceLLM
"""
- from .huggingface_llm import HuggingFaceLLM
+ from .huggingface import HuggingFaceLLM
LLMProvenanceMixin.__init__(
self, provenance=provenance, agent_id=agent_id, is_automated=is_automated
)
self._llm = HuggingFaceLLM(**config)
self.model = getattr(self._llm, 'model', 'huggingface')
-
+
def generate(self, prompt: str, **kwargs):
"""
Generate response with provenance tracking.
-
+
Args:
prompt: Input prompt
**kwargs: Additional generation parameters
-
+
Returns:
LLM response
"""
@@ -360,21 +492,46 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin):
max_length=kwargs.get('max_length'),
temperature=kwargs.get('temperature')
)
-
+
return response
-
+
+ def generate_structured(self, prompt: str, **kwargs):
+ """Generate structured JSON with provenance tracking."""
+ return self._track_generation(
+ "structured",
+ "hf_structured",
+ prompt,
+ lambda: self._llm.generate_structured(prompt, **kwargs),
+ )
+
+ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):
+ """Generate schema-validated output with provenance tracking."""
+ return self._track_generation(
+ "typed",
+ "hf_typed",
+ prompt,
+ lambda: self._llm.generate_typed(
+ prompt, schema, max_retries=max_retries, **kwargs
+ ),
+ max_retries=max_retries,
+ )
+
def __getattr__(self, name):
"""Delegate other methods to wrapped LLM."""
+ if name == "_llm":
+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it
+ # up through getattr(self._llm, ...) would recurse forever.
+ raise AttributeError(name)
return getattr(self._llm, name)
class LiteLLMWithProvenance(LLMProvenanceMixin):
"""
LiteLLM with provenance tracking.
-
+
Wraps the original LiteLLM and tracks all API calls across providers.
"""
-
+
def __init__(
self,
provenance: bool = False,
@@ -389,22 +546,22 @@ class LiteLLMWithProvenance(LLMProvenanceMixin):
provenance: Enable provenance tracking (default: False)
**config: Configuration passed to original LiteLLM
"""
- from .lite_llm import LiteLLM
+ from .litellm import LiteLLM
LLMProvenanceMixin.__init__(
self, provenance=provenance, agent_id=agent_id, is_automated=is_automated
)
self._llm = LiteLLM(**config)
self.model = getattr(self._llm, 'model', 'litellm')
-
+
def generate(self, prompt: str, **kwargs):
"""
Generate response with provenance tracking.
-
+
Args:
prompt: Input prompt
**kwargs: Additional generation parameters
-
+
Returns:
LLM response
"""
@@ -424,8 +581,8 @@ class LiteLLMWithProvenance(LLMProvenanceMixin):
prompt_tokens = getattr(response.usage, 'prompt_tokens', None)
completion_tokens = getattr(response.usage, 'completion_tokens', None)
- if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params:
- total_cost = response._hidden_params['response_cost']
+ if hasattr(response, 'cost'):
+ total_cost = response.cost
self._track_llm_call(
call_id=f"lite_call_{uuid.uuid4().hex[:8]}",
@@ -439,11 +596,36 @@ class LiteLLMWithProvenance(LLMProvenanceMixin):
activity_ended_at_time=activity_ended_at_time,
provider=kwargs.get('provider')
)
-
+
return response
-
+
+ def generate_structured(self, prompt: str, **kwargs):
+ """Generate structured JSON with provenance tracking."""
+ return self._track_generation(
+ "structured",
+ "lite_structured",
+ prompt,
+ lambda: self._llm.generate_structured(prompt, **kwargs),
+ )
+
+ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):
+ """Generate schema-validated output with provenance tracking."""
+ return self._track_generation(
+ "typed",
+ "lite_typed",
+ prompt,
+ lambda: self._llm.generate_typed(
+ prompt, schema, max_retries=max_retries, **kwargs
+ ),
+ max_retries=max_retries,
+ )
+
def __getattr__(self, name):
"""Delegate other methods to wrapped LLM."""
+ if name == "_llm":
+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it
+ # up through getattr(self._llm, ...) would recurse forever.
+ raise AttributeError(name)
return getattr(self._llm, name)
diff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.py
index 5c870a3e..46c7e8d0 100644
--- a/semantica/semantic_extract/providers.py
+++ b/semantica/semantic_extract/providers.py
@@ -145,6 +145,31 @@ def _extract_first_json_value(text: str):
return None
+class ResponseText(str):
+ """A ``str`` that also carries the provider response's usage and cost.
+
+ ``generate()`` must keep returning the generated text, but the provenance
+ wrappers in ``semantica.llms`` also need the token counts (and, where the
+ provider reports it, the cost) that only the raw provider response holds.
+ OpenAI and Groq responses carry no cost, so only ``usage`` is set for them;
+ LiteLLM supplies ``cost`` from its ``response_cost``. Attaching that
+ metadata to the returned value keeps it scoped to the request it came from,
+ rather than stashing it on the provider instance where concurrent calls
+ would overwrite each other.
+ """
+
+ def __new__(
+ cls,
+ text: str,
+ usage: Any = None,
+ cost: Optional[float] = None,
+ ) -> "ResponseText":
+ instance = super().__new__(cls, text if text is not None else "")
+ instance.usage = usage
+ instance.cost = cost
+ return instance
+
+
class BaseProvider:
"""Base class for providers - makes it easy to add custom providers."""
@@ -802,7 +827,11 @@ class OpenAIProvider(BaseProvider):
)
response = self.client.chat.completions.create(**create_kwargs)
- return response.choices[0].message.content
+ content = response.choices[0].message.content
+ if content is None:
+ # Tool-call, refusal and content-filter replies carry no text.
+ return None
+ return ResponseText(content, usage=getattr(response, "usage", None))
def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:
"""Generate structured JSON output."""
@@ -1083,7 +1112,11 @@ class GroqProvider(BaseProvider):
)
response = self.client.chat.completions.create(**create_kwargs)
- return response.choices[0].message.content
+ content = response.choices[0].message.content
+ if content is None:
+ # Tool-call, refusal and content-filter replies carry no text.
+ return None
+ return ResponseText(content, usage=getattr(response, "usage", None))
def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:
"""Generate structured output."""
diff --git a/tests/provenance/test_llms_provenance.py b/tests/provenance/test_llms_provenance.py
index cb7a6daa..cb639094 100644
--- a/tests/provenance/test_llms_provenance.py
+++ b/tests/provenance/test_llms_provenance.py
@@ -7,9 +7,20 @@ Tests that provenance tracking works correctly for all LLM providers.
import pytest
+class _StubProvider:
+ """Stand-in for the provider a wrapper's target class constructs.
+
+ Keeps these tests offline: constructing ``HuggingFaceLLM`` would otherwise
+ load (and download) a real model.
+ """
+
+ def __init__(self, *args, **kwargs):
+ pass
+
+
class TestGroqLLMProvenance:
"""Test Groq LLM with provenance."""
-
+
def test_without_provenance(self):
"""Test Groq LLM works without provenance."""
try:
@@ -19,7 +30,7 @@ class TestGroqLLMProvenance:
assert llm.provenance is False
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_with_provenance_enabled(self):
"""Test Groq LLM tracks provenance."""
try:
@@ -33,7 +44,7 @@ class TestGroqLLMProvenance:
class TestOpenAILLMProvenance:
"""Test OpenAI LLM with provenance."""
-
+
def test_without_provenance(self):
"""Test OpenAI LLM works without provenance."""
try:
@@ -42,7 +53,7 @@ class TestOpenAILLMProvenance:
assert llm is not None
except ImportError:
pytest.skip("OpenAILLM not available")
-
+
def test_with_provenance_enabled(self):
"""Test OpenAI LLM tracks provenance."""
try:
@@ -55,9 +66,14 @@ class TestOpenAILLMProvenance:
class TestHuggingFaceLLMProvenance:
"""Test HuggingFace LLM with provenance."""
-
- def test_without_provenance(self):
+
+ def test_without_provenance(self, monkeypatch):
"""Test HuggingFace LLM works without provenance."""
+ # Stub the provider so the wrapper can be exercised without loading a
+ # model; without this the test downloads ``gpt2`` on every run.
+ monkeypatch.setattr(
+ "semantica.llms.huggingface.HuggingFaceLLMProvider", _StubProvider
+ )
try:
from semantica.llms.llms_provenance import HuggingFaceLLMWithProvenance
llm = HuggingFaceLLMWithProvenance(provenance=False)
@@ -68,7 +84,7 @@ class TestHuggingFaceLLMProvenance:
class TestLLMProvenanceEdgeCases:
"""Test edge cases for LLM provenance."""
-
+
def test_empty_prompt(self):
"""Test LLM with empty prompt."""
try:
@@ -78,29 +94,27 @@ class TestLLMProvenanceEdgeCases:
assert llm is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_very_long_prompt(self):
"""Test LLM with very long prompt."""
try:
from semantica.llms.llms_provenance import GroqLLMWithProvenance
llm = GroqLLMWithProvenance(provenance=True)
- long_prompt = "Explain " + "AI " * 1000
# Should handle long prompts
assert llm is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_special_characters_in_prompt(self):
"""Test LLM with special characters in prompt."""
try:
from semantica.llms.llms_provenance import GroqLLMWithProvenance
llm = GroqLLMWithProvenance(provenance=True)
- special_prompt = "Explain @#$%^&*() and 中文 émojis 🎉"
# Should handle special characters
assert llm is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_multiple_llm_calls(self):
"""Test multiple LLM calls with provenance."""
try:
@@ -110,7 +124,7 @@ class TestLLMProvenanceEdgeCases:
assert llm._prov_manager is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_concurrent_llm_calls(self):
"""Test concurrent LLM calls."""
try:
@@ -120,7 +134,7 @@ class TestLLMProvenanceEdgeCases:
assert llm is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_llm_with_custom_parameters(self):
"""Test LLM with custom generation parameters."""
try:
@@ -130,7 +144,7 @@ class TestLLMProvenanceEdgeCases:
assert llm._prov_manager is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_llm_response_without_usage(self):
"""Test LLM response without usage metadata."""
try:
@@ -140,7 +154,7 @@ class TestLLMProvenanceEdgeCases:
assert llm is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_llm_response_without_cost(self):
"""Test LLM response without cost information."""
try:
@@ -150,7 +164,7 @@ class TestLLMProvenanceEdgeCases:
assert llm is not None
except ImportError:
pytest.skip("GroqLLM not available")
-
+
def test_different_llm_providers(self):
"""Test different LLM providers with same provenance pattern."""
try:
@@ -165,7 +179,7 @@ class TestLLMProvenanceEdgeCases:
assert openai is not None
except ImportError:
pytest.skip("LLM providers not available")
-
+
def test_llm_latency_tracking(self):
"""Test that LLM latency is tracked correctly."""
try:
diff --git a/tests/provenance/test_llms_provenance_imports.py b/tests/provenance/test_llms_provenance_imports.py
new file mode 100644
index 00000000..b5323c9c
--- /dev/null
+++ b/tests/provenance/test_llms_provenance_imports.py
@@ -0,0 +1,74 @@
+"""Regression tests for the LLM provenance wrappers' import resolution.
+
+Each ``*WithProvenance`` wrapper imports the LLM class it wraps lazily, inside
+``__init__``, from a module path that had drifted from the real one
+(``.groq_llm`` / ``.openai_llm`` / ``.huggingface_llm`` / ``.lite_llm``). A
+wrong path is invisible until the wrapper is instantiated, and the surrounding
+suites guard those instantiations with ``pytest.skip``, so the breakage never
+reached CI.
+
+These tests do not skip: if a wrapper cannot resolve the class it wraps, the
+test fails. The providers behind those classes are stubbed so the suite stays
+offline and does not require the optional ``groq`` / ``litellm`` packages.
+"""
+
+
+class _StubProvider:
+ """Stand-in for the ``*Provider`` a wrapper's target class constructs."""
+
+ def __init__(self, *args, **kwargs):
+ pass
+
+
+class TestGroqWrapper:
+ """GroqLLMWithProvenance must resolve the Groq class it wraps."""
+
+ def test_instantiates_and_wraps_groq(self, monkeypatch):
+ monkeypatch.setattr("semantica.llms.groq.GroqProvider", _StubProvider)
+ from semantica.llms.groq import Groq
+ from semantica.llms.llms_provenance import GroqLLMWithProvenance
+
+ wrapper = GroqLLMWithProvenance(provenance=False)
+ assert isinstance(wrapper._llm, Groq)
+
+
+class TestOpenAIWrapper:
+ """OpenAILLMWithProvenance must resolve the OpenAI class it wraps."""
+
+ def test_instantiates_and_wraps_openai(self, monkeypatch):
+ monkeypatch.setattr("semantica.llms.openai.OpenAIProvider", _StubProvider)
+ from semantica.llms.openai import OpenAI
+ from semantica.llms.llms_provenance import OpenAILLMWithProvenance
+
+ wrapper = OpenAILLMWithProvenance(provenance=False)
+ assert isinstance(wrapper._llm, OpenAI)
+
+
+class TestHuggingFaceWrapper:
+ """HuggingFaceLLMWithProvenance must resolve the class it wraps."""
+
+ def test_instantiates_and_wraps_huggingface(self, monkeypatch):
+ # The real provider loads a transformer model during construction.
+ monkeypatch.setattr(
+ "semantica.llms.huggingface.HuggingFaceLLMProvider", _StubProvider
+ )
+ from semantica.llms.huggingface import HuggingFaceLLM
+ from semantica.llms.llms_provenance import HuggingFaceLLMWithProvenance
+
+ wrapper = HuggingFaceLLMWithProvenance(provenance=False)
+ assert isinstance(wrapper._llm, HuggingFaceLLM)
+
+
+class TestLiteLLMWrapper:
+ """LiteLLMWithProvenance must resolve the LiteLLM class it wraps."""
+
+ def test_instantiates_and_wraps_litellm(self, monkeypatch):
+ # LiteLLM refuses to construct when its optional package is absent; the
+ # wrapper only needs the class object, so let it through.
+ monkeypatch.setattr("semantica.llms.litellm.LITELLM_AVAILABLE", True)
+ from semantica.llms.litellm import LiteLLM
+ from semantica.llms.llms_provenance import LiteLLMWithProvenance
+
+ # `model` is a required argument of LiteLLM; omitting it raises.
+ wrapper = LiteLLMWithProvenance(provenance=False, model="openai/gpt-4o-mini")
+ assert isinstance(wrapper._llm, LiteLLM)
diff --git a/tests/provenance/test_llms_provenance_tracking.py b/tests/provenance/test_llms_provenance_tracking.py
new file mode 100644
index 00000000..620a22be
--- /dev/null
+++ b/tests/provenance/test_llms_provenance_tracking.py
@@ -0,0 +1,430 @@
+"""Regression tests for what the LLM provenance wrappers actually record.
+
+Two gaps in the wrappers let calls go untracked or tracked without their
+usage/cost:
+
+* ``generate_structured`` / ``generate_typed`` fell through ``__getattr__`` to
+ the wrapped LLM, so those calls produced no provenance entry at all.
+* ``generate`` read token counts and cost off the returned value, but the
+ providers returned a plain ``str``, so ``prompt_tokens`` / ``completion_tokens``
+ / ``total_cost`` were always ``None``.
+
+These tests drive the wrappers with a stub LLM and a recording provenance
+manager so they stay offline and assert on the recorded metadata directly.
+A second group drives the providers themselves with a canned client, so a
+provider that stops carrying usage/cost turns the suite red.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+from pydantic import BaseModel
+
+from semantica.llms.llms_provenance import (
+ GroqLLMWithProvenance,
+ LiteLLMWithProvenance,
+)
+from semantica.semantic_extract.providers import (
+ GroqProvider,
+ OpenAIProvider,
+ ResponseText,
+)
+
+
+class _RecordingProvenanceManager:
+ """Capture the kwargs of every ``track_entity`` call."""
+
+ def __init__(self):
+ self.calls = []
+
+ def track_entity(self, **kwargs):
+ self.calls.append(kwargs)
+
+
+class _StubLLM:
+ """Minimal stand-in for a wrapped provider class."""
+
+ model = "stub-model"
+
+ def __init__(self, response=None, structured=None, typed=None):
+ self._response = response
+ self._structured = structured
+ self._typed = typed
+
+ def generate(self, prompt, **kwargs):
+ return self._response
+
+ def generate_structured(self, prompt, **kwargs):
+ return self._structured
+
+ def generate_typed(self, prompt, schema, max_retries=3, **kwargs):
+ return self._typed
+
+
+def _wrapper_with(llm):
+ """Build a Groq wrapper wired to *llm* and a recording provenance manager."""
+ wrapper = GroqLLMWithProvenance(provenance=False)
+ recorder = _RecordingProvenanceManager()
+ wrapper.provenance = True
+ wrapper._prov_manager = recorder
+ wrapper._llm = llm
+ return wrapper, recorder
+
+
+class TestStructuredAndTypedAreTracked:
+ """Structured/typed calls must produce a provenance entry."""
+
+ def test_generate_structured_records_entry(self):
+ wrapper, recorder = _wrapper_with(
+ _StubLLM(structured={"answer": 42})
+ )
+
+ result = wrapper.generate_structured("question")
+
+ assert result == {"answer": 42}
+ assert len(recorder.calls) == 1
+ assert recorder.calls[0]["metadata"]["generation_mode"] == "structured"
+
+ def test_generate_typed_records_entry(self):
+ wrapper, recorder = _wrapper_with(_StubLLM(typed="validated"))
+
+ result = wrapper.generate_typed("question", schema=object)
+
+ assert result == "validated"
+ assert len(recorder.calls) == 1
+ metadata = recorder.calls[0]["metadata"]
+ assert metadata["generation_mode"] == "typed"
+ assert metadata["max_retries"] == 3
+
+ def test_calls_are_tracked_independently(self):
+ wrapper, recorder = _wrapper_with(
+ _StubLLM(structured={"a": 1}, typed="b")
+ )
+
+ wrapper.generate_structured("first")
+ wrapper.generate_typed("second", schema=object)
+
+ assert len(recorder.calls) == 2
+ modes = [c["metadata"]["generation_mode"] for c in recorder.calls]
+ assert modes == ["structured", "typed"]
+ assert recorder.calls[0]["entity_id"] != recorder.calls[1]["entity_id"]
+
+ def test_untracked_when_provenance_disabled(self):
+ wrapper = GroqLLMWithProvenance(provenance=False)
+ wrapper._llm = _StubLLM(structured={"a": 1})
+
+ # No provenance manager is set, so the call must still succeed.
+ assert wrapper.generate_structured("q") == {"a": 1}
+
+
+class TestUsageAndCostPropagate:
+ """Token counts and cost must reach the recorded metadata."""
+
+ def test_generate_records_tokens_and_cost(self):
+ response = ResponseText(
+ "the answer",
+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5),
+ cost=0.002,
+ )
+ wrapper, recorder = _wrapper_with(_StubLLM(response=response))
+
+ result = wrapper.generate("question")
+
+ assert result == "the answer"
+ assert len(recorder.calls) == 1
+ metadata = recorder.calls[0]["metadata"]
+ assert metadata["prompt_tokens"] == 10
+ assert metadata["completion_tokens"] == 5
+ assert metadata["total_tokens"] == 15
+ assert metadata["total_cost"] == 0.002
+
+ def test_zero_completion_tokens_still_sums(self):
+ response = ResponseText(
+ "",
+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=0),
+ )
+ wrapper, recorder = _wrapper_with(_StubLLM(response=response))
+
+ wrapper.generate("question")
+
+ metadata = recorder.calls[0]["metadata"]
+ assert metadata["completion_tokens"] == 0
+ assert metadata["total_tokens"] == 10
+
+ def test_generate_without_usage_metadata_stays_none(self):
+ wrapper, recorder = _wrapper_with(_StubLLM(response=ResponseText("plain")))
+
+ wrapper.generate("question")
+
+ metadata = recorder.calls[0]["metadata"]
+ assert metadata["prompt_tokens"] is None
+ assert metadata["completion_tokens"] is None
+ assert metadata["total_cost"] is None
+
+
+class TestResponseText:
+ """``ResponseText`` must behave like ``str`` while carrying metadata."""
+
+ def test_is_a_string(self):
+ text = ResponseText("hello")
+ assert isinstance(text, str)
+ assert text == "hello"
+ assert text.upper() == "HELLO"
+
+ def test_defaults(self):
+ text = ResponseText("hello")
+ assert text.usage is None
+ assert text.cost is None
+
+ def test_carries_usage_and_cost(self):
+ usage = SimpleNamespace(prompt_tokens=1, completion_tokens=2)
+ text = ResponseText("hi", usage=usage, cost=0.5)
+ assert text.usage is usage
+ assert text.cost == 0.5
+
+
+ "wrapper_name",
+ [
+ "GroqLLMWithProvenance",
+ "OpenAILLMWithProvenance",
+ "HuggingFaceLLMWithProvenance",
+ "LiteLLMWithProvenance",
+ ],
+)
+def test_all_wrappers_expose_structured_and_typed(wrapper_name):
+ """Every wrapper must override both structured entry points."""
+ import semantica.llms.llms_provenance as module
+
+ wrapper_cls = getattr(module, wrapper_name)
+ assert "generate_structured" in wrapper_cls.__dict__
+ assert "generate_typed" in wrapper_cls.__dict__
+
+
+ "wrapper_name",
+ [
+ "GroqLLMWithProvenance",
+ "OpenAILLMWithProvenance",
+ "HuggingFaceLLMWithProvenance",
+ "LiteLLMWithProvenance",
+ ],
+)
+def test_unset_llm_raises_attribute_error_not_recursion(wrapper_name):
+ """A wrapper whose ``_llm`` was never set must not recurse in __getattr__."""
+ import semantica.llms.llms_provenance as module
+
+ wrapper = object.__new__(getattr(module, wrapper_name))
+
+ with pytest.raises(AttributeError):
+ wrapper.some_missing_attribute
+
+
+def test_copy_of_wrapper_does_not_recurse():
+ """copy.copy() probes attributes before _llm exists on the new object."""
+ import copy
+
+ wrapper = GroqLLMWithProvenance(provenance=False)
+ wrapper._llm = _StubLLM(response="ok")
+
+ clone = copy.copy(wrapper)
+
+ assert clone.generate("q") == "ok"
+
+
+class _FakeCompletions:
+ """Only the ``chat.completions.create`` surface the providers use."""
+
+ def __init__(self, response):
+ self._response = response
+
+ def create(self, **kwargs):
+ return self._response
+
+
+class _FakeChat:
+ def __init__(self, response):
+ self.completions = _FakeCompletions(response)
+
+
+class _FakeClient:
+ """Stand-in for an OpenAI/Groq client, backed by a canned response."""
+
+ def __init__(self, response):
+ self.chat = _FakeChat(response)
+
+
+def _chat_response(content):
+ # OpenAI and Groq responses report usage but no cost.
+ return SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content=content))],
+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5),
+ )
+
+
+class TestProvidersCarryUsageAndCost:
+ """The fix lives in the providers, so drive them rather than a stub."""
+
+ def test_openai_provider_returns_response_text(self):
+ provider = OpenAIProvider(api_key="test-key")
+ provider.client = _FakeClient(_chat_response("hello"))
+
+ result = provider.generate("question")
+
+ assert isinstance(result, ResponseText)
+ assert result == "hello"
+ assert result.usage is not None
+ assert result.usage.prompt_tokens == 10
+
+ def test_groq_provider_returns_response_text(self):
+ provider = GroqProvider(api_key="test-key")
+ provider.client = _FakeClient(_chat_response("hi"))
+
+ result = provider.generate("question")
+
+ assert isinstance(result, ResponseText)
+ assert result == "hi"
+ assert result.usage is not None
+
+ def test_wrapper_records_usage_from_real_provider(self):
+ provider = GroqProvider(api_key="test-key")
+ provider.client = _FakeClient(_chat_response("hello"))
+ wrapper, recorder = _wrapper_with(provider)
+
+ result = wrapper.generate("question")
+
+ assert result == "hello"
+ metadata = recorder.calls[0]["metadata"]
+ assert metadata["prompt_tokens"] == 10
+ assert metadata["completion_tokens"] == 5
+ assert metadata["total_tokens"] == 15
+
+ def test_litellm_returns_response_text_with_cost(self, monkeypatch):
+ import semantica.llms.litellm as litellm_module
+
+ response = SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content="answer"))],
+ usage=SimpleNamespace(prompt_tokens=3, completion_tokens=4),
+ _hidden_params={"response_cost": 0.007},
+ )
+ monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)
+ monkeypatch.setattr(litellm_module, "completion", lambda **kwargs: response)
+
+ llm = litellm_module.LiteLLM(model="openai/gpt-4o")
+ result = llm.generate("question")
+
+ assert isinstance(result, ResponseText)
+ assert result == "answer"
+ assert result.usage is not None
+ assert result.cost == 0.007
+ # The rest of _hidden_params (api_base, response headers) is not kept.
+ assert not hasattr(result, "_hidden_params")
+
+ def test_litellm_missing_response_cost_is_none(self, monkeypatch):
+ import semantica.llms.litellm as litellm_module
+
+ response = SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content="answer"))],
+ usage=None,
+ _hidden_params={"api_base": "https://example.invalid"},
+ )
+ monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)
+ monkeypatch.setattr(litellm_module, "completion", lambda **kwargs: response)
+
+ result = litellm_module.LiteLLM(model="openai/gpt-4o").generate("question")
+
+ assert result == "answer"
+ assert result.cost is None
+
+
+class TestNoneReplyStaysNone:
+ """A reply with no text (tool call, refusal, content filter) stays ``None``."""
+
+ @pytest.mark.parametrize("provider_cls", [OpenAIProvider, GroqProvider])
+ def test_none_reply_stays_none(self, provider_cls):
+ provider = provider_cls(api_key="test-key")
+ provider.client = _FakeClient(_chat_response(None))
+
+ assert provider.generate("question") is None
+
+ def test_litellm_none_reply_stays_none(self, monkeypatch):
+ import semantica.llms.litellm as litellm_module
+
+ response = SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content=None))],
+ usage=None,
+ _hidden_params={},
+ )
+ monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)
+ monkeypatch.setattr(litellm_module, "completion", lambda **kwargs: response)
+
+ llm = litellm_module.LiteLLM(model="openai/gpt-4o")
+
+ assert llm.generate("question") is None
+
+ def test_wrapper_records_none_reply_without_stringifying(self):
+ wrapper, recorder = _wrapper_with(_StubLLM(response=None))
+
+ assert wrapper.generate("question") is None
+ metadata = recorder.calls[0]["metadata"]
+ assert metadata["response_preview"] is None
+ assert metadata["prompt_tokens"] is None
+
+
+class _Answer(BaseModel):
+ text: str
+ confidence: float
+
+
+class TestStructuredAndTypedPreviews:
+ """Structured/typed previews must serialize the whole result."""
+
+ def test_typed_model_with_text_field_records_full_json(self):
+ answer = _Answer(text="short", confidence=0.9)
+ wrapper, recorder = _wrapper_with(_StubLLM(typed=answer))
+
+ assert wrapper.generate_typed("question", schema=_Answer) is answer
+ preview = recorder.calls[0]["metadata"]["response_preview"]
+ assert preview == answer.model_dump_json()
+
+ def test_structured_dict_records_json(self):
+ wrapper, recorder = _wrapper_with(_StubLLM(structured={"answer": 42}))
+
+ wrapper.generate_structured("question")
+
+ preview = recorder.calls[0]["metadata"]["response_preview"]
+ assert preview == '{"answer": 42}'
+
+ def test_long_structured_preview_is_truncated(self):
+ wrapper, recorder = _wrapper_with(
+ _StubLLM(structured={"items": list(range(500))})
+ )
+
+ wrapper.generate_structured("question")
+
+ assert len(recorder.calls[0]["metadata"]["response_preview"]) == 200
+
+ def test_none_result_records_none(self):
+ wrapper, recorder = _wrapper_with(_StubLLM(structured=None))
+
+ assert wrapper.generate_structured("question") is None
+ assert recorder.calls[0]["metadata"]["response_preview"] is None
+
+
+def test_litellm_wrapper_reads_cost_from_response_text(monkeypatch):
+ monkeypatch.setattr("semantica.llms.litellm.LITELLM_AVAILABLE", True)
+ wrapper = LiteLLMWithProvenance(provenance=False, model="openai/gpt-4o")
+ recorder = _RecordingProvenanceManager()
+ wrapper.provenance = True
+ wrapper._prov_manager = recorder
+ wrapper._llm = _StubLLM(
+ response=ResponseText(
+ "answer",
+ usage=SimpleNamespace(prompt_tokens=3, completion_tokens=4),
+ cost=0.007,
+ )
+ )
+
+ wrapper.generate("question")
+
+ assert recorder.calls[0]["metadata"]["total_cost"] == 0.007