Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(llms): resolve broken imports in llms provenance wrappers (#1912)
Kevin6d7364df8722
diff --git a/semantica/llms/litellm.py b/semantica/llms/litellm.pyindex 2db65528..7e63b56d 100644--- a/semantica/llms/litellm.py+++ b/semantica/llms/litellm.py@@ -13,6 +13,7 @@ from pydantic import BaseModelfrom ..utils.exceptions import ProcessingErrorfrom ..utils.logging import get_logger+from ..semantic_extract.providers import ResponseTextlogger = get_logger("llms.litellm")@@ -168,16 +169,32 @@ class LiteLLM:**options)+ # Metadata the provenance wrappers read back off the returned value.+ # Only usage and cost are kept: the rest of _hidden_params can hold+ # api_base and provider response headers.+ hidden_params = getattr(response, '_hidden_params', None) or {}+ usage = getattr(response, 'usage', None)+ if usage is None and isinstance(response, dict):+ usage = response.get('usage')+ meta = {+ "usage": usage,+ "cost": hidden_params.get('response_cost'),+ }++ def _wrap(content):+ # Tool-call, refusal and content-filter replies carry no text.+ return None if content is None else ResponseText(content, **meta)+# Extract text from responseif hasattr(response, 'choices') and len(response.choices) > 0:- return response.choices[0].message.content+ return _wrap(response.choices[0].message.content)elif isinstance(response, dict):if 'choices' in response and len(response['choices']) > 0:- return response['choices'][0]['message']['content']+ return _wrap(response['choices'][0]['message']['content'])elif 'content' in response:- return response['content']+ return _wrap(response['content'])elif isinstance(response, str):- return response+ return _wrap(response)raise ProcessingError(f"Unexpected response format from LiteLLM: {type(response)}")diff --git a/semantica/llms/llms_provenance.py b/semantica/llms/llms_provenance.pyindex f6277fff..287f92c1 100644--- a/semantica/llms/llms_provenance.py+++ b/semantica/llms/llms_provenance.py@@ -7,7 +7,8 @@ This module provides provenance tracking for all LLM operations:- HuggingFace LLM- LiteLLM-Tracks: model name, tokens (prompt/completion), cost, latency, prompts, responses+Tracks: model name, tokens (prompt/completion), latency, prompts, responses, and+cost where the provider reports it (LiteLLM; OpenAI and Groq do not)All classes wrap the original LLM providers and add optional provenance trackingwithout modifying existing functionality.@@ -17,15 +18,15 @@ Usage:GroqLLMWithProvenance,OpenAILLMWithProvenance)-+# Enable provenance trackingllm = GroqLLMWithProvenance(provenance=True)response = llm.generate("What is artificial intelligence?")-+# Provenance automatically tracks:# - Model used# - Token counts- # - API costs+ # - API costs (LiteLLM only; OpenAI and Groq do not report cost)# - Latency# - Prompt and response previews@@ -41,20 +42,43 @@ Author: Semantica ContributorsLicense: MIT"""-from typing import Optional, Dict, Any+from typing import Optional, Anyfrom datetime import datetime+import jsonimport timeimport uuid+from pydantic import BaseModel+++def _preview_result(result: Any) -> Optional[str]:+ """Render a structured/typed result as a 200-char preview.++ Serializes the whole value, so a Pydantic model with a ``text`` or+ ``content`` field is not reduced to that one field.+ """+ if result is None:+ return None+ if isinstance(result, BaseModel):+ preview = result.model_dump_json()+ elif isinstance(result, (dict, list)):+ try:+ preview = json.dumps(result, default=str)+ except (TypeError, ValueError):+ preview = str(result)+ else:+ preview = str(result)+ return preview[:200]+class LLMProvenanceMixin:"""Mixin to add provenance tracking to any LLM provider.-+This mixin provides common provenance infrastructure for trackingLLM API calls including tokens, costs, and performance metrics."""-+def __init__(self,provenance: bool = False,@@ -102,14 +126,19 @@ class LLMProvenanceMixin:**metadata: Additional metadata (tokens, cost, latency, etc.)"""if self.provenance and self._prov_manager:- # Extract response text+ # Extract response text; a reply with no text stays None rather+ # than being recorded as the string "None".response_text = response- if hasattr(response, 'text'):- response_text = response.text- elif hasattr(response, 'content'):- response_text = response.content- elif not isinstance(response, str):- response_text = str(response)+ if response is None:+ response_preview = None+ else:+ if hasattr(response, 'text'):+ response_text = response.text+ elif hasattr(response, 'content'):+ response_text = response.content+ elif not isinstance(response, str):+ response_text = str(response)+ response_preview = str(response_text)[:200]# Typed Activity timing (issue #825, Part B Tier 1): popped out so# it populates real fields, not the opaque metadata blob.@@ -129,24 +158,67 @@ class LLMProvenanceMixin:metadata={"model": getattr(self, 'model', 'unknown'),"prompt_preview": prompt[:200] if len(prompt) > 200 else prompt,- "response_preview": response_text[:200] if len(str(response_text)) > 200 else str(response_text),+ "response_preview": response_preview,**metadata})+ def _track_generation(+ self,+ kind: str,+ call_id_prefix: str,+ prompt: str,+ invoke,+ **metadata,+ ):+ """Run one generation call and record a provenance entry for it.++ Args:+ kind: Generation mode label (e.g. ``"structured"``, ``"typed"``),+ recorded as ``generation_mode``.+ call_id_prefix: Prefix for the generated call id.+ prompt: Input prompt.+ invoke: Zero-argument callable performing the wrapped LLM call.+ **metadata: Extra metadata recorded with the entry.++ Returns:+ Whatever ``invoke`` returns, unchanged.+ """+ start_time = time.time()+ activity_started_at_time = datetime.utcnow().isoformat()+ result = invoke()+ elapsed = time.time() - start_time+ activity_ended_at_time = datetime.utcnow().isoformat()++ if self.provenance:+ # Pass the serialized preview rather than the raw result, so+ # _track_llm_call does not pick a model's .text/.content field.+ self._track_llm_call(+ call_id=f"{call_id_prefix}_{uuid.uuid4().hex[:8]}",+ prompt=prompt,+ response=_preview_result(result),+ generation_mode=kind,+ latency_seconds=elapsed,+ activity_started_at_time=activity_started_at_time,+ activity_ended_at_time=activity_ended_at_time,+ **metadata,+ )++ return result+class GroqLLMWithProvenance(LLMProvenanceMixin):"""Groq LLM with provenance tracking.-+Wraps the original GroqLLM and tracks all API calls with complete metadata.-+Example:>>> llm = GroqLLMWithProvenance(provenance=True, model="llama-3.1-70b")>>> response = llm.generate("Explain quantum computing")- >>> # API call is tracked with model, tokens, cost, latency+ >>> # API call is tracked with model, tokens, latency (Groq reports no cost)"""-+def __init__(self,provenance: bool = False,@@ -161,22 +233,22 @@ class GroqLLMWithProvenance(LLMProvenanceMixin):provenance: Enable provenance tracking (default: False)**config: Configuration passed to original GroqLLM"""- from .groq_llm import GroqLLM+ from .groq import GroqLLMProvenanceMixin.__init__(self, provenance=provenance, agent_id=agent_id, is_automated=is_automated)- self._llm = GroqLLM(**config)+ self._llm = Groq(**config)self.model = getattr(self._llm, 'model', 'groq')-+def generate(self, prompt: str, **kwargs):"""Generate response with provenance tracking.-+Args:prompt: Input prompt**kwargs: Additional generation parameters-+Returns:LLM response (same format as original GroqLLM)"""@@ -205,7 +277,12 @@ class GroqLLMWithProvenance(LLMProvenanceMixin):response=response,prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,- total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None,+ total_tokens=(+ (prompt_tokens + completion_tokens)+ if prompt_tokens is not None+ and completion_tokens is not None+ else None+ ),total_cost=total_cost,latency_seconds=elapsed,activity_started_at_time=activity_started_at_time,@@ -214,21 +291,46 @@ class GroqLLMWithProvenance(LLMProvenanceMixin):max_tokens=kwargs.get('max_tokens'),top_p=kwargs.get('top_p'))-+return response-++ def generate_structured(self, prompt: str, **kwargs):+ """Generate structured JSON with provenance tracking."""+ return self._track_generation(+ "structured",+ "groq_structured",+ prompt,+ lambda: self._llm.generate_structured(prompt, **kwargs),+ )++ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):+ """Generate schema-validated output with provenance tracking."""+ return self._track_generation(+ "typed",+ "groq_typed",+ prompt,+ lambda: self._llm.generate_typed(+ prompt, schema, max_retries=max_retries, **kwargs+ ),+ max_retries=max_retries,+ )+def __getattr__(self, name):"""Delegate other methods to wrapped LLM."""+ if name == "_llm":+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it+ # up through getattr(self._llm, ...) would recurse forever.+ raise AttributeError(name)return getattr(self._llm, name)class OpenAILLMWithProvenance(LLMProvenanceMixin):"""OpenAI LLM with provenance tracking.-+Wraps the original OpenAILLM and tracks all API calls."""-+def __init__(self,provenance: bool = False,@@ -243,22 +345,22 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin):provenance: Enable provenance tracking (default: False)**config: Configuration passed to original OpenAILLM"""- from .openai_llm import OpenAILLM+ from .openai import OpenAILLMProvenanceMixin.__init__(self, provenance=provenance, agent_id=agent_id, is_automated=is_automated)- self._llm = OpenAILLM(**config)+ self._llm = OpenAI(**config)self.model = getattr(self._llm, 'model', 'openai')-+def generate(self, prompt: str, **kwargs):"""Generate response with provenance tracking.-+Args:prompt: Input prompt**kwargs: Additional generation parameters-+Returns:LLM response"""@@ -287,7 +389,12 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin):response=response,prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,- total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None,+ total_tokens=(+ (prompt_tokens + completion_tokens)+ if prompt_tokens is not None+ and completion_tokens is not None+ else None+ ),total_cost=total_cost,latency_seconds=elapsed,activity_started_at_time=activity_started_at_time,@@ -295,21 +402,46 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin):temperature=kwargs.get('temperature'),max_tokens=kwargs.get('max_tokens'))-+return response-++ def generate_structured(self, prompt: str, **kwargs):+ """Generate structured JSON with provenance tracking."""+ return self._track_generation(+ "structured",+ "openai_structured",+ prompt,+ lambda: self._llm.generate_structured(prompt, **kwargs),+ )++ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):+ """Generate schema-validated output with provenance tracking."""+ return self._track_generation(+ "typed",+ "openai_typed",+ prompt,+ lambda: self._llm.generate_typed(+ prompt, schema, max_retries=max_retries, **kwargs+ ),+ max_retries=max_retries,+ )+def __getattr__(self, name):"""Delegate other methods to wrapped LLM."""+ if name == "_llm":+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it+ # up through getattr(self._llm, ...) would recurse forever.+ raise AttributeError(name)return getattr(self._llm, name)class HuggingFaceLLMWithProvenance(LLMProvenanceMixin):"""HuggingFace LLM with provenance tracking.-+Wraps the original HuggingFaceLLM and tracks all generations."""-+def __init__(self,provenance: bool = False,@@ -324,22 +456,22 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin):provenance: Enable provenance tracking (default: False)**config: Configuration passed to original HuggingFaceLLM"""- from .huggingface_llm import HuggingFaceLLM+ from .huggingface import HuggingFaceLLMLLMProvenanceMixin.__init__(self, provenance=provenance, agent_id=agent_id, is_automated=is_automated)self._llm = HuggingFaceLLM(**config)self.model = getattr(self._llm, 'model', 'huggingface')-+def generate(self, prompt: str, **kwargs):"""Generate response with provenance tracking.-+Args:prompt: Input prompt**kwargs: Additional generation parameters-+Returns:LLM response"""@@ -360,21 +492,46 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin):max_length=kwargs.get('max_length'),temperature=kwargs.get('temperature'))-+return response-++ def generate_structured(self, prompt: str, **kwargs):+ """Generate structured JSON with provenance tracking."""+ return self._track_generation(+ "structured",+ "hf_structured",+ prompt,+ lambda: self._llm.generate_structured(prompt, **kwargs),+ )++ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):+ """Generate schema-validated output with provenance tracking."""+ return self._track_generation(+ "typed",+ "hf_typed",+ prompt,+ lambda: self._llm.generate_typed(+ prompt, schema, max_retries=max_retries, **kwargs+ ),+ max_retries=max_retries,+ )+def __getattr__(self, name):"""Delegate other methods to wrapped LLM."""+ if name == "_llm":+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it+ # up through getattr(self._llm, ...) would recurse forever.+ raise AttributeError(name)return getattr(self._llm, name)class LiteLLMWithProvenance(LLMProvenanceMixin):"""LiteLLM with provenance tracking.-+Wraps the original LiteLLM and tracks all API calls across providers."""-+def __init__(self,provenance: bool = False,@@ -389,22 +546,22 @@ class LiteLLMWithProvenance(LLMProvenanceMixin):provenance: Enable provenance tracking (default: False)**config: Configuration passed to original LiteLLM"""- from .lite_llm import LiteLLM+ from .litellm import LiteLLMLLMProvenanceMixin.__init__(self, provenance=provenance, agent_id=agent_id, is_automated=is_automated)self._llm = LiteLLM(**config)self.model = getattr(self._llm, 'model', 'litellm')-+def generate(self, prompt: str, **kwargs):"""Generate response with provenance tracking.-+Args:prompt: Input prompt**kwargs: Additional generation parameters-+Returns:LLM response"""@@ -424,8 +581,8 @@ class LiteLLMWithProvenance(LLMProvenanceMixin):prompt_tokens = getattr(response.usage, 'prompt_tokens', None)completion_tokens = getattr(response.usage, 'completion_tokens', None)- if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params:- total_cost = response._hidden_params['response_cost']+ if hasattr(response, 'cost'):+ total_cost = response.costself._track_llm_call(call_id=f"lite_call_{uuid.uuid4().hex[:8]}",@@ -439,11 +596,36 @@ class LiteLLMWithProvenance(LLMProvenanceMixin):activity_ended_at_time=activity_ended_at_time,provider=kwargs.get('provider'))-+return response-++ def generate_structured(self, prompt: str, **kwargs):+ """Generate structured JSON with provenance tracking."""+ return self._track_generation(+ "structured",+ "lite_structured",+ prompt,+ lambda: self._llm.generate_structured(prompt, **kwargs),+ )++ def generate_typed(self, prompt: str, schema, max_retries: int = 3, **kwargs):+ """Generate schema-validated output with provenance tracking."""+ return self._track_generation(+ "typed",+ "lite_typed",+ prompt,+ lambda: self._llm.generate_typed(+ prompt, schema, max_retries=max_retries, **kwargs+ ),+ max_retries=max_retries,+ )+def __getattr__(self, name):"""Delegate other methods to wrapped LLM."""+ if name == "_llm":+ # _llm is unset (failed __init__, copy.copy, unpickling); looking it+ # up through getattr(self._llm, ...) would recurse forever.+ raise AttributeError(name)return getattr(self._llm, name)diff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.pyindex 5c870a3e..46c7e8d0 100644--- a/semantica/semantic_extract/providers.py+++ b/semantica/semantic_extract/providers.py@@ -145,6 +145,31 @@ def _extract_first_json_value(text: str):return None+class ResponseText(str):+ """A ``str`` that also carries the provider response's usage and cost.++ ``generate()`` must keep returning the generated text, but the provenance+ wrappers in ``semantica.llms`` also need the token counts (and, where the+ provider reports it, the cost) that only the raw provider response holds.+ OpenAI and Groq responses carry no cost, so only ``usage`` is set for them;+ LiteLLM supplies ``cost`` from its ``response_cost``. Attaching that+ metadata to the returned value keeps it scoped to the request it came from,+ rather than stashing it on the provider instance where concurrent calls+ would overwrite each other.+ """++ def __new__(+ cls,+ text: str,+ usage: Any = None,+ cost: Optional[float] = None,+ ) -> "ResponseText":+ instance = super().__new__(cls, text if text is not None else "")+ instance.usage = usage+ instance.cost = cost+ return instance++class BaseProvider:"""Base class for providers - makes it easy to add custom providers."""@@ -802,7 +827,11 @@ class OpenAIProvider(BaseProvider):)response = self.client.chat.completions.create(**create_kwargs)- return response.choices[0].message.content+ content = response.choices[0].message.content+ if content is None:+ # Tool-call, refusal and content-filter replies carry no text.+ return None+ return ResponseText(content, usage=getattr(response, "usage", None))def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:"""Generate structured JSON output."""@@ -1083,7 +1112,11 @@ class GroqProvider(BaseProvider):)response = self.client.chat.completions.create(**create_kwargs)- return response.choices[0].message.content+ content = response.choices[0].message.content+ if content is None:+ # Tool-call, refusal and content-filter replies carry no text.+ return None+ return ResponseText(content, usage=getattr(response, "usage", None))def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:"""Generate structured output."""diff --git a/tests/provenance/test_llms_provenance.py b/tests/provenance/test_llms_provenance.pyindex cb7a6daa..cb639094 100644--- a/tests/provenance/test_llms_provenance.py+++ b/tests/provenance/test_llms_provenance.py@@ -7,9 +7,20 @@ Tests that provenance tracking works correctly for all LLM providers.import pytest+class _StubProvider:+ """Stand-in for the provider a wrapper's target class constructs.++ Keeps these tests offline: constructing ``HuggingFaceLLM`` would otherwise+ load (and download) a real model.+ """++ def __init__(self, *args, **kwargs):+ pass++class TestGroqLLMProvenance:"""Test Groq LLM with provenance."""-+def test_without_provenance(self):"""Test Groq LLM works without provenance."""try:@@ -19,7 +30,7 @@ class TestGroqLLMProvenance:assert llm.provenance is Falseexcept ImportError:pytest.skip("GroqLLM not available")-+def test_with_provenance_enabled(self):"""Test Groq LLM tracks provenance."""try:@@ -33,7 +44,7 @@ class TestGroqLLMProvenance:class TestOpenAILLMProvenance:"""Test OpenAI LLM with provenance."""-+def test_without_provenance(self):"""Test OpenAI LLM works without provenance."""try:@@ -42,7 +53,7 @@ class TestOpenAILLMProvenance:assert llm is not Noneexcept ImportError:pytest.skip("OpenAILLM not available")-+def test_with_provenance_enabled(self):"""Test OpenAI LLM tracks provenance."""try:@@ -55,9 +66,14 @@ class TestOpenAILLMProvenance:class TestHuggingFaceLLMProvenance:"""Test HuggingFace LLM with provenance."""-- def test_without_provenance(self):++ def test_without_provenance(self, monkeypatch):"""Test HuggingFace LLM works without provenance."""+ # Stub the provider so the wrapper can be exercised without loading a+ # model; without this the test downloads ``gpt2`` on every run.+ monkeypatch.setattr(+ "semantica.llms.huggingface.HuggingFaceLLMProvider", _StubProvider+ )try:from semantica.llms.llms_provenance import HuggingFaceLLMWithProvenancellm = HuggingFaceLLMWithProvenance(provenance=False)@@ -68,7 +84,7 @@ class TestHuggingFaceLLMProvenance:class TestLLMProvenanceEdgeCases:"""Test edge cases for LLM provenance."""-+def test_empty_prompt(self):"""Test LLM with empty prompt."""try:@@ -78,29 +94,27 @@ class TestLLMProvenanceEdgeCases:assert llm is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_very_long_prompt(self):"""Test LLM with very long prompt."""try:from semantica.llms.llms_provenance import GroqLLMWithProvenancellm = GroqLLMWithProvenance(provenance=True)- long_prompt = "Explain " + "AI " * 1000# Should handle long promptsassert llm is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_special_characters_in_prompt(self):"""Test LLM with special characters in prompt."""try:from semantica.llms.llms_provenance import GroqLLMWithProvenancellm = GroqLLMWithProvenance(provenance=True)- special_prompt = "Explain @#$%^&*() and 中文 émojis 🎉"# Should handle special charactersassert llm is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_multiple_llm_calls(self):"""Test multiple LLM calls with provenance."""try:@@ -110,7 +124,7 @@ class TestLLMProvenanceEdgeCases:assert llm._prov_manager is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_concurrent_llm_calls(self):"""Test concurrent LLM calls."""try:@@ -120,7 +134,7 @@ class TestLLMProvenanceEdgeCases:assert llm is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_llm_with_custom_parameters(self):"""Test LLM with custom generation parameters."""try:@@ -130,7 +144,7 @@ class TestLLMProvenanceEdgeCases:assert llm._prov_manager is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_llm_response_without_usage(self):"""Test LLM response without usage metadata."""try:@@ -140,7 +154,7 @@ class TestLLMProvenanceEdgeCases:assert llm is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_llm_response_without_cost(self):"""Test LLM response without cost information."""try:@@ -150,7 +164,7 @@ class TestLLMProvenanceEdgeCases:assert llm is not Noneexcept ImportError:pytest.skip("GroqLLM not available")-+def test_different_llm_providers(self):"""Test different LLM providers with same provenance pattern."""try:@@ -165,7 +179,7 @@ class TestLLMProvenanceEdgeCases:assert openai is not Noneexcept ImportError:pytest.skip("LLM providers not available")-+def test_llm_latency_tracking(self):"""Test that LLM latency is tracked correctly."""try:diff --git a/tests/provenance/test_llms_provenance_imports.py b/tests/provenance/test_llms_provenance_imports.pynew file mode 100644index 00000000..b5323c9c--- /dev/null+++ b/tests/provenance/test_llms_provenance_imports.py@@ -0,0 +1,74 @@+"""Regression tests for the LLM provenance wrappers' import resolution.++Each ``*WithProvenance`` wrapper imports the LLM class it wraps lazily, inside+``__init__``, from a module path that had drifted from the real one+(``.groq_llm`` / ``.openai_llm`` / ``.huggingface_llm`` / ``.lite_llm``). A+wrong path is invisible until the wrapper is instantiated, and the surrounding+suites guard those instantiations with ``pytest.skip``, so the breakage never+reached CI.++These tests do not skip: if a wrapper cannot resolve the class it wraps, the+test fails. The providers behind those classes are stubbed so the suite stays+offline and does not require the optional ``groq`` / ``litellm`` packages.+"""+++class _StubProvider:+ """Stand-in for the ``*Provider`` a wrapper's target class constructs."""++ def __init__(self, *args, **kwargs):+ pass+++class TestGroqWrapper:+ """GroqLLMWithProvenance must resolve the Groq class it wraps."""++ def test_instantiates_and_wraps_groq(self, monkeypatch):+ monkeypatch.setattr("semantica.llms.groq.GroqProvider", _StubProvider)+ from semantica.llms.groq import Groq+ from semantica.llms.llms_provenance import GroqLLMWithProvenance++ wrapper = GroqLLMWithProvenance(provenance=False)+ assert isinstance(wrapper._llm, Groq)+++class TestOpenAIWrapper:+ """OpenAILLMWithProvenance must resolve the OpenAI class it wraps."""++ def test_instantiates_and_wraps_openai(self, monkeypatch):+ monkeypatch.setattr("semantica.llms.openai.OpenAIProvider", _StubProvider)+ from semantica.llms.openai import OpenAI+ from semantica.llms.llms_provenance import OpenAILLMWithProvenance++ wrapper = OpenAILLMWithProvenance(provenance=False)+ assert isinstance(wrapper._llm, OpenAI)+++class TestHuggingFaceWrapper:+ """HuggingFaceLLMWithProvenance must resolve the class it wraps."""++ def test_instantiates_and_wraps_huggingface(self, monkeypatch):+ # The real provider loads a transformer model during construction.+ monkeypatch.setattr(+ "semantica.llms.huggingface.HuggingFaceLLMProvider", _StubProvider+ )+ from semantica.llms.huggingface import HuggingFaceLLM+ from semantica.llms.llms_provenance import HuggingFaceLLMWithProvenance++ wrapper = HuggingFaceLLMWithProvenance(provenance=False)+ assert isinstance(wrapper._llm, HuggingFaceLLM)+++class TestLiteLLMWrapper:+ """LiteLLMWithProvenance must resolve the LiteLLM class it wraps."""++ def test_instantiates_and_wraps_litellm(self, monkeypatch):+ # LiteLLM refuses to construct when its optional package is absent; the+ # wrapper only needs the class object, so let it through.+ monkeypatch.setattr("semantica.llms.litellm.LITELLM_AVAILABLE", True)+ from semantica.llms.litellm import LiteLLM+ from semantica.llms.llms_provenance import LiteLLMWithProvenance++ # `model` is a required argument of LiteLLM; omitting it raises.+ wrapper = LiteLLMWithProvenance(provenance=False, model="openai/gpt-4o-mini")+ assert isinstance(wrapper._llm, LiteLLM)diff --git a/tests/provenance/test_llms_provenance_tracking.py b/tests/provenance/test_llms_provenance_tracking.pynew file mode 100644index 00000000..620a22be--- /dev/null+++ b/tests/provenance/test_llms_provenance_tracking.py@@ -0,0 +1,430 @@+"""Regression tests for what the LLM provenance wrappers actually record.++Two gaps in the wrappers let calls go untracked or tracked without their+usage/cost:++* ``generate_structured`` / ``generate_typed`` fell through ``__getattr__`` to+ the wrapped LLM, so those calls produced no provenance entry at all.+* ``generate`` read token counts and cost off the returned value, but the+ providers returned a plain ``str``, so ``prompt_tokens`` / ``completion_tokens``+ / ``total_cost`` were always ``None``.++These tests drive the wrappers with a stub LLM and a recording provenance+manager so they stay offline and assert on the recorded metadata directly.+A second group drives the providers themselves with a canned client, so a+provider that stops carrying usage/cost turns the suite red.+"""++from types import SimpleNamespace++import pytest+from pydantic import BaseModel++from semantica.llms.llms_provenance import (+ GroqLLMWithProvenance,+ LiteLLMWithProvenance,+)+from semantica.semantic_extract.providers import (+ GroqProvider,+ OpenAIProvider,+ ResponseText,+)+++class _RecordingProvenanceManager:+ """Capture the kwargs of every ``track_entity`` call."""++ def __init__(self):+ self.calls = []++ def track_entity(self, **kwargs):+ self.calls.append(kwargs)+++class _StubLLM:+ """Minimal stand-in for a wrapped provider class."""++ model = "stub-model"++ def __init__(self, response=None, structured=None, typed=None):+ self._response = response+ self._structured = structured+ self._typed = typed++ def generate(self, prompt, **kwargs):+ return self._response++ def generate_structured(self, prompt, **kwargs):+ return self._structured++ def generate_typed(self, prompt, schema, max_retries=3, **kwargs):+ return self._typed+++def _wrapper_with(llm):+ """Build a Groq wrapper wired to *llm* and a recording provenance manager."""+ wrapper = GroqLLMWithProvenance(provenance=False)+ recorder = _RecordingProvenanceManager()+ wrapper.provenance = True+ wrapper._prov_manager = recorder+ wrapper._llm = llm+ return wrapper, recorder+++class TestStructuredAndTypedAreTracked:+ """Structured/typed calls must produce a provenance entry."""++ def test_generate_structured_records_entry(self):+ wrapper, recorder = _wrapper_with(+ _StubLLM(structured={"answer": 42})+ )++ result = wrapper.generate_structured("question")++ assert result == {"answer": 42}+ assert len(recorder.calls) == 1+ assert recorder.calls[0]["metadata"]["generation_mode"] == "structured"++ def test_generate_typed_records_entry(self):+ wrapper, recorder = _wrapper_with(_StubLLM(typed="validated"))++ result = wrapper.generate_typed("question", schema=object)++ assert result == "validated"+ assert len(recorder.calls) == 1+ metadata = recorder.calls[0]["metadata"]+ assert metadata["generation_mode"] == "typed"+ assert metadata["max_retries"] == 3++ def test_calls_are_tracked_independently(self):+ wrapper, recorder = _wrapper_with(+ _StubLLM(structured={"a": 1}, typed="b")+ )++ wrapper.generate_structured("first")+ wrapper.generate_typed("second", schema=object)++ assert len(recorder.calls) == 2+ modes = [c["metadata"]["generation_mode"] for c in recorder.calls]+ assert modes == ["structured", "typed"]+ assert recorder.calls[0]["entity_id"] != recorder.calls[1]["entity_id"]++ def test_untracked_when_provenance_disabled(self):+ wrapper = GroqLLMWithProvenance(provenance=False)+ wrapper._llm = _StubLLM(structured={"a": 1})++ # No provenance manager is set, so the call must still succeed.+ assert wrapper.generate_structured("q") == {"a": 1}+++class TestUsageAndCostPropagate:+ """Token counts and cost must reach the recorded metadata."""++ def test_generate_records_tokens_and_cost(self):+ response = ResponseText(+ "the answer",+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5),+ cost=0.002,+ )+ wrapper, recorder = _wrapper_with(_StubLLM(response=response))++ result = wrapper.generate("question")++ assert result == "the answer"+ assert len(recorder.calls) == 1+ metadata = recorder.calls[0]["metadata"]+ assert metadata["prompt_tokens"] == 10+ assert metadata["completion_tokens"] == 5+ assert metadata["total_tokens"] == 15+ assert metadata["total_cost"] == 0.002++ def test_zero_completion_tokens_still_sums(self):+ response = ResponseText(+ "",+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=0),+ )+ wrapper, recorder = _wrapper_with(_StubLLM(response=response))++ wrapper.generate("question")++ metadata = recorder.calls[0]["metadata"]+ assert metadata["completion_tokens"] == 0+ assert metadata["total_tokens"] == 10++ def test_generate_without_usage_metadata_stays_none(self):+ wrapper, recorder = _wrapper_with(_StubLLM(response=ResponseText("plain")))++ wrapper.generate("question")++ metadata = recorder.calls[0]["metadata"]+ assert metadata["prompt_tokens"] is None+ assert metadata["completion_tokens"] is None+ assert metadata["total_cost"] is None+++class TestResponseText:+ """``ResponseText`` must behave like ``str`` while carrying metadata."""++ def test_is_a_string(self):+ text = ResponseText("hello")+ assert isinstance(text, str)+ assert text == "hello"+ assert text.upper() == "HELLO"++ def test_defaults(self):+ text = ResponseText("hello")+ assert text.usage is None+ assert text.cost is None++ def test_carries_usage_and_cost(self):+ usage = SimpleNamespace(prompt_tokens=1, completion_tokens=2)+ text = ResponseText("hi", usage=usage, cost=0.5)+ assert text.usage is usage+ assert text.cost == 0.5+++ "wrapper_name",+ [+ "GroqLLMWithProvenance",+ "OpenAILLMWithProvenance",+ "HuggingFaceLLMWithProvenance",+ "LiteLLMWithProvenance",+ ],+)+def test_all_wrappers_expose_structured_and_typed(wrapper_name):+ """Every wrapper must override both structured entry points."""+ import semantica.llms.llms_provenance as module++ wrapper_cls = getattr(module, wrapper_name)+ assert "generate_structured" in wrapper_cls.__dict__+ assert "generate_typed" in wrapper_cls.__dict__+++ "wrapper_name",+ [+ "GroqLLMWithProvenance",+ "OpenAILLMWithProvenance",+ "HuggingFaceLLMWithProvenance",+ "LiteLLMWithProvenance",+ ],+)+def test_unset_llm_raises_attribute_error_not_recursion(wrapper_name):+ """A wrapper whose ``_llm`` was never set must not recurse in __getattr__."""+ import semantica.llms.llms_provenance as module++ wrapper = object.__new__(getattr(module, wrapper_name))++ with pytest.raises(AttributeError):+ wrapper.some_missing_attribute+++def test_copy_of_wrapper_does_not_recurse():+ """copy.copy() probes attributes before _llm exists on the new object."""+ import copy++ wrapper = GroqLLMWithProvenance(provenance=False)+ wrapper._llm = _StubLLM(response="ok")++ clone = copy.copy(wrapper)++ assert clone.generate("q") == "ok"+++class _FakeCompletions:+ """Only the ``chat.completions.create`` surface the providers use."""++ def __init__(self, response):+ self._response = response++ def create(self, **kwargs):+ return self._response+++class _FakeChat:+ def __init__(self, response):+ self.completions = _FakeCompletions(response)+++class _FakeClient:+ """Stand-in for an OpenAI/Groq client, backed by a canned response."""++ def __init__(self, response):+ self.chat = _FakeChat(response)+++def _chat_response(content):+ # OpenAI and Groq responses report usage but no cost.+ return SimpleNamespace(+ choices=[SimpleNamespace(message=SimpleNamespace(content=content))],+ usage=SimpleNamespace(prompt_tokens=10, completion_tokens=5),+ )+++class TestProvidersCarryUsageAndCost:+ """The fix lives in the providers, so drive them rather than a stub."""++ def test_openai_provider_returns_response_text(self):+ provider = OpenAIProvider(api_key="test-key")+ provider.client = _FakeClient(_chat_response("hello"))++ result = provider.generate("question")++ assert isinstance(result, ResponseText)+ assert result == "hello"+ assert result.usage is not None+ assert result.usage.prompt_tokens == 10++ def test_groq_provider_returns_response_text(self):+ provider = GroqProvider(api_key="test-key")+ provider.client = _FakeClient(_chat_response("hi"))++ result = provider.generate("question")++ assert isinstance(result, ResponseText)+ assert result == "hi"+ assert result.usage is not None++ def test_wrapper_records_usage_from_real_provider(self):+ provider = GroqProvider(api_key="test-key")+ provider.client = _FakeClient(_chat_response("hello"))+ wrapper, recorder = _wrapper_with(provider)++ result = wrapper.generate("question")++ assert result == "hello"+ metadata = recorder.calls[0]["metadata"]+ assert metadata["prompt_tokens"] == 10+ assert metadata["completion_tokens"] == 5+ assert metadata["total_tokens"] == 15++ def test_litellm_returns_response_text_with_cost(self, monkeypatch):+ import semantica.llms.litellm as litellm_module++ response = SimpleNamespace(+ choices=[SimpleNamespace(message=SimpleNamespace(content="answer"))],+ usage=SimpleNamespace(prompt_tokens=3, completion_tokens=4),+ _hidden_params={"response_cost": 0.007},+ )+ monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)+ monkeypatch.setattr(litellm_module, "completion", lambda **kwargs: response)++ llm = litellm_module.LiteLLM(model="openai/gpt-4o")+ result = llm.generate("question")++ assert isinstance(result, ResponseText)+ assert result == "answer"+ assert result.usage is not None+ assert result.cost == 0.007+ # The rest of _hidden_params (api_base, response headers) is not kept.+ assert not hasattr(result, "_hidden_params")++ def test_litellm_missing_response_cost_is_none(self, monkeypatch):+ import semantica.llms.litellm as litellm_module++ response = SimpleNamespace(+ choices=[SimpleNamespace(message=SimpleNamespace(content="answer"))],+ usage=None,+ _hidden_params={"api_base": "https://example.invalid"},+ )+ monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)+ monkeypatch.setattr(litellm_module, "completion", lambda **kwargs: response)++ result = litellm_module.LiteLLM(model="openai/gpt-4o").generate("question")++ assert result == "answer"+ assert result.cost is None+++class TestNoneReplyStaysNone:+ """A reply with no text (tool call, refusal, content filter) stays ``None``."""++ @pytest.mark.parametrize("provider_cls", [OpenAIProvider, GroqProvider])+ def test_none_reply_stays_none(self, provider_cls):+ provider = provider_cls(api_key="test-key")+ provider.client = _FakeClient(_chat_response(None))++ assert provider.generate("question") is None++ def test_litellm_none_reply_stays_none(self, monkeypatch):+ import semantica.llms.litellm as litellm_module++ response = SimpleNamespace(+ choices=[SimpleNamespace(message=SimpleNamespace(content=None))],+ usage=None,+ _hidden_params={},+ )+ monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)+ monkeypatch.setattr(litellm_module, "completion", lambda **kwargs: response)++ llm = litellm_module.LiteLLM(model="openai/gpt-4o")++ assert llm.generate("question") is None++ def test_wrapper_records_none_reply_without_stringifying(self):+ wrapper, recorder = _wrapper_with(_StubLLM(response=None))++ assert wrapper.generate("question") is None+ metadata = recorder.calls[0]["metadata"]+ assert metadata["response_preview"] is None+ assert metadata["prompt_tokens"] is None+++class _Answer(BaseModel):+ text: str+ confidence: float+++class TestStructuredAndTypedPreviews:+ """Structured/typed previews must serialize the whole result."""++ def test_typed_model_with_text_field_records_full_json(self):+ answer = _Answer(text="short", confidence=0.9)+ wrapper, recorder = _wrapper_with(_StubLLM(typed=answer))++ assert wrapper.generate_typed("question", schema=_Answer) is answer+ preview = recorder.calls[0]["metadata"]["response_preview"]+ assert preview == answer.model_dump_json()++ def test_structured_dict_records_json(self):+ wrapper, recorder = _wrapper_with(_StubLLM(structured={"answer": 42}))++ wrapper.generate_structured("question")++ preview = recorder.calls[0]["metadata"]["response_preview"]+ assert preview == '{"answer": 42}'++ def test_long_structured_preview_is_truncated(self):+ wrapper, recorder = _wrapper_with(+ _StubLLM(structured={"items": list(range(500))})+ )++ wrapper.generate_structured("question")++ assert len(recorder.calls[0]["metadata"]["response_preview"]) == 200++ def test_none_result_records_none(self):+ wrapper, recorder = _wrapper_with(_StubLLM(structured=None))++ assert wrapper.generate_structured("question") is None+ assert recorder.calls[0]["metadata"]["response_preview"] is None+++def test_litellm_wrapper_reads_cost_from_response_text(monkeypatch):+ monkeypatch.setattr("semantica.llms.litellm.LITELLM_AVAILABLE", True)+ wrapper = LiteLLMWithProvenance(provenance=False, model="openai/gpt-4o")+ recorder = _RecordingProvenanceManager()+ wrapper.provenance = True+ wrapper._prov_manager = recorder+ wrapper._llm = _StubLLM(+ response=ResponseText(+ "answer",+ usage=SimpleNamespace(prompt_tokens=3, completion_tokens=4),+ cost=0.007,+ )+ )++ wrapper.generate("question")++ assert recorder.calls[0]["metadata"]["total_cost"] == 0.007