Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix: prevent stale provider reuse when API credentials change (#1665)
Sameer Kadam4e930551643b
diff --git a/semantica/semantic_extract/providers.py b/semantica/semantic_extract/providers.pyindex 1bf5a388..5c870a3e 100644--- a/semantica/semantic_extract/providers.py+++ b/semantica/semantic_extract/providers.py@@ -70,6 +70,7 @@ License: MIT"""import json+import threadingimport timefrom typing import Any, Dict, List, Optional, Union, Type@@ -1843,19 +1844,223 @@ class HuggingFaceModelLoader:class ProviderPool:- """Pool for reusing provider instances."""+ """Pool for reusing provider instances.++ Thread-safety+ -------------+ ``get()`` uses per-key ``threading.Event`` objects to serialise+ concurrent first-time construction of the *same* key without holding+ any lock during the (potentially slow) provider construction itself.+ Unrelated keys are therefore built concurrently.++ A short-lived ``_meta_lock`` guards only the in-memory bookkeeping+ dicts (``_providers``, ``_in_progress``); it is never held while a+ provider constructor runs.++ Same-thread re-entrancy (a provider constructor that calls+ ``create_provider()`` for the *same key* it is currently building) is+ detected via a ``threading.local`` set and handled by constructing a+ fresh instance directly rather than waiting on the in-progress event+ (which would deadlock). Nested calls for *different* keys proceed+ normally through the pool.++ Credential isolation+ --------------------+ Every *built-in* provider that is not shadowed by a registered custom+ provider resolves its API key through the fallback chain+ ``explicit kwarg → config.get_api_key() → env var`` inside its+ ``__init__``. ``get()`` mirrors that same resolution *before*+ computing the cache key so that the effective credential is always+ part of the key. Two calls that resolve to different credentials+ therefore receive different cached instances.++ Custom providers registered via ``provider_registry`` (even those+ registered under a built-in name such as ``"openai"``) are never+ given an automatically injected ``api_key``; only kwargs explicitly+ supplied by the caller are forwarded to them.++ Providers that do not use an API key (``ollama``, ``huggingface_llm``)+ are unaffected: ``_resolve_api_key`` returns ``None`` for them and no+ kwarg is injected.+ """++ # Names of *built-in* providers that accept an ``api_key`` constructor+ # argument and whose credential must be resolved before key computation.+ # Credential injection is skipped when a custom provider is registered+ # under one of these names.+ _API_KEY_PROVIDERS = frozenset(+ {"openai", "gemini", "groq", "anthropic", "deepseek", "novita"}+ )def __init__(self):self._providers: Dict[str, BaseProvider] = {}+ # Per-key in-progress events: present while a builder thread is+ # constructing the provider for that key.+ self._in_progress: Dict[str, threading.Event] = {}+ # Guards _providers and _in_progress; never held during construction.+ self._meta_lock = threading.Lock()+ # Per-thread set of keys currently being built by this thread.+ # Used to detect same-key re-entrancy.+ self._local = threading.local()self.logger = get_logger("provider_pool")- def get(self, name: str, **kwargs) -> BaseProvider:- """Get or create a provider instance."""- # Create a cache key from name and kwargs- # Filter out non-hashable items or volatile args if any- # For now, we assume kwargs are configuration options that should match+ # ------------------------------------------------------------------+ # Credential resolution+ # ------------------------------------------------------------------++ def _resolve_api_key(self, name: str, kwargs: dict) -> Optional[str]:+ """Return the effective API key for *name* using the same fallback+ chain that the built-in provider's ``__init__`` would use:++ 1. Explicit ``api_key`` kwarg (already supplied by the caller).+ 2. ``config.get_api_key(name)`` — checks the in-memory Config+ singleton first, then falls back to the ``{NAME}_API_KEY``+ environment variable.++ Returns ``None`` when:+ * the provider name is not in ``_API_KEY_PROVIDERS`` (e.g. Ollama,+ HuggingFace), or+ * a custom provider is registered under *name* — custom providers+ must receive only the kwargs the caller explicitly supplied, never+ an automatically injected ``api_key``, or+ * no key can be found anywhere.++ The returned value is intentionally *not* logged so that+ credentials do not appear in log output.+ """+ if name.lower() not in self._API_KEY_PROVIDERS:+ return None++ # If a custom provider is registered under this name, do not inject+ # an api_key — the custom class may not accept that parameter.+ if provider_registry.get(name):+ return None++ # Explicit kwarg takes precedence.+ explicit = kwargs.get("api_key")+ if explicit:+ return explicit- # Helper to make dict hashable+ # Mirror the built-in provider __init__ fallback:+ # config singleton → env var.+ return config.get_api_key(name.lower()) or None++ # ------------------------------------------------------------------+ # Public interface+ # ------------------------------------------------------------------++ def get(self, name: str, **kwargs) -> BaseProvider:+ """Return a cached provider for the given *name* and configuration.++ For built-in API-key providers (when not shadowed by a custom+ registration), the effective API key is resolved and injected into+ *kwargs* before the cache key is computed so that two callers+ whose credentials differ receive different cached instances even+ when neither passed an explicit ``api_key``.++ Concurrent first-time requests for the same key are serialised via+ a per-key ``threading.Event``; the provider is constructed exactly+ once. If construction raises, any threads waiting on that key retry+ and become the new builder (or find a result from a concurrent+ successful attempt).++ If the calling thread is already building this exact key (i.e. a+ provider constructor calls back into the pool for the same key), a+ fresh instance is returned directly to avoid deadlock. Nested calls+ for *different* keys go through the normal pool path.+ """+ # Resolve the effective credential and normalise kwargs so the key+ # captures it even when the caller relied on env-var resolution.+ resolved_key = self._resolve_api_key(name, kwargs)+ if resolved_key and not kwargs.get("api_key"):+ kwargs = {**kwargs, "api_key": resolved_key}++ key = self._make_key(name, **kwargs)++ # ---------- fast path (no lock) ----------+ provider = self._providers.get(key)+ if provider is not None:+ return provider++ # ---------- same-thread re-entrancy guard ----------+ # If *this thread* is already building this key (nested call from+ # inside a constructor), bypass the pool entirely to avoid waiting+ # on an event that this thread will never signal.+ building = getattr(self._local, "building", None)+ if building is None:+ building = set()+ self._local.building = building++ if key in building:+ self.logger.debug(+ "Re-entrant get() for %s — constructing fresh instance to avoid deadlock",+ name,+ )+ return self._create_provider(name, **kwargs)++ # ---------- slow path: serialise per-key ----------+ while True:+ with self._meta_lock:+ # Re-check under the meta-lock.+ provider = self._providers.get(key)+ if provider is not None:+ return provider++ existing_event = self._in_progress.get(key)+ if existing_event is not None:+ # Another thread is building; we will wait outside the lock.+ wait_event = existing_event+ else:+ # This thread becomes the builder.+ wait_event = None+ build_event = threading.Event()+ self._in_progress[key] = build_event++ if wait_event is not None:+ # Wait for the builder thread; then re-enter the loop so we+ # pick up the result (or retry if it failed).+ wait_event.wait()+ # After the event fires, check if the result was stored.+ provider = self._providers.get(key)+ if provider is not None:+ return provider+ # Builder failed — loop again: this thread will become the+ # next builder if no other thread already is.+ continue++ # This thread is the builder — construct outside every lock.+ building.add(key)+ try:+ self.logger.debug("Creating new provider instance for %s", name)+ provider = self._create_provider(name, **kwargs)+ with self._meta_lock:+ self._providers[key] = provider+ self._in_progress.pop(key, None)+ build_event.set()+ return provider+ except Exception:+ # Signal waiting threads so they don't block forever, then+ # remove the in-progress entry so a subsequent call can retry.+ with self._meta_lock:+ self._in_progress.pop(key, None)+ build_event.set()+ raise+ finally:+ building.discard(key)++ # ------------------------------------------------------------------+ # Internal helpers+ # ------------------------------------------------------------------++ @staticmethod+ def _make_key(name: str, **kwargs) -> str:+ """Compute a deterministic string key from *name* and *kwargs*.++ All kwargs values participate in the key (including ``api_key``+ when present) so that instances with different configurations are+ never aliased. Nested dicts and lists are normalised to tuples+ so the resulting string is stable.+ """def make_hashable(value):if isinstance(value, dict):return tuple(sorted((k, make_hashable(v)) for k, v in value.items()))@@ -1865,22 +2070,13 @@ class ProviderPool:key_parts = [name]for k, v in sorted(kwargs.items()):- # Skip some keys if they shouldn't affect pooling?- # For now, all init args matter for the instance identity.key_parts.append((k, make_hashable(v)))-- key = str(tuple(key_parts))-- if key in self._providers:- return self._providers[key]-- self.logger.debug(f"Creating new provider instance for {name}")- provider = self._create_provider(name, **kwargs)- self._providers[key] = provider- return provider+ return str(tuple(key_parts))def _create_provider(self, name: str, **kwargs) -> BaseProvider:- """Internal creation logic."""+ """Instantiate and return a new provider. Called only when the+ cache does not yet contain an entry for the resolved key.+ """# Check registry firstcustom_provider = provider_registry.get(name)if custom_provider:@@ -1907,8 +2103,18 @@ class ProviderPool:return provider_class(**kwargs)def clear(self):- """Clear the provider pool."""- self._providers.clear()+ """Clear the provider pool.++ Waits for any in-progress constructions to complete or fail before+ discarding cached entries, so callers that obtained a reference just+ before ``clear()`` still hold a valid object.+ """+ with self._meta_lock:+ self._providers.clear()+ # In-progress events are intentionally left intact: threads+ # waiting on them will re-try after the event fires and find+ # an empty pool, becoming new builders. This avoids forcing+ # waiting threads to deal with a stale event after clear().# Global provider pooldiff --git a/tests/semantic_extract/test_provider_pool_isolation.py b/tests/semantic_extract/test_provider_pool_isolation.pynew file mode 100644index 00000000..25d3fdcb--- /dev/null+++ b/tests/semantic_extract/test_provider_pool_isolation.py@@ -0,0 +1,964 @@+"""+Regression tests for ProviderPool credential isolation and concurrency safety.++Issue+-----+``ProviderPool`` computed its cache key from the raw kwargs passed to+``create_provider()``. When callers omitted ``api_key`` and relied on+environment-variable resolution inside the provider ``__init__``, the+resolved credential was *not* part of the pool key. Rotating+``OPENAI_API_KEY`` (or any provider env var) between two calls with+otherwise identical arguments caused the second call to receive the+stale cached provider — a credential cross-contamination bug.++There was also an unsynchronised check-then-create-then-store sequence+in ``ProviderPool.get()`` that let concurrent threads construct duplicate+providers for the same key.++Fix+---+* ``ProviderPool.get()`` now resolves the effective API key (via the same+ ``config.get_api_key() → env-var`` fallback chain the provider+ ``__init__`` uses) *before* computing the cache key, and injects it into+ kwargs so the resolved credential is always represented in the key.+* A ``threading.Lock`` with double-checked locking serialises first-time+ construction so that concurrent callers with the same key always receive+ the same instance.++Tests in this module+---------------------+TestCredentialIsolation+ - env-key rotation creates a fresh provider+ - explicit api_key=A and api_key=B are isolated+ - same effective config still reuses the cached provider+ - use_pool=False always bypasses the pool+ - api_key is not leaked into debug log output++TestConcurrencySafety+ - concurrent first-time requests for the same config produce exactly one+ canonical provider (single construction, same instance returned to all)+"""++import os+import threading+import unittest+from unittest.mock import MagicMock, patch++from semantica.semantic_extract.providers import (+ ProviderPool,+ create_provider,+)+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++def _make_pool() -> ProviderPool:+ """Return a fresh, isolated ProviderPool for each test."""+ return ProviderPool()+++def _mock_openai_cls(instances: list):+ """Return a side_effect callable that appends each new mock to *instances*."""+ def factory(*args, **kwargs):+ m = MagicMock(name=f"OpenAIProvider-{len(instances)}")+ instances.append(m)+ return m+ return factory+++# ---------------------------------------------------------------------------+# Credential isolation+# ---------------------------------------------------------------------------++class TestCredentialIsolation(unittest.TestCase):+ """ProviderPool must isolate providers by effective credential."""++ def setUp(self):+ # Every test gets its own pool so there is no cross-test state.+ self.pool = _make_pool()++ # ------------------------------------------------------------------+ # Core bug: env-key rotation must yield a new provider+ # ------------------------------------------------------------------++ def test_env_key_rotation_creates_fresh_provider(self):+ """+ Rotating the environment API key between two otherwise-identical+ calls must produce two distinct provider instances.++ This is the primary regression test: before the fix the pool key+ did not contain the env-var-resolved credential, so both calls+ received the same (stale) instance.+ """+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-key-A"}, clear=False):+ p1 = self.pool.get("openai", model="gpt-4")++ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-key-B"}, clear=False):+ p2 = self.pool.get("openai", model="gpt-4")++ # Two different credentials → two different instances+ self.assertIsNot(+ p1,+ p2,+ "Rotating OPENAI_API_KEY must produce a new provider instance, "+ "not reuse the one initialised with the old key.",+ )+ self.assertEqual(len(instances), 2, "Expected exactly two provider constructions.")++ def test_env_key_rotation_via_create_provider(self):+ """Same assertion exercised through the public ``create_provider`` API."""+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-env-A"}, clear=False):+ p1 = self.pool.get("openai", model="gpt-4")++ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-env-B"}, clear=False):+ p2 = self.pool.get("openai", model="gpt-4")++ self.assertIsNot(p1, p2)+ self.assertEqual(len(instances), 2)++ # ------------------------------------------------------------------+ # Explicit api_key isolation (must still work after the fix)+ # ------------------------------------------------------------------++ def test_explicit_different_keys_are_isolated(self):+ """Two calls with different explicit api_key values must not share an instance."""+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ p1 = self.pool.get("openai", api_key="sk-A", model="gpt-4")+ p2 = self.pool.get("openai", api_key="sk-B", model="gpt-4")++ self.assertIsNot(p1, p2)+ self.assertEqual(len(instances), 2)++ def test_explicit_same_key_reuses_instance(self):+ """Two calls with the same explicit api_key and model must share one instance."""+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ p1 = self.pool.get("openai", api_key="sk-same", model="gpt-4")+ p2 = self.pool.get("openai", api_key="sk-same", model="gpt-4")++ self.assertIs(p1, p2)+ self.assertEqual(len(instances), 1, "Same config must not construct a second instance.")++ # ------------------------------------------------------------------+ # Same effective credential reuses the cached provider+ # ------------------------------------------------------------------++ def test_same_env_key_reuses_provider(self):+ """If the env key does not change, repeat calls must return the cached instance."""+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-stable"}, clear=False):+ p1 = self.pool.get("openai", model="gpt-4")+ p2 = self.pool.get("openai", model="gpt-4")++ self.assertIs(p1, p2, "Same config must reuse the cached provider.")+ self.assertEqual(len(instances), 1)++ def test_explicit_key_matches_env_key_reuses_provider(self):+ """+ If the caller passes api_key=X explicitly and the env var is also X,+ repeated calls must still reuse the same instance (key must be stable).+ """+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-same-X"}, clear=False):+ p1 = self.pool.get("openai", api_key="sk-same-X", model="gpt-4")+ # Second call omits explicit key — env var resolves to same value+ p2 = self.pool.get("openai", model="gpt-4")++ self.assertIs(p1, p2)+ self.assertEqual(len(instances), 1)++ # ------------------------------------------------------------------+ # use_pool=False+ # ------------------------------------------------------------------++ def test_use_pool_false_always_creates_fresh_instance(self):+ """use_pool=False must bypass the pool and always construct a new provider."""+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-any"}, clear=False):+ p1 = create_provider("openai", use_pool=False, model="gpt-4")+ p2 = create_provider("openai", use_pool=False, model="gpt-4")++ self.assertIsNot(p1, p2)+ self.assertEqual(len(instances), 2)++ # ------------------------------------------------------------------+ # Providers that do not use api_key are unaffected+ # ------------------------------------------------------------------++ def test_ollama_provider_unaffected_no_api_key(self):+ """+ OllamaProvider does not use an API key. The pool must still work+ correctly (same config → same instance) without injecting a spurious+ api_key kwarg.+ """+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OllamaProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ p1 = self.pool.get("ollama", model="llama2", base_url="http://localhost:11434")+ p2 = self.pool.get("ollama", model="llama2", base_url="http://localhost:11434")++ self.assertIs(p1, p2)+ self.assertEqual(len(instances), 1)++ # Verify api_key was NOT injected into the pool key for Ollama.+ pool_key = ProviderPool._make_key(+ "ollama", model="llama2", base_url="http://localhost:11434"+ )+ self.assertNotIn("api_key", pool_key)++ # ------------------------------------------------------------------+ # api_key must not appear in log output+ # ------------------------------------------------------------------++ def test_api_key_not_logged(self):+ """+ The debug log message emitted when a new provider is created must+ not contain the raw API key value.+ """+ import logging++ log_records = []++ class CapturingHandler(logging.Handler):+ def emit(self, record):+ log_records.append(self.format(record))++ handler = CapturingHandler()+ logging.getLogger().addHandler(handler)+ try:+ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls([]),+ ):+ with patch.dict(+ os.environ, {"OPENAI_API_KEY": "sk-secret-should-not-log"}, clear=False+ ):+ self.pool.get("openai", model="gpt-4")+ finally:+ logging.getLogger().removeHandler(handler)++ combined = "\n".join(log_records)+ self.assertNotIn(+ "sk-secret-should-not-log",+ combined,+ "Raw API key value must not appear in log output.",+ )+++# ---------------------------------------------------------------------------+# Concurrency safety+# ---------------------------------------------------------------------------++class TestConcurrencySafety(unittest.TestCase):+ """ProviderPool.get() must serialise first-time construction under concurrency."""++ def setUp(self):+ self.pool = _make_pool()++ def test_concurrent_first_access_single_construction(self):+ """+ N threads simultaneously requesting the same (provider, config) for+ the first time must result in exactly ONE provider construction and+ all threads receiving the SAME instance.++ This is the regression test for the check-then-create race:+ before the fix, concurrent threads could each pass the ``if key in+ self._providers`` check before any one of them stored its result,+ causing N duplicate constructions and non-deterministic aliasing.+ """+ num_threads = 20+ results: list = [None] * num_threads+ construction_count = 0+ construction_lock = threading.Lock()++ # Introduce a small delay inside the constructor to widen the race+ # window so that — without the fix — multiple threads would almost+ # certainly construct duplicate providers.+ import time++ def slow_factory(*args, **kwargs):+ nonlocal construction_count+ time.sleep(0.02) # 20 ms — enough to expose the race+ with construction_lock:+ construction_count += 1+ m = MagicMock(name=f"provider-{construction_count}")+ return m++ barrier = threading.Barrier(num_threads)++ def worker(idx):+ barrier.wait() # all threads start simultaneously+ results[idx] = self.pool.get("openai", api_key="sk-concurrent", model="gpt-4")++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=slow_factory,+ ):+ threads = [threading.Thread(target=worker, args=(i,)) for i in range(num_threads)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=5)++ # Exactly one construction must have occurred.+ self.assertEqual(+ construction_count,+ 1,+ f"Expected exactly 1 provider construction under concurrency, "+ f"got {construction_count}.",+ )++ # Every thread must have received the same instance.+ canonical = results[0]+ self.assertIsNotNone(canonical)+ for idx, result in enumerate(results):+ self.assertIs(+ result,+ canonical,+ f"Thread {idx} received a different provider instance than thread 0.",+ )++ def test_clear_is_thread_safe(self):+ """+ ``clear()`` must not corrupt the pool's internal state when called+ concurrently with ``get()`` calls.+ """+ import time++ instances: list = []+ errors: list = []++ def factory(*args, **kwargs):+ time.sleep(0.001)+ m = MagicMock()+ instances.append(m)+ return m++ stop_event = threading.Event()++ def getter():+ while not stop_event.is_set():+ try:+ self.pool.get("openai", api_key="sk-x", model="gpt-4")+ except Exception as exc:+ errors.append(exc)++ def clearer():+ for _ in range(20):+ self.pool.clear()+ time.sleep(0.005)++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=factory,+ ):+ getter_threads = [threading.Thread(target=getter) for _ in range(4)]+ clear_thread = threading.Thread(target=clearer)++ for t in getter_threads:+ t.start()+ clear_thread.start()++ clear_thread.join(timeout=5)+ stop_event.set()+ for t in getter_threads:+ t.join(timeout=5)++ self.assertEqual(+ errors,+ [],+ f"No exceptions expected during concurrent get/clear, got: {errors}",+ )+++# ---------------------------------------------------------------------------+# _make_key and _resolve_api_key unit tests+# ---------------------------------------------------------------------------++class TestProviderPoolInternals(unittest.TestCase):+ """Unit tests for ProviderPool helper methods."""++ def setUp(self):+ self.pool = _make_pool()++ def test_make_key_is_deterministic(self):+ """_make_key must return the same string for equivalent kwargs regardless of insertion order."""+ k1 = ProviderPool._make_key("openai", api_key="sk-x", model="gpt-4", temperature=0.5)+ k2 = ProviderPool._make_key("openai", model="gpt-4", temperature=0.5, api_key="sk-x")+ self.assertEqual(k1, k2)++ def test_make_key_differs_for_different_api_keys(self):+ k1 = ProviderPool._make_key("openai", api_key="sk-A", model="gpt-4")+ k2 = ProviderPool._make_key("openai", api_key="sk-B", model="gpt-4")+ self.assertNotEqual(k1, k2)++ def test_make_key_nested_dict_is_stable(self):+ k1 = ProviderPool._make_key("openai", opts={"a": 1, "b": 2})+ k2 = ProviderPool._make_key("openai", opts={"b": 2, "a": 1})+ self.assertEqual(k1, k2)++ def test_resolve_api_key_explicit_kwarg(self):+ """Explicit kwarg is returned as-is."""+ result = self.pool._resolve_api_key("openai", {"api_key": "sk-explicit"})+ self.assertEqual(result, "sk-explicit")++ def test_resolve_api_key_env_var_fallback(self):+ """Falls back to env var when no explicit kwarg."""+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-from-env"}, clear=False):+ result = self.pool._resolve_api_key("openai", {})+ self.assertEqual(result, "sk-from-env")++ def test_resolve_api_key_none_for_ollama(self):+ """Ollama is not in _API_KEY_PROVIDERS; must return None."""+ result = self.pool._resolve_api_key("ollama", {})+ self.assertIsNone(result)++ def test_resolve_api_key_none_for_huggingface_llm(self):+ """HuggingFace LLM provider must return None."""+ result = self.pool._resolve_api_key("huggingface_llm", {})+ self.assertIsNone(result)++ def test_resolve_api_key_none_when_no_key_anywhere(self):+ """When no key is set anywhere, must return None (not raise)."""+ env_key = "OPENAI_API_KEY"+ original = os.environ.pop(env_key, None)+ try:+ # Also ensure config singleton has no key cached+ with patch(+ "semantica.semantic_extract.providers.config.get_api_key",+ return_value=None,+ ):+ result = self.pool._resolve_api_key("openai", {})+ self.assertIsNone(result)+ finally:+ if original is not None:+ os.environ[env_key] = original++ def test_all_api_key_providers_covered(self):+ """All built-in API-key-using providers must be listed in _API_KEY_PROVIDERS."""+ expected = {"openai", "gemini", "groq", "anthropic", "deepseek", "novita"}+ self.assertEqual(ProviderPool._API_KEY_PROVIDERS, expected)+++if __name__ == "__main__":+ unittest.main(verbosity=2)+++# ---------------------------------------------------------------------------+# HIGH-1 regression: custom provider shadowing a built-in API-key provider+# ---------------------------------------------------------------------------++class TestCustomProviderShadowing(unittest.TestCase):+ """Custom providers registered under a built-in name must not receive+ an automatically injected ``api_key`` kwarg."""++ def setUp(self):+ self.pool = _make_pool()++ def tearDown(self):+ # Remove any registrations added during tests.+ from semantica.semantic_extract.registry import provider_registry as _reg+ for name in ("openai", "groq", "anthropic", "gemini"):+ _reg.unregister(name)++ def test_custom_provider_under_builtin_name_no_api_key_injection(self):+ """+ Regression test for HIGH-1.++ A custom provider registered under the name ``"openai"`` must NOT+ receive an automatically injected ``api_key`` kwarg, even when+ ``OPENAI_API_KEY`` is set in the environment.++ Before the fix, ``_resolve_api_key`` would resolve the env-var key+ and inject it regardless of whether the registered class accepted it,+ causing a ``TypeError`` for any custom class that does not declare+ an ``api_key`` parameter.+ """+ from semantica.semantic_extract.registry import provider_registry++ class MyCustomProvider:+ """Custom provider that does NOT accept api_key."""+ def __init__(self, model="custom-default"):+ self.model = model++ def is_available(self):+ return True++ provider_registry.register("openai", MyCustomProvider)++ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-should-not-inject"}, clear=False):+ # Must not raise TypeError — no api_key should be injected.+ provider = self.pool.get("openai", model="custom-v1")++ self.assertIsInstance(provider, MyCustomProvider)+ self.assertEqual(provider.model, "custom-v1")++ def test_custom_provider_receives_explicit_caller_kwargs_unchanged(self):+ """Explicit caller kwargs (other than api_key) must still reach the custom class."""+ from semantica.semantic_extract.registry import provider_registry++ received_kwargs = {}++ class CapturingProvider:+ def __init__(self, **kwargs):+ received_kwargs.update(kwargs)++ def is_available(self):+ return True++ provider_registry.register("groq", CapturingProvider)++ self.pool.get("groq", model="custom-model", timeout=30)++ self.assertEqual(received_kwargs.get("model"), "custom-model")+ self.assertEqual(received_kwargs.get("timeout"), 30)+ self.assertNotIn(+ "api_key",+ received_kwargs,+ "api_key must not be auto-injected into a custom provider.",+ )++ def test_custom_provider_explicit_api_key_caller_passes_through(self):+ """If the *caller* explicitly passes api_key, it must reach the custom class."""+ from semantica.semantic_extract.registry import provider_registry++ received_kwargs = {}++ class CapturingProvider:+ def __init__(self, **kwargs):+ received_kwargs.update(kwargs)++ def is_available(self):+ return True++ provider_registry.register("anthropic", CapturingProvider)++ self.pool.get("anthropic", api_key="sk-caller-explicit", model="claude-custom")++ self.assertEqual(+ received_kwargs.get("api_key"),+ "sk-caller-explicit",+ "Explicit caller api_key must be forwarded to the custom provider.",+ )++ def test_builtin_provider_still_resolves_env_key_when_no_custom_registration(self):+ """Built-in providers must continue to have their env key resolved."""+ instances = []++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=_mock_openai_cls(instances),+ ):+ with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-builtin-env"}, clear=False):+ p1 = self.pool.get("openai", model="gpt-4")+ p2 = self.pool.get("openai", model="gpt-4")++ # Same env key → same instance (key stable).+ self.assertIs(p1, p2)+ self.assertEqual(len(instances), 1)++ def test_builtin_provider_env_rotation_after_deregistering_custom(self):+ """+ After a custom provider is unregistered, the built-in env-key+ resolution must resume correctly.+ """+ from semantica.semantic_extract.registry import provider_registry++ class TempCustom:+ def __init__(self, model="x"):+ pass+ def is_available(self):+ return True++ provider_registry.register("gemini", TempCustom)+ # While registered: no key injection+ with patch.dict(os.environ, {"GEMINI_API_KEY": "sk-gemini"}, clear=False):+ p_custom = self.pool.get("gemini", model="custom")+ self.assertIsInstance(p_custom, TempCustom)++ provider_registry.unregister("gemini")+ self.pool.clear() # reset so built-in path is exercised fresh++ with patch(+ "semantica.semantic_extract.providers.GeminiProvider",+ side_effect=_mock_openai_cls([]),+ ) as MockGemini:+ with patch.dict(os.environ, {"GEMINI_API_KEY": "sk-gemini"}, clear=False):+ self.pool.get("gemini", model="gemini-pro")++ # Built-in path was taken (GeminiProvider was instantiated).+ MockGemini.assert_called_once()+ # api_key must have been passed (env-resolved).+ call_kwargs = MockGemini.call_args[1]+ self.assertIn("api_key", call_kwargs)+++# ---------------------------------------------------------------------------+# HIGH-2 regression: nested / re-entrant provider creation+# ---------------------------------------------------------------------------++class TestNestedProviderCreation(unittest.TestCase):+ """Provider constructors must be able to call ``create_provider()`` or+ ``pool.get()`` again without deadlocking."""++ def setUp(self):+ self.pool = _make_pool()++ def tearDown(self):+ from semantica.semantic_extract.registry import provider_registry as _reg+ for name in ("openai", "groq", "anthropic"):+ _reg.unregister(name)++ # ------------------------------------------------------------------+ # Core re-entrancy (different keys) — primary regression test+ # ------------------------------------------------------------------++ def test_nested_different_key_no_deadlock(self):+ """+ Regression test for HIGH-2.++ A provider constructor that calls ``pool.get()`` for a *different*+ key must not deadlock.++ Before the fix, the single non-reentrant ``threading.Lock`` was held+ for the entire duration of ``_create_provider()``. Any re-entrant+ call to ``pool.get()`` (for any key) would block forever attempting+ to re-acquire the same lock.+ """+ from semantica.semantic_extract.registry import provider_registry++ inner_instances = []++ class InnerProvider:+ def __init__(self, **kwargs):+ inner_instances.append(self)++ def is_available(self):+ return True++ class OuterProvider:+ """Outer provider whose constructor depends on InnerProvider."""+ def __init__(self, **kwargs):+ # Nested call for a completely different provider/key.+ self.inner = self.pool.get("groq", api_key="sk-inner", model="llama")++ def is_available(self):+ return True++ OuterProvider.pool = self.pool+ provider_registry.register("openai", OuterProvider)+ provider_registry.register("groq", InnerProvider)++ deadline = threading.Event()+ error_holder = [None]+ result_holder = [None]++ def attempt():+ try:+ result_holder[0] = self.pool.get(+ "openai", api_key="sk-outer", model="gpt-4"+ )+ except Exception as exc:+ error_holder[0] = exc+ finally:+ deadline.set()++ t = threading.Thread(target=attempt, daemon=True)+ t.start()+ t.join(timeout=3)++ self.assertTrue(+ deadline.is_set(),+ "Nested pool.get() for a different key deadlocked (thread did not return).",+ )+ if error_holder[0]:+ self.fail(f"Unexpected exception: {error_holder[0]}")++ outer = result_holder[0]+ self.assertIsInstance(outer, OuterProvider)+ self.assertIsInstance(outer.inner, InnerProvider)+ self.assertEqual(len(inner_instances), 1)++ def test_nested_different_key_inner_is_cached(self):+ """The inner provider created during outer construction is cached in the pool."""+ from semantica.semantic_extract.registry import provider_registry++ class Inner:+ def __init__(self, **kw): pass+ def is_available(self): return True++ class Outer:+ def __init__(self, **kw):+ self.inner = pool.get("groq", api_key="sk-i", model="m")+ def is_available(self): return True++ pool = self.pool+ provider_registry.register("openai", Outer)+ provider_registry.register("groq", Inner)++ pool.get("openai", api_key="sk-o", model="m")++ # The inner provider must now be in the pool — same instance returned.+ inner2 = pool.get("groq", api_key="sk-i", model="m")+ self.assertIsInstance(inner2, Inner)++ # ------------------------------------------------------------------+ # Same-key re-entrancy: must not deadlock (RecursionError is acceptable)+ # ------------------------------------------------------------------++ def test_same_key_reentrant_raises_recursion_not_deadlock(self):+ """+ A constructor that calls pool.get() for the EXACT same key it is+ currently building is inherently infinitely recursive (the pool+ cannot meaningfully satisfy such a request). The pool must not+ deadlock; raising RecursionError is the correct outcome.+ """+ from semantica.semantic_extract.registry import provider_registry++ pool = self.pool++ class SelfReferential:+ def __init__(self, **kw):+ # Same name + same kwargs → same key → infinite recursion.+ pool.get("anthropic", api_key="sk-x", model="claude")++ provider_registry.register("anthropic", SelfReferential)++ deadline = threading.Event()+ exception_holder = [None]++ def attempt():+ try:+ pool.get("anthropic", api_key="sk-x", model="claude")+ except RecursionError:+ exception_holder[0] = "RecursionError"+ except Exception as e:+ exception_holder[0] = e+ finally:+ deadline.set()++ t = threading.Thread(target=attempt, daemon=True)+ t.start()+ t.join(timeout=5)++ self.assertTrue(+ deadline.is_set(),+ "Same-key re-entrancy caused a deadlock (thread did not return within 5s).",+ )+ self.assertEqual(+ exception_holder[0],+ "RecursionError",+ f"Expected RecursionError for infinite same-key re-entrancy, "+ f"got: {exception_holder[0]}",+ )++ # ------------------------------------------------------------------+ # Construction failure: waiting threads must not hang+ # ------------------------------------------------------------------++ def test_construction_failure_unblocks_waiting_threads(self):+ """+ If provider construction raises, threads that queued up waiting for+ the same key must not hang indefinitely — they must eventually get+ a chance to build (or fail) themselves.+ """+ import time+ from semantica.semantic_extract.registry import provider_registry++ pool = self.pool+ attempt_count = 0+ succeeded = threading.Event()+ errors: list = []++ # First construction raises; subsequent ones succeed.+ class FlakyProvider:+ def __init__(self, **kw):+ nonlocal attempt_count+ attempt_count += 1+ if attempt_count == 1:+ raise RuntimeError("first construction deliberately fails")+ self.ok = True++ def is_available(self):+ return True++ provider_registry.register("groq", FlakyProvider)++ results: list = [None, None]+ barriers = threading.Barrier(2)++ def worker(idx):+ barriers.wait() # start simultaneously+ try:+ results[idx] = pool.get("groq", api_key="sk-flaky", model="m")+ if results[idx] is not None:+ succeeded.set()+ except Exception as e:+ errors.append(e)++ threads = [threading.Thread(target=worker, args=(i,)) for i in range(2)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=5)++ # At least one thread must have obtained a valid provider.+ self.assertTrue(+ succeeded.is_set() or any(r is not None for r in results),+ "All threads failed or hung after construction failure.",+ )+ # The first construction must have failed (attempt_count >= 2 means+ # at least one retry occurred after the initial failure).+ self.assertGreaterEqual(attempt_count, 2)++ # ------------------------------------------------------------------+ # Same-key concurrent: still serialised after the redesign+ # ------------------------------------------------------------------++ def test_same_key_concurrent_single_construction_after_redesign(self):+ """+ Verify the per-key Event design still ensures exactly one provider+ is constructed when many threads race for the same key simultaneously.+ (Keeps the concurrency guarantee from the original fix.)+ """+ import time++ construction_count = 0+ count_lock = threading.Lock()+ num_threads = 16+ results: list = [None] * num_threads++ def slow_factory(*args, **kwargs):+ nonlocal construction_count+ time.sleep(0.02)+ with count_lock:+ construction_count += 1+ return MagicMock(name=f"p-{construction_count}")++ barrier = threading.Barrier(num_threads)++ def worker(idx):+ barrier.wait()+ results[idx] = self.pool.get(+ "openai", api_key="sk-concurrent", model="gpt-4"+ )++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ side_effect=slow_factory,+ ):+ threads = [+ threading.Thread(target=worker, args=(i,)) for i in range(num_threads)+ ]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=5)++ self.assertEqual(+ construction_count,+ 1,+ f"Expected exactly 1 construction, got {construction_count}.",+ )+ canonical = results[0]+ self.assertIsNotNone(canonical)+ for idx, r in enumerate(results):+ self.assertIs(r, canonical, f"Thread {idx} got a different instance.")++ # ------------------------------------------------------------------+ # Different-key concurrent: must NOT serialize unrelated keys+ # ------------------------------------------------------------------++ def test_different_keys_build_concurrently(self):+ """+ Two threads building *different* keys must not be serialised by the+ pool: both constructions should run in parallel (verified by timing).+ """+ import time++ DELAY = 0.1 # 100 ms construction delay per provider++ started = threading.Barrier(2)+ results = {}++ def build_a():+ started.wait()+ results["a"] = self.pool.get("openai", api_key="sk-a", model="gpt-4a")++ def build_b():+ started.wait()+ results["b"] = self.pool.get("openai", api_key="sk-b", model="gpt-4b")++ with patch(+ "semantica.semantic_extract.providers.OpenAIProvider",+ ) as MockCls:+ MockCls.side_effect = [+ MagicMock(name="provider-a"),+ MagicMock(name="provider-b"),+ ]++ ta = threading.Thread(target=build_a)+ tb = threading.Thread(target=build_b)++ t0 = __import__("time").monotonic()+ ta.start()+ tb.start()+ ta.join(timeout=5)+ tb.join(timeout=5)+ elapsed = __import__("time").monotonic() - t0++ self.assertIn("a", results)+ self.assertIn("b", results)+ # If serialised, elapsed ≥ 2 * DELAY. If parallel, elapsed ≈ DELAY.+ # We use 1.5 * DELAY as a generous threshold.+ self.assertLess(+ elapsed,+ DELAY * 1.5 + 0.3, # 0.3 s of scheduling slack+ f"Different-key builds took {elapsed:.3f}s — expected parallel (~{DELAY}s), "+ f"not serialised (~{2*DELAY}s).",+ )+++if __name__ == "__main__":+ unittest.main(verbosity=2)