Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
refactor(ner): remove dead _extract_with_spacy method and unused self.nlp (#1220)
Yunare Maiae12eec40a1fd
diff --git a/semantica/semantic_extract/ner_extractor.py b/semantica/semantic_extract/ner_extractor.pyindex a920efe1..89df3e1e 100644--- a/semantica/semantic_extract/ner_extractor.py+++ b/semantica/semantic_extract/ner_extractor.py@@ -139,17 +139,19 @@ class NERExtractor:if not self.progress_tracker.enabled:self.progress_tracker.enabled = True- # Initialize spaCy model if ML method is used- self.nlp = None+ # Validate the spaCy runtime up front if ML method is used. The model+ # itself is loaded lazily by extract_entities_ml() through the+ # process-level cache in methods.py; this instance only tracks whether+ # ML dispatch should be attempted at all.self._ml_runtime_usable = Trueif "ml" in self.method and SPACY_AVAILABLE:try:# Deferred import: keeps semantic_extract.methods out of the- # module-level import graph and routes loading through the+ # module-level import graph and routes validation through the# process-level cache so repeated NERExtractor constructions# never pay the ~120 ms spacy.load() cost more than once.from .methods import load_spacy_model- self.nlp = load_spacy_model(self.model_name)+ load_spacy_model(self.model_name)except OSError:self.logger.warning(f"spaCy model {self.model_name} not found. ML method will fallback."@@ -535,42 +537,6 @@ class NERExtractor:return processed- def _extract_with_spacy(- self, text: str, min_confidence: float, entity_types: Optional[List[str]]- ) -> List[Entity]:- """Extract entities using spaCy."""- entities = []-- doc = self.nlp(text)-- for ent in doc.ents:- # Filter by entity types if specified- if entity_types and ent.label_ not in entity_types:- continue-- # Get confidence if available- confidence = 1.0- if hasattr(ent, "confidence"):- confidence = ent.confidence- elif hasattr(ent, "score"):- confidence = ent.score-- if confidence >= min_confidence:- entities.append(- Entity(- text=ent.text,- label=ent.label_,- start_char=ent.start_char,- end_char=ent.end_char,- confidence=confidence,- metadata={- "lemma": ent.lemma_ if hasattr(ent, "lemma_") else ent.text- },- )- )-- return entities-def _extract_fallback(self, text: str) -> List[Entity]:"""Fallback entity extraction using simple patterns."""entities = []diff --git a/tests/split/test_spacy_model_cache.py b/tests/split/test_spacy_model_cache.pyindex 00012030..18b57d1a 100644--- a/tests/split/test_spacy_model_cache.py+++ b/tests/split/test_spacy_model_cache.py@@ -216,14 +216,13 @@ class TestNERExtractorSpacyModelCache:monkeypatch.setattr(se_methods, "spacy", _fake_spacy(fake_load))- e1 = NERExtractor(method="ml")- e2 = NERExtractor(method="ml")- e3 = NERExtractor(method="ml", model="en_core_web_sm")+ NERExtractor(method="ml")+ NERExtractor(method="ml")+ NERExtractor(method="ml", model="en_core_web_sm")assert len(calls) == 1, ("repeated NERExtractor constructions should not reload the model")- assert e1.nlp is e2.nlp is e3.nlpdef test_ner_extractor_and_split_callers_share_one_cached_model(self, monkeypatch):"""NERExtractor, SemanticChunker, and split_by_sentences must all use@@ -248,20 +247,23 @@ class TestNERExtractorSpacyModelCache:def test_ner_extractor_distinct_model_names_load_separately(self, monkeypatch):"""Different model names must produce separate cache entries."""calls = []+ loaded = {}def fake_load(name, **kwargs):calls.append(name)- return _nlp_mock()+ nlp = _nlp_mock()+ loaded[name] = nlp+ return nlpmonkeypatch.setattr(se_methods, "spacy", _fake_spacy(fake_load))- sm = NERExtractor(method="ml", model="en_core_web_sm")- lg = NERExtractor(method="ml", model="en_core_web_lg")- sm_again = NERExtractor(method="ml", model="en_core_web_sm")+ NERExtractor(method="ml", model="en_core_web_sm")+ NERExtractor(method="ml", model="en_core_web_lg")+ NERExtractor(method="ml", model="en_core_web_sm")assert calls == ["en_core_web_sm", "en_core_web_lg"]- assert sm.nlp is sm_again.nlp- assert sm.nlp is not lg.nlp+ # Same model name -> same cached Language object; different names -> different objects.+ assert loaded["en_core_web_sm"] is not loaded["en_core_web_lg"]def test_ner_extractor_failed_load_not_cached_and_retried(self, monkeypatch):"""A missing model must not poison the cache. A subsequent construction@@ -274,31 +276,33 @@ class TestNERExtractorSpacyModelCache:monkeypatch.setattr(se_methods, "spacy", _fake_spacy(failing_load))- # Construction with missing model: nlp must remain None, no crash- extractor1 = NERExtractor(method="ml")- assert extractor1.nlp is None+ # Construction with missing model: must not raise+ NERExtractor(method="ml")assert len(attempts) == 1, "one load attempt expected for the missing model"# Second construction: must retry (cache must not hold the failure)- extractor2 = NERExtractor(method="ml")- assert extractor2.nlp is None+ NERExtractor(method="ml")assert len(attempts) == 2, "a failed load must not be cached"# Now install a working model and verify recovery+ loaded_models = {}+def working_load(name, **_kwargs):attempts.append(name)- return _nlp_mock()+ nlp = _nlp_mock()+ loaded_models[name] = nlp+ return nlpmonkeypatch.setattr(se_methods, "spacy", _fake_spacy(working_load))- extractor3 = NERExtractor(method="ml")- extractor4 = NERExtractor(method="ml")+ NERExtractor(method="ml")+ NERExtractor(method="ml")- assert extractor3.nlp is not None- assert extractor3.nlp is extractor4.nlpassert len(attempts) == 3, ("exactly one successful load expected after the model becomes available")+ # Confirm the recovered model is cached and shared across callers.+ assert se_methods.load_spacy_model("en_core_web_sm") is loaded_models["en_core_web_sm"]def test_ner_extractor_non_ml_method_does_not_load_model(self, monkeypatch):"""NERExtractor with a non-ml method must not touch the spaCy cache."""diff --git a/tests/test_ner_configurations.py b/tests/test_ner_configurations.pyindex 15c2568a..8224158f 100644--- a/tests/test_ner_configurations.py+++ b/tests/test_ner_configurations.py@@ -33,10 +33,24 @@ class TestNERConfigurations(unittest.TestCase):# Mock LLM providermock_provider = MagicMock()mock_provider.is_available.return_value = True- mock_provider.generate_structured.return_value = [- {"text": "Apple Inc.", "label": "ORG", "start": 0, "end": 10, "confidence": 0.95},- {"text": "Steve Jobs", "label": "PERSON", "start": 26, "end": 36, "confidence": 0.98}+ # The LLM path uses generate_typed (Pydantic schema validation), not+ # generate_structured. Build a typed response object whose .entities+ # carries simple namespace-like items.+ def _entity(text, label, start, end, confidence):+ item = MagicMock()+ item.text = text+ item.label = label+ item.start = start+ item.end = end+ item.confidence = confidence+ return item++ typed_response = MagicMock()+ typed_response.entities = [+ _entity("Apple Inc.", "ORG", 0, 10, 0.95),+ _entity("Steve Jobs", "PERSON", 26, 36, 0.98),]+ mock_provider.generate_typed.return_value = typed_responsemock_create_provider.return_value = mock_provider# Initialize extractor with LLM method@@ -56,7 +70,7 @@ class TestNERConfigurations(unittest.TestCase):self.assertEqual(len(entities), 2)self.assertEqual(entities[0].text, "Apple Inc.")self.assertEqual(entities[0].label, "ORG")- self.assertEqual(entities[0].metadata["extraction_method"], "llm")+ self.assertEqual(entities[0].metadata["extraction_method"], "llm_typed")self.assertEqual(entities[0].metadata["model"], "gpt-4")@patch('semantica.semantic_extract.methods.spacy')@@ -115,7 +129,6 @@ class TestNERConfigurations(unittest.TestCase):with patch('semantica.semantic_extract.ner_extractor.SPACY_AVAILABLE', True):extractor = NERExtractor(method="ml", model="en_core_web_sm")- self.assertIsNone(extractor.nlp)self.assertFalse(extractor._ml_runtime_usable)@patch('semantica.semantic_extract.methods.get_entity_method')@@ -183,7 +196,7 @@ class TestNERConfigurations(unittest.TestCase):self.assertTrue(len(entities) >= 2)texts = [e.text for e in entities]- self.assertIn("Apple Inc", texts) # Regex pattern does not capture the trailing dot+ self.assertIn("Apple Inc.", texts) # The ORG pattern includes the trailing dot via (?:\.|\b)# Actually methods.py regex: r"\b([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s+(?:Inc|Corp|LLC|Ltd|Company))\b"# "Apple Inc." -> "Apple Inc" (dot is outside \b if not matched?)# Let's check the result strictly