Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
Fix #839: Support persistent backends in DecisionEmbeddingPipeline
Sameer63057d272f40e852
diff --git a/semantica/vector_store/decision_embedding_pipeline.py b/semantica/vector_store/decision_embedding_pipeline.pyindex 5f1146a1..070614d8 100644--- a/semantica/vector_store/decision_embedding_pipeline.py+++ b/semantica/vector_store/decision_embedding_pipeline.py@@ -312,9 +312,12 @@ class DecisionEmbeddingPipeline:query_result = self.process_decision(query_decision, store_embeddings=False)# Get candidate embeddings from vector store- candidate_embeddings = self._get_candidate_embeddings(filters)+ fetch_limit = limit * 5 if limit else 100+ candidate_embeddings = self._get_candidate_embeddings(+ query_result["semantic_embedding"], limit=fetch_limit, filters=filters+ )- if not candidate_embeddings:+ if not candidate_embeddings["embeddings"]:return []# Calculate similarities@@ -329,10 +332,30 @@ class DecisionEmbeddingPipeline:query_result["structural_embedding"],candidate_embeddings["embeddings"],candidate_embeddings["metadata"],- top_k=limit,+ top_k=len(candidate_embeddings["embeddings"]),weights=weights,filters=filters)++ # Patch semantic similarity for backends that don't return vectors+ # using the search scores+ actual_weights = weights or (self.semantic_weight, self.structural_weight)+ for sim in similarities:+ idx = sim["index"]+ original_vec = candidate_embeddings["embeddings"][idx][0]+ # If vector is all zeros (placeholder), we use the score from search_vectors+ if not np.any(original_vec):+ score = candidate_embeddings["scores"][idx]+ # Simple normalization if score looks like a distance (FAISS)+ if score > 1.0:+ score = 1.0 / (1.0 + score)+ sim["semantic_similarity"] = score+ # Recompute overall similarity+ sim["similarity"] = (actual_weights[0] * score) + (actual_weights[1] * sim["structural_similarity"])++ # Re-sort after patching+ similarities.sort(key=lambda x: x["similarity"], reverse=True)+ similarities = similarities[:limit]else:# Use semantic similarity onlysimilarities = self._find_semantic_similar(@@ -340,7 +363,8 @@ class DecisionEmbeddingPipeline:candidate_embeddings["embeddings"],candidate_embeddings["metadata"],limit,- filters+ filters,+ candidate_scores=candidate_embeddings.get("scores"))return similarities@@ -759,27 +783,24 @@ class DecisionEmbeddingPipeline:except Exception as e:self.logger.warning(f"Failed to pre-generate structural embeddings: {e}")- def _get_candidate_embeddings(self, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:+ def _get_candidate_embeddings(+ self, query_vector: Optional[np.ndarray] = None, limit: int = 10000, filters: Optional[Dict[str, Any]] = None+ ) -> Dict[str, Any]:"""Get candidate embeddings from vector store."""if not self.vector_store:- return {"embeddings": [], "metadata": []}+ return {"embeddings": [], "metadata": [], "scores": []}++ if query_vector is None:+ query_vector = np.zeros(self.embedding_dimension)++ results = self.vector_store.search_vectors(query_vector, k=limit, **(filters or {}))- # Get all vectors from storeembeddings = []metadata = []+ scores = []- for vector_id, vector in self.vector_store.vectors.items():- vector_metadata = self.vector_store.metadata.get(vector_id, {})-- # Apply filters- if filters:- match = True- for key, value in filters.items():- if key not in vector_metadata or vector_metadata[key] != value:- match = False- break- if not match:- continue+ for res in results:+ vector_metadata = res.get("metadata", {})# Extract structural embedding if availablestruct_emb = vector_metadata.get("structural_embedding")@@ -788,10 +809,15 @@ class DecisionEmbeddingPipeline:else:struct_emb = np.zeros(self.node_embedding_dimension)+ vector = res.get("vector")+ if vector is None:+ vector = np.zeros(self.embedding_dimension)+embeddings.append((vector, struct_emb))metadata.append(vector_metadata)+ scores.append(res.get("score", 0.0))- return {"embeddings": embeddings, "metadata": metadata}+ return {"embeddings": embeddings, "metadata": metadata, "scores": scores}def _find_semantic_similar(self,@@ -799,16 +825,22 @@ class DecisionEmbeddingPipeline:candidate_embeddings: List[Tuple[np.ndarray, np.ndarray]],candidate_metadata: List[Dict[str, Any]],limit: int,- filters: Optional[Dict[str, Any]] = None+ filters: Optional[Dict[str, Any]] = None,+ candidate_scores: Optional[List[float]] = None) -> List[Dict[str, Any]]:"""Find similar decisions using semantic similarity only."""similarities = []for i, (sem_emb, struct_emb) in enumerate(candidate_embeddings):# Calculate semantic similarity- similarity = self.hybrid_calculator._calculate_similarity(- query_embedding, sem_emb, "cosine"- )+ if not np.any(sem_emb) and candidate_scores:+ similarity = candidate_scores[i]+ if similarity > 1.0:+ similarity = 1.0 / (1.0 + similarity)+ else:+ similarity = self.hybrid_calculator._calculate_similarity(+ query_embedding, sem_emb, "cosine"+ )similarities.append({"similarity": similarity,diff --git a/semantica/vector_store/decision_vector_methods.py b/semantica/vector_store/decision_vector_methods.pyindex 550712b1..b38a0298 100644--- a/semantica/vector_store/decision_vector_methods.py+++ b/semantica/vector_store/decision_vector_methods.py@@ -365,6 +365,19 @@ def get_decision_statistics(outcome_counts = {}confidence_values = []+ # Not all backends expose internal metadata dictionaries (e.g. FAISS).+ if not hasattr(store, "metadata"):+ return {+ "total_decisions": 0,+ "categories": {},+ "outcomes": {},+ "average_confidence": 0.0,+ "min_confidence": 0.0,+ "max_confidence": 0.0,+ "has_structural_embeddings": False,+ "warning": "Statistics not supported for persistent backends without full metadata access."+ }+for metadata in store.metadata.values():# Count by categorycategory = metadata.get("category", "unknown")diff --git a/tests/vector_store/test_decision_embedding_pipeline.py b/tests/vector_store/test_decision_embedding_pipeline.pyindex 0572517c..9a1b5351 100644--- a/tests/vector_store/test_decision_embedding_pipeline.py+++ b/tests/vector_store/test_decision_embedding_pipeline.py@@ -393,6 +393,53 @@ class TestDecisionEmbeddingPipelineEdgeCases:assert all("decision_data" in result for result in results)assert all("vector_id" in result for result in results)+ def test_find_similar_decisions_real_faiss_backend(self):+ """Regression test for FAISS backend crashing due to .vectors access (issue #839)"""+ try:+ from semantica.vector_store import VectorStore+ except ImportError:+ pytest.skip("VectorStore not available")++ # Create a real VectorStore with FAISS backend+ vs = VectorStore(backend="faiss", config={"dimension": 384})+ vs.initialize_decision_pipeline()++ # Store a couple of dummy decisions to ensure index has data+ vs.store_decision(scenario="A test decision 1", category="test")+ vs.store_decision(scenario="A test decision 2", category="test")++ # This will call _get_candidate_embeddings without a mock+ # Before the fix, this crashed with AttributeError: 'VectorStore' object has no attribute 'vectors'+ results = vs.search_decisions("test query", limit=2)++ # Assert it returns results and doesn't crash+ assert isinstance(results, list)+ assert len(results) <= 2+++ def test_find_similar_decisions_real_inmemory_backend(self):+ """Regression test for inmemory backend preserving behavior after issue #839 fix"""+ try:+ from semantica.vector_store import VectorStore+ except ImportError:+ pytest.skip("VectorStore not available")++ vs = VectorStore(backend="inmemory", config={"dimension": 384})+ vs.initialize_decision_pipeline()++ vs.store_decision(scenario="Apple product launch", category="tech")+ vs.store_decision(scenario="Microsoft earnings report", category="tech")+ vs.store_decision(scenario="Local bakery opens", category="food")++ results = vs.search_decisions("software company", limit=2)++ assert isinstance(results, list)+ assert len(results) <= 2++ if len(results) > 0:+ assert "similarity" in results[0]+ assert "semantic_similarity" in results[0]+ assert "structural_similarity" in results[0]if __name__ == "__main__":pytest.main([__file__])