Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

Fix #839: Support persistent backends in DecisionEmbeddingPipeline

Sameer63057d272f40e852
diff --git a/semantica/vector_store/decision_embedding_pipeline.py b/semantica/vector_store/decision_embedding_pipeline.py
index 5f1146a1..070614d8 100644
--- a/semantica/vector_store/decision_embedding_pipeline.py
+++ b/semantica/vector_store/decision_embedding_pipeline.py
@@ -312,9 +312,12 @@ class DecisionEmbeddingPipeline:
query_result = self.process_decision(query_decision, store_embeddings=False)
# Get candidate embeddings from vector store
- candidate_embeddings = self._get_candidate_embeddings(filters)
+ fetch_limit = limit * 5 if limit else 100
+ candidate_embeddings = self._get_candidate_embeddings(
+ query_result["semantic_embedding"], limit=fetch_limit, filters=filters
+ )
- if not candidate_embeddings:
+ if not candidate_embeddings["embeddings"]:
return []
# Calculate similarities
@@ -329,10 +332,30 @@ class DecisionEmbeddingPipeline:
query_result["structural_embedding"],
candidate_embeddings["embeddings"],
candidate_embeddings["metadata"],
- top_k=limit,
+ top_k=len(candidate_embeddings["embeddings"]),
weights=weights,
filters=filters
)
+
+ # Patch semantic similarity for backends that don't return vectors
+ # using the search scores
+ actual_weights = weights or (self.semantic_weight, self.structural_weight)
+ for sim in similarities:
+ idx = sim["index"]
+ original_vec = candidate_embeddings["embeddings"][idx][0]
+ # If vector is all zeros (placeholder), we use the score from search_vectors
+ if not np.any(original_vec):
+ score = candidate_embeddings["scores"][idx]
+ # Simple normalization if score looks like a distance (FAISS)
+ if score > 1.0:
+ score = 1.0 / (1.0 + score)
+ sim["semantic_similarity"] = score
+ # Recompute overall similarity
+ sim["similarity"] = (actual_weights[0] * score) + (actual_weights[1] * sim["structural_similarity"])
+
+ # Re-sort after patching
+ similarities.sort(key=lambda x: x["similarity"], reverse=True)
+ similarities = similarities[:limit]
else:
# Use semantic similarity only
similarities = self._find_semantic_similar(
@@ -340,7 +363,8 @@ class DecisionEmbeddingPipeline:
candidate_embeddings["embeddings"],
candidate_embeddings["metadata"],
limit,
- filters
+ filters,
+ candidate_scores=candidate_embeddings.get("scores")
)
return similarities
@@ -759,27 +783,24 @@ class DecisionEmbeddingPipeline:
except Exception as e:
self.logger.warning(f"Failed to pre-generate structural embeddings: {e}")
- def _get_candidate_embeddings(self, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+ def _get_candidate_embeddings(
+ self, query_vector: Optional[np.ndarray] = None, limit: int = 10000, filters: Optional[Dict[str, Any]] = None
+ ) -> Dict[str, Any]:
"""Get candidate embeddings from vector store."""
if not self.vector_store:
- return {"embeddings": [], "metadata": []}
+ return {"embeddings": [], "metadata": [], "scores": []}
+
+ if query_vector is None:
+ query_vector = np.zeros(self.embedding_dimension)
+
+ results = self.vector_store.search_vectors(query_vector, k=limit, **(filters or {}))
- # Get all vectors from store
embeddings = []
metadata = []
+ scores = []
- for vector_id, vector in self.vector_store.vectors.items():
- vector_metadata = self.vector_store.metadata.get(vector_id, {})
-
- # Apply filters
- if filters:
- match = True
- for key, value in filters.items():
- if key not in vector_metadata or vector_metadata[key] != value:
- match = False
- break
- if not match:
- continue
+ for res in results:
+ vector_metadata = res.get("metadata", {})
# Extract structural embedding if available
struct_emb = vector_metadata.get("structural_embedding")
@@ -788,10 +809,15 @@ class DecisionEmbeddingPipeline:
else:
struct_emb = np.zeros(self.node_embedding_dimension)
+ vector = res.get("vector")
+ if vector is None:
+ vector = np.zeros(self.embedding_dimension)
+
embeddings.append((vector, struct_emb))
metadata.append(vector_metadata)
+ scores.append(res.get("score", 0.0))
- return {"embeddings": embeddings, "metadata": metadata}
+ return {"embeddings": embeddings, "metadata": metadata, "scores": scores}
def _find_semantic_similar(
self,
@@ -799,16 +825,22 @@ class DecisionEmbeddingPipeline:
candidate_embeddings: List[Tuple[np.ndarray, np.ndarray]],
candidate_metadata: List[Dict[str, Any]],
limit: int,
- filters: Optional[Dict[str, Any]] = None
+ filters: Optional[Dict[str, Any]] = None,
+ candidate_scores: Optional[List[float]] = None
) -> List[Dict[str, Any]]:
"""Find similar decisions using semantic similarity only."""
similarities = []
for i, (sem_emb, struct_emb) in enumerate(candidate_embeddings):
# Calculate semantic similarity
- similarity = self.hybrid_calculator._calculate_similarity(
- query_embedding, sem_emb, "cosine"
- )
+ if not np.any(sem_emb) and candidate_scores:
+ similarity = candidate_scores[i]
+ if similarity > 1.0:
+ similarity = 1.0 / (1.0 + similarity)
+ else:
+ similarity = self.hybrid_calculator._calculate_similarity(
+ query_embedding, sem_emb, "cosine"
+ )
similarities.append({
"similarity": similarity,
diff --git a/semantica/vector_store/decision_vector_methods.py b/semantica/vector_store/decision_vector_methods.py
index 550712b1..b38a0298 100644
--- a/semantica/vector_store/decision_vector_methods.py
+++ b/semantica/vector_store/decision_vector_methods.py
@@ -365,6 +365,19 @@ def get_decision_statistics(
outcome_counts = {}
confidence_values = []
+ # Not all backends expose internal metadata dictionaries (e.g. FAISS).
+ if not hasattr(store, "metadata"):
+ return {
+ "total_decisions": 0,
+ "categories": {},
+ "outcomes": {},
+ "average_confidence": 0.0,
+ "min_confidence": 0.0,
+ "max_confidence": 0.0,
+ "has_structural_embeddings": False,
+ "warning": "Statistics not supported for persistent backends without full metadata access."
+ }
+
for metadata in store.metadata.values():
# Count by category
category = metadata.get("category", "unknown")
diff --git a/tests/vector_store/test_decision_embedding_pipeline.py b/tests/vector_store/test_decision_embedding_pipeline.py
index 0572517c..9a1b5351 100644
--- a/tests/vector_store/test_decision_embedding_pipeline.py
+++ b/tests/vector_store/test_decision_embedding_pipeline.py
@@ -393,6 +393,53 @@ class TestDecisionEmbeddingPipelineEdgeCases:
assert all("decision_data" in result for result in results)
assert all("vector_id" in result for result in results)
+ def test_find_similar_decisions_real_faiss_backend(self):
+ """Regression test for FAISS backend crashing due to .vectors access (issue #839)"""
+ try:
+ from semantica.vector_store import VectorStore
+ except ImportError:
+ pytest.skip("VectorStore not available")
+
+ # Create a real VectorStore with FAISS backend
+ vs = VectorStore(backend="faiss", config={"dimension": 384})
+ vs.initialize_decision_pipeline()
+
+ # Store a couple of dummy decisions to ensure index has data
+ vs.store_decision(scenario="A test decision 1", category="test")
+ vs.store_decision(scenario="A test decision 2", category="test")
+
+ # This will call _get_candidate_embeddings without a mock
+ # Before the fix, this crashed with AttributeError: 'VectorStore' object has no attribute 'vectors'
+ results = vs.search_decisions("test query", limit=2)
+
+ # Assert it returns results and doesn't crash
+ assert isinstance(results, list)
+ assert len(results) <= 2
+
+
+ def test_find_similar_decisions_real_inmemory_backend(self):
+ """Regression test for inmemory backend preserving behavior after issue #839 fix"""
+ try:
+ from semantica.vector_store import VectorStore
+ except ImportError:
+ pytest.skip("VectorStore not available")
+
+ vs = VectorStore(backend="inmemory", config={"dimension": 384})
+ vs.initialize_decision_pipeline()
+
+ vs.store_decision(scenario="Apple product launch", category="tech")
+ vs.store_decision(scenario="Microsoft earnings report", category="tech")
+ vs.store_decision(scenario="Local bakery opens", category="food")
+
+ results = vs.search_decisions("software company", limit=2)
+
+ assert isinstance(results, list)
+ assert len(results) <= 2
+
+ if len(results) > 0:
+ assert "similarity" in results[0]
+ assert "semantic_similarity" in results[0]
+ assert "structural_similarity" in results[0]
if __name__ == "__main__":
pytest.main([__file__])