💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Move vector normalization to vectors module, closes #584
davidmezzettibbbb4a20e2e4
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 92b02968..c8f4d5cb 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -26,7 +26,7 @@ from .search import Explain, Query, Search, Terms# pylint: disable=C0302,R0904class Embeddings:"""- Embeddings is the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts+ Embeddings databases are the engine that delivers semantic search. Data is transformed into embeddings vectors where similar conceptswill produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used to find resultsthat have the same meaning, not necessarily the same keywords."""@@ -121,9 +121,6 @@ class Embeddings:self.reducer = Reducer(embeddings, self.config["pca"])self.reducer(embeddings)- # Normalize embeddings- self.normalize(embeddings)-# Save index dimensionsself.config["dimensions"] = dimensions@@ -177,9 +174,6 @@ class Embeddings:if self.reducer:self.reducer(embeddings)- # Normalize embeddings- self.normalize(embeddings)-# Append embeddings to the indexself.ann.append(embeddings)@@ -328,9 +322,6 @@ class Embeddings:if self.reducer:self.reducer(embeddings)- # Normalize embeddings- self.normalize(embeddings)-return embeddingsdef count(self):@@ -1035,17 +1026,3 @@ class Embeddings:config["columns"] = self.config["columns"]return config-- def normalize(self, embeddings):- """- Normalizes embeddings using L2 normalization. Operation applied directly on array.-- Args:- embeddings: input embeddings matrix- """-- # Calculation is different for matrices vs vectors- if len(embeddings.shape) > 1:- embeddings /= np.linalg.norm(embeddings, axis=1)[:, np.newaxis]- else:- embeddings /= np.linalg.norm(embeddings)