Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Move vector normalization to vectors module, closes #584

davidmezzettibbbb4a20e2e4
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index 92b02968..c8f4d5cb 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -26,7 +26,7 @@ from .search import Explain, Query, Search, Terms
# pylint: disable=C0302,R0904
class Embeddings:
"""
- Embeddings is the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts
+ Embeddings databases are the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts
will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used to find results
that have the same meaning, not necessarily the same keywords.
"""
@@ -121,9 +121,6 @@ class Embeddings:
self.reducer = Reducer(embeddings, self.config["pca"])
self.reducer(embeddings)
- # Normalize embeddings
- self.normalize(embeddings)
-
# Save index dimensions
self.config["dimensions"] = dimensions
@@ -177,9 +174,6 @@ class Embeddings:
if self.reducer:
self.reducer(embeddings)
- # Normalize embeddings
- self.normalize(embeddings)
-
# Append embeddings to the index
self.ann.append(embeddings)
@@ -328,9 +322,6 @@ class Embeddings:
if self.reducer:
self.reducer(embeddings)
- # Normalize embeddings
- self.normalize(embeddings)
-
return embeddings
def count(self):
@@ -1035,17 +1026,3 @@ class Embeddings:
config["columns"] = self.config["columns"]
return config
-
- def normalize(self, embeddings):
- """
- Normalizes embeddings using L2 normalization. Operation applied directly on array.
-
- Args:
- embeddings: input embeddings matrix
- """
-
- # Calculation is different for matrices vs vectors
- if len(embeddings.shape) > 1:
- embeddings /= np.linalg.norm(embeddings, axis=1)[:, np.newaxis]
- else:
- embeddings /= np.linalg.norm(embeddings)