💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Refactor API code into smaller components #63
davidmezzetti975ba144df11
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.pyindex d5fc9c22..943b5352 100644--- a/src/python/txtai/api/base.py+++ b/src/python/txtai/api/base.py@@ -2,11 +2,7 @@API module"""-import os-import pickle-import tempfile--from ..embeddings import Embeddings+from ..embeddings import Documents, Embeddingsfrom ..pipeline import Extractor, Labels, Similarity@@ -24,7 +20,7 @@ class API:"""# Initialize member variables- self.config, self.documents, self.batch = config, None, 0+ self.config, self.documents = config, Noneself.embeddings, self.extractor, self.labels, self.similar = None, None, None, None# Create/load embeddings index depending on writable flag@@ -127,26 +123,10 @@ class API:if self.embeddings and self.config.get("writable"):# Create documents file if not already openif not self.documents:- self.documents = tempfile.NamedTemporaryFile(mode="wb", suffix=".docs", delete=False)+ self.documents = Documents()# Add batch- pickle.dump([(document["id"], document["text"], None) for document in documents], self.documents)- self.batch += 1-- def stream(self):- """- Generator that streams documents previously queued with add.- """-- # Open stream file- with open(self.documents.name, "rb") as queue:- # Read each batch- for _ in range(self.batch):- documents = pickle.load(queue)-- # Yield each document- for document in documents:- yield document+ documents.add([(document["id"], document["text"], None) for document in documents])def index(self):"""@@ -155,25 +135,19 @@ class API:"""if self.embeddings and self.config.get("writable") and self.documents:- # Close streaming file- self.documents.close()-# Build scoring index if scoring method providedif self.config.get("scoring"):- self.embeddings.score(self.stream())+ self.embeddings.score(self.documents)# Build embeddings index- self.embeddings.index(self.stream())+ self.embeddings.index(self.documents)# Save indexself.embeddings.save(self.config["path"])- # Cleanup stream file- os.remove(self.documents.name)-- # Reset document parameters+ # Reset document stream+ self.documents.close()self.documents = None- self.batch = 0def similarity(self, query, texts):"""diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.pynew file mode 100644index 00000000..b70d4ba7--- /dev/null+++ b/src/python/txtai/embeddings/__init__.py@@ -0,0 +1,6 @@+"""+Embeddings imports+"""++from .base import Embeddings+from .documents import Documentsdiff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings/base.pysimilarity index 99%rename from src/python/txtai/embeddings.pyrename to src/python/txtai/embeddings/base.pyindex 8588de9d..17e00244 100644--- a/src/python/txtai/embeddings.py+++ b/src/python/txtai/embeddings/base.py@@ -10,9 +10,9 @@ import numpy as npfrom sklearn.decomposition import TruncatedSVD-from .ann import ANNFactory-from .scoring import ScoringFactory-from .vectors import VectorsFactory+from ..ann import ANNFactory+from ..scoring import ScoringFactory+from ..vectors import VectorsFactoryclass Embeddings:diff --git a/src/python/txtai/embeddings/documents.py b/src/python/txtai/embeddings/documents.pynew file mode 100644index 00000000..3e04da1f--- /dev/null+++ b/src/python/txtai/embeddings/documents.py@@ -0,0 +1,71 @@+"""+Stream module+"""++import os+import pickle+import tempfile+++class Documents:+ """+ Streams documents to temporary storage. Allows queuing large volumes of content for later indexing.+ """++ def __init__(self):+ """+ Creates a new DocumentStream.+ """++ self.documents = None+ self.batch = 0++ def __iter__(self):+ """+ Streams all queued documents.+ """++ # Close streaming file+ self.documents.close()++ # Open stream file+ with open(self.documents.name, "rb") as queue:+ # Read each batch+ for _ in range(self.batch):+ documents = pickle.load(queue)++ # Yield each document+ yield from documents++ def add(self, documents):+ """+ Adds a batch of documents for indexing.++ Args:+ documents: list of (id, data, tag) tuples++ Returns:+ documents+ """++ # Create documents file if not already open+ if not self.documents:+ self.documents = tempfile.NamedTemporaryFile(mode="wb", suffix=".docs", delete=False)++ # Add batch+ pickle.dump(documents, self.documents)+ self.batch += 1++ return documents++ def close(self):+ """+ Closes and resets this instance. New sets of documents can be added with additional calls to add.+ """++ # Cleanup stream file+ os.remove(self.documents.name)++ # Reset document parameters+ self.documents = None+ self.batch = 0