Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Refactor API code into smaller components #63

davidmezzetti975ba144df11
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index d5fc9c22..943b5352 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -2,11 +2,7 @@
API module
"""
-import os
-import pickle
-import tempfile
-
-from ..embeddings import Embeddings
+from ..embeddings import Documents, Embeddings
from ..pipeline import Extractor, Labels, Similarity
@@ -24,7 +20,7 @@ class API:
"""
# Initialize member variables
- self.config, self.documents, self.batch = config, None, 0
+ self.config, self.documents = config, None
self.embeddings, self.extractor, self.labels, self.similar = None, None, None, None
# Create/load embeddings index depending on writable flag
@@ -127,26 +123,10 @@ class API:
if self.embeddings and self.config.get("writable"):
# Create documents file if not already open
if not self.documents:
- self.documents = tempfile.NamedTemporaryFile(mode="wb", suffix=".docs", delete=False)
+ self.documents = Documents()
# Add batch
- pickle.dump([(document["id"], document["text"], None) for document in documents], self.documents)
- self.batch += 1
-
- def stream(self):
- """
- Generator that streams documents previously queued with add.
- """
-
- # Open stream file
- with open(self.documents.name, "rb") as queue:
- # Read each batch
- for _ in range(self.batch):
- documents = pickle.load(queue)
-
- # Yield each document
- for document in documents:
- yield document
+ documents.add([(document["id"], document["text"], None) for document in documents])
def index(self):
"""
@@ -155,25 +135,19 @@ class API:
"""
if self.embeddings and self.config.get("writable") and self.documents:
- # Close streaming file
- self.documents.close()
-
# Build scoring index if scoring method provided
if self.config.get("scoring"):
- self.embeddings.score(self.stream())
+ self.embeddings.score(self.documents)
# Build embeddings index
- self.embeddings.index(self.stream())
+ self.embeddings.index(self.documents)
# Save index
self.embeddings.save(self.config["path"])
- # Cleanup stream file
- os.remove(self.documents.name)
-
- # Reset document parameters
+ # Reset document stream
+ self.documents.close()
self.documents = None
- self.batch = 0
def similarity(self, query, texts):
"""
diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.py
new file mode 100644
index 00000000..b70d4ba7
--- /dev/null
+++ b/src/python/txtai/embeddings/__init__.py
@@ -0,0 +1,6 @@
+"""
+Embeddings imports
+"""
+
+from .base import Embeddings
+from .documents import Documents
diff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings/base.py
similarity index 99%
rename from src/python/txtai/embeddings.py
rename to src/python/txtai/embeddings/base.py
index 8588de9d..17e00244 100644
--- a/src/python/txtai/embeddings.py
+++ b/src/python/txtai/embeddings/base.py
@@ -10,9 +10,9 @@ import numpy as np
from sklearn.decomposition import TruncatedSVD
-from .ann import ANNFactory
-from .scoring import ScoringFactory
-from .vectors import VectorsFactory
+from ..ann import ANNFactory
+from ..scoring import ScoringFactory
+from ..vectors import VectorsFactory
class Embeddings:
diff --git a/src/python/txtai/embeddings/documents.py b/src/python/txtai/embeddings/documents.py
new file mode 100644
index 00000000..3e04da1f
--- /dev/null
+++ b/src/python/txtai/embeddings/documents.py
@@ -0,0 +1,71 @@
+"""
+Stream module
+"""
+
+import os
+import pickle
+import tempfile
+
+
+class Documents:
+ """
+ Streams documents to temporary storage. Allows queuing large volumes of content for later indexing.
+ """
+
+ def __init__(self):
+ """
+ Creates a new DocumentStream.
+ """
+
+ self.documents = None
+ self.batch = 0
+
+ def __iter__(self):
+ """
+ Streams all queued documents.
+ """
+
+ # Close streaming file
+ self.documents.close()
+
+ # Open stream file
+ with open(self.documents.name, "rb") as queue:
+ # Read each batch
+ for _ in range(self.batch):
+ documents = pickle.load(queue)
+
+ # Yield each document
+ yield from documents
+
+ def add(self, documents):
+ """
+ Adds a batch of documents for indexing.
+
+ Args:
+ documents: list of (id, data, tag) tuples
+
+ Returns:
+ documents
+ """
+
+ # Create documents file if not already open
+ if not self.documents:
+ self.documents = tempfile.NamedTemporaryFile(mode="wb", suffix=".docs", delete=False)
+
+ # Add batch
+ pickle.dump(documents, self.documents)
+ self.batch += 1
+
+ return documents
+
+ def close(self):
+ """
+ Closes and resets this instance. New sets of documents can be added with additional calls to add.
+ """
+
+ # Cleanup stream file
+ os.remove(self.documents.name)
+
+ # Reset document parameters
+ self.documents = None
+ self.batch = 0