Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Refactor code into smaller modules #63

davidmezzettie6b2db47c41f
diff --git a/src/python/txtai/ann.py b/src/python/txtai/ann.py
deleted file mode 100644
index eb473c2c..00000000
--- a/src/python/txtai/ann.py
+++ /dev/null
@@ -1,258 +0,0 @@
-"""
-ANN (Approximate Nearest Neighbors) module
-"""
-
-import numpy as np
-
-# pylint: disable=E0611
-from annoy import AnnoyIndex
-from hnswlib import Index
-
-# Conditionally import Faiss as it's only supported on Linux/macOS
-try:
- import faiss
-
- FAISS = True
-except ImportError:
- FAISS = False
-
-
-class ANN:
- """
- Base class for ANN models.
- """
-
- @staticmethod
- def create(config):
- """
- Create an ANN model.
-
- Args:
- config: index configuration parameters
-
- Returns:
- ANN
- """
-
- # ANN model
- model = None
- backend = config.get("backend")
-
- # Default backend if not provided, based on available libraries
- if not backend:
- backend = "faiss" if FAISS else "annoy"
-
- # Create ANN instance
- if backend == "annoy":
- model = Annoy(config)
- elif backend == "hnsw":
- model = HNSW(config)
- else:
- # Raise error if trying to create a Faiss index without Faiss installed
- if not FAISS:
- raise ImportError("Faiss library is not installed")
-
- model = Faiss(config)
-
- # Store config back
- config["backend"] = backend
-
- return model
-
- def __init__(self, config):
- """
- Creates a new ANN model.
- """
-
- # ANN index
- self.model = None
-
- # Model configuration
- self.config = config
-
- def load(self, path):
- """
- Loads an ANN model at path.
- """
-
- def index(self, embeddings):
- """
- Builds an ANN model.
-
- Args:
- embeddings: embeddings array
- """
-
- def search(self, queries, limit):
- """
- Searches ANN model for query. Returns topn results.
-
- Args:
- queries: queries array
- limit: maximum results
-
- Returns:
- query results
- """
-
- def save(self, path):
- """
- Saves an ANN model at path.
- """
-
- def setting(self, name, default=None):
- """
- Looks up backend specific setting.
-
- Args:
- name: setting name
- default: default value when setting not found
-
- Returns:
- setting value
- """
-
- # Get the backend-specific config object
- backend = self.config.get(self.config["backend"])
-
- # Get setting value, set default value if not found
- setting = backend.get(name) if backend else None
- return setting if setting else default
-
-
-class Annoy(ANN):
- """
- Builds an ANN model using the Annoy library.
- """
-
- def load(self, path):
- # Load index
- self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])
- self.model.load(path)
-
- def index(self, embeddings):
- # Inner product is equal to cosine similarity on normalized vectors
- self.config["metric"] = "dot"
-
- # Create index
- self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])
-
- # Add items
- for x in range(embeddings.shape[0]):
- self.model.add_item(x, embeddings[x])
-
- # Build index
- self.model.build(self.setting("ntrees", 10))
-
- def search(self, queries, limit):
- # Lookup search k setting
- searchk = self.setting("searchk", -1)
-
- # Annoy doesn't have a built in batch query method
- results = []
- for query in queries:
- # Run the query
- ids, scores = self.model.get_nns_by_vector(query, n=limit, search_k=searchk, include_distances=True)
-
- # Map results to [(id, score)]
- results.append(list(zip(ids, scores)))
-
- return results
-
- def save(self, path):
- # Write index
- self.model.save(path)
-
-
-class Faiss(ANN):
- """
- Builds an ANN model using the Faiss library.
- """
-
- def load(self, path):
- # Load index
- self.model = faiss.read_index(path)
-
- def index(self, embeddings):
- # Lookup components setting
- components = self.setting("components")
-
- # Create embeddings index. Inner product is equal to cosine similarity on normalized vectors.
- if components:
- params = components
- elif self.config.get("quantize"):
- params = "IVF100,SQ8" if embeddings.shape[0] >= 5000 else "IDMap,SQ8"
- else:
- params = "IVF100,Flat" if embeddings.shape[0] >= 5000 else "IDMap,Flat"
-
- self.model = faiss.index_factory(embeddings.shape[1], params, faiss.METRIC_INNER_PRODUCT)
-
- # Train model
- self.model.train(embeddings)
- self.model.add_with_ids(embeddings, np.array(range(embeddings.shape[0])))
-
- def search(self, queries, limit):
- # Run the query
- self.model.nprobe = self.setting("nprobe", 6)
- scores, ids = self.model.search(queries, limit)
-
- # Map results to [(id, score)]
- results = []
- for x, score in enumerate(scores):
- results.append(list(zip(ids[x].tolist(), score.tolist())))
-
- return results
-
- def save(self, path):
- # Write index
- faiss.write_index(self.model, path)
-
-
-class HNSW(ANN):
- """
- Builds an ANN model using the hnswlib library.
- """
-
- def load(self, path):
- # Load index
- self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])
- self.model.load_index(path)
-
- def index(self, embeddings):
- # Inner product is equal to cosine similarity on normalized vectors
- self.config["metric"] = "ip"
-
- # Lookup index settings
- efconstruction = self.setting("efconstruction", 200)
- m = self.setting("m", 16)
- seed = self.setting("randomseed", 100)
-
- # Create index
- self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])
- self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)
-
- # Add items
- self.model.add_items(embeddings, np.array(range(embeddings.shape[0])))
-
- def search(self, queries, limit):
- # Set ef query param
- ef = self.setting("efsearch")
- if ef:
- self.model.set_ef(ef)
-
- # Run the query
- ids, distances = self.model.knn_query(queries, k=limit)
-
- # Map results to [(id, score)]
- results = []
- for x, distance in enumerate(distances):
- # Convert distances to similarity scores
- scores = [1 - d for d in distance]
-
- results.append(list(zip(ids[x], scores)))
-
- return results
-
- def save(self, path):
- # Write index
- self.model.save_index(path)
diff --git a/src/python/txtai/ann/__init__.py b/src/python/txtai/ann/__init__.py
new file mode 100644
index 00000000..880534d2
--- /dev/null
+++ b/src/python/txtai/ann/__init__.py
@@ -0,0 +1,9 @@
+"""
+ANN imports
+"""
+
+from .annoy import Annoy
+from .base import ANN
+from .factory import ANNFactory
+from .faiss import Faiss
+from .hnsw import HNSW
diff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/annoy.py
new file mode 100644
index 00000000..28540f95
--- /dev/null
+++ b/src/python/txtai/ann/annoy.py
@@ -0,0 +1,51 @@
+"""
+Annoy module
+"""
+
+from annoy import AnnoyIndex
+
+from .base import ANN
+
+
+class Annoy(ANN):
+ """
+ Builds an ANN model using the Annoy library.
+ """
+
+ def load(self, path):
+ # Load index
+ self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])
+ self.model.load(path)
+
+ def index(self, embeddings):
+ # Inner product is equal to cosine similarity on normalized vectors
+ self.config["metric"] = "dot"
+
+ # Create index
+ self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])
+
+ # Add items
+ for x in range(embeddings.shape[0]):
+ self.model.add_item(x, embeddings[x])
+
+ # Build index
+ self.model.build(self.setting("ntrees", 10))
+
+ def search(self, queries, limit):
+ # Lookup search k setting
+ searchk = self.setting("searchk", -1)
+
+ # Annoy doesn't have a built in batch query method
+ results = []
+ for query in queries:
+ # Run the query
+ ids, scores = self.model.get_nns_by_vector(query, n=limit, search_k=searchk, include_distances=True)
+
+ # Map results to [(id, score)]
+ results.append(list(zip(ids, scores)))
+
+ return results
+
+ def save(self, path):
+ # Write index
+ self.model.save(path)
diff --git a/src/python/txtai/ann/base.py b/src/python/txtai/ann/base.py
new file mode 100644
index 00000000..8386b68f
--- /dev/null
+++ b/src/python/txtai/ann/base.py
@@ -0,0 +1,69 @@
+"""
+ANN (Approximate Nearest Neighbors) module
+"""
+
+
+class ANN:
+ """
+ Base class for ANN models.
+ """
+
+ def __init__(self, config):
+ """
+ Creates a new ANN model.
+ """
+
+ # ANN index
+ self.model = None
+
+ # Model configuration
+ self.config = config
+
+ def load(self, path):
+ """
+ Loads an ANN model at path.
+ """
+
+ def index(self, embeddings):
+ """
+ Builds an ANN model.
+
+ Args:
+ embeddings: embeddings array
+ """
+
+ def search(self, queries, limit):
+ """
+ Searches ANN model for query. Returns topn results.
+
+ Args:
+ queries: queries array
+ limit: maximum results
+
+ Returns:
+ query results
+ """
+
+ def save(self, path):
+ """
+ Saves an ANN model at path.
+ """
+
+ def setting(self, name, default=None):
+ """
+ Looks up backend specific setting.
+
+ Args:
+ name: setting name
+ default: default value when setting not found
+
+ Returns:
+ setting value
+ """
+
+ # Get the backend-specific config object
+ backend = self.config.get(self.config["backend"])
+
+ # Get setting value, set default value if not found
+ setting = backend.get(name) if backend else None
+ return setting if setting else default
diff --git a/src/python/txtai/ann/factory.py b/src/python/txtai/ann/factory.py
new file mode 100644
index 00000000..9ddb3d0d
--- /dev/null
+++ b/src/python/txtai/ann/factory.py
@@ -0,0 +1,57 @@
+"""
+Factory module
+"""
+
+from .annoy import Annoy
+from .hnsw import HNSW
+
+# Conditionally import Faiss as it's only supported on Linux/macOS
+try:
+ from .faiss import Faiss
+
+ FAISS = True
+except ImportError:
+ FAISS = False
+
+
+class ANNFactory:
+ """
+ Methods to create ANN models.
+ """
+
+ @staticmethod
+ def create(config):
+ """
+ Create an ANN model.
+
+ Args:
+ config: index configuration parameters
+
+ Returns:
+ ANN
+ """
+
+ # ANN model
+ model = None
+ backend = config.get("backend")
+
+ # Default backend if not provided, based on available libraries
+ if not backend:
+ backend = "faiss" if FAISS else "annoy"
+
+ # Create ANN instance
+ if backend == "annoy":
+ model = Annoy(config)
+ elif backend == "hnsw":
+ model = HNSW(config)
+ else:
+ # Raise error if trying to create a Faiss index without Faiss installed
+ if not FAISS:
+ raise ImportError("Faiss library is not installed")
+
+ model = Faiss(config)
+
+ # Store config back
+ config["backend"] = backend
+
+ return model
diff --git a/src/python/txtai/ann/faiss.py b/src/python/txtai/ann/faiss.py
new file mode 100644
index 00000000..714f111c
--- /dev/null
+++ b/src/python/txtai/ann/faiss.py
@@ -0,0 +1,59 @@
+"""
+Faiss module
+"""
+
+import numpy as np
+
+from .base import ANN
+
+# Conditionally import Faiss as it's only supported on Linux/macOS
+try:
+ from faiss import index_factory, METRIC_INNER_PRODUCT, read_index, write_index
+
+ FAISS = True
+except ImportError:
+ FAISS = False
+
+
+class Faiss(ANN):
+ """
+ Builds an ANN model using the Faiss library.
+ """
+
+ def load(self, path):
+ # Load index
+ self.model = read_index(path)
+
+ def index(self, embeddings):
+ # Lookup components setting
+ components = self.setting("components")
+
+ # Create embeddings index. Inner product is equal to cosine similarity on normalized vectors.
+ if components:
+ params = components
+ elif self.config.get("quantize"):
+ params = "IVF100,SQ8" if embeddings.shape[0] >= 5000 else "IDMap,SQ8"
+ else:
+ params = "IVF100,Flat" if embeddings.shape[0] >= 5000 else "IDMap,Flat"
+
+ self.model = index_factory(embeddings.shape[1], params, METRIC_INNER_PRODUCT)
+
+ # Train model
+ self.model.train(embeddings)
+ self.model.add_with_ids(embeddings, np.array(range(embeddings.shape[0])))
+
+ def search(self, queries, limit):
+ # Run the query
+ self.model.nprobe = self.setting("nprobe", 6)
+ scores, ids = self.model.search(queries, limit)
+
+ # Map results to [(id, score)]
+ results = []
+ for x, score in enumerate(scores):
+ results.append(list(zip(ids[x].tolist(), score.tolist())))
+
+ return results
+
+ def save(self, path):
+ # Write index
+ write_index(self.model, path)
diff --git a/src/python/txtai/ann/hnsw.py b/src/python/txtai/ann/hnsw.py
new file mode 100644
index 00000000..2754c118
--- /dev/null
+++ b/src/python/txtai/ann/hnsw.py
@@ -0,0 +1,60 @@
+"""
+HNSW module
+"""
+
+import numpy as np
+
+# pylint: disable=E0611
+from hnswlib import Index
+
+from .base import ANN
+
+
+class HNSW(ANN):
+ """
+ Builds an ANN model using the hnswlib library.
+ """
+
+ def load(self, path):
+ # Load index
+ self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])
+ self.model.load_index(path)
+
+ def index(self, embeddings):
+ # Inner product is equal to cosine similarity on normalized vectors
+ self.config["metric"] = "ip"
+
+ # Lookup index settings
+ efconstruction = self.setting("efconstruction", 200)
+ m = self.setting("m", 16)
+ seed = self.setting("randomseed", 100)
+
+ # Create index
+ self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])
+ self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)
+
+ # Add items
+ self.model.add_items(embeddings, np.array(range(embeddings.shape[0])))
+
+ def search(self, queries, limit):
+ # Set ef query param
+ ef = self.setting("efsearch")
+ if ef:
+ self.model.set_ef(ef)
+
+ # Run the query
+ ids, distances = self.model.knn_query(queries, k=limit)
+
+ # Map results to [(id, score)]
+ results = []
+ for x, distance in enumerate(distances):
+ # Convert distances to similarity scores
+ scores = [1 - d for d in distance]
+
+ results.append(list(zip(ids[x], scores)))
+
+ return results
+
+ def save(self, path):
+ # Write index
+ self.model.save_index(path)
diff --git a/src/python/txtai/api.py b/src/python/txtai/api.py
index d03f71f8..a3bd7a24 100644
--- a/src/python/txtai/api.py
+++ b/src/python/txtai/api.py
@@ -13,8 +13,7 @@ import yaml
from fastapi import Body, FastAPI, Request
from .embeddings import Embeddings
-from .extractor import Extractor
-from .pipeline import Labels, Similarity
+from .pipeline import Extractor, Labels, Similarity
# API instance
app = FastAPI()
diff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings.py
index 8efffc2d..8588de9d 100644
--- a/src/python/txtai/embeddings.py
+++ b/src/python/txtai/embeddings.py
@@ -10,9 +10,9 @@ import numpy as np
from sklearn.decomposition import TruncatedSVD
-from .ann import ANN
-from .scoring import Scoring
-from .vectors import Vectors
+from .ann import ANNFactory
+from .scoring import ScoringFactory
+from .vectors import VectorsFactory
class Embeddings:
@@ -44,7 +44,7 @@ class Embeddings:
self.lsa = None
# Embedding scoring method - weighs each word in a sentence
- self.scoring = Scoring.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None
+ self.scoring = ScoringFactory.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None
# Sentence vectors model
self.model = self.loadVectors() if self.config else None
@@ -57,7 +57,7 @@ class Embeddings:
vector model
"""
- return Vectors.create(self.config, self.scoring)
+ return VectorsFactory.create(self.config, self.scoring)
def score(self, documents):
"""
@@ -104,7 +104,7 @@ class Embeddings:
self.config["dimensions"] = dimensions
# Create embeddings index
- self.embeddings = ANN.create(self.config)
+ self.embeddings = ANNFactory.create(self.config)
# Build the index
self.embeddings.index(embeddings)
@@ -307,7 +307,7 @@ class Embeddings:
self.config["path"] = os.path.join(path, self.config["path"])
# Sentence embeddings index
- self.embeddings = ANN.create(self.config)
+ self.embeddings = ANNFactory.create(self.config)
self.embeddings.load("%s/embeddings" % path)
# Dimensionality reduction
@@ -317,7 +317,7 @@ class Embeddings:
# Embedding scoring
if self.config.get("scoring"):
- self.scoring = Scoring.create(self.config["scoring"])
+ self.scoring = ScoringFactory.create(self.config["scoring"])
self.scoring.load(path)
# Sentence vectors model - transforms text into sentence embeddings
diff --git a/src/python/txtai/extractor/__init__.py b/src/python/txtai/extractor/__init__.py
new file mode 100644
index 00000000..e83805a6
--- /dev/null
+++ b/src/python/txtai/extractor/__init__.py
@@ -0,0 +1,5 @@
+"""
+Defined to maintain backwards compatibility
+"""
+
+from ..pipeline.extractor import Extractor
diff --git a/src/python/txtai/pipeline.py b/src/python/txtai/pipeline.py
deleted file mode 100644
index 535698bf..00000000
--- a/src/python/txtai/pipeline.py
+++ /dev/null
@@ -1,162 +0,0 @@
-"""
-Pipeline module
-"""
-
-import numpy as np
-import torch
-
-from transformers import pipeline
-
-
-class Pipeline:
- """
- Light wrapper around Hugging Face's pipeline component for selected tasks. Adds support for model
- quantization and minor interface changes.
- """
-
- def __init__(self, task, path=None, quantize=False, gpu=False, model=None):
- """
- Loads a new pipeline model.
-
- Args:
- task: pipeline task or category
- path: optional path to model, accepts Hugging Face model hub id or local path,
- uses default model for task if not provided
- quantize: if model should be quantized, defaults to False
- gpu: if gpu inference should be used (only works if GPUs are available)
- model: optional existing pipeline model to wrap
- """
-
- if model:
- # Check if input model is a Pipeline or a HF pipeline
- self.pipeline = model.pipeline if isinstance(model, Pipeline) else model
- else:
- # Enable GPU inference if explicitly set and a GPU is available
- gpu = gpu and torch.cuda.is_available()
-
- # Transformer pipeline task
- self.pipeline = pipeline(task, model=path, tokenizer=path, device=0 if gpu else -1)
-
- # Model quantization. Compresses model to int8 precision, improves runtime performance. Only supported on CPU.
- if not gpu and quantize:
- # pylint: disable=E1101
- self.pipeline.model = torch.quantization.quantize_dynamic(self.pipeline.model, {torch.nn.Linear}, dtype=torch.qint8)
-
-
-class Questions(Pipeline):
- """
- Runs extractive QA for a series of questions and contexts.
- """
-
- def __init__(self, path=None, quantize=False, gpu=False, model=None):
- super().__init__("question-answering", path, quantize, gpu, model)
-
- def __call__(self, questions, contexts):
- """
- Runs a extractive question-answering model against each question-context pair, finding the best answers.
-
- Args:
- questions: list of questions
- contexts: list of contexts to pull answers from
-
- Returns:
- list of answers
- """
-
- answers = []
-
- for x, question in enumerate(questions):
- if question and contexts[x]:
- # Run the QA pipeline
- result = self.pipeline(question=question, context=contexts[x])
-
- # Get answer and score
- answer, score = result["answer"], result["score"]
-
- # Require score to be at least 0.05
- if score < 0.05:
- answer = None
-
- # Add answer
- answers.append(answer)
- else:
- answers.append(None)
-
- return answers
-
-
-class Labels(Pipeline):
- """
- Applies a zero shot classifier to text using a list of labels.
- """
-
- def __init__(self, path=None, quantize=False, gpu=True, model=None):
- super().__init__("zero-shot-classification", path, quantize, gpu, model)
-
- def __call__(self, text, labels, multiclass=False):
- """
- Applies a zero shot classifier to text using a list of labels. Returns a list of
- (id, score) sorted by highest score, where id is the index in labels.
-
- This method supports text as a string or a list. If the input is a string, the return
- type is a 1D list of (id, score). If text is a list, a 2D list of (id, score) is
- returned with a row per string.
-
- Args:
- text: text|list
- labels: list of labels
-
- Returns:
- list of (id, score)
- """
-
- # Run ZSL pipeline
- results = self.pipeline(text, labels, multi_class=multiclass)
-
- # Convert results to a list if necessary
- if not isinstance(results, list):
- results = [results]
-
- # Build list of (id, score)
- scores = []
- for result in results:
- scores.append([(labels.index(label), result["scores"][x]) for x, label in enumerate(result["labels"])])
-
- return scores[0] if isinstance(text, str) else scores
-
-
-class Similarity(Labels):
- """
- Computes similarity between query and list of text using a zero shot classifier.
- """
-
- def __call__(self, query, texts, multiclass=True):
- """
- Computes the similarity between query and list of text. Returns a list of
- (id, score) sorted by highest score, where id is the index in texts.
-
- This method supports query as a string or a list. If the input is a string,
- the return type is a 1D list of (id, score). If text is a list, a 2D list
- of (id, score) is returned with a row per string.
-
- Args:
- query: query text|list
- texts: list of texts
-
- Returns:
- list of (id, score)
- """
-
- # Call Labels pipeline for texts using input query as the candidate label
- scores = super().__call__(texts, [query] if isinstance(query, str) else query, multiclass)
-
- # Sort on query index id
- scores = [[score for _, score in sorted(row)] for row in scores]
-
- # Transpose axes to get a list of text scores for each query
- scores = np.array(scores).T.tolist()
-
- # Build list of (id, score) per query sorted by highest score
- scores = [sorted(enumerate(row), key=lambda x: x[1], reverse=True) for row in scores]
-
- return scores[0] if isinstance(query, str) else scores
diff --git a/src/python/txtai/pipeline/__init__.py b/src/python/txtai/pipeline/__init__.py
new file mode 100644
index 00000000..78010353
--- /dev/null
+++ b/src/python/txtai/pipeline/__init__.py
@@ -0,0 +1,11 @@
+"""
+Pipeline imports
+"""
+
+from .base import Pipeline
+from .extractor import Extractor
+from .hfpipeline import HFPipeline
+from .labels import Labels
+from .questions import Questions
+from .similarity import Similarity
+from .tokenizer import Tokenizer
diff --git a/src/python/txtai/pipeline/base.py b/src/python/txtai/pipeline/base.py
new file mode 100644
index 00000000..6fad9358
--- /dev/null
+++ b/src/python/txtai/pipeline/base.py
@@ -0,0 +1,9 @@
+"""
+Pipeline module
+"""
+
+
+class Pipeline:
+ """
+ Base class for all Pipelines. The only interface requirement is to define a __call___ method.
+ """
diff --git a/src/python/txtai/extractor.py b/src/python/txtai/pipeline/extractor.py
similarity index 98%
rename from src/python/txtai/extractor.py
rename to src/python/txtai/pipeline/extractor.py
index 98a7bf49..40bc7fc8 100644
--- a/src/python/txtai/extractor.py
+++ b/src/python/txtai/pipeline/extractor.py
@@ -4,11 +4,12 @@ Extractor module
from nltk.tokenize import sent_tokenize
-from .pipeline import Questions
+from .base import Pipeline
+from .questions import Questions
from .tokenizer import Tokenizer
-class Extractor:
+class Extractor(Pipeline):
"""
Class that uses an extractive question-answering model to extract content from a given text context.
"""
diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.py
new file mode 100644
index 00000000..c701d3d3
--- /dev/null
+++ b/src/python/txtai/pipeline/hfpipeline.py
@@ -0,0 +1,44 @@
+"""
+Hugging Face pipeline wrapper module
+"""
+
+import torch
+
+from transformers import pipeline
+
+from .base import Pipeline
+
+
+class HFPipeline(Pipeline):
+ """
+ Light wrapper around Hugging Face's pipeline component for selected tasks. Adds support for model
+ quantization and minor interface changes.
+ """
+
+ def __init__(self, task, path=None, quantize=False, gpu=False, model=None):
+ """
+ Loads a new pipeline model.
+
+ Args:
+ task: pipeline task or category
+ path: optional path to model, accepts Hugging Face model hub id or local path,
+ uses default model for task if not provided
+ quantize: if model should be quantized, defaults to False
+ gpu: if gpu inference should be used (only works if GPUs are available)
+ model: optional existing pipeline model to wrap
+ """
+
+ if model:
+ # Check if input model is a Pipeline or a HF pipeline
+ self.pipeline = model.pipeline if isinstance(model, Pipeline) else model
+ else:
+ # Enable GPU inference if explicitly set and a GPU is available
+ gpu = gpu and torch.cuda.is_available()
+
+ # Transformer pipeline task
+ self.pipeline = pipeline(task, model=path, tokenizer=path, device=0 if gpu else -1)
+
+ # Model quantization. Compresses model to int8 precision, improves runtime performance. Only supported on CPU.
+ if not gpu and quantize:
+ # pylint: disable=E1101
+ self.pipeline.model = torch.quantization.quantize_dynamic(self.pipeline.model, {torch.nn.Linear}, dtype=torch.qint8)
diff --git a/src/python/txtai/pipeline/labels.py b/src/python/txtai/pipeline/labels.py
new file mode 100644
index 00000000..ef657c9e
--- /dev/null
+++ b/src/python/txtai/pipeline/labels.py
@@ -0,0 +1,45 @@
+"""
+Labels module
+"""
+
+from .hfpipeline import HFPipeline
+
+
+class Labels(HFPipeline):
+ """
+ Applies a zero shot classifier to text using a list of labels.
+ """
+
+ def __init__(self, path=None, quantize=False, gpu=True, model=None):
+ super().__init__("zero-shot-classification", path, quantize, gpu, model)
+
+ def __call__(self, text, labels, multiclass=False):
+ """
+ Applies a zero shot classifier to text using a list of labels. Returns a list of
+ (id, score) sorted by highest score, where id is the index in labels.
+
+ This method supports text as a string or a list. If the input is a string, the return
+ type is a 1D list of (id, score). If text is a list, a 2D list of (id, score) is
+ returned with a row per string.
+
+ Args:
+ text: text|list
+ labels: list of labels
+
+ Returns:
+ list of (id, score)
+ """
+
+ # Run ZSL pipeline
+ results = self.pipeline(text, labels, multi_class=multiclass)
+
+ # Convert results to a list if necessary
+ if not isinstance(results, list):
+ results = [results]
+
+ # Build list of (id, score)
+ scores = []
+ for result in results:
+ scores.append([(labels.index(label), result["scores"][x]) for x, label in enumerate(result["labels"])])
+
+ return scores[0] if isinstance(text, str) else scores
diff --git a/src/python/txtai/pipeline/questions.py b/src/python/txtai/pipeline/questions.py
new file mode 100644
index 00000000..db23fb3a
--- /dev/null
+++ b/src/python/txtai/pipeline/questions.py
@@ -0,0 +1,47 @@
+"""
+Questions module
+"""
+
+from .hfpipeline import HFPipeline
+
+
+class Questions(HFPipeline):
+ """
+ Runs extractive QA for a series of questions and contexts.
+ """
+
+ def __init__(self, path=None, quantize=False, gpu=False, model=None):
+ super().__init__("question-answering", path, quantize, gpu, model)
+
+ def __call__(self, questions, contexts):
+ """
+ Runs a extractive question-answering model against each question-context pair, finding the best answers.
+
+ Args:
+ questions: list of questions
+ contexts: list of contexts to pull answers from
+
+ Returns:
+ list of answers
+ """
+
+ answers = []
+
+ for x, question in enumerate(questions):
+ if question and contexts[x]:
+ # Run the QA pipeline
+ result = self.pipeline(question=question, context=contexts[x])
+
+ # Get answer and score
+ answer, score = result["answer"], result["score"]
+
+ # Require score to be at least 0.05
+ if score < 0.05:
+ answer = None
+
+ # Add answer
+ answers.append(answer)
+ else:
+ answers.append(None)
+
+ return answers
diff --git a/src/python/txtai/pipeline/similarity.py b/src/python/txtai/pipeline/similarity.py
new file mode 100644
index 00000000..2784fda7
--- /dev/null
+++ b/src/python/txtai/pipeline/similarity.py
@@ -0,0 +1,44 @@
+"""
+Similarity module
+"""
+
+import numpy as np
+
+from .labels import Labels
+
+
+class Similarity(Labels):
+ """
+ Computes similarity between query and list of text using a zero shot classifier.
+ """
+
+ def __call__(self, query, texts, multiclass=True):
+ """
+ Computes the similarity between query and list of text. Returns a list of
+ (id, score) sorted by highest score, where id is the index in texts.
+
+ This method supports query as a string or a list. If the input is a string,
+ the return type is a 1D list of (id, score). If text is a list, a 2D list
+ of (id, score) is returned with a row per string.
+
+ Args:
+ query: query text|list
+ texts: list of text
+
+ Returns:
+ list of (id, score)
+ """
+
+ # Call Labels pipeline for texts using input query as the candidate label
+ scores = super().__call__(texts, [query] if isinstance(query, str) else query, multiclass)
+
+ # Sort on query index id
+ scores = [[score for _, score in sorted(row)] for row in scores]
+
+ # Transpose axes to get a list of text scores for each query
+ scores = np.array(scores).T.tolist()
+
+ # Build list of (id, score) per query sorted by highest score
+ scores = [sorted(enumerate(row), key=lambda x: x[1], reverse=True) for row in scores]
+
+ return scores[0] if isinstance(query, str) else scores
diff --git a/src/python/txtai/tokenizer.py b/src/python/txtai/pipeline/tokenizer.py
similarity index 73%
rename from src/python/txtai/tokenizer.py
rename to src/python/txtai/pipeline/tokenizer.py
index ff9a67de..197cfa51 100644
--- a/src/python/txtai/tokenizer.py
+++ b/src/python/txtai/pipeline/tokenizer.py
@@ -5,10 +5,12 @@ Tokenizer module
import re
import string
+from .base import Pipeline
-class Tokenizer:
+
+class Tokenizer(Pipeline):
"""
- Text tokenization methods.
+ Tokenizes text into a list of tokens. Primarily designed for English text.
"""
# fmt: off
@@ -30,6 +32,19 @@ class Tokenizer:
list of tokens
"""
+ return Tokenizer()(text)
+
+ def __call__(self, text):
+ """
+ Tokenizes input text into a list of tokens. Filters tokens that match a specific pattern and removes stop words.
+
+ Args:
+ text: input text
+
+ Returns:
+ list of tokens
+ """
+
# Convert to all lowercase, split on whitespace, strip punctuation
tokens = [token.strip(string.punctuation) for token in text.lower().split()]
diff --git a/src/python/txtai/scoring/__init__.py b/src/python/txtai/scoring/__init__.py
new file mode 100644
index 00000000..c4850c9b
--- /dev/null
+++ b/src/python/txtai/scoring/__init__.py
@@ -0,0 +1,8 @@
+"""
+Scoring imports
+"""
+
+from .base import Scoring
+from .bm25 import BM25
+from .factory import ScoringFactory
+from .sif import SIF
diff --git a/src/python/txtai/scoring.py b/src/python/txtai/scoring/base.py
similarity index 75%
rename from src/python/txtai/scoring.py
rename to src/python/txtai/scoring/base.py
index 3b5958dd..ded6d4cc 100644
--- a/src/python/txtai/scoring.py
+++ b/src/python/txtai/scoring/base.py
@@ -7,7 +7,7 @@ import pickle
from collections import Counter
-from .tokenizer import Tokenizer
+from ..pipeline.tokenizer import Tokenizer
class Scoring:
@@ -15,28 +15,6 @@ class Scoring:
Base scoring object. Default method scores documents using TF-IDF.
"""
- @staticmethod
- def create(method):
- """
- Factory method to construct a Scoring object.
-
- Args:
- method: scoring method (bm25, sif, tfidf)
-
- Returns:
- Scoring object
- """
-
- if method == "bm25":
- return BM25()
- if method == "sif":
- return SIF()
- if method == "tfidf":
- # Default scoring object implements tf-idf
- return Scoring()
-
- return None
-
def __init__(self):
"""
Initializes backing statistic objects.
@@ -194,42 +172,3 @@ class Scoring:
"""
return idf
-
-
-class BM25(Scoring):
- """
- BM25 scoring. Scores using Apache Lucene's version of BM25 which adds 1 to prevent
- negative scores.
- """
-
- def __init__(self, k1=0.1, b=0.75):
- super().__init__()
-
- # BM25 configurable parameters
- self.k1 = k1
- self.b = b
-
- def computeIDF(self, freq):
- # Calculate BM25 IDF score
- return math.log(1 + (self.total - freq + 0.5) / (freq + 0.5))
-
- def score(self, freq, idf, length):
- # Calculate BM25 score
- k = self.k1 * ((1 - self.b) + self.b * length / self.avgdl)
- return idf * (freq * (self.k1 + 1)) / (freq + k)
-
-
-class SIF(Scoring):
- """
- Smooth Inverse Frequency (SIF) scoring.
- """
-
- def __init__(self, a=0.001):
- super().__init__()
-
- # SIF configurable parameters
- self.a = a
-
- def score(self, freq, idf, length):
- # Calculate SIF score
- return self.a / (self.a + freq / self.tokens)
diff --git a/src/python/txtai/scoring/bm25.py b/src/python/txtai/scoring/bm25.py
new file mode 100644
index 00000000..b5ba668e
--- /dev/null
+++ b/src/python/txtai/scoring/bm25.py
@@ -0,0 +1,30 @@
+"""
+BM25 module
+"""
+
+import math
+
+from .base import Scoring
+
+
+class BM25(Scoring):
+ """
+ BM25 scoring. Scores using Apache Lucene's version of BM25 which adds 1 to prevent
+ negative scores.
+ """
+
+ def __init__(self, k1=0.1, b=0.75):
+ super().__init__()
+
+ # BM25 configurable parameters
+ self.k1 = k1
+ self.b = b
+
+ def computeIDF(self, freq):
+ # Calculate BM25 IDF score
+ return math.log(1 + (self.total - freq + 0.5) / (freq + 0.5))
+
+ def score(self, freq, idf, length):
+ # Calculate BM25 score
+ k = self.k1 * ((1 - self.b) + self.b * length / self.avgdl)
+ return idf * (freq * (self.k1 + 1)) / (freq + k)
diff --git a/src/python/txtai/scoring/factory.py b/src/python/txtai/scoring/factory.py
new file mode 100644
index 00000000..413c69b5
--- /dev/null
+++ b/src/python/txtai/scoring/factory.py
@@ -0,0 +1,35 @@
+"""
+Factory module
+"""
+
+from .base import Scoring
+from .bm25 import BM25
+from .sif import SIF
+
+
+class ScoringFactory:
+ """
+ Methods to create Scoring models.
+ """
+
+ @staticmethod
+ def create(method):
+ """
+ Factory method to construct a Scoring object.
+
+ Args:
+ method: scoring method (bm25, sif, tfidf)
+
+ Returns:
+ Scoring
+ """
+
+ if method == "bm25":
+ return BM25()
+ if method == "sif":
+ return SIF()
+ if method == "tfidf":
+ # Default scoring object implements tf-idf
+ return Scoring()
+
+ return None
diff --git a/src/python/txtai/scoring/sif.py b/src/python/txtai/scoring/sif.py
new file mode 100644
index 00000000..308ee899
--- /dev/null
+++ b/src/python/txtai/scoring/sif.py
@@ -0,0 +1,21 @@
+"""
+SIF module
+"""
+
+from .base import Scoring
+
+
+class SIF(Scoring):
+ """
+ Smooth Inverse Frequency (SIF) scoring.
+ """
+
+ def __init__(self, a=0.001):
+ super().__init__()
+
+ # SIF configurable parameters
+ self.a = a
+
+ def score(self, freq, idf, length):
+ # Calculate SIF score
+ return self.a / (self.a + freq / self.tokens)
diff --git a/src/python/txtai/tokenizer/__init__.py b/src/python/txtai/tokenizer/__init__.py
new file mode 100644
index 00000000..0ec75bba
--- /dev/null
+++ b/src/python/txtai/tokenizer/__init__.py
@@ -0,0 +1,5 @@
+"""
+Defined to maintain backwards compatibility
+"""
+
+from ..pipeline.tokenizer import Tokenizer
diff --git a/src/python/txtai/vectors.py b/src/python/txtai/vectors.py
deleted file mode 100644
index bccd710a..00000000
--- a/src/python/txtai/vectors.py
+++ /dev/null
@@ -1,313 +0,0 @@
-"""
-Vectors module
-"""
-
-import os
-import pickle
-import tempfile
-
-from errno import ENOENT
-from multiprocessing import Pool
-
-import fasttext
-import numpy as np
-
-from pymagnitude import converter, Magnitude
-from sentence_transformers import SentenceTransformer
-from sentence_transformers.models import Pooling, Transformer
-
-from .tokenizer import Tokenizer
-
-# Multiprocessing helper methods
-# pylint: disable=W0603
-VECTORS = None
-
-
-def create(config, scoring):
- """
- Multiprocessing helper method. Creates a global embeddings object to be accessed in a new subprocess.
-
- Args:
- config: vector configuration
- scoring: scoring instance
- """
-
- global VECTORS
-
- # Create a global embedding object using configuration and saved
- VECTORS = WordVectors(config, scoring)
-
-
-def transform(document):
- """
- Multiprocessing helper method. Transforms document into an embeddings vector.
-
- Args:
- document: (id, text|tokens, tags)
-
- Returns:
- (id, embedding)
- """
-
- global VECTORS
-
- return (document[0], VECTORS.transform(document))
-
-
-class Vectors:
- """
- Base class for sentence embeddings/vector models.
- """
-
- @staticmethod
- def create(config, scoring):
- """
- Create a Vectors model instance.
-
- Args:
- config: vector configuration
- scoring: scoring instance
- """
-
- # Derive vector type
- transformers = config.get("method") == "transformers"
-
- # Create vector model instance
- return TransformersVectors(config, scoring) if transformers else WordVectors(config, scoring)
-
- def __init__(self, config, scoring):
- # Store parameters
- self.config = config
- self.scoring = scoring
-
- # Detect if this is an initialized configuration
- self.initialized = "dimensions" in config
-
- # Enables optional string tokenization
- self.tokenize = "tokenize" not in config or config["tokenize"]
-
- # pylint: disable=E1111
- self.model = self.load(config["path"])
-
- def load(self, path):
- """
- Loads vector model at path.
-
- Args:
- path: path to word vector model
-
- Returns:
- vector model
- """
-
- def index(self, documents):
- """
- Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,
- number of dimensions and temporary file with embeddings.
-
- Args:
- documents: list of (id, text|tokens, tags)
-
- Returns:
- (ids, dimensions, stream)
- """
-
- def transform(self, document):
- """
- Transforms document into an embeddings vector.
-
- Args:
- document: (id, text|tokens, tags)
-
- Returns:
- embeddings vector
- """
-
-
-class WordVectors(Vectors):
- """
- Builds sentence embeddings/vectors using weighted word embeddings.
- """
-
- def load(self, path):
- # Ensure that vector path exists
- if not path or not os.path.isfile(path):
- raise IOError(ENOENT, "Vector model file not found", path)
-
- # Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded
- return Magnitude(path, case_insensitive=True, blocking=not self.initialized)
-
- def index(self, documents):
- ids, dimensions, stream = [], None, None
-
- # Shared objects with Pool
- args = (self.config, self.scoring)
-
- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
- with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:
- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
- stream = output.name
- for uid, embedding in pool.imap(transform, documents):
- if not dimensions:
- # Set number of dimensions for embeddings
- dimensions = embedding.shape[0]
-
- ids.append(uid)
- pickle.dump(embedding, output)
-
- return (ids, dimensions, stream)
-
- def transform(self, document):
- # Convert to tokens if necessary
- if isinstance(document[1], str):
- document = (document[0], Tokenizer.tokenize(document[1]), document[2])
-
- # Generate weights for each vector using a scoring method
- weights = self.scoring.weights(document) if self.scoring else None
-
- # pylint: disable=E1133
- if weights and [x for x in weights if x > 0]:
- # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.
- embedding = np.average(self.lookup(document[1]), weights=np.array(weights, dtype=np.float32), axis=0)
- else:
- # If no weights, use mean
- embedding = np.mean(self.lookup(document[1]), axis=0)
-
- return embedding
-
- def lookup(self, tokens):
- """
- Queries word vectors for given list of input tokens.
-
- Args:
- tokens: list of tokens to query
-
- Returns:
- word vectors array
- """
-
- return self.model.query(tokens)
-
- @staticmethod
- def build(data, size, mincount, path):
- """
- Builds fastText vectors from a file.
-
- Args:
- data: path to input data file
- size: number of vector dimensions
- mincount: minimum number of occurrences required to register a token
- path: path to output file
- """
-
- # Train on data file using largest dimension size
- model = fasttext.train_unsupervised(data, dim=size, minCount=mincount)
-
- # Output file path
- print("Building %d dimension model" % size)
-
- # Output vectors in vec/txt format
- with open(path + ".txt", "w") as output:
- words = model.get_words()
- output.write("%d %d\n" % (len(words), model.get_dimension()))
-
- for word in words:
- # Skip end of line token
- if word != "</s>":
- vector = model.get_word_vector(word)
- data = ""
- for v in vector:
- data += " " + str(v)
-
- output.write(word + data + "\n")
-
- # Build magnitude vectors database
- print("Converting vectors to magnitude format")
- converter.convert(path + ".txt", path + ".magnitude", subword=True)
-
-
-class TransformersVectors(Vectors):
- """
- Builds sentence embeddings/vectors using the transformers library.
- """
-
- def load(self, path):
- model = Transformer(path)
- pooling = Pooling(model.get_word_embedding_dimension())
-
- return SentenceTransformer(modules=[model, pooling])
-
- def index(self, documents):
- ids, dimensions, stream = [], None, None
-
- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
- stream = output.name
- batch = []
- for document in documents:
- batch.append(document)
-
- if len(batch) == 500:
- # Convert batch to embeddings
- uids, dimensions = self.batch(batch, output)
- ids.extend(uids)
-
- batch = []
-
- # Final batch
- if batch:
- uids, dimensions = self.batch(batch, output)
- ids.extend(uids)
-
- return (ids, dimensions, stream)
-
- def transform(self, document):
- # Convert input document to text and build embeddings
- return self.model.encode([self.text(document[1])], show_progress_bar=False)[0]
-
- def batch(self, documents, output):
- """
- Builds a batch of embeddings.
-
- Args:
- documents: list of documents used to build embeddings
- output: output temp file to store embeddings
-
- Returns:
- (ids, dimensions) list of ids and number of dimensions in embeddings
- """
-
- # Extract ids and convert input documents to text
- ids = [uid for uid, _, _ in documents]
- documents = [self.text(text) for _, text, _ in documents]
- dimensions = None
-
- # Build embeddings
- embeddings = self.model.encode(documents, show_progress_bar=False)
- for embedding in embeddings:
- if not dimensions:
- # Set number of dimensions for embeddings
- dimensions = embedding.shape[0]
-
- pickle.dump(embedding, output)
-
- return (ids, dimensions)
-
- def text(self, text):
- """
- Converts input into text that can be processed by transformer models. This method supports
- optional string tokenization and joins tokenized input into text.
-
- Args:
- text: text|tokens
- """
-
- # Optional string tokenization
- if self.tokenize and isinstance(text, str):
- text = Tokenizer.tokenize(text)
-
- # Transformer models require string input
- if isinstance(text, list):
- text = " ".join(text)
-
- return text
diff --git a/src/python/txtai/vectors/__init__.py b/src/python/txtai/vectors/__init__.py
new file mode 100644
index 00000000..5bedd266
--- /dev/null
+++ b/src/python/txtai/vectors/__init__.py
@@ -0,0 +1,8 @@
+"""
+Vectors import
+"""
+
+from .base import Vectors
+from .factory import VectorsFactory
+from .transformers import TransformersVectors
+from .words import WordVectors
diff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.py
new file mode 100644
index 00000000..cbb101c0
--- /dev/null
+++ b/src/python/txtai/vectors/base.py
@@ -0,0 +1,57 @@
+"""
+Vectors module
+"""
+
+
+class Vectors:
+ """
+ Base class for sentence embeddings/vector models.
+ """
+
+ def __init__(self, config, scoring):
+ # Store parameters
+ self.config = config
+ self.scoring = scoring
+
+ # Detect if this is an initialized configuration
+ self.initialized = "dimensions" in config
+
+ # Enables optional string tokenization
+ self.tokenize = "tokenize" not in config or config["tokenize"]
+
+ # pylint: disable=E1111
+ self.model = self.load(config["path"])
+
+ def load(self, path):
+ """
+ Loads vector model at path.
+
+ Args:
+ path: path to word vector model
+
+ Returns:
+ vector model
+ """
+
+ def index(self, documents):
+ """
+ Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,
+ number of dimensions and temporary file with embeddings.
+
+ Args:
+ documents: list of (id, text|tokens, tags)
+
+ Returns:
+ (ids, dimensions, stream)
+ """
+
+ def transform(self, document):
+ """
+ Transforms document into an embeddings vector.
+
+ Args:
+ document: (id, text|tokens, tags)
+
+ Returns:
+ embeddings vector
+ """
diff --git a/src/python/txtai/vectors/factory.py b/src/python/txtai/vectors/factory.py
new file mode 100644
index 00000000..41417fe7
--- /dev/null
+++ b/src/python/txtai/vectors/factory.py
@@ -0,0 +1,31 @@
+"""
+Factory module
+"""
+
+from .transformers import TransformersVectors
+from .words import WordVectors
+
+
+class VectorsFactory:
+ """
+ Methods to create Vectors models.
+ """
+
+ @staticmethod
+ def create(config, scoring):
+ """
+ Create a Vectors model instance.
+
+ Args:
+ config: vector configuration
+ scoring: scoring instance
+
+ Returns:
+ Vectors
+ """
+
+ # Derive vector type
+ transformers = config.get("method") == "transformers"
+
+ # Create vector model instance
+ return TransformersVectors(config, scoring) if transformers else WordVectors(config, scoring)
diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.py
new file mode 100644
index 00000000..7e17f06f
--- /dev/null
+++ b/src/python/txtai/vectors/transformers.py
@@ -0,0 +1,100 @@
+"""
+Transformers module
+"""
+
+import pickle
+import tempfile
+
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.models import Pooling, Transformer
+
+from ..pipeline.tokenizer import Tokenizer
+
+from .base import Vectors
+
+
+class TransformersVectors(Vectors):
+ """
+ Builds sentence embeddings/vectors using the transformers library.
+ """
+
+ def load(self, path):
+ model = Transformer(path)
+ pooling = Pooling(model.get_word_embedding_dimension())
+
+ return SentenceTransformer(modules=[model, pooling])
+
+ def index(self, documents):
+ ids, dimensions, stream = [], None, None
+
+ # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
+ stream = output.name
+ batch = []
+ for document in documents:
+ batch.append(document)
+
+ if len(batch) == 500:
+ # Convert batch to embeddings
+ uids, dimensions = self.batch(batch, output)
+ ids.extend(uids)
+
+ batch = []
+
+ # Final batch
+ if batch:
+ uids, dimensions = self.batch(batch, output)
+ ids.extend(uids)
+
+ return (ids, dimensions, stream)
+
+ def transform(self, document):
+ # Convert input document to text and build embeddings
+ return self.model.encode([self.text(document[1])], show_progress_bar=False)[0]
+
+ def batch(self, documents, output):
+ """
+ Builds a batch of embeddings.
+
+ Args:
+ documents: list of documents used to build embeddings
+ output: output temp file to store embeddings
+
+ Returns:
+ (ids, dimensions) list of ids and number of dimensions in embeddings
+ """
+
+ # Extract ids and convert input documents to text
+ ids = [uid for uid, _, _ in documents]
+ documents = [self.text(text) for _, text, _ in documents]
+ dimensions = None
+
+ # Build embeddings
+ embeddings = self.model.encode(documents, show_progress_bar=False)
+ for embedding in embeddings:
+ if not dimensions:
+ # Set number of dimensions for embeddings
+ dimensions = embedding.shape[0]
+
+ pickle.dump(embedding, output)
+
+ return (ids, dimensions)
+
+ def text(self, text):
+ """
+ Converts input into text that can be processed by transformer models. This method supports
+ optional string tokenization and joins tokenized input into text.
+
+ Args:
+ text: text|tokens
+ """
+
+ # Optional string tokenization
+ if self.tokenize and isinstance(text, str):
+ text = Tokenizer.tokenize(text)
+
+ # Transformer models require string input
+ if isinstance(text, list):
+ text = " ".join(text)
+
+ return text
diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.py
new file mode 100644
index 00000000..1cac062e
--- /dev/null
+++ b/src/python/txtai/vectors/words.py
@@ -0,0 +1,155 @@
+"""
+Word Vectors module
+"""
+
+import os
+import pickle
+import tempfile
+
+from errno import ENOENT
+from multiprocessing import Pool
+
+import fasttext
+import numpy as np
+
+from pymagnitude import converter, Magnitude
+
+from .base import Vectors
+from ..pipeline.tokenizer import Tokenizer
+
+# Multiprocessing helper methods
+# pylint: disable=W0603
+VECTORS = None
+
+
+def create(config, scoring):
+ """
+ Multiprocessing helper method. Creates a global embeddings object to be accessed in a new subprocess.
+
+ Args:
+ config: vector configuration
+ scoring: scoring instance
+ """
+
+ global VECTORS
+
+ # Create a global embedding object using configuration and saved
+ VECTORS = WordVectors(config, scoring)
+
+
+def transform(document):
+ """
+ Multiprocessing helper method. Transforms document into an embeddings vector.
+
+ Args:
+ document: (id, text|tokens, tags)
+
+ Returns:
+ (id, embedding)
+ """
+
+ global VECTORS
+
+ return (document[0], VECTORS.transform(document))
+
+
+class WordVectors(Vectors):
+ """
+ Builds sentence embeddings/vectors using weighted word embeddings.
+ """
+
+ def load(self, path):
+ # Ensure that vector path exists
+ if not path or not os.path.isfile(path):
+ raise IOError(ENOENT, "Vector model file not found", path)
+
+ # Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded
+ return Magnitude(path, case_insensitive=True, blocking=not self.initialized)
+
+ def index(self, documents):
+ ids, dimensions, stream = [], None, None
+
+ # Shared objects with Pool
+ args = (self.config, self.scoring)
+
+ # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
+ with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:
+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
+ stream = output.name
+ for uid, embedding in pool.imap(transform, documents):
+ if not dimensions:
+ # Set number of dimensions for embeddings
+ dimensions = embedding.shape[0]
+
+ ids.append(uid)
+ pickle.dump(embedding, output)
+
+ return (ids, dimensions, stream)
+
+ def transform(self, document):
+ # Convert to tokens if necessary
+ if isinstance(document[1], str):
+ document = (document[0], Tokenizer.tokenize(document[1]), document[2])
+
+ # Generate weights for each vector using a scoring method
+ weights = self.scoring.weights(document) if self.scoring else None
+
+ # pylint: disable=E1133
+ if weights and [x for x in weights if x > 0]:
+ # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.
+ embedding = np.average(self.lookup(document[1]), weights=np.array(weights, dtype=np.float32), axis=0)
+ else:
+ # If no weights, use mean
+ embedding = np.mean(self.lookup(document[1]), axis=0)
+
+ return embedding
+
+ def lookup(self, tokens):
+ """
+ Queries word vectors for given list of input tokens.
+
+ Args:
+ tokens: list of tokens to query
+
+ Returns:
+ word vectors array
+ """
+
+ return self.model.query(tokens)
+
+ @staticmethod
+ def build(data, size, mincount, path):
+ """
+ Builds fastText vectors from a file.
+
+ Args:
+ data: path to input data file
+ size: number of vector dimensions
+ mincount: minimum number of occurrences required to register a token
+ path: path to output file
+ """
+
+ # Train on data file using largest dimension size
+ model = fasttext.train_unsupervised(data, dim=size, minCount=mincount)
+
+ # Output file path
+ print("Building %d dimension model" % size)
+
+ # Output vectors in vec/txt format
+ with open(path + ".txt", "w") as output:
+ words = model.get_words()
+ output.write("%d %d\n" % (len(words), model.get_dimension()))
+
+ for word in words:
+ # Skip end of line token
+ if word != "</s>":
+ vector = model.get_word_vector(word)
+ data = ""
+ for v in vector:
+ data += " " + str(v)
+
+ output.write(word + data + "\n")
+
+ # Build magnitude vectors database
+ print("Converting vectors to magnitude format")
+ converter.convert(path + ".txt", path + ".magnitude", subword=True)
diff --git a/test/python/testann.py b/test/python/testann.py
index 6386b631..526080ea 100644
--- a/test/python/testann.py
+++ b/test/python/testann.py
@@ -8,7 +8,7 @@ import unittest
import numpy as np
-from txtai.ann import ANN
+from txtai.ann import ANNFactory
class TestANN(unittest.TestCase):
@@ -104,7 +104,7 @@ class TestANN(unittest.TestCase):
if params:
config.update(params)
- model = ANN.create(config)
+ model = ANNFactory.create(config)
model.index(data)
return model
diff --git a/test/python/testscoring.py b/test/python/testscoring.py
index d4beeabf..f5f3c203 100644
--- a/test/python/testscoring.py
+++ b/test/python/testscoring.py
@@ -6,7 +6,7 @@ import os
import tempfile
import unittest
-from txtai.scoring import Scoring
+from txtai.scoring import ScoringFactory
class TestScoring(unittest.TestCase):
@@ -61,7 +61,7 @@ class TestScoring(unittest.TestCase):
Test unknown method
"""
- self.assertIsNone(Scoring.create("unknown"))
+ self.assertIsNone(ScoringFactory.create("unknown"))
def method(self, method, data=None):
"""
@@ -78,7 +78,7 @@ class TestScoring(unittest.TestCase):
# Derive input data
data = data if data else self.data
- model = Scoring.create(method)
+ model = ScoringFactory.create(method)
model.index(data)
keys = [k for k, v in sorted(model.idf.items(), key=lambda x: x[1])]
diff --git a/test/python/testtransformers.py b/test/python/testtransformers.py
index e63f5740..aa963e1a 100644
--- a/test/python/testtransformers.py
+++ b/test/python/testtransformers.py
@@ -7,7 +7,7 @@ import unittest
import numpy as np
-from txtai.vectors import Vectors
+from txtai.vectors import VectorsFactory
class TestTransformersVectors(unittest.TestCase):
@@ -21,7 +21,7 @@ class TestTransformersVectors(unittest.TestCase):
Create single TransformersVectors instance.
"""
- cls.model = Vectors.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)
+ cls.model = VectorsFactory.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)
def testIndex(self):
"""
diff --git a/test/python/testwordvectors.py b/test/python/testwordvectors.py
index 9b23062c..51c8d7ca 100644
--- a/test/python/testwordvectors.py
+++ b/test/python/testwordvectors.py
@@ -6,7 +6,7 @@ import os
import tempfile
import unittest
-from txtai.vectors import WordVectors, Vectors
+from txtai.vectors import WordVectors, VectorsFactory
class TestWordVectors(unittest.TestCase):
@@ -35,13 +35,13 @@ class TestWordVectors(unittest.TestCase):
"""
config = {"path": self.path}
- model = Vectors.create(config, None)
+ model = VectorsFactory.create(config, None)
self.assertFalse(model.initialized)
config["ids"] = ["0", "1"]
config["dimensions"] = 300
- model = Vectors.create(config, None)
+ model = VectorsFactory.create(config, None)
self.assertTrue(model.initialized)
@@ -53,7 +53,7 @@ class TestWordVectors(unittest.TestCase):
# Generate data
documents = [(x, "This is a test", None) for x in range(1000)]
- model = Vectors.create({"path": self.path}, None)
+ model = VectorsFactory.create({"path": self.path}, None)
ids, dimension, stream = model.index(documents)
@@ -66,7 +66,7 @@ class TestWordVectors(unittest.TestCase):
Test word vector lookup
"""
- model = Vectors.create({"path": self.path}, None)
+ model = VectorsFactory.create({"path": self.path}, None)
self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))
def testNoExist(self):
@@ -76,12 +76,12 @@ class TestWordVectors(unittest.TestCase):
# Test non-existent path raises an exception
with self.assertRaises(IOError):
- Vectors.create({"path": os.path.join(tempfile.gettempdir(), "noexist")}, None)
+ VectorsFactory.create({"path": os.path.join(tempfile.gettempdir(), "noexist")}, None)
def testTransform(self):
"""
Test word vector transform
"""
- model = Vectors.create({"path": self.path}, None)
+ model = VectorsFactory.create({"path": self.path}, None)
self.assertEqual(len(model.transform((None, ["txtai"], None))), 300)