💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Refactor code into smaller modules #63
davidmezzettie6b2db47c41f
diff --git a/src/python/txtai/ann.py b/src/python/txtai/ann.pydeleted file mode 100644index eb473c2c..00000000--- a/src/python/txtai/ann.py+++ /dev/null@@ -1,258 +0,0 @@-"""-ANN (Approximate Nearest Neighbors) module-"""--import numpy as np--# pylint: disable=E0611-from annoy import AnnoyIndex-from hnswlib import Index--# Conditionally import Faiss as it's only supported on Linux/macOS-try:- import faiss-- FAISS = True-except ImportError:- FAISS = False---class ANN:- """- Base class for ANN models.- """-- @staticmethod- def create(config):- """- Create an ANN model.-- Args:- config: index configuration parameters-- Returns:- ANN- """-- # ANN model- model = None- backend = config.get("backend")-- # Default backend if not provided, based on available libraries- if not backend:- backend = "faiss" if FAISS else "annoy"-- # Create ANN instance- if backend == "annoy":- model = Annoy(config)- elif backend == "hnsw":- model = HNSW(config)- else:- # Raise error if trying to create a Faiss index without Faiss installed- if not FAISS:- raise ImportError("Faiss library is not installed")-- model = Faiss(config)-- # Store config back- config["backend"] = backend-- return model-- def __init__(self, config):- """- Creates a new ANN model.- """-- # ANN index- self.model = None-- # Model configuration- self.config = config-- def load(self, path):- """- Loads an ANN model at path.- """-- def index(self, embeddings):- """- Builds an ANN model.-- Args:- embeddings: embeddings array- """-- def search(self, queries, limit):- """- Searches ANN model for query. Returns topn results.-- Args:- queries: queries array- limit: maximum results-- Returns:- query results- """-- def save(self, path):- """- Saves an ANN model at path.- """-- def setting(self, name, default=None):- """- Looks up backend specific setting.-- Args:- name: setting name- default: default value when setting not found-- Returns:- setting value- """-- # Get the backend-specific config object- backend = self.config.get(self.config["backend"])-- # Get setting value, set default value if not found- setting = backend.get(name) if backend else None- return setting if setting else default---class Annoy(ANN):- """- Builds an ANN model using the Annoy library.- """-- def load(self, path):- # Load index- self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])- self.model.load(path)-- def index(self, embeddings):- # Inner product is equal to cosine similarity on normalized vectors- self.config["metric"] = "dot"-- # Create index- self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])-- # Add items- for x in range(embeddings.shape[0]):- self.model.add_item(x, embeddings[x])-- # Build index- self.model.build(self.setting("ntrees", 10))-- def search(self, queries, limit):- # Lookup search k setting- searchk = self.setting("searchk", -1)-- # Annoy doesn't have a built in batch query method- results = []- for query in queries:- # Run the query- ids, scores = self.model.get_nns_by_vector(query, n=limit, search_k=searchk, include_distances=True)-- # Map results to [(id, score)]- results.append(list(zip(ids, scores)))-- return results-- def save(self, path):- # Write index- self.model.save(path)---class Faiss(ANN):- """- Builds an ANN model using the Faiss library.- """-- def load(self, path):- # Load index- self.model = faiss.read_index(path)-- def index(self, embeddings):- # Lookup components setting- components = self.setting("components")-- # Create embeddings index. Inner product is equal to cosine similarity on normalized vectors.- if components:- params = components- elif self.config.get("quantize"):- params = "IVF100,SQ8" if embeddings.shape[0] >= 5000 else "IDMap,SQ8"- else:- params = "IVF100,Flat" if embeddings.shape[0] >= 5000 else "IDMap,Flat"-- self.model = faiss.index_factory(embeddings.shape[1], params, faiss.METRIC_INNER_PRODUCT)-- # Train model- self.model.train(embeddings)- self.model.add_with_ids(embeddings, np.array(range(embeddings.shape[0])))-- def search(self, queries, limit):- # Run the query- self.model.nprobe = self.setting("nprobe", 6)- scores, ids = self.model.search(queries, limit)-- # Map results to [(id, score)]- results = []- for x, score in enumerate(scores):- results.append(list(zip(ids[x].tolist(), score.tolist())))-- return results-- def save(self, path):- # Write index- faiss.write_index(self.model, path)---class HNSW(ANN):- """- Builds an ANN model using the hnswlib library.- """-- def load(self, path):- # Load index- self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])- self.model.load_index(path)-- def index(self, embeddings):- # Inner product is equal to cosine similarity on normalized vectors- self.config["metric"] = "ip"-- # Lookup index settings- efconstruction = self.setting("efconstruction", 200)- m = self.setting("m", 16)- seed = self.setting("randomseed", 100)-- # Create index- self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])- self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)-- # Add items- self.model.add_items(embeddings, np.array(range(embeddings.shape[0])))-- def search(self, queries, limit):- # Set ef query param- ef = self.setting("efsearch")- if ef:- self.model.set_ef(ef)-- # Run the query- ids, distances = self.model.knn_query(queries, k=limit)-- # Map results to [(id, score)]- results = []- for x, distance in enumerate(distances):- # Convert distances to similarity scores- scores = [1 - d for d in distance]-- results.append(list(zip(ids[x], scores)))-- return results-- def save(self, path):- # Write index- self.model.save_index(path)diff --git a/src/python/txtai/ann/__init__.py b/src/python/txtai/ann/__init__.pynew file mode 100644index 00000000..880534d2--- /dev/null+++ b/src/python/txtai/ann/__init__.py@@ -0,0 +1,9 @@+"""+ANN imports+"""++from .annoy import Annoy+from .base import ANN+from .factory import ANNFactory+from .faiss import Faiss+from .hnsw import HNSWdiff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/annoy.pynew file mode 100644index 00000000..28540f95--- /dev/null+++ b/src/python/txtai/ann/annoy.py@@ -0,0 +1,51 @@+"""+Annoy module+"""++from annoy import AnnoyIndex++from .base import ANN+++class Annoy(ANN):+ """+ Builds an ANN model using the Annoy library.+ """++ def load(self, path):+ # Load index+ self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])+ self.model.load(path)++ def index(self, embeddings):+ # Inner product is equal to cosine similarity on normalized vectors+ self.config["metric"] = "dot"++ # Create index+ self.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])++ # Add items+ for x in range(embeddings.shape[0]):+ self.model.add_item(x, embeddings[x])++ # Build index+ self.model.build(self.setting("ntrees", 10))++ def search(self, queries, limit):+ # Lookup search k setting+ searchk = self.setting("searchk", -1)++ # Annoy doesn't have a built in batch query method+ results = []+ for query in queries:+ # Run the query+ ids, scores = self.model.get_nns_by_vector(query, n=limit, search_k=searchk, include_distances=True)++ # Map results to [(id, score)]+ results.append(list(zip(ids, scores)))++ return results++ def save(self, path):+ # Write index+ self.model.save(path)diff --git a/src/python/txtai/ann/base.py b/src/python/txtai/ann/base.pynew file mode 100644index 00000000..8386b68f--- /dev/null+++ b/src/python/txtai/ann/base.py@@ -0,0 +1,69 @@+"""+ANN (Approximate Nearest Neighbors) module+"""+++class ANN:+ """+ Base class for ANN models.+ """++ def __init__(self, config):+ """+ Creates a new ANN model.+ """++ # ANN index+ self.model = None++ # Model configuration+ self.config = config++ def load(self, path):+ """+ Loads an ANN model at path.+ """++ def index(self, embeddings):+ """+ Builds an ANN model.++ Args:+ embeddings: embeddings array+ """++ def search(self, queries, limit):+ """+ Searches ANN model for query. Returns topn results.++ Args:+ queries: queries array+ limit: maximum results++ Returns:+ query results+ """++ def save(self, path):+ """+ Saves an ANN model at path.+ """++ def setting(self, name, default=None):+ """+ Looks up backend specific setting.++ Args:+ name: setting name+ default: default value when setting not found++ Returns:+ setting value+ """++ # Get the backend-specific config object+ backend = self.config.get(self.config["backend"])++ # Get setting value, set default value if not found+ setting = backend.get(name) if backend else None+ return setting if setting else defaultdiff --git a/src/python/txtai/ann/factory.py b/src/python/txtai/ann/factory.pynew file mode 100644index 00000000..9ddb3d0d--- /dev/null+++ b/src/python/txtai/ann/factory.py@@ -0,0 +1,57 @@+"""+Factory module+"""++from .annoy import Annoy+from .hnsw import HNSW++# Conditionally import Faiss as it's only supported on Linux/macOS+try:+ from .faiss import Faiss++ FAISS = True+except ImportError:+ FAISS = False+++class ANNFactory:+ """+ Methods to create ANN models.+ """++ @staticmethod+ def create(config):+ """+ Create an ANN model.++ Args:+ config: index configuration parameters++ Returns:+ ANN+ """++ # ANN model+ model = None+ backend = config.get("backend")++ # Default backend if not provided, based on available libraries+ if not backend:+ backend = "faiss" if FAISS else "annoy"++ # Create ANN instance+ if backend == "annoy":+ model = Annoy(config)+ elif backend == "hnsw":+ model = HNSW(config)+ else:+ # Raise error if trying to create a Faiss index without Faiss installed+ if not FAISS:+ raise ImportError("Faiss library is not installed")++ model = Faiss(config)++ # Store config back+ config["backend"] = backend++ return modeldiff --git a/src/python/txtai/ann/faiss.py b/src/python/txtai/ann/faiss.pynew file mode 100644index 00000000..714f111c--- /dev/null+++ b/src/python/txtai/ann/faiss.py@@ -0,0 +1,59 @@+"""+Faiss module+"""++import numpy as np++from .base import ANN++# Conditionally import Faiss as it's only supported on Linux/macOS+try:+ from faiss import index_factory, METRIC_INNER_PRODUCT, read_index, write_index++ FAISS = True+except ImportError:+ FAISS = False+++class Faiss(ANN):+ """+ Builds an ANN model using the Faiss library.+ """++ def load(self, path):+ # Load index+ self.model = read_index(path)++ def index(self, embeddings):+ # Lookup components setting+ components = self.setting("components")++ # Create embeddings index. Inner product is equal to cosine similarity on normalized vectors.+ if components:+ params = components+ elif self.config.get("quantize"):+ params = "IVF100,SQ8" if embeddings.shape[0] >= 5000 else "IDMap,SQ8"+ else:+ params = "IVF100,Flat" if embeddings.shape[0] >= 5000 else "IDMap,Flat"++ self.model = index_factory(embeddings.shape[1], params, METRIC_INNER_PRODUCT)++ # Train model+ self.model.train(embeddings)+ self.model.add_with_ids(embeddings, np.array(range(embeddings.shape[0])))++ def search(self, queries, limit):+ # Run the query+ self.model.nprobe = self.setting("nprobe", 6)+ scores, ids = self.model.search(queries, limit)++ # Map results to [(id, score)]+ results = []+ for x, score in enumerate(scores):+ results.append(list(zip(ids[x].tolist(), score.tolist())))++ return results++ def save(self, path):+ # Write index+ write_index(self.model, path)diff --git a/src/python/txtai/ann/hnsw.py b/src/python/txtai/ann/hnsw.pynew file mode 100644index 00000000..2754c118--- /dev/null+++ b/src/python/txtai/ann/hnsw.py@@ -0,0 +1,60 @@+"""+HNSW module+"""++import numpy as np++# pylint: disable=E0611+from hnswlib import Index++from .base import ANN+++class HNSW(ANN):+ """+ Builds an ANN model using the hnswlib library.+ """++ def load(self, path):+ # Load index+ self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])+ self.model.load_index(path)++ def index(self, embeddings):+ # Inner product is equal to cosine similarity on normalized vectors+ self.config["metric"] = "ip"++ # Lookup index settings+ efconstruction = self.setting("efconstruction", 200)+ m = self.setting("m", 16)+ seed = self.setting("randomseed", 100)++ # Create index+ self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])+ self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)++ # Add items+ self.model.add_items(embeddings, np.array(range(embeddings.shape[0])))++ def search(self, queries, limit):+ # Set ef query param+ ef = self.setting("efsearch")+ if ef:+ self.model.set_ef(ef)++ # Run the query+ ids, distances = self.model.knn_query(queries, k=limit)++ # Map results to [(id, score)]+ results = []+ for x, distance in enumerate(distances):+ # Convert distances to similarity scores+ scores = [1 - d for d in distance]++ results.append(list(zip(ids[x], scores)))++ return results++ def save(self, path):+ # Write index+ self.model.save_index(path)diff --git a/src/python/txtai/api.py b/src/python/txtai/api.pyindex d03f71f8..a3bd7a24 100644--- a/src/python/txtai/api.py+++ b/src/python/txtai/api.py@@ -13,8 +13,7 @@ import yamlfrom fastapi import Body, FastAPI, Requestfrom .embeddings import Embeddings-from .extractor import Extractor-from .pipeline import Labels, Similarity+from .pipeline import Extractor, Labels, Similarity# API instanceapp = FastAPI()diff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings.pyindex 8efffc2d..8588de9d 100644--- a/src/python/txtai/embeddings.py+++ b/src/python/txtai/embeddings.py@@ -10,9 +10,9 @@ import numpy as npfrom sklearn.decomposition import TruncatedSVD-from .ann import ANN-from .scoring import Scoring-from .vectors import Vectors+from .ann import ANNFactory+from .scoring import ScoringFactory+from .vectors import VectorsFactoryclass Embeddings:@@ -44,7 +44,7 @@ class Embeddings:self.lsa = None# Embedding scoring method - weighs each word in a sentence- self.scoring = Scoring.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None+ self.scoring = ScoringFactory.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None# Sentence vectors modelself.model = self.loadVectors() if self.config else None@@ -57,7 +57,7 @@ class Embeddings:vector model"""- return Vectors.create(self.config, self.scoring)+ return VectorsFactory.create(self.config, self.scoring)def score(self, documents):"""@@ -104,7 +104,7 @@ class Embeddings:self.config["dimensions"] = dimensions# Create embeddings index- self.embeddings = ANN.create(self.config)+ self.embeddings = ANNFactory.create(self.config)# Build the indexself.embeddings.index(embeddings)@@ -307,7 +307,7 @@ class Embeddings:self.config["path"] = os.path.join(path, self.config["path"])# Sentence embeddings index- self.embeddings = ANN.create(self.config)+ self.embeddings = ANNFactory.create(self.config)self.embeddings.load("%s/embeddings" % path)# Dimensionality reduction@@ -317,7 +317,7 @@ class Embeddings:# Embedding scoringif self.config.get("scoring"):- self.scoring = Scoring.create(self.config["scoring"])+ self.scoring = ScoringFactory.create(self.config["scoring"])self.scoring.load(path)# Sentence vectors model - transforms text into sentence embeddingsdiff --git a/src/python/txtai/extractor/__init__.py b/src/python/txtai/extractor/__init__.pynew file mode 100644index 00000000..e83805a6--- /dev/null+++ b/src/python/txtai/extractor/__init__.py@@ -0,0 +1,5 @@+"""+Defined to maintain backwards compatibility+"""++from ..pipeline.extractor import Extractordiff --git a/src/python/txtai/pipeline.py b/src/python/txtai/pipeline.pydeleted file mode 100644index 535698bf..00000000--- a/src/python/txtai/pipeline.py+++ /dev/null@@ -1,162 +0,0 @@-"""-Pipeline module-"""--import numpy as np-import torch--from transformers import pipeline---class Pipeline:- """- Light wrapper around Hugging Face's pipeline component for selected tasks. Adds support for model- quantization and minor interface changes.- """-- def __init__(self, task, path=None, quantize=False, gpu=False, model=None):- """- Loads a new pipeline model.-- Args:- task: pipeline task or category- path: optional path to model, accepts Hugging Face model hub id or local path,- uses default model for task if not provided- quantize: if model should be quantized, defaults to False- gpu: if gpu inference should be used (only works if GPUs are available)- model: optional existing pipeline model to wrap- """-- if model:- # Check if input model is a Pipeline or a HF pipeline- self.pipeline = model.pipeline if isinstance(model, Pipeline) else model- else:- # Enable GPU inference if explicitly set and a GPU is available- gpu = gpu and torch.cuda.is_available()-- # Transformer pipeline task- self.pipeline = pipeline(task, model=path, tokenizer=path, device=0 if gpu else -1)-- # Model quantization. Compresses model to int8 precision, improves runtime performance. Only supported on CPU.- if not gpu and quantize:- # pylint: disable=E1101- self.pipeline.model = torch.quantization.quantize_dynamic(self.pipeline.model, {torch.nn.Linear}, dtype=torch.qint8)---class Questions(Pipeline):- """- Runs extractive QA for a series of questions and contexts.- """-- def __init__(self, path=None, quantize=False, gpu=False, model=None):- super().__init__("question-answering", path, quantize, gpu, model)-- def __call__(self, questions, contexts):- """- Runs a extractive question-answering model against each question-context pair, finding the best answers.-- Args:- questions: list of questions- contexts: list of contexts to pull answers from-- Returns:- list of answers- """-- answers = []-- for x, question in enumerate(questions):- if question and contexts[x]:- # Run the QA pipeline- result = self.pipeline(question=question, context=contexts[x])-- # Get answer and score- answer, score = result["answer"], result["score"]-- # Require score to be at least 0.05- if score < 0.05:- answer = None-- # Add answer- answers.append(answer)- else:- answers.append(None)-- return answers---class Labels(Pipeline):- """- Applies a zero shot classifier to text using a list of labels.- """-- def __init__(self, path=None, quantize=False, gpu=True, model=None):- super().__init__("zero-shot-classification", path, quantize, gpu, model)-- def __call__(self, text, labels, multiclass=False):- """- Applies a zero shot classifier to text using a list of labels. Returns a list of- (id, score) sorted by highest score, where id is the index in labels.-- This method supports text as a string or a list. If the input is a string, the return- type is a 1D list of (id, score). If text is a list, a 2D list of (id, score) is- returned with a row per string.-- Args:- text: text|list- labels: list of labels-- Returns:- list of (id, score)- """-- # Run ZSL pipeline- results = self.pipeline(text, labels, multi_class=multiclass)-- # Convert results to a list if necessary- if not isinstance(results, list):- results = [results]-- # Build list of (id, score)- scores = []- for result in results:- scores.append([(labels.index(label), result["scores"][x]) for x, label in enumerate(result["labels"])])-- return scores[0] if isinstance(text, str) else scores---class Similarity(Labels):- """- Computes similarity between query and list of text using a zero shot classifier.- """-- def __call__(self, query, texts, multiclass=True):- """- Computes the similarity between query and list of text. Returns a list of- (id, score) sorted by highest score, where id is the index in texts.-- This method supports query as a string or a list. If the input is a string,- the return type is a 1D list of (id, score). If text is a list, a 2D list- of (id, score) is returned with a row per string.-- Args:- query: query text|list- texts: list of texts-- Returns:- list of (id, score)- """-- # Call Labels pipeline for texts using input query as the candidate label- scores = super().__call__(texts, [query] if isinstance(query, str) else query, multiclass)-- # Sort on query index id- scores = [[score for _, score in sorted(row)] for row in scores]-- # Transpose axes to get a list of text scores for each query- scores = np.array(scores).T.tolist()-- # Build list of (id, score) per query sorted by highest score- scores = [sorted(enumerate(row), key=lambda x: x[1], reverse=True) for row in scores]-- return scores[0] if isinstance(query, str) else scoresdiff --git a/src/python/txtai/pipeline/__init__.py b/src/python/txtai/pipeline/__init__.pynew file mode 100644index 00000000..78010353--- /dev/null+++ b/src/python/txtai/pipeline/__init__.py@@ -0,0 +1,11 @@+"""+Pipeline imports+"""++from .base import Pipeline+from .extractor import Extractor+from .hfpipeline import HFPipeline+from .labels import Labels+from .questions import Questions+from .similarity import Similarity+from .tokenizer import Tokenizerdiff --git a/src/python/txtai/pipeline/base.py b/src/python/txtai/pipeline/base.pynew file mode 100644index 00000000..6fad9358--- /dev/null+++ b/src/python/txtai/pipeline/base.py@@ -0,0 +1,9 @@+"""+Pipeline module+"""+++class Pipeline:+ """+ Base class for all Pipelines. The only interface requirement is to define a __call___ method.+ """diff --git a/src/python/txtai/extractor.py b/src/python/txtai/pipeline/extractor.pysimilarity index 98%rename from src/python/txtai/extractor.pyrename to src/python/txtai/pipeline/extractor.pyindex 98a7bf49..40bc7fc8 100644--- a/src/python/txtai/extractor.py+++ b/src/python/txtai/pipeline/extractor.py@@ -4,11 +4,12 @@ Extractor modulefrom nltk.tokenize import sent_tokenize-from .pipeline import Questions+from .base import Pipeline+from .questions import Questionsfrom .tokenizer import Tokenizer-class Extractor:+class Extractor(Pipeline):"""Class that uses an extractive question-answering model to extract content from a given text context."""diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.pynew file mode 100644index 00000000..c701d3d3--- /dev/null+++ b/src/python/txtai/pipeline/hfpipeline.py@@ -0,0 +1,44 @@+"""+Hugging Face pipeline wrapper module+"""++import torch++from transformers import pipeline++from .base import Pipeline+++class HFPipeline(Pipeline):+ """+ Light wrapper around Hugging Face's pipeline component for selected tasks. Adds support for model+ quantization and minor interface changes.+ """++ def __init__(self, task, path=None, quantize=False, gpu=False, model=None):+ """+ Loads a new pipeline model.++ Args:+ task: pipeline task or category+ path: optional path to model, accepts Hugging Face model hub id or local path,+ uses default model for task if not provided+ quantize: if model should be quantized, defaults to False+ gpu: if gpu inference should be used (only works if GPUs are available)+ model: optional existing pipeline model to wrap+ """++ if model:+ # Check if input model is a Pipeline or a HF pipeline+ self.pipeline = model.pipeline if isinstance(model, Pipeline) else model+ else:+ # Enable GPU inference if explicitly set and a GPU is available+ gpu = gpu and torch.cuda.is_available()++ # Transformer pipeline task+ self.pipeline = pipeline(task, model=path, tokenizer=path, device=0 if gpu else -1)++ # Model quantization. Compresses model to int8 precision, improves runtime performance. Only supported on CPU.+ if not gpu and quantize:+ # pylint: disable=E1101+ self.pipeline.model = torch.quantization.quantize_dynamic(self.pipeline.model, {torch.nn.Linear}, dtype=torch.qint8)diff --git a/src/python/txtai/pipeline/labels.py b/src/python/txtai/pipeline/labels.pynew file mode 100644index 00000000..ef657c9e--- /dev/null+++ b/src/python/txtai/pipeline/labels.py@@ -0,0 +1,45 @@+"""+Labels module+"""++from .hfpipeline import HFPipeline+++class Labels(HFPipeline):+ """+ Applies a zero shot classifier to text using a list of labels.+ """++ def __init__(self, path=None, quantize=False, gpu=True, model=None):+ super().__init__("zero-shot-classification", path, quantize, gpu, model)++ def __call__(self, text, labels, multiclass=False):+ """+ Applies a zero shot classifier to text using a list of labels. Returns a list of+ (id, score) sorted by highest score, where id is the index in labels.++ This method supports text as a string or a list. If the input is a string, the return+ type is a 1D list of (id, score). If text is a list, a 2D list of (id, score) is+ returned with a row per string.++ Args:+ text: text|list+ labels: list of labels++ Returns:+ list of (id, score)+ """++ # Run ZSL pipeline+ results = self.pipeline(text, labels, multi_class=multiclass)++ # Convert results to a list if necessary+ if not isinstance(results, list):+ results = [results]++ # Build list of (id, score)+ scores = []+ for result in results:+ scores.append([(labels.index(label), result["scores"][x]) for x, label in enumerate(result["labels"])])++ return scores[0] if isinstance(text, str) else scoresdiff --git a/src/python/txtai/pipeline/questions.py b/src/python/txtai/pipeline/questions.pynew file mode 100644index 00000000..db23fb3a--- /dev/null+++ b/src/python/txtai/pipeline/questions.py@@ -0,0 +1,47 @@+"""+Questions module+"""++from .hfpipeline import HFPipeline+++class Questions(HFPipeline):+ """+ Runs extractive QA for a series of questions and contexts.+ """++ def __init__(self, path=None, quantize=False, gpu=False, model=None):+ super().__init__("question-answering", path, quantize, gpu, model)++ def __call__(self, questions, contexts):+ """+ Runs a extractive question-answering model against each question-context pair, finding the best answers.++ Args:+ questions: list of questions+ contexts: list of contexts to pull answers from++ Returns:+ list of answers+ """++ answers = []++ for x, question in enumerate(questions):+ if question and contexts[x]:+ # Run the QA pipeline+ result = self.pipeline(question=question, context=contexts[x])++ # Get answer and score+ answer, score = result["answer"], result["score"]++ # Require score to be at least 0.05+ if score < 0.05:+ answer = None++ # Add answer+ answers.append(answer)+ else:+ answers.append(None)++ return answersdiff --git a/src/python/txtai/pipeline/similarity.py b/src/python/txtai/pipeline/similarity.pynew file mode 100644index 00000000..2784fda7--- /dev/null+++ b/src/python/txtai/pipeline/similarity.py@@ -0,0 +1,44 @@+"""+Similarity module+"""++import numpy as np++from .labels import Labels+++class Similarity(Labels):+ """+ Computes similarity between query and list of text using a zero shot classifier.+ """++ def __call__(self, query, texts, multiclass=True):+ """+ Computes the similarity between query and list of text. Returns a list of+ (id, score) sorted by highest score, where id is the index in texts.++ This method supports query as a string or a list. If the input is a string,+ the return type is a 1D list of (id, score). If text is a list, a 2D list+ of (id, score) is returned with a row per string.++ Args:+ query: query text|list+ texts: list of text++ Returns:+ list of (id, score)+ """++ # Call Labels pipeline for texts using input query as the candidate label+ scores = super().__call__(texts, [query] if isinstance(query, str) else query, multiclass)++ # Sort on query index id+ scores = [[score for _, score in sorted(row)] for row in scores]++ # Transpose axes to get a list of text scores for each query+ scores = np.array(scores).T.tolist()++ # Build list of (id, score) per query sorted by highest score+ scores = [sorted(enumerate(row), key=lambda x: x[1], reverse=True) for row in scores]++ return scores[0] if isinstance(query, str) else scoresdiff --git a/src/python/txtai/tokenizer.py b/src/python/txtai/pipeline/tokenizer.pysimilarity index 73%rename from src/python/txtai/tokenizer.pyrename to src/python/txtai/pipeline/tokenizer.pyindex ff9a67de..197cfa51 100644--- a/src/python/txtai/tokenizer.py+++ b/src/python/txtai/pipeline/tokenizer.py@@ -5,10 +5,12 @@ Tokenizer moduleimport reimport string+from .base import Pipeline-class Tokenizer:++class Tokenizer(Pipeline):"""- Text tokenization methods.+ Tokenizes text into a list of tokens. Primarily designed for English text."""# fmt: off@@ -30,6 +32,19 @@ class Tokenizer:list of tokens"""+ return Tokenizer()(text)++ def __call__(self, text):+ """+ Tokenizes input text into a list of tokens. Filters tokens that match a specific pattern and removes stop words.++ Args:+ text: input text++ Returns:+ list of tokens+ """+# Convert to all lowercase, split on whitespace, strip punctuationtokens = [token.strip(string.punctuation) for token in text.lower().split()]diff --git a/src/python/txtai/scoring/__init__.py b/src/python/txtai/scoring/__init__.pynew file mode 100644index 00000000..c4850c9b--- /dev/null+++ b/src/python/txtai/scoring/__init__.py@@ -0,0 +1,8 @@+"""+Scoring imports+"""++from .base import Scoring+from .bm25 import BM25+from .factory import ScoringFactory+from .sif import SIFdiff --git a/src/python/txtai/scoring.py b/src/python/txtai/scoring/base.pysimilarity index 75%rename from src/python/txtai/scoring.pyrename to src/python/txtai/scoring/base.pyindex 3b5958dd..ded6d4cc 100644--- a/src/python/txtai/scoring.py+++ b/src/python/txtai/scoring/base.py@@ -7,7 +7,7 @@ import picklefrom collections import Counter-from .tokenizer import Tokenizer+from ..pipeline.tokenizer import Tokenizerclass Scoring:@@ -15,28 +15,6 @@ class Scoring:Base scoring object. Default method scores documents using TF-IDF."""- @staticmethod- def create(method):- """- Factory method to construct a Scoring object.-- Args:- method: scoring method (bm25, sif, tfidf)-- Returns:- Scoring object- """-- if method == "bm25":- return BM25()- if method == "sif":- return SIF()- if method == "tfidf":- # Default scoring object implements tf-idf- return Scoring()-- return None-def __init__(self):"""Initializes backing statistic objects.@@ -194,42 +172,3 @@ class Scoring:"""return idf---class BM25(Scoring):- """- BM25 scoring. Scores using Apache Lucene's version of BM25 which adds 1 to prevent- negative scores.- """-- def __init__(self, k1=0.1, b=0.75):- super().__init__()-- # BM25 configurable parameters- self.k1 = k1- self.b = b-- def computeIDF(self, freq):- # Calculate BM25 IDF score- return math.log(1 + (self.total - freq + 0.5) / (freq + 0.5))-- def score(self, freq, idf, length):- # Calculate BM25 score- k = self.k1 * ((1 - self.b) + self.b * length / self.avgdl)- return idf * (freq * (self.k1 + 1)) / (freq + k)---class SIF(Scoring):- """- Smooth Inverse Frequency (SIF) scoring.- """-- def __init__(self, a=0.001):- super().__init__()-- # SIF configurable parameters- self.a = a-- def score(self, freq, idf, length):- # Calculate SIF score- return self.a / (self.a + freq / self.tokens)diff --git a/src/python/txtai/scoring/bm25.py b/src/python/txtai/scoring/bm25.pynew file mode 100644index 00000000..b5ba668e--- /dev/null+++ b/src/python/txtai/scoring/bm25.py@@ -0,0 +1,30 @@+"""+BM25 module+"""++import math++from .base import Scoring+++class BM25(Scoring):+ """+ BM25 scoring. Scores using Apache Lucene's version of BM25 which adds 1 to prevent+ negative scores.+ """++ def __init__(self, k1=0.1, b=0.75):+ super().__init__()++ # BM25 configurable parameters+ self.k1 = k1+ self.b = b++ def computeIDF(self, freq):+ # Calculate BM25 IDF score+ return math.log(1 + (self.total - freq + 0.5) / (freq + 0.5))++ def score(self, freq, idf, length):+ # Calculate BM25 score+ k = self.k1 * ((1 - self.b) + self.b * length / self.avgdl)+ return idf * (freq * (self.k1 + 1)) / (freq + k)diff --git a/src/python/txtai/scoring/factory.py b/src/python/txtai/scoring/factory.pynew file mode 100644index 00000000..413c69b5--- /dev/null+++ b/src/python/txtai/scoring/factory.py@@ -0,0 +1,35 @@+"""+Factory module+"""++from .base import Scoring+from .bm25 import BM25+from .sif import SIF+++class ScoringFactory:+ """+ Methods to create Scoring models.+ """++ @staticmethod+ def create(method):+ """+ Factory method to construct a Scoring object.++ Args:+ method: scoring method (bm25, sif, tfidf)++ Returns:+ Scoring+ """++ if method == "bm25":+ return BM25()+ if method == "sif":+ return SIF()+ if method == "tfidf":+ # Default scoring object implements tf-idf+ return Scoring()++ return Nonediff --git a/src/python/txtai/scoring/sif.py b/src/python/txtai/scoring/sif.pynew file mode 100644index 00000000..308ee899--- /dev/null+++ b/src/python/txtai/scoring/sif.py@@ -0,0 +1,21 @@+"""+SIF module+"""++from .base import Scoring+++class SIF(Scoring):+ """+ Smooth Inverse Frequency (SIF) scoring.+ """++ def __init__(self, a=0.001):+ super().__init__()++ # SIF configurable parameters+ self.a = a++ def score(self, freq, idf, length):+ # Calculate SIF score+ return self.a / (self.a + freq / self.tokens)diff --git a/src/python/txtai/tokenizer/__init__.py b/src/python/txtai/tokenizer/__init__.pynew file mode 100644index 00000000..0ec75bba--- /dev/null+++ b/src/python/txtai/tokenizer/__init__.py@@ -0,0 +1,5 @@+"""+Defined to maintain backwards compatibility+"""++from ..pipeline.tokenizer import Tokenizerdiff --git a/src/python/txtai/vectors.py b/src/python/txtai/vectors.pydeleted file mode 100644index bccd710a..00000000--- a/src/python/txtai/vectors.py+++ /dev/null@@ -1,313 +0,0 @@-"""-Vectors module-"""--import os-import pickle-import tempfile--from errno import ENOENT-from multiprocessing import Pool--import fasttext-import numpy as np--from pymagnitude import converter, Magnitude-from sentence_transformers import SentenceTransformer-from sentence_transformers.models import Pooling, Transformer--from .tokenizer import Tokenizer--# Multiprocessing helper methods-# pylint: disable=W0603-VECTORS = None---def create(config, scoring):- """- Multiprocessing helper method. Creates a global embeddings object to be accessed in a new subprocess.-- Args:- config: vector configuration- scoring: scoring instance- """-- global VECTORS-- # Create a global embedding object using configuration and saved- VECTORS = WordVectors(config, scoring)---def transform(document):- """- Multiprocessing helper method. Transforms document into an embeddings vector.-- Args:- document: (id, text|tokens, tags)-- Returns:- (id, embedding)- """-- global VECTORS-- return (document[0], VECTORS.transform(document))---class Vectors:- """- Base class for sentence embeddings/vector models.- """-- @staticmethod- def create(config, scoring):- """- Create a Vectors model instance.-- Args:- config: vector configuration- scoring: scoring instance- """-- # Derive vector type- transformers = config.get("method") == "transformers"-- # Create vector model instance- return TransformersVectors(config, scoring) if transformers else WordVectors(config, scoring)-- def __init__(self, config, scoring):- # Store parameters- self.config = config- self.scoring = scoring-- # Detect if this is an initialized configuration- self.initialized = "dimensions" in config-- # Enables optional string tokenization- self.tokenize = "tokenize" not in config or config["tokenize"]-- # pylint: disable=E1111- self.model = self.load(config["path"])-- def load(self, path):- """- Loads vector model at path.-- Args:- path: path to word vector model-- Returns:- vector model- """-- def index(self, documents):- """- Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,- number of dimensions and temporary file with embeddings.-- Args:- documents: list of (id, text|tokens, tags)-- Returns:- (ids, dimensions, stream)- """-- def transform(self, document):- """- Transforms document into an embeddings vector.-- Args:- document: (id, text|tokens, tags)-- Returns:- embeddings vector- """---class WordVectors(Vectors):- """- Builds sentence embeddings/vectors using weighted word embeddings.- """-- def load(self, path):- # Ensure that vector path exists- if not path or not os.path.isfile(path):- raise IOError(ENOENT, "Vector model file not found", path)-- # Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded- return Magnitude(path, case_insensitive=True, blocking=not self.initialized)-- def index(self, documents):- ids, dimensions, stream = [], None, None-- # Shared objects with Pool- args = (self.config, self.scoring)-- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage- with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:- stream = output.name- for uid, embedding in pool.imap(transform, documents):- if not dimensions:- # Set number of dimensions for embeddings- dimensions = embedding.shape[0]-- ids.append(uid)- pickle.dump(embedding, output)-- return (ids, dimensions, stream)-- def transform(self, document):- # Convert to tokens if necessary- if isinstance(document[1], str):- document = (document[0], Tokenizer.tokenize(document[1]), document[2])-- # Generate weights for each vector using a scoring method- weights = self.scoring.weights(document) if self.scoring else None-- # pylint: disable=E1133- if weights and [x for x in weights if x > 0]:- # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.- embedding = np.average(self.lookup(document[1]), weights=np.array(weights, dtype=np.float32), axis=0)- else:- # If no weights, use mean- embedding = np.mean(self.lookup(document[1]), axis=0)-- return embedding-- def lookup(self, tokens):- """- Queries word vectors for given list of input tokens.-- Args:- tokens: list of tokens to query-- Returns:- word vectors array- """-- return self.model.query(tokens)-- @staticmethod- def build(data, size, mincount, path):- """- Builds fastText vectors from a file.-- Args:- data: path to input data file- size: number of vector dimensions- mincount: minimum number of occurrences required to register a token- path: path to output file- """-- # Train on data file using largest dimension size- model = fasttext.train_unsupervised(data, dim=size, minCount=mincount)-- # Output file path- print("Building %d dimension model" % size)-- # Output vectors in vec/txt format- with open(path + ".txt", "w") as output:- words = model.get_words()- output.write("%d %d\n" % (len(words), model.get_dimension()))-- for word in words:- # Skip end of line token- if word != "</s>":- vector = model.get_word_vector(word)- data = ""- for v in vector:- data += " " + str(v)-- output.write(word + data + "\n")-- # Build magnitude vectors database- print("Converting vectors to magnitude format")- converter.convert(path + ".txt", path + ".magnitude", subword=True)---class TransformersVectors(Vectors):- """- Builds sentence embeddings/vectors using the transformers library.- """-- def load(self, path):- model = Transformer(path)- pooling = Pooling(model.get_word_embedding_dimension())-- return SentenceTransformer(modules=[model, pooling])-- def index(self, documents):- ids, dimensions, stream = [], None, None-- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:- stream = output.name- batch = []- for document in documents:- batch.append(document)-- if len(batch) == 500:- # Convert batch to embeddings- uids, dimensions = self.batch(batch, output)- ids.extend(uids)-- batch = []-- # Final batch- if batch:- uids, dimensions = self.batch(batch, output)- ids.extend(uids)-- return (ids, dimensions, stream)-- def transform(self, document):- # Convert input document to text and build embeddings- return self.model.encode([self.text(document[1])], show_progress_bar=False)[0]-- def batch(self, documents, output):- """- Builds a batch of embeddings.-- Args:- documents: list of documents used to build embeddings- output: output temp file to store embeddings-- Returns:- (ids, dimensions) list of ids and number of dimensions in embeddings- """-- # Extract ids and convert input documents to text- ids = [uid for uid, _, _ in documents]- documents = [self.text(text) for _, text, _ in documents]- dimensions = None-- # Build embeddings- embeddings = self.model.encode(documents, show_progress_bar=False)- for embedding in embeddings:- if not dimensions:- # Set number of dimensions for embeddings- dimensions = embedding.shape[0]-- pickle.dump(embedding, output)-- return (ids, dimensions)-- def text(self, text):- """- Converts input into text that can be processed by transformer models. This method supports- optional string tokenization and joins tokenized input into text.-- Args:- text: text|tokens- """-- # Optional string tokenization- if self.tokenize and isinstance(text, str):- text = Tokenizer.tokenize(text)-- # Transformer models require string input- if isinstance(text, list):- text = " ".join(text)-- return textdiff --git a/src/python/txtai/vectors/__init__.py b/src/python/txtai/vectors/__init__.pynew file mode 100644index 00000000..5bedd266--- /dev/null+++ b/src/python/txtai/vectors/__init__.py@@ -0,0 +1,8 @@+"""+Vectors import+"""++from .base import Vectors+from .factory import VectorsFactory+from .transformers import TransformersVectors+from .words import WordVectorsdiff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.pynew file mode 100644index 00000000..cbb101c0--- /dev/null+++ b/src/python/txtai/vectors/base.py@@ -0,0 +1,57 @@+"""+Vectors module+"""+++class Vectors:+ """+ Base class for sentence embeddings/vector models.+ """++ def __init__(self, config, scoring):+ # Store parameters+ self.config = config+ self.scoring = scoring++ # Detect if this is an initialized configuration+ self.initialized = "dimensions" in config++ # Enables optional string tokenization+ self.tokenize = "tokenize" not in config or config["tokenize"]++ # pylint: disable=E1111+ self.model = self.load(config["path"])++ def load(self, path):+ """+ Loads vector model at path.++ Args:+ path: path to word vector model++ Returns:+ vector model+ """++ def index(self, documents):+ """+ Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,+ number of dimensions and temporary file with embeddings.++ Args:+ documents: list of (id, text|tokens, tags)++ Returns:+ (ids, dimensions, stream)+ """++ def transform(self, document):+ """+ Transforms document into an embeddings vector.++ Args:+ document: (id, text|tokens, tags)++ Returns:+ embeddings vector+ """diff --git a/src/python/txtai/vectors/factory.py b/src/python/txtai/vectors/factory.pynew file mode 100644index 00000000..41417fe7--- /dev/null+++ b/src/python/txtai/vectors/factory.py@@ -0,0 +1,31 @@+"""+Factory module+"""++from .transformers import TransformersVectors+from .words import WordVectors+++class VectorsFactory:+ """+ Methods to create Vectors models.+ """++ @staticmethod+ def create(config, scoring):+ """+ Create a Vectors model instance.++ Args:+ config: vector configuration+ scoring: scoring instance++ Returns:+ Vectors+ """++ # Derive vector type+ transformers = config.get("method") == "transformers"++ # Create vector model instance+ return TransformersVectors(config, scoring) if transformers else WordVectors(config, scoring)diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.pynew file mode 100644index 00000000..7e17f06f--- /dev/null+++ b/src/python/txtai/vectors/transformers.py@@ -0,0 +1,100 @@+"""+Transformers module+"""++import pickle+import tempfile++from sentence_transformers import SentenceTransformer+from sentence_transformers.models import Pooling, Transformer++from ..pipeline.tokenizer import Tokenizer++from .base import Vectors+++class TransformersVectors(Vectors):+ """+ Builds sentence embeddings/vectors using the transformers library.+ """++ def load(self, path):+ model = Transformer(path)+ pooling = Pooling(model.get_word_embedding_dimension())++ return SentenceTransformer(modules=[model, pooling])++ def index(self, documents):+ ids, dimensions, stream = [], None, None++ # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:+ stream = output.name+ batch = []+ for document in documents:+ batch.append(document)++ if len(batch) == 500:+ # Convert batch to embeddings+ uids, dimensions = self.batch(batch, output)+ ids.extend(uids)++ batch = []++ # Final batch+ if batch:+ uids, dimensions = self.batch(batch, output)+ ids.extend(uids)++ return (ids, dimensions, stream)++ def transform(self, document):+ # Convert input document to text and build embeddings+ return self.model.encode([self.text(document[1])], show_progress_bar=False)[0]++ def batch(self, documents, output):+ """+ Builds a batch of embeddings.++ Args:+ documents: list of documents used to build embeddings+ output: output temp file to store embeddings++ Returns:+ (ids, dimensions) list of ids and number of dimensions in embeddings+ """++ # Extract ids and convert input documents to text+ ids = [uid for uid, _, _ in documents]+ documents = [self.text(text) for _, text, _ in documents]+ dimensions = None++ # Build embeddings+ embeddings = self.model.encode(documents, show_progress_bar=False)+ for embedding in embeddings:+ if not dimensions:+ # Set number of dimensions for embeddings+ dimensions = embedding.shape[0]++ pickle.dump(embedding, output)++ return (ids, dimensions)++ def text(self, text):+ """+ Converts input into text that can be processed by transformer models. This method supports+ optional string tokenization and joins tokenized input into text.++ Args:+ text: text|tokens+ """++ # Optional string tokenization+ if self.tokenize and isinstance(text, str):+ text = Tokenizer.tokenize(text)++ # Transformer models require string input+ if isinstance(text, list):+ text = " ".join(text)++ return textdiff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.pynew file mode 100644index 00000000..1cac062e--- /dev/null+++ b/src/python/txtai/vectors/words.py@@ -0,0 +1,155 @@+"""+Word Vectors module+"""++import os+import pickle+import tempfile++from errno import ENOENT+from multiprocessing import Pool++import fasttext+import numpy as np++from pymagnitude import converter, Magnitude++from .base import Vectors+from ..pipeline.tokenizer import Tokenizer++# Multiprocessing helper methods+# pylint: disable=W0603+VECTORS = None+++def create(config, scoring):+ """+ Multiprocessing helper method. Creates a global embeddings object to be accessed in a new subprocess.++ Args:+ config: vector configuration+ scoring: scoring instance+ """++ global VECTORS++ # Create a global embedding object using configuration and saved+ VECTORS = WordVectors(config, scoring)+++def transform(document):+ """+ Multiprocessing helper method. Transforms document into an embeddings vector.++ Args:+ document: (id, text|tokens, tags)++ Returns:+ (id, embedding)+ """++ global VECTORS++ return (document[0], VECTORS.transform(document))+++class WordVectors(Vectors):+ """+ Builds sentence embeddings/vectors using weighted word embeddings.+ """++ def load(self, path):+ # Ensure that vector path exists+ if not path or not os.path.isfile(path):+ raise IOError(ENOENT, "Vector model file not found", path)++ # Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded+ return Magnitude(path, case_insensitive=True, blocking=not self.initialized)++ def index(self, documents):+ ids, dimensions, stream = [], None, None++ # Shared objects with Pool+ args = (self.config, self.scoring)++ # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage+ with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:+ stream = output.name+ for uid, embedding in pool.imap(transform, documents):+ if not dimensions:+ # Set number of dimensions for embeddings+ dimensions = embedding.shape[0]++ ids.append(uid)+ pickle.dump(embedding, output)++ return (ids, dimensions, stream)++ def transform(self, document):+ # Convert to tokens if necessary+ if isinstance(document[1], str):+ document = (document[0], Tokenizer.tokenize(document[1]), document[2])++ # Generate weights for each vector using a scoring method+ weights = self.scoring.weights(document) if self.scoring else None++ # pylint: disable=E1133+ if weights and [x for x in weights if x > 0]:+ # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.+ embedding = np.average(self.lookup(document[1]), weights=np.array(weights, dtype=np.float32), axis=0)+ else:+ # If no weights, use mean+ embedding = np.mean(self.lookup(document[1]), axis=0)++ return embedding++ def lookup(self, tokens):+ """+ Queries word vectors for given list of input tokens.++ Args:+ tokens: list of tokens to query++ Returns:+ word vectors array+ """++ return self.model.query(tokens)++ @staticmethod+ def build(data, size, mincount, path):+ """+ Builds fastText vectors from a file.++ Args:+ data: path to input data file+ size: number of vector dimensions+ mincount: minimum number of occurrences required to register a token+ path: path to output file+ """++ # Train on data file using largest dimension size+ model = fasttext.train_unsupervised(data, dim=size, minCount=mincount)++ # Output file path+ print("Building %d dimension model" % size)++ # Output vectors in vec/txt format+ with open(path + ".txt", "w") as output:+ words = model.get_words()+ output.write("%d %d\n" % (len(words), model.get_dimension()))++ for word in words:+ # Skip end of line token+ if word != "</s>":+ vector = model.get_word_vector(word)+ data = ""+ for v in vector:+ data += " " + str(v)++ output.write(word + data + "\n")++ # Build magnitude vectors database+ print("Converting vectors to magnitude format")+ converter.convert(path + ".txt", path + ".magnitude", subword=True)diff --git a/test/python/testann.py b/test/python/testann.pyindex 6386b631..526080ea 100644--- a/test/python/testann.py+++ b/test/python/testann.py@@ -8,7 +8,7 @@ import unittestimport numpy as np-from txtai.ann import ANN+from txtai.ann import ANNFactoryclass TestANN(unittest.TestCase):@@ -104,7 +104,7 @@ class TestANN(unittest.TestCase):if params:config.update(params)- model = ANN.create(config)+ model = ANNFactory.create(config)model.index(data)return modeldiff --git a/test/python/testscoring.py b/test/python/testscoring.pyindex d4beeabf..f5f3c203 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring.py@@ -6,7 +6,7 @@ import osimport tempfileimport unittest-from txtai.scoring import Scoring+from txtai.scoring import ScoringFactoryclass TestScoring(unittest.TestCase):@@ -61,7 +61,7 @@ class TestScoring(unittest.TestCase):Test unknown method"""- self.assertIsNone(Scoring.create("unknown"))+ self.assertIsNone(ScoringFactory.create("unknown"))def method(self, method, data=None):"""@@ -78,7 +78,7 @@ class TestScoring(unittest.TestCase):# Derive input datadata = data if data else self.data- model = Scoring.create(method)+ model = ScoringFactory.create(method)model.index(data)keys = [k for k, v in sorted(model.idf.items(), key=lambda x: x[1])]diff --git a/test/python/testtransformers.py b/test/python/testtransformers.pyindex e63f5740..aa963e1a 100644--- a/test/python/testtransformers.py+++ b/test/python/testtransformers.py@@ -7,7 +7,7 @@ import unittestimport numpy as np-from txtai.vectors import Vectors+from txtai.vectors import VectorsFactoryclass TestTransformersVectors(unittest.TestCase):@@ -21,7 +21,7 @@ class TestTransformersVectors(unittest.TestCase):Create single TransformersVectors instance."""- cls.model = Vectors.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)+ cls.model = VectorsFactory.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)def testIndex(self):"""diff --git a/test/python/testwordvectors.py b/test/python/testwordvectors.pyindex 9b23062c..51c8d7ca 100644--- a/test/python/testwordvectors.py+++ b/test/python/testwordvectors.py@@ -6,7 +6,7 @@ import osimport tempfileimport unittest-from txtai.vectors import WordVectors, Vectors+from txtai.vectors import WordVectors, VectorsFactoryclass TestWordVectors(unittest.TestCase):@@ -35,13 +35,13 @@ class TestWordVectors(unittest.TestCase):"""config = {"path": self.path}- model = Vectors.create(config, None)+ model = VectorsFactory.create(config, None)self.assertFalse(model.initialized)config["ids"] = ["0", "1"]config["dimensions"] = 300- model = Vectors.create(config, None)+ model = VectorsFactory.create(config, None)self.assertTrue(model.initialized)@@ -53,7 +53,7 @@ class TestWordVectors(unittest.TestCase):# Generate datadocuments = [(x, "This is a test", None) for x in range(1000)]- model = Vectors.create({"path": self.path}, None)+ model = VectorsFactory.create({"path": self.path}, None)ids, dimension, stream = model.index(documents)@@ -66,7 +66,7 @@ class TestWordVectors(unittest.TestCase):Test word vector lookup"""- model = Vectors.create({"path": self.path}, None)+ model = VectorsFactory.create({"path": self.path}, None)self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))def testNoExist(self):@@ -76,12 +76,12 @@ class TestWordVectors(unittest.TestCase):# Test non-existent path raises an exceptionwith self.assertRaises(IOError):- Vectors.create({"path": os.path.join(tempfile.gettempdir(), "noexist")}, None)+ VectorsFactory.create({"path": os.path.join(tempfile.gettempdir(), "noexist")}, None)def testTransform(self):"""Test word vector transform"""- model = Vectors.create({"path": self.path}, None)+ model = VectorsFactory.create({"path": self.path}, None)self.assertEqual(len(model.transform((None, ["txtai"], None))), 300)