💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add object storage, closes #183
davidmezzetti119306e911c8
diff --git a/docs/embeddings.md b/docs/embeddings.mdindex aeee6afb..fd6b775b 100644--- a/docs/embeddings.md+++ b/docs/embeddings.md@@ -1,5 +1,5 @@# Embeddings-Embeddings is the engine that delivers semantic search. Text is transformed into embeddings vectors where similar concepts will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used find results that have the same meaning, not necessarily the same keywords.+Embeddings is the engine that delivers semantic search. Data is transformed into embeddings vectors where similar concepts will produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used find results that have the same meaning, not necessarily the same keywords.Embeddings parameters are set through the constructor. Examples below.diff --git a/docs/install.md b/docs/install.mdindex aa0acc90..1c60896f 100644--- a/docs/install.md+++ b/docs/install.md@@ -49,6 +49,22 @@ Serve txtai via a web API.pip install txtai[api]```+### Database++Additional content storage options++```+pip install txtai[database]+```++### Model++Additional non-standard models++```+pip install txtai[model]+```+### PipelineAll pipelines - default install comes with most common pipelines.diff --git a/setup.py b/setup.pyindex f67277d6..057566c7 100644--- a/setup.py+++ b/setup.py@@ -19,6 +19,8 @@ extras["api"] = ["uvicorn>=0.12.1",]+extras["database"] = ["pillow>=7.2.0"]+extras["model"] = ["onnxruntime>=1.8.1"]extras["pipeline"] = [@@ -47,7 +49,7 @@ extras["similarity"] = [extras["workflow"] = ["apache-libcloud>=3.3.1", "pillow>=7.2.0", "requests>=2.26.0", "xmltodict>=0.12.0"]-extras["all"] = extras["api"] + extras["model"] + extras["pipeline"] + extras["similarity"] + extras["workflow"]+extras["all"] = extras["api"] + extras["database"] + extras["model"] + extras["pipeline"] + extras["similarity"] + extras["workflow"]setup(name="txtai",diff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/annoy.pyindex 89e50bbe..f31f8b06 100644--- a/src/python/txtai/ann/annoy.py+++ b/src/python/txtai/ann/annoy.py@@ -12,6 +12,7 @@ except ImportError:from .base import ANN+# pylint: disable=W0223class Annoy(ANN):"""diff --git a/src/python/txtai/database/__init__.py b/src/python/txtai/database/__init__.pyindex 465f6b3a..6ac4f3a8 100644--- a/src/python/txtai/database/__init__.py+++ b/src/python/txtai/database/__init__.py@@ -3,6 +3,7 @@ Database imports"""from .base import Database+from .encoder import *from .factory import DatabaseFactoryfrom .sqlite import SQLitefrom .sql import *diff --git a/src/python/txtai/database/base.py b/src/python/txtai/database/base.pyindex a2cd3676..c0ac6a87 100644--- a/src/python/txtai/database/base.py+++ b/src/python/txtai/database/base.py@@ -4,6 +4,7 @@ Database moduleimport logging+from .encoder import EncoderFactoryfrom .sql import Expression, SQL, SQLException# Logging configuration@@ -31,6 +32,10 @@ class Database:# SQL parserself.sql = SQL(self)+ # Load objects encoder+ encoder = self.config.get("objects")+ self.encoder = EncoderFactory.create(encoder) if encoder else None+def load(self, path):"""Loads a database path.@@ -68,7 +73,7 @@ class Database:sequentially as deletes could have caused indexid gaps.Args:- columns: optional list of document columns used to rebuild text+ columns: optional list of document columns used to rebuild data"""raise NotImplementedError@@ -93,7 +98,7 @@ class Database:def ids(self, ids):"""Retrieves the internal indexids for a list of ids. Multiple indexids may be present for an id in cases- where text is segmented.+ where data is segmented.Args:ids: list of document idsdiff --git a/src/python/txtai/database/encoder/__init__.py b/src/python/txtai/database/encoder/__init__.pynew file mode 100644index 00000000..90a596f4--- /dev/null+++ b/src/python/txtai/database/encoder/__init__.py@@ -0,0 +1,8 @@+"""+Encoder imports+"""++from .base import Encoder+from .factory import EncoderFactory+from .image import ImageEncoder+from .pickle import PickleEncoderdiff --git a/src/python/txtai/database/encoder/base.py b/src/python/txtai/database/encoder/base.pynew file mode 100644index 00000000..42d5949c--- /dev/null+++ b/src/python/txtai/database/encoder/base.py@@ -0,0 +1,34 @@+"""+Encoder module+"""++from io import BytesIO+++class Encoder:+ """+ Encodes and decodes object content. The base encoder works only with byte arrays. It can be extended to encode different datatypes.+ """++ def encode(self, obj):+ """+ Encodes an object to a byte array using the encoder.++ Returns:+ encoded object as a byte array+ """++ return obj++ def decode(self, data):+ """+ Decodes input byte array into an object using this encoder.++ Args:+ data: encoded data++ Returns:+ decoded object+ """++ return BytesIO(data)diff --git a/src/python/txtai/database/encoder/factory.py b/src/python/txtai/database/encoder/factory.pynew file mode 100644index 00000000..ded7bf65--- /dev/null+++ b/src/python/txtai/database/encoder/factory.py@@ -0,0 +1,55 @@+"""+Encoder factory module+"""++from .base import Encoder+++class EncoderFactory:+ """+ Encoder factory. Creates new Encoder instances.+ """++ @staticmethod+ def get(encoder):+ """+ Gets a new instance of encoder class.++ Args:+ encoder: Encoder instance class++ Returns:+ Encoder class+ """++ # Local task if no package+ if "." not in encoder:+ # Get parent package+ encoder = ".".join(__name__.split(".")[:-1]) + "." + encoder.capitalize() + "Encoder"++ parts = encoder.split(".")+ module = ".".join(parts[:-1])+ m = __import__(module)+ for comp in parts[1:]:+ m = getattr(m, comp)++ return m++ @staticmethod+ def create(encoder):+ """+ Creates a new Encoder instance.++ Args:+ encoder: Encoder instance class++ Returns:+ Encoder+ """++ # Return default encoder+ if encoder is True:+ return Encoder()++ # Get Encoder instance+ return EncoderFactory.get(encoder)()diff --git a/src/python/txtai/database/encoder/image.py b/src/python/txtai/database/encoder/image.pynew file mode 100644index 00000000..1e25a087--- /dev/null+++ b/src/python/txtai/database/encoder/image.py@@ -0,0 +1,43 @@+"""+ImageEncoder module+"""++from io import BytesIO++from .base import Encoder++# Conditional import+try:+ from PIL import Image++ PIL = True+except ImportError:+ PIL = False+++class ImageEncoder(Encoder):+ """+ Encodes and decodes Image objects as compressed binary content, using the original image's algorithm.+ """++ def __init__(self):+ """+ Creates a new ImageEncoder.+ """++ if not PIL:+ raise ImportError('ImageEncoder is not available - install "database" extra to enable')++ def encode(self, obj):+ # Create byte stream+ output = BytesIO()++ # Write image to byte stream+ obj.save(output, format=obj.format, quality="keep")++ # Return byte array+ return output.getvalue()++ def decode(self, data):+ # Return a PIL image+ return Image.open(BytesIO(data))diff --git a/src/python/txtai/database/encoder/pickle.py b/src/python/txtai/database/encoder/pickle.pynew file mode 100644index 00000000..48260d4c--- /dev/null+++ b/src/python/txtai/database/encoder/pickle.py@@ -0,0 +1,21 @@+"""+PickleEncoder module+"""++import pickle++from .base import Encoder+++class PickleEncoder(Encoder):+ """+ Encodes and decodes objects using the Python pickle package.+ """++ def encode(self, obj):+ # Pickle object+ return pickle.dumps(obj, protocol=4)++ def decode(self, data):+ # Unpickle to object+ return pickle.loads(data)diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.pyindex a3b177cd..4625a0a0 100644--- a/src/python/txtai/database/sqlite.py+++ b/src/python/txtai/database/sqlite.py@@ -51,6 +51,19 @@ class SQLite(Database):INSERT_DOCUMENT = "INSERT INTO documents VALUES (?, ?, ?, ?)"DELETE_DOCUMENTS = "DELETE FROM documents WHERE id IN (SELECT id FROM batch)"+ # Objects - stores binary content+ CREATE_OBJECTS = """+ CREATE TABLE IF NOT EXISTS objects (+ id TEXT PRIMARY KEY,+ object BLOB,+ tags TEXT,+ entry DATETIME+ )+ """++ INSERT_OBJECT = "INSERT INTO objects VALUES (?, ?, ?, ?)"+ DELETE_OBJECTS = "DELETE FROM objects WHERE id IN (SELECT id FROM batch)"+# Sections - stores section textCREATE_SECTIONS = """CREATE TABLE IF NOT EXISTS %s (@@ -67,9 +80,9 @@ class SQLite(Database):DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"COPY_SECTIONS = ("INSERT INTO %s SELECT indexid-1, id, text, tags, entry FROM (SELECT row_number() OVER (ORDER BY indexid) AS indexid, "- + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d on s.id = d.id)"+ + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d ON s.id = d.id)")- STREAM_SECTIONS = "SELECT id, text, tags FROM %s ORDER BY indexid"+ STREAM_SECTIONS = "SELECT s.id, s.text, object, s.tags FROM %s s LEFT JOIN objects o ON s.id = o.id ORDER BY indexid"DROP_SECTIONS = "DROP TABLE sections"RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"@@ -77,7 +90,12 @@ class SQLite(Database):SELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"# Partial sql clauses- TABLE_CLAUSE = "SELECT %s FROM sections s LEFT JOIN documents d ON s.id = d.id LEFT JOIN scores sc on s.indexid=sc.indexid"+ TABLE_CLAUSE = (+ "SELECT %s FROM sections s "+ + "LEFT JOIN documents d ON s.id = d.id "+ + "LEFT JOIN objects o ON s.id = o.id "+ + "LEFT JOIN scores sc ON s.indexid = sc.indexid"+ )IDS_CLAUSE = "s.indexid in (SELECT indexid from batch WHERE batch=%s)"def __init__(self, config):@@ -111,19 +129,22 @@ class SQLite(Database):# Insert documentsfor uid, document, tags in documents:if isinstance(document, dict):- # Insert document as JSON- self.cursor.execute(SQLite.INSERT_DOCUMENT, [uid, json.dumps(document), tags, entry])+ # Insert document and use return value for sections table+ document = self.insertdocument(uid, document, tags, entry)- # Extract text, if any- document = document.get("text")+ if document:+ if isinstance(document, list):+ # Join tokens to text+ document = " ".join(document)+ elif not isinstance(document, str):+ # If object support is enabled, save object+ self.insertobject(uid, document, tags, entry)- if isinstance(document, list):- # Join tokens to text- document = " ".join(document)+ # Clear section text for objects, even when objects aren't inserted+ document = None- if document:# Save text section- self.cursor.execute(SQLite.INSERT_SECTION, [index, uid, document, tags, entry])+ self.insertsection(index, uid, document, tags, entry)index += 1def delete(self, ids):@@ -131,8 +152,9 @@ class SQLite(Database):# Batch idsself.batch(ids=ids)- # Delete all documents and sections by id+ # Delete all documents, objects and sections by idself.cursor.execute(SQLite.DELETE_DOCUMENTS)+ self.cursor.execute(SQLite.DELETE_OBJECTS)self.cursor.execute(SQLite.DELETE_SECTIONS)def reindex(self, columns=None):@@ -153,7 +175,11 @@ class SQLite(Database):# Stream new resultsself.cursor.execute(SQLite.STREAM_SECTIONS % name)- yield from self.cursor+ for uid, text, obj, tags in self.cursor:+ if not text and self.encoder and obj:+ yield (uid, self.encoder.decode(obj), tags)+ else:+ yield (uid, text, tags)# Swap as new tableself.cursor.execute(SQLite.DROP_SECTIONS)@@ -201,7 +227,7 @@ class SQLite(Database):def resolve(self, name, alias=False, compound=False):# Standard column namessections = ["indexid", "id", "tags", "entry"]- noprefix = ["data", "score", "text"]+ noprefix = ["data", "object", "score", "text"]# Alias JSON column expressionsif alias:@@ -279,7 +305,11 @@ class SQLite(Database):# Copy columns to result. In cases with duplicate column names, find one with a valuefor x, column in enumerate(columns):if column not in result or result[column] is None:- result[column] = row[x]+ # Decode object+ if self.encoder and column == "object":+ result[column] = self.encoder.decode(row[x])+ else:+ result[column] = row[x]results.append(result)@@ -299,9 +329,73 @@ class SQLite(Database):# Create initial schema and indicesself.cursor.execute(SQLite.CREATE_DOCUMENTS)+ self.cursor.execute(SQLite.CREATE_OBJECTS)self.cursor.execute(SQLite.CREATE_SECTIONS % name)self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)+ def insertdocument(self, uid, document, tags, entry):+ """+ Inserts a document.++ Args:+ uid: unique id+ document: input document+ tags: document tags+ entry: generated entry date++ Returns:+ section value+ """++ # Make a copy of document before changing+ document = document.copy()++ # Get and remove object field from document+ obj = document.pop("object") if "object" in document else None++ # Insert document as JSON+ self.cursor.execute(SQLite.INSERT_DOCUMENT, [uid, json.dumps(document), tags, entry])++ # Get value of text field+ text = document.get("text")++ # If both text and object are set, insert object as it won't otherwise be used+ if text and obj:+ self.insertobject(uid, obj, tags, entry)++ # Return value to use for section - use text if available otherwise use object+ return text if text else obj++ def insertobject(self, uid, obj, tags, entry):+ """+ Inserts an object.++ Args:+ uid: unique id+ obj: input object+ tags: object tags+ entry: generated entry date+ """++ # If object support is enabled, save object+ if self.encoder:+ self.cursor.execute(SQLite.INSERT_OBJECT, [uid, self.encoder.encode(obj), tags, entry])++ def insertsection(self, index, uid, text, tags, entry):+ """+ Inserts a section.++ Args:+ index: index id+ uid: unique id+ text: section text+ tags: section tags+ entry: generated entry date+ """++ # Save text section+ self.cursor.execute(SQLite.INSERT_SECTION, [index, uid, text, tags, entry])+def copy(self, path):"""Copies the current database into path. This method will use the backup API if the current connection has no uncommitted changes.diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 2742b5fe..3f8cc871 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -2,6 +2,7 @@Embeddings module"""+import jsonimport pickleimport osimport shutil@@ -17,10 +18,11 @@ from .archive import Archivefrom .reducer import Reducerfrom .search import Search+# pylint: disable=R0904class Embeddings:"""- Embeddings is the engine that delivers semantic search. Text is transformed into embeddings vectors where similar concepts+ Embeddings is the engine that delivers semantic search. Data is transformed into embeddings vectors where similar conceptswill produce similar vectors. Indexes both large and small are built with these vectors. The indexes are used find resultsthat have the same meaning, not necessarily the same keywords."""@@ -41,7 +43,7 @@ class Embeddings:# Dimensionality reduction and scoring models - word vectors onlyself.reducer, self.scoring = None, None- # Embeddings vector model - transforms text into similarity vectors+ # Embeddings vector model - transforms data into similarity vectorsself.model = None# Approximate nearest neighbor index@@ -61,7 +63,7 @@ class Embeddings:Builds a scoring index. Only used by word vectors models.Args:- documents: list of (id, dict|text|tokens, tags)+ documents: list of (id, data, tags)"""# Build scoring index over documents@@ -73,7 +75,7 @@ class Embeddings:Builds an embeddings index. This method overwrites an existing index.Args:- documents: list of (id, dict|text|tokens, tags)+ documents: list of (id, data, tags)reindex: if this is a reindex operation in which case database creation is skipped, defaults to False"""@@ -118,7 +120,7 @@ class Embeddings:this method runs a standard index operation.Args:- documents: list of (id, dict|text|tokens, tags)+ documents: list of (id, data, tags)"""# Run standard insert if index doesn't exist@@ -202,12 +204,14 @@ class Embeddings:Args:config: new config- columns: optional list of document columns used to rebuild text+ columns: optional list of document columns used to rebuild data"""if self.database:- # Keep content parameter to ensure database is preserved+ # Keep content and objects parameters to ensure database is preservedconfig["content"] = self.config["content"]+ if "objects" in self.config:+ config["objects"] = self.config["objects"]# Reset configurationself.configure(config)@@ -217,10 +221,10 @@ class Embeddings:def transform(self, document):"""- Transforms document into an embeddings vector. Document text will be tokenized if not pre-tokenized.+ Transforms document into an embeddings vector.Args:- document: (id, text|tokens, tags)+ document: (id, data, tags)Returns:embeddings vector@@ -241,10 +245,10 @@ class Embeddings:def batchtransform(self, documents):"""- Transforms documents into embeddings vectors. Document text will be tokenized if not pre-tokenized.+ Transforms documents into embeddings vectors.Args:- documents: list of (id, text|tokens, tags)+ documents: list of (id, data, tags)Returns:embeddings vectors@@ -269,7 +273,7 @@ class Embeddings:on if a database is available.Args:- query: query text|tokens+ query: input querylimit: maximum resultsReturns:@@ -285,7 +289,7 @@ class Embeddings:on if a database is available.Args:- queries: queries text|tokens+ queries: input querieslimit: maximum resultsReturns:@@ -294,29 +298,29 @@ class Embeddings:return Search(self)(queries, limit)- def similarity(self, query, texts):+ def similarity(self, query, data):"""- Computes the similarity between query and list of text. Returns a list of- (id, score) sorted by highest score, where id is the index in texts.+ Computes the similarity between query and list of data. Returns a list of+ (id, score) sorted by highest score, where id is the index in data.Args:- query: query text|tokens- texts: list of text|tokens+ query: input query+ data: list of dataReturns:list of (id, score)"""- return self.batchsimilarity([query], texts)[0]+ return self.batchsimilarity([query], data)[0]- def batchsimilarity(self, queries, texts):+ def batchsimilarity(self, queries, data):"""- Computes the similarity between list of queries and list of text. Returns a list- of (id, score) sorted by highest score per query, where id is the index in texts.+ Computes the similarity between list of queries and list of data. Returns a list+ of (id, score) sorted by highest score per query, where id is the index in data.Args:- queries: queries text|tokens- texts: list of text|tokens+ queries: input queries+ data: list of dataReturns:list of (id, score) per query@@ -324,10 +328,10 @@ class Embeddings:# Convert queries to embedding vectorsqueries = np.array([self.transform((None, query, None)) for query in queries])- texts = np.array([self.transform((None, text, None)) for text in texts])+ data = np.array([self.transform((None, row, None)) for row in data])# Dot product on normalized vectors is equal to cosine similarity- scores = np.dot(queries, texts.T).tolist()+ scores = np.dot(queries, data.T).tolist()# Add index and sort desc based on scorereturn [sorted(enumerate(score), key=lambda x: x[1], reverse=True) for score in scores]@@ -367,7 +371,7 @@ class Embeddings:self.scoring = ScoringFactory.create(self.config["scoring"])self.scoring.load(f"{path}/scoring")- # Sentence vectors model - transforms text to embeddings vectors+ # Sentence vectors model - transforms data to embeddings vectorsself.model = self.loadvectors()# Document database - stores document content@@ -444,6 +448,13 @@ class Embeddings:self.database.close()self.database = None+ def info(self):+ """+ Prints the current embeddings index configuration.+ """++ print(json.dumps(self.config, sort_keys=True, indent=2))+def configure(self, config):"""Sets the configuration for this embeddings index and loads config-driven models.@@ -464,7 +475,7 @@ class Embeddings:else:self.reducer, self.scoring = None, None- # Sentence vectors model - transforms text to embeddings vectors+ # Sentence vectors model - transforms data to embeddings vectorsself.model = self.loadvectors() if self.config else Nonedef loadvectors(self):@@ -482,14 +493,14 @@ class Embeddings:Transforms documents into embeddings vectors.Args:- documents: list of (id, dict|text|tokens, tags)+ documents: list of (id, data, tags)Returns:(document ids, dimensions, embeddings)"""# Transform documents to embeddings vectors- ids, dimensions, stream = self.model.index(self.gettext(documents))+ ids, dimensions, stream = self.model.index(self.getinputs(documents))# Load streamed embeddings back to memoryembeddings = np.empty((len(ids), dimensions), dtype=np.float32)@@ -502,27 +513,31 @@ class Embeddings:return (ids, dimensions, embeddings)- def gettext(self, documents):+ def getinputs(self, documents):"""- Selects documents that have text to vectorize for similarity indexing.+ Prepares documents for vector models.Must be one of the following:- - text- - list of text tokens- - dict with the field "text"+ - dict with the field "text" or "object"+ - not a dict++ Otherwise, the document is not passed to the vector models.Args:- documents: list of (id, dict|text|tokens, tags)+ documents: list of (id, data, tags)Returns:filtered documents"""for document in documents:- if isinstance(document[1], (str, list)):+ if isinstance(document[1], dict):+ if "text" in document[1]:+ yield (document[0], document[1]["text"], document[2])+ elif "object" in document[1]:+ yield (document[0], document[1]["object"], document[2])+ else:yield document- elif isinstance(document[1], dict) and "text" in document[1]:- yield (document[0], document[1]["text"], document[2])def checkarchive(self, path):"""diff --git a/src/python/txtai/embeddings/search.py b/src/python/txtai/embeddings/search.pyindex 93bbf795..a1edcb9d 100644--- a/src/python/txtai/embeddings/search.py+++ b/src/python/txtai/embeddings/search.py@@ -97,7 +97,7 @@ class Search:if "similar" in query:# Get handle to similar queriesfor params in query["similar"]:- # Store the query index and similar text argument (first argument)+ # Store the query index and similar query argument (first argument)equeries.append((x, params[0]))# Second argument is similar candidate limitdiff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.pyindex cbc1eeaa..ce8398bc 100644--- a/src/python/txtai/vectors/base.py+++ b/src/python/txtai/vectors/base.py@@ -42,7 +42,7 @@ class Vectors:number of dimensions and temporary file with embeddings.Args:- documents: list of (id, text|tokens, tags)+ documents: list of (id, data, tags)Returns:(ids, dimensions, stream)@@ -55,7 +55,7 @@ class Vectors:Transforms document into an embeddings vector.Args:- document: (id, text|tokens, tags)+ document: (id, data, tags)Returns:embeddings vectordiff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.pyindex ef649406..4f437101 100644--- a/src/python/txtai/vectors/transformers.py+++ b/src/python/txtai/vectors/transformers.py@@ -70,8 +70,8 @@ class TransformersVectors(Vectors):return (ids, dimensions, stream)def transform(self, document):- # Convert input document to text and build embeddings- return self.model.encode([self.text(document[1])])[0]+ # Prepare input document for transformers model and build embeddings+ return self.model.encode([self.prepare(document[1])])[0]def batch(self, documents, output):"""@@ -85,9 +85,9 @@ class TransformersVectors(Vectors):(ids, dimensions) list of ids and number of dimensions in embeddings"""- # Extract ids and convert input documents to text+ # Extract ids and prepare input documents for transformers modelids = [uid for uid, _, _ in documents]- documents = [self.text(text) for _, text, _ in documents]+ documents = [self.prepare(data) for _, data, _ in documents]dimensions = None# Build embeddings@@ -101,21 +101,24 @@ class TransformersVectors(Vectors):return (ids, dimensions)- def text(self, text):+ def prepare(self, data):"""- Converts input into text that can be processed by transformer models. This method supports- optional string tokenization and joins tokenized input into text.+ Prepares input data for transformers model. This method supports optional string tokenization and+ joins tokenized input into text.Args:- text: text|tokens+ data: input data++ Returns:+ data formatted for transformers model"""# Optional string tokenization- if self.tokenize and isinstance(text, str):- text = Tokenizer.tokenize(text)+ if self.tokenize and isinstance(data, str):+ data = Tokenizer.tokenize(data)- # Transformer models require string input- if isinstance(text, list):- text = " ".join(text)+ # Convert token list to string+ if isinstance(data, list):+ data = " ".join(data)- return text+ return datadiff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.pyindex d0150727..36439d18 100644--- a/src/python/txtai/vectors/words.py+++ b/src/python/txtai/vectors/words.py@@ -48,7 +48,7 @@ def transform(document):Multiprocessing helper method. Transforms document into an embeddings vector.Args:- document: (id, dict|text|tokens, tags)+ document: (id, data, tags)Returns:(id, embedding)