💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add reindex parameter for API, closes #1111
davidmezzetti11b32da720f0
diff --git a/docs/embeddings/configuration/general.md b/docs/embeddings/configuration/general.mdindex 640d60d0..7fe0e3a9 100644--- a/docs/embeddings/configuration/general.md+++ b/docs/embeddings/configuration/general.md@@ -79,3 +79,5 @@ format: json|pickle```Sets the configuration storage format. Defaults to `json`.++Note that `pickle` configuration is deprecated and will raise an error with the default settings. Its only kept for reading legacy indexes and requires setting the `ALLOW_PICKLE` environment variable. Only enable for local and/or trusted sources.diff --git a/src/python/txtai/api/routers/embeddings.py b/src/python/txtai/api/routers/embeddings.pyindex 32dcff32..df139cdc 100644--- a/src/python/txtai/api/routers/embeddings.py+++ b/src/python/txtai/api/routers/embeddings.py@@ -14,7 +14,7 @@ from .. import applicationfrom ..responses import ResponseFactoryfrom ..route import EncodingAPIRoute-from ...app import ReadOnlyError+from ...app import ReadOnlyError, ReindexDisabledErrorfrom ...graph import Graphrouter = APIRouter(route_class=EncodingAPIRoute)@@ -198,7 +198,7 @@ def reindex(config: dict = Body(...), function: str = Body(default=None)):try:application.get().reindex(config, function)- except ReadOnlyError as e:+ except (ReadOnlyError, ReindexDisabledError) as e:raise HTTPException(status_code=403, detail=e.args[0]) from ediff --git a/src/python/txtai/app/__init__.py b/src/python/txtai/app/__init__.pyindex 95768fb6..b7c0ff42 100644--- a/src/python/txtai/app/__init__.py+++ b/src/python/txtai/app/__init__.py@@ -2,4 +2,4 @@App imports"""-from .base import Application, ReadOnlyError+from .base import Application, ReadOnlyError, ReindexDisabledErrordiff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.pyindex 8254bd59..6e868978 100644--- a/src/python/txtai/app/base.py+++ b/src/python/txtai/app/base.py@@ -566,6 +566,10 @@ class Application:if not self.config.get("writable"):raise ReadOnlyError("Attempting to reindex a read-only index (writable != True)")+ # Raise error if not re-indexable+ if not self.config.get("reindex"):+ raise ReindexDisabledError("Attempting to reindex with reindexing disabled (reindex != True)")+if self.embeddings:with self.lock:# Resolve function, if necessary@@ -826,3 +830,9 @@ class ReadOnlyError(Exception):"""Error raised when trying to modify a read-only index"""+++class ReindexDisabledError(Exception):+ """+ Error raised when trying to reindex with reindexing disabled+ """diff --git a/test/python/testapi/testapiembeddings.py b/test/python/testapi/testapiembeddings.pyindex 4ad99df1..fa9262fb 100644--- a/test/python/testapi/testapiembeddings.py+++ b/test/python/testapi/testapiembeddings.py@@ -21,6 +21,9 @@ path: %s# Allow indexing of documentswritable: True+# Allow reindexing+reindex: True+# Questions settingsquestions:path: distilbert-base-cased-distilled-squad@@ -92,6 +95,20 @@ rag:output: flatten"""+# Configuration for reindexing disabled+REINDEXDISABLED = """+# Index file path+path: %s++# Allow indexing of documents+writable: True++# Embeddings settings+embeddings:+ path: sentence-transformers/nli-mpnet-base-v2+ content: True+"""+# Configuration for rerankerRERANK = """# Index file path@@ -114,6 +131,7 @@ reranker:"""+# pylint: disable=R0904class TestEmbeddings(unittest.TestCase):"""API tests for embeddings indices.@@ -346,7 +364,7 @@ class TestEmbeddings(unittest.TestCase):Test read-only API instance"""- # Re-create read-only model+ # Re-create application with a read-only indexself.client = TestEmbeddings.start(READONLY)# Test search@@ -370,7 +388,7 @@ class TestEmbeddings(unittest.TestCase):Test API instance with custom functions"""- # Re-create model with custom functions+ # Re-create application with custom functionsself.client = TestEmbeddings.start(FUNCTIONS)# Index data@@ -409,7 +427,7 @@ class TestEmbeddings(unittest.TestCase):Test RAG via API"""- # Re-create model with custom functions+ # Re-create application with a RAG pipelineself.client = TestEmbeddings.start(RAG)# Index data@@ -422,12 +440,27 @@ class TestEmbeddings(unittest.TestCase):response = self.client.post("batchrag", json={"queries": ["bear", "bear"]}).json()self.assertEqual(len(response), 2)+ def testXReindexDisabled(self):+ """+ Test reindexing is disabled+ """++ # Re-create application with reindexing disabled+ self.client = TestEmbeddings.start(REINDEXDISABLED)++ # Index data+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(self.data)])+ self.client.get("index")++ # Assert error raised+ self.assertEqual(self.client.post("reindex", json={"config": {"path": "sentence-transformers/nli-mpnet-base-v2"}}).status_code, 403)+def testXRerank(self):"""Test rerank via API"""- # Re-create model with custom functions+ # Re-create application with a reranker pipelineself.client = TestEmbeddings.start(RERANK)# Index data