Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add reindex parameter for API, closes #1111

davidmezzetti11b32da720f0
diff --git a/docs/embeddings/configuration/general.md b/docs/embeddings/configuration/general.md
index 640d60d0..7fe0e3a9 100644
--- a/docs/embeddings/configuration/general.md
+++ b/docs/embeddings/configuration/general.md
@@ -79,3 +79,5 @@ format: json|pickle
```
Sets the configuration storage format. Defaults to `json`.
+
+Note that `pickle` configuration is deprecated and will raise an error with the default settings. Its only kept for reading legacy indexes and requires setting the `ALLOW_PICKLE` environment variable. Only enable for local and/or trusted sources.
diff --git a/src/python/txtai/api/routers/embeddings.py b/src/python/txtai/api/routers/embeddings.py
index 32dcff32..df139cdc 100644
--- a/src/python/txtai/api/routers/embeddings.py
+++ b/src/python/txtai/api/routers/embeddings.py
@@ -14,7 +14,7 @@ from .. import application
from ..responses import ResponseFactory
from ..route import EncodingAPIRoute
-from ...app import ReadOnlyError
+from ...app import ReadOnlyError, ReindexDisabledError
from ...graph import Graph
router = APIRouter(route_class=EncodingAPIRoute)
@@ -198,7 +198,7 @@ def reindex(config: dict = Body(...), function: str = Body(default=None)):
try:
application.get().reindex(config, function)
- except ReadOnlyError as e:
+ except (ReadOnlyError, ReindexDisabledError) as e:
raise HTTPException(status_code=403, detail=e.args[0]) from e
diff --git a/src/python/txtai/app/__init__.py b/src/python/txtai/app/__init__.py
index 95768fb6..b7c0ff42 100644
--- a/src/python/txtai/app/__init__.py
+++ b/src/python/txtai/app/__init__.py
@@ -2,4 +2,4 @@
App imports
"""
-from .base import Application, ReadOnlyError
+from .base import Application, ReadOnlyError, ReindexDisabledError
diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.py
index 8254bd59..6e868978 100644
--- a/src/python/txtai/app/base.py
+++ b/src/python/txtai/app/base.py
@@ -566,6 +566,10 @@ class Application:
if not self.config.get("writable"):
raise ReadOnlyError("Attempting to reindex a read-only index (writable != True)")
+ # Raise error if not re-indexable
+ if not self.config.get("reindex"):
+ raise ReindexDisabledError("Attempting to reindex with reindexing disabled (reindex != True)")
+
if self.embeddings:
with self.lock:
# Resolve function, if necessary
@@ -826,3 +830,9 @@ class ReadOnlyError(Exception):
"""
Error raised when trying to modify a read-only index
"""
+
+
+class ReindexDisabledError(Exception):
+ """
+ Error raised when trying to reindex with reindexing disabled
+ """
diff --git a/test/python/testapi/testapiembeddings.py b/test/python/testapi/testapiembeddings.py
index 4ad99df1..fa9262fb 100644
--- a/test/python/testapi/testapiembeddings.py
+++ b/test/python/testapi/testapiembeddings.py
@@ -21,6 +21,9 @@ path: %s
# Allow indexing of documents
writable: True
+# Allow reindexing
+reindex: True
+
# Questions settings
questions:
path: distilbert-base-cased-distilled-squad
@@ -92,6 +95,20 @@ rag:
output: flatten
"""
+# Configuration for reindexing disabled
+REINDEXDISABLED = """
+# Index file path
+path: %s
+
+# Allow indexing of documents
+writable: True
+
+# Embeddings settings
+embeddings:
+ path: sentence-transformers/nli-mpnet-base-v2
+ content: True
+"""
+
# Configuration for reranker
RERANK = """
# Index file path
@@ -114,6 +131,7 @@ reranker:
"""
+# pylint: disable=R0904
class TestEmbeddings(unittest.TestCase):
"""
API tests for embeddings indices.
@@ -346,7 +364,7 @@ class TestEmbeddings(unittest.TestCase):
Test read-only API instance
"""
- # Re-create read-only model
+ # Re-create application with a read-only index
self.client = TestEmbeddings.start(READONLY)
# Test search
@@ -370,7 +388,7 @@ class TestEmbeddings(unittest.TestCase):
Test API instance with custom functions
"""
- # Re-create model with custom functions
+ # Re-create application with custom functions
self.client = TestEmbeddings.start(FUNCTIONS)
# Index data
@@ -409,7 +427,7 @@ class TestEmbeddings(unittest.TestCase):
Test RAG via API
"""
- # Re-create model with custom functions
+ # Re-create application with a RAG pipeline
self.client = TestEmbeddings.start(RAG)
# Index data
@@ -422,12 +440,27 @@ class TestEmbeddings(unittest.TestCase):
response = self.client.post("batchrag", json={"queries": ["bear", "bear"]}).json()
self.assertEqual(len(response), 2)
+ def testXReindexDisabled(self):
+ """
+ Test reindexing is disabled
+ """
+
+ # Re-create application with reindexing disabled
+ self.client = TestEmbeddings.start(REINDEXDISABLED)
+
+ # Index data
+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(self.data)])
+ self.client.get("index")
+
+ # Assert error raised
+ self.assertEqual(self.client.post("reindex", json={"config": {"path": "sentence-transformers/nli-mpnet-base-v2"}}).status_code, 403)
+
def testXRerank(self):
"""
Test rerank via API
"""
- # Re-create model with custom functions
+ # Re-create application with a reranker pipeline
self.client = TestEmbeddings.start(RERANK)
# Index data