Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add tabular pipeline to API and API tests, #134

davidmezzetti29533c95e864
diff --git a/src/python/txtai/api/application.py b/src/python/txtai/api/application.py
index c7eab1fb..75ab35e9 100644
--- a/src/python/txtai/api/application.py
+++ b/src/python/txtai/api/application.py
@@ -12,7 +12,7 @@ from .base import API
from .factory import Factory
# pylint: disable=R0401
-from .routers import embeddings, extractor, labels, segmentation, similarity, summary, textractor, transcription, translation, workflow
+from .routers import embeddings, extractor, labels, segmentation, similarity, summary, tabular, textractor, transcription, translation, workflow
# API instance
app = FastAPI()
@@ -60,6 +60,7 @@ def start():
("segmentation", segmentation.router),
("similarity", similarity.router),
("summary", summary.router),
+ ("tabular", tabular.router),
("textractor", textractor.router),
("transcription", transcription.router),
("translation", translation.router),
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index 8a0afefe..60dd0e04 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -53,7 +53,7 @@ class API:
self.pipelines = {}
# Default pipelines
- pipelines = ["extractor", "labels", "segmentation", "similarity", "summary", "textractor", "transcription", "translation"]
+ pipelines = ["extractor", "labels", "segmentation", "similarity", "summary", "tabular", "textractor", "transcription", "translation"]
# Add custom pipelines
for key in self.config:
diff --git a/src/python/txtai/api/routers/__init__.py b/src/python/txtai/api/routers/__init__.py
index 338bae5d..2ab87bfa 100644
--- a/src/python/txtai/api/routers/__init__.py
+++ b/src/python/txtai/api/routers/__init__.py
@@ -7,6 +7,7 @@ from . import extractor
from . import segmentation
from . import similarity
from . import summary
+from . import tabular
from . import textractor
from . import transcription
from . import translation
diff --git a/src/python/txtai/api/routers/tabular.py b/src/python/txtai/api/routers/tabular.py
new file mode 100644
index 00000000..367550c1
--- /dev/null
+++ b/src/python/txtai/api/routers/tabular.py
@@ -0,0 +1,41 @@
+"""
+Defines API paths for tabular endpoints.
+"""
+
+from typing import List
+
+from fastapi import APIRouter, Body
+
+from .. import application
+
+router = APIRouter()
+
+
[email protected]("/tabular")
+def tabular(file: str):
+ """
+ Splits tabular data into rows and columns.
+
+ Args:
+ file: file to process
+
+ Returns:
+ list of (id, text, tag) elements
+ """
+
+ return application.get().pipeline("tabular", (file,))
+
+
[email protected]("/batchtabular")
+def batchtabular(files: List[str] = Body(...)):
+ """
+ Splits tabular data into rows and columns.
+
+ Args:
+ files: list of files to process
+
+ Returns:
+ list of (id, text, tag) elements
+ """
+
+ return application.get().pipeline("tabular", (files,))
diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.py
index 8482378c..6f65ef31 100644
--- a/test/python/testapi/testpipelines.py
+++ b/test/python/testapi/testpipelines.py
@@ -33,6 +33,9 @@ similarity:
summary:
path: t5-small
+# Tabular
+tabular:
+
# Text extraction
textractor:
@@ -180,6 +183,26 @@ class TestPipelines(unittest.TestCase):
summaries = self.client.post("batchsummary", json={"texts": [self.text, self.text], "maxlength": 10}).json()
self.assertEqual(len(summaries), 2)
+ def testTabular(self):
+ """
+ Test tabular via API
+ """
+
+ results = self.client.get("tabular?file=%s" % Utils.PATH + "/tabular.csv").json()
+
+ # Check length of results is as expected
+ self.assertEqual(len(results), 6)
+
+ def testTabularBatch(self):
+ """
+ Test batch tabular via API
+ """
+
+ path = Utils.PATH + "/tabular.csv"
+
+ texts = self.client.post("batchtabular", json=[path, path]).json()
+ self.assertEqual(len(texts), 2)
+
def testTextractor(self):
"""
Test textractor via API