Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add retrieval task, closes #150

davidmezzettida67b22c04c7
diff --git a/src/python/txtai/workflow/task/__init__.py b/src/python/txtai/workflow/task/__init__.py
index 6e2be8ab..473e03d9 100644
--- a/src/python/txtai/workflow/task/__init__.py
+++ b/src/python/txtai/workflow/task/__init__.py
@@ -6,6 +6,7 @@ from .base import Task
from .factory import TaskFactory
from .file import FileTask
from .image import ImageTask
+from .retrieve import RetrieveTask
from .service import ServiceTask
from .storage import StorageTask
from .url import UrlTask
diff --git a/src/python/txtai/workflow/task/retrieve.py b/src/python/txtai/workflow/task/retrieve.py
new file mode 100644
index 00000000..dea6b165
--- /dev/null
+++ b/src/python/txtai/workflow/task/retrieve.py
@@ -0,0 +1,47 @@
+"""
+RetrieveTask module
+"""
+
+import os
+import tempfile
+
+from urllib.request import urlretrieve
+
+from .url import UrlTask
+
+
+class RetrieveTask(UrlTask):
+ """
+ Task that retrieves urls (local or remote) to a local directory.
+ """
+
+ def __init__(self, action=None, select=None, unpack=True, directory=None):
+ super().__init__(action, select, unpack)
+
+ # Create default temporary directory if not specified
+ if not directory:
+ # Save tempdir to prevent content from being deleted until this task is out of scope
+ self.tempdir = tempfile.TemporaryDirectory()
+ directory = self.tempdir.name
+
+ # Create output directory if necessary
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+
+ self.directory = directory
+
+ def prepare(self, element):
+ # Run super() method to format element
+ element = super().prepare(element)
+
+ # Extract file name
+ _, name = os.path.split(element)
+
+ # Derive output path
+ path = os.path.join(self.directory, name)
+
+ # Retrieve URL
+ urlretrieve(element, os.path.join(self.directory, name))
+
+ # Return new file path
+ return path
diff --git a/test/python/testapi/testworkflow.py b/test/python/testapi/testworkflow.py
index 23bf066c..107e509e 100644
--- a/test/python/testapi/testworkflow.py
+++ b/test/python/testapi/testworkflow.py
@@ -22,12 +22,20 @@ writable: true
embeddings:
path: sentence-transformers/nli-mpnet-base-v2
+# Labels
+labels:
+ path: prajjwal1/bert-medium-mnli
+
# Text segmentation
segmentation:
sentences: true
# Workflow definitions
workflow:
+ labels:
+ tasks:
+ - action: labels
+ args: [[positive, negative]]
segment:
tasks:
- action: segmentation
@@ -182,6 +190,9 @@ class TestWorkflow(unittest.TestCase):
"""
text = "This is a test sentence. And another sentence to split."
+ results = self.client.post("workflow", json={"name": "labels", "elements": [text]}).json()
+ self.assertEqual(len(results), 2)
+
results = self.client.post("workflow", json={"name": "segment", "elements": [text]}).json()
self.assertEqual(len(results), 2)
diff --git a/test/python/testworkflow.py b/test/python/testworkflow.py
index bf638f46..5e1a581d 100644
--- a/test/python/testworkflow.py
+++ b/test/python/testworkflow.py
@@ -6,7 +6,7 @@ import unittest
from txtai.embeddings import Documents, Embeddings
from txtai.pipeline import Summary, Translation, Textractor
-from txtai.workflow import Workflow, Task, FileTask, ImageTask, StorageTask, WorkflowTask
+from txtai.workflow import Workflow, Task, FileTask, ImageTask, RetrieveTask, StorageTask, WorkflowTask
# pylint: disable = C0411
from utils import Utils
@@ -77,12 +77,22 @@ class TestWorkflow(unittest.TestCase):
Tests an image task
"""
- workflow = Workflow([ImageTask()], 1)
+ workflow = Workflow([ImageTask()])
- results = list(workflow(["file://" + Utils.PATH + "/books.jpg"]))
+ results = list(workflow([Utils.PATH + "/books.jpg"]))
self.assertEqual(results[0].size, (1024, 682))
+ def testRetrieveWorkflow(self):
+ """
+ Tests a retrieve task
+ """
+
+ workflow = Workflow([RetrieveTask()])
+
+ results = list(workflow(["file://" + Utils.PATH + "/books.jpg"]))
+ self.assertTrue(results[0].endswith("books.jpg"))
+
def testStorageWorkflow(self):
"""
Tests a storage task