💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add retrieval task, closes #150
davidmezzettida67b22c04c7
diff --git a/src/python/txtai/workflow/task/__init__.py b/src/python/txtai/workflow/task/__init__.pyindex 6e2be8ab..473e03d9 100644--- a/src/python/txtai/workflow/task/__init__.py+++ b/src/python/txtai/workflow/task/__init__.py@@ -6,6 +6,7 @@ from .base import Taskfrom .factory import TaskFactoryfrom .file import FileTaskfrom .image import ImageTask+from .retrieve import RetrieveTaskfrom .service import ServiceTaskfrom .storage import StorageTaskfrom .url import UrlTaskdiff --git a/src/python/txtai/workflow/task/retrieve.py b/src/python/txtai/workflow/task/retrieve.pynew file mode 100644index 00000000..dea6b165--- /dev/null+++ b/src/python/txtai/workflow/task/retrieve.py@@ -0,0 +1,47 @@+"""+RetrieveTask module+"""++import os+import tempfile++from urllib.request import urlretrieve++from .url import UrlTask+++class RetrieveTask(UrlTask):+ """+ Task that retrieves urls (local or remote) to a local directory.+ """++ def __init__(self, action=None, select=None, unpack=True, directory=None):+ super().__init__(action, select, unpack)++ # Create default temporary directory if not specified+ if not directory:+ # Save tempdir to prevent content from being deleted until this task is out of scope+ self.tempdir = tempfile.TemporaryDirectory()+ directory = self.tempdir.name++ # Create output directory if necessary+ if not os.path.exists(directory):+ os.makedirs(directory)++ self.directory = directory++ def prepare(self, element):+ # Run super() method to format element+ element = super().prepare(element)++ # Extract file name+ _, name = os.path.split(element)++ # Derive output path+ path = os.path.join(self.directory, name)++ # Retrieve URL+ urlretrieve(element, os.path.join(self.directory, name))++ # Return new file path+ return pathdiff --git a/test/python/testapi/testworkflow.py b/test/python/testapi/testworkflow.pyindex 23bf066c..107e509e 100644--- a/test/python/testapi/testworkflow.py+++ b/test/python/testapi/testworkflow.py@@ -22,12 +22,20 @@ writable: trueembeddings:path: sentence-transformers/nli-mpnet-base-v2+# Labels+labels:+ path: prajjwal1/bert-medium-mnli+# Text segmentationsegmentation:sentences: true# Workflow definitionsworkflow:+ labels:+ tasks:+ - action: labels+ args: [[positive, negative]]segment:tasks:- action: segmentation@@ -182,6 +190,9 @@ class TestWorkflow(unittest.TestCase):"""text = "This is a test sentence. And another sentence to split."+ results = self.client.post("workflow", json={"name": "labels", "elements": [text]}).json()+ self.assertEqual(len(results), 2)+results = self.client.post("workflow", json={"name": "segment", "elements": [text]}).json()self.assertEqual(len(results), 2)diff --git a/test/python/testworkflow.py b/test/python/testworkflow.pyindex bf638f46..5e1a581d 100644--- a/test/python/testworkflow.py+++ b/test/python/testworkflow.py@@ -6,7 +6,7 @@ import unittestfrom txtai.embeddings import Documents, Embeddingsfrom txtai.pipeline import Summary, Translation, Textractor-from txtai.workflow import Workflow, Task, FileTask, ImageTask, StorageTask, WorkflowTask+from txtai.workflow import Workflow, Task, FileTask, ImageTask, RetrieveTask, StorageTask, WorkflowTask# pylint: disable = C0411from utils import Utils@@ -77,12 +77,22 @@ class TestWorkflow(unittest.TestCase):Tests an image task"""- workflow = Workflow([ImageTask()], 1)+ workflow = Workflow([ImageTask()])- results = list(workflow(["file://" + Utils.PATH + "/books.jpg"]))+ results = list(workflow([Utils.PATH + "/books.jpg"]))self.assertEqual(results[0].size, (1024, 682))+ def testRetrieveWorkflow(self):+ """+ Tests a retrieve task+ """++ workflow = Workflow([RetrieveTask()])++ results = list(workflow(["file://" + Utils.PATH + "/books.jpg"]))+ self.assertTrue(results[0].endswith("books.jpg"))+def testStorageWorkflow(self):"""Tests a storage task