💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add safeopen parameter for Textractor, closes #1077
davidmezzetti511f47925147
diff --git a/docs/pipeline/data/textractor.md b/docs/pipeline/data/textractor.mdindex 89a5d45a..53645846 100644--- a/docs/pipeline/data/textractor.md+++ b/docs/pipeline/data/textractor.md@@ -16,6 +16,8 @@ The [backend](../filetohtml/#txtai.pipeline.FileToHTML.__init__) parameter setsSee the [FiletoHTML](../filetohtml) and [HTMLToMarkdown](../htmltomd) pipelines to learn more on the dependencies necessary for each of those pipelines.+Note that the default parameters enable access to all local files and all URLs. The `safeopen` parameter limits this to only files in a specified directory (defaults to temp directory) and public URLs. When running the textractor pipeline through an Application or via the API, `safeopen` defaults to True. It can be disabled if desired via configuration.+## ExampleThe following shows a simple example using this pipeline.diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.pyindex bceb8f87..e8068627 100644--- a/src/python/txtai/app/base.py+++ b/src/python/txtai/app/base.py@@ -139,6 +139,10 @@ class Application:config["embeddings"] = Noneconfig["similarity"] = self.pipelines["similarity"]+ elif pipeline == "textractor":+ # Default to safeopen enabled+ config["safeopen"] = config.get("safeopen", True)+self.pipelines[pipeline] = PipelineFactory.create(config, pipeline)def createworkflows(self):diff --git a/src/python/txtai/pipeline/data/textractor.py b/src/python/txtai/pipeline/data/textractor.pyindex fa7835f6..49e8c2b7 100644--- a/src/python/txtai/pipeline/data/textractor.py+++ b/src/python/txtai/pipeline/data/textractor.py@@ -3,7 +3,9 @@ Textractor module"""import contextlib+import ipaddressimport os+import socketimport tempfilefrom urllib.parse import urlparse@@ -32,7 +34,8 @@ class Textractor(Segmentation):chunker=None,headers=None,backend="available",- **kwargs+ safeopen=False,+ **kwargs,):super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext, chunker, **kwargs)@@ -48,6 +51,9 @@ class Textractor(Segmentation):# HTTP headersself.headers = headers if headers else {}+ # Safe open mode. When set only local temp urls (or a specified directory) and non-private URLs are supported+ self.safeopen = os.path.abspath(tempfile.gettempdir() if isinstance(safeopen, bool) else safeopen) if safeopen else safeopen+def text(self, text):# Check if text is a valid file path or urlpath, exists = self.valid(text)@@ -96,6 +102,10 @@ class Textractor(Segmentation):# Check if this is a local file path or local file urlexists = os.path.exists(path)+ # Safe open validation+ if not self.safecheck(path):+ raise IOError(f"Safeopen URL validation failed: {path}")+# Consider local files and HTTP urls validreturn (path if exists or urlparse(path).scheme in ("http", "https") else None, exists)@@ -137,3 +147,59 @@ class Textractor(Segmentation):# Remote filewith contextlib.closing(urlopen(Request(url, headers=self.headers))) as connection:return connection.read()++ def safecheck(self, url):+ """+ Safe open url validation. Validates a local path is within the safeopen directory and+ that URLs are a public HTTP(s) URLs.++ Args:+ url: input url++ Returns:+ True if url is valid, false otherwise+ """++ # Default to allow all urls when safe open is disabled+ valid = True++ if self.safeopen:+ if os.path.exists(url):+ # Validate local file is in safe path+ path = os.path.realpath(url)+ prefix = os.path.commonprefix([self.safeopen, path])+ valid = prefix == self.safeopen+ else:+ # URL validation+ valid = url.lower().startswith("http") and not self.isprivateurl(url)++ return valid++ def isprivateurl(self, url):+ """+ Checks if URL refers to a private/internal IP address.++ Args:+ url: input url++ Returns:+ True if this is a private url, false otherwise+ """++ # Assume the url is private until proven otherwise+ private = True++ try:+ host = urlparse(url).hostname+ if host:+ # Resolve IP Address+ ip = ipaddress.ip_address(socket.gethostbyname(host))++ # Check if it's private+ private = ip.is_private or ip.is_loopback++ # pylint: disable=W0718+ except Exception:+ pass++ return privatediff --git a/test/python/testapi/testapipipeline.py b/test/python/testapi/testapipipeline.pyindex 3dd3d3fa..618f67b3 100644--- a/test/python/testapi/testapipipeline.py+++ b/test/python/testapi/testapipipeline.py@@ -312,6 +312,11 @@ class TestPipeline(unittest.TestCase):# Check length of text is as expectedself.assertEqual(len(text), 2471)+ # Check invalid URLs+ for url in ["http://192.168.1.1/path", "http://127.0.0.1/path", "http://invalid", "/etc/config"]:+ with self.assertRaises(IOError):+ self.client.get(f"textract?file={url}").json()+def testTextractorBatch(self):"""Test batch textractor via API