Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add safeopen parameter for Textractor, closes #1077

davidmezzetti511f47925147
diff --git a/docs/pipeline/data/textractor.md b/docs/pipeline/data/textractor.md
index 89a5d45a..53645846 100644
--- a/docs/pipeline/data/textractor.md
+++ b/docs/pipeline/data/textractor.md
@@ -16,6 +16,8 @@ The [backend](../filetohtml/#txtai.pipeline.FileToHTML.__init__) parameter sets
See the [FiletoHTML](../filetohtml) and [HTMLToMarkdown](../htmltomd) pipelines to learn more on the dependencies necessary for each of those pipelines.
+Note that the default parameters enable access to all local files and all URLs. The `safeopen` parameter limits this to only files in a specified directory (defaults to temp directory) and public URLs. When running the textractor pipeline through an Application or via the API, `safeopen` defaults to True. It can be disabled if desired via configuration.
+
## Example
The following shows a simple example using this pipeline.
diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.py
index bceb8f87..e8068627 100644
--- a/src/python/txtai/app/base.py
+++ b/src/python/txtai/app/base.py
@@ -139,6 +139,10 @@ class Application:
config["embeddings"] = None
config["similarity"] = self.pipelines["similarity"]
+ elif pipeline == "textractor":
+ # Default to safeopen enabled
+ config["safeopen"] = config.get("safeopen", True)
+
self.pipelines[pipeline] = PipelineFactory.create(config, pipeline)
def createworkflows(self):
diff --git a/src/python/txtai/pipeline/data/textractor.py b/src/python/txtai/pipeline/data/textractor.py
index fa7835f6..49e8c2b7 100644
--- a/src/python/txtai/pipeline/data/textractor.py
+++ b/src/python/txtai/pipeline/data/textractor.py
@@ -3,7 +3,9 @@ Textractor module
"""
import contextlib
+import ipaddress
import os
+import socket
import tempfile
from urllib.parse import urlparse
@@ -32,7 +34,8 @@ class Textractor(Segmentation):
chunker=None,
headers=None,
backend="available",
- **kwargs
+ safeopen=False,
+ **kwargs,
):
super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext, chunker, **kwargs)
@@ -48,6 +51,9 @@ class Textractor(Segmentation):
# HTTP headers
self.headers = headers if headers else {}
+ # Safe open mode. When set only local temp urls (or a specified directory) and non-private URLs are supported
+ self.safeopen = os.path.abspath(tempfile.gettempdir() if isinstance(safeopen, bool) else safeopen) if safeopen else safeopen
+
def text(self, text):
# Check if text is a valid file path or url
path, exists = self.valid(text)
@@ -96,6 +102,10 @@ class Textractor(Segmentation):
# Check if this is a local file path or local file url
exists = os.path.exists(path)
+ # Safe open validation
+ if not self.safecheck(path):
+ raise IOError(f"Safeopen URL validation failed: {path}")
+
# Consider local files and HTTP urls valid
return (path if exists or urlparse(path).scheme in ("http", "https") else None, exists)
@@ -137,3 +147,59 @@ class Textractor(Segmentation):
# Remote file
with contextlib.closing(urlopen(Request(url, headers=self.headers))) as connection:
return connection.read()
+
+ def safecheck(self, url):
+ """
+ Safe open url validation. Validates a local path is within the safeopen directory and
+ that URLs are a public HTTP(s) URLs.
+
+ Args:
+ url: input url
+
+ Returns:
+ True if url is valid, false otherwise
+ """
+
+ # Default to allow all urls when safe open is disabled
+ valid = True
+
+ if self.safeopen:
+ if os.path.exists(url):
+ # Validate local file is in safe path
+ path = os.path.realpath(url)
+ prefix = os.path.commonprefix([self.safeopen, path])
+ valid = prefix == self.safeopen
+ else:
+ # URL validation
+ valid = url.lower().startswith("http") and not self.isprivateurl(url)
+
+ return valid
+
+ def isprivateurl(self, url):
+ """
+ Checks if URL refers to a private/internal IP address.
+
+ Args:
+ url: input url
+
+ Returns:
+ True if this is a private url, false otherwise
+ """
+
+ # Assume the url is private until proven otherwise
+ private = True
+
+ try:
+ host = urlparse(url).hostname
+ if host:
+ # Resolve IP Address
+ ip = ipaddress.ip_address(socket.gethostbyname(host))
+
+ # Check if it's private
+ private = ip.is_private or ip.is_loopback
+
+ # pylint: disable=W0718
+ except Exception:
+ pass
+
+ return private
diff --git a/test/python/testapi/testapipipeline.py b/test/python/testapi/testapipipeline.py
index 3dd3d3fa..618f67b3 100644
--- a/test/python/testapi/testapipipeline.py
+++ b/test/python/testapi/testapipipeline.py
@@ -312,6 +312,11 @@ class TestPipeline(unittest.TestCase):
# Check length of text is as expected
self.assertEqual(len(text), 2471)
+ # Check invalid URLs
+ for url in ["http://192.168.1.1/path", "http://127.0.0.1/path", "http://invalid", "/etc/config"]:
+ with self.assertRaises(IOError):
+ self.client.get(f"textract?file={url}").json()
+
def testTextractorBatch(self):
"""
Test batch textractor via API