💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Improve textractor pipeline, closes #748
davidmezzettiab8ada9bf6fa
diff --git a/src/python/txtai/pipeline/data/textractor.py b/src/python/txtai/pipeline/data/textractor.pyindex 5a32cd47..767d9a96 100644--- a/src/python/txtai/pipeline/data/textractor.py+++ b/src/python/txtai/pipeline/data/textractor.py@@ -4,14 +4,16 @@ Textractor moduleimport contextlibimport os+import tempfilefrom subprocess import Popen-from urllib.request import urlopen+from urllib.parse import urlparse+from urllib.request import urlopen, Request# Conditional importtry:from bs4 import BeautifulSoup, NavigableString- from tika import parser+ from tika import detector, parserTIKA = Trueexcept ImportError:@@ -25,36 +27,48 @@ class Textractor(Segmentation):Extracts text from files."""- def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False, tika=True, sections=False):+ def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False, tika=True, sections=False, headers=None):if not TIKA:raise ImportError('Textractor pipeline is not available - install "pipeline" extra to enable')super().__init__(sentences, lines, paragraphs, minlength, join, sections)- # Determine if Tika (default if Java is available) or Beautiful Soup should be used- # Beautiful Soup only supports HTML, Tika supports a wide variety of file formats, including HTML.+ # Determine if Apache Tika (default if Java is available) or Beautiful Soup should be used+ # Beautiful Soup only supports HTML, Tika supports a wide variety of file formats.self.tika = self.checkjava() if tika else False# HTML to Text extractor- self.extract = Extract(self.sections)+ self.extract = Extract(self.paragraphs, self.sections)++ # HTTP headers+ self.headers = headers if headers else {}def text(self, text):- # Use Tika if available- if self.tika:- # Format file urls as local file paths- text = text.replace("file://", "")+ # Check if text is a valid file path or url+ path, exists = self.valid(text)++ if not path:+ # Not a valid file path, treat input as data+ html = text++ elif self.tika:+ # Use Tika if available+ # Retrieve remote file, if necessary+ path = path if exists else self.download(path)# Parse content to XHTML- parsed = parser.from_file(text, xmlContent=True)- text = parsed["content"]+ html = self.html(path)++ # Delete temporary file+ if not exists:+ os.remove(path)+else:- # Fallback to XHTML-only support, read data from url/path- text = f"file://{text}" if os.path.exists(text) else text- with contextlib.closing(urlopen(text)) as connection:- text = connection.read()+ # Read data from url/path+ html = self.retrieve(path)# Extract text from HTML- return self.extract(text)+ return self.extract(html)def checkjava(self, path=None):"""@@ -80,6 +94,86 @@ class Textractor(Segmentation):return True+ def valid(self, path):+ """+ Checks if path is a valid local file or web url. Returns path if valid along with a flag+ denoting if the path exists locally.++ Args:+ path: path to check++ Returns:+ (path, exists)+ """++ # Convert file urls to local paths+ path = path.replace("file://", "")++ # Check if this is a local file path or local file url+ exists = os.path.exists(path)++ # Consider local files and HTTP urls valid+ return (path if exists or urlparse(path).scheme in ("http", "https") else None, exists)++ def download(self, url):+ """+ Downloads content of url to a temporary file.++ Args:+ url: input url++ Returns:+ temporary file path+ """++ with tempfile.NamedTemporaryFile(mode="wb", delete=False) as output:+ path = output.name++ # Retrieve and write data to temporary file+ output.write(self.retrieve(url))++ return path++ def retrieve(self, url):+ """+ Retrieves content from url.++ Args:+ url: input url++ Returns:+ data+ """++ # Local file+ if os.path.exists(url):+ with open(url, "rb") as f:+ return f.read()++ # Remote file+ with contextlib.closing(urlopen(Request(url, headers=self.headers))) as connection:+ return connection.read()++ def html(self, path):+ """+ Parses content to HTML using Apache Tika.++ Args:+ path: file path++ Returns:+ html+ """++ # Skip if file is already HTML+ mimetype = detector.from_file(path)+ if mimetype in ("text/html", "text/xhtml"):+ return self.retrieve(path)++ # Parse content to XHTML+ parsed = parser.from_file(path, xmlContent=True)+ return parsed["content"]+class Extract:"""@@ -87,14 +181,16 @@ class Extract:Visual formatting is not included (bold, italic, styling etc)."""- def __init__(self, sections):+ def __init__(self, paragraphs, sections):"""Create a new Extract instance.Args:+ paragraphs: True if paragraph parsing enable, False otherwisesections: True if section parsing enabled, False otherwise"""+ self.paragraphs = paragraphsself.sections = sectionsdef __call__(self, html):@@ -111,30 +207,38 @@ class Extract:# HTML Parsersoup = BeautifulSoup(html, features="html.parser")- # Extract text from each body element+ # Ignore script and style tags+ for script in soup.find_all(["script", "style"]):+ script.decompose()++ # Check if articles are embedded in this html+ article = soup.find("article") is not None++ # Extract text from each section elementnodes = []- for body in soup.find_all("body"):- nodes.append(self.process(body))+ for node in soup.find_all("article" if article else "body"):+ nodes.append(self.process(node, article))# Return extracted text, fallback to default text extraction if no nodes found- return "\n".join(nodes) if nodes else soup.get_text()+ return "\n".join(self.metadata(soup) + nodes) if nodes else soup.get_text()- def process(self, node):+ def process(self, node, article):"""Extracts text from a node. This method applies transforms for containers, tables, lists and text.Page breaks are detected and reflected in the output text as a page break character.Args:node: input node+ article: True if the main section node is an articleReturns:node text"""if node.name == "table":- return self.table(node)+ return self.table(node, article)if node.name in ("ul", "ol"):- return self.items(node)+ return self.items(node, article)# Get page break symbol, if availablepage = node.name and node.get("class") and "page" in node.get("class")@@ -143,17 +247,47 @@ class Extract:children = self.children(node)# Join elements into text- text = "\n".join(self.process(node) for node in children) if self.iscontainer(node, children) else self.text(node)+ if self.iscontainer(node, children):+ texts = [self.process(node, article) for node in children]+ text = "\n".join(text for text in texts if text or not article)+ else:+ text = self.text(node, article)# Add page breaks, if section parsing enabled. Otherwise add node text.return f"{text}\f" if page and self.sections else text- def text(self, node):+ def metadata(self, node):+ """+ Builds a metadata section. The metadata section consists of the title and+ description fields.++ Args:+ node: input document node++ Returns:+ metadata as a list+ """++ title = node.find("title")+ metadata = [title.text] if title else []++ description = node.find("meta", attrs={"name": "description"})+ if description and description["content"]:+ metadata.append(f"\n{description['content']}")++ # Add separator+ if metadata:+ metadata.append("\f" if self.sections else "\n\n")++ return metadata++ def text(self, node, article):"""Text handler. This method flattens a node and it's children to text.Args:node: input node+ article: True if the main section node is an articleReturns:node text@@ -166,15 +300,21 @@ class Extract:# Join text elementstext = "".join(x.text for x in items)+ # Article text processing+ text = self.articletext(node, text) if article else text+# Return text, strip leading/trailing whitespace if this is a string only node- return text if node.name else text.strip()+ text = text if node.name and text else text.strip()- def table(self, node):+ return text++ def table(self, node, article):"""Table handler. This method transforms a HTML table into a Markdown formatted table.Args:node: input node+ article: True if the main section node is an articleReturns:table as markdown@@ -189,7 +329,7 @@ class Extract:columns = row.find_all(lambda tag: tag.name in ("th", "td"))# Add columns with separator- elements.append(f"|{'|'.join(self.process(column) for column in columns)}|")+ elements.append(f"|{'|'.join(self.process(column, article) for column in columns)}|")# If there are multiple rows, add header format rowif not header and len(rows) > 1:@@ -199,12 +339,13 @@ class Extract:# Join elements together as stringreturn "\n".join(elements)- def items(self, node):+ def items(self, node, article):"""List handler. This method transforms a HTML ordered/unordered list into a Markdown formatted list.Args:node: input node+ article: True if the main section node is an articleReturns:list as markdown@@ -215,8 +356,12 @@ class Extract:# Unordered lists use dashes. Ordered lists use numbers.prefix = "-" if node.name == "ul" else f"{x + 1}."+ # List item text+ text = self.process(element, article)+# Add list element- elements.append(f" {prefix} {self.process(element)}")+ if text:+ elements.append(f" {prefix} {text}")# Join elements together as stringreturn "\n".join(elements)@@ -234,7 +379,7 @@ class Extract:True if this is a container element, False otherwise"""- return children and (node.name in ("div", "body") or not any(isinstance(x, NavigableString) for x in children))+ return children and (node.name in ("div", "body", "article") or not any(isinstance(x, NavigableString) for x in children))def children(self, node):"""@@ -252,3 +397,68 @@ class Extract:return [node for node in node.contents if node.name or node.text.strip()]return None++ def articletext(self, node, text):+ """+ Transforms node text using article parsing rules. Article parsing is designed to extract text content from web articles.+ It ignores navigation headers and other superfluous elements.++ Args:+ node: input node+ text: current text++ Returns:+ article text+ """++ # Check if text is valid article text+ text = text if (node.name in ["p", "th", "td", "li", "a"] or self.isheader(node)) and not self.islink(node) else ""+ if text:+ # Replace non-breaking space with newline+ text = text.replace("\xa0", "\n")++ # Format paragraph whitespace+ if node.name == "p":+ text = f"{text.strip()}\n\n" if self.paragraphs else f"{text.strip()}\n"++ # Format header whitespace+ if self.sections and self.isheader(node):+ text = f"\f{text.strip()}\n"++ return text++ def isheader(self, node):+ """+ Checks if node is a header node.++ Args:+ node: input node++ Returns:+ True if node is a header node, False otherwise+ """++ return node.name in ["h1", "h2", "h3", "h4", "h5", "h6"]++ def islink(self, node):+ """+ Checks if node is a link node. This method does not consider links without tables as link nodes.++ Args:+ node: input node++ Returns:+ True if node is a link node, False otherwise+ """++ # Check if this is a link node or link container+ link, parent = False, node+ while parent:+ if parent.name == "a":+ link = True+ break++ parent = parent.parent++ # Return if this node or any parents are a link. Ignore links in table cells.+ return link and node.parent.name not in ("th", "td")diff --git a/test/python/testpipeline/testtextractor.py b/test/python/testpipeline/testtextractor.pyindex f5d0707d..5a0336ad 100644--- a/test/python/testpipeline/testtextractor.py+++ b/test/python/testpipeline/testtextractor.py@@ -32,6 +32,17 @@ class TestTextractor(unittest.TestCase):textractor = Textractor()self.assertFalse(textractor.checkjava("1112444abc"))+ def testHTML(self):+ """+ Tests input as HTML strings+ """++ textractor = Textractor()++ # Extract from HTML string+ text = textractor("<html><body><p>This is a test</p></body></html>")+ self.assertEqual(text, "This is a test")+def testLines(self):"""Test extraction to lines@@ -110,3 +121,12 @@ class TestTextractor(unittest.TestCase):# Check for table headerself.assertTrue("|---|" in text)++ def testURL(self):+ """+ Test parsing a remote URL+ """++ textractor = Textractor(sections=True, join=True)+ text = textractor("https://github.com/neuml/txtai")+ self.assertTrue("txtai is an all-in-one embeddings database" in text)