Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Format nested HTML lists without duplicating items (#1286)

Mohammad Hijjawi3f33fb2918bc
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.py
index 9bc6a74c..6b78f35d 100644
--- a/src/python/txtai/pipeline/data/htmltomd.py
+++ b/src/python/txtai/pipeline/data/htmltomd.py
@@ -4,6 +4,8 @@ HTMLToMarkdown module
import re
+from textwrap import indent
+
# Conditional import
try:
from bs4 import BeautifulSoup, NavigableString
@@ -253,10 +255,19 @@ class HTMLToMarkdown(Pipeline):
list as markdown
"""
- elements = []
- for x, element in enumerate(node.find_all("li")):
+ elements, count = [], 0
+ for element in node.find_all(("li", "ul", "ol"), recursive=False):
+ # Nested list that is a direct child of this list
+ if element.name != "li":
+ self.nested(element, article, elements)
+ continue
+
# Unordered lists use dashes. Ordered lists use numbers.
- prefix = "-" if node.name == "ul" else f"{x + 1}."
+ count += 1
+ prefix = "-" if node.name == "ul" else f"{count}."
+
+ # Detach nested lists, these are formatted separately
+ nested = [child.extract() for child in element.find_all(("ul", "ol"), recursive=False)]
# List item text
text = self.process(element, article)
@@ -265,9 +276,28 @@ class HTMLToMarkdown(Pipeline):
if text:
elements.append(f"{prefix} {text}")
+ # Add nested lists
+ for child in nested:
+ self.nested(child, article, elements)
+
# Join elements together as string
return "\n".join(elements)
+ def nested(self, node, article, elements):
+ """
+ Nested list handler. Formats a nested list and adds it to elements indented with a tab,
+ which is preserved by text cleaning.
+
+ Args:
+ node: nested list node
+ article: True if the main section node is an article
+ elements: list of formatted list elements
+ """
+
+ text = self.items(node, article)
+ if text:
+ elements.append(indent(text, "\t"))
+
def code(self, node):
"""
Code block handler. This method transforms a HTML pre or code block into a Markdown formatted
diff --git a/test/python/testapi/testapipipeline.py b/test/python/testapi/testapipipeline.py
index 53c668d8..0405da25 100644
--- a/test/python/testapi/testapipipeline.py
+++ b/test/python/testapi/testapipipeline.py
@@ -316,7 +316,7 @@ class TestPipeline(unittest.TestCase):
text = self.client.get(f"textract?file={Utils.PATH}/article.pdf").json()
# Check length of text is as expected
- self.assertEqual(len(text), 2471)
+ self.assertEqual(len(text), 2472)
# Check invalid URLs
for url in ["http://192.168.1.1/path", "http://127.0.0.1/path", "http://invalid", "/etc/config", "/tmp/txtai-1/test"]:
@@ -331,7 +331,7 @@ class TestPipeline(unittest.TestCase):
path = Utils.PATH + "/article.pdf"
texts = self.client.post("batchtextract", json=[path, path]).json()
- self.assertEqual((len(texts[0]), len(texts[1])), (2471, 2471))
+ self.assertEqual((len(texts[0]), len(texts[1])), (2472, 2472))
def testTextToSpeech(self):
"""
diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.py
index 8c3f990c..80a5f2f1 100644
--- a/test/python/testpipeline/testdata/testtextractor.py
+++ b/test/python/testpipeline/testdata/testtextractor.py
@@ -113,6 +113,19 @@ class TestTextractor(unittest.TestCase):
self.assertMarkdown("<ul><li>Test1</li><li>Test2</li></ul>", "- Test1\n- Test2")
self.assertMarkdown("<ol><li>Test1</li><li>Test2</li></ol>", "1. Test1\n2. Test2")
+ # Nested lists
+ self.assertMarkdown("<ul><li>Test1<ul><li>Test2</li></ul></li><li>Test3</li></ul>", "- Test1\n\t- Test2\n- Test3")
+ self.assertMarkdown(
+ "<ol><li>Test1<ol><li>Test2</li><li>Test3</li></ol></li><li>Test4</li></ol>", "1. Test1\n\t1. Test2\n\t2. Test3\n2. Test4"
+ )
+ self.assertMarkdown(
+ "<ul><li>Test1<ol><li>Test2<ul><li>Test3</li><li>Test4</li></ul></li><li>Test5</li></ol></li><li>Test6</li></ul>",
+ "- Test1\n\t1. Test2\n\t\t- Test3\n\t\t- Test4\n\t2. Test5\n- Test6",
+ )
+
+ # Nested list that is a direct child of the parent list
+ self.assertMarkdown("<ol><li>Test1</li><ol><li>Test2</li></ol><li>Test3</li></ol>", "1. Test1\n\t1. Test2\n2. Test3")
+
# Code
self.assertMarkdown("<code>This is a test</code>", "```\nThis is a test\n```")
self.assertMarkdown("<pre>This is a test</pre>", "```\nThis is a test\n```")
@@ -198,7 +211,7 @@ class TestTextractor(unittest.TestCase):
text = textractor(Utils.PATH + "/article.pdf")
# Check length of text is as expected
- self.assertEqual(len(text), 2471)
+ self.assertEqual(len(text), 2472)
def testTable(self):
"""
@@ -235,7 +248,7 @@ class TestTextractor(unittest.TestCase):
path, text = textractor(Utils.PATH + "/article.pdf")
self.assertEqual(path, Utils.PATH + "/article.pdf")
- self.assertEqual(len(text), 2471)
+ self.assertEqual(len(text), 2472)
def testURL(self):
"""