💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Format nested HTML lists without duplicating items (#1286)
Mohammad Hijjawi3f33fb2918bc
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.pyindex 9bc6a74c..6b78f35d 100644--- a/src/python/txtai/pipeline/data/htmltomd.py+++ b/src/python/txtai/pipeline/data/htmltomd.py@@ -4,6 +4,8 @@ HTMLToMarkdown moduleimport re+from textwrap import indent+# Conditional importtry:from bs4 import BeautifulSoup, NavigableString@@ -253,10 +255,19 @@ class HTMLToMarkdown(Pipeline):list as markdown"""- elements = []- for x, element in enumerate(node.find_all("li")):+ elements, count = [], 0+ for element in node.find_all(("li", "ul", "ol"), recursive=False):+ # Nested list that is a direct child of this list+ if element.name != "li":+ self.nested(element, article, elements)+ continue+# Unordered lists use dashes. Ordered lists use numbers.- prefix = "-" if node.name == "ul" else f"{x + 1}."+ count += 1+ prefix = "-" if node.name == "ul" else f"{count}."++ # Detach nested lists, these are formatted separately+ nested = [child.extract() for child in element.find_all(("ul", "ol"), recursive=False)]# List item texttext = self.process(element, article)@@ -265,9 +276,28 @@ class HTMLToMarkdown(Pipeline):if text:elements.append(f"{prefix} {text}")+ # Add nested lists+ for child in nested:+ self.nested(child, article, elements)+# Join elements together as stringreturn "\n".join(elements)+ def nested(self, node, article, elements):+ """+ Nested list handler. Formats a nested list and adds it to elements indented with a tab,+ which is preserved by text cleaning.++ Args:+ node: nested list node+ article: True if the main section node is an article+ elements: list of formatted list elements+ """++ text = self.items(node, article)+ if text:+ elements.append(indent(text, "\t"))+def code(self, node):"""Code block handler. This method transforms a HTML pre or code block into a Markdown formatteddiff --git a/test/python/testapi/testapipipeline.py b/test/python/testapi/testapipipeline.pyindex 53c668d8..0405da25 100644--- a/test/python/testapi/testapipipeline.py+++ b/test/python/testapi/testapipipeline.py@@ -316,7 +316,7 @@ class TestPipeline(unittest.TestCase):text = self.client.get(f"textract?file={Utils.PATH}/article.pdf").json()# Check length of text is as expected- self.assertEqual(len(text), 2471)+ self.assertEqual(len(text), 2472)# Check invalid URLsfor url in ["http://192.168.1.1/path", "http://127.0.0.1/path", "http://invalid", "/etc/config", "/tmp/txtai-1/test"]:@@ -331,7 +331,7 @@ class TestPipeline(unittest.TestCase):path = Utils.PATH + "/article.pdf"texts = self.client.post("batchtextract", json=[path, path]).json()- self.assertEqual((len(texts[0]), len(texts[1])), (2471, 2471))+ self.assertEqual((len(texts[0]), len(texts[1])), (2472, 2472))def testTextToSpeech(self):"""diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.pyindex 8c3f990c..80a5f2f1 100644--- a/test/python/testpipeline/testdata/testtextractor.py+++ b/test/python/testpipeline/testdata/testtextractor.py@@ -113,6 +113,19 @@ class TestTextractor(unittest.TestCase):self.assertMarkdown("<ul><li>Test1</li><li>Test2</li></ul>", "- Test1\n- Test2")self.assertMarkdown("<ol><li>Test1</li><li>Test2</li></ol>", "1. Test1\n2. Test2")+ # Nested lists+ self.assertMarkdown("<ul><li>Test1<ul><li>Test2</li></ul></li><li>Test3</li></ul>", "- Test1\n\t- Test2\n- Test3")+ self.assertMarkdown(+ "<ol><li>Test1<ol><li>Test2</li><li>Test3</li></ol></li><li>Test4</li></ol>", "1. Test1\n\t1. Test2\n\t2. Test3\n2. Test4"+ )+ self.assertMarkdown(+ "<ul><li>Test1<ol><li>Test2<ul><li>Test3</li><li>Test4</li></ul></li><li>Test5</li></ol></li><li>Test6</li></ul>",+ "- Test1\n\t1. Test2\n\t\t- Test3\n\t\t- Test4\n\t2. Test5\n- Test6",+ )++ # Nested list that is a direct child of the parent list+ self.assertMarkdown("<ol><li>Test1</li><ol><li>Test2</li></ol><li>Test3</li></ol>", "1. Test1\n\t1. Test2\n2. Test3")+# Codeself.assertMarkdown("<code>This is a test</code>", "```\nThis is a test\n```")self.assertMarkdown("<pre>This is a test</pre>", "```\nThis is a test\n```")@@ -198,7 +211,7 @@ class TestTextractor(unittest.TestCase):text = textractor(Utils.PATH + "/article.pdf")# Check length of text is as expected- self.assertEqual(len(text), 2471)+ self.assertEqual(len(text), 2472)def testTable(self):"""@@ -235,7 +248,7 @@ class TestTextractor(unittest.TestCase):path, text = textractor(Utils.PATH + "/article.pdf")self.assertEqual(path, Utils.PATH + "/article.pdf")- self.assertEqual(len(text), 2471)+ self.assertEqual(len(text), 2472)def testURL(self):"""