Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep list items wrapped in inline elements in article mode (#1421)

George Tonmoy Royc604b5ef6747
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.py
index 72f627cb..f11afa18 100644
--- a/src/python/txtai/pipeline/data/htmltomd.py
+++ b/src/python/txtai/pipeline/data/htmltomd.py
@@ -423,8 +423,8 @@ class HTMLToMarkdown(Pipeline):
# List of valid text nodes
valid = ("p", "th", "td", "li", "a", "b", "strong", "i", "em")
- # Check if this node is valid or it's part of a paragraph or table cell
- valid = node.name in valid or node.find_parent(("p", "th", "td")) is not None
+ # Check if this node is valid or it's part of a paragraph, list item or table cell
+ valid = node.name in valid or node.find_parent(("p", "li", "th", "td")) is not None
# Check if text is valid article text
text = text if (valid or self.isheader(node)) and not self.islink(node) else ""
diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.py
index 3c88a2bb..69336b3d 100644
--- a/test/python/testpipeline/testdata/testtextractor.py
+++ b/test/python/testpipeline/testdata/testtextractor.py
@@ -205,6 +205,29 @@ class TestTextractor(unittest.TestCase):
html = "<html><body><article><p><span>Test1</span></p><p><span>Test2</span></p></article></body></html>"
self.assertEqual(textractor(html), ["Test1", "Test2"])
+ def testHTMLArticleListSpan(self):
+ """
+ Test article list items with text wrapped in inline elements
+ """
+
+ textractor = Textractor()
+
+ for section in ["article", "main"]:
+ with self.subTest(section=section):
+ html = f"<html><body><{section}><p>Test</p><ul><li><span>Test1</span></li><li><span>Test2</span></li></ul></{section}></body></html>"
+ self.assertEqual(textractor(html), "Test\n\n- Test1\n- Test2")
+
+ # Nested lists
+ html = "<html><body><article><p>Test</p><ol><li><span>Test1</span><ul><li><span>Test2</span></li></ul></li></ol></article></body></html>"
+ self.assertEqual(textractor(html), "Test\n\n1. Test1\n\t- Test2")
+
+ # Link only list items are still skipped
+ html = (
+ "<html><body><article><p>Test</p><ul><li><span>Test1</span></li>"
+ "<li><span><a href='link'>Home</a></span></li></ul></article></body></html>"
+ )
+ self.assertEqual(textractor(html), "Test\n\n- Test1")
+
def testHTMLListStart(self):
"""
Test ordered lists retain their starting number