💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep list items wrapped in inline elements in article mode (#1421)
George Tonmoy Royc604b5ef6747
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.pyindex 72f627cb..f11afa18 100644--- a/src/python/txtai/pipeline/data/htmltomd.py+++ b/src/python/txtai/pipeline/data/htmltomd.py@@ -423,8 +423,8 @@ class HTMLToMarkdown(Pipeline):# List of valid text nodesvalid = ("p", "th", "td", "li", "a", "b", "strong", "i", "em")- # Check if this node is valid or it's part of a paragraph or table cell- valid = node.name in valid or node.find_parent(("p", "th", "td")) is not None+ # Check if this node is valid or it's part of a paragraph, list item or table cell+ valid = node.name in valid or node.find_parent(("p", "li", "th", "td")) is not None# Check if text is valid article texttext = text if (valid or self.isheader(node)) and not self.islink(node) else ""diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.pyindex 3c88a2bb..69336b3d 100644--- a/test/python/testpipeline/testdata/testtextractor.py+++ b/test/python/testpipeline/testdata/testtextractor.py@@ -205,6 +205,29 @@ class TestTextractor(unittest.TestCase):html = "<html><body><article><p><span>Test1</span></p><p><span>Test2</span></p></article></body></html>"self.assertEqual(textractor(html), ["Test1", "Test2"])+ def testHTMLArticleListSpan(self):+ """+ Test article list items with text wrapped in inline elements+ """++ textractor = Textractor()++ for section in ["article", "main"]:+ with self.subTest(section=section):+ html = f"<html><body><{section}><p>Test</p><ul><li><span>Test1</span></li><li><span>Test2</span></li></ul></{section}></body></html>"+ self.assertEqual(textractor(html), "Test\n\n- Test1\n- Test2")++ # Nested lists+ html = "<html><body><article><p>Test</p><ol><li><span>Test1</span><ul><li><span>Test2</span></li></ul></li></ol></article></body></html>"+ self.assertEqual(textractor(html), "Test\n\n1. Test1\n\t- Test2")++ # Link only list items are still skipped+ html = (+ "<html><body><article><p>Test</p><ul><li><span>Test1</span></li>"+ "<li><span><a href='link'>Home</a></span></li></ul></article></body></html>"+ )+ self.assertEqual(textractor(html), "Test\n\n- Test1")+def testHTMLListStart(self):"""Test ordered lists retain their starting number