💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix HTMLToMarkdown crash on a meta description tag with no content attribute (#1140)
Andrew Chen7c77c134e28c
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.pyindex 721610d3..9bc6a74c 100644--- a/src/python/txtai/pipeline/data/htmltomd.py+++ b/src/python/txtai/pipeline/data/htmltomd.py@@ -135,7 +135,7 @@ class HTMLToMarkdown(Pipeline):metadata = [f"**{title.text.strip()}**"] if title and title.text else []description = node.find("meta", attrs={"name": "description"})- if description and description["content"]:+ if description and description.get("content"):metadata.append(f"\n*{description['content'].strip()}*")# Add separatordiff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.pyindex 1c2acf25..b0ad064c 100644--- a/test/python/testpipeline/testdata/testtextractor.py+++ b/test/python/testpipeline/testdata/testtextractor.py@@ -140,6 +140,16 @@ class TestTextractor(unittest.TestCase):self.assertMarkdown("<p>This is a <strong><em>test</em></strong></p>", "This is a **test**")self.assertMarkdown("<p>This is a <em><strong>test</strong></em></p>", "This is a *test*")+ def testHTMLMetadataNoContent(self):+ """+ Test HTML with a meta description tag that has no content attribute+ """++ # A <meta name="description"> without a content attribute must not crash+ textractor = Textractor()+ html = "<html><head><meta name='description'></head><body><p>This is a test</p></body></html>"+ self.assertEqual(textractor(html), "This is a test")+def testParagraphs(self):"""Test extraction to paragraphs