Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix HTMLToMarkdown crash on a meta description tag with no content attribute (#1140)

Andrew Chen7c77c134e28c
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.py
index 721610d3..9bc6a74c 100644
--- a/src/python/txtai/pipeline/data/htmltomd.py
+++ b/src/python/txtai/pipeline/data/htmltomd.py
@@ -135,7 +135,7 @@ class HTMLToMarkdown(Pipeline):
metadata = [f"**{title.text.strip()}**"] if title and title.text else []
description = node.find("meta", attrs={"name": "description"})
- if description and description["content"]:
+ if description and description.get("content"):
metadata.append(f"\n*{description['content'].strip()}*")
# Add separator
diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.py
index 1c2acf25..b0ad064c 100644
--- a/test/python/testpipeline/testdata/testtextractor.py
+++ b/test/python/testpipeline/testdata/testtextractor.py
@@ -140,6 +140,16 @@ class TestTextractor(unittest.TestCase):
self.assertMarkdown("<p>This is a <strong><em>test</em></strong></p>", "This is a **test**")
self.assertMarkdown("<p>This is a <em><strong>test</strong></em></p>", "This is a *test*")
+ def testHTMLMetadataNoContent(self):
+ """
+ Test HTML with a meta description tag that has no content attribute
+ """
+
+ # A <meta name="description"> without a content attribute must not crash
+ textractor = Textractor()
+ html = "<html><head><meta name='description'></head><body><p>This is a test</p></body></html>"
+ self.assertEqual(textractor(html), "This is a test")
+
def testParagraphs(self):
"""
Test extraction to paragraphs