💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep the backticks of inline code in HTMLToMarkdown (#1461)
Lukas060be73da62d
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.pyindex f1cad0cf..e4d720f7 100644--- a/src/python/txtai/pipeline/data/htmltomd.py+++ b/src/python/txtai/pipeline/data/htmltomd.py@@ -104,7 +104,8 @@ class HTMLToMarkdown(Pipeline):return self.items(node, article)if node.name in ("code", "pre"):- return self.code(node)+ # Code inside a paragraph, list item, table cell or heading is inline code, not a code block+ return f"`{node.text.strip()}`" if self.isinline(node) else self.code(node)if node.name == "table":return self.table(node, article)@@ -204,6 +205,8 @@ class HTMLToMarkdown(Pipeline):text = f"{space}**{text.strip()}** "elif target.name in ("i", "em"):text = f"{space}*{text.strip()}* "+ elif target.name == "code":+ text = f"{space}`{text.strip()}` "elif target.name == "a":# Links without a target are added as plain texthref = target.get("href")@@ -440,6 +443,19 @@ class HTMLToMarkdown(Pipeline):return text+ def isinline(self, node):+ """+ Checks if node is an inline code node, a code element inside a paragraph, list item, table cell or heading.++ Args:+ node: input node++ Returns:+ True if node is an inline code node, False otherwise+ """++ return node.name == "code" and node.parent is not None and (node.parent.name in ("p", "li", "td", "th", "a") or self.isheader(node.parent))+def isheader(self, node):"""Checks if node is a header node.diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.pyindex 10c602bb..c095a4c2 100644--- a/test/python/testpipeline/testdata/testtextractor.py+++ b/test/python/testpipeline/testdata/testtextractor.py@@ -134,6 +134,15 @@ class TestTextractor(unittest.TestCase):self.assertMarkdown("<code>This is a test</code>", "```\nThis is a test\n```")self.assertMarkdown("<pre>This is a test</pre>", "```\nThis is a test\n```")+ # Inline code keeps its backticks, also when it is all a list item, table cell or heading holds+ self.assertMarkdown("<p>Run <code>a*b*c</code> now</p>", "Run `a*b*c` now")+ self.assertMarkdown("<ul><li><code>pip install txtai</code></li></ul>", "- `pip install txtai`")+ self.assertMarkdown(+ "<table><tr><th>Option</th><th>Default</th></tr><tr><td><code>path</code></td><td>None</td></tr></table>",+ "|Option|Default|\n|---|---|\n|`path`|None|",+ )+ self.assertMarkdown("<h2>The <code>config</code> file</h2>", "## The `config` file")+# Tablesself.assertMarkdown("<table><tr><th>Header1</th><th>Header2</th></tr><tr><td>Test1</td><td>Test2</td></tr></table>",