Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep the backticks of inline code in HTMLToMarkdown (#1461)

Lukas060be73da62d
diff --git a/src/python/txtai/pipeline/data/htmltomd.py b/src/python/txtai/pipeline/data/htmltomd.py
index f1cad0cf..e4d720f7 100644
--- a/src/python/txtai/pipeline/data/htmltomd.py
+++ b/src/python/txtai/pipeline/data/htmltomd.py
@@ -104,7 +104,8 @@ class HTMLToMarkdown(Pipeline):
return self.items(node, article)
if node.name in ("code", "pre"):
- return self.code(node)
+ # Code inside a paragraph, list item, table cell or heading is inline code, not a code block
+ return f"`{node.text.strip()}`" if self.isinline(node) else self.code(node)
if node.name == "table":
return self.table(node, article)
@@ -204,6 +205,8 @@ class HTMLToMarkdown(Pipeline):
text = f"{space}**{text.strip()}** "
elif target.name in ("i", "em"):
text = f"{space}*{text.strip()}* "
+ elif target.name == "code":
+ text = f"{space}`{text.strip()}` "
elif target.name == "a":
# Links without a target are added as plain text
href = target.get("href")
@@ -440,6 +443,19 @@ class HTMLToMarkdown(Pipeline):
return text
+ def isinline(self, node):
+ """
+ Checks if node is an inline code node, a code element inside a paragraph, list item, table cell or heading.
+
+ Args:
+ node: input node
+
+ Returns:
+ True if node is an inline code node, False otherwise
+ """
+
+ return node.name == "code" and node.parent is not None and (node.parent.name in ("p", "li", "td", "th", "a") or self.isheader(node.parent))
+
def isheader(self, node):
"""
Checks if node is a header node.
diff --git a/test/python/testpipeline/testdata/testtextractor.py b/test/python/testpipeline/testdata/testtextractor.py
index 10c602bb..c095a4c2 100644
--- a/test/python/testpipeline/testdata/testtextractor.py
+++ b/test/python/testpipeline/testdata/testtextractor.py
@@ -134,6 +134,15 @@ class TestTextractor(unittest.TestCase):
self.assertMarkdown("<code>This is a test</code>", "```\nThis is a test\n```")
self.assertMarkdown("<pre>This is a test</pre>", "```\nThis is a test\n```")
+ # Inline code keeps its backticks, also when it is all a list item, table cell or heading holds
+ self.assertMarkdown("<p>Run <code>a*b*c</code> now</p>", "Run `a*b*c` now")
+ self.assertMarkdown("<ul><li><code>pip install txtai</code></li></ul>", "- `pip install txtai`")
+ self.assertMarkdown(
+ "<table><tr><th>Option</th><th>Default</th></tr><tr><td><code>path</code></td><td>None</td></tr></table>",
+ "|Option|Default|\n|---|---|\n|`path`|None|",
+ )
+ self.assertMarkdown("<h2>The <code>config</code> file</h2>", "## The `config` file")
+
# Tables
self.assertMarkdown(
"<table><tr><th>Header1</th><th>Header2</th></tr><tr><td>Test1</td><td>Test2</td></tr></table>",