Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep inline HTML text in its line when parsing an upload (#11818)

Lukasdcd0e1ca670a
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.py
index d36ff9121b7..7114aebbcba 100644
--- a/studio/backend/core/rag/parsers.py
+++ b/studio/backend/core/rag/parsers.py
@@ -48,30 +48,82 @@ def _page(text: str, page_number: int | None) -> Page:
return Page(text = text, page_number = page_number, char_count = len(text))
+_HTML_SKIP_TAGS = frozenset(("script", "style", "template"))
+_HTML_BLOCK_TAGS = frozenset(
+ "address article aside blockquote br caption center dd details dialog dir div dl dt"
+ " fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr legend li"
+ " listing main menu nav ol optgroup option p plaintext pre search section summary table"
+ " td text textarea th title tr ul xmp".split()
+)
+_HTML_PRE_TAGS = frozenset(("listing", "plaintext", "pre", "textarea", "xmp"))
+# Atomic inline boxes: their text never runs into a neighbour's, but they do not break the line.
+_HTML_BOX_TAGS = frozenset(("button", "img", "input", "select"))
+
+
class _Stripper(HTMLParser):
- """Collect visible text, skipping <script>/<style>."""
+ """Collect visible text, one line per block element."""
def __init__(self) -> None:
super().__init__()
self._skip = 0
+ self._pre = 0
+ self._templates: list[bool] = []
+ self._line: list[str] = []
self.out: list[str] = []
+ def _flush(self) -> None:
+ text = "".join(self._line)
+ self._line = []
+ # Whitespace inside <pre>/<textarea> is content; elsewhere it is layout.
+ text = text.strip("\n") if self._pre else " ".join(text.split())
+ if text.strip():
+ self.out.append(text)
+
def handle_starttag(self, tag, attrs):
- if tag in ("script", "style"):
+ if tag == "template":
+ # A declarative shadow root (shadowrootmode=open|closed) is rendered; other templates are inert.
+ inert = (dict(attrs).get("shadowrootmode") or "").lower() not in ("open", "closed")
+ self._templates.append(inert)
+ self._skip += inert
+ elif tag in _HTML_SKIP_TAGS:
self._skip += 1
+ elif tag in _HTML_BLOCK_TAGS and not self._skip:
+ self._flush()
+ if tag in _HTML_PRE_TAGS:
+ self._pre += 1
+ elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:
+ self._line.append(" ")
+ elif tag == "tspan" and not self._skip and any(k in ("x", "y") for k, _ in attrs):
+ # An absolute x/y starts a new SVG text chunk (a separate label or line).
+ self._flush()
def handle_endtag(self, tag):
- if tag in ("script", "style") and self._skip:
- self._skip -= 1
+ if tag == "template":
+ if self._templates and self._templates.pop() and self._skip:
+ self._skip -= 1
+ elif tag in _HTML_SKIP_TAGS:
+ if self._skip:
+ self._skip -= 1
+ elif tag in _HTML_BLOCK_TAGS and not self._skip:
+ self._flush()
+ if tag in _HTML_PRE_TAGS and self._pre:
+ self._pre -= 1
+ elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:
+ self._line.append(" ")
def handle_data(self, data):
- if not self._skip and data.strip():
- self.out.append(data.strip())
+ if not self._skip:
+ self._line.append(data)
+
+ def close(self):
+ super().close()
+ self._flush()
def _html(raw: str) -> list[Page]:
parser = _Stripper()
parser.feed(raw)
+ parser.close()
return [_page("\n".join(parser.out), 1)]
diff --git a/studio/backend/tests/test_rag_parsing.py b/studio/backend/tests/test_rag_parsing.py
index f6a91ad1a72..6bcaa73360b 100644
--- a/studio/backend/tests/test_rag_parsing.py
+++ b/studio/backend/tests/test_rag_parsing.py
@@ -554,3 +554,96 @@ def test_docx_skips_placeholder_rows_and_cells_but_keeps_columns(tmp_path):
text = "\n".join(pg.text for pg in parsers.parse(str(path)))
assert "PROMPT" not in text
assert "Name | | END" in text
+
+
+def _parse_html(tmp_path, body):
+ from core.rag import parsers
+
+ path = tmp_path / "page.html"
+ path.write_text(f"<html><body>{body}</body></html>", encoding = "utf-8")
+ return "\n".join(p.text for p in parsers.parse(str(path)))
+
+
+def test_html_keeps_inline_elements_in_their_line(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ '<p>The <b>quick</b> brown fox jumps over the <a href="#">lazy</a> dog.</p>'
+ "<p>It is un<b>believ</b>able.</p>",
+ )
+ assert text == "The quick brown fox jumps over the lazy dog.\nIt is unbelievable."
+
+
+def test_html_adjacent_buttons_stay_separate_words(tmp_path):
+ text = _parse_html(
+ tmp_path, "<p>Click <button>Accept</button><button>Decline</button> to go on.</p>"
+ )
+ assert text == "Click Accept Decline to go on."
+
+
+def test_html_block_elements_start_new_lines(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<h1>Install <em>guide</em></h1><ul><li>One</li><li>Two <i>items</i></li></ul>"
+ "<p>line one<br>line two</p><table><tr><td>cell a</td><td>cell b</td></tr></table>",
+ )
+ assert text == "Install guide\nOne\nTwo items\nline one\nline two\ncell a\ncell b"
+
+
+def test_html_legend_and_options_stay_separate_words(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<fieldset><legend>Size</legend>Pick one</fieldset>"
+ "<select><option>Small</option><option>Large</option></select>",
+ )
+ assert text == "Size\nPick one\nSmall\nLarge"
+
+
+def test_html_keeps_text_after_the_last_block(tmp_path):
+ text = _parse_html(tmp_path, "<p>First</p>Trailing <b>text</b>")
+ assert text == "First\nTrailing text"
+
+
+def test_html_pre_keeps_its_layout(tmp_path):
+ text = _parse_html(tmp_path, "<p>Code:</p><pre>def f():\n return 1</pre>")
+ assert text == "Code:\ndef f():\n return 1"
+
+
+def test_html_textarea_keeps_its_layout_and_svg_labels_stay_apart(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<textarea>line one\n indented</textarea>"
+ '<svg><text x="0">Revenue</text><text x="0" y="20">Cost</text></svg>',
+ )
+ assert text == "line one\n indented\nRevenue\nCost"
+
+
+def test_html_positioned_svg_tspans_are_separate_labels(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ '<svg><text><tspan x="0" y="0">Revenue</tspan><tspan x="0" dy="20">Cost</tspan>'
+ "</text><text>Bold<tspan>er</tspan></text></svg>",
+ )
+ assert text == "Revenue\nCost\nBolder"
+
+
+def test_html_skips_script_style_and_template(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<p>Visible</p><script>var x = 1;</script><style>p { color: red }</style>"
+ "<template><p>Inert until cloned</p></template>",
+ )
+ assert text == "Visible"
+
+
+def test_html_keeps_declarative_shadow_root_text(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ '<my-card><template shadowrootmode="open"><p>Shadow text</p>'
+ "<template><p>inert</p></template></template></my-card><p>After</p>",
+ )
+ assert text == "Shadow text\nAfter"
+
+
+def test_html_template_blocks_do_not_split_visible_text(tmp_path):
+ text = _parse_html(tmp_path, "<p>Hello <template><div>hidden</div></template>world</p>")
+ assert text == "Hello world"