Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep inline HTML text in its line when parsing an upload (#11818)
Lukasdcd0e1ca670a
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex d36ff9121b7..7114aebbcba 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -48,30 +48,82 @@ def _page(text: str, page_number: int | None) -> Page:return Page(text = text, page_number = page_number, char_count = len(text))+_HTML_SKIP_TAGS = frozenset(("script", "style", "template"))+_HTML_BLOCK_TAGS = frozenset(+ "address article aside blockquote br caption center dd details dialog dir div dl dt"+ " fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr legend li"+ " listing main menu nav ol optgroup option p plaintext pre search section summary table"+ " td text textarea th title tr ul xmp".split()+)+_HTML_PRE_TAGS = frozenset(("listing", "plaintext", "pre", "textarea", "xmp"))+# Atomic inline boxes: their text never runs into a neighbour's, but they do not break the line.+_HTML_BOX_TAGS = frozenset(("button", "img", "input", "select"))++class _Stripper(HTMLParser):- """Collect visible text, skipping <script>/<style>."""+ """Collect visible text, one line per block element."""def __init__(self) -> None:super().__init__()self._skip = 0+ self._pre = 0+ self._templates: list[bool] = []+ self._line: list[str] = []self.out: list[str] = []+ def _flush(self) -> None:+ text = "".join(self._line)+ self._line = []+ # Whitespace inside <pre>/<textarea> is content; elsewhere it is layout.+ text = text.strip("\n") if self._pre else " ".join(text.split())+ if text.strip():+ self.out.append(text)+def handle_starttag(self, tag, attrs):- if tag in ("script", "style"):+ if tag == "template":+ # A declarative shadow root (shadowrootmode=open|closed) is rendered; other templates are inert.+ inert = (dict(attrs).get("shadowrootmode") or "").lower() not in ("open", "closed")+ self._templates.append(inert)+ self._skip += inert+ elif tag in _HTML_SKIP_TAGS:self._skip += 1+ elif tag in _HTML_BLOCK_TAGS and not self._skip:+ self._flush()+ if tag in _HTML_PRE_TAGS:+ self._pre += 1+ elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:+ self._line.append(" ")+ elif tag == "tspan" and not self._skip and any(k in ("x", "y") for k, _ in attrs):+ # An absolute x/y starts a new SVG text chunk (a separate label or line).+ self._flush()def handle_endtag(self, tag):- if tag in ("script", "style") and self._skip:- self._skip -= 1+ if tag == "template":+ if self._templates and self._templates.pop() and self._skip:+ self._skip -= 1+ elif tag in _HTML_SKIP_TAGS:+ if self._skip:+ self._skip -= 1+ elif tag in _HTML_BLOCK_TAGS and not self._skip:+ self._flush()+ if tag in _HTML_PRE_TAGS and self._pre:+ self._pre -= 1+ elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:+ self._line.append(" ")def handle_data(self, data):- if not self._skip and data.strip():- self.out.append(data.strip())+ if not self._skip:+ self._line.append(data)++ def close(self):+ super().close()+ self._flush()def _html(raw: str) -> list[Page]:parser = _Stripper()parser.feed(raw)+ parser.close()return [_page("\n".join(parser.out), 1)]diff --git a/studio/backend/tests/test_rag_parsing.py b/studio/backend/tests/test_rag_parsing.pyindex f6a91ad1a72..6bcaa73360b 100644--- a/studio/backend/tests/test_rag_parsing.py+++ b/studio/backend/tests/test_rag_parsing.py@@ -554,3 +554,96 @@ def test_docx_skips_placeholder_rows_and_cells_but_keeps_columns(tmp_path):text = "\n".join(pg.text for pg in parsers.parse(str(path)))assert "PROMPT" not in textassert "Name | | END" in text+++def _parse_html(tmp_path, body):+ from core.rag import parsers++ path = tmp_path / "page.html"+ path.write_text(f"<html><body>{body}</body></html>", encoding = "utf-8")+ return "\n".join(p.text for p in parsers.parse(str(path)))+++def test_html_keeps_inline_elements_in_their_line(tmp_path):+ text = _parse_html(+ tmp_path,+ '<p>The <b>quick</b> brown fox jumps over the <a href="#">lazy</a> dog.</p>'+ "<p>It is un<b>believ</b>able.</p>",+ )+ assert text == "The quick brown fox jumps over the lazy dog.\nIt is unbelievable."+++def test_html_adjacent_buttons_stay_separate_words(tmp_path):+ text = _parse_html(+ tmp_path, "<p>Click <button>Accept</button><button>Decline</button> to go on.</p>"+ )+ assert text == "Click Accept Decline to go on."+++def test_html_block_elements_start_new_lines(tmp_path):+ text = _parse_html(+ tmp_path,+ "<h1>Install <em>guide</em></h1><ul><li>One</li><li>Two <i>items</i></li></ul>"+ "<p>line one<br>line two</p><table><tr><td>cell a</td><td>cell b</td></tr></table>",+ )+ assert text == "Install guide\nOne\nTwo items\nline one\nline two\ncell a\ncell b"+++def test_html_legend_and_options_stay_separate_words(tmp_path):+ text = _parse_html(+ tmp_path,+ "<fieldset><legend>Size</legend>Pick one</fieldset>"+ "<select><option>Small</option><option>Large</option></select>",+ )+ assert text == "Size\nPick one\nSmall\nLarge"+++def test_html_keeps_text_after_the_last_block(tmp_path):+ text = _parse_html(tmp_path, "<p>First</p>Trailing <b>text</b>")+ assert text == "First\nTrailing text"+++def test_html_pre_keeps_its_layout(tmp_path):+ text = _parse_html(tmp_path, "<p>Code:</p><pre>def f():\n return 1</pre>")+ assert text == "Code:\ndef f():\n return 1"+++def test_html_textarea_keeps_its_layout_and_svg_labels_stay_apart(tmp_path):+ text = _parse_html(+ tmp_path,+ "<textarea>line one\n indented</textarea>"+ '<svg><text x="0">Revenue</text><text x="0" y="20">Cost</text></svg>',+ )+ assert text == "line one\n indented\nRevenue\nCost"+++def test_html_positioned_svg_tspans_are_separate_labels(tmp_path):+ text = _parse_html(+ tmp_path,+ '<svg><text><tspan x="0" y="0">Revenue</tspan><tspan x="0" dy="20">Cost</tspan>'+ "</text><text>Bold<tspan>er</tspan></text></svg>",+ )+ assert text == "Revenue\nCost\nBolder"+++def test_html_skips_script_style_and_template(tmp_path):+ text = _parse_html(+ tmp_path,+ "<p>Visible</p><script>var x = 1;</script><style>p { color: red }</style>"+ "<template><p>Inert until cloned</p></template>",+ )+ assert text == "Visible"+++def test_html_keeps_declarative_shadow_root_text(tmp_path):+ text = _parse_html(+ tmp_path,+ '<my-card><template shadowrootmode="open"><p>Shadow text</p>'+ "<template><p>inert</p></template></template></my-card><p>After</p>",+ )+ assert text == "Shadow text\nAfter"+++def test_html_template_blocks_do_not_split_visible_text(tmp_path):+ text = _parse_html(tmp_path, "<p>Hello <template><div>hidden</div></template>world</p>")+ assert text == "Hello world"