Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep HTML table rows together in knowledge bases and sources (#13186)
Nilay1a384c3f9d49
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex 5c0e4b92a02..5d089a1882c 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -57,45 +57,171 @@ _HTML_BLOCK_TAGS = frozenset(" td text textarea th title tr ul xmp".split())_HTML_PRE_TAGS = frozenset(("listing", "plaintext", "pre", "textarea", "xmp"))-# Atomic inline boxes: their text never runs into a neighbour's, but they do not break the line.+# atomic inline boxes separate neighbouring text without breaking the line_HTML_BOX_TAGS = frozenset(("button", "img", "input", "select"))+_HTML_ROW_GROUPS = frozenset(("thead", "tbody", "tfoot"))+_HTML_TABLE_DEPTH = 32+++class _HtmlPre(str):+ pass+++class _HtmlTable:+ def __init__(+ self,+ sink: list,+ nested: bool = False,+ ) -> None:+ self.sink = sink+ self.nested = nested+ self.row: list[list | None] | None = None+ self.spans: list[int] = []+++def _html_span(attrs, name: str, limit: int) -> int:+ try:+ value = int(dict(attrs).get(name) or 1)+ except ValueError:+ return 1+ # rowspan="0" runs to the end of its row group+ return limit if value == 0 and name == "rowspan" else min(max(value, 1), limit)class _Stripper(HTMLParser):- """Collect visible text, one line per block element."""+ """collect visible text, one line per block element."""- def __init__(self) -> None:+ def __init__(self, span_budget: int = 0) -> None:super().__init__()self._skip = 0self._pre = 0self._templates: list[bool] = []self._line: list[str] = []self.out: list[str] = []+ self._tables: list[_HtmlTable] = []+ self._span_budget = span_budget+ self._deep = 0++ def _in_table(self) -> bool:+ return bool(self._tables) and not self._skip and not self._deep++ def _end_cell(self) -> None:+ self._flush()+ self.out = self._tables[-1].sink++ def _end_row(self) -> None:+ self._end_cell()+ table = self._tables[-1]+ row, table.row = table.row, None+ if row is None:+ return+ missing = max(len(table.spans) - len(row), 0)+ for col in range(len(row), len(table.spans)):+ table.spans[col] = max(table.spans[col] - 1, 0)+ self._span_budget -= missing+ if self._span_budget < 0:+ table.spans.clear()+ else:+ row.extend([None] * missing)+ while table.spans and not table.spans[-1]:+ table.spans.pop()+ if len(row) == 1:+ table.sink.extend(line for line in row[0] or () if line is not None)+ return+ cells: list[str] = []+ below: list[str] = []+ for cell in row:+ lines = cell or []+ parent: list[str] = []+ in_nested = False+ for line in lines:+ if line is None:+ in_nested = not in_nested+ elif in_nested:+ below.append(line)+ else:+ parent.append(line)+ parts: list[str] = []+ previous_pre = False+ for line in parent:+ is_pre = isinstance(line, _HtmlPre)+ if parts:+ parts.append("\n" if previous_pre or is_pre else " ")+ parts.append(str(line) if is_pre else " ".join(line.split()))+ previous_pre = is_pre+ cells.append("".join(parts))+ if any(cells):+ table.sink.append(" | ".join(cells))+ table.sink.extend(below)++ def _start_cell(self, attrs) -> None:+ self._end_cell()+ table = self._tables[-1]+ if table.row is None:+ table.row = []+ row, spans = table.row, table.spans+ start = len(row)+ while len(row) < len(spans) and spans[len(row)]:+ spans[len(row)] -= 1+ row.append(None)+ self._flush()+ cell: list[str] = []+ rowspan = _html_span(attrs, "rowspan", 65534)+ colspan = _html_span(attrs, "colspan", 1000)+ # each spanned slot adds an empty field, so stop when fields outnumber input characters.+ self._span_budget -= len(row) - start + colspan - 1+ if self._span_budget < 0:+ rowspan = colspan = 1+ spans[:] = [0] * len(spans)+ for i in range(colspan):+ if len(row) < len(spans):+ spans[len(row)] = rowspan - 1+ else:+ spans.append(rowspan - 1)+ row.append(None if i else cell)+ self.out = celldef _flush(self) -> None:text = "".join(self._line)self._line = []- # Whitespace inside <pre>/<textarea> is content; elsewhere it is layout.+ # whitespace inside <pre>/<textarea> is content; elsewhere it is layouttext = text.strip("\n") if self._pre else " ".join(text.split())if text.strip():- self.out.append(text)+ self.out.append(_HtmlPre(text) if self._pre else text)def handle_starttag(self, tag, attrs):if tag == "template":- # A declarative shadow root (shadowrootmode=open|closed) is rendered; other templates are inert.+ # declarative shadow roots render; other templates are inertinert = (dict(attrs).get("shadowrootmode") or "").lower() not in ("open", "closed")self._templates.append(inert)self._skip += inertelif tag in _HTML_SKIP_TAGS:self._skip += 1+ elif tag in ("td", "th") and self._in_table():+ self._start_cell(attrs)+ elif tag == "tr" and self._in_table():+ self._end_row()+ self._flush()+ self._tables[-1].row = []+ elif tag in _HTML_ROW_GROUPS and self._in_table():+ self._end_row()+ self._tables[-1].spans = []elif tag in _HTML_BLOCK_TAGS and not self._skip:self._flush()+ # each nesting level copies lines into its parent, so deeply nested tables read as plain blocks+ if tag == "table" and (self._deep or len(self._tables) >= _HTML_TABLE_DEPTH):+ self._deep += 1+ elif tag == "table":+ nested = bool(self._tables and self.out is not self._tables[-1].sink)+ if nested:+ self.out.append(None)+ self._tables.append(_HtmlTable(self.out, nested))if tag in _HTML_PRE_TAGS:self._pre += 1elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:self._line.append(" ")elif tag == "tspan" and not self._skip and any(k in ("x", "y") for k, _ in attrs):- # An absolute x/y starts a new SVG text chunk (a separate label or line).+ # absolute x/y coordinates start a separate SVG text label or lineself._flush()def handle_endtag(self, tag):@@ -105,8 +231,22 @@ class _Stripper(HTMLParser):elif tag in _HTML_SKIP_TAGS:if self._skip:self._skip -= 1+ elif tag in ("td", "th") and self._in_table():+ self._end_cell()+ elif tag in _HTML_ROW_GROUPS and self._in_table():+ self._end_row()+ self._tables[-1].spans = []+ elif tag in ("tr", "table") and self._in_table():+ self._end_row()+ if tag == "table":+ table = self._tables.pop()+ if table.nested:+ self.out.append(None)+ self._flush()elif tag in _HTML_BLOCK_TAGS and not self._skip:self._flush()+ if tag == "table" and self._deep:+ self._deep -= 1if tag in _HTML_PRE_TAGS and self._pre:self._pre -= 1elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:@@ -118,19 +258,23 @@ class _Stripper(HTMLParser):def close(self):super().close()+ while self._tables:+ self._end_row()+ table = self._tables.pop()+ if table.nested:+ self.out.append(None)self._flush()def _html(raw: str) -> list[Page]:- parser = _Stripper()+ parser = _Stripper(len(raw))parser.feed(raw)parser.close()return [_page("\n".join(parser.out), 1)]-# pymupdf4llm rebuilds text from positioned glyphs and mangles complex-shaping scripts (RTL forms,-# Indic matras to U+FFFD), so fall back to PyMuPDF's logical-order get_text(); thresholds mirror-# unslothai/unsloth#5351.+# pymupdf4llm can corrupt positioned RTL and Indic glyphs; use logical-order+# get_text() at unslothai/unsloth#5351 thresholds_SHAPED_PRESENTATION_FORMS = re.compile("[\ufb1d-\ufdff\ufe70-\ufefc]")_PDF_FALLBACK_MIN_BAD_GLYPHS = 5_PDF_FALLBACK_BAD_GLYPH_RATIO = 0.0005diff --git a/studio/backend/tests/test_rag_parsing.py b/studio/backend/tests/test_rag_parsing.pyindex b4b579ef6ae..bab99e2bf20 100644--- a/studio/backend/tests/test_rag_parsing.py+++ b/studio/backend/tests/test_rag_parsing.py@@ -860,7 +860,98 @@ def test_html_block_elements_start_new_lines(tmp_path):"<h1>Install <em>guide</em></h1><ul><li>One</li><li>Two <i>items</i></li></ul>""<p>line one<br>line two</p><table><tr><td>cell a</td><td>cell b</td></tr></table>",)- assert text == "Install guide\nOne\nTwo items\nline one\nline two\ncell a\ncell b"+ assert text == "Install guide\nOne\nTwo items\nline one\nline two\ncell a | cell b"+++def test_html_table_rows_keep_their_columns(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><th>Plan</th><th>Price</th><th>Support</th><th>Seats</th></tr>"+ "<tr><td>Starter</td><td>$9</td><td><p>Email</p><p>Chat</p></td><td>1</td></tr>"+ "<tr><td>Team<td>$49<td><td>10</table><p>After</p>",+ )+ assert text == (+ "Plan | Price | Support | Seats\nStarter | $9 | Email Chat | 1\nTeam | $49 | | 10\nAfter"+ )+++def test_html_table_spans_keep_columns_aligned(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><th>Plan</th><th colspan=2>Price</th></tr>"+ "<tr><td rowspan=2>Pro</td><td>Monthly</td><td>$20</td></tr>"+ "<tr><td>Annual</td><td>$200</td></tr></table>",+ )+ assert text == "Plan | Price | \nPro | Monthly | $20\n | Annual | $200"+++def test_html_trailing_rowspan_keeps_columns_aligned(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><td>Item</td><td rowspan=2>Notes</td></tr><tr><td>Next</td></tr></table>",+ )+ assert text == "Item | Notes\nNext | "+++def test_html_table_preserves_preformatted_cell_whitespace(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><td><pre>a\n b<br> c</pre></td><td>d</td></tr></table>",+ )+ assert text == "a\n b\n c | d"+++def test_html_layout_and_nested_tables_keep_their_lines(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><td><h1>Title</h1><p>Intro</p>"+ "<table><tr><td>Name<table><tr><td>x</td><td>y</td></tr></table></td><td>Value</td></tr></table>"+ "<p>Outro</p></td></tr></table>",+ )+ assert text == "Title\nIntro\nName | Value\nx | y\nOutro"+++def test_html_text_after_a_nested_table_stays_in_its_parent_cell(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><td>Before<table><tr><td>x</td><td>y</td></tr></table>After</td>"+ "<td>Peer</td></tr></table>",+ )+ assert text == "Before After | Peer\nx | y"+++def test_html_table_spans_are_capped_by_the_file_size(tmp_path):+ html = "<table><tr><td colspan=1000 rowspan=65534>x" + "<tr><td>a" * 2000+ text = _parse_html(tmp_path, html)+ assert text.count("a") == 2000 and len(text) < 4 * len(html)+++def test_html_empty_rows_spend_the_span_budget(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><tr><td colspan=50 rowspan=65534>x" + "<tr>" * 100 + "<tr><td>a<td>b</table>",+ )+ assert text.splitlines()[-1] == "a | b"+++def test_html_rowspan_stops_at_its_row_group(tmp_path):+ text = _parse_html(+ tmp_path,+ "<table><thead><tr><th rowspan=3>Plan</th><th>Price</th></tr></thead>"+ "<tbody><tr><td rowspan=0>Team</td><td>$49</td></tr><tr><td>$490</td></tr></tbody>"+ "<tbody><tr><td>Pro</td><td>$99</td></tr></tbody></table>",+ )+ assert text == "Plan | Price\nTeam | $49\n | $490\nPro | $99"+++def test_html_tables_nested_past_the_depth_cap_read_as_blocks(tmp_path):+ text = _parse_html(tmp_path, "<table><tr><td>" * 32 + "<table><tr><td>a<td>b" + "<p>end")+ assert text == "a\nb\nend"+++def test_html_stray_cell_end_keeps_words_apart(tmp_path):+ text = _parse_html(tmp_path, "<table><tr><td>Total</td>Note</td>Extra</tr></table>")+ assert text == "Note\nExtra\nTotal"def test_html_legend_and_options_stay_separate_words(tmp_path):