Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep HTML table rows together in knowledge bases and sources (#13186)

Nilay1a384c3f9d49
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.py
index 5c0e4b92a02..5d089a1882c 100644
--- a/studio/backend/core/rag/parsers.py
+++ b/studio/backend/core/rag/parsers.py
@@ -57,45 +57,171 @@ _HTML_BLOCK_TAGS = frozenset(
" td text textarea th title tr ul xmp".split()
)
_HTML_PRE_TAGS = frozenset(("listing", "plaintext", "pre", "textarea", "xmp"))
-# Atomic inline boxes: their text never runs into a neighbour's, but they do not break the line.
+# atomic inline boxes separate neighbouring text without breaking the line
_HTML_BOX_TAGS = frozenset(("button", "img", "input", "select"))
+_HTML_ROW_GROUPS = frozenset(("thead", "tbody", "tfoot"))
+_HTML_TABLE_DEPTH = 32
+
+
+class _HtmlPre(str):
+ pass
+
+
+class _HtmlTable:
+ def __init__(
+ self,
+ sink: list,
+ nested: bool = False,
+ ) -> None:
+ self.sink = sink
+ self.nested = nested
+ self.row: list[list | None] | None = None
+ self.spans: list[int] = []
+
+
+def _html_span(attrs, name: str, limit: int) -> int:
+ try:
+ value = int(dict(attrs).get(name) or 1)
+ except ValueError:
+ return 1
+ # rowspan="0" runs to the end of its row group
+ return limit if value == 0 and name == "rowspan" else min(max(value, 1), limit)
class _Stripper(HTMLParser):
- """Collect visible text, one line per block element."""
+ """collect visible text, one line per block element."""
- def __init__(self) -> None:
+ def __init__(self, span_budget: int = 0) -> None:
super().__init__()
self._skip = 0
self._pre = 0
self._templates: list[bool] = []
self._line: list[str] = []
self.out: list[str] = []
+ self._tables: list[_HtmlTable] = []
+ self._span_budget = span_budget
+ self._deep = 0
+
+ def _in_table(self) -> bool:
+ return bool(self._tables) and not self._skip and not self._deep
+
+ def _end_cell(self) -> None:
+ self._flush()
+ self.out = self._tables[-1].sink
+
+ def _end_row(self) -> None:
+ self._end_cell()
+ table = self._tables[-1]
+ row, table.row = table.row, None
+ if row is None:
+ return
+ missing = max(len(table.spans) - len(row), 0)
+ for col in range(len(row), len(table.spans)):
+ table.spans[col] = max(table.spans[col] - 1, 0)
+ self._span_budget -= missing
+ if self._span_budget < 0:
+ table.spans.clear()
+ else:
+ row.extend([None] * missing)
+ while table.spans and not table.spans[-1]:
+ table.spans.pop()
+ if len(row) == 1:
+ table.sink.extend(line for line in row[0] or () if line is not None)
+ return
+ cells: list[str] = []
+ below: list[str] = []
+ for cell in row:
+ lines = cell or []
+ parent: list[str] = []
+ in_nested = False
+ for line in lines:
+ if line is None:
+ in_nested = not in_nested
+ elif in_nested:
+ below.append(line)
+ else:
+ parent.append(line)
+ parts: list[str] = []
+ previous_pre = False
+ for line in parent:
+ is_pre = isinstance(line, _HtmlPre)
+ if parts:
+ parts.append("\n" if previous_pre or is_pre else " ")
+ parts.append(str(line) if is_pre else " ".join(line.split()))
+ previous_pre = is_pre
+ cells.append("".join(parts))
+ if any(cells):
+ table.sink.append(" | ".join(cells))
+ table.sink.extend(below)
+
+ def _start_cell(self, attrs) -> None:
+ self._end_cell()
+ table = self._tables[-1]
+ if table.row is None:
+ table.row = []
+ row, spans = table.row, table.spans
+ start = len(row)
+ while len(row) < len(spans) and spans[len(row)]:
+ spans[len(row)] -= 1
+ row.append(None)
+ self._flush()
+ cell: list[str] = []
+ rowspan = _html_span(attrs, "rowspan", 65534)
+ colspan = _html_span(attrs, "colspan", 1000)
+ # each spanned slot adds an empty field, so stop when fields outnumber input characters.
+ self._span_budget -= len(row) - start + colspan - 1
+ if self._span_budget < 0:
+ rowspan = colspan = 1
+ spans[:] = [0] * len(spans)
+ for i in range(colspan):
+ if len(row) < len(spans):
+ spans[len(row)] = rowspan - 1
+ else:
+ spans.append(rowspan - 1)
+ row.append(None if i else cell)
+ self.out = cell
def _flush(self) -> None:
text = "".join(self._line)
self._line = []
- # Whitespace inside <pre>/<textarea> is content; elsewhere it is layout.
+ # whitespace inside <pre>/<textarea> is content; elsewhere it is layout
text = text.strip("\n") if self._pre else " ".join(text.split())
if text.strip():
- self.out.append(text)
+ self.out.append(_HtmlPre(text) if self._pre else text)
def handle_starttag(self, tag, attrs):
if tag == "template":
- # A declarative shadow root (shadowrootmode=open|closed) is rendered; other templates are inert.
+ # declarative shadow roots render; other templates are inert
inert = (dict(attrs).get("shadowrootmode") or "").lower() not in ("open", "closed")
self._templates.append(inert)
self._skip += inert
elif tag in _HTML_SKIP_TAGS:
self._skip += 1
+ elif tag in ("td", "th") and self._in_table():
+ self._start_cell(attrs)
+ elif tag == "tr" and self._in_table():
+ self._end_row()
+ self._flush()
+ self._tables[-1].row = []
+ elif tag in _HTML_ROW_GROUPS and self._in_table():
+ self._end_row()
+ self._tables[-1].spans = []
elif tag in _HTML_BLOCK_TAGS and not self._skip:
self._flush()
+ # each nesting level copies lines into its parent, so deeply nested tables read as plain blocks
+ if tag == "table" and (self._deep or len(self._tables) >= _HTML_TABLE_DEPTH):
+ self._deep += 1
+ elif tag == "table":
+ nested = bool(self._tables and self.out is not self._tables[-1].sink)
+ if nested:
+ self.out.append(None)
+ self._tables.append(_HtmlTable(self.out, nested))
if tag in _HTML_PRE_TAGS:
self._pre += 1
elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:
self._line.append(" ")
elif tag == "tspan" and not self._skip and any(k in ("x", "y") for k, _ in attrs):
- # An absolute x/y starts a new SVG text chunk (a separate label or line).
+ # absolute x/y coordinates start a separate SVG text label or line
self._flush()
def handle_endtag(self, tag):
@@ -105,8 +231,22 @@ class _Stripper(HTMLParser):
elif tag in _HTML_SKIP_TAGS:
if self._skip:
self._skip -= 1
+ elif tag in ("td", "th") and self._in_table():
+ self._end_cell()
+ elif tag in _HTML_ROW_GROUPS and self._in_table():
+ self._end_row()
+ self._tables[-1].spans = []
+ elif tag in ("tr", "table") and self._in_table():
+ self._end_row()
+ if tag == "table":
+ table = self._tables.pop()
+ if table.nested:
+ self.out.append(None)
+ self._flush()
elif tag in _HTML_BLOCK_TAGS and not self._skip:
self._flush()
+ if tag == "table" and self._deep:
+ self._deep -= 1
if tag in _HTML_PRE_TAGS and self._pre:
self._pre -= 1
elif tag in _HTML_BOX_TAGS and not self._skip and not self._pre:
@@ -118,19 +258,23 @@ class _Stripper(HTMLParser):
def close(self):
super().close()
+ while self._tables:
+ self._end_row()
+ table = self._tables.pop()
+ if table.nested:
+ self.out.append(None)
self._flush()
def _html(raw: str) -> list[Page]:
- parser = _Stripper()
+ parser = _Stripper(len(raw))
parser.feed(raw)
parser.close()
return [_page("\n".join(parser.out), 1)]
-# pymupdf4llm rebuilds text from positioned glyphs and mangles complex-shaping scripts (RTL forms,
-# Indic matras to U+FFFD), so fall back to PyMuPDF's logical-order get_text(); thresholds mirror
-# unslothai/unsloth#5351.
+# pymupdf4llm can corrupt positioned RTL and Indic glyphs; use logical-order
+# get_text() at unslothai/unsloth#5351 thresholds
_SHAPED_PRESENTATION_FORMS = re.compile("[\ufb1d-\ufdff\ufe70-\ufefc]")
_PDF_FALLBACK_MIN_BAD_GLYPHS = 5
_PDF_FALLBACK_BAD_GLYPH_RATIO = 0.0005
diff --git a/studio/backend/tests/test_rag_parsing.py b/studio/backend/tests/test_rag_parsing.py
index b4b579ef6ae..bab99e2bf20 100644
--- a/studio/backend/tests/test_rag_parsing.py
+++ b/studio/backend/tests/test_rag_parsing.py
@@ -860,7 +860,98 @@ def test_html_block_elements_start_new_lines(tmp_path):
"<h1>Install <em>guide</em></h1><ul><li>One</li><li>Two <i>items</i></li></ul>"
"<p>line one<br>line two</p><table><tr><td>cell a</td><td>cell b</td></tr></table>",
)
- assert text == "Install guide\nOne\nTwo items\nline one\nline two\ncell a\ncell b"
+ assert text == "Install guide\nOne\nTwo items\nline one\nline two\ncell a | cell b"
+
+
+def test_html_table_rows_keep_their_columns(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><th>Plan</th><th>Price</th><th>Support</th><th>Seats</th></tr>"
+ "<tr><td>Starter</td><td>$9</td><td><p>Email</p><p>Chat</p></td><td>1</td></tr>"
+ "<tr><td>Team<td>$49<td><td>10</table><p>After</p>",
+ )
+ assert text == (
+ "Plan | Price | Support | Seats\nStarter | $9 | Email Chat | 1\nTeam | $49 | | 10\nAfter"
+ )
+
+
+def test_html_table_spans_keep_columns_aligned(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><th>Plan</th><th colspan=2>Price</th></tr>"
+ "<tr><td rowspan=2>Pro</td><td>Monthly</td><td>$20</td></tr>"
+ "<tr><td>Annual</td><td>$200</td></tr></table>",
+ )
+ assert text == "Plan | Price | \nPro | Monthly | $20\n | Annual | $200"
+
+
+def test_html_trailing_rowspan_keeps_columns_aligned(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><td>Item</td><td rowspan=2>Notes</td></tr><tr><td>Next</td></tr></table>",
+ )
+ assert text == "Item | Notes\nNext | "
+
+
+def test_html_table_preserves_preformatted_cell_whitespace(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><td><pre>a\n b<br> c</pre></td><td>d</td></tr></table>",
+ )
+ assert text == "a\n b\n c | d"
+
+
+def test_html_layout_and_nested_tables_keep_their_lines(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><td><h1>Title</h1><p>Intro</p>"
+ "<table><tr><td>Name<table><tr><td>x</td><td>y</td></tr></table></td><td>Value</td></tr></table>"
+ "<p>Outro</p></td></tr></table>",
+ )
+ assert text == "Title\nIntro\nName | Value\nx | y\nOutro"
+
+
+def test_html_text_after_a_nested_table_stays_in_its_parent_cell(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><td>Before<table><tr><td>x</td><td>y</td></tr></table>After</td>"
+ "<td>Peer</td></tr></table>",
+ )
+ assert text == "Before After | Peer\nx | y"
+
+
+def test_html_table_spans_are_capped_by_the_file_size(tmp_path):
+ html = "<table><tr><td colspan=1000 rowspan=65534>x" + "<tr><td>a" * 2000
+ text = _parse_html(tmp_path, html)
+ assert text.count("a") == 2000 and len(text) < 4 * len(html)
+
+
+def test_html_empty_rows_spend_the_span_budget(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><tr><td colspan=50 rowspan=65534>x" + "<tr>" * 100 + "<tr><td>a<td>b</table>",
+ )
+ assert text.splitlines()[-1] == "a | b"
+
+
+def test_html_rowspan_stops_at_its_row_group(tmp_path):
+ text = _parse_html(
+ tmp_path,
+ "<table><thead><tr><th rowspan=3>Plan</th><th>Price</th></tr></thead>"
+ "<tbody><tr><td rowspan=0>Team</td><td>$49</td></tr><tr><td>$490</td></tr></tbody>"
+ "<tbody><tr><td>Pro</td><td>$99</td></tr></tbody></table>",
+ )
+ assert text == "Plan | Price\nTeam | $49\n | $490\nPro | $99"
+
+
+def test_html_tables_nested_past_the_depth_cap_read_as_blocks(tmp_path):
+ text = _parse_html(tmp_path, "<table><tr><td>" * 32 + "<table><tr><td>a<td>b" + "<p>end")
+ assert text == "a\nb\nend"
+
+
+def test_html_stray_cell_end_keeps_words_apart(tmp_path):
+ text = _parse_html(tmp_path, "<table><tr><td>Total</td>Note</td>Extra</tr></table>")
+ assert text == "Note\nExtra\nTotal"
def test_html_legend_and_options_stay_separate_words(tmp_path):