Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep merged table cells in their columns when reading web pages (#13098)
Nilaycceb77064e21
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex 33ad49f01e1..d6f13257aa9 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -100,6 +100,14 @@ def _is_hidden_element(attr_dict: dict) -> bool:return _style_hides_element(attr_dict.get("style") or "")+def _span_attr(attr_dict: dict, name: str) -> int:+ try:+ value = int(attr_dict.get(name) or 1)+ except ValueError:+ return 1+ return value if value >= 0 else 1++def _is_aria_heading(attr_dict: dict) -> bool:"""True for ``role="heading"``, which titles a page just as ``h1``-``h6`` does.@@ -192,6 +200,12 @@ _BLOCK_TAGS = frozenset(})_HEADING_TAGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})+# half the 16K fetch result: repeated span cells must leave room for the source text after the table+_MAX_SPAN_CHARS = 8_000+# a long rowspan cell is usually page layout, so only its first row keeps the text+_MAX_REPEATED_CELL_CHARS = 200+# floor per scope candidate once the page-wide total is spent, so an earlier decoy cannot starve a later article+_MIN_SCOPE_SPAN_CHARS = 256_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}# measured density: 0.94-1.00 for link lists, 0.13-0.90 for content headers@@ -226,6 +240,7 @@ class _HeaderFrame:"outer_in_pre","outer_in_code","outer_bq_depth",+ "outer_table_depth",)def __init__(@@ -237,6 +252,7 @@ class _HeaderFrame:in_code: int,bq_depth: int,list_depth: int,+ table_depth: int,):self.depth = depthself.parts: list[str] = []@@ -254,11 +270,16 @@ class _HeaderFrame:self.outer_in_pre = in_preself.outer_in_code = in_codeself.outer_bq_depth = bq_depth+ self.outer_table_depth = table_depth# both exclude heading text: it is kept anyway, so it must not vote on dropping the restself.text_chars: int = 0self.link_chars: int = 0- def render(self, closed_by_own_tag: bool) -> str:+ def render(+ self,+ closed_by_own_tag: bool,+ strip: bool | None = None,+ ) -> str:"""The buffer, or only its headings when the header is link furniture.Without a matching ``</header>`` the header may have adopted the page@@ -270,7 +291,11 @@ class _HeaderFrame:# droppable chars only: headings survive, and blank structure _cleanup collapses must not inflate a tiny headerdroppable = self.rendered_chars - self.heading_charsbig_enough = self.text_chars >= _HEADER_MIN_CHARS or droppable >= _HEADER_MAX_RENDERED_CHARS- if big_enough and self.link_chars >= _HEADER_LINK_DENSITY * self.text_chars:+ if (+ strip+ if strip is not None+ else (big_enough and self.link_chars >= _HEADER_LINK_DENSITY * self.text_chars)+ ):self.stripped = True# the closing tag's blank line lands after the heading mark pops, so terminate it herereturn headings + "\n\n" if headings.strip() else headings@@ -339,6 +364,41 @@ class SiteLinks:return self.clean("".join(out))+class _TableFrame:+ """Outer table state suspended while a nested table is rendered."""++ __slots__ = (+ "current_row",+ "cell_parts",+ "in_cell",+ "cell_seq",+ "header_row_done",+ "row_has_th",+ "is_first_row",+ "cell_colspan",+ "cell_rowspan",+ "row_spans",+ "in_row",+ "outer_bq_depth",+ "parts",+ )++ def __init__(self, renderer: _MarkdownRenderer):+ self.current_row = renderer._current_row+ self.cell_parts = renderer._cell_parts+ self.in_cell = renderer._in_cell+ self.cell_seq = renderer._cell_seq+ self.header_row_done = renderer._header_row_done+ self.row_has_th = renderer._row_has_th+ self.is_first_row = renderer._is_first_row+ self.cell_colspan = renderer._cell_colspan+ self.cell_rowspan = renderer._cell_rowspan+ self.row_spans = renderer._row_spans+ self.in_row = renderer._in_row+ self.outer_bq_depth = len(renderer._bq_stack)+ self.parts: list[str] = []++class _MarkdownRenderer(HTMLParser):"""HTMLParser subclass that emits Markdown tokens into a list.@@ -351,7 +411,10 @@ class _MarkdownRenderer(HTMLParser):self,scope_tags: frozenset[str] | None = None,strip_header: bool = False,+ span_char_limit: int = _MAX_SPAN_CHARS,+ header_decisions: list[bool] | None = None,site_links: SiteLinks | None = None,+ page_span_limit: int | None = None,):super().__init__(convert_charrefs = False)self._site_links = site_links@@ -375,6 +438,8 @@ class _MarkdownRenderer(HTMLParser):# Open <header> buffers, innermost last. Empty unless strip_header.self._strip_header = strip_headerself._header_stack: list[_HeaderFrame] = []+ self.header_decisions: list[bool] = []+ self._header_decisions = header_decisions# Furniture chars removed, so a candidate is sized as the page wrote it.self._dropped_chars: int = 0self._seg_dropped_start: int = 0@@ -408,9 +473,20 @@ class _MarkdownRenderer(HTMLParser):self._cell_parts: list[str] = []self._in_cell: bool = Falseself._cell_seq: int = 0+ self._next_cell_seq: int = 0self._header_row_done: bool = Falseself._row_has_th: bool = Falseself._is_first_row: bool = False+ self._cell_colspan: int = 1+ self._cell_rowspan: int = 1+ self._row_spans: dict[int, tuple[str, int]] = {}+ self._table_stack: list[_TableFrame] = []+ self._span_chars: int = 0+ self._has_generated_spans: bool = False+ self._span_char_limit: int = min(_MAX_SPAN_CHARS, max(0, span_char_limit))+ self._scope_span_limit: int = self._span_char_limit+ self._page_span_left: int | None = page_span_limit+ self._in_row: bool = Falseself._in_pre: bool = Falseself._pre_parts: list[str] = []@@ -432,6 +508,8 @@ class _MarkdownRenderer(HTMLParser):return self._cell_seq != frame.outer_cell_seqif self._in_pre:return not frame.outer_in_pre+ if self._table_stack and self._table_depth() > frame.outer_table_depth:+ return Truereturn len(self._bq_stack) > frame.outer_bq_depthdef _emit(self, text: str) -> None:@@ -470,6 +548,8 @@ class _MarkdownRenderer(HTMLParser):self._cell_parts.append(text)elif self._in_pre:self._pre_parts.append(text)+ elif self._table_stack and len(self._bq_stack) <= self._table_stack[-1].outer_bq_depth:+ self._table_stack[-1].parts.append(text)elif self._bq_stack:self._bq_stack[-1].append(text)else:@@ -524,18 +604,133 @@ class _MarkdownRenderer(HTMLParser):return "\n".join(prefixed)# Table helpers: flush open cells/rows so omitted </td>/</tr> don't lose data.+ def _table_depth(self) -> int:+ return len(self._table_stack) + int(self._in_table)++ def _start_table(self) -> None:+ if self._in_table:+ self._table_stack.append(_TableFrame(self))+ self._in_table = True+ self._current_row = []+ self._cell_parts = []+ self._in_cell = False+ self._header_row_done = False+ self._row_has_th = False+ self._is_first_row = True+ self._cell_colspan = 1+ self._cell_rowspan = 1+ self._row_spans = {}+ self._in_row = False+ self._emit("\n\n")++ def _finish_table(self) -> None:+ self._finish_cell()+ self._finish_row()+ self._row_spans = {}+ self._emit("\n")+ if not self._table_stack:+ self._in_table = False+ return++ frame = self._table_stack.pop()+ nested = "".join(frame.parts)+ self._current_row = frame.current_row+ self._cell_parts = frame.cell_parts+ self._in_cell = frame.in_cell+ self._cell_seq = frame.cell_seq+ self._header_row_done = frame.header_row_done+ self._row_has_th = frame.row_has_th+ self._is_first_row = frame.is_first_row+ self._cell_colspan = frame.cell_colspan+ self._cell_rowspan = frame.cell_rowspan+ self._row_spans = frame.row_spans+ self._in_row = frame.in_row+ self._in_table = True+ self._emit_replay(nested)+def _finish_cell(self) -> None:if not self._in_cell:returnself._in_cell = Falsecell_text = "".join(self._cell_parts).strip().replace("\n", " ")- cell_text = cell_text.replace("|", "\\|")- self._current_row.append(cell_text)+ if not self._table_stack:+ cell_text = cell_text.replace("|", "\\|")self._cell_parts = []+ self._fill_spanned_cells(0)+ col = len(self._current_row)+ self._current_row.append(cell_text)+ if self._span_chars >= self._span_char_limit:+ return+ outer_cells = sum(frame.in_cell for frame in self._table_stack)+ extra_col_cost = 9 + 2 * outer_cells if not self._header_row_done else 3 + outer_cells+ # a nested table flattens and drops empty cells, so its colspan padding would only spend budget+ extra_cols = (+ 0+ if self._table_stack+ else min(+ self._cell_colspan - 1,+ (self._span_char_limit - self._span_chars) // extra_col_cost,+ )+ )+ self._current_row.extend([""] * extra_cols)+ generated = extra_col_cost * extra_cols+ self._span_chars += generated+ self._has_generated_spans |= bool(extra_cols)+ if self._cell_rowspan > 1:+ repeated = cell_text if self._visible_len(cell_text) <= _MAX_REPEATED_CELL_CHARS else ""+ for i in range(extra_cols + 1):+ self._row_spans[col + i] = (repeated if i == 0 else "", self._cell_rowspan)++ def _visible_len(self, text: str) -> int:+ # site-link markers vanish from the output, so they must not spend the span budget+ if self._site_links is not None:+ text = self._site_links.clean(text)+ # budgets assume four ASCII chars per token; a CJK char is about one token+ return len(text) + 3 * sum(not ch.isascii() for ch in text)++ def _fill_spanned_cells(self, width: int) -> None:+ outer_cells = sum(frame.in_cell for frame in self._table_stack)+ while self._span_chars < self._span_char_limit and (+ len(self._current_row) < width or len(self._current_row) in self._row_spans+ ):+ span = self._row_spans.get(len(self._current_row))+ text = span[0] if span else ""+ cost = self._visible_len(text) + 3 + (text.count("|") + 1) * outer_cells+ if self._span_chars + cost > self._span_char_limit:+ self._span_chars = self._span_char_limit+ break+ self._current_row.append(text)+ self._span_chars += cost+ self._has_generated_spans = Truedef _finish_row(self) -> None:+ in_row, self._in_row = self._in_row, False+ if not self._current_row and not (in_row and self._row_spans):+ return+ if not self._current_row:+ # a wholly generated row also adds its opening pipe, newline, and enclosing quote prefixes+ row_cost = (+ 3 + 2 * len(self._bq_stack) + sum(frame.in_cell for frame in self._table_stack)+ )+ self._span_chars = min(self._span_char_limit, self._span_chars + row_cost)+ self._fill_spanned_cells(max(self._row_spans, default = -1) + 1)+ if self._span_chars >= self._span_char_limit:+ self._row_spans = {}+ else:+ self._row_spans = {+ col: (text, rows - 1) for col, (text, rows) in self._row_spans.items() if rows > 1+ }if not self._current_row:return+ if self._table_stack:+ # nested tables flatten into the outer cell: replaying their pipes re-escaped them per level+ line = " ".join(cell for cell in self._current_row if cell)+ if line:+ self._emit_replay(line + "\n")+ self._is_first_row = False+ self._current_row = []+ self._row_has_th = False+ returnline = "| " + " | ".join(self._current_row) + " |"self._emit_replay(line + "\n")if not self._header_row_done and (self._row_has_th or self._is_first_row):@@ -629,7 +824,13 @@ class _MarkdownRenderer(HTMLParser):self._header_stack[-1].link_chars += frame.link_charsself._header_stack[-1].heading_parts.extend(frame.heading_parts)self._header_stack[-1].heading_chars += frame.heading_chars- out = frame.render(closed_by_own_tag)+ strip = (+ self._header_decisions[len(self.header_decisions)]+ if self._header_decisions is not None+ else None+ )+ out = frame.render(closed_by_own_tag, strip)+ self.header_decisions.append(frame.stripped)if frame.stripped:self._dropped_chars += max(0, frame.rendered_chars - frame.heading_chars)self._emit(out)@@ -713,6 +914,7 @@ class _MarkdownRenderer(HTMLParser):self._inline_code_depth,len(self._bq_stack),len(self._list_stack),+ self._table_depth(),))elif _is_hidden_element(attr_dict):@@ -720,6 +922,12 @@ class _MarkdownRenderer(HTMLParser):if self._scope_tags is not None and tag in self._scope_tags:self._flush_header_frames()if self._scope_depth == 0:+ if self._page_span_left is not None:+ self._page_span_left = max(0, self._page_span_left - self._span_chars)+ self._span_char_limit = min(+ self._scope_span_limit, max(self._page_span_left, _MIN_SCOPE_SPAN_CHARS)+ )+ self._span_chars = 0self._scope_seg_start = len(self._out)self._seg_dropped_start = self._dropped_charsself._seg_heading_texts = []@@ -846,20 +1054,27 @@ class _MarkdownRenderer(HTMLParser):self._emit("`")elif tag == "table":- self._in_table = True- self._header_row_done = False- self._is_first_row = True- self._emit("\n\n")+ self._start_table()elif tag == "tr":self._finish_cell()self._finish_row()+ self._in_row = True++ elif tag in ("thead", "tbody", "tfoot"):+ self._finish_cell()+ self._finish_row()+ self._row_spans = {}elif tag in ("th", "td"):self._finish_cell()self._cell_parts = []self._in_cell = True- self._cell_seq += 1+ self._next_cell_seq += 1+ self._cell_seq = self._next_cell_seq+ self._cell_colspan = min(max(1, _span_attr(attr_dict, "colspan")), 1000)+ # rowspan="0" runs to the end of the row group+ self._cell_rowspan = _span_attr(attr_dict, "rowspan") or 65534if tag == "th":self._row_has_th = True@@ -926,11 +1141,13 @@ class _MarkdownRenderer(HTMLParser):self._finish_cell()self._finish_row()- elif tag == "table":+ elif tag in ("thead", "tbody", "tfoot"):self._finish_cell()self._finish_row()- self._in_table = False- self._emit("\n")+ self._row_spans = {}++ elif tag == "table":+ self._finish_table()def _text_suppressed(self) -> bool:if self._skip_depth or self._hidden_marks:@@ -989,6 +1206,8 @@ class _MarkdownRenderer(HTMLParser):self._inline_code_depth -= 1self._emit("`")+ while self._table_stack:+ self._finish_table()self._finish_cell()self._finish_row()@@ -1133,10 +1352,17 @@ def _new_renderer(source_html: str,scope_tags: frozenset[str] | None,strip_header: bool,- site_links: SiteLinks | None,+ site_links: SiteLinks | None = None,+ span_char_limit: int | None = None,+ header_decisions: list[bool] | None = None,) -> _MarkdownRenderer:renderer = _MarkdownRenderer(- scope_tags = scope_tags, strip_header = strip_header, site_links = site_links+ scope_tags = scope_tags,+ strip_header = strip_header,+ site_links = site_links,+ span_char_limit = 2 * len(source_html) if span_char_limit is None else span_char_limit,+ header_decisions = header_decisions,+ page_span_limit = 2 * len(source_html),)renderer.feed(source_html)renderer.close()@@ -1149,12 +1375,17 @@ def _render(scope_tags: frozenset[str] | None,strip_header: bool = False,site_links: SiteLinks | None = None,+ span_char_limit: int | None = None,) -> str:- return _cleanup("".join(_new_renderer(source_html, scope_tags, strip_header, site_links)._out))+ renderer = _new_renderer(source_html, scope_tags, strip_header, site_links, span_char_limit)+ return _cleanup("".join(renderer._out))def _select_main_scope_render(- source_html: str, tag: str, site_links: SiteLinks | None+ source_html: str,+ tag: str,+ site_links: SiteLinks | None,+ span_char_limit: int | None = None,) -> tuple[int, str]:"""Length and boilerplate-stripped render of the largest single ``<tag>``subtree. Sizing candidates one at a time stops many tiny sibling cards from@@ -1170,19 +1401,35 @@ def _select_main_scope_render(furniture can never be the majority of a score. Uncapped, a teaser with a1000 link header outranked a sibling holding five times its real text."""renderer = _new_renderer(- source_html, frozenset({tag}), strip_header = True, site_links = site_links+ source_html,+ frozenset({tag}),+ strip_header = True,+ site_links = site_links,+ span_char_limit = span_char_limit,+ )+ # generated span cells must not lift a scope past the content gate or rank; only span pages pay this second pass+ scoring = (+ _new_renderer(+ source_html,+ frozenset({tag}),+ strip_header = True,+ span_char_limit = 0,+ header_decisions = renderer.header_decisions,+ )+ if renderer._has_generated_spans+ else renderer)- dropped = renderer.scope_dropped- heading_prose = renderer.scope_heading_prosebest_len = 0best_render = ""for i, seg in enumerate(renderer.scope_segments):rendered = _strip_boilerplate_lines(_cleanup(seg), site_links)- measured = site_links.clean(rendered) if site_links is not None else rendered- prose = _visible_chars(measured) - heading_prose[i]+ scored = _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)+ if site_links is not None:+ scored = site_links.clean(scored)+ prose = _visible_chars(scored) - scoring.scope_heading_prose[i]if prose < _MIN_MAIN_CONTENT_CHARS:continue- size = len(measured) + min(dropped[i], len(measured))+ size = len(scored) + min(scoring.scope_dropped[i], len(scored))if size > best_len:best_len = sizebest_render = rendered@@ -1255,6 +1502,7 @@ def html_to_markdown(*,main_content: bool = False,site_links: SiteLinks | None = None,+ max_span_chars: int | None = None,) -> str:"""Convert HTML to Markdown (headings, links, emphasis, lists, tables, blockquotes, code, entities).@@ -1268,20 +1516,34 @@ def html_to_markdown(fragments from the result.``site_links`` records the links back into the page's own site; the output is unchanged.++ ``max_span_chars`` caps the cells generated for ``rowspan``/``colspan``, so a caller with a+ smaller result budget keeps room for the text after a table."""source_html = source_html.replace("\r\n", "\n").replace("\r", "\n")+ span_limit = 2 * len(source_html)+ if max_span_chars is not None:+ span_limit = min(span_limit, max_span_chars)rendered = ""if main_content:for scope_tag in ("article", "main"):# Render only the chosen subtree so sibling <article>/<main> elements do not leak in.- length, rendered = _select_main_scope_render(source_html, scope_tag, site_links)+ length, rendered = _select_main_scope_render(+ source_html, scope_tag, site_links, span_limit+ )if length >= _MIN_MAIN_CONTENT_CHARS:breakelse:rendered = _strip_boilerplate_lines(- _render(source_html, None, strip_header = True, site_links = site_links),+ _render(+ source_html,+ None,+ strip_header = True,+ site_links = site_links,+ span_char_limit = span_limit,+ ),site_links,)else:- rendered = _render(source_html, None, site_links = site_links)+ rendered = _render(source_html, None, site_links = site_links, span_char_limit = span_limit)return site_links.finish(rendered) if site_links is not None else rendereddiff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex db374381d1d..15c2b842f36 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -17128,7 +17128,7 @@ def _fetch_page_text(# Markdown README that merely opens with a block tag is kept as-is (see _HTML_DOCUMENT_RE).if _looks_like_html_document(body):from ._html_to_md import html_to_markdown- converted = html_to_markdown(body, main_content = True)+ converted = html_to_markdown(body, main_content = True, max_span_chars = max_chars // 2)readme_body = converted if converted.strip() else bodyif readme_body.strip():return _truncate_page_text(@@ -17158,7 +17158,10 @@ def _fetch_page_text(from ._html_to_md import SiteLinks, html_to_markdownsite_links = SiteLinks(url)- text = html_to_markdown(body, main_content = True, site_links = site_links)+ # generated span cells get half the window budget, as html_to_markdown's own cap does for 16K+ text = html_to_markdown(+ body, main_content = True, site_links = site_links, max_span_chars = max_chars // 2+ )# a page that fits keeps same-site links so the model can follow them.if text and len(text) <= max_chars and len(text) <= _dense_char_limit(text, max_chars):return textdiff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex 9b4680927e5..0038cab68d9 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -23,7 +23,7 @@ _BACKEND_DIR = str(Path(__file__).resolve().parent.parent)if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)-from core.inference._html_to_md import _is_aria_heading, html_to_markdown+from core.inference._html_to_md import SiteLinks, _is_aria_heading, html_to_markdownfrom core.inference.tools import (_fetch_page_text,_fetch_url_raw,@@ -2225,6 +2225,187 @@ def test_table_nested_in_a_header_inside_a_cell_keeps_its_columns():assert html_to_markdown(f"<body>{html}</body>", main_content = True) == html_to_markdown(f"<body>{html}</body>")+ continued = "<table><tr><td><header><table><tr><td>x</td></tr></table></header>after</td><td>sibling</td></tr></table>"+ assert html_to_markdown(continued, main_content = True) == html_to_markdown(continued)+++def test_nested_tables_flatten_into_their_outer_cell():+ nested = "<table><tr><td>A</td><td><table><tr><td>x | y</td><td>z</td></tr></table></td><td>C</td></tr></table>"+ assert html_to_markdown(nested).splitlines() == ["| A | x \\| y z | C |", "| --- | --- | --- |"]+ deep = "<table><tr><td>" * 500 + "x" + "</td></tr></table>" * 500+ assert html_to_markdown(deep) == "| x |\n| --- |"+ article = "<article>" + "<table><tr><td>" * 8 + "x" + "</td></tr></table>" * 8 + "</article>"+ main = "<main>" + "<p>Real page body.</p>" * 20 + "</main>"+ assert "Real page body." in html_to_markdown(article + main, main_content = True)+++def test_spanned_table_cells_stay_in_their_columns():+ html = (+ "<table>"+ "<tr><th>Rank</th><th>Nation</th><th>Gold</th><th>Silver</th><th>Bronze</th><th>Total</th></tr>"+ "<tr><td rowspan='2'>25</td><th>Latvia</th><td>0</td><td>1</td><td>0</td><td>1</td></tr>"+ "<tr><th>Estonia</th><td>0</td><td>1</td><td>0</td><td>1</td></tr>"+ "<tr><td>27</td><th>Spain</th><td>0</td><td>0</td><td>1</td><td rowspan='2'>1</td></tr>"+ "<tr><td>28</td><th>Chile</th><td>0</td><td>0</td><td>1</td></tr>"+ "<tr><th colspan='2'>Totals (5 entries)</th><td>0</td><td>2</td><td>2</td><td>4</td></tr>"+ "</table>"+ )+ lines = html_to_markdown(f"<body>{html}</body>").splitlines()+ assert lines == [+ "| Rank | Nation | Gold | Silver | Bronze | Total |",+ "| --- | --- | --- | --- | --- | --- |",+ "| 25 | Latvia | 0 | 1 | 0 | 1 |",+ "| 25 | Estonia | 0 | 1 | 0 | 1 |",+ "| 27 | Spain | 0 | 0 | 1 | 1 |",+ "| 28 | Chile | 0 | 0 | 1 | 1 |",+ "| Totals (5 entries) | | 0 | 2 | 2 | 4 |",+ ]+++def test_spanned_site_links_strip_on_every_repeated_row():+ html = (+ "<table><tr><th>Rank</th><th>Nation</th></tr>"+ "<tr><td rowspan='2'><a href='/r25'>25</a></td><td>Estonia</td></tr>"+ "<tr><td>Georgia</td></tr>"+ "<tr><td colspan='2'><a href='https://ex.com/t'>Totals | all</a></td></tr></table>"+ )+ site_links = SiteLinks("https://ex.com/page")+ full = html_to_markdown(f"<body>{html}</body>", site_links = site_links)+ assert full.splitlines()[2:] == [+ "| [25](/r25) | Estonia |",+ "| [25](/r25) | Georgia |",+ "| [Totals \\| all](https://ex.com/t) | |",+ ]+ assert site_links.strip(full).splitlines()[2:] == [+ "| 25 | Estonia |",+ "| 25 | Georgia |",+ "| Totals \\| all | |",+ ]+++def test_table_spans_do_not_multiply_the_page_size():+ small_wide = "<table><tr>" + "<td colspan='1000'>x</td>" * 3 + "</tr></table>"+ wide = "<table><tr>" + "<td colspan='1000'>x" * 30000 + "</table>"+ tall = "<table><tr><td rowspan='65534'>" + "word " * 2000 + "<tr><td>b" * 5000 + "</table>"+ for html in (small_wide, wide, tall):+ assert len(html_to_markdown(html)) < 3 * len(html)++ repeated = (+ "<table><tr><td rowspan='1001'>"+ + "x" * 100+ + "</td></tr>"+ + "<tr></tr>" * 1000+ + "</table>"+ )+ rendered = html_to_markdown(repeated + "<p>Article sentinel.</p>")+ assert rendered.index("Article sentinel.") < 16_000+ quoted = (+ "<blockquote><table><tr><td rowspan='3000'>x</td></tr>"+ + "<tr></tr>" * 2999+ + "</table></blockquote><p>Article sentinel.</p>"+ )+ assert html_to_markdown(quoted).index("Article sentinel.") < 16_000+ nested = (+ "<table><tr><td>"+ + "s" * 7000+ + "<table><tr><td rowspan='3000'>x</td></tr>"+ + "<tr></tr>" * 2999+ + "</table></td></tr></table><p>Article sentinel.</p>"+ )+ assert html_to_markdown(nested).index("Article sentinel.") < 16_000+++def test_table_spans_fit_a_small_fetch_budget(monkeypatch):+ tall = (+ "<html><body><table><tr><td rowspan='3000'>x</td></tr>"+ + "<tr></tr>" * 2999+ + "</table><p>Article sentinel.</p></body></html>"+ )+ assert html_to_markdown(tall, max_span_chars = 1000).index("Article sentinel.") < 2000++ def fake_fetch(url, **kwargs):+ return None, tall, "text/html"++ monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)+ assert "Article sentinel." in _fetch_page_text("https://example.com/t", max_chars = 2000)+++def test_table_span_budget_is_page_wide_and_token_weighted():+ from core.inference._html_to_md import _new_renderer++ candidate = "<article><table><tr><td colspan='1000'>x</td></tr></table></article>"+ page = candidate * 3000+ renderer = _new_renderer(page, frozenset({"article"}), True)+ assert sum(len(seg) for seg in renderer.scope_segments) < 10 * len(page)++ cell = "中文表格单元内容很长" * 20+ rows = "".join(f"<tr><td>r{i}</td></tr>" for i in range(1, 50))+ html = f"<table><tr><td rowspan='50'>{cell}</td><td>r0</td></tr>{rows}</table><p>Article sentinel.</p>"+ assert html_to_markdown(html, max_span_chars = 3404).count(cell) == 1+++def test_nested_colspan_padding_does_not_spend_the_span_budget():+ layout = "<table><tr><td><table><tr><td colspan='1000'>x</td></tr></table></td></tr></table>"+ data = (+ "<table><tr><th>Rank</th><th>Nation</th></tr>"+ "<tr><td rowspan='2'>25</td><td>Estonia</td></tr><tr><td>Georgia</td></tr></table>"+ )+ pad = "<p>" + "Body text. " * 400 + "</p>"+ assert html_to_markdown(pad + layout + data).splitlines()[-1] == "| 25 | Georgia |"+++def test_generated_table_spans_do_not_make_a_main_content_candidate():+ body = "<main><p>" + "Real page body. " * 20 + "</p></main>"+ decoys = (+ "<article><table><tr><td rowspan='51'>x</td></tr>"+ + "<tr></tr>" * 50+ + "</table></article>",+ "<article><blockquote><table><tr><td rowspan='101'>x</td></tr>"+ + "<tr></tr>" * 100+ + "</table></blockquote></article>",+ "<article><header><table><tr><td colspan='1000'><a href='/nav'>"+ + "Navigation " * 12+ + "</a></td></tr></table></header><p>"+ + "Teaser. " * 12+ + "</p></article>",+ )+ for decoy in decoys:+ out = html_to_markdown(f"<body>{decoy}{body}</body>", main_content = True)+ assert "Real page body." in out++ exhausted = (+ "<article><table><tr><td colspan='1000' rowspan='2'>x</td></tr><tr></tr></table></article>"+ )+ substantive = (+ "<article><p>"+ + "Real page body. " * 20+ + "</p><table><tr><td rowspan='2'>rank</td><td>one</td></tr><tr><td>two</td></tr></table></article>"+ )+ assert "| rank | two |" in html_to_markdown(exhausted + substantive, main_content = True)+++def test_table_spans_stop_at_row_groups_tables_and_long_cells():+ def rows(html):+ return html_to_markdown(f"<body>{html}</body>").splitlines()++ covered_row = "<table><tr><th>A</th><th>B</th></tr><tr><td rowspan=2>a</td><td rowspan=2>b</td></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>"+ assert rows(covered_row)[-3:] == ["| a | b |", "| a | b |", "| c | d |"]+ footer = "<table><thead><tr><th>Item</th><th>Qty</th></tr></thead><tbody><tr><td>x</td><td rowspan=2>5</td></tr></tbody><tfoot><tr><td>Total</td><td>5</td></tr></tfoot></table>"+ assert rows(footer)[-1] == "| Total | 5 |"+ nested = "<table><tr><td>T1</td><td><table><tr><td rowspan=2>GK</td><td>P1</td></tr></table></td></tr><tr><td>T2</td><td>S2</td></tr></table>"+ assert rows(nested)[-1] == "| T2 | S2 |"+ nested_under_span = "<table><tr><td rowspan=2>A</td><td><table><tr><td>B</td></tr></table></td></tr><tr><td>C</td></tr></table>"+ assert rows(nested_under_span)[-1] == "| A | C |"+ to_group_end = "<table><tr><th>G</th><th>N</th></tr><tbody><tr><th rowspan=0>g</th><td>1</td></tr><tr><td>2</td></tr></tbody><tbody><tr><td>h</td><td>3</td></tr></tbody></table>"+ assert rows(to_group_end)[-3:] == ["| g | 1 |", "| g | 2 |", "| h | 3 |"]+ sidebar = (+ "<table><tr><td rowspan=5>"+ + "Article body text. " * 120+ + "</td><td>Home</td></tr>"+ + "<tr><td>Link</td></tr>" * 4+ + "</table>"+ )+ assert html_to_markdown(sidebar).count("Article body text.") == 120def test_truncated_header_and_blockquote_keep_source_order():@@ -2232,7 +2413,7 @@ def test_truncated_header_and_blockquote_keep_source_order():assert out.index("Title") < out.index("Quote")-# Headers interact with every buffer, so enumerate the grid: that is where the one-off bugs live.+# headers interact with every buffer, where one-off bugs occur_GRID_HEADINGS = {"h1": "<h1>Page Title</h1>","aria": "<div role='heading'>Page Title</div>",