Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep exponents when the model reads a web page (#13183)
Nilay76dac379bc3e
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex 3c7b5c23a98..948bd49a5ff 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -17,8 +17,10 @@ error placeholders, session banners, cookie prompts) from the result.from __future__ import annotationsimport html+import itertoolsimport reimport secrets+import unicodedatafrom html.parser import HTMLParserfrom urllib.parse import urlsplit@@ -208,6 +210,125 @@ _MAX_REPEATED_CELL_CHARS = 200_MIN_SCOPE_SPAN_CHARS = 256_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}+# after a word, digits joined by commas / en dashes are citations (claim<sup>1,3–5</sup>); one separator per+# step keeps the match linear. Hyphen / minus stay exponents (10<sup>-3</sup>), and so do thousands (1,000)+_CITATION_LIST = re.compile(r"\d+(?:\s*[–,]\s*\d+)+")+_THOUSANDS = re.compile(r"\d{1,3}(?:,\d{3})+")+_PLAIN_SUFFIXES = frozenset(+ {"st", "nd", "rd", "th", "tm", "sm", "mc", "md", "(tm)", "(sm)", "(r)", "(c)", "mr", "m.r."}+)+# French / Romance ordinals after a digit (1er, 2e, 1º, 2ª); after a letter "e" can be Euler's number+# XVe siècle, François Ier: a Roman numeral takes ordinals like a digit+_ROMAN_NUMERAL_TAIL = re.compile(r"(?<![^\W\d_])[IVXLCDM]+$")+# French superior abbreviations: Mme, Mlle, Mgr, Dr, Pr, no, St, Cie; keyed on the whole base word+_SUPERIOR_ABBREVIATIONS = {+ "M": frozenset({"me", "mes", "lle", "lles", "gr", "e", "r", "rs", "s"}),+ "D": frozenset({"r", "rs", "re", "res"}),+ "P": frozenset({"r", "rs", "re", "res"}),+ "n": frozenset({"o", "os"}),+ "N": frozenset({"o", "os"}),+ "S": frozenset({"t", "te", "ts", "tes", "r"}),+ "J": frozenset({"r"}),+ "C": frozenset({"ie", "ies"}),+}+_LAST_WORD = re.compile(r"(?<![^\W\d_])[^\W\d_]+$")+++def _last_word(text: str) -> str:+ match = _LAST_WORD.search(text)+ return match.group() if match else ""+++# French second / seconde (2d, 2de, 2ds, 2des) only after a 2: 10<sup>d</sup> stays a power+_SECOND_SUFFIXES = frozenset({"d", "de", "ds", "des"})+_DIGIT_ORDINAL_SUFFIXES = frozenset(+ {+ "e",+ "es",+ "er",+ "ers",+ "re",+ "res",+ "ère",+ "ères",+ "ème",+ "èmes",+ "eme",+ "emes",+ "nd",+ "nde",+ "bis",+ "ter",+ "quater",+ "quinquies",+ "am",+ "pm",+ "a.m.",+ "p.m.",+ "ndes",+ "nds",+ "º",+ "ª",+ "o",+ "a",+ }+)+_MD_DELIMITERS = "*_`"+_STRIP_MD_DELIMITERS = str.maketrans("", "", _MD_DELIMITERS)+# SiteLinks wraps same-site links in invisible \x00 markers; the base is the text before them+_SITE_LINK_MARKER_TAIL = re.compile(r"\x00[0-9a-f]+:\d+:[se]\x00$")+# parts a base lookup reads back: enough for delimiters and link markers, bounded on hostile pages+_SUP_BASE_SCAN_PARTS = 8+_SUP_BASE_SCAN_CHARS = 128+# a caret binds one token: a signed number or one letter goes bare, anything longer in parentheses+_BARE_EXPONENT = re.compile(r"[-+−]?(?:\d+(?:[.,]\d+)?|[^\W\d_])")+# split cents: $19<sup>99</sup> is a price, not an exponent+_PRICE_TAIL = re.compile(r"(\S)\s?\d(?:[\d,.'’]|[ \u00a0\u202f]\d)*$")+# ISO 4217 codes: CHF 19<sup>95</sup> is a price like $19<sup>99</sup>+_CURRENCY_CODES = frozenset(+ (+ "AED AFN ALL AMD ANG AOA ARS AUD AWG AZN BAM BBD BDT BGN BHD BIF BMD BND BOB BRL BSD BTN "+ "BWP BYN BZD CAD CDF CHF CLP CNY COP CRC CUP CVE CZK DJF DKK DOP DZD EGP ERN ETB EUR FJD "+ "FKP GBP GEL GHS GIP GMD GNF GTQ GYD HKD HNL HTG HUF IDR ILS INR IQD IRR ISK JMD JOD JPY "+ "KES KGS KHR KMF KPW KRW KWD KYD KZT LAK LBP LKR LRD LSL LYD MAD MDL MGA MKD MMK MNT MOP "+ "MRU MUR MVR MWK MXN MYR MZN NAD NGN NIO NOK NPR NZD OMR PAB PEN PGK PHP PKR PLN PYG QAR "+ "RON RSD RUB RWF SAR SBD SCR SDG SEK SGD SHP SLE SOS SRD SSP SVC STN SYP SZL THB TJS TMT TND "+ "TOP TRY TTD TWD TZS UAH UGX USD UYU UZS VED VES VND VUV WST XAF XCD XCG XOF XPF YER ZAR ZMW "+ "ZWG ZWL"+ ).split()+)+_THREE_DECIMAL_CURRENCIES = frozenset({"BHD", "IQD", "JOD", "KWD", "LYD", "OMR", "TND"})+# no minor unit: JPY 10<sup>12</sup> is a power, never cents+_ZERO_DECIMAL_CURRENCIES = frozenset(+ "BIF CLP DJF GNF ISK JPY KMF KRW PYG RWF UGX VND VUV XAF XOF XPF".split()+)+_CODE_PRICE_TAIL = re.compile(r"\b([A-Z]{3})[ \u00a0\u202f]?\d(?:[\d,.'’]|[ \u00a0\u202f]\d)*$")+# note markers that keep their plain-text form, like Wikipedia's class="reference"+_FOOTNOTE_CLASSES = frozenset({"reference", "footnote", "footnote-ref", "noteref", "fn", "cite"})+# class token parts (split on - and _) starting with these mark a note too: footnote-reference, citation+_FOOTNOTE_CLASS_PREFIXES = (+ "footnote",+ "noteref",+ "cite",+ "citation",+ "endnote",+ "fnref",+ "reference",+)+_CLASS_PART_SPLIT = re.compile(r"[-_]")+# deeper <sup> nests render as plain text: each tracked level rescans its whole suffix on close+_MAX_SUP_DEPTH = 8+++def _visible_tail(text: str) -> str:+ """*text* without the emphasis/code delimiters and link markers the renderer appended."""+ while True:+ trimmed = _SITE_LINK_MARKER_TAIL.sub("", text.rstrip(_MD_DELIMITERS))+ if trimmed == text:+ return text+ text = trimmed++# measured density: 0.94-1.00 for link lists, 0.13-0.90 for content headers_HEADER_LINK_DENSITY = 0.93# below this the ratio is noise: link lists start at 182 chars, link-dense headers stop at 93@@ -599,6 +720,12 @@ class _MarkdownRenderer(HTMLParser):# Blockquote state: stack of buffers so nested blockquotes get the right ">" depth.self._bq_stack: list[list[str]] = []+ # per open <sup>: its _open_tags index, then (output list, start), the (copy, start) pairs that tee the+ # same text, the base text and whether a currency amount precedes it+ self._sup_starts: list[+ tuple[int, tuple[list[str], int, list[tuple[list[str], int]], str, int] | None]+ ] = []+def _nested_buffer_open(self, frame: _HeaderFrame) -> bool:"""True when a side buffer opened *inside* *frame* still holds content.@@ -641,22 +768,106 @@ class _MarkdownRenderer(HTMLParser):# Tally once, on the emit reaching the frame; counting again on flush doubled it.if frame is not None and not nested_open:frame.rendered_chars += len(measured.strip())- frame.parts.append(text)- return+ elif self._in_link and self._heading_marks:+ self._link_heading_parts.append(text)+ self._emit_target().append(text)++ def _emit_target(self) -> list[str]:+ frame = self._header_stack[-1] if self._header_stack else None+ if frame is not None and not self._nested_buffer_open(frame):+ return frame.partsif self._in_link:- self._link_text_parts.append(text)- if self._heading_marks:- self._link_heading_parts.append(text)- elif self._in_cell:- self._cell_parts.append(text)- elif self._in_pre:- self._pre_parts.append(text)- elif self._table_stack and len(self._bq_stack) <= self._table_stack[-1].outer_bq_depth:- self._table_stack[-1].parts.append(text)- elif self._bq_stack:- self._bq_stack[-1].append(text)- else:- self._out.append(text)+ return self._link_text_parts+ if self._in_cell:+ return self._cell_parts+ if self._in_pre:+ return self._pre_parts+ if self._table_stack and len(self._bq_stack) <= self._table_stack[-1].outer_bq_depth:+ return self._table_stack[-1].parts+ if self._bq_stack:+ return self._bq_stack[-1]+ return self._out++ def _sup_base(self, target: list[str]) -> str:+ """The visible text a <sup> raises (base = its last character), or "" when none: after whitespace or+ sentence punctuation it is a footnote marker (``fact.<sup>1</sup>``,+ ``<a href="#fn1"><sup>1</sup></a>``) or a fraction numerator (``<sup>1</sup>⁄``)."""+ for part in itertools.islice(reversed(target), _SUP_BASE_SCAN_PARTS):+ part = part[-_SUP_BASE_SCAN_CHARS:]+ part = _visible_tail(part)+ if part:+ base = part[-1]+ return part if base.isalnum() or base in ")]}|)]}" else ""+ return ""++ @staticmethod+ def _after_price(target: list[str]) -> int:+ """Digits of the minor unit when *target* ends in a currency amount ($19 -> 2, KWD 19 -> 3), else 0."""+ context = _visible_tail("".join(p[-40:] for p in target[-8:])[-40:])+ context = context.translate(_STRIP_MD_DELIMITERS)+ price = _PRICE_TAIL.search(context)+ # any Unicode currency sign (Sc): $, €, ₺, ₱, ...; or an ISO code: CHF 19+ if price and unicodedata.category(price.group(1)) == "Sc":+ # ₫ ₲ ₩ have no minor unit (¥ is shared with two-decimal CNY, so it stays 2)+ return 0 if price.group(1) in "₫₲₩" else 2+ code = _CODE_PRICE_TAIL.search(context)+ if (+ not code+ or code.group(1) not in _CURRENCY_CODES+ or code.group(1) in _ZERO_DECIMAL_CURRENCIES+ ):+ return 0+ return 3 if code.group(1) in _THREE_DECIMAL_CURRENCIES else 2++ def _sup_copies(self) -> list[tuple[list[str], int]]:+ copies = [self._link_heading_parts, self._seg_heading_texts]+ if self._header_stack:+ copies.append(self._header_stack[-1].heading_parts)+ return [(copy, len(copy)) for copy in copies]++ def _finish_sup(self, opened) -> None:+ if opened is None or opened[0] is not self._emit_target():+ return+ target, start, copies, base, after_price = opened+ joined = "".join(target[start:])+ raw = joined.strip()+ shown = self._site_links.clean(raw) if self._site_links is not None else raw+ visible = shown.strip(_MD_DELIMITERS)+ if (+ not visible+ or "\n" in visible+ or visible[0] == "["+ # $19<sup>.99</sup> / €19<sup>,99</sup> are split cents; 10<sup>.5</sup> with no currency is a power+ or (visible[0] in ".," and after_price)+ or not any(c.isalnum() for c in visible)+ or visible.lower() in _PLAIN_SUFFIXES+ or (+ len(_last_word(base)) > 1+ and _CITATION_LIST.fullmatch(visible)+ and ("–" in visible or not _THOUSANDS.fullmatch(visible))+ )+ or (base[-1] == "2" and visible.lower() in _SECOND_SUFFIXES)+ or (+ (base[-1].isdigit() or _ROMAN_NUMERAL_TAIL.search(base))+ and visible.lower() in _DIGIT_ORDINAL_SUFFIXES+ )+ or visible in _SUPERIOR_ABBREVIATIONS.get(_last_word(base), ())+ # split cents are two digits; $2<sup>n</sup> or USD 10<sup>6</sup> stays an exponent+ or (len(visible) == after_price and visible.isdigit())+ ):+ return+ # only the emphasis wrapping a whole exponent is renderer syntax; an inner * is an operator+ token = shown.strip(_MD_DELIMITERS)+ bare = _BARE_EXPONENT.fullmatch(token) or (+ token.startswith("(") and token.endswith(")") and token.count("(") == 1+ )+ exponent = f"^{raw}" if bare else f"^({raw})"+ exponent += joined[len(joined.rstrip()) :]+ target[start:] = [exponent]+ # headings are teed into these copies; a stale one renders "E=mc2" or skews the prose gate+ for copy, copy_start in copies:+ if "".join(copy[copy_start:]) == joined:+ copy[copy_start:] = [exponent]def _seg_heading_prose(self) -> int:"""Heading characters in this segment that the gate would otherwise read as@@ -879,6 +1090,9 @@ class _MarkdownRenderer(HTMLParser):if name in _IMPLICIT_CLOSERS:self._closable_open -= 1del self._open_tags[index:]+ # a <sup> an ancestor closed (<p>x<sup>2</p>) is gone; a kept frame would fill the depth cap+ while self._sup_starts and self._sup_starts[-1][0] >= index:+ self._sup_starts.pop()def _close_implicit(self, tag: str) -> None:"""HTML5 optional-end-tag recovery for a start tag about to open.@@ -1116,6 +1330,36 @@ class _MarkdownRenderer(HTMLParser):elif tag == "br":self._emit("\n")+ elif tag == "sup":+ target = self._emit_target()+ reference = (+ any(+ token in _FOOTNOTE_CLASSES+ or any(+ part == "fn" or part.startswith(_FOOTNOTE_CLASS_PREFIXES)+ for part in _CLASS_PART_SPLIT.split(token)+ )+ for token in (attr_dict.get("class") or "").lower().split()+ )+ or "doc-noteref" in (attr_dict.get("role") or "").lower().split()+ )+ # past the cap nothing is tracked, so the stack stays bounded on hostile pages+ if len(self._sup_starts) < _MAX_SUP_DEPTH:+ self._sup_starts.append(+ (+ len(self._open_tags) - 1,+ None+ if reference or not (base := self._sup_base(target))+ else (+ target,+ len(target),+ self._sup_copies(),+ base,+ self._after_price(target) if base[-1].isdigit() else 0,+ ),+ )+ )+elif tag in _BLOCK_TAGS:if not self._li_marker_pending:self._emit("\n\n")@@ -1201,6 +1445,17 @@ class _MarkdownRenderer(HTMLParser):if self._skip_depth:return+ closing_sup = None+ # O(1): only a <sup> that is still the innermost open tag converts; malformed nesting drops on unwind+ if (+ tag == "sup"+ and self._sup_starts+ and self._open_tags+ and self._open_tags[-1] == "sup"+ and self._sup_starts[-1][0] == len(self._open_tags) - 1+ ):+ closing_sup = self._sup_starts.pop()[1]+if not self._exit_tag(tag):return@@ -1218,6 +1473,9 @@ class _MarkdownRenderer(HTMLParser):elif tag in _INLINE_EMPHASIS:self._emit(_INLINE_EMPHASIS[tag])+ elif tag == "sup":+ self._finish_sup(closing_sup)+elif tag in _BLOCK_TAGS:self._emit("\n\n")diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex dcea7059972..dfc0da83bf6 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -869,6 +869,232 @@ def test_page_that_fits_keeps_its_links(monkeypatch):assert "[[1]](#cite_note-1)" in out+ "markup, expected",+ [+ ("(2<sup>53</sup> – 1)", "(2^53 – 1)"),+ ("2<sup>−52</sup>", "2^−52"),+ ("1.898×10<sup>27</sup> kg", "1.898×10^27 kg"),+ ("6.02214076×10<sup>23</sup> mol<sup>−1</sup>", "6.02214076×10^23 mol^−1"),+ ("2<sup><i>n</i>+1</sup> nodes", "2^(*n*+1) nodes"),+ ("2<sup>n + 1</sup>", "2^(n + 1)"),+ ("the 1<sup>st</sup> and 2<sup>nd</sup>", "the 1st and 2nd"),+ ("Intel<sup>®</sup> Core<sup>™</sup> i7", "Intel® Core™ i7"),+ ("now $19<sup>.99</sup> only", "now $19.99 only"),+ ("price<sup>*</sup> and terms<sup>†</sup>", "price* and terms†"),+ ("10<sup>6 </sup>years", "10^6 years"),+ ("Add <sup>1</sup>⁄<sub>2</sub> cup", "Add 1⁄2 cup"),+ ("Add 1 <sup>1</sup>/<sub>2</sub> cups", "Add 1 1/2 cups"),+ ("<sup>1</sup> Footnote text", "1 Footnote text"),+ ("Add <em><sup>1</sup></em>⁄ cup", "Add *1*⁄ cup"),+ ("the 1<sup><em>st</em></sup> one", "the 1*st* one"),+ ("a<b><sup>2</sup></b>", "a**^2**"),+ ("x<sup>n×2</sup>", "x^(n×2)"),+ ("x<sup>2n</sup> and y<sup>n2</sup>", "x^(2n) and y^(n2)"),+ ("x<sup>2<em>n</em></sup>", "x^(2*n*)"),+ ("x<sup>2*3</sup> and x<sup><em>n</em></sup>", "x^(2*3) and x^*n*"),+ ("e<sup>iπ</sup> + 1 = 0", "e^(iπ) + 1 = 0"),+ ("now $19<sup>99</sup> or € 1,299<sup>95</sup>", "now $1999 or € 1,29995"),+ ("<b>$19</b><sup>99</sup>", "**$19**99"),+ ("$<b>19</b><sup>99</sup>", "$**19**99"),+ ("Brand<sup>TM</sup> and Service<sup>SM</sup>", "BrandTM and ServiceSM"),+ ("el 1<sup>º</sup> y la 2<sup>ª</sup>, x<sup>a</sup>", "el 1º y la 2ª, x^a"),+ (+ "le XV<sup>e</sup> siècle, François I<sup>er</sup>, MAX<sup>e</sup>",+ "le XVe siècle, François Ier, MAX^e",+ ),+ ("<span>$</span> <b>19</b><sup>99</sup>", "$ **19**99"),+ (+ "₺19<sup>99</sup> or ₱19<sup>99</sup>, a19<sup>2</sup>",+ "₺1999 or ₱1999, a19^2",+ ),+ (+ "CHF 19<sup>95</sup> or USD 19<sup>99</sup>, ABC 10<sup>3</sup>",+ "CHF 1995 or USD 1999, ABC 10^3",+ ),+ (+ "AED 19<sup>99</sup>, TWD 19<sup>99</sup>, ILS 19<sup>99</sup>",+ "AED 1999, TWD 1999, ILS 1999",+ ),+ (+ "ZWG 19<sup>99</sup>, les 1<sup>ers</sup> et 1<sup>res</sup>",+ "ZWG 1999, les 1ers et 1res",+ ),+ ("$2<sup>n</sup> and USD 10<sup>6</sup>, $19<sup>99</sup>", "$2^n and USD 10^6, $1999"),+ (+ 'XCG 19<sup>99</sup>, claim<sup role="doc-noteref presentation">2</sup>',+ "XCG 1999, claim2",+ ),+ ("le P<sup>r</sup> Martin et les P<sup>rs</sup>", "le Pr Martin et les Prs"),+ ("CHF 1’299<sup>95</sup> or CHF 1'299<sup>95</sup>", "CHF 1’29995 or CHF 1'29995"),+ ("Marque<sup>MC</sup> et Produit<sup>MD</sup>", "MarqueMC et ProduitMD"),+ ("Brand<sup>(TM)</sup> and Other<sup>(R)</sup>", "Brand(TM) and Other(R)"),+ (+ "KWD 19<sup>950</sup>, BHD 1<sup>234</sup>, USD 10<sup>100</sup>",+ "KWD 19950, BHD 1234, USD 10^100",+ ),+ ("M<sup>r</sup>, M<sup>s</sup> and M<sup>rs</sup> Smith", "Mr, Ms and Mrs Smith"),+ ("JPY 10<sup>12</sup> and KRW 10<sup>12</sup>", "JPY 10^12 and KRW 10^12"),+ ("les 1<sup>ères</sup> places", "les 1ères places"),+ ("John J<sup>r</sup> and John S<sup>r</sup>", "John Jr and John Sr"),+ ("VED 19<sup>99</sup> and S<sup>T</sup>", "VED 1999 and S^T"),+ (+ 'claim<sup class="footnote-reference">2</sup>, la 2<sup>de</sup>, le 2<sup>d</sup>',+ "claim2, la 2de, le 2d",+ ),+ (+ "la P<sup>re</sup> Durand, 12<sup>bis</sup> rue, article 3<sup>ter</sup>",+ "la Pre Durand, 12bis rue, article 3ter",+ ),+ (+ 'claim<sup class="citation">2</sup> and fact<sup class="endnote-ref">3</sup>',+ "claim2 and fact3",+ ),+ ("la D<sup>re</sup> Roy et les D<sup>res</sup>", "la Dre Roy et les Dres"),+ (+ 'x<sup class="excited">2</sup> and 10<sup>.5</sup>, $19<sup>.99</sup>',+ "x^2 and 10^(.5), $19.99",+ ),+ (+ 'claim<sup class="fnref">2</sup>, fact<sup class="fn-ref">3</sup>, Marca<sup>MR</sup>',+ "claim2, fact3, MarcaMR",+ ),+ (+ "les S<sup>ts</sup>, les S<sup>tes</sup> et les C<sup>ies</sup>",+ "les Sts, les Stes et les Cies",+ ),+ ("les 2<sup>nds</sup>", "les 2nds"),+ ("(a+b)<sup>2</sup> and 「引用」<sup>1</sup>", "(a+b)^2 and 「引用」1"),+ (+ '€19<sup>,99</sup> and claim<sup class="reference-number">2</sup>',+ "€19,99 and claim2",+ ),+ (+ "SVC 19<sup>99</sup>, claim<sup>1–3</sup>, fact<sup>2,5</sup>, 10<sup>-3</sup>",+ "SVC 1999, claim1–3, fact2,5, 10^-3",+ ),+ ("x<sup>2,5</sup> and 10<sup>1,000</sup>", "x^2,5 and 10^1,000"),+ ("₫10<sup>12</sup> and ₩10<sup>12</sup>", "₫10^12 and ₩10^12"),+ ("claim<sup>1,3–5</sup>", "claim1,3–5"),+ ("Doors 9:30<sup>pm</sup>, ends 11<sup>p.m.</sup>", "Doors 9:30pm, ends 11p.m."),+ ("10<sup>6–8</sup> CFU/mL and 10<sup>d</sup>", "10^(6–8) CFU/mL and 10^d"),+ ("(x+1)<sup>2–4</sup>", "(x+1)^(2–4)"),+ (+ "M<sup>me</sup> Dupont, D<sup>r</sup> Martin, n<sup>o</sup> 5, Om<sup>e</sup>",+ "Mme Dupont, Dr Martin, no 5, Om^e",+ ),+ ("€1.299<sup>95</sup> or €1 299<sup>95</sup>", "€1.29995 or €1 29995"),+ (+ 'claim<sup role="doc-noteref">1</sup> and fact<sup class="footnote">2</sup>',+ "claim1 and fact2",+ ),+ ("x<sup>-n</sup> and 10<sup>2.5</sup>", "x^-n and 10^2.5"),+ ("now $19<sup><em>.99</em></sup>", "now $19*.99*"),+ ("A fact.<sup>1</sup> Next, a list,<sup>2</sup>", "A fact.1 Next, a list,2"),+ ("(a+b)<sup>2</sup> and km<sup>2</sup>.", "(a+b)^2 and km^2."),+ ("le 1<sup>er</sup> mai, 2<sup>e</sup> et x<sup>e</sup>", "le 1er mai, 2e et x^e"),+ ],+)+def test_superscripts_keep_their_exponent(markup, expected):+ assert html_to_markdown(f"<p>{markup}</p>") == expected+++ "title",+ [+ "<h1><a href='/p'>E=mc<sup>2</sup></a></h1>",+ "<a href='/p'><h1>E=mc<sup>2</sup></h1></a>",+ ],+)+def test_linked_header_title_keeps_one_exponent(title):+ nav = "".join(f"<a href='/s{i}'>Section number {i}</a> " for i in range(12))+ html = f"<header>{title}{nav}</header><p>{'Body text here. ' * 40}</p>"+ out = html_to_markdown(html, main_content = True)+ assert out.count("E=mc") == 1+ assert "E=mc^2](/p)" in out+++def test_stripped_header_keeps_the_exponent_in_its_heading():+ nav = "".join(f"<a href='/s{i}'>Section number {i}</a> " for i in range(12))+ html = f"<header><h1>E=mc<sup>2</sup></h1>{nav}</header><p>{'Body text here. ' * 40}</p>"+ out = html_to_markdown(html, main_content = True)+ assert "# E=mc^2" in out+ assert "E=mc2" not in out+++def test_deeply_nested_superscripts_track_a_bounded_depth():+ out = html_to_markdown("<p>x" + "<sup>a" * 50 + "</sup>" * 50 + "</p>")+ assert out.translate(str.maketrans("", "", "^()")) == "x" + "a" * 50+ assert out.count("^") == 8+++def test_linked_footnote_wrapping_its_superscript_renders_unchanged():+ html = '<p>text<a role="doc-noteref" href="#fn1"><sup>1</sup></a> more</p>'+ assert html_to_markdown(html) == "text[1](#fn1) more"+++def test_exponent_headings_do_not_read_as_body_prose():+ heads = "".join(f"<h2>x<sup>n+{i}</sup></h2>" for i in range(67))+ body = "<p>" + "Real document body sentence. " * 12 + "</p>"+ html = f"<html><body><article>{heads}</article><div>{body}</div></body></html>"+ assert "Real document body" in html_to_markdown(html, main_content = True)+++def test_same_site_linked_base_keeps_its_exponent():+ site_links = SiteLinks("https://e.com/page")+ html = '<p>5 <a href="/metre">m</a><sup>2</sup> and <a href="https://x.org/m">m</a><sup>3</sup></p>'+ out = site_links.clean(html_to_markdown(html, site_links = site_links))+ assert out == "5 [m](/metre)^2 and [m](https://x.org/m)^3"+++def test_empty_superscripts_do_not_rescan_the_page():+ start = time.perf_counter()+ html_to_markdown("<p>" + "<i></i><sup></sup>" * 8000 + "</p>")+ assert time.perf_counter() - start < 2+ start = time.perf_counter()+ html_to_markdown("<p>" + "*" * 250000 + "<sup></sup>" * 23000 + "</p>")+ assert time.perf_counter() - start < 3+ start = time.perf_counter()+ html_to_markdown("<p>claim<sup>" + "1–" * 16000 + "x</sup></p>")+ assert time.perf_counter() - start < 2+ start = time.perf_counter()+ html_to_markdown(+ "<p>" + "<b>" + "word 1 " * 40000 + "</b>" * 1 + "9" + "<sup></sup>" * 20000 + "</p>"+ )+ assert time.perf_counter() - start < 3+++def test_unclosed_superscripts_do_not_exhaust_the_depth_cap():+ html = "<p>x<sup>2</p>" * 10 + "<p>1.898×10<sup>27</sup> kg</p>"+ assert html_to_markdown(html).endswith("1.898×10^27 kg")+++def test_untracked_superscripts_keep_the_stack_bounded():+ from core.inference._html_to_md import _MarkdownRenderer++ renderer = _MarkdownRenderer()+ renderer.feed("<p>" + "<sup>" * 5000 + "x")+ assert len(renderer._sup_starts) <= 8+++def test_footnote_superscripts_render_unchanged():+ html = (+ '<p>mass<sup class="reference"><a href="#cite_note-12">[12]</a></sup> and '+ "volume<sup>[13]</sup></p>"+ )+ assert html_to_markdown(html) == "mass[[12]](#cite_note-12) and volume[13]"+++def test_fetched_page_keeps_exponents_beside_footnotes(monkeypatch):+ body = (+ "<html><body><main><article><p>Jupiter has a mass of 1.898×10<sup>27</sup> kg"+ '<sup class="reference"><a href="#cite_note-12">[12]</a></sup> and a surface area of '+ "6.1419×10<sup>10</sup> km<sup>2</sup>.</p></article></main></body></html>"+ )+ out = _page_text(monkeypatch, "https://en.wikipedia.org/wiki/Jupiter", body, "text/html")+ assert "1.898×10^27 kg[[12]](#cite_note-12) and a surface area of 6.1419×10^10 km^2." in out++def test_page_cut_by_the_room_left_drops_its_site_link_urls(monkeypatch):from core.inference import tools