Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep exponents when the model reads a web page (#13183)

Nilay76dac379bc3e
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index 3c7b5c23a98..948bd49a5ff 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -17,8 +17,10 @@ error placeholders, session banners, cookie prompts) from the result.
from __future__ import annotations
import html
+import itertools
import re
import secrets
+import unicodedata
from html.parser import HTMLParser
from urllib.parse import urlsplit
@@ -208,6 +210,125 @@ _MAX_REPEATED_CELL_CHARS = 200
_MIN_SCOPE_SPAN_CHARS = 256
_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}
+# after a word, digits joined by commas / en dashes are citations (claim<sup>1,3–5</sup>); one separator per
+# step keeps the match linear. Hyphen / minus stay exponents (10<sup>-3</sup>), and so do thousands (1,000)
+_CITATION_LIST = re.compile(r"\d+(?:\s*[–,]\s*\d+)+")
+_THOUSANDS = re.compile(r"\d{1,3}(?:,\d{3})+")
+_PLAIN_SUFFIXES = frozenset(
+ {"st", "nd", "rd", "th", "tm", "sm", "mc", "md", "(tm)", "(sm)", "(r)", "(c)", "mr", "m.r."}
+)
+# French / Romance ordinals after a digit (1er, 2e, 1º, 2ª); after a letter "e" can be Euler's number
+# XVe siècle, François Ier: a Roman numeral takes ordinals like a digit
+_ROMAN_NUMERAL_TAIL = re.compile(r"(?<![^\W\d_])[IVXLCDM]+$")
+# French superior abbreviations: Mme, Mlle, Mgr, Dr, Pr, no, St, Cie; keyed on the whole base word
+_SUPERIOR_ABBREVIATIONS = {
+ "M": frozenset({"me", "mes", "lle", "lles", "gr", "e", "r", "rs", "s"}),
+ "D": frozenset({"r", "rs", "re", "res"}),
+ "P": frozenset({"r", "rs", "re", "res"}),
+ "n": frozenset({"o", "os"}),
+ "N": frozenset({"o", "os"}),
+ "S": frozenset({"t", "te", "ts", "tes", "r"}),
+ "J": frozenset({"r"}),
+ "C": frozenset({"ie", "ies"}),
+}
+_LAST_WORD = re.compile(r"(?<![^\W\d_])[^\W\d_]+$")
+
+
+def _last_word(text: str) -> str:
+ match = _LAST_WORD.search(text)
+ return match.group() if match else ""
+
+
+# French second / seconde (2d, 2de, 2ds, 2des) only after a 2: 10<sup>d</sup> stays a power
+_SECOND_SUFFIXES = frozenset({"d", "de", "ds", "des"})
+_DIGIT_ORDINAL_SUFFIXES = frozenset(
+ {
+ "e",
+ "es",
+ "er",
+ "ers",
+ "re",
+ "res",
+ "ère",
+ "ères",
+ "ème",
+ "èmes",
+ "eme",
+ "emes",
+ "nd",
+ "nde",
+ "bis",
+ "ter",
+ "quater",
+ "quinquies",
+ "am",
+ "pm",
+ "a.m.",
+ "p.m.",
+ "ndes",
+ "nds",
+ "º",
+ "ª",
+ "o",
+ "a",
+ }
+)
+_MD_DELIMITERS = "*_`"
+_STRIP_MD_DELIMITERS = str.maketrans("", "", _MD_DELIMITERS)
+# SiteLinks wraps same-site links in invisible \x00 markers; the base is the text before them
+_SITE_LINK_MARKER_TAIL = re.compile(r"\x00[0-9a-f]+:\d+:[se]\x00$")
+# parts a base lookup reads back: enough for delimiters and link markers, bounded on hostile pages
+_SUP_BASE_SCAN_PARTS = 8
+_SUP_BASE_SCAN_CHARS = 128
+# a caret binds one token: a signed number or one letter goes bare, anything longer in parentheses
+_BARE_EXPONENT = re.compile(r"[-+−]?(?:\d+(?:[.,]\d+)?|[^\W\d_])")
+# split cents: $19<sup>99</sup> is a price, not an exponent
+_PRICE_TAIL = re.compile(r"(\S)\s?\d(?:[\d,.'’]|[ \u00a0\u202f]\d)*$")
+# ISO 4217 codes: CHF 19<sup>95</sup> is a price like $19<sup>99</sup>
+_CURRENCY_CODES = frozenset(
+ (
+ "AED AFN ALL AMD ANG AOA ARS AUD AWG AZN BAM BBD BDT BGN BHD BIF BMD BND BOB BRL BSD BTN "
+ "BWP BYN BZD CAD CDF CHF CLP CNY COP CRC CUP CVE CZK DJF DKK DOP DZD EGP ERN ETB EUR FJD "
+ "FKP GBP GEL GHS GIP GMD GNF GTQ GYD HKD HNL HTG HUF IDR ILS INR IQD IRR ISK JMD JOD JPY "
+ "KES KGS KHR KMF KPW KRW KWD KYD KZT LAK LBP LKR LRD LSL LYD MAD MDL MGA MKD MMK MNT MOP "
+ "MRU MUR MVR MWK MXN MYR MZN NAD NGN NIO NOK NPR NZD OMR PAB PEN PGK PHP PKR PLN PYG QAR "
+ "RON RSD RUB RWF SAR SBD SCR SDG SEK SGD SHP SLE SOS SRD SSP SVC STN SYP SZL THB TJS TMT TND "
+ "TOP TRY TTD TWD TZS UAH UGX USD UYU UZS VED VES VND VUV WST XAF XCD XCG XOF XPF YER ZAR ZMW "
+ "ZWG ZWL"
+ ).split()
+)
+_THREE_DECIMAL_CURRENCIES = frozenset({"BHD", "IQD", "JOD", "KWD", "LYD", "OMR", "TND"})
+# no minor unit: JPY 10<sup>12</sup> is a power, never cents
+_ZERO_DECIMAL_CURRENCIES = frozenset(
+ "BIF CLP DJF GNF ISK JPY KMF KRW PYG RWF UGX VND VUV XAF XOF XPF".split()
+)
+_CODE_PRICE_TAIL = re.compile(r"\b([A-Z]{3})[ \u00a0\u202f]?\d(?:[\d,.'’]|[ \u00a0\u202f]\d)*$")
+# note markers that keep their plain-text form, like Wikipedia's class="reference"
+_FOOTNOTE_CLASSES = frozenset({"reference", "footnote", "footnote-ref", "noteref", "fn", "cite"})
+# class token parts (split on - and _) starting with these mark a note too: footnote-reference, citation
+_FOOTNOTE_CLASS_PREFIXES = (
+ "footnote",
+ "noteref",
+ "cite",
+ "citation",
+ "endnote",
+ "fnref",
+ "reference",
+)
+_CLASS_PART_SPLIT = re.compile(r"[-_]")
+# deeper <sup> nests render as plain text: each tracked level rescans its whole suffix on close
+_MAX_SUP_DEPTH = 8
+
+
+def _visible_tail(text: str) -> str:
+ """*text* without the emphasis/code delimiters and link markers the renderer appended."""
+ while True:
+ trimmed = _SITE_LINK_MARKER_TAIL.sub("", text.rstrip(_MD_DELIMITERS))
+ if trimmed == text:
+ return text
+ text = trimmed
+
+
# measured density: 0.94-1.00 for link lists, 0.13-0.90 for content headers
_HEADER_LINK_DENSITY = 0.93
# below this the ratio is noise: link lists start at 182 chars, link-dense headers stop at 93
@@ -599,6 +720,12 @@ class _MarkdownRenderer(HTMLParser):
# Blockquote state: stack of buffers so nested blockquotes get the right ">" depth.
self._bq_stack: list[list[str]] = []
+ # per open <sup>: its _open_tags index, then (output list, start), the (copy, start) pairs that tee the
+ # same text, the base text and whether a currency amount precedes it
+ self._sup_starts: list[
+ tuple[int, tuple[list[str], int, list[tuple[list[str], int]], str, int] | None]
+ ] = []
+
def _nested_buffer_open(self, frame: _HeaderFrame) -> bool:
"""True when a side buffer opened *inside* *frame* still holds content.
@@ -641,22 +768,106 @@ class _MarkdownRenderer(HTMLParser):
# Tally once, on the emit reaching the frame; counting again on flush doubled it.
if frame is not None and not nested_open:
frame.rendered_chars += len(measured.strip())
- frame.parts.append(text)
- return
+ elif self._in_link and self._heading_marks:
+ self._link_heading_parts.append(text)
+ self._emit_target().append(text)
+
+ def _emit_target(self) -> list[str]:
+ frame = self._header_stack[-1] if self._header_stack else None
+ if frame is not None and not self._nested_buffer_open(frame):
+ return frame.parts
if self._in_link:
- self._link_text_parts.append(text)
- if self._heading_marks:
- self._link_heading_parts.append(text)
- elif self._in_cell:
- self._cell_parts.append(text)
- elif self._in_pre:
- self._pre_parts.append(text)
- elif self._table_stack and len(self._bq_stack) <= self._table_stack[-1].outer_bq_depth:
- self._table_stack[-1].parts.append(text)
- elif self._bq_stack:
- self._bq_stack[-1].append(text)
- else:
- self._out.append(text)
+ return self._link_text_parts
+ if self._in_cell:
+ return self._cell_parts
+ if self._in_pre:
+ return self._pre_parts
+ if self._table_stack and len(self._bq_stack) <= self._table_stack[-1].outer_bq_depth:
+ return self._table_stack[-1].parts
+ if self._bq_stack:
+ return self._bq_stack[-1]
+ return self._out
+
+ def _sup_base(self, target: list[str]) -> str:
+ """The visible text a <sup> raises (base = its last character), or "" when none: after whitespace or
+ sentence punctuation it is a footnote marker (``fact.<sup>1</sup>``,
+ ``<a href="#fn1"><sup>1</sup></a>``) or a fraction numerator (``<sup>1</sup>&frasl;``)."""
+ for part in itertools.islice(reversed(target), _SUP_BASE_SCAN_PARTS):
+ part = part[-_SUP_BASE_SCAN_CHARS:]
+ part = _visible_tail(part)
+ if part:
+ base = part[-1]
+ return part if base.isalnum() or base in ")]}|)]}" else ""
+ return ""
+
+ @staticmethod
+ def _after_price(target: list[str]) -> int:
+ """Digits of the minor unit when *target* ends in a currency amount ($19 -> 2, KWD 19 -> 3), else 0."""
+ context = _visible_tail("".join(p[-40:] for p in target[-8:])[-40:])
+ context = context.translate(_STRIP_MD_DELIMITERS)
+ price = _PRICE_TAIL.search(context)
+ # any Unicode currency sign (Sc): $, €, ₺, ₱, ...; or an ISO code: CHF 19
+ if price and unicodedata.category(price.group(1)) == "Sc":
+ # ₫ ₲ ₩ have no minor unit (¥ is shared with two-decimal CNY, so it stays 2)
+ return 0 if price.group(1) in "₫₲₩" else 2
+ code = _CODE_PRICE_TAIL.search(context)
+ if (
+ not code
+ or code.group(1) not in _CURRENCY_CODES
+ or code.group(1) in _ZERO_DECIMAL_CURRENCIES
+ ):
+ return 0
+ return 3 if code.group(1) in _THREE_DECIMAL_CURRENCIES else 2
+
+ def _sup_copies(self) -> list[tuple[list[str], int]]:
+ copies = [self._link_heading_parts, self._seg_heading_texts]
+ if self._header_stack:
+ copies.append(self._header_stack[-1].heading_parts)
+ return [(copy, len(copy)) for copy in copies]
+
+ def _finish_sup(self, opened) -> None:
+ if opened is None or opened[0] is not self._emit_target():
+ return
+ target, start, copies, base, after_price = opened
+ joined = "".join(target[start:])
+ raw = joined.strip()
+ shown = self._site_links.clean(raw) if self._site_links is not None else raw
+ visible = shown.strip(_MD_DELIMITERS)
+ if (
+ not visible
+ or "\n" in visible
+ or visible[0] == "["
+ # $19<sup>.99</sup> / €19<sup>,99</sup> are split cents; 10<sup>.5</sup> with no currency is a power
+ or (visible[0] in ".," and after_price)
+ or not any(c.isalnum() for c in visible)
+ or visible.lower() in _PLAIN_SUFFIXES
+ or (
+ len(_last_word(base)) > 1
+ and _CITATION_LIST.fullmatch(visible)
+ and ("–" in visible or not _THOUSANDS.fullmatch(visible))
+ )
+ or (base[-1] == "2" and visible.lower() in _SECOND_SUFFIXES)
+ or (
+ (base[-1].isdigit() or _ROMAN_NUMERAL_TAIL.search(base))
+ and visible.lower() in _DIGIT_ORDINAL_SUFFIXES
+ )
+ or visible in _SUPERIOR_ABBREVIATIONS.get(_last_word(base), ())
+ # split cents are two digits; $2<sup>n</sup> or USD 10<sup>6</sup> stays an exponent
+ or (len(visible) == after_price and visible.isdigit())
+ ):
+ return
+ # only the emphasis wrapping a whole exponent is renderer syntax; an inner * is an operator
+ token = shown.strip(_MD_DELIMITERS)
+ bare = _BARE_EXPONENT.fullmatch(token) or (
+ token.startswith("(") and token.endswith(")") and token.count("(") == 1
+ )
+ exponent = f"^{raw}" if bare else f"^({raw})"
+ exponent += joined[len(joined.rstrip()) :]
+ target[start:] = [exponent]
+ # headings are teed into these copies; a stale one renders "E=mc2" or skews the prose gate
+ for copy, copy_start in copies:
+ if "".join(copy[copy_start:]) == joined:
+ copy[copy_start:] = [exponent]
def _seg_heading_prose(self) -> int:
"""Heading characters in this segment that the gate would otherwise read as
@@ -879,6 +1090,9 @@ class _MarkdownRenderer(HTMLParser):
if name in _IMPLICIT_CLOSERS:
self._closable_open -= 1
del self._open_tags[index:]
+ # a <sup> an ancestor closed (<p>x<sup>2</p>) is gone; a kept frame would fill the depth cap
+ while self._sup_starts and self._sup_starts[-1][0] >= index:
+ self._sup_starts.pop()
def _close_implicit(self, tag: str) -> None:
"""HTML5 optional-end-tag recovery for a start tag about to open.
@@ -1116,6 +1330,36 @@ class _MarkdownRenderer(HTMLParser):
elif tag == "br":
self._emit("\n")
+ elif tag == "sup":
+ target = self._emit_target()
+ reference = (
+ any(
+ token in _FOOTNOTE_CLASSES
+ or any(
+ part == "fn" or part.startswith(_FOOTNOTE_CLASS_PREFIXES)
+ for part in _CLASS_PART_SPLIT.split(token)
+ )
+ for token in (attr_dict.get("class") or "").lower().split()
+ )
+ or "doc-noteref" in (attr_dict.get("role") or "").lower().split()
+ )
+ # past the cap nothing is tracked, so the stack stays bounded on hostile pages
+ if len(self._sup_starts) < _MAX_SUP_DEPTH:
+ self._sup_starts.append(
+ (
+ len(self._open_tags) - 1,
+ None
+ if reference or not (base := self._sup_base(target))
+ else (
+ target,
+ len(target),
+ self._sup_copies(),
+ base,
+ self._after_price(target) if base[-1].isdigit() else 0,
+ ),
+ )
+ )
+
elif tag in _BLOCK_TAGS:
if not self._li_marker_pending:
self._emit("\n\n")
@@ -1201,6 +1445,17 @@ class _MarkdownRenderer(HTMLParser):
if self._skip_depth:
return
+ closing_sup = None
+ # O(1): only a <sup> that is still the innermost open tag converts; malformed nesting drops on unwind
+ if (
+ tag == "sup"
+ and self._sup_starts
+ and self._open_tags
+ and self._open_tags[-1] == "sup"
+ and self._sup_starts[-1][0] == len(self._open_tags) - 1
+ ):
+ closing_sup = self._sup_starts.pop()[1]
+
if not self._exit_tag(tag):
return
@@ -1218,6 +1473,9 @@ class _MarkdownRenderer(HTMLParser):
elif tag in _INLINE_EMPHASIS:
self._emit(_INLINE_EMPHASIS[tag])
+ elif tag == "sup":
+ self._finish_sup(closing_sup)
+
elif tag in _BLOCK_TAGS:
self._emit("\n\n")
diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.py
index dcea7059972..dfc0da83bf6 100644
--- a/studio/backend/tests/test_web_fetch_extraction.py
+++ b/studio/backend/tests/test_web_fetch_extraction.py
@@ -869,6 +869,232 @@ def test_page_that_fits_keeps_its_links(monkeypatch):
assert "[[1]](#cite_note-1)" in out
+ "markup, expected",
+ [
+ ("(2<sup>53</sup> &ndash; 1)", "(2^53 – 1)"),
+ ("2<sup>&minus;52</sup>", "2^−52"),
+ ("1.898&times;10<sup>27</sup> kg", "1.898×10^27 kg"),
+ ("6.02214076&times;10<sup>23</sup> mol<sup>&minus;1</sup>", "6.02214076×10^23 mol^−1"),
+ ("2<sup><i>n</i>+1</sup> nodes", "2^(*n*+1) nodes"),
+ ("2<sup>n + 1</sup>", "2^(n + 1)"),
+ ("the 1<sup>st</sup> and 2<sup>nd</sup>", "the 1st and 2nd"),
+ ("Intel<sup>&reg;</sup> Core<sup>&trade;</sup> i7", "Intel® Core™ i7"),
+ ("now $19<sup>.99</sup> only", "now $19.99 only"),
+ ("price<sup>*</sup> and terms<sup>&dagger;</sup>", "price* and terms†"),
+ ("10<sup>6 </sup>years", "10^6 years"),
+ ("Add <sup>1</sup>&frasl;<sub>2</sub> cup", "Add 1⁄2 cup"),
+ ("Add 1 <sup>1</sup>/<sub>2</sub> cups", "Add 1 1/2 cups"),
+ ("<sup>1</sup> Footnote text", "1 Footnote text"),
+ ("Add <em><sup>1</sup></em>&frasl; cup", "Add *1*⁄ cup"),
+ ("the 1<sup><em>st</em></sup> one", "the 1*st* one"),
+ ("a<b><sup>2</sup></b>", "a**^2**"),
+ ("x<sup>n&times;2</sup>", "x^(n×2)"),
+ ("x<sup>2n</sup> and y<sup>n2</sup>", "x^(2n) and y^(n2)"),
+ ("x<sup>2<em>n</em></sup>", "x^(2*n*)"),
+ ("x<sup>2*3</sup> and x<sup><em>n</em></sup>", "x^(2*3) and x^*n*"),
+ ("e<sup>i&pi;</sup> + 1 = 0", "e^(iπ) + 1 = 0"),
+ ("now $19<sup>99</sup> or &euro; 1,299<sup>95</sup>", "now $1999 or € 1,29995"),
+ ("<b>$19</b><sup>99</sup>", "**$19**99"),
+ ("$<b>19</b><sup>99</sup>", "$**19**99"),
+ ("Brand<sup>TM</sup> and Service<sup>SM</sup>", "BrandTM and ServiceSM"),
+ ("el 1<sup>º</sup> y la 2<sup>ª</sup>, x<sup>a</sup>", "el 1º y la 2ª, x^a"),
+ (
+ "le XV<sup>e</sup> siècle, François I<sup>er</sup>, MAX<sup>e</sup>",
+ "le XVe siècle, François Ier, MAX^e",
+ ),
+ ("<span>$</span> <b>19</b><sup>99</sup>", "$ **19**99"),
+ (
+ "&#8378;19<sup>99</sup> or &#8369;19<sup>99</sup>, a19<sup>2</sup>",
+ "₺1999 or ₱1999, a19^2",
+ ),
+ (
+ "CHF 19<sup>95</sup> or USD 19<sup>99</sup>, ABC 10<sup>3</sup>",
+ "CHF 1995 or USD 1999, ABC 10^3",
+ ),
+ (
+ "AED 19<sup>99</sup>, TWD 19<sup>99</sup>, ILS 19<sup>99</sup>",
+ "AED 1999, TWD 1999, ILS 1999",
+ ),
+ (
+ "ZWG 19<sup>99</sup>, les 1<sup>ers</sup> et 1<sup>res</sup>",
+ "ZWG 1999, les 1ers et 1res",
+ ),
+ ("$2<sup>n</sup> and USD 10<sup>6</sup>, $19<sup>99</sup>", "$2^n and USD 10^6, $1999"),
+ (
+ 'XCG 19<sup>99</sup>, claim<sup role="doc-noteref presentation">2</sup>',
+ "XCG 1999, claim2",
+ ),
+ ("le P<sup>r</sup> Martin et les P<sup>rs</sup>", "le Pr Martin et les Prs"),
+ ("CHF 1’299<sup>95</sup> or CHF 1'299<sup>95</sup>", "CHF 1’29995 or CHF 1'29995"),
+ ("Marque<sup>MC</sup> et Produit<sup>MD</sup>", "MarqueMC et ProduitMD"),
+ ("Brand<sup>(TM)</sup> and Other<sup>(R)</sup>", "Brand(TM) and Other(R)"),
+ (
+ "KWD 19<sup>950</sup>, BHD 1<sup>234</sup>, USD 10<sup>100</sup>",
+ "KWD 19950, BHD 1234, USD 10^100",
+ ),
+ ("M<sup>r</sup>, M<sup>s</sup> and M<sup>rs</sup> Smith", "Mr, Ms and Mrs Smith"),
+ ("JPY 10<sup>12</sup> and KRW 10<sup>12</sup>", "JPY 10^12 and KRW 10^12"),
+ ("les 1<sup>ères</sup> places", "les 1ères places"),
+ ("John J<sup>r</sup> and John S<sup>r</sup>", "John Jr and John Sr"),
+ ("VED 19<sup>99</sup> and S<sup>T</sup>", "VED 1999 and S^T"),
+ (
+ 'claim<sup class="footnote-reference">2</sup>, la 2<sup>de</sup>, le 2<sup>d</sup>',
+ "claim2, la 2de, le 2d",
+ ),
+ (
+ "la P<sup>re</sup> Durand, 12<sup>bis</sup> rue, article 3<sup>ter</sup>",
+ "la Pre Durand, 12bis rue, article 3ter",
+ ),
+ (
+ 'claim<sup class="citation">2</sup> and fact<sup class="endnote-ref">3</sup>',
+ "claim2 and fact3",
+ ),
+ ("la D<sup>re</sup> Roy et les D<sup>res</sup>", "la Dre Roy et les Dres"),
+ (
+ 'x<sup class="excited">2</sup> and 10<sup>.5</sup>, $19<sup>.99</sup>',
+ "x^2 and 10^(.5), $19.99",
+ ),
+ (
+ 'claim<sup class="fnref">2</sup>, fact<sup class="fn-ref">3</sup>, Marca<sup>MR</sup>',
+ "claim2, fact3, MarcaMR",
+ ),
+ (
+ "les S<sup>ts</sup>, les S<sup>tes</sup> et les C<sup>ies</sup>",
+ "les Sts, les Stes et les Cies",
+ ),
+ ("les 2<sup>nds</sup>", "les 2nds"),
+ ("(a+b)<sup>2</sup> and 「引用」<sup>1</sup>", "(a+b)^2 and 「引用」1"),
+ (
+ '&euro;19<sup>,99</sup> and claim<sup class="reference-number">2</sup>',
+ "€19,99 and claim2",
+ ),
+ (
+ "SVC 19<sup>99</sup>, claim<sup>1&ndash;3</sup>, fact<sup>2,5</sup>, 10<sup>-3</sup>",
+ "SVC 1999, claim1–3, fact2,5, 10^-3",
+ ),
+ ("x<sup>2,5</sup> and 10<sup>1,000</sup>", "x^2,5 and 10^1,000"),
+ ("&#8363;10<sup>12</sup> and &#8361;10<sup>12</sup>", "₫10^12 and ₩10^12"),
+ ("claim<sup>1,3&ndash;5</sup>", "claim1,3–5"),
+ ("Doors 9:30<sup>pm</sup>, ends 11<sup>p.m.</sup>", "Doors 9:30pm, ends 11p.m."),
+ ("10<sup>6&ndash;8</sup> CFU/mL and 10<sup>d</sup>", "10^(6–8) CFU/mL and 10^d"),
+ ("(x+1)<sup>2&ndash;4</sup>", "(x+1)^(2–4)"),
+ (
+ "M<sup>me</sup> Dupont, D<sup>r</sup> Martin, n<sup>o</sup> 5, Om<sup>e</sup>",
+ "Mme Dupont, Dr Martin, no 5, Om^e",
+ ),
+ ("&euro;1.299<sup>95</sup> or &euro;1 299<sup>95</sup>", "€1.29995 or €1 29995"),
+ (
+ 'claim<sup role="doc-noteref">1</sup> and fact<sup class="footnote">2</sup>',
+ "claim1 and fact2",
+ ),
+ ("x<sup>-n</sup> and 10<sup>2.5</sup>", "x^-n and 10^2.5"),
+ ("now $19<sup><em>.99</em></sup>", "now $19*.99*"),
+ ("A fact.<sup>1</sup> Next, a list,<sup>2</sup>", "A fact.1 Next, a list,2"),
+ ("(a+b)<sup>2</sup> and km<sup>2</sup>.", "(a+b)^2 and km^2."),
+ ("le 1<sup>er</sup> mai, 2<sup>e</sup> et x<sup>e</sup>", "le 1er mai, 2e et x^e"),
+ ],
+)
+def test_superscripts_keep_their_exponent(markup, expected):
+ assert html_to_markdown(f"<p>{markup}</p>") == expected
+
+
+ "title",
+ [
+ "<h1><a href='/p'>E=mc<sup>2</sup></a></h1>",
+ "<a href='/p'><h1>E=mc<sup>2</sup></h1></a>",
+ ],
+)
+def test_linked_header_title_keeps_one_exponent(title):
+ nav = "".join(f"<a href='/s{i}'>Section number {i}</a> " for i in range(12))
+ html = f"<header>{title}{nav}</header><p>{'Body text here. ' * 40}</p>"
+ out = html_to_markdown(html, main_content = True)
+ assert out.count("E=mc") == 1
+ assert "E=mc^2](/p)" in out
+
+
+def test_stripped_header_keeps_the_exponent_in_its_heading():
+ nav = "".join(f"<a href='/s{i}'>Section number {i}</a> " for i in range(12))
+ html = f"<header><h1>E=mc<sup>2</sup></h1>{nav}</header><p>{'Body text here. ' * 40}</p>"
+ out = html_to_markdown(html, main_content = True)
+ assert "# E=mc^2" in out
+ assert "E=mc2" not in out
+
+
+def test_deeply_nested_superscripts_track_a_bounded_depth():
+ out = html_to_markdown("<p>x" + "<sup>a" * 50 + "</sup>" * 50 + "</p>")
+ assert out.translate(str.maketrans("", "", "^()")) == "x" + "a" * 50
+ assert out.count("^") == 8
+
+
+def test_linked_footnote_wrapping_its_superscript_renders_unchanged():
+ html = '<p>text<a role="doc-noteref" href="#fn1"><sup>1</sup></a> more</p>'
+ assert html_to_markdown(html) == "text[1](#fn1) more"
+
+
+def test_exponent_headings_do_not_read_as_body_prose():
+ heads = "".join(f"<h2>x<sup>n+{i}</sup></h2>" for i in range(67))
+ body = "<p>" + "Real document body sentence. " * 12 + "</p>"
+ html = f"<html><body><article>{heads}</article><div>{body}</div></body></html>"
+ assert "Real document body" in html_to_markdown(html, main_content = True)
+
+
+def test_same_site_linked_base_keeps_its_exponent():
+ site_links = SiteLinks("https://e.com/page")
+ html = '<p>5 <a href="/metre">m</a><sup>2</sup> and <a href="https://x.org/m">m</a><sup>3</sup></p>'
+ out = site_links.clean(html_to_markdown(html, site_links = site_links))
+ assert out == "5 [m](/metre)^2 and [m](https://x.org/m)^3"
+
+
+def test_empty_superscripts_do_not_rescan_the_page():
+ start = time.perf_counter()
+ html_to_markdown("<p>" + "<i></i><sup></sup>" * 8000 + "</p>")
+ assert time.perf_counter() - start < 2
+ start = time.perf_counter()
+ html_to_markdown("<p>" + "*" * 250000 + "<sup></sup>" * 23000 + "</p>")
+ assert time.perf_counter() - start < 3
+ start = time.perf_counter()
+ html_to_markdown("<p>claim<sup>" + "1–" * 16000 + "x</sup></p>")
+ assert time.perf_counter() - start < 2
+ start = time.perf_counter()
+ html_to_markdown(
+ "<p>" + "<b>" + "word 1 " * 40000 + "</b>" * 1 + "9" + "<sup></sup>" * 20000 + "</p>"
+ )
+ assert time.perf_counter() - start < 3
+
+
+def test_unclosed_superscripts_do_not_exhaust_the_depth_cap():
+ html = "<p>x<sup>2</p>" * 10 + "<p>1.898&times;10<sup>27</sup> kg</p>"
+ assert html_to_markdown(html).endswith("1.898×10^27 kg")
+
+
+def test_untracked_superscripts_keep_the_stack_bounded():
+ from core.inference._html_to_md import _MarkdownRenderer
+
+ renderer = _MarkdownRenderer()
+ renderer.feed("<p>" + "<sup>" * 5000 + "x")
+ assert len(renderer._sup_starts) <= 8
+
+
+def test_footnote_superscripts_render_unchanged():
+ html = (
+ '<p>mass<sup class="reference"><a href="#cite_note-12">[12]</a></sup> and '
+ "volume<sup>[13]</sup></p>"
+ )
+ assert html_to_markdown(html) == "mass[[12]](#cite_note-12) and volume[13]"
+
+
+def test_fetched_page_keeps_exponents_beside_footnotes(monkeypatch):
+ body = (
+ "<html><body><main><article><p>Jupiter has a mass of 1.898&times;10<sup>27</sup> kg"
+ '<sup class="reference"><a href="#cite_note-12">[12]</a></sup> and a surface area of '
+ "6.1419&times;10<sup>10</sup> km<sup>2</sup>.</p></article></main></body></html>"
+ )
+ out = _page_text(monkeypatch, "https://en.wikipedia.org/wiki/Jupiter", body, "text/html")
+ assert "1.898×10^27 kg[[12]](#cite_note-12) and a surface area of 6.1419×10^10 km^2." in out
+
+
def test_page_cut_by_the_room_left_drops_its_site_link_urls(monkeypatch):
from core.inference import tools