Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(arxiv): read the TeX from alttext when the annotation child is absent (#6456)

Lukas4b9664cd1e66
diff --git a/changelog.d/6456.bugfix.md b/changelog.d/6456.bugfix.md
new file mode 100644
index 0000000000..933d95dbaf
--- /dev/null
+++ b/changelog.d/6456.bugfix.md
@@ -0,0 +1,8 @@
+Equations are recovered from arXiv HTML whose TeX lives only in a `<math>`
+element's `alttext` attribute (#6414). When the `annotation` child is absent or
+empty, `alttext` is read instead of abandoning the page to the PDF fallback. An
+explicit annotation still wins where the two disagree, and `alttext` that is
+empty or whitespace-only is not treated as a source.
+
+The `annotation` matcher stays anchored, so `annotation-xml` — which carries
+MathML-Content, not TeX — is still not mistaken for a TeX source (#6418).
diff --git a/src/local_deep_research/research_library/downloaders/arxiv.py b/src/local_deep_research/research_library/downloaders/arxiv.py
index cbd960d3b0..4c39c9c622 100644
--- a/src/local_deep_research/research_library/downloaders/arxiv.py
+++ b/src/local_deep_research/research_library/downloaders/arxiv.py
@@ -8,7 +8,7 @@ from enum import StrEnum
from typing import Dict, Final, Optional
from urllib.parse import urlparse
-from bs4 import BeautifulSoup, NavigableString
+from bs4 import BeautifulSoup, NavigableString, Tag
from loguru import logger
from ...constants import USER_AGENT
@@ -257,18 +257,47 @@ class ArxivDownloader(HTMLDownloader):
)
return None
+ @staticmethod
+ def _tex_for_math_element(math: Tag) -> str:
+ """The TeX one <math> element carries, or ``""`` when it carries none.
+
+ Two sources, annotation first because it is the explicit one: an
+ ``<annotation encoding="application/x-tex">`` child, then the
+ ``alttext`` attribute LaTeXML always writes onto the element itself.
+ An empty annotation falls through rather than abandoning the page --
+ an element can carry an empty annotation and a usable one beside it, or
+ an empty annotation and a usable attribute, and there is no reason to
+ prefer the empty one. Every x-tex annotation is examined for that
+ reason, not just the first descendant.
+
+ ``<annotation-xml>`` is deliberately not a source: it holds a
+ different encoding, and the anchored matcher already excludes it.
+ """
+ for annotation in math.find_all(
+ _TEX_ANNOTATION_LOCAL_NAME,
+ attrs={"encoding": "application/x-tex"},
+ ):
+ tex = annotation.get_text().strip()
+ if tex:
+ return tex
+ alttext = math.get("alttext")
+ return alttext.strip() if isinstance(alttext, str) else ""
+
@staticmethod
def _rewrite_math_to_tex(soup: BeautifulSoup, arxiv_id: str) -> bool:
"""Replace every <math> element with its TeX annotation, in place.
- Returns False when the rendition must be abandoned for the PDF. Three
- distinct cases do that, each logged at debug so they are told apart
- in a report: the page carries more than MAX_MATH_ELEMENTS equations,
- a <math> has no ``<annotation encoding="application/x-tex">`` child,
- or that annotation is empty. The second is not hypothetical -- LaTeXML
- also carries the TeX in the ``alttext`` attribute and the parallel
- <annotation> markup depends on how arXiv invokes it, so a whole class
- of renditions may take this exit.
+ Returns False when the rendition must be abandoned for the PDF. Two
+ distinct cases do that, each logged at debug so they are told apart in
+ a report: the page carries more than MAX_MATH_ELEMENTS equations, or a
+ <math> offers no TeX at all.
+
+ "No TeX at all" is narrower than it used to be. The annotation child is
+ not the only place LaTeXML writes the TeX -- it always writes it into
+ the ``alttext`` attribute as well, while the parallel <annotation>
+ markup depends on how arXiv invokes it, so a rendition carrying only
+ ``alttext`` used to take the PDF exit with its TeX sitting right there
+ on the element (#6414). Both sources are read now, annotation first.
Each element is rewritten in place -- renamed to an inline <span>
holding the TeX -- rather than with ``Tag.replace_with``. replace_with
@@ -290,23 +319,12 @@ class ArxivDownloader(HTMLDownloader):
return False
for math in math_elements:
- annotation = math.find(
- _TEX_ANNOTATION_LOCAL_NAME,
- attrs={"encoding": "application/x-tex"},
- )
- if annotation is None:
- logger.debug(
- "arXiv HTML for {} has a <math> element with no "
- '<annotation encoding="application/x-tex"> child; '
- "falling back to the PDF",
- arxiv_id,
- )
- return False
- tex = annotation.get_text().strip()
+ tex = ArxivDownloader._tex_for_math_element(math)
if not tex:
logger.debug(
- "arXiv HTML for {} has a <math> element whose TeX "
- "annotation is empty; falling back to the PDF",
+ "arXiv HTML for {} has a <math> element carrying neither a "
+ 'non-empty <annotation encoding="application/x-tex"> child '
+ "nor a non-empty alttext attribute; falling back to the PDF",
arxiv_id,
)
return False
diff --git a/tests/research_library/downloaders/test_arxiv_html_text.py b/tests/research_library/downloaders/test_arxiv_html_text.py
index 1a0d7c73f6..e01944b176 100644
--- a/tests/research_library/downloaders/test_arxiv_html_text.py
+++ b/tests/research_library/downloaders/test_arxiv_html_text.py
@@ -839,3 +839,180 @@ def test_html_accepts_matching_rendition(
extract.assert_called_once_with(
mocker.ANY, f"https://arxiv.org/abs/{requested_id}"
)
+
+
+# ---------------------------------------------------------------------------
+# TeX sources on a <math> element (#6414) and the anchored matchers (#6418)
+# ---------------------------------------------------------------------------
+
+
+def _page_with(formula: str) -> str:
+ """One rendition carrying `formula`, with enough prose to survive the
+ shared extraction pipeline."""
+ prose = (
+ "This article carries typeset equations and enough surrounding prose "
+ "that the shared extraction pipeline keeps the body. Each equation is "
+ "followed by discussion of the bounds it implies. "
+ ) * 8
+ return (
+ "<html><head><title>Equation</title></head>"
+ "<body><article><p>" + prose + formula + "</p></article></body></html>"
+ )
+
+
+def test_alttext_is_read_when_the_annotation_child_is_absent(downloader):
+ # Given a rendition whose TeX lives only in alttext. LaTeXML always writes
+ # it there; the parallel <annotation> markup depends on how arXiv invokes
+ # LaTeXML, so this shape used to take the PDF exit with its TeX in hand.
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with('<math alttext="x^2"><mi>x</mi></math>'), "lxml"
+ )
+
+ # When the rewrite runs
+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")
+
+ # Then the page is kept and the TeX is in the text
+ assert rewritten is True
+ assert soup.find_all("math") == []
+ assert "x^2" in soup.get_text()
+
+
+def test_alttext_is_read_on_a_namespace_prefixed_element(downloader):
+ # The shape #5617 widened the matcher for, carrying only annotation-xml --
+ # a different encoding, and not a TeX source.
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with(
+ '<m:math alttext="\\alpha"><m:semantics><m:mi>a</m:mi>'
+ '<m:annotation-xml encoding="MathML-Content"><m:ci>a</m:ci>'
+ "</m:annotation-xml></m:semantics></m:math>"
+ ),
+ "lxml",
+ )
+
+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")
+
+ assert rewritten is True
+ assert "\\alpha" in soup.get_text()
+
+
+def test_an_empty_annotation_falls_through_to_alttext(downloader):
+ # An element can carry an empty annotation and a usable attribute, and
+ # there is no reason to prefer the empty one.
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with(
+ '<math alttext="x^2"><semantics>'
+ '<annotation encoding="application/x-tex"></annotation>'
+ "</semantics></math>"
+ ),
+ "lxml",
+ )
+
+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")
+
+ assert rewritten is True
+ assert "x^2" in soup.get_text()
+
+
+def test_the_annotation_still_wins_over_alttext(downloader):
+ # Order matters and must be observable: the annotation is the explicit
+ # source, so a disagreement resolves to it.
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with(
+ '<math alttext="FROM_ALTTEXT"><semantics>'
+ '<annotation encoding="application/x-tex">FROM_ANNOTATION'
+ "</annotation></semantics></math>"
+ ),
+ "lxml",
+ )
+
+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")
+
+ assert rewritten is True
+ text = soup.get_text()
+ assert "FROM_ANNOTATION" in text
+ assert "FROM_ALTTEXT" not in text
+
+
+def test_neither_source_still_falls_back_to_the_pdf(downloader):
+ # The exit is narrower now, not gone: a <math> with no annotation and no
+ # alttext carries no TeX, and the MathML must not be left in the text.
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(_page_with("<math><mi>x</mi></math>"), "lxml")
+
+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")
+
+ assert rewritten is False
+ # The comment above promises the MathML is not left behind; assert it,
+ # rather than trusting that returning False is enough.
+ assert "<mi>" not in soup.get_text()
+ assert "<math" not in soup.get_text()
+
+
+def test_an_empty_annotation_falls_through_to_a_later_one(downloader):
+ """``find`` returns the FIRST matching descendant, so an element carrying
+ an empty x-tex annotation ahead of a usable one used to take the PDF exit
+ with its TeX sitting right there.
+ """
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with(
+ "<math><semantics>"
+ '<annotation encoding="application/x-tex"></annotation>'
+ '<annotation encoding="application/x-tex">x^2</annotation>'
+ "</semantics></math>"
+ ),
+ "lxml",
+ )
+
+ assert downloader._rewrite_math_to_tex(soup, "2501.12345v2") is True
+ assert "x^2" in soup.get_text()
+
+
+def test_an_empty_alttext_is_not_a_tex_source(downloader):
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with('<math alttext=" "><mi>x</mi></math>'), "lxml"
+ )
+
+ assert downloader._rewrite_math_to_tex(soup, "2501.12345v2") is False
+
+
+def test_the_local_name_matchers_are_anchored(downloader):
+ """#6418: the matchers are `(?:^|:)math$` and `(?:^|:)annotation$` so that
+ `<m:math>` is rewritten alongside `<math>`. Nothing pinned the anchors --
+ replacing both with unanchored `math` / `annotation` left every test green,
+ although the unanchored form would rewrite `<mathjax>` and would read
+ `<annotation-xml>` as a TeX annotation.
+ """
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(
+ _page_with(
+ "<mathjax>NOT_MATH</mathjax>"
+ '<math alttext="real"><semantics>'
+ '<annotation-xml encoding="MathML-Content">NOT_TEX</annotation-xml>'
+ "</semantics></math>"
+ ),
+ "lxml",
+ )
+
+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")
+
+ assert rewritten is True
+ # `<mathjax>` is not a <math> element: it survives untouched.
+ assert soup.find("mathjax") is not None
+ assert "NOT_MATH" in soup.get_text()
+ # `<annotation-xml>` is not a TeX annotation: alttext supplied the TeX.
+ assert "real" in soup.get_text()
+ assert "NOT_TEX" not in soup.get_text()