~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(arxiv): read the TeX from alttext when the annotation child is absent (#6456)
Lukas4b9664cd1e66
diff --git a/changelog.d/6456.bugfix.md b/changelog.d/6456.bugfix.mdnew file mode 100644index 0000000000..933d95dbaf--- /dev/null+++ b/changelog.d/6456.bugfix.md@@ -0,0 +1,8 @@+Equations are recovered from arXiv HTML whose TeX lives only in a `<math>`+element's `alttext` attribute (#6414). When the `annotation` child is absent or+empty, `alttext` is read instead of abandoning the page to the PDF fallback. An+explicit annotation still wins where the two disagree, and `alttext` that is+empty or whitespace-only is not treated as a source.++The `annotation` matcher stays anchored, so `annotation-xml` — which carries+MathML-Content, not TeX — is still not mistaken for a TeX source (#6418).diff --git a/src/local_deep_research/research_library/downloaders/arxiv.py b/src/local_deep_research/research_library/downloaders/arxiv.pyindex cbd960d3b0..4c39c9c622 100644--- a/src/local_deep_research/research_library/downloaders/arxiv.py+++ b/src/local_deep_research/research_library/downloaders/arxiv.py@@ -8,7 +8,7 @@ from enum import StrEnumfrom typing import Dict, Final, Optionalfrom urllib.parse import urlparse-from bs4 import BeautifulSoup, NavigableString+from bs4 import BeautifulSoup, NavigableString, Tagfrom loguru import loggerfrom ...constants import USER_AGENT@@ -257,18 +257,47 @@ class ArxivDownloader(HTMLDownloader):)return None+ @staticmethod+ def _tex_for_math_element(math: Tag) -> str:+ """The TeX one <math> element carries, or ``""`` when it carries none.++ Two sources, annotation first because it is the explicit one: an+ ``<annotation encoding="application/x-tex">`` child, then the+ ``alttext`` attribute LaTeXML always writes onto the element itself.+ An empty annotation falls through rather than abandoning the page --+ an element can carry an empty annotation and a usable one beside it, or+ an empty annotation and a usable attribute, and there is no reason to+ prefer the empty one. Every x-tex annotation is examined for that+ reason, not just the first descendant.++ ``<annotation-xml>`` is deliberately not a source: it holds a+ different encoding, and the anchored matcher already excludes it.+ """+ for annotation in math.find_all(+ _TEX_ANNOTATION_LOCAL_NAME,+ attrs={"encoding": "application/x-tex"},+ ):+ tex = annotation.get_text().strip()+ if tex:+ return tex+ alttext = math.get("alttext")+ return alttext.strip() if isinstance(alttext, str) else ""+@staticmethoddef _rewrite_math_to_tex(soup: BeautifulSoup, arxiv_id: str) -> bool:"""Replace every <math> element with its TeX annotation, in place.- Returns False when the rendition must be abandoned for the PDF. Three- distinct cases do that, each logged at debug so they are told apart- in a report: the page carries more than MAX_MATH_ELEMENTS equations,- a <math> has no ``<annotation encoding="application/x-tex">`` child,- or that annotation is empty. The second is not hypothetical -- LaTeXML- also carries the TeX in the ``alttext`` attribute and the parallel- <annotation> markup depends on how arXiv invokes it, so a whole class- of renditions may take this exit.+ Returns False when the rendition must be abandoned for the PDF. Two+ distinct cases do that, each logged at debug so they are told apart in+ a report: the page carries more than MAX_MATH_ELEMENTS equations, or a+ <math> offers no TeX at all.++ "No TeX at all" is narrower than it used to be. The annotation child is+ not the only place LaTeXML writes the TeX -- it always writes it into+ the ``alttext`` attribute as well, while the parallel <annotation>+ markup depends on how arXiv invokes it, so a rendition carrying only+ ``alttext`` used to take the PDF exit with its TeX sitting right there+ on the element (#6414). Both sources are read now, annotation first.Each element is rewritten in place -- renamed to an inline <span>holding the TeX -- rather than with ``Tag.replace_with``. replace_with@@ -290,23 +319,12 @@ class ArxivDownloader(HTMLDownloader):return Falsefor math in math_elements:- annotation = math.find(- _TEX_ANNOTATION_LOCAL_NAME,- attrs={"encoding": "application/x-tex"},- )- if annotation is None:- logger.debug(- "arXiv HTML for {} has a <math> element with no "- '<annotation encoding="application/x-tex"> child; '- "falling back to the PDF",- arxiv_id,- )- return False- tex = annotation.get_text().strip()+ tex = ArxivDownloader._tex_for_math_element(math)if not tex:logger.debug(- "arXiv HTML for {} has a <math> element whose TeX "- "annotation is empty; falling back to the PDF",+ "arXiv HTML for {} has a <math> element carrying neither a "+ 'non-empty <annotation encoding="application/x-tex"> child '+ "nor a non-empty alttext attribute; falling back to the PDF",arxiv_id,)return Falsediff --git a/tests/research_library/downloaders/test_arxiv_html_text.py b/tests/research_library/downloaders/test_arxiv_html_text.pyindex 1a0d7c73f6..e01944b176 100644--- a/tests/research_library/downloaders/test_arxiv_html_text.py+++ b/tests/research_library/downloaders/test_arxiv_html_text.py@@ -839,3 +839,180 @@ def test_html_accepts_matching_rendition(extract.assert_called_once_with(mocker.ANY, f"https://arxiv.org/abs/{requested_id}")+++# ---------------------------------------------------------------------------+# TeX sources on a <math> element (#6414) and the anchored matchers (#6418)+# ---------------------------------------------------------------------------+++def _page_with(formula: str) -> str:+ """One rendition carrying `formula`, with enough prose to survive the+ shared extraction pipeline."""+ prose = (+ "This article carries typeset equations and enough surrounding prose "+ "that the shared extraction pipeline keeps the body. Each equation is "+ "followed by discussion of the bounds it implies. "+ ) * 8+ return (+ "<html><head><title>Equation</title></head>"+ "<body><article><p>" + prose + formula + "</p></article></body></html>"+ )+++def test_alttext_is_read_when_the_annotation_child_is_absent(downloader):+ # Given a rendition whose TeX lives only in alttext. LaTeXML always writes+ # it there; the parallel <annotation> markup depends on how arXiv invokes+ # LaTeXML, so this shape used to take the PDF exit with its TeX in hand.+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with('<math alttext="x^2"><mi>x</mi></math>'), "lxml"+ )++ # When the rewrite runs+ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")++ # Then the page is kept and the TeX is in the text+ assert rewritten is True+ assert soup.find_all("math") == []+ assert "x^2" in soup.get_text()+++def test_alttext_is_read_on_a_namespace_prefixed_element(downloader):+ # The shape #5617 widened the matcher for, carrying only annotation-xml --+ # a different encoding, and not a TeX source.+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with(+ '<m:math alttext="\\alpha"><m:semantics><m:mi>a</m:mi>'+ '<m:annotation-xml encoding="MathML-Content"><m:ci>a</m:ci>'+ "</m:annotation-xml></m:semantics></m:math>"+ ),+ "lxml",+ )++ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")++ assert rewritten is True+ assert "\\alpha" in soup.get_text()+++def test_an_empty_annotation_falls_through_to_alttext(downloader):+ # An element can carry an empty annotation and a usable attribute, and+ # there is no reason to prefer the empty one.+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with(+ '<math alttext="x^2"><semantics>'+ '<annotation encoding="application/x-tex"></annotation>'+ "</semantics></math>"+ ),+ "lxml",+ )++ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")++ assert rewritten is True+ assert "x^2" in soup.get_text()+++def test_the_annotation_still_wins_over_alttext(downloader):+ # Order matters and must be observable: the annotation is the explicit+ # source, so a disagreement resolves to it.+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with(+ '<math alttext="FROM_ALTTEXT"><semantics>'+ '<annotation encoding="application/x-tex">FROM_ANNOTATION'+ "</annotation></semantics></math>"+ ),+ "lxml",+ )++ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")++ assert rewritten is True+ text = soup.get_text()+ assert "FROM_ANNOTATION" in text+ assert "FROM_ALTTEXT" not in text+++def test_neither_source_still_falls_back_to_the_pdf(downloader):+ # The exit is narrower now, not gone: a <math> with no annotation and no+ # alttext carries no TeX, and the MathML must not be left in the text.+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(_page_with("<math><mi>x</mi></math>"), "lxml")++ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")++ assert rewritten is False+ # The comment above promises the MathML is not left behind; assert it,+ # rather than trusting that returning False is enough.+ assert "<mi>" not in soup.get_text()+ assert "<math" not in soup.get_text()+++def test_an_empty_annotation_falls_through_to_a_later_one(downloader):+ """``find`` returns the FIRST matching descendant, so an element carrying+ an empty x-tex annotation ahead of a usable one used to take the PDF exit+ with its TeX sitting right there.+ """+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with(+ "<math><semantics>"+ '<annotation encoding="application/x-tex"></annotation>'+ '<annotation encoding="application/x-tex">x^2</annotation>'+ "</semantics></math>"+ ),+ "lxml",+ )++ assert downloader._rewrite_math_to_tex(soup, "2501.12345v2") is True+ assert "x^2" in soup.get_text()+++def test_an_empty_alttext_is_not_a_tex_source(downloader):+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with('<math alttext=" "><mi>x</mi></math>'), "lxml"+ )++ assert downloader._rewrite_math_to_tex(soup, "2501.12345v2") is False+++def test_the_local_name_matchers_are_anchored(downloader):+ """#6418: the matchers are `(?:^|:)math$` and `(?:^|:)annotation$` so that+ `<m:math>` is rewritten alongside `<math>`. Nothing pinned the anchors --+ replacing both with unanchored `math` / `annotation` left every test green,+ although the unanchored form would rewrite `<mathjax>` and would read+ `<annotation-xml>` as a TeX annotation.+ """+ from bs4 import BeautifulSoup++ soup = BeautifulSoup(+ _page_with(+ "<mathjax>NOT_MATH</mathjax>"+ '<math alttext="real"><semantics>'+ '<annotation-xml encoding="MathML-Content">NOT_TEX</annotation-xml>'+ "</semantics></math>"+ ),+ "lxml",+ )++ rewritten = downloader._rewrite_math_to_tex(soup, "2501.12345v2")++ assert rewritten is True+ # `<mathjax>` is not a <math> element: it survives untouched.+ assert soup.find("mathjax") is not None+ assert "NOT_MATH" in soup.get_text()+ # `<annotation-xml>` is not a TeX annotation: alttext supplied the TeX.+ assert "real" in soup.get_text()+ assert "NOT_TEX" not in soup.get_text()