Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Offline tests: HTML fixtures are snapshots of a real GitHub page, so no network is needed."""
from __future__ import annotations
import email
import sys
import time
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from core.inference._html_to_md import SiteLinks, _is_aria_heading, html_to_markdown
from core.inference.tools import (
_fetch_page_text,
_fetch_url_raw,
_github_repo_readme_api_url,
_looks_like_html,
)
_GITHUB_HIDDEN_ERROR_BLOCK = """
<div data-show-on-forbidden-error hidden>
<div class="Box">
<div class="blankslate-container">
<h3 class="blankslate-heading">Uh oh!</h3>
<p class="blankslate-description">
<p class="color-fg-muted my-2 mb-2 ws-normal">There was an error while loading.
<a class="Link--inTextBlock" href="" aria-label="Please reload this page">Please reload this page</a>.</p>
</p>
</div>
</div>
</div>
"""
_GITHUB_PAGE = f"""<!DOCTYPE html>
<html lang="en">
<head><title>unslothai/unsloth</title></head>
<body>
<a class="px-2 py-4" href="#start-of-content">Skip to content</a>
<header class="Header-old">
<div class="AppHeader-globalBar">
<a href="/login">Sign in</a>
<a href="/signup">Sign up</a>
</div>
</header>
<div class="js-notification-shelf"></div>
<div hidden>
You signed in with another tab or window. Reload to refresh your session.
You signed out in another tab or window. Reload to refresh your session.
You switched accounts on another tab or window. Reload to refresh your session.
Dismiss alert
</div>
<template>{{{{ message }}}}</template>
{_GITHUB_HIDDEN_ERROR_BLOCK}
<main id="js-repo-pjax-container">
{_GITHUB_HIDDEN_ERROR_BLOCK}
<div id="repository-container-header">
<a href="/unslothai">unslothai</a> / <a href="/unslothai/unsloth">unsloth</a>
<a href="/login?return_to=%2Funslothai%2Funsloth">Notifications</a>
You must be signed in to change notification settings
</div>
<div class="repository-content">
<table aria-labelledby="folders-and-files">
<tr><th>Name</th><th>Last commit message</th></tr>
<tr><td><a href="/unslothai/unsloth/tree/main/unsloth">unsloth</a></td><td></td></tr>
</table>
<article class="markdown-body entry-content container-lg" itemprop="text">
<h1>Unsloth Studio</h1>
<p>Unsloth Studio lets you run and train models locally. Fine-tune and
run LLMs on Windows, Linux and macOS with a single install command,
then export to GGUF, Ollama, vLLM or Hugging Face when you are done.</p>
<h2>Install</h2>
<pre>curl -fsSL https://unsloth.ai/install.sh | sh</pre>
<p>See the <a href="https://unsloth.ai/docs">documentation</a> for
quickstarts, notebooks, and fine-tuning guides for every major model
family including Llama, Gemma, Qwen and DeepSeek.</p>
</article>
</div>
<div class="Layout-sidebar">
<h2>Languages</h2>
<ul>
<li><a href="/unslothai/unsloth/search?l=javascript">JavaScript 89.3%</a></li>
<li><a href="/unslothai/unsloth/search?l=python">Python 9.7%</a></li>
</ul>
</div>
</main>
<footer>
<a href="https://docs.github.com">Docs</a>
<a href="https://github.com/contact">Contact</a>
</footer>
<div aria-live="polite" aria-hidden="true">You can't perform that action at this time.</div>
</body>
</html>
"""
@pytest.mark.parametrize(
"html, present_first, present_second, absent",
[
pytest.param(
"<body><p>visible</p><div hidden><p>secret error text</p></div><p>after</p></body>",
"visible",
"after",
"secret error text",
id = "hidden_attribute_subtree_is_dropped",
),
pytest.param(
"<body><p>visible</p>"
'<div style="display:none">secret loading block</div>'
"<p>after</p></body>",
"visible",
"after",
"secret loading block",
id = "inline_style_display_none_subtree_is_dropped",
),
pytest.param(
'<body><p>before</p><hr aria-hidden="true"><p>after</p></body>',
"before",
"after",
"---",
id = "hidden_void_element_is_suppressed",
),
pytest.param(
"<body><p>one<br hidden>two</p></body>",
"one",
"two",
"one\ntwo",
id = "hidden_void_br_emits_no_break",
),
pytest.param(
"<body><p>Skip to content</p><div hidden>gone</div><main><p>hello</p></main></body>",
"Skip to content",
"hello",
"gone",
id = "default_conversion_unscoped_and_unstripped",
),
],
)
def test_hidden_subtrees_are_dropped_from_the_conversion(
html, present_first, present_second, absent
):
out = html_to_markdown(html)
assert present_first in out
assert present_second in out
assert absent not in out
@pytest.mark.parametrize(
"html, absent",
[
pytest.param(
'<body><p>keep</p><span aria-hidden="true">decoration</span></body>',
"decoration",
id = "aria_hidden_true_subtree_is_dropped",
),
pytest.param(
'<body><p>keep</p><span style="visibility:hidden">ghost</span></body>',
"ghost",
id = "inline_style_visibility_hidden_subtree_is_dropped",
),
pytest.param(
'<body><p>keep</p><div style="display:none !important">gone</div></body>',
"gone",
id = "inline_style_display_none_important_is_dropped",
),
pytest.param(
'<body><p>keep</p><div style="color: red; display : none ; margin:0">gone</div></body>',
"gone",
id = "inline_style_display_none_among_other_declarations",
),
# hidden is enumerated: invalid or empty values, including "false", still hide.
pytest.param(
'<body><p>keep</p><div hidden="false">not rendered</div></body>',
"not rendered",
id = "hidden_false_is_still_hidden",
),
],
)
def test_hidden_markers_other_than_the_attribute_are_dropped(html, absent):
out = html_to_markdown(html)
assert "keep" in out
assert absent not in out
def test_aria_hidden_false_subtree_is_kept():
html = '<body><span aria-hidden="false">still here</span></body>'
assert "still here" in html_to_markdown(html)
def test_inline_style_visible_display_is_kept():
html = (
"<body>"
'<div style="display:block">block kept</div>'
'<div style="visibility:visible">visible kept</div>'
'<a style="background:url(none.png)">link kept</a>'
"</body>"
)
out = html_to_markdown(html)
assert "block kept" in out
assert "visible kept" in out
assert "link kept" in out
@pytest.mark.parametrize(
"html, absent, present",
[
pytest.param(
"<body><div><p hidden>gone</div><p>kept</p></body>",
"gone",
"kept",
id = "hidden_recovers_from_omitted_close_tags",
),
pytest.param(
"<body><p hidden>secret<hr>kept text</body>",
"secret",
"kept text",
id = "hr_implicitly_closes_hidden_paragraph",
),
pytest.param(
"<body><table><tr>"
"<td hidden>outer<table><tr><td>secret cell</td></tr></table></td>"
"<td>visible cell</td>"
"</tr></table></body>",
"secret cell",
"visible cell",
id = "nested_hidden_table_does_not_leak_inner_cells",
),
],
)
def test_hidden_regions_end_at_the_implied_close_tag(html, absent, present):
out = html_to_markdown(html)
assert absent not in out
assert present in out
def test_nested_hidden_regions():
html = "<body><div hidden><div hidden>inner</div>outer</div><p>ok</p></body>"
out = html_to_markdown(html)
assert "inner" not in out
assert "outer" not in out
assert "ok" in out
def test_hidden_paragraph_omitted_close_does_not_swallow_siblings():
html = (
"<body><div><p hidden>secret<p>visible one</p><p>visible two</p></div><p>after</p></body>"
)
out = html_to_markdown(html)
assert "secret" not in out
assert "visible one" in out
assert "visible two" in out
assert "after" in out
@pytest.mark.parametrize(
"html, present_first, present_second",
[
pytest.param(
"<body><ul><li hidden>secret<li>shown A</li><li>shown B</li></ul></body>",
"shown A",
"shown B",
id = "hidden_list_item_omitted_close_keeps_following_items",
),
pytest.param(
"<body><p hidden><span>secret<div>visible div</div><p>visible paragraph</body>",
"visible div",
"visible paragraph",
id = "hidden_paragraph_with_inline_child_implicitly_closed_by_block",
),
pytest.param(
"<body><ul><li hidden><span>secret<li>visible item</ul><p>after</p></body>",
"visible item",
"after",
id = "hidden_list_item_with_inline_child_closed_by_next_item",
),
],
)
def test_hidden_regions_with_inline_children_end_at_the_implied_close(
html, present_first, present_second
):
out = html_to_markdown(html)
assert "secret" not in out
assert present_first in out
assert present_second in out
def test_skipped_tag_implicitly_closes_hidden_paragraph():
# optional-close bookkeeping must run before skipped blocks or an unclosed hidden paragraph hides later siblings
for skipped in ("nav", "footer"):
html = f"<body><p hidden>secret<{skipped}>chrome</{skipped}>VISIBLE</body>"
out = html_to_markdown(html)
assert "secret" not in out
assert "chrome" not in out
assert "VISIBLE" in out
@pytest.mark.parametrize("skipped", ["nav", "footer", "button"])
@pytest.mark.parametrize("main_content", [False, True])
def test_skipped_tag_closed_by_its_parent_keeps_the_rest_of_the_page(skipped, main_content):
html = (
f"<body><div><{skipped}><div><a href=/x>Menu</a></div></div>"
"<main><h1>Title</h1><p>Real text of the page that must reach the model.</p></main></body>"
)
out = html_to_markdown(html, main_content = main_content)
assert "Menu" not in out
assert "Real text of the page" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_html_block_start_breaks_out_of_svg(main_content):
html = (
"<body><div><svg><div><a href=/x>Menu</a></div></div>"
"<main><h1>Title</h1><p>Real text of the page.</p></main></body>"
)
out = html_to_markdown(html, main_content = main_content)
assert "Menu" in out
assert "Real text of the page" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_heading_button_after_foreign_breakout_is_kept(main_content):
html = "<svg><h1><button>Title</button></h1><p>Body text</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Title" in out and "Body text" in out
@pytest.mark.parametrize(
"outside",
[
"<div><template><p>Hidden</div><p>Template text</p></template></div>",
"<nav><template><p>Hidden</nav><p>Template text</p></template></nav>",
],
)
def test_end_tag_outside_a_template_keeps_it_skipped(outside):
out = html_to_markdown(outside + "<p>After</p>")
assert "Hidden" not in out and "Template text" not in out
assert "After" in out
@pytest.mark.parametrize(
"html",
[
"<form><nav>Hidden</form>Menu text</nav><p>After</p>",
"<span><nav>Hidden</span>Menu text</nav><p>After</p>",
"<a><nav>Hidden</a>Menu text</nav><p>After</p>",
"<b><nav>Hidden</b>Menu text</nav><p>After</p>",
"<div><select><option>Hidden</div>Menu text</select></div><p>After</p>",
(
"<div><table><tr><td><nav>Hidden</div>Menu text</nav></td></tr></table>"
"</div><p>After</p>"
),
"<nav><table><tr><td>Hidden</nav>Menu text</td></tr></table></nav><p>After</p>",
"<nav><select><option>Hidden</nav>Menu text</select></nav><p>After</p>",
"<body><nav>Hidden</body>Menu text</nav><p>After</p>",
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_ancestor_end_tag_that_does_not_pop_a_skipped_element_keeps_it_skipped(html, main_content):
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_matching_skipped_root_cannot_close_across_select_scope(main_content):
html = "<datalist><select><option>Hidden</datalist>Menu text</select></datalist><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_dialog_skip_does_not_recover_through_implicitly_closed_paragraph(main_content):
html = "<p>intro<dialog>Hidden</p>Menu text</dialog><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "intro" in out and "After" in out
@pytest.mark.parametrize("block", ["center", "dir", "listing", "search", "summary"])
@pytest.mark.parametrize("main_content", [False, True])
def test_block_start_closes_paragraph_before_skipped_form_chrome(block, main_content):
html = f"<p>intro<{block}><datalist>Hidden</p>Menu text</datalist></{block}><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "intro" in out and "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_paragraph_start_recovery_stops_at_button_scope_barrier(main_content):
html = "<p hidden><object><center>SECRET</center></object><p>AFTER</p>"
out = html_to_markdown(html, main_content = main_content)
assert "SECRET" not in out
assert "AFTER" in out
@pytest.mark.parametrize("table_tag", ["caption", "tbody", "td", "tfoot", "th", "thead", "tr"])
@pytest.mark.parametrize("main_content", [False, True])
def test_table_only_tag_outside_table_cannot_recover_skipped_content(table_tag, main_content):
html = f"<{table_tag}><nav>Hidden</{table_tag}>Menu text</nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_caption_start_closes_the_active_cell_before_skip_recovery(main_content):
html = "<table><tr><td><caption><nav>Hidden</td>Menu text</nav></caption></table><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_table_cell_start_reconciles_while_content_is_skipped(main_content):
html = "<table><tr><td><nav>Hidden<td></nav>Visible</td></tr></table>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "Visible" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_nested_button_start_replaces_outer_skipped_button(main_content):
html = "<button>Title<button>Inner</button><nav>Hidden</button>Menu text</nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Title" not in out and "Inner" not in out
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("foreign", ["<svg><g>", "<math><mrow>"])
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_button_start_does_not_close_html_button(foreign, main_content):
closing = "</g></svg>" if foreign.startswith("<svg") else "</mrow></math>"
html = (
f"<button>Title{foreign}<button>Inner</button>{closing}"
"<nav>Hidden</button>Menu text</nav><p>After</p>"
)
out = html_to_markdown(html, main_content = main_content)
assert "Title" not in out and "Inner" not in out and "Hidden" not in out
assert "Menu text" in out and "After" in out
@pytest.mark.parametrize("root, descendant", [("svg", "foreignObject"), ("math", "annotation-xml")])
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_root_end_closes_through_foreign_descendant(root, descendant, main_content):
html = f"<{root}><{descendant}>Hidden</{root}><p>Real text</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "Real text" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_end_tag_closes_through_integration_point(main_content):
html = "<svg><g><foreignObject></g><p>Visible</p></svg><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Visible" in out and "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_breakout_keeps_html_skipped_ancestor(main_content):
html = "<nav><svg><g><foreignObject></g><p>Hidden</p></nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "After" in out
@pytest.mark.parametrize(
"html",
[
"<svg><foreignObject><div>Hidden</svg>Menu text</div></foreignObject></svg>",
"<math><mtext><div>Hidden</math>Menu text</div></mtext></math>",
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_root_end_stops_at_html_integration_descendant(html, main_content):
out = html_to_markdown(html + "<p>After</p>", main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_generic_datalist_end_stops_at_special_descendant(main_content):
html = "<datalist><nav>Hidden</datalist>Menu text</nav></datalist><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("skipped", ["dialog", "footer", "nav"])
@pytest.mark.parametrize("main_content", [False, True])
def test_scoped_block_skip_end_pops_special_descendant(skipped, main_content):
html = f"<{skipped}><div>Hidden</{skipped}><main>Visible article</main>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "Visible article" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_nested_form_start_is_ignored_before_datalist_end(main_content):
html = "<form><datalist>Hidden<form>ignored</datalist><p>Visible article</p></form>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "ignored" not in out
assert "Visible article" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_form_end_clears_html_form_pointer(main_content):
html = "<form><svg><g>Hidden</form></g></svg><form hidden><p>SECRET</p></form><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "SECRET" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_matching_foreign_form_end_keeps_html_form_pointer(main_content):
html = "<form><svg><form>Hidden</form></svg><form hidden><p>SECRET</p></form><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "SECRET" in out and "After" in out
@pytest.mark.parametrize("item", ["dd", "dt", "li"])
@pytest.mark.parametrize("main_content", [False, True])
def test_list_item_start_closes_paragraph_before_skip_recovery(item, main_content):
html = f"<p>intro<{item}><datalist>Hidden</p>Menu text</datalist><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "intro" in out and "After" in out
@pytest.mark.parametrize("heading", ["h1", "h2", "h3", "h4", "h5", "h6"])
@pytest.mark.parametrize("main_content", [False, True])
def test_heading_start_closes_open_heading_before_skip_recovery(heading, main_content):
html = f"<{heading}>Title<h2>Sub</h2><nav>Hidden</{heading}>Menu text</nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Title" in out and "Sub" in out and "After" in out
assert "Hidden" not in out and "Menu text" not in out
@pytest.mark.parametrize("main_content", [False, True])
def test_heading_start_only_pops_a_current_heading(main_content):
html = "<h1><span>Title<h2>Sub</h2><nav>Hidden</h1>Menu text</nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Title" in out and "Sub" in out and "Menu text" in out and "After" in out
assert "Hidden" not in out
@pytest.mark.parametrize("main_content", [False, True])
def test_mismatched_heading_end_reconciles_before_skip_recovery(main_content):
html = "<h1>Title</h2><nav>Hidden</h1>Menu text</nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Title" in out and "After" in out
assert "Hidden" not in out and "Menu text" not in out
@pytest.mark.parametrize("main_content", [False, True])
def test_table_start_closes_active_table_outside_cell(main_content):
html = "<table><table><tr><td>X</td></tr></table><nav>Hidden</table>Menu text</nav><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "X" in out and "After" in out
assert "Hidden" not in out and "Menu text" not in out
@pytest.mark.parametrize("main_content", [False, True])
def test_table_start_pops_skipped_descendant_of_active_table(main_content):
html = "<table><nav>Hidden<table><tr><td>X</td></tr></table><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "X" in out and "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_table_start_inside_caption_remains_nested(main_content):
html = (
"<table><caption><nav>Hidden<table><tr><td>X</td></tr></table>"
"Menu text</nav></caption></table><p>After</p>"
)
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "X" not in out and "Menu text" not in out
assert "After" in out
@pytest.mark.parametrize("main_content", [False, True])
def test_custom_select_keeps_table_and_following_paragraph_suppressed(main_content):
html = "<select><table>Hidden</select><p>Real text</p>"
assert html_to_markdown(html, main_content = main_content) == ""
@pytest.mark.parametrize(
"html",
[
"<section><nav>Hidden</section><p>Visible text</p>",
"<ul><li><nav>Hidden</li><li>Visible text</li></ul>",
"<table><tr><td><nav>Hidden</td><td>Visible text</td></tr></table>",
"<h1><nav>Hidden</h2><p>Visible text</p>",
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_ancestor_end_tag_that_pops_a_skipped_element_ends_suppression(html, main_content):
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out
assert "Visible text" in out
@pytest.mark.parametrize(
"opening",
["<blockquote>", "<pre>", "<a href='/real'>", "<ol><li>"],
)
def test_recovered_ancestor_flushes_open_content_buffers_before_scoring(opening):
real = "real article prose " * 20
decoy = "unrelated decoy prose " * 20
html = f"<article>{opening}{real}<nav>Hidden</article><main><p>{decoy}</p></main>"
out = html_to_markdown(html, main_content = True)
assert "real article prose" in out
assert "unrelated decoy prose" not in out
def test_mismatched_heading_recovery_closes_the_open_heading_before_scoring():
real = "real article prose " * 20
decoy = "unrelated decoy prose " * 20
html = (
f"<article><h1>Real title<nav>Hidden</h2><p>{real}</p></article>"
f"<main><p>{decoy}</p></main>"
)
out = html_to_markdown(html, main_content = True)
assert "real article prose" in out
assert "unrelated decoy prose" not in out
@pytest.mark.parametrize(
"foreign",
[
"<svg><foreignObject><p>Hidden</div>Menu text</p></foreignObject></svg>",
"<svg><desc><p>Hidden</div>Menu text</p></desc></svg>",
"<svg><title>Hidden</div>Menu text</title></svg>",
"<math><mi><p>Hidden</div>Menu text</p></mi></math>",
"<math><mo><p>Hidden</div>Menu text</p></mo></math>",
"<math><mn><p>Hidden</div>Menu text</p></mn></math>",
"<math><ms><p>Hidden</div>Menu text</p></ms></math>",
"<math><mtext><p>Hidden</div>Menu text</p></mtext></math>",
'<math><annotation-xml encoding="text/html"><p>Hidden</div>Menu text</p>'
"</annotation-xml></math>",
'<math><annotation-xml encoding="application/xhtml+xml"><p>Hidden</div>Menu text</p>'
"</annotation-xml></math>",
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_foreign_content_scope_barriers_keep_skipped_text_hidden(foreign, main_content):
html = f"<div>{foreign}</div><p>After</p>"
out = html_to_markdown(html, main_content = main_content)
assert "Hidden" not in out and "Menu text" not in out
assert "After" in out
def test_html_element_named_like_foreign_barrier_does_not_block_recovery():
html = "<div><nav><foreignobject>Hidden</div>Visible</foreignobject></nav><p>After</p>"
out = html_to_markdown(html)
assert "Hidden" not in out
assert "Visible" in out and "After" in out
@pytest.mark.parametrize(
"build",
[
lambda n: "<body><nav><ul>" + "<li>x" * n + "</a>" * n + "</ul></nav>",
lambda n: "<body>"
+ "<div>" * n
+ "<nav><template>x"
+ "</div>" * n
+ "</template>"
+ "</span>" * n,
],
)
def test_stray_end_tags_in_a_skipped_menu_stay_linear(build):
import time
timings = []
for count in (1000, 8000):
html = build(count)
runs = []
# Best of five: one descheduled call on a shared runner must not read as growth.
for _ in range(5):
start = time.perf_counter()
html_to_markdown(html, main_content = True)
runs.append(time.perf_counter() - start)
timings.append(min(runs))
# eight times the input should remain well below quadratic growth
assert timings[1] < timings[0] * 24, timings
@pytest.mark.parametrize(
"html, heading",
[
(
'<h3 data-state="closed"><button type="button" aria-controls="r1" aria-expanded="false">'
'What is the right plan for me?<span aria-hidden="true">v</span></button></h3>'
'<div id="r1" role="region"><p>Answer text.</p></div>',
"### What is the right plan for me?",
),
(
'<h2 class="accordion-header">\n <button class="accordion-button" type="button" '
'aria-expanded="true" aria-controls="c1">\n What is the right plan for me?\n </button>\n</h2>'
'<div id="c1"><div class="accordion-body">Answer text.</div></div>',
"What is the right plan for me?",
),
(
"<h3><button>What is the right plan for me?</button></h3><p>Answer text.</p>",
"### What is the right plan for me?",
),
(
'<div role="heading" aria-level="3"><button aria-expanded="false">'
"What is the right plan for me?</button></div><p>Answer text.</p>",
"What is the right plan for me?",
),
],
)
def test_accordion_question_in_a_heading_button_is_kept(html, heading):
out = html_to_markdown(html)
assert heading in out
assert "Answer text." in out
def test_buttons_that_are_not_a_heading_title_are_still_dropped():
html = (
"<h4><span>Create Artifacts</span><button aria-expanded='false'>"
"<span class='sr-only'>More information</span></button></h4>"
"<p>Body text.</p><button>Subscribe</button>"
)
out = html_to_markdown(html)
assert "#### Create Artifacts" in out
assert "Body text." in out
assert "More information" not in out
assert "Subscribe" not in out
@pytest.mark.parametrize(
"html",
[
"<h4>Create<button>More information</button></h4><p>Body text.</p>",
"<h4>é<button>More information</button></h4><p>Body text.</p>",
"<hgroup><h1>Create</h1><h2></h2><button>More information</button></hgroup><p>Body text.</p>",
],
)
def test_a_button_after_entity_or_nested_heading_text_is_dropped(html):
out = html_to_markdown(html)
assert "Body text." in out
assert "More information" not in out
@pytest.mark.parametrize(
"between", ["", "<!-- </h4> -->", "<span hidden>" + "x" * 5000 + "</span>"]
)
def test_a_leading_control_before_the_heading_title_is_dropped(between):
html = (
f"<h4><button aria-expanded='false'>More information</button>{between}<span>Create Artifacts</span></h4>"
"<p>Body text.</p>"
)
out = html_to_markdown(html)
assert "#### Create Artifacts" in out
assert "More information" not in out
@pytest.mark.parametrize(
"html, heading",
[
(
"<hgroup><h1><button>Question</button></h1><p>Sub</p></hgroup><p>Answer text.</p>",
"# Question",
),
(
"<h3><button hidden>old</button><button>Question</button></h3><p>Answer text.</p>",
"### Question",
),
(
"<h3><span hidden><button>old</button></span><button>Question</button></h3><p>Answer text.</p>",
"### Question",
),
("<h3><button>Question</button><br></h3><p>Answer text.</p>", "### Question"),
("<h3><em><button>Question</button></em></h3><p>Answer text.</p>", "### *Question*"),
(
"<h3><button aria-label='Settings'><svg><path/></svg></button><button>Question</button></h3>"
"<p>Answer text.</p>",
"### Question",
),
("<ul><li><h3><button>Question</button><li>Answer text.</ul>", "### Question"),
(
"<ul><li hidden>old<li><h3><button>Question</button></h3>Answer text.</ul>",
"### Question",
),
("<p hidden>old<h3><button>Question</button></h3><p>Answer text.</p>", "### Question"),
(
'<script>const x="<h3><button>"</script><!-- <h2><button> -->'
"<h3><button>Question</button></h3><p>Answer text.</p>",
"### Question",
),
("<h3><!-- </h3> --><button>Question</button></h3><p>Answer text.</p>", "### Question"),
(
"<title-card>Card</title-card><h3><button>Question</button></h3><p>Answer text.</p>",
"### Question",
),
(
'<div data-example="<script>"></div><h3 data-template="</h3>"><button>Question</button></h3>'
"<p>Answer text.</p>",
"### Question",
),
("<p>a\nb</p>\n<h3>\n <button>Question</button></h3><p>Answer text.</p>", "Question"),
(
'<div role="presentation heading" aria-level="3"><button>Question</button></div><p>Answer text.</p>',
"Question",
),
],
)
def test_heading_button_title_edge_cases(html, heading):
out = html_to_markdown(html)
assert heading in out
assert "Answer text." in out
assert "old" not in out
@pytest.mark.parametrize(
"cut", ["<h3><button>Question", "<h3><button>Question</button>", "<h3><button>Question<svg>"]
)
def test_heading_button_cut_by_the_fetch_cap_keeps_its_title(cut):
assert "### Question" in html_to_markdown("<p>Answer text.</p>" + cut)
@pytest.mark.parametrize("title", ["<em>Question</em> one", "First<br>Second", "Q & A"])
def test_a_heading_button_title_renders_like_the_plain_heading(title):
plain = html_to_markdown(f"<h3>{title}</h3><p>Answer text.</p>")
assert html_to_markdown(f"<h3><button>{title}</button></h3><p>Answer text.</p>") == plain
def test_visible_void_hr_still_renders():
html = "<body><p>a</p><hr><p>b</p></body>"
out = html_to_markdown(html)
assert "---" in out
@pytest.mark.parametrize(
"html, expected",
[
(
"<ul>\n<li>\n<p>First item</p>\n</li>\n<li>\n<p>Second item</p>\n</li>\n</ul>",
"* First item\n\n* Second item",
),
("<ol><li><p>One</p></li><li><div>Two</div></li></ol>", "1. One\n\n2. Two"),
("<ul><li><p>a</p><ul><li><p>b</p></li></ul></li></ul>", "* a\n\n * b"),
("<ul><li></li><p>outside</p></ul>", "*\n\noutside"),
],
)
def test_block_opening_list_item_stays_on_marker_line(html, expected):
assert html_to_markdown(html) == expected
def test_empty_header_does_not_consume_the_list_marker():
html = "<ul><li><header></header><p>text text text</p></li></ul>"
assert html_to_markdown(html, main_content = True) == "* text text text"
def test_github_page_main_content_keeps_readme_only():
out = html_to_markdown(_GITHUB_PAGE, main_content = True)
assert "Unsloth Studio" in out
assert "install.sh" in out
assert "documentation" in out
assert "Uh oh!" not in out
assert "There was an error while loading" not in out
assert "Please reload this page" not in out
assert "You can't perform that action at this time" not in out
assert "Skip to content" not in out
assert "Sign in" not in out
assert "Reload to refresh your session" not in out
assert "JavaScript 89.3%" not in out
assert "Languages" not in out
assert "Last commit message" not in out
def test_main_scope_used_when_no_article():
html = """
<body>
<header><a href="/login">Sign in</a></header>
<main><h1>Doc title</h1><p>%s</p></main>
<footer>footer junk</footer>
</body>
""" % ("Body text. " * 40)
out = html_to_markdown(html, main_content = True)
assert "Doc title" in out
assert "Body text." in out
assert "Sign in" not in out
assert "footer junk" not in out
def test_main_content_falls_back_to_full_document():
html = "<body><h1>Tiny</h1><p>Just a short page.</p></body>"
out = html_to_markdown(html, main_content = True)
assert "Tiny" in out
assert "Just a short page." in out
def test_tiny_article_stub_does_not_hijack_scope():
body_text = "Real content paragraph. " * 30
html = f"<body><article>ad</article><main><p>{body_text}</p></main></body>"
out = html_to_markdown(html, main_content = True)
assert "Real content paragraph." in out
def test_sibling_articles_do_not_leak_after_main_selected():
real = "Main article body content for selection. " * 20
card = "Unrelated related-post card teaser blurb. " * 3
cards = "".join(f"<article><p>{card}</p></article>" for _ in range(5))
html = f"<body><article><h1>Real</h1><p>{real}</p></article>{cards}</body>"
out = html_to_markdown(html, main_content = True)
assert "Main article body content" in out
assert "Unrelated related-post" not in out
def test_boilerplate_filter_preserves_phrase_inside_real_prose():
body = (
"<article><h1>Authentication</h1>"
"<p>We use cookies to authenticate API requests and keep sessions safe.</p>"
"<p>%s</p></article>"
) % ("Additional documentation content to select the article. " * 8)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "We use cookies to authenticate API requests" in out
def test_boilerplate_filter_still_drops_standalone_and_stacked_furniture():
body = (
"<article>"
"<p>Skip to content</p>"
"<p>You signed in with another tab or window. Reload to refresh your session.</p>"
"<p>Real README body. %s</p>"
"</article>"
) % ("Genuine documentation text. " * 8)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Real README body." in out
assert "Skip to content" not in out
assert "Reload to refresh your session" not in out
def test_boilerplate_not_stripped_inside_code_fences():
html = (
"<body><article><p>%s</p>"
"<pre>assert 'There was an error while loading' in page</pre>"
"</article></body>" % ("Prose. " * 40)
)
out = html_to_markdown(html, main_content = True)
assert "There was an error while loading" in out
def test_aside_callout_inside_article_is_kept():
body = (
"<article><h1>Guide</h1>"
"<p>%s</p>"
"<aside class='admonition warning'><strong>Warning:</strong> "
"This operation is destructive and cannot be undone.</aside>"
"<p>Trailing paragraph.</p></article>"
) % ("Documentation body text to select the article scope. " * 6)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "This operation is destructive and cannot be undone." in out
assert "Warning:" in out
out_full = html_to_markdown(f"<body>{body}</body>")
assert "This operation is destructive and cannot be undone." in out_full
def test_github_repo_url_maps_to_readme_api():
assert (
_github_repo_readme_api_url("https://github.com/unslothai/unsloth")
== "https://api.github.com/repos/unslothai/unsloth/readme"
)
assert (
_github_repo_readme_api_url("https://github.com/unslothai/unsloth/")
== "https://api.github.com/repos/unslothai/unsloth/readme"
)
assert (
_github_repo_readme_api_url("http://www.github.com/owner/repo.git")
== "https://api.github.com/repos/owner/repo/readme"
)
def test_github_non_repo_urls_are_not_rewritten():
for url in (
"https://github.com/unslothai/unsloth/tree/main/studio",
"https://github.com/unslothai/unsloth/issues/123",
"https://github.com/topics/llm",
"https://github.com/orgs/unslothai/repositories",
"https://github.com/login/oauth",
"https://github.com/unslothai",
"https://example.com/owner/repo",
"https://raw.githubusercontent.com/owner/repo/main/README.md",
):
assert _github_repo_readme_api_url(url) is None, url
def test_fetch_page_text_prefers_github_readme(monkeypatch):
calls = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
):
calls.append((url, extra_headers))
assert url == "https://api.github.com/repos/unslothai/unsloth/readme"
return None, "# Unsloth\n\nFine-tune LLMs faster.", "text/plain"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://github.com/unslothai/unsloth")
assert "Fine-tune LLMs faster." in out
assert "README of https://github.com/unslothai/unsloth" in out
assert len(calls) == 1
assert calls[0][1]["Accept"] == "application/vnd.github.raw+json"
def test_fetch_page_text_keeps_html_readme_from_api(monkeypatch):
html_readme = (
"<!doctype html><html><body>"
"<h1>Project Title</h1>"
"<p>Install with the one-line script and read the docs.</p>"
"</body></html>"
)
calls = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
):
calls.append(url)
assert url == "https://api.github.com/repos/unslothai/unsloth/readme"
return None, html_readme, "text/html"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://github.com/unslothai/unsloth")
assert "README of https://github.com/unslothai/unsloth" in out
assert "Project Title" in out
assert "Install with the one-line script" in out
assert "<html" not in out
assert len(calls) == 1
def test_fetch_page_text_falls_back_to_html_when_readme_api_fails(monkeypatch):
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
if url.startswith("https://api.github.com/"):
return "Failed to fetch URL: HTTP 403 rate limited", "", ""
return None, _GITHUB_PAGE, "text/html"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://github.com/unslothai/unsloth")
assert "Unsloth Studio" in out
assert "Uh oh!" not in out
assert "There was an error while loading" not in out
def test_fetch_page_text_reads_github_file_link_as_raw_file(monkeypatch):
source = "<!doctype html>\n<html>\n <body><script>run()</script></body>\n</html>\n"
calls = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
calls.append(url)
if url.startswith("https://raw.githubusercontent.com/"):
return None, source, "text/plain"
return None, _GITHUB_PAGE, "text/html"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text(
"https://github.com/unslothai/unsloth/blob/main/studio/frontend/index.html?plain=1#L3"
)
assert calls == [
"https://raw.githubusercontent.com/unslothai/unsloth/main/studio/frontend/index.html"
]
assert out == source.strip()
def test_fetch_page_text_falls_back_to_github_file_page_when_raw_fails(monkeypatch):
calls = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
calls.append(url)
if url.startswith("https://raw.githubusercontent.com/"):
return "Failed to fetch URL: HTTP 404 Not Found", "", ""
return None, _GITHUB_PAGE, "text/html"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://github.com/owner/repo/blob/v1.2/src/app.py")
assert calls == [
"https://raw.githubusercontent.com/owner/repo/v1.2/src/app.py",
"https://github.com/owner/repo/blob/v1.2/src/app.py",
]
assert "Unsloth Studio" in out
def test_fetch_page_text_keeps_semicolon_in_github_file_name(monkeypatch):
calls = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
calls.append(url)
return None, "file body", "text/plain"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
_fetch_page_text("https://github.com/owner/repo/blob/main/t/foo;bar.py")
assert calls == ["https://raw.githubusercontent.com/owner/repo/main/t/foo;bar.py"]
@pytest.mark.parametrize(
"url",
[
"https://github.com/unslothai/unsloth",
"https://github.com/unslothai/unsloth/tree/main/studio",
"https://github.com/unslothai/unsloth/blob/main",
"https://github.com/unslothai/unsloth/issues/123",
"https://example.com/owner/repo/blob/main/app.py",
],
)
def test_fetch_page_text_leaves_other_github_urls_on_their_page(monkeypatch, url):
calls = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
calls.append(url)
return "Failed to fetch URL: HTTP 403", "", ""
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
_fetch_page_text(url)
assert not any("raw.githubusercontent.com" in c for c in calls)
assert calls[-1] == url
_RAW_TEXT_PAGE = "line one\n indented code\nline three"
_HTML_FRAGMENT = "<article><h1>Doc Title</h1><p>Readable fragment body.</p></article>"
def _page_text(monkeypatch, url, body, content_type):
"""``_fetch_page_text`` with the fetch stubbed to answer `body` under `content_type`."""
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
return None, body, content_type
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
return _fetch_page_text(url)
@pytest.mark.parametrize(
"url, body, content_type, present, absent",
[
pytest.param(
"https://raw.githubusercontent.com/o/r/main/file.txt",
_RAW_TEXT_PAGE,
"text/plain",
[" indented code"],
[],
id = "fetch_page_text_non_html_returned_raw",
),
pytest.param(
"https://github.com/unslothai/unsloth/tree/main",
_GITHUB_PAGE,
"text/html",
["Unsloth Studio"],
["Uh oh!"],
id = "fetch_page_text_html_conversion",
),
pytest.param(
"https://example.com/fragment",
_HTML_FRAGMENT,
"",
["Doc Title", "Readable fragment body."],
["<article"],
id = "fetch_page_text_missing_content_type_fragment_converted",
),
pytest.param(
"https://example.com/no-content-type.txt",
_RAW_TEXT_PAGE,
"",
[" indented code"],
[],
id = "fetch_page_text_missing_content_type_plain_text_raw",
),
pytest.param(
"https://example.com/mislabeled",
_GITHUB_PAGE,
"text/plain",
["Unsloth Studio"],
["<html"],
id = "fetch_page_text_mislabeled_text_plain_html_converted",
),
],
)
def test_fetch_page_text_content_type_handling(
monkeypatch, url, body, content_type, present, absent
):
out = _page_text(monkeypatch, url, body, content_type)
for fragment in present:
assert fragment in out
for fragment in absent:
assert fragment not in out
def test_fetch_page_text_propagates_fetch_errors(monkeypatch):
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
return "Failed to fetch URL: HTTP 404 Not Found", "", ""
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
assert _fetch_page_text("https://example.com/missing") == (
"Failed to fetch URL: HTTP 404 Not Found"
)
_WIKI_URL = "https://en.wikipedia.org/wiki/Python_(programming_language)"
def _wiki_article(paragraphs):
body = "".join(
f'<p>Paragraph {i} links <a href="https://en.wikipedia.org/wiki/Some_Long_Article_Title_{i}">'
f'topic {i}</a>, <a href="/wiki/Another_Related_Page_{i}">a related page</a> and a note'
f'<sup><a href="#cite_note-{i}">[{i}]</a></sup>.</p>'
for i in range(paragraphs)
)
return (
f"<html><body><main><article>{body}"
"<p>Python 3.0 was released on 3 December 2008.</p>"
'<p>See <a href="https://www.python.org/doc/">the docs</a>.</p>'
"</article></main></body></html>"
)
def test_long_page_spends_its_budget_on_text_not_links_into_the_site(monkeypatch):
out = _page_text(monkeypatch, _WIKI_URL, _wiki_article(150), "text/html")
assert "Python 3.0 was released on 3 December 2008." in out
assert "topic 149, a related page and a note[149]." in out
assert "[the docs](https://www.python.org/doc/)" in out
assert "wikipedia.org" not in out
assert "/wiki/" not in out
assert "#cite_note" not in out
def test_page_that_fits_keeps_its_links(monkeypatch):
out = _page_text(monkeypatch, _WIKI_URL, _wiki_article(3), "text/html")
assert "[topic 1](https://en.wikipedia.org/wiki/Some_Long_Article_Title_1)" in out
assert "[a related page](/wiki/Another_Related_Page_1)" in out
assert "[[1]](#cite_note-1)" in out
@pytest.mark.parametrize(
"markup, expected",
[
("(2<sup>53</sup> – 1)", "(2^53 – 1)"),
("2<sup>−52</sup>", "2^−52"),
("1.898×10<sup>27</sup> kg", "1.898×10^27 kg"),
("6.02214076×10<sup>23</sup> mol<sup>−1</sup>", "6.02214076×10^23 mol^−1"),
("2<sup><i>n</i>+1</sup> nodes", "2^(*n*+1) nodes"),
("2<sup>n + 1</sup>", "2^(n + 1)"),
("the 1<sup>st</sup> and 2<sup>nd</sup>", "the 1st and 2nd"),
("Intel<sup>®</sup> Core<sup>™</sup> i7", "Intel® Core™ i7"),
("now $19<sup>.99</sup> only", "now $19.99 only"),
("price<sup>*</sup> and terms<sup>†</sup>", "price* and terms†"),
("10<sup>6 </sup>years", "10^6 years"),
("Add <sup>1</sup>⁄<sub>2</sub> cup", "Add 1⁄2 cup"),
("Add 1 <sup>1</sup>/<sub>2</sub> cups", "Add 1 1/2 cups"),
("<sup>1</sup> Footnote text", "1 Footnote text"),
("Add <em><sup>1</sup></em>⁄ cup", "Add *1*⁄ cup"),
("the 1<sup><em>st</em></sup> one", "the 1*st* one"),
("a<b><sup>2</sup></b>", "a**^2**"),
("x<sup>n×2</sup>", "x^(n×2)"),
("x<sup>2n</sup> and y<sup>n2</sup>", "x^(2n) and y^(n2)"),
("x<sup>2<em>n</em></sup>", "x^(2*n*)"),
("x<sup>2*3</sup> and x<sup><em>n</em></sup>", "x^(2*3) and x^*n*"),
("e<sup>iπ</sup> + 1 = 0", "e^(iπ) + 1 = 0"),
("now $19<sup>99</sup> or € 1,299<sup>95</sup>", "now $1999 or € 1,29995"),
("<b>$19</b><sup>99</sup>", "**$19**99"),
("$<b>19</b><sup>99</sup>", "$**19**99"),
("Brand<sup>TM</sup> and Service<sup>SM</sup>", "BrandTM and ServiceSM"),
("el 1<sup>º</sup> y la 2<sup>ª</sup>, x<sup>a</sup>", "el 1º y la 2ª, x^a"),
(
"le XV<sup>e</sup> siècle, François I<sup>er</sup>, MAX<sup>e</sup>",
"le XVe siècle, François Ier, MAX^e",
),
("<span>$</span> <b>19</b><sup>99</sup>", "$ **19**99"),
(
"₺19<sup>99</sup> or ₱19<sup>99</sup>, a19<sup>2</sup>",
"₺1999 or ₱1999, a19^2",
),
(
"CHF 19<sup>95</sup> or USD 19<sup>99</sup>, ABC 10<sup>3</sup>",
"CHF 1995 or USD 1999, ABC 10^3",
),
(
"AED 19<sup>99</sup>, TWD 19<sup>99</sup>, ILS 19<sup>99</sup>",
"AED 1999, TWD 1999, ILS 1999",
),
(
"ZWG 19<sup>99</sup>, les 1<sup>ers</sup> et 1<sup>res</sup>",
"ZWG 1999, les 1ers et 1res",
),
("$2<sup>n</sup> and USD 10<sup>6</sup>, $19<sup>99</sup>", "$2^n and USD 10^6, $1999"),
(
'XCG 19<sup>99</sup>, claim<sup role="doc-noteref presentation">2</sup>',
"XCG 1999, claim2",
),
("le P<sup>r</sup> Martin et les P<sup>rs</sup>", "le Pr Martin et les Prs"),
("CHF 1’299<sup>95</sup> or CHF 1'299<sup>95</sup>", "CHF 1’29995 or CHF 1'29995"),
("Marque<sup>MC</sup> et Produit<sup>MD</sup>", "MarqueMC et ProduitMD"),
("Brand<sup>(TM)</sup> and Other<sup>(R)</sup>", "Brand(TM) and Other(R)"),
(
"KWD 19<sup>950</sup>, BHD 1<sup>234</sup>, USD 10<sup>100</sup>",
"KWD 19950, BHD 1234, USD 10^100",
),
("M<sup>r</sup>, M<sup>s</sup> and M<sup>rs</sup> Smith", "Mr, Ms and Mrs Smith"),
("JPY 10<sup>12</sup> and KRW 10<sup>12</sup>", "JPY 10^12 and KRW 10^12"),
("les 1<sup>ères</sup> places", "les 1ères places"),
("John J<sup>r</sup> and John S<sup>r</sup>", "John Jr and John Sr"),
("VED 19<sup>99</sup> and S<sup>T</sup>", "VED 1999 and S^T"),
(
'claim<sup class="footnote-reference">2</sup>, la 2<sup>de</sup>, le 2<sup>d</sup>',
"claim2, la 2de, le 2d",
),
(
"la P<sup>re</sup> Durand, 12<sup>bis</sup> rue, article 3<sup>ter</sup>",
"la Pre Durand, 12bis rue, article 3ter",
),
(
'claim<sup class="citation">2</sup> and fact<sup class="endnote-ref">3</sup>',
"claim2 and fact3",
),
("la D<sup>re</sup> Roy et les D<sup>res</sup>", "la Dre Roy et les Dres"),
(
'x<sup class="excited">2</sup> and 10<sup>.5</sup>, $19<sup>.99</sup>',
"x^2 and 10^(.5), $19.99",
),
(
'claim<sup class="fnref">2</sup>, fact<sup class="fn-ref">3</sup>, Marca<sup>MR</sup>',
"claim2, fact3, MarcaMR",
),
(
"les S<sup>ts</sup>, les S<sup>tes</sup> et les C<sup>ies</sup>",
"les Sts, les Stes et les Cies",
),
("les 2<sup>nds</sup>", "les 2nds"),
("(a+b)<sup>2</sup> and 「引用」<sup>1</sup>", "(a+b)^2 and 「引用」1"),
(
'€19<sup>,99</sup> and claim<sup class="reference-number">2</sup>',
"€19,99 and claim2",
),
(
"SVC 19<sup>99</sup>, claim<sup>1–3</sup>, fact<sup>2,5</sup>, 10<sup>-3</sup>",
"SVC 1999, claim1–3, fact2,5, 10^-3",
),
("x<sup>2,5</sup> and 10<sup>1,000</sup>", "x^2,5 and 10^1,000"),
("₫10<sup>12</sup> and ₩10<sup>12</sup>", "₫10^12 and ₩10^12"),
("claim<sup>1,3–5</sup>", "claim1,3–5"),
("Doors 9:30<sup>pm</sup>, ends 11<sup>p.m.</sup>", "Doors 9:30pm, ends 11p.m."),
("10<sup>6–8</sup> CFU/mL and 10<sup>d</sup>", "10^(6–8) CFU/mL and 10^d"),
("(x+1)<sup>2–4</sup>", "(x+1)^(2–4)"),
(
"M<sup>me</sup> Dupont, D<sup>r</sup> Martin, n<sup>o</sup> 5, Om<sup>e</sup>",
"Mme Dupont, Dr Martin, no 5, Om^e",
),
("€1.299<sup>95</sup> or €1 299<sup>95</sup>", "€1.29995 or €1 29995"),
(
'claim<sup role="doc-noteref">1</sup> and fact<sup class="footnote">2</sup>',
"claim1 and fact2",
),
("x<sup>-n</sup> and 10<sup>2.5</sup>", "x^-n and 10^2.5"),
("now $19<sup><em>.99</em></sup>", "now $19*.99*"),
("A fact.<sup>1</sup> Next, a list,<sup>2</sup>", "A fact.1 Next, a list,2"),
("(a+b)<sup>2</sup> and km<sup>2</sup>.", "(a+b)^2 and km^2."),
("le 1<sup>er</sup> mai, 2<sup>e</sup> et x<sup>e</sup>", "le 1er mai, 2e et x^e"),
],
)
def test_superscripts_keep_their_exponent(markup, expected):
assert html_to_markdown(f"<p>{markup}</p>") == expected
@pytest.mark.parametrize(
"title",
[
"<h1><a href='/p'>E=mc<sup>2</sup></a></h1>",
"<a href='/p'><h1>E=mc<sup>2</sup></h1></a>",
],
)
def test_linked_header_title_keeps_one_exponent(title):
nav = "".join(f"<a href='/s{i}'>Section number {i}</a> " for i in range(12))
html = f"<header>{title}{nav}</header><p>{'Body text here. ' * 40}</p>"
out = html_to_markdown(html, main_content = True)
assert out.count("E=mc") == 1
assert "E=mc^2](/p)" in out
def test_stripped_header_keeps_the_exponent_in_its_heading():
nav = "".join(f"<a href='/s{i}'>Section number {i}</a> " for i in range(12))
html = f"<header><h1>E=mc<sup>2</sup></h1>{nav}</header><p>{'Body text here. ' * 40}</p>"
out = html_to_markdown(html, main_content = True)
assert "# E=mc^2" in out
assert "E=mc2" not in out
def test_deeply_nested_superscripts_track_a_bounded_depth():
out = html_to_markdown("<p>x" + "<sup>a" * 50 + "</sup>" * 50 + "</p>")
assert out.translate(str.maketrans("", "", "^()")) == "x" + "a" * 50
assert out.count("^") == 8
def test_linked_footnote_wrapping_its_superscript_renders_unchanged():
html = '<p>text<a role="doc-noteref" href="#fn1"><sup>1</sup></a> more</p>'
assert html_to_markdown(html) == "text[1](#fn1) more"
def test_exponent_headings_do_not_read_as_body_prose():
heads = "".join(f"<h2>x<sup>n+{i}</sup></h2>" for i in range(67))
body = "<p>" + "Real document body sentence. " * 12 + "</p>"
html = f"<html><body><article>{heads}</article><div>{body}</div></body></html>"
assert "Real document body" in html_to_markdown(html, main_content = True)
def test_same_site_linked_base_keeps_its_exponent():
site_links = SiteLinks("https://e.com/page")
html = '<p>5 <a href="/metre">m</a><sup>2</sup> and <a href="https://x.org/m">m</a><sup>3</sup></p>'
out = site_links.clean(html_to_markdown(html, site_links = site_links))
assert out == "5 [m](/metre)^2 and [m](https://x.org/m)^3"
def test_empty_superscripts_do_not_rescan_the_page():
start = time.perf_counter()
html_to_markdown("<p>" + "<i></i><sup></sup>" * 8000 + "</p>")
assert time.perf_counter() - start < 2
start = time.perf_counter()
html_to_markdown("<p>" + "*" * 250000 + "<sup></sup>" * 23000 + "</p>")
assert time.perf_counter() - start < 3
start = time.perf_counter()
html_to_markdown("<p>claim<sup>" + "1–" * 16000 + "x</sup></p>")
assert time.perf_counter() - start < 2
start = time.perf_counter()
html_to_markdown(
"<p>" + "<b>" + "word 1 " * 40000 + "</b>" * 1 + "9" + "<sup></sup>" * 20000 + "</p>"
)
assert time.perf_counter() - start < 3
def test_unclosed_superscripts_do_not_exhaust_the_depth_cap():
html = "<p>x<sup>2</p>" * 10 + "<p>1.898×10<sup>27</sup> kg</p>"
assert html_to_markdown(html).endswith("1.898×10^27 kg")
def test_untracked_superscripts_keep_the_stack_bounded():
from core.inference._html_to_md import _MarkdownRenderer
renderer = _MarkdownRenderer()
renderer.feed("<p>" + "<sup>" * 5000 + "x")
assert len(renderer._sup_starts) <= 8
def test_footnote_superscripts_render_unchanged():
html = (
'<p>mass<sup class="reference"><a href="#cite_note-12">[12]</a></sup> and '
"volume<sup>[13]</sup></p>"
)
assert html_to_markdown(html) == "mass[[12]](#cite_note-12) and volume[13]"
def test_fetched_page_keeps_exponents_beside_footnotes(monkeypatch):
body = (
"<html><body><main><article><p>Jupiter has a mass of 1.898×10<sup>27</sup> kg"
'<sup class="reference"><a href="#cite_note-12">[12]</a></sup> and a surface area of '
"6.1419×10<sup>10</sup> km<sup>2</sup>.</p></article></main></body></html>"
)
out = _page_text(monkeypatch, "https://en.wikipedia.org/wiki/Jupiter", body, "text/html")
assert "1.898×10^27 kg[[12]](#cite_note-12) and a surface area of 6.1419×10^10 km^2." in out
@pytest.mark.parametrize(
"markup, expected",
[
("<p>R&D and Q&A at AT&T.</p>", "R&D and Q&A at AT&T."),
# &format is no reference and &lang is one only with its ";", so both stay as written
(
"<p>See https://example.com/v1?key=abc&format=json&lang=en for details.</p>",
"See https://example.com/v1?key=abc&format=json&lang=en for details.",
),
(
"<pre>curl 'https://example.com/v1?api=1&map_action=map&b=2'</pre>",
"```\ncurl 'https://example.com/v1?api=1&map_action=map&b=2'\n```",
),
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_a_bare_ampersand_stays_as_written(markup, expected, main_content):
assert html_to_markdown(markup, main_content = main_content) == expected
@pytest.mark.parametrize(
"markup, expected",
[
# a decimal reference may drop its ";", even before a letter a-f
(
"<p>©Copyright 2026</p><h2>Next</h2><p>More & text.</p>",
"©Copyright 2026\n\n## Next\n\nMore & text.",
),
# "&#" without digits is text, with or without a ";" later in the page
(
"<p>Use &# here</p><h2>Next</h2><p>More text.</p>",
"Use &# here\n\n## Next\n\nMore text.",
),
(
"<p>C&#; and C&#x; typos</p><h2>Next</h2><p>More text.</p>",
"C&#; and C&#x; typos\n\n## Next\n\nMore text.",
),
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_a_malformed_numeric_reference_keeps_the_rest_of_the_page(markup, expected, main_content):
assert html_to_markdown(markup, main_content = main_content) == expected
@pytest.mark.parametrize("main_content", [False, True])
def test_references_decode_as_a_browser_shows_them(main_content):
# legacy names such as & and ¬ decode without their ";" in text
html = "<p><b> & © AB & x ¬it; &foo;</p>"
assert html_to_markdown(html, main_content = main_content) == "<b> & © AB & x ¬it; &foo;"
@pytest.mark.parametrize(
"markup, expected",
[
# and a literal U+00A0 are both kept, as a browser keeps them
("<p>10 GB and 10\xa0GB</p>", "10\xa0GB and 10\xa0GB"),
("<div>for x in y:<br> print(x)</div>", "for x in y:\n\xa0\xa0print(x)"),
("<div>if x:<br>  print(x)</div>", "if x:\n\u2003\u2003print(x)"),
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_no_break_spaces_are_kept(markup, expected, main_content):
assert html_to_markdown(markup, main_content = main_content) == expected
@pytest.mark.parametrize("tag", ["span", "code", "pre"])
@pytest.mark.parametrize("nbsp", [" ", "\xa0", " "])
def test_kept_no_break_spaces_do_not_dilute_a_link_header(nbsp, tag):
links = "".join(f'<a href="/s{i}">Section {i}</a> ' for i in range(20))
prose = "Article prose. " * 20
html = (
f"<article><header>{links}<{tag}>Menu{nbsp * 10}x</{tag}></header><p>{prose}</p></article>"
)
assert html_to_markdown(html, main_content = True) == prose.strip()
@pytest.mark.parametrize("sep", [" ", " "])
@pytest.mark.parametrize("main_content", [False, True])
def test_a_caption_keeps_the_no_break_spaces_between_its_tabs(main_content, sep):
# Javadoc 8 method summary: the caption's tabs are separated only by
html = (
f'<table><caption><span>All Methods</span><span class="tabEnd">{sep}</span>'
f'<span><a href="javascript:show(1);">Static Methods</a></span><span class="tabEnd">{sep}</span>'
"</caption><tr><th>Modifier</th><th>Method</th></tr><tr><td>a</td><td>b</td></tr></table>"
)
out = html_to_markdown(html, main_content = main_content)
kept = {" ": "\xa0", " ": "\u2003"}[sep]
assert out.startswith(
f"All Methods{kept}[Static Methods](javascript:show(1);){kept}| Modifier |"
)
@pytest.mark.parametrize("nbsp", [" ", "\xa0", " ", " "])
def test_no_break_space_padding_does_not_make_a_stub_the_main_content(nbsp):
body = "Body prose that the page is about. " * 20
html = f"<main><article><p>a{nbsp * 220}b</p></article><p>{body}</p></main>"
assert body.strip() in html_to_markdown(html, main_content = True)
def test_no_break_space_padding_in_a_stripped_header_does_not_outrank_a_longer_article():
links = "".join(f'<a href="/s{i}">Section {i}</a> ' for i in range(20))
body = "Actual article body. " * 48
html = (
f'<article><header>{links}<span>Menu{" " * 200}x</span></header>'
f"<p>{'Teaser prose. ' * 40}</p></article><article><p>{body}</p></article>"
)
assert body.strip() in html_to_markdown(html, main_content = True)
def test_no_break_space_padding_does_not_make_a_byline_header_navigation():
title = (
"Practical techniques for building reliable Python applications "
"and deploying them with confidence in production"
)
html = (
f'<article><header><a href="/read?token={"x" * 650}">{title}</a>'
f'<p>by{" " * 20}Ann</p></header>'
f'<p>{"This is the main article body. " * 30}</p></article>'
)
assert title in html_to_markdown(html, main_content = True)
def test_no_break_space_padding_does_not_outrank_a_longer_article():
teaser = "Teaser " + " " * 3000 + " ".join(["teaser words"] * 20)
body = "Body prose that the page is about. " * 60
html = f"<article><p>{teaser}</p></article><article><p>{body}</p></article>"
assert body.strip() in html_to_markdown(html, main_content = True)
@pytest.mark.parametrize("main_content", [False, True])
def test_a_decimal_reference_past_the_int_digit_limit_still_renders(main_content):
ones, zeros = "1" * 5000, "0" * 5000
html = (
f'<p><a href="/x?q=&#{ones};">[&#{zeros}65;] [&#{ones};] [&#{ones}a]</a></p>'
f"<p>[&#{ones}"
)
out = html_to_markdown(html, main_content = main_content)
assert out == "[[A] [\ufffd] [\ufffda]](/x?q=\ufffd)\n\n[\ufffd"
@pytest.mark.parametrize(
"markup, expected",
[
# Pygments 2.21 HtmlFormatter(linenos="table"): each <pre> sits in a table cell.
(
'<div class="highlight"><table class="highlighttable"><tr><td class="linenos">'
'<div class="linenodiv"><pre><span class="normal">1</span>\n'
'<span class="normal">2</span></pre></div></td><td class="code"><div><pre><span></span>'
'<span class="k">if</span> <span class="n">a</span> <span class="o"><</span> '
'<span class="n">b</span> <span class="ow">and</span> <span class="n">c</span> '
'<span class="o">></span> <span class="mi">0</span><span class="p">:</span>\n'
' <span class="nb">print</span><span class="p">(</span>'
'<span class="s2">"a & b"</span><span class="p">)</span>\n'
"</pre></div></td></tr></table></div>\n",
'| ``` 1 2 ``` | ``` if a < b and c > 0: print("a & b") ``` |\n| --- | --- |',
),
# an RFC section link inside a <pre> keeps its
(
'<pre>in <a href="./rfc5741#section-2">Section 2 of RFC 5741</a>.</pre>',
"```\nin Section\xa02 of RFC 5741.\n```",
),
(
'<pre>fn new() -> <a href="/std/vec">Vec<T></a></pre>',
"```\nfn new() -> Vec<T>\n```",
),
],
)
@pytest.mark.parametrize("main_content", [False, True])
def test_escaped_code_stays_inside_its_code_block(markup, expected, main_content):
assert html_to_markdown(markup, main_content = main_content) == expected
def test_page_cut_by_the_room_left_drops_its_site_link_urls(monkeypatch):
from core.inference import tools
context = tools._REQUEST_CONTEXT_TOKENS.set(32768)
room = tools._REQUEST_RESULT_BUDGET.set(1200)
try:
out = _page_text(monkeypatch, _WIKI_URL, _wiki_article(60), "text/html")
finally:
tools._REQUEST_RESULT_BUDGET.reset(room)
tools._REQUEST_CONTEXT_TOKENS.reset(context)
assert "(truncated," in out
assert "wikipedia.org" not in out
def test_dropping_site_link_urls_leaves_code_samples_alone(monkeypatch):
filler = "".join(
f"<p>Guide paragraph {i} with enough words to run long.</p>" for i in range(400)
)
page = (
'<html><body><main><p>See <a href="#install">Install</a> or <a href="#usage">Usage</a>.</p>'
"<pre>[Install](#install)</pre><p>Inline <code>[Usage](#usage)</code>.</p>"
f"{filler}</main></body></html>"
)
out = _page_text(monkeypatch, "https://docs.example.com/guide", page, "text/html")
assert "```\n[Install](#install)\n```" in out
assert "Inline `[Usage](#usage)`." in out
def test_dropping_site_link_urls_only_rewrites_links_emitted_by_renderer(monkeypatch):
filler = "".join(
f"<p>Guide paragraph {i} with enough words to run long.</p>" for i in range(400)
)
page = (
"<html><body><main>"
"<p>Literal Markdown: [Install](#install).</p>"
'<p>Inline code: <code><a href="#install">Install</a></code>.</p>'
'<p>Actual link: <a href="#install">Install</a>.</p>'
f"{filler}</main></body></html>"
)
out = _page_text(monkeypatch, "https://docs.example.com/guide", page, "text/html")
assert out.startswith(
"Literal Markdown: [Install](#install).\n\n"
"Inline code: `[Install](#install)`.\n\n"
"Actual link: Install."
)
def test_dropping_site_link_urls_keeps_the_same_article(monkeypatch):
linked = "".join(
f'<p>Story one fact {i} see <a href="/topics/a-very-long-topic-slug-number-{i}">topic {i}</a>.</p>'
for i in range(300)
)
plain = "".join(
f"<p>Story two paragraph {i} with plain prose and no links.</p>" for i in range(300)
)
page = f"<html><body><article><h1>STORY ONE</h1>{linked}</article><article><h1>STORY TWO</h1>{plain}</article></body></html>"
out = _page_text(monkeypatch, "https://news.example.com/story-one", page, "text/html")
assert out.startswith("# STORY ONE\n\nStory one fact 0 see topic 0.")
assert "STORY TWO" not in out
def test_dropping_site_link_urls_keeps_a_link_header_as_furniture(monkeypatch):
labels = "machine learning,deep learning,pytorch,cuda,transformers,lora,gguf,llama,quantization,finetuning,inference,qlora"
tags = "".join(
f'<a href="/category/topics/{t}/archive/page/1/?ref=card-header">{t}</a> '
for t in labels.split(",")
)
prose = "".join(
f"<p>Main page prose the user asked about, paragraph {i}, with enough words.</p>"
for i in range(250)
)
page = (
f"<html><body><main><h1>Main Post</h1>{prose}<aside><article><header>{tags}</header>"
"<p>A short teaser for a related post that is just long enough to read like a real sentence or two here.</p>"
"</article></aside></main></body></html>"
)
out = _page_text(monkeypatch, "https://blog.example.com/main-post", page, "text/html")
assert out.startswith("# Main Post\n\nMain page prose the user asked about, paragraph 0")
def test_looks_like_html():
assert _looks_like_html("<!DOCTYPE html><html></html>")
assert _looks_like_html("\n <HTML lang='en'>")
assert not _looks_like_html("# Markdown README\n\n<h1>embedded html later</h1>")
assert not _looks_like_html("plain text")
def test_looks_like_html_markdown_with_leading_fenced_example_stays_markdown():
fenced = (
"```html\n<!DOCTYPE html>\n<html><body><div>hi</div></body></html>\n```\n\n# Real README\n"
)
assert not _looks_like_html(fenced)
assert not _looks_like_html("Use the <html> element to start a page.")
assert not _looks_like_html('<p align="center"><img src="logo.png"></p>\n\n# Project\n')
assert not _looks_like_html('<div align="center">\n\n# Project\n\n</div>\n')
assert not _looks_like_html('<h1 align="center">Project</h1>\n\nMarkdown body.\n')
assert not _looks_like_html("<https://example.com> is the homepage")
def test_looks_like_html_detects_bare_fragments():
assert _looks_like_html("<body><p>hello</p></body>")
assert _looks_like_html("\n<article><h1>Title</h1><p>Body</p></article>")
assert _looks_like_html("<section>content</section>")
def test_looks_like_html_leading_table_stays_markdown():
assert not _looks_like_html("<table><tr><td>cell</td></tr></table>")
assert not _looks_like_html(
'<table align="center"><tr><td><img src="logo.png"></td></tr></table>\n\n# Project\n'
)
assert not _looks_like_html("<tr><td>cell</td></tr>")
@pytest.mark.parametrize(
"md_readme, first, second, third",
[
pytest.param(
"```html\n"
"<!DOCTYPE html>\n"
"<html><body><h1>Demo</h1></body></html>\n"
"```\n\n"
"# My Project\n\nInstall and run.\n",
"```html",
"<!DOCTYPE html>",
"# My Project",
id = "fetch_page_text_keeps_markdown_readme_with_html_example",
),
pytest.param(
'<table align="center">\n'
'<tr><td><img src="logo.png"></td><td>Badges</td></tr>\n'
"</table>\n\n"
"# My Project\n\n"
"- feature one\n"
"- feature two\n\n"
"```python\nprint('hi')\n```\n",
"- feature one\n- feature two",
"```python",
"# My Project",
id = "fetch_page_text_keeps_markdown_readme_with_leading_table",
),
pytest.param(
"<blockquote>Note: pre-release.</blockquote>\n\n"
"# My Project\n\n"
"Install:\n\n"
"- step one\n"
"- step two\n\n"
"```bash\npip install myproject\n```\n",
"# My Project",
"- step one",
"```bash",
id = "fetch_page_text_markdown_readme_with_leading_block_tag_stays_markdown",
),
],
)
def test_fetch_page_text_keeps_a_markdown_readme_verbatim(
monkeypatch, md_readme, first, second, third
):
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
assert url == "https://api.github.com/repos/unslothai/unsloth/readme"
return (None, md_readme, "text/plain")
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://github.com/unslothai/unsloth")
assert "README of https://github.com/unslothai/unsloth" in out
assert first in out
assert second in out
assert third in out
def test_fetch_url_raw_missing_content_type_reported_empty(monkeypatch):
# get_content_type() defaults to text/plain; _fetch_url_raw must report "" so sniffing fires.
import email
import urllib.request
class _FakeResp:
headers = email.message_from_string("")
def __init__(self):
self._body = b"<html><body>hello</body></html>"
def read(self, n = -1):
body, self._body = self._body, b""
return body
class _FakeOpener:
def open(
self,
req,
timeout = None,
):
return _FakeResp()
monkeypatch.setattr(
"core.inference.tools._validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: _FakeOpener())
err, body, content_type = _fetch_url_raw("https://example.com/")
assert err is None
assert "hello" in body
assert content_type == ""
def _four_address_getaddrinfo(hostname, port, *args, **kwargs):
import socket as _socket
return [
(_socket.AF_INET6, _socket.SOCK_STREAM, 0, "", ("2606:4700::1", port, 0, 0)),
(_socket.AF_INET6, _socket.SOCK_STREAM, 0, "", ("2606:4700::2", port, 0, 0)),
(_socket.AF_INET, _socket.SOCK_STREAM, 0, "", ("104.16.0.1", port)),
(_socket.AF_INET, _socket.SOCK_STREAM, 0, "", ("104.16.0.2", port)),
]
def test_validate_and_resolve_host_returns_every_validated_address(monkeypatch):
import socket as _socket
from core.inference import tools as tools_mod
monkeypatch.setattr(_socket, "getaddrinfo", _four_address_getaddrinfo)
ok, reason, ips = tools_mod._validate_and_resolve_host("example.com", 443)
assert (ok, reason) == (True, "")
assert ips == ["2606:4700::1", "2606:4700::2", "104.16.0.1", "104.16.0.2"]
class _FakeSocket:
def __init__(self, address):
self.address = address
def settimeout(self, value):
self.timeout = value
def _recording_create_connection(
monkeypatch,
unreachable = (),
stall = (),
answers_above = None,
overrun = 1.0,
):
"""``socket.create_connection`` on a virtual clock: a dial charges its timeout
rather than sleeping it. *stall* burns the whole dial (*overrun* times it),
*answers_above* is an ``(ip, seconds)`` that connects only when given more."""
calls = []
spent = [0.0]
real_monotonic = time.monotonic
monkeypatch.setattr(time, "monotonic", lambda: real_monotonic() + spent[0])
def create(
address,
timeout = None,
source_address = None,
):
calls.append((address[0], timeout))
if address[0] in stall:
spent[0] += timeout * overrun
raise TimeoutError("timed out")
if address[0] in unreachable:
raise OSError(101, f"unreachable {address[0]}")
if answers_above is not None and address[0] == answers_above[0]:
if timeout is None or timeout <= answers_above[1]:
spent[0] += timeout
raise TimeoutError("timed out")
spent[0] += answers_above[1]
return _FakeSocket(address)
return calls, create
@pytest.mark.parametrize("timeout", [30, None])
def test_pinned_dial_walks_to_the_next_address_when_first_is_unreachable(monkeypatch, timeout):
import socket as _socket
from core.inference import tools as tools_mod
# Only the third answers, so trying just the ends cannot pass.
addresses = ("2606:4700::1", "2606:4700::2", "104.16.0.1", "104.16.0.2")
unreachable = set(addresses) - {addresses[2]}
calls, create = _recording_create_connection(monkeypatch, unreachable = unreachable)
monkeypatch.setattr(_socket, "create_connection", create)
dial = tools_mod._pinned_create_connection(addresses)
sock = dial((addresses[0], 443), timeout)
assert sock.address == (addresses[2], 443)
assert [ip for ip, _timeout in calls] == list(addresses[:3])
capped = tools_mod._PINNED_DIAL_TIMEOUT if timeout else None
assert [dialled for _ip, dialled in calls] == [capped] * 3
calls.clear()
assert dial(("proxy.corp", 3128), timeout).address == ("proxy.corp", 3128)
assert [ip for ip, _timeout in calls] == ["proxy.corp"]
calls.clear()
with pytest.raises(OSError):
tools_mod._pinned_create_connection(addresses[:2])((addresses[0], 443), timeout)
assert len(calls) == (2 if timeout is None else 4)
@pytest.mark.parametrize("overrun,budget", [(1.0, 0.3), (4.0, 0.001)])
def test_pinned_dial_asks_for_no_more_than_the_callers_timeout(monkeypatch, overrun, budget):
import socket as _socket
from core.inference import tools as tools_mod
addresses = ("2606:4700::1", "2606:4700::2", "104.16.0.1")
calls, create = _recording_create_connection(
monkeypatch,
stall = set(addresses),
overrun = overrun,
)
monkeypatch.setattr(_socket, "create_connection", create)
dial = tools_mod._pinned_create_connection(addresses)
with pytest.raises(OSError):
dial((addresses[0], 443), budget)
assert sum(timeout for _ip, timeout in calls) <= budget
assert all(timeout >= 0 for _ip, timeout in calls)
assert [ip for ip, _timeout in calls] == list(addresses) * 2
def test_pinned_dial_probe_stays_affordable_for_many_records(monkeypatch):
import socket as _socket
from core.inference import tools as tools_mod
addresses = tuple(f"198.51.100.{i}" for i in range(31))
calls, create = _recording_create_connection(
monkeypatch,
stall = set(addresses[:-1]),
unreachable = {addresses[-1]},
)
monkeypatch.setattr(_socket, "create_connection", create)
dial = tools_mod._pinned_create_connection(addresses)
with pytest.raises(OSError, match = addresses[-1]):
dial((addresses[0], 443), 1.0)
probes = [timeout for _ip, timeout in calls[: len(addresses)]]
assert sum(probes) == pytest.approx(tools_mod._PINNED_PROBE_BUDGET)
assert calls[len(addresses)][1] == pytest.approx(
(1 - tools_mod._PINNED_PROBE_BUDGET) / len(addresses),
rel = 0.05,
)
def test_pinned_dial_shares_what_is_left_so_no_address_strands_the_next(monkeypatch):
import socket as _socket
from core.inference import tools as tools_mod
monkeypatch.setattr(tools_mod, "_PINNED_DIAL_TIMEOUT", 0.02)
addresses = ("192.0.2.1", "2606:4700::2", "104.16.0.1")
calls, create = _recording_create_connection(
monkeypatch,
unreachable = {addresses[0]},
stall = {addresses[1]},
answers_above = (addresses[2], 0.05),
)
monkeypatch.setattr(_socket, "create_connection", create)
dial = tools_mod._pinned_create_connection(addresses)
sock = dial((addresses[0], 443), 0.6)
assert sock.address == (addresses[2], 443)
assert [ip for ip, _timeout in calls] == list(addresses) * 2
assert [timeout for _ip, timeout in calls[:3]] == [0.02] * 3
assert calls[3][1] == pytest.approx((0.6 - 0.04) / 3, rel = 0.01)
assert calls[4][1] == pytest.approx(calls[3][1] * 3 / 2, rel = 0.01)
assert sock.timeout == 0.6
def test_pinned_https_connection_walks_a_bracketed_ipv6_host(monkeypatch):
# http.client strips the brackets; the walk must still see a pinned address.
import socket as _socket
from core.inference import tools as tools_mod
addresses = ("2606:2800:220:1::1", "93.184.216.34")
calls, create = _recording_create_connection(monkeypatch, unreachable = {addresses[0]})
monkeypatch.setattr(_socket, "create_connection", create)
handler = tools_mod._SNIHTTPSHandler("example.com", addresses)
conn = handler._sni_connection(f"[{addresses[0]}]", timeout = 30)
sock = conn._create_connection((conn.host, conn.port), conn.timeout, None)
assert sock.address == (addresses[1], 443)
assert [ip for ip, _timeout in calls] == list(addresses)
name = "example.com"
for conn in (
tools_mod._SNIHTTPSHandler(name, (), name)._sni_connection(name),
tools_mod._PinnedHTTPHandler((), name)._pinned_connection(name),
):
with pytest.raises(OSError, match = "no proxy took it"):
conn._create_connection((name, 443), 30, None)
def test_pinned_http_handler_walks_through_a_real_opener(monkeypatch):
import socket as _socket
import urllib.request
from core.inference import tools as tools_mod
addresses = ("2606:4700::1", "2606:4700::2", "203.0.113.9")
dialled = []
real_create_connection = _socket.create_connection
server = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
server.bind(("127.0.0.1", 0))
server.listen(1)
def create(
address,
timeout = None,
source_address = None,
):
dialled.append(address[0])
if address[0] != addresses[2]:
raise OSError(101, "Network is unreachable")
sock = real_create_connection(server.getsockname(), timeout)
conn, _addr = server.accept()
conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\n\r\nthird one ok")
return sock
monkeypatch.setattr(_socket, "create_connection", create)
opener = urllib.request.build_opener(
tools_mod._PinnedHTTPHandler(addresses),
urllib.request.ProxyHandler({}),
)
with opener.open(f"http://[{addresses[0]}]/page", timeout = 5) as resp:
body = resp.read()
server.close()
assert body == b"third one ok"
assert dialled == list(addresses)
def test_fetch_url_raw_pins_one_address_and_hands_urllib_every_address(monkeypatch):
import socket as _socket
import urllib.error
import urllib.request
from core.inference import tools as tools_mod
class _FakeResp:
headers = email.message_from_string("Content-Type: text/plain\n")
def __init__(self):
self._body = b"slow origin answered"
def read(self, n = -1):
body, self._body = self._body, b""
return body
seen = {"timeouts": []}
class _SlowOriginOpener:
def open(
self,
req,
timeout = None,
):
seen["timeouts"].append(timeout)
seen["url"] = req.full_url
if timeout is not None and timeout < 9:
raise urllib.error.URLError(TimeoutError("timed out"))
return _FakeResp()
def fake_build_opener(*handlers):
seen["handlers"] = handlers
return _SlowOriginOpener()
monkeypatch.setattr(_socket, "getaddrinfo", _four_address_getaddrinfo)
monkeypatch.setattr(urllib.request, "getproxies", dict)
monkeypatch.setattr(urllib.request, "build_opener", fake_build_opener)
err, body, _content_type = tools_mod._fetch_url_raw(
"https://example.com/",
timeout = 30,
deadline = time.monotonic() + 30,
)
assert err is None
assert "slow origin answered" in body
assert seen["url"] == "https://[2606:4700::1]/"
assert seen["timeouts"] == [pytest.approx(30, abs = 1)]
addresses = ("2606:4700::1", "2606:4700::2", "104.16.0.1", "104.16.0.2")
for handler_type in (tools_mod._SNIHTTPSHandler, tools_mod._PinnedHTTPHandler):
handler = next(h for h in seen["handlers"] if isinstance(h, handler_type))
assert handler._addresses == addresses
@pytest.mark.parametrize(
"disable_dns_pinning,proxied,expected_url",
[
(False, False, "https://203.0.113.7:8443/page?q=1"),
(False, True, "https://203.0.113.7:8443/page?q=1"),
# The opt-out only applies to proxied fetches; direct ones would reopen DNS rebinding.
(True, False, "https://203.0.113.7:8443/page?q=1"),
(True, True, "https://example.com:8443/page?q=1"),
],
)
@pytest.mark.parametrize("proxy_hostname", [None, True])
@pytest.mark.parametrize("failure", [None, "Failed to resolve host: x", "Blocked: x"])
def test_fetch_url_raw_dns_pinning_proxy_opt_out(
monkeypatch, disable_dns_pinning, proxied, expected_url, proxy_hostname, failure
):
import email
import urllib.request
import core.inference.tools as tools_mod
class _FakeResp:
headers = email.message_from_string("Content-Type: text/plain\n")
def __init__(self):
self._body = b"ok"
def read(self, n = -1):
body, self._body = self._body, b""
return body
requested = []
class _FakeOpener:
def open(
self,
req,
timeout = None,
):
requested.append(req)
return _FakeResp()
resolved = []
def resolve(host, port):
resolved.append((host, port))
return (False, failure, []) if failure else (True, "", ["203.0.113.7"])
monkeypatch.setenv("UNSLOTH_STUDIO_DISABLE_DNS_PINNING", "1" if disable_dns_pinning else "0")
built = []
def fake_build_opener(*handlers):
built.append(handlers)
return _FakeOpener()
monkeypatch.setattr(tools_mod, "_validate_and_resolve_host", resolve)
monkeypatch.setattr(urllib.request, "build_opener", fake_build_opener)
# Patch lookups, not env: getproxies/proxy_bypass read system settings on macOS/Windows.
monkeypatch.setattr(
urllib.request,
"getproxies",
lambda: {"https": "http://proxy.corp:3128"} if proxied else {},
)
monkeypatch.setattr(urllib.request, "proxy_bypass", lambda host: False)
# Embedded credentials are refused; see test_fetch_url_raw_rejects_embedded_credentials.
url = "https://example.com:8443/page?q=1"
err, body, _content_type = tools_mod._fetch_url_raw(url, proxy_hostname = proxy_hostname)
named = proxied and (proxy_hostname or disable_dns_pinning)
if failure and not (named and failure.startswith("Failed to resolve")):
assert err == failure and not requested
return
assert err is None
assert body == "ok"
assert resolved == [("example.com", 8443)]
assert [req.full_url for req in requested] == [url if named else expected_url]
assert requested[0].get_header("Host") == "example.com:8443"
for handler_type in (tools_mod._SNIHTTPSHandler, tools_mod._PinnedHTTPHandler):
handler = next(h for h in built[0] if isinstance(h, handler_type))
assert handler._addresses == (
() if (proxied and not named) or failure else ("203.0.113.7",)
)
assert handler._origin == ("example.com" if named else None)
@pytest.mark.parametrize(
"host",
["intranet", "printer.local", "nas.lan", "router.home.arpa", "home.arpa", "wiki.intranet"],
)
def test_fetch_url_raw_named_proxy_fetch_refuses_local_names(monkeypatch, host):
import socket
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import core.inference.tools as tools_mod
seen = []
class Proxy(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
seen.append(self.path)
self.send_response(200)
self.send_header("Content-Length", "2")
self.end_headers()
self.wfile.write(b"ok")
server = ThreadingHTTPServer(("127.0.0.1", 0), Proxy)
threading.Thread(target = server.serve_forever, daemon = True).start()
for name in ("NO_PROXY", "no_proxy", "ALL_PROXY", "all_proxy", "HTTPS_PROXY", "https_proxy"):
monkeypatch.delenv(name, raising = False)
monkeypatch.setenv("HTTP_PROXY", f"http://127.0.0.1:{server.server_address[1]}")
monkeypatch.setenv("http_proxy", f"http://127.0.0.1:{server.server_address[1]}")
real_getaddrinfo = socket.getaddrinfo
def nxdomain(name, *args, **kwargs):
if name == host:
raise socket.gaierror(socket.EAI_NONAME, "Name or service not known")
return real_getaddrinfo(name, *args, **kwargs)
monkeypatch.setattr(socket, "getaddrinfo", nxdomain)
try:
err, _, _ = tools_mod._fetch_url_raw(
f"http://{host}/page",
deadline = time.monotonic() + 4,
proxy_hostname = True,
)
finally:
server.shutdown()
assert err == f"Blocked: {host} is a local name."
assert seen == []
def test_fetch_url_raw_proxy_scheme_key_case_insensitive(monkeypatch):
# Windows registry proxy keys keep their case while ProxyHandler lowercases them.
import email
import urllib.request
import core.inference.tools as tools_mod
class _FakeResp:
headers = email.message_from_string("Content-Type: text/plain\n")
def __init__(self):
self._body = b"ok"
def read(self, n = -1):
body, self._body = self._body, b""
return body
requested = []
built = []
class _FakeOpener:
def open(
self,
req,
timeout = None,
):
requested.append(req)
return _FakeResp()
monkeypatch.setenv("UNSLOTH_STUDIO_DISABLE_DNS_PINNING", "1")
monkeypatch.setattr(
tools_mod,
"_validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "getproxies", lambda: {"HTTPS": "http://proxy.corp:3128"})
monkeypatch.setattr(urllib.request, "proxy_bypass", lambda host: False)
monkeypatch.setattr(
urllib.request,
"build_opener",
lambda *handlers: built.append(handlers) or _FakeOpener(),
)
err, body, _content_type = tools_mod._fetch_url_raw("https://example.com:8443/page?q=1")
assert err is None
assert body == "ok"
assert [req.full_url for req in requested] == ["https://example.com:8443/page?q=1"]
assert not [h for h in built[0] if isinstance(h, urllib.request.ProxyHandler) and not h.proxies]
@pytest.mark.parametrize(
"no_proxy,disable_dns_pinning,expected_url",
[
# urllib tests NO_PROXY against Request.host, so port-qualified entries need the port.
("example.com:8443", True, "https://203.0.113.7:8443/page?q=1"),
("example.com", True, "https://203.0.113.7:8443/page?q=1"),
("other.example", True, "https://example.com:8443/page?q=1"),
("example.com", False, "https://203.0.113.7:8443/page?q=1"),
],
)
def test_fetch_url_raw_no_proxy_routing(monkeypatch, no_proxy, disable_dns_pinning, expected_url):
# Real getproxies/proxy_bypass, not stubs: the point is to agree with urllib.
import email
import urllib.request
import core.inference.tools as tools_mod
class _FakeResp:
headers = email.message_from_string("Content-Type: text/plain\n")
def __init__(self):
self._body = b"ok"
def read(self, n = -1):
body, self._body = self._body, b""
return body
requested = []
built = []
class _FakeOpener:
def open(
self,
req,
timeout = None,
):
requested.append(req)
return _FakeResp()
def fake_build_opener(*handlers):
built.append(handlers)
return _FakeOpener()
for var in ("http_proxy", "https_proxy", "no_proxy", "all_proxy", "REQUEST_METHOD"):
monkeypatch.delenv(var, raising = False)
monkeypatch.delenv(var.upper(), raising = False)
monkeypatch.setenv("https_proxy", "http://proxy.corp:3128")
monkeypatch.setenv("no_proxy", no_proxy)
monkeypatch.setenv("UNSLOTH_STUDIO_DISABLE_DNS_PINNING", "1" if disable_dns_pinning else "0")
monkeypatch.setattr(
tools_mod,
"_validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "build_opener", fake_build_opener)
err, body, _content_type = tools_mod._fetch_url_raw("https://example.com:8443/page?q=1")
assert err is None
assert body == "ok"
assert [req.full_url for req in requested] == [expected_url]
bypassed = expected_url.startswith("https://203.0.113.7")
empty_proxy_handlers = [
h for h in built[0] if isinstance(h, urllib.request.ProxyHandler) and not h.proxies
]
assert bool(empty_proxy_handlers) is bypassed
def test_fetch_url_raw_rejects_embedded_credentials(monkeypatch):
import core.inference.tools as tools_mod
def resolve(host, port):
raise AssertionError("must be rejected before DNS resolution")
monkeypatch.setattr(tools_mod, "_validate_and_resolve_host", resolve)
err, body, _content_type = tools_mod._fetch_url_raw(
"https://user:[email protected]:8443/page?q=1"
)
assert err is not None and "credentials" in err
assert body == ""
def test_fetch_page_text_missing_content_type_html_sniffed(monkeypatch):
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
return None, _GITHUB_PAGE, ""
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://example.com/no-content-type")
assert "Unsloth Studio" in out
assert "<html" not in out
assert "Uh oh!" not in out
def test_nested_hidden_list_does_not_leak_child_items():
html = (
"<body><ul>"
"<li hidden>parent<ul><li>secret child</li></ul></li>"
"<li>visible sibling</li>"
"</ul></body>"
)
out = html_to_markdown(html)
assert "parent" not in out
assert "secret child" not in out
assert "visible sibling" in out
def test_nested_hidden_list_with_omitted_closes_stays_suppressed():
html = (
"<body><ul>"
"<li hidden>parent<ul><li>secret child<ul><li>deeper secret</ul></li></ul>"
"<li>visible sibling"
"</ul></body>"
)
out = html_to_markdown(html)
assert "parent" not in out
assert "secret child" not in out
assert "deeper secret" not in out
assert "visible sibling" in out
def test_many_tiny_articles_do_not_displace_substantial_main():
cards = "".join(
f"<article><h2>Teaser {i}</h2><p>Advertisement card blurb.</p></article>" for i in range(12)
)
main_body = "Authoritative main documentation content. " * 30
html = f"<body>{cards}<main><h1>Real page</h1><p>{main_body}</p></main></body>"
out = html_to_markdown(html, main_content = True)
assert "Authoritative main documentation content." in out
assert "Advertisement card blurb." not in out
def test_single_substantial_article_still_preferred_over_main():
# GitHub README pages mix a substantial <article> with repository furniture.
article_body = "Real README documentation body text. " * 20
html = (
"<body><main>"
f"<article><h1>Guide</h1><p>{article_body}</p></article>"
"<div><h2>Languages</h2><p>JavaScript 89.3%</p></div>"
"</main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "Real README documentation body text." in out
assert "JavaScript 89.3%" not in out
def test_one_card_does_not_stand_in_for_a_listing_main():
cards = "".join(
f"<article><h2>Plan {i}</h2><p>{f'Plan {i} feature and price detail. ' * 8}</p></article>"
for i in range(6)
)
html = f"<body><main><h1>Pricing</h1>{cards}</main></body>"
out = html_to_markdown(html, main_content = True)
for i in range(6):
assert f"Plan {i} feature and price detail." in out
def test_article_listing_without_main_uses_the_document():
cards = "".join(
f"<article><h2>Plan {i}</h2><p>{f'Plan {i} feature and price detail. ' * 8}</p></article>"
for i in range(6)
)
out = html_to_markdown(f"<body><h1>Pricing</h1>{cards}</body>", main_content = True)
for i in range(6):
assert f"Plan {i} feature and price detail." in out
def test_post_body_outside_article_beats_author_bio_card():
post = "Main post body paragraph with the actual story. " * 30
bio = "Author bio describing the writer and their work. " * 6
related = "".join(
f"<article class='related'><h3>Related {i}</h3><p>{'Teaser for another post. ' * 9}</p></article>"
for i in range(3)
)
html = (
"<body><main><h1>Post title</h1>"
f"<div class='post-content'><p>{post}</p></div>"
f"<article class='author-card'><p>{bio}</p></article>{related}"
"</main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "Main post body paragraph" in out
def test_lone_readme_article_is_kept_over_repo_page_chrome():
readme = "Short README describing the library. " * 8
rows = "".join(
f"<tr><td><a href='/o/r/tree/main/dir{i}'>dir{i}</a></td>"
f"<td><a href='/o/r/commit/{i}'>Update the dir{i} module and its tests</a></td><td>2 days ago</td></tr>"
for i in range(25)
)
html = (
"<body><main><h2>Repository files navigation</h2>"
f"<table><tr><th>Name</th><th>Last commit message</th><th>Last commit date</th></tr>{rows}</table>"
f"<article class='markdown-body'><h1>Lib</h1><p>{readme}</p></article>"
"<div><h2>About</h2><p>A small library for doing one thing well.</p></div>"
"</main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "Short README describing the library." in out
assert "Last commit message" not in out
def test_link_heavy_comments_do_not_pull_main_over_the_post():
post = "The post explains the topic in full detail here. " * 50
comments = "".join(
f"<article class='comment'><p><a href='https://example.com/author/{i}?{'utm_source=comments&' * 20}'>Reader {i}</a> "
f"says: {'Thanks for writing this up. ' * 9}</p></article>"
for i in range(6)
)
html = (
f"<body><main><article class='post'><h1>Post</h1><p>{post}</p></article>"
f"<section id='comments'>{comments}</section></main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "The post explains the topic" in out
assert "Thanks for writing this up." not in out
def test_generated_table_spans_do_not_pull_main_over_the_post():
post = "Primary article prose with real details. " * 30
related = "Related card teaser. " * 12
rows = "".join(f"<tr><td>row {i}</td></tr>" for i in range(1, 80))
table = f"<table><tr><td rowspan='80'>repeated marker</td><td>row 0</td></tr>{rows}</table>"
html = (
"<body><main>"
f"<article><h1>Primary</h1><p>{post}</p></article>"
f"<article><p>{related}</p></article>{table}"
"</main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "Primary article prose" in out
assert "Related card teaser." not in out
@pytest.mark.parametrize("hide", ["hidden", "aria-hidden='true'", "style='display:none'"])
def test_hidden_duplicate_article_is_not_a_second_card(hide):
post = "The post explains the topic in full detail here. " * 20
comments = "".join(
f"<div class='comment'><p>Reader {i} says: {'Thanks for writing this up, it helped. ' * 6}</p></div>"
for i in range(8)
)
html = (
f"<body><main><article><h1>Post</h1><p>{post}</p></article>"
f"<article {hide}><p>Duplicate</p></article><section>{comments}</section></main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "The post explains the topic" in out
assert "Thanks for writing this up" not in out
def test_truncated_open_article_scope_is_scored_and_preferred():
# _fetch_url_raw may truncate before </article>, so the open scope must exclude page chrome.
chrome = "<nav>Skip to content</nav><div>Repository file tree and page chrome.</div>"
article_body = "Real README documentation body text. " * 20
html = f"<body>{chrome}<article><h1>Guide</h1><p>{article_body}</p>"
out = html_to_markdown(html, main_content = True)
assert "Real README documentation body text." in out
assert "Repository file tree and page chrome." not in out
def test_truncated_open_main_scope_is_scored_and_preferred():
chrome = "<nav>Skip to content</nav><div>Repository file tree and page chrome.</div>"
main_body = "Authoritative main documentation content. " * 30
html = f"<body>{chrome}<main><h1>Doc</h1><p>{main_body}</p>"
out = html_to_markdown(html, main_content = True)
assert "Authoritative main documentation content." in out
assert "Repository file tree and page chrome." not in out
def test_fetch_url_raw_overall_deadline_aborts_across_redirects(monkeypatch):
# 5s per hop vs an 8s budget: aborts on hop 3, well before the 5-hop cap.
import urllib.request
from urllib.error import HTTPError
import core.inference.tools as tools_mod
clock = {"t": 1000.0}
monkeypatch.setattr(tools_mod.time, "monotonic", lambda: clock["t"])
hops = {"n": 0}
class _RedirectingOpener:
def open(
self,
req,
timeout = None,
):
clock["t"] += 5.0
hops["n"] += 1
raise HTTPError(
req.full_url,
302,
"Found",
{"Location": "https://example.com/next"},
None,
)
monkeypatch.setattr(
tools_mod,
"_validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: _RedirectingOpener())
err, body, content_type = tools_mod._fetch_url_raw(
"https://example.com/start",
timeout = 30,
deadline = clock["t"] + 8.0,
)
assert err == "Failed to fetch URL: timed out."
assert body == ""
assert hops["n"] < 5
def test_fetch_url_raw_host_headers_follow_each_hop(monkeypatch):
# A header chosen for unsloth.ai must not ride a redirect to another site.
import urllib.request
from urllib.error import HTTPError
import core.inference.tools as tools_mod
sent = []
class _Opener:
def open(
self,
req,
timeout = None,
):
sent.append((req.get_header("Host"), req.get_header("X-unsloth-studio")))
raise HTTPError(
req.full_url, 302, "Found", {"Location": "https://example.com/next"}, None
)
monkeypatch.setattr(
tools_mod,
"_validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: _Opener())
tools_mod._fetch_url_raw(
"https://unsloth.ai/",
host_headers = lambda host: {"X-Unsloth-Studio": "1"} if host == "unsloth.ai" else {},
)
assert sent[0] == ("unsloth.ai", "1")
assert sent[1] == ("example.com", None)
def test_fetch_url_raw_cancel_event_aborts_before_network(monkeypatch):
import threading
import urllib.request
import core.inference.tools as tools_mod
ev = threading.Event()
ev.set()
opened = {"n": 0}
class _Opener:
def open(
self,
req,
timeout = None,
):
opened["n"] += 1
raise AssertionError("network must not be touched after cancel")
monkeypatch.setattr(
tools_mod,
"_validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: _Opener())
err, body, content_type = tools_mod._fetch_url_raw(
"https://example.com/",
cancel_event = ev,
)
assert err == "Failed to fetch URL: cancelled."
assert opened["n"] == 0
def test_fetch_page_text_shares_one_deadline_across_readme_and_fallback(monkeypatch):
seen_deadlines = []
def fake_fetch(
url,
timeout = 30,
extra_headers = None,
deadline = None,
cancel_event = None,
meta_out = None,
):
seen_deadlines.append(deadline)
return "Failed to fetch URL: HTTP 429 rate limited", "", ""
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
out = _fetch_page_text("https://github.com/unslothai/unsloth", timeout = 30)
assert out == "Failed to fetch URL: HTTP 429 rate limited"
assert len(seen_deadlines) == 2
assert seen_deadlines[0] is not None
assert seen_deadlines[0] == seen_deadlines[1]
def test_fetch_url_raw_deadline_aborts_slow_body(monkeypatch):
import email
import urllib.request
import core.inference.tools as tools_mod
clock = {"t": 1000.0}
monkeypatch.setattr(tools_mod.time, "monotonic", lambda: clock["t"])
class _DrippingResp:
headers = email.message_from_string("")
def read(self, n = -1):
clock["t"] += 10.0
return b"x" * 16
def close(self):
pass
class _Opener:
def open(
self,
req,
timeout = None,
):
return _DrippingResp()
monkeypatch.setattr(
tools_mod,
"_validate_and_resolve_host",
lambda host, port: (True, "", ["203.0.113.7"]),
)
monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: _Opener())
err, body, content_type = tools_mod._fetch_url_raw(
"https://example.com/",
timeout = 30,
deadline = clock["t"] + 5.0,
)
assert err == "Failed to fetch URL: timed out."
assert body == ""
def test_read_capped_body_deadline_bounds_a_real_socket_drip():
# read(n) waits for n bytes; only a real socket shows a one-byte drip past the deadline.
import socket
import threading
import urllib.request
from core.inference.tools import _read_capped_body
server = socket.create_server(("127.0.0.1", 0))
stop = threading.Event()
def _drip():
conn, _ = server.accept()
with conn:
conn.recv(4096)
conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 1000000\r\n\r\n")
try:
while not stop.is_set():
conn.sendall(b"x")
time.sleep(0.05)
except OSError:
pass
threading.Thread(target = _drip, daemon = True).start()
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
resp = opener.open(f"http://127.0.0.1:{server.getsockname()[1]}/", timeout = 15)
started = time.monotonic()
try:
err, _body = _read_capped_body(resp, 1_000_000, 15, started + 0.5, None)
except TimeoutError:
err = "timed out"
finally:
stop.set()
resp.close()
server.close()
assert err is not None
assert time.monotonic() - started < 5
def test_resolve_with_budget_aborts_on_slow_resolver(monkeypatch):
import threading
import core.inference.tools as tools_mod
clock = {"t": 1000.0}
monkeypatch.setattr(tools_mod.time, "monotonic", lambda: clock["t"])
release = threading.Event()
def slow_resolve(host, port):
release.wait(5.0)
return True, "", ["203.0.113.7"]
monkeypatch.setattr(tools_mod, "_validate_and_resolve_host", slow_resolve)
def advance_past_deadline():
import time as _t
_t.sleep(0.1)
clock["t"] += 100.0
t = threading.Thread(target = advance_past_deadline, daemon = True)
t.start()
try:
ok, reason, ip = tools_mod._resolve_with_budget(
"example.com",
443,
1005.0,
None,
)
finally:
release.set()
assert ok is False
assert reason == "Failed to fetch URL: timed out."
def test_web_search_query_cancelled_skips_search(monkeypatch):
import sys
import threading
import types
import core.inference.tools as tools_mod
ev = threading.Event()
ev.set()
called = {"n": 0}
class _DDGS:
def __init__(self, *a, **k):
called["n"] += 1
def text(self, *a, **k):
called["n"] += 1
return []
fake_mod = types.ModuleType("ddgs")
fake_mod.DDGS = _DDGS
monkeypatch.setitem(sys.modules, "ddgs", fake_mod)
out = tools_mod._web_search("some query", cancel_event = ev)
assert out == "Search cancelled."
assert called["n"] == 0
def test_looks_like_html_document_only_matches_real_documents():
from core.inference.tools import _looks_like_html_document
assert _looks_like_html_document("<!doctype html><html><body>x</body></html>")
assert _looks_like_html_document("\n <HTML lang='en'>")
assert _looks_like_html_document("<body><h1>x</h1></body>")
for frag in (
"<blockquote>q</blockquote>",
"<ul><li>x</li></ul>",
"<pre>x</pre>",
"<dl><dt>x</dt></dl>",
):
assert not _looks_like_html_document(frag), frag
def _interlanguage_list(count: int) -> str:
return "".join(
f'<li class="interlanguage-link">'
f'<a href="https://x{i}.wikipedia.org/wiki/K">Lang{i}</a></li>'
for i in range(count)
)
def test_in_main_header_language_list_does_not_displace_article():
body = (
"<main><header><h1>Cat</h1><div id='p-lang-btn'><ul>%s</ul></div></header>"
"<div id='mw-content-text'><p>%s</p></div></main>"
) % (_interlanguage_list(300), "The cat (Felis catus) is a small mammal. " * 30)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Felis catus" in out
assert "Lang0" not in out
assert "x0.wikipedia.org" not in out
assert "# Cat" in out
assert out.index("Felis catus") < 200
def test_header_title_kept_when_article_is_shorter_than_its_language_list():
body = ("<main><header><h1>Stub</h1><ul>%s</ul></header><p>%s</p></main>") % (
_interlanguage_list(300),
"Short article body. " * 15,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Short article body." in out
assert "Lang0" not in out
assert "# Stub" in out
def test_link_only_article_header_reduces_to_its_heading():
body = ("<article><header><h1>Post title</h1><ul>%s</ul></header><p>%s</p></article>") % (
_interlanguage_list(300),
"Real article content. " * 40,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "# Post title" in out
assert "Real article content." in out
assert "Lang0" not in out
def test_article_header_byline_and_date_are_kept():
body = (
"<article><header><h1>Why Rust</h1><p>By Jane Doe</p>"
"<time>2026-07-12</time><p>A summary of what this essay argues.</p></header>"
"<p>%s</p></article>"
) % ("Real article content. " * 40)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "# Why Rust" in out
assert "By Jane Doe" in out
assert "2026-07-12" in out
assert "A summary of what this essay argues." in out
def test_small_link_header_is_left_alone():
body = (
"<article><header><h1>Post title</h1>"
"<a href='/subscribe'>Subscribe now</a></header><p>%s</p></article>"
) % ("Real article content. " * 40)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Subscribe now" in out
assert "Real article content." in out
def test_unclosed_header_does_not_swallow_the_body():
body = "<main><header><h1>Title</h1><p>%s</p></main>" % ("Article body text. " * 40)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Article body text." in out
assert "Title" in out
def test_unclosed_header_with_many_headings_keeps_body():
sections = "".join(f"<h2>Section {i}</h2><p>{'Body prose here. ' * 10}</p>" for i in range(12))
body = f"<main><header><h1>T</h1>{sections}</main>"
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Body prose here." in out
assert "Section 0" in out
def test_header_strip_applies_without_article_or_main():
body = ("<header><h1>Site name</h1><ul>%s</ul></header><p>%s</p>") % (
_interlanguage_list(300),
"Page prose without a main landmark. " * 20,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Page prose without a main landmark." in out
assert "# Site name" in out
assert "Lang0" not in out
def test_header_kept_in_unscoped_conversion():
body = "<header><h1>Site</h1><a href='/x'>Nav link</a></header><p>Text.</p>"
out = html_to_markdown(f"<body>{body}</body>")
assert "Nav link" in out
assert "Text." in out
def test_unclosed_header_in_truncated_scope_keeps_body():
sections = "".join(
f"<h2>Section {i} of the article</h2><p>{'Body prose here. ' * 10}</p>" for i in range(12)
)
out = html_to_markdown(
f"<body><main><header><h1>T</h1>{sections}",
main_content = True,
)
assert "Body prose here." in out
assert "Section 0 of the article" in out
def test_sibling_card_does_not_beat_an_article_with_a_swallowed_body():
real = "<article><header><h1>Real</h1><p>%s</p></article>" % ("Real article body. " * 40)
card = "<article><p>%s</p></article>" % ("Related card teaser. " * 12)
out = html_to_markdown(f"<body>{real}{card}</body>", main_content = True)
assert "Real article body." in out
assert "Related card teaser." not in out
def test_text_heavy_header_is_kept_even_when_longer_than_the_body():
body = "<main><header><h1>Title</h1><p>%s</p></header><p>%s</p></main>" % (
"Introductory hero text. " * 30,
"The real body prose. " * 20,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "The real body prose." in out
assert "Introductory hero text." in out
assert "# Title" in out
def test_unclosed_link_in_unclosed_header_keeps_the_body():
body = "<main><header><h1>Title</h1><a href='/'>Home<p>%s</p>" % ("Article body text. " * 40)
out = html_to_markdown(f"<body>{body}", main_content = True)
assert "Article body text." in out
assert "# Title" in out
def test_unclosed_link_does_not_hand_the_scope_to_a_sibling_card():
real = "<article><header><h1>Real</h1><a href='/'>Home<p>%s</p></article>" % ("REAL " * 60)
card = "<article><p>%s</p></article>" % ("CARD " * 30)
out = html_to_markdown(f"<body>{real}{card}</body>", main_content = True)
assert "REAL" in out
assert "CARD" not in out
def test_entity_encoded_body_is_not_lost_to_an_unclosed_header():
out = html_to_markdown(
"<body><main><header><h1>T</h1><p>%s</p>" % ("α" * 400),
main_content = True,
)
assert out.count("α") == 400
def test_header_closed_by_an_ancestor_is_kept_whole():
body = "<div><header><h1>Site</h1><ul>%s</ul></div><p>%s</p>" % (
_interlanguage_list(300),
"The real body prose. " * 20,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "The real body prose." in out
assert "# Site" in out
assert "Lang0" in out
def test_unclosed_header_does_not_strip_a_short_article_it_adopted():
body = "<main><header><h1>T</h1><ul>%s</ul><p>%s</p></main>" % (
_interlanguage_list(300),
"Short real article. " * 4,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Short real article." in out
def test_heading_inside_a_nested_buffer_is_kept_and_the_links_still_go():
body = (
"<main><header><blockquote><h1>Page Title</h1></blockquote><ul>%s</ul></header><p>%s</p></main>"
% (
_interlanguage_list(400),
"Article body. " * 30,
)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Page Title" in out
assert "Lang0" not in out
assert out.index("Article body.") < 16000
def test_long_hrefs_count_toward_the_header_size_floor():
nav = "".join('<a href="https://e.com/p?%s=%d">L%d</a>' % ("q" * 1000, i, i) for i in range(30))
body = "<main><header>%s</header><p>%s</p></main>" % (nav, "Article body. " * 30)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "L0" not in out
assert out.index("Article body.") < 16000
@pytest.mark.parametrize(
"template, filler, repeats, present, also_present",
[
pytest.param(
"<article><header><h1><a href='/p'>%s</a></h1><p>By Jane Doe, July 2026</p></header><p>%s</p></article>",
"A Very Long Linked Headline About Assorted Things In The World Today ",
5,
"By Jane Doe",
"Very Long Linked",
id = "linked_heading_does_not_condemn_the_byline_beside_it",
),
pytest.param(
"<article><header><h1>T</h1><a name='intro'>%s</a><p>Byline</p></header><p>%s</p></article>",
"Introductory prose that renders as plain text. ",
8,
"Introductory prose",
"Byline",
id = "anchor_without_href_is_prose_not_link_furniture",
),
pytest.param(
"<article><header><h1><a href='/p?%s'>Title</a></h1>"
"<a href='/author/jane'>Jane</a></header><p>%s</p></article>",
"q",
900,
"Title",
"Jane",
id = "a_long_heading_href_does_not_condemn_the_rest_of_the_header",
),
],
)
def test_header_link_density_is_measured_on_rendered_text(
template, filler, repeats, present, also_present
):
body = template % (filler * repeats, "Article body. " * 30)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert present in out
assert also_present in out
def test_linked_heading_is_not_emitted_twice():
body = (
"<main><header><h1><a href='/post'>Title</a></h1><ul>%s</ul></header><p>%s</p></main>"
% (
_interlanguage_list(300),
"Article body. " * 30,
)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "# [Title](/post)" in out
assert out.count("Title") == 1
def test_article_still_beats_a_bigger_card_after_its_header_goes():
real = "<article><header><h1>Real</h1><ul>%s</ul></header><p>%s</p></article>" % (
_interlanguage_list(300),
"Short real body. " * 14,
)
card = "<article><p>%s</p></article>" % ("Related card teaser text here. " * 12)
out = html_to_markdown(f"<body>{real}{card}</body>", main_content = True)
assert "Short real body." in out
assert "Related card teaser" not in out
def test_furniture_only_card_does_not_suppress_the_main_it_sits_in():
card = "<article><header>%s</header><p>%s</p></article>" % (
"".join('<a href="/l%d">Language %d</a>' % (i, i) for i in range(120)),
"Short teaser about the related thing, read more.",
)
body = "<p>%s</p>" % ("The real article body the reader wants. " * 40)
out = html_to_markdown(f"<body><main>{body}{card}</main></body>", main_content = True)
assert "The real article body the reader wants." in out
assert "Language 7" not in out
def test_a_preserved_heading_is_terminated():
body = "<main><header><h1>Title</h1><ul>%s</ul></header>Article body text here.</main>" % (
_interlanguage_list(300)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "# TitleArticle" not in out
assert out.startswith("# Title")
assert "Article body text here." in out
def test_scope_holding_only_furniture_does_not_win_or_blank_the_page():
body = "<main><header><ul>%s</ul></header></main><p>%s</p>" % (
_interlanguage_list(300),
"Real page body prose. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Real page body prose." in out
def test_header_inside_an_enclosing_link_renders_once():
body = "<main><a href='/x'><header><h1>T</h1></header></a><p>%s</p></main>" % (
"Article body. " * 30
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert out.count("[# T](/x)") == 1
def test_table_nested_in_a_header_inside_a_cell_keeps_its_columns():
html = "<table><tr><td><header><table><tr><td>x</td></tr></table></header></td></tr></table>"
assert html_to_markdown(f"<body>{html}</body>", main_content = True) == html_to_markdown(
f"<body>{html}</body>"
)
continued = "<table><tr><td><header><table><tr><td>x</td></tr></table></header>after</td><td>sibling</td></tr></table>"
assert html_to_markdown(continued, main_content = True) == html_to_markdown(continued)
def test_nested_tables_flatten_into_their_outer_cell():
nested = "<table><tr><td>A</td><td><table><tr><td>x | y</td><td>z</td></tr></table></td><td>C</td></tr></table>"
assert html_to_markdown(nested).splitlines() == ["| A | x \\| y z | C |", "| --- | --- | --- |"]
deep = "<table><tr><td>" * 500 + "x" + "</td></tr></table>" * 500
assert html_to_markdown(deep) == "| x |\n| --- |"
article = "<article>" + "<table><tr><td>" * 8 + "x" + "</td></tr></table>" * 8 + "</article>"
main = "<main>" + "<p>Real page body.</p>" * 20 + "</main>"
assert "Real page body." in html_to_markdown(article + main, main_content = True)
def test_spanned_table_cells_stay_in_their_columns():
html = (
"<table>"
"<tr><th>Rank</th><th>Nation</th><th>Gold</th><th>Silver</th><th>Bronze</th><th>Total</th></tr>"
"<tr><td rowspan='2'>25</td><th>Latvia</th><td>0</td><td>1</td><td>0</td><td>1</td></tr>"
"<tr><th>Estonia</th><td>0</td><td>1</td><td>0</td><td>1</td></tr>"
"<tr><td>27</td><th>Spain</th><td>0</td><td>0</td><td>1</td><td rowspan='2'>1</td></tr>"
"<tr><td>28</td><th>Chile</th><td>0</td><td>0</td><td>1</td></tr>"
"<tr><th colspan='2'>Totals (5 entries)</th><td>0</td><td>2</td><td>2</td><td>4</td></tr>"
"</table>"
)
lines = html_to_markdown(f"<body>{html}</body>").splitlines()
assert lines == [
"| Rank | Nation | Gold | Silver | Bronze | Total |",
"| --- | --- | --- | --- | --- | --- |",
"| 25 | Latvia | 0 | 1 | 0 | 1 |",
"| 25 | Estonia | 0 | 1 | 0 | 1 |",
"| 27 | Spain | 0 | 0 | 1 | 1 |",
"| 28 | Chile | 0 | 0 | 1 | 1 |",
"| Totals (5 entries) | | 0 | 2 | 2 | 4 |",
]
def test_spanned_site_links_strip_on_every_repeated_row():
html = (
"<table><tr><th>Rank</th><th>Nation</th></tr>"
"<tr><td rowspan='2'><a href='/r25'>25</a></td><td>Estonia</td></tr>"
"<tr><td>Georgia</td></tr>"
"<tr><td colspan='2'><a href='https://ex.com/t'>Totals | all</a></td></tr></table>"
)
site_links = SiteLinks("https://ex.com/page")
full = html_to_markdown(f"<body>{html}</body>", site_links = site_links)
assert full.splitlines()[2:] == [
"| [25](/r25) | Estonia |",
"| [25](/r25) | Georgia |",
"| [Totals \\| all](https://ex.com/t) | |",
]
assert site_links.strip(full).splitlines()[2:] == [
"| 25 | Estonia |",
"| 25 | Georgia |",
"| Totals \\| all | |",
]
def test_table_spans_do_not_multiply_the_page_size():
small_wide = "<table><tr>" + "<td colspan='1000'>x</td>" * 3 + "</tr></table>"
wide = "<table><tr>" + "<td colspan='1000'>x" * 30000 + "</table>"
tall = "<table><tr><td rowspan='65534'>" + "word " * 2000 + "<tr><td>b" * 5000 + "</table>"
for html in (small_wide, wide, tall):
assert len(html_to_markdown(html)) < 3 * len(html)
repeated = (
"<table><tr><td rowspan='1001'>"
+ "x" * 100
+ "</td></tr>"
+ "<tr></tr>" * 1000
+ "</table>"
)
rendered = html_to_markdown(repeated + "<p>Article sentinel.</p>")
assert rendered.index("Article sentinel.") < 16_000
quoted = (
"<blockquote><table><tr><td rowspan='3000'>x</td></tr>"
+ "<tr></tr>" * 2999
+ "</table></blockquote><p>Article sentinel.</p>"
)
assert html_to_markdown(quoted).index("Article sentinel.") < 16_000
nested = (
"<table><tr><td>"
+ "s" * 7000
+ "<table><tr><td rowspan='3000'>x</td></tr>"
+ "<tr></tr>" * 2999
+ "</table></td></tr></table><p>Article sentinel.</p>"
)
assert html_to_markdown(nested).index("Article sentinel.") < 16_000
def test_table_spans_fit_a_small_fetch_budget(monkeypatch):
tall = (
"<html><body><table><tr><td rowspan='3000'>x</td></tr>"
+ "<tr></tr>" * 2999
+ "</table><p>Article sentinel.</p></body></html>"
)
assert html_to_markdown(tall, max_span_chars = 1000).index("Article sentinel.") < 2000
def fake_fetch(url, **kwargs):
return None, tall, "text/html"
monkeypatch.setattr("core.inference.tools._fetch_url_raw", fake_fetch)
assert "Article sentinel." in _fetch_page_text("https://example.com/t", max_chars = 2000)
def test_table_span_budget_is_page_wide_and_token_weighted():
from core.inference._html_to_md import _new_renderer
candidate = "<article><table><tr><td colspan='1000'>x</td></tr></table></article>"
page = candidate * 3000
renderer = _new_renderer(page, frozenset({"article"}), True)
assert sum(len(seg) for seg in renderer.scope_segments) < 10 * len(page)
cell = "中文表格单元内容很长" * 20
rows = "".join(f"<tr><td>r{i}</td></tr>" for i in range(1, 50))
html = f"<table><tr><td rowspan='50'>{cell}</td><td>r0</td></tr>{rows}</table><p>Article sentinel.</p>"
assert html_to_markdown(html, max_span_chars = 3404).count(cell) == 1
def test_nested_colspan_padding_does_not_spend_the_span_budget():
layout = "<table><tr><td><table><tr><td colspan='1000'>x</td></tr></table></td></tr></table>"
data = (
"<table><tr><th>Rank</th><th>Nation</th></tr>"
"<tr><td rowspan='2'>25</td><td>Estonia</td></tr><tr><td>Georgia</td></tr></table>"
)
pad = "<p>" + "Body text. " * 400 + "</p>"
assert html_to_markdown(pad + layout + data).splitlines()[-1] == "| 25 | Georgia |"
def test_generated_table_spans_do_not_make_a_main_content_candidate():
body = "<main><p>" + "Real page body. " * 20 + "</p></main>"
decoys = (
"<article><table><tr><td rowspan='51'>x</td></tr>"
+ "<tr></tr>" * 50
+ "</table></article>",
"<article><blockquote><table><tr><td rowspan='101'>x</td></tr>"
+ "<tr></tr>" * 100
+ "</table></blockquote></article>",
"<article><header><table><tr><td colspan='1000'><a href='/nav'>"
+ "Navigation " * 12
+ "</a></td></tr></table></header><p>"
+ "Teaser. " * 12
+ "</p></article>",
)
for decoy in decoys:
out = html_to_markdown(f"<body>{decoy}{body}</body>", main_content = True)
assert "Real page body." in out
exhausted = (
"<article><table><tr><td colspan='1000' rowspan='2'>x</td></tr><tr></tr></table></article>"
)
substantive = (
"<article><p>"
+ "Real page body. " * 20
+ "</p><table><tr><td rowspan='2'>rank</td><td>one</td></tr><tr><td>two</td></tr></table></article>"
)
assert "| rank | two |" in html_to_markdown(exhausted + substantive, main_content = True)
def test_table_spans_stop_at_row_groups_tables_and_long_cells():
def rows(html):
return html_to_markdown(f"<body>{html}</body>").splitlines()
covered_row = "<table><tr><th>A</th><th>B</th></tr><tr><td rowspan=2>a</td><td rowspan=2>b</td></tr><tr></tr><tr><td>c</td><td>d</td></tr></table>"
assert rows(covered_row)[-3:] == ["| a | b |", "| a | b |", "| c | d |"]
footer = "<table><thead><tr><th>Item</th><th>Qty</th></tr></thead><tbody><tr><td>x</td><td rowspan=2>5</td></tr></tbody><tfoot><tr><td>Total</td><td>5</td></tr></tfoot></table>"
assert rows(footer)[-1] == "| Total | 5 |"
nested = "<table><tr><td>T1</td><td><table><tr><td rowspan=2>GK</td><td>P1</td></tr></table></td></tr><tr><td>T2</td><td>S2</td></tr></table>"
assert rows(nested)[-1] == "| T2 | S2 |"
nested_under_span = "<table><tr><td rowspan=2>A</td><td><table><tr><td>B</td></tr></table></td></tr><tr><td>C</td></tr></table>"
assert rows(nested_under_span)[-1] == "| A | C |"
to_group_end = "<table><tr><th>G</th><th>N</th></tr><tbody><tr><th rowspan=0>g</th><td>1</td></tr><tr><td>2</td></tr></tbody><tbody><tr><td>h</td><td>3</td></tr></tbody></table>"
assert rows(to_group_end)[-3:] == ["| g | 1 |", "| g | 2 |", "| h | 3 |"]
sidebar = (
"<table><tr><td rowspan=5>"
+ "Article body text. " * 120
+ "</td><td>Home</td></tr>"
+ "<tr><td>Link</td></tr>" * 4
+ "</table>"
)
assert html_to_markdown(sidebar).count("Article body text.") == 120
def test_truncated_header_and_blockquote_keep_source_order():
out = html_to_markdown("<body><main><header><h1>Title</h1><blockquote>Quote", main_content = True)
assert out.index("Title") < out.index("Quote")
# headers interact with every buffer, where one-off bugs occur
_GRID_HEADINGS = {
"h1": "<h1>Page Title</h1>",
"aria": "<div role='heading'>Page Title</div>",
"in_quote": "<blockquote><h1>Page Title</h1></blockquote>",
"is_link": "<a role='heading' href='/t'>Page Title</a>",
"none": "",
}
_GRID_WRAPPERS = {
"bare": ("", ""),
"quote": ("<blockquote>", "</blockquote>"),
"cell": ("<table><tr><td>", "</td></tr></table>"),
"link": ("<a href='/x'>", "</a>"),
"item": ("<ul><li>", "</li></ul>"),
"pre": ("<pre>", "</pre>"),
}
_GRID_NESTED = {
"bare": ("", ""),
"quote": ("<blockquote>", "</blockquote>"),
"cell": ("<table><tr><td>", "</td></tr></table>"),
"pre": ("<pre>", "</pre>"),
}
_GRID_BODY = "Article body sentence. " * 30
@pytest.mark.parametrize("heading", sorted(_GRID_HEADINGS))
@pytest.mark.parametrize("wrapper", sorted(_GRID_WRAPPERS))
@pytest.mark.parametrize("nested", sorted(_GRID_NESTED))
@pytest.mark.parametrize("inner", ("link_dense", "content"))
@pytest.mark.parametrize("close_header", (True, False))
@pytest.mark.parametrize("close_nested", (True, False))
def test_header_survives_every_buffer_combination(
heading, wrapper, nested, inner, close_header, close_nested
):
open_wrap, close_wrap = _GRID_WRAPPERS[wrapper]
open_nest, close_nest = _GRID_NESTED[nested]
payload = (
f"<ul>{_interlanguage_list(300)}</ul>"
if inner == "link_dense"
else "<p>By Jane Doe, 12 July 2026</p><p>A summary of the argument.</p>"
)
header = (
f"<header>{_GRID_HEADINGS[heading]}{open_nest}{payload}"
f"{close_nest if close_nested else ''}{'</header>' if close_header else ''}"
)
html = (
f"<body><main>{open_wrap}{header}{close_wrap if close_header else ''}"
f"<p>{_GRID_BODY}</p></main></body>"
)
out = html_to_markdown(html, main_content = True)
assert "Article body sentence." in out, "body lost"
assert out.strip(), "empty output"
assert out.index("Article body sentence.") < 16000, "body pushed past the fetch cap"
well_formed = close_header and close_nested and "pre" not in (wrapper, nested)
if _GRID_HEADINGS[heading] and well_formed:
assert out.count("Page Title") == 1, "title duplicated or lost"
def test_a_stub_cannot_outrank_a_real_article_on_removed_navigation():
stub = "<article><header><h1>%s</h1><ul>%s</ul></header></article>" % (
"A Long Title That Exceeds Fifty Characters Easily Here",
_interlanguage_list(300),
)
real = "<article><p>%s</p></article>" % ("Genuine full article body text. " * 20)
out = html_to_markdown(f"<body>{stub}{real}</body>", main_content = True)
assert "Genuine full article body" in out
def test_unclosed_link_in_a_closed_header_counts_as_furniture():
body = "<main><header><h1>T</h1><a href='/nav'>%s</header><p>%s</p></main>" % (
"Navigation label text. " * 20,
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Navigation label text." not in out
assert "Article body." in out
def test_enclosing_anchor_text_counts_toward_header_density():
body = "<main><a href='/nav'><header>%s</header></a><p>%s</p></main>" % (
"".join(f"<span>Nav label {i}</span>" for i in range(40)),
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Nav label 0" not in out
assert "Article body." in out
def test_truncated_header_flushes_into_its_enclosing_buffer():
assert html_to_markdown(
"<body><main><a href='/x'>before<header><h1>Title", main_content = True
).startswith("[before")
assert html_to_markdown(
"<body><main><table><tr><td><header><h1>Title", main_content = True
).startswith("|")
def test_empty_blocks_do_not_inflate_the_header_size():
body = (
"<article><header><h1>%s</h1>%s<a href='/x'>L</a></header></article>"
"<article><p>%s</p></article>"
% (
"A Fairly Long Heading Title Here" * 2,
"<div></div>" * 500,
"Genuine full article body text. " * 20,
)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Genuine full article body" in out
def test_blockquoted_heading_is_not_counted_as_retained_prose():
stub = "<article><blockquote><header><h1>%s</h1><ul>%s</ul></header></blockquote></article>" % (
"A Long Title That Exceeds Fifty Characters Easily Here",
_interlanguage_list(300),
)
real = "<article><p>%s</p></article>" % ("Genuine full article body text. " * 20)
out = html_to_markdown(f"<body>{stub}{real}</body>", main_content = True)
assert "Genuine full article body" in out
def test_list_left_open_in_a_header_does_not_indent_the_body():
body = "<main><header><h1>T</h1><ul>%s</header><ul><li>Body item one</li></ul></main>" % (
_interlanguage_list(300)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
item = next(line for line in out.split("\n") if "Body item one" in line)
assert item.startswith("*"), item
def test_deeply_nested_headers_do_not_blow_up_quadratically():
chunk = "<p>%s</p>" % ("filler text here. " * 100)
def build(depth):
return (
"<body><main>"
+ "".join(f"<header>{chunk}" for _ in range(depth))
+ "</header>" * depth
+ "<p>body</p></main></body>"
)
import time
timings = []
for depth in (20, 80):
html = build(depth)
runs = []
# Best of five: one descheduled call on a shared runner must not read as growth.
for _ in range(5):
start = time.perf_counter()
html_to_markdown(html, main_content = True)
runs.append(time.perf_counter() - start)
timings.append(min(runs))
assert timings[1] < timings[0] * 12, timings
def test_linked_heading_text_is_counted_once_for_the_size_floor():
body = (
"<article><header><h1><a href='/p'>%s</a></h1>"
"<a href='/author'>Jane</a></header><p>%s</p></article>"
% ("Headline word " * 60, "Article body. " * 30)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Jane" in out
def test_fenced_code_counts_as_retained_content():
code = "<pre>%s</pre>" % "\n".join("# comment line %d" % i for i in range(16))
article = "<article><header><h1>T</h1><ul>%s</ul></header>%s</article>" % (
_interlanguage_list(300),
code,
)
sibling = "<article><p>%s</p></article>" % ("Sibling teaser text here. " * 12)
out = html_to_markdown(f"<body>{article}{sibling}</body>", main_content = True)
assert "comment line 1" in out
assert "Sibling teaser" not in out
_FENCE = "`" * 3
def test_dropped_furniture_cannot_dominate_sibling_ranking():
teaser = "<article><header>%s</header><p>%s</p></article>" % (
"".join('<a href="/l%d">Lang%d</a>' % (i, i) for i in range(1000)),
"Teaser words here. " * 20,
)
real = "<article><p>%s</p></article>" % ("The genuine article body text goes here. " * 55)
out = html_to_markdown(f"<body>{teaser}{real}</body>", main_content = True)
assert "The genuine article body text goes here." in out
assert "Teaser words here." not in out
def test_literal_bracket_paren_is_prose_not_a_destination():
article = "<article><p>%s](%s) %s</p></article>" % (
"Real article prose that the reader wants to see. " * 3,
"y" * 100,
"Tail prose to finish the paragraph off here.",
)
sibling = "<article><p>%s</p></article>" % ("Unrelated sibling teaser words. " * 8)
out = html_to_markdown(f"<body>{article}{sibling}</body>", main_content = True)
assert "Real article prose that the reader wants to see." in out
assert "Unrelated sibling teaser" not in out
def test_hand_preserved_heading_reaches_the_eligibility_tally():
card = (
'<article><header><a href="/h"><div role="heading">%s</div>%s</a><ul>%s</ul></header></article>'
% (
"Card Title Words " * 14,
"nav text " * 40,
_interlanguage_list(300),
)
)
real = "<p>%s</p>" % ("The real page body text here. " * 30)
out = html_to_markdown(f"<body><main>{real}{card}</main></body>", main_content = True)
assert "The real page body text here." in out
def test_pre_inside_a_table_cell_is_drained_before_the_row():
body = "<main><header><h1>T</h1><table><tr><td><pre>CODEMARKER</header><p>%s</p></main>" % (
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "CODEMARKER|" not in out
assert "Article body." in out
def test_post_processing_respects_the_widened_fence():
code = "<pre>%s\nskip to content\n\nreal code line \nmore code</pre>" % _FENCE
body = "<article>%s<p>%s</p></article>" % (code, "Body text here. " * 20)
out = html_to_markdown(f"<body><main>{body}</main></body>", main_content = True)
assert "skip to content" in out
assert "skip to content\n\nreal code line" in out
assert "real code line " in out
def test_unbalanced_destination_keeps_scoring_the_rest_of_the_line():
article = '<article><p><a href="/docs/(draft">Doc</a> %s</p></article>' % (
"Substantial article prose continues here. " * 6,
)
sibling = "<article><p>%s</p></article>" % ("Sibling teaser text. " * 12)
out = html_to_markdown(f"<body>{article}{sibling}</body>", main_content = True)
assert "Substantial article prose continues here." in out
assert "Sibling teaser" not in out
def test_structural_headings_do_not_satisfy_the_eligibility_gate():
card = '<article><header><div role="heading">%s</div><ul>%s</ul></header></article>' % (
"Card Title Words " * 14,
_interlanguage_list(300),
)
real = "<article><p>%s</p></article>" % ("The real article body text here. " * 20)
out = html_to_markdown(f"<body>{real}{card}</body>", main_content = True)
assert "The real article body text here." in out
assert "Card Title Words" not in out
def test_anchor_wrapping_a_heading_preserves_only_the_heading():
bulk = " ".join("NavWord%04d" % i for i in range(1200))
body = (
'<main><header><a href="/home"><h1>Title</h1>%s</a><ul>%s</ul></header><p>%s</p></main>'
% (
bulk,
_interlanguage_list(300),
"Article body. " * 30,
)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Title" in out
assert "NavWord0000" not in out
assert out.index("Article body.") < 16000
def test_link_destination_scan_balances_parentheses():
query = "utm_source=x&" * 25
card = '<article><header>%s</header><p><a href="/card(foo)?%s">Read</a></p></article>' % (
"".join('<a href="/l%d">Lang%d</a>' % (i, i) for i in range(120)),
query,
)
real = "<article><p>%s</p></article>" % ("The genuine article body text here. " * 20)
out = html_to_markdown(f"<body>{real}{card}</body>", main_content = True)
assert "The genuine article body text here." in out
assert "/card(foo)?" not in out
def test_literal_fence_inside_pre_does_not_end_the_code_region():
code = "<pre>%s\n%s</pre>" % (
_FENCE,
"\n".join("# code line %d" % i for i in range(20)),
)
article = "<article><header><h1>T</h1><ul>%s</ul></header>%s</article>" % (
_interlanguage_list(300),
code,
)
sibling = "<article><p>%s</p></article>" % ("Sibling teaser text here. " * 12)
out = html_to_markdown(f"<body>{article}{sibling}</body>", main_content = True)
assert "code line 1" in out
assert "Sibling teaser" not in out
def test_heading_through_a_nested_buffer_is_emitted_once():
body = (
'<main><header><div role="heading"><blockquote>UniqueTitle</blockquote></div><ul>%s</ul></header><p>%s</p></main>'
% (
_interlanguage_list(300),
"Article body. " * 30,
)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert out.count("UniqueTitle") == 1
def test_late_code_end_tag_after_a_recovered_header_is_a_no_op():
body = "<main><header><h1>T</h1><code>navcode<ul>%s</ul></header><p>%s</p></code></main>" % (
_interlanguage_list(300),
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert out.count("`") % 2 == 0
assert "Article body." in out
def test_header_text_is_sized_after_whitespace_collapses():
byline = '<a href="/a">Jane%sDoe</a><time>July 2026</time>' % (" " * 300)
body = "<main><header><h1>T</h1>%s</header><p>%s</p></main>" % (
byline,
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Jane Doe" in out
assert "July 2026" in out
def test_link_destinations_do_not_count_as_retained_prose():
query = "utm_source=x&" * 25
card = '<article><header>%s</header><p><a href="/card?%s">Read</a></p></article>' % (
"".join('<a href="/l%d">Lang%d</a>' % (i, i) for i in range(120)),
query,
)
real = "<article><p>%s</p></article>" % ("The genuine article body text here. " * 20)
out = html_to_markdown(f"<body>{real}{card}</body>", main_content = True)
assert "The genuine article body text here." in out
assert "/card?" not in out
def test_late_end_tag_cannot_replay_a_recovered_pre_block():
body = "<main><header><h1>T</h1><ul>%s</ul><pre>%s</header><p>%s</p></pre></main>" % (
_interlanguage_list(300),
"NAVJUNK_MARKER\n" * 3,
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "NAVJUNK_MARKER" not in out
assert out.index("Article body.") < 16000
def test_aria_heading_accepts_a_fallback_role_token_list():
assert _is_aria_heading({"role": "heading"})
assert _is_aria_heading({"role": "future-role heading"})
assert _is_aria_heading({"role": "HEADING"})
assert not _is_aria_heading({"role": "banner"})
assert not _is_aria_heading({})
body = (
'<main><header><div role="future-role heading">Page Title</div><ul>%s</ul></header><p>%s</p></main>'
% (
_interlanguage_list(300),
"Article body. " * 30,
)
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Page Title" in out
assert "Lang0" not in out
@pytest.mark.parametrize(
("heading_markup", "marker"),
[
("<h1>Page Title</h1>", "Page Title"),
("<table><tr><td><h1>Page Title</h1></td></tr></table>", "Page Title"),
("<div role='heading' aria-level='1'>Page Title</div>", "Page Title"),
("<a role='heading' href='/title'>Page Title</a>", "Page Title"),
("<a href='/post'><h1>Page Title</h1></a>", "Page Title"),
("<hgroup><h1>Title</h1><p>Subtitle here</p></hgroup>", "Subtitle here"),
],
)
def test_heading_survives_a_stripped_header(heading_markup, marker):
body = "<main><header>%s<ul>%s</ul></header><p>%s</p></main>" % (
heading_markup,
_interlanguage_list(300),
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert marker in out
assert "Lang0" not in out
assert "Article body." in out
@pytest.mark.parametrize(
"body_markup",
[
"<main><blockquote><header><h1>T</h1><ul>{links}</ul></header></blockquote><p>{body}</p></main>",
"<main><header><h1>T</h1><blockquote><ul>{links}</ul></header><p>{body}</p></main>",
"<main><header><h1>T</h1><table><tr><td><ul>{links}</ul></header><p>{body}</p></main>",
],
)
def test_link_list_is_stripped_through_a_nested_buffer(body_markup):
body = body_markup.format(links = _interlanguage_list(300), body = "Article body. " * 30)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert "Lang0" not in out
assert out.index("Article body.") < 16000
def test_hash_prefixed_prose_still_wins_its_scope():
lines = "".join("<p>#include <header_%02d.h></p>" % i for i in range(14))
article = "<article><header><h1>T</h1><ul>%s</ul></header>%s</article>" % (
_interlanguage_list(300),
lines,
)
sibling = "<article><p>%s</p></article>" % ("Sibling teaser text here. " * 12)
out = html_to_markdown(f"<body>{article}{sibling}</body>", main_content = True)
assert "#include" in out
assert "Sibling teaser" not in out
def test_header_size_is_independent_of_the_buffer_it_renders_through():
links = "".join(
'<a href="/very/long/section/path/number/%03d/index">L%03d</a>' % (i, i) for i in range(14)
)
wrapped = {
"bare": links,
"blockquote": f"<blockquote>{links}</blockquote>",
"cell": f"<table><tr><td>{links}</td></tr></table>",
}
kept = set()
for inner in wrapped.values():
body = "<main><header><h1>T</h1>%s</header><p>%s</p></main>" % (
inner,
"Article body. " * 30,
)
kept.add("L000" in html_to_markdown(f"<body>{body}</body>", main_content = True))
assert len(kept) == 1
def test_nested_inline_code_closes_every_span_it_opened():
body = "<main><article><p><code><code>x</code></code></p><p>%s</p></article></main>" % (
"Body text here. " * 20,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert out.count("`") % 2 == 0
assert "``x``" in out
def test_header_inside_open_inline_code_leaves_delimiters_paired():
body = "<main><code>head<header><h1>T</h1></header>tail</code><p>%s</p></main>" % (
"Article body. " * 30,
)
out = html_to_markdown(f"<body>{body}</body>", main_content = True)
assert out.count("`") % 2 == 0
assert "`head`" not in out
assert "Article body." in out
def test_nested_blockquote_prose_does_not_backtrack():
# Heading detection must scan linearly: a regex backtracks exponentially on "> > > ... prose".
import time
html = (
"<body><main>"
+ "<blockquote>" * 40
+ "<p>prose text here</p>"
+ "</blockquote>" * 40
+ "</main></body>"
)
start = time.perf_counter()
html_to_markdown(html, main_content = True)
assert time.perf_counter() - start < 1.0
def test_deeply_nested_tags_stay_linear():
import time
def build(count):
return (
"<body><main>"
+ "<header><h1>T</h1>" * count
+ "</header>" * count
+ "<p>body</p></main></body>"
)
timings = []
for count in (1000, 4000):
html = build(count)
runs = []
# Best of five: one descheduled call on a shared runner must not read as growth.
for _ in range(5):
start = time.perf_counter()
html_to_markdown(html, main_content = True)
runs.append(time.perf_counter() - start)
timings.append(min(runs))
assert timings[1] < timings[0] * 12, timings
_KATEX_HTML_COPY = (
'<span class="katex-html" aria-hidden="true"><span class="base">'
'<span class="mord mathnormal">y</span><span class="mrel">=</span>'
'<span class="mord mathnormal">x</span><span class="mord mathnormal">A</span>'
'<span class="mord mathnormal">T</span><span class="mbin">+</span>'
'<span class="mord mathnormal">b</span></span></span>'
)
@pytest.mark.parametrize(
"markup, expected",
[
(
'incoming data: <span class="katex"><span class="katex-mathml"><math '
'xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>y</mi><mo>=</mo>'
"<mi>x</mi><msup><mi>A</mi><mi>T</mi></msup><mo>+</mo><mi>b</mi></mrow>"
'<annotation encoding="application/x-tex">y = xA^T + b</annotation></semantics>'
f"</math></span>{_KATEX_HTML_COPY}</span>.",
"incoming data: $y = xA^T + b$.",
),
(
'keys of dimension <math class="ltx_Math" alttext="d_{k}" display="inline">'
"<semantics><msub><mi>d</mi><mi>k</mi></msub>"
'<annotation-xml encoding="MathML-Content"><apply><csymbol cd="ml">subscript'
"</csymbol><ci>d</ci><ci>k</ci></apply></annotation-xml>"
'<annotation encoding="application/x-tex">d_{k}</annotation>'
'<annotation encoding="application/x-llamapun">ITALIC_d start_POSTSUBSCRIPT'
"</annotation></semantics></math>, and values",
"keys of dimension $d_{k}$, and values",
),
(
'the form <math xmlns="http://www.w3.org/1998/Math/MathML"><mrow>\n<mstyle><mi>a'
"</mi><msup><mi>x</mi><mrow><mn>2</mn></mrow></msup><mo>+</mo><mi>b</mi><mi>x</mi>"
"<mo>+</mo><mi>c</mi><mo>=</mo><mn>0</mn></mstyle>\n</mrow></math>, with",
"the form $ax^{2}+bx+c=0$, with",
),
(
"roots <math><mi>x</mi><mo>=</mo><mfrac><mrow><mo>−</mo><mi>b</mi>"
"<mo>±</mo><msqrt><msup><mi>b</mi><mn>2</mn></msup><mo>−</mo><mn>4</mn>"
"<mi>a</mi><mi>c</mi></msqrt></mrow><mrow><mn>2</mn><mi>a</mi></mrow></mfrac></math>",
"roots $x=\\frac{−b±\\sqrt{b^{2}−4ac}}{2a}$",
),
(
"sum <math><munderover><mo>∑</mo><mrow><mi>i</mi><mo>=</mo><mn>1</mn></mrow>"
"<mi>n</mi></munderover><msub><mi>x</mi><mi>i</mi></msub></math> and "
"<math><mi>sin</mi><mo>⁡</mo><mi>x</mi></math>",
"sum $∑_{i=1}^{n}x_{i}$ and $sin x$",
),
(
"<math><mtable><mtr><mtd><mi>a</mi></mtd><mtd><mi>b</mi></mtd></mtr>"
"<mtr><mtd><mi>c</mi></mtd><mtd><mi>d</mi></mtd></mtr></mtable></math>",
"$a & b \\\\ c & d$",
),
(
"set <math><mo>{</mo><mover><mi>x</mi><mo>¯</mo></mover><mo>></mo><mn>50</mn>"
"<mi>%</mi><mo>}</mo><mtext> for all </mtext><mi>n</mi>"
"<mphantom><mi>y</mi></mphantom></math>",
"set $\\{\\overset{¯}{x}>50\\%\\}\\text{ for all }n$",
),
],
)
def test_formulas_reach_the_model_once(markup, expected):
assert html_to_markdown(f"<p>{markup}</p>") == expected
def test_display_formula_gets_its_own_paragraph():
html = (
'<p>We compute:</p><math alttext="E=mc^{2}" display="block"><mi>E</mi></math>'
"<p>where c is the speed of light.</p>"
)
assert html_to_markdown(html) == (
"We compute:\n\n$$E=mc^{2}$$\n\nwhere c is the speed of light."
)
@pytest.mark.parametrize(
"html",
[
'<p>Shown</p><div hidden><math alttext="x"><mi>x</mi></math></div>',
'<p>Shown</p><math style="display:none" alttext="x"><mi>x</mi></math>',
'<p>Shown</p><nav><math alttext="x"><mi>x</mi></math></nav>',
"<p>Shown</p><math><mi></mi></math>",
],
)
def test_hidden_or_empty_formulas_stay_out(html):
assert html_to_markdown(html) == "Shown"
def test_formula_counts_as_article_text():
prose = "Attention weighs every key against the query. " * 6
html = (
f"<nav>Menu</nav><article><p>{prose}keys of dimension "
'<math alttext="d_{k}"><mi>d</mi></math>.</p></article>'
)
out = html_to_markdown(html, main_content = True)
assert out.count("$d_{k}$") == 1 and "Menu" not in out
@pytest.mark.parametrize(
"html",
[
'<p>Value <math alttext=" "><mi>x</mi></math> here.</p>',
'<p>Value <math alttext="x"><semantics><mi>y</mi>'
'<annotation encoding="application/x-tex"> </annotation></semantics></math> here.</p>',
],
)
def test_blank_tex_source_falls_back(html):
assert html_to_markdown(html) == "Value $x$ here."
@pytest.mark.parametrize("main_content", [False, True])
@pytest.mark.parametrize(
"tail",
[
"<math><mi>x</mi><mo>+</mo><mi>y</mi>",
'<math alttext="x+y"><semantics><mi>z</mi><annotation encoding="application/x-tex">x+',
"<math><semantics><mi>x</mi><mo>+</mo><mi>y</mi>"
'<annotation encoding="application/x-tex">x+',
],
)
def test_formula_cut_off_at_end_of_page_is_kept(tail, main_content):
assert html_to_markdown(f"<p>Then {tail}", main_content = main_content) == "Then $x+y$"