Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
test: guard donor navigation across public pages (#3421)
zhifu gaoefee7346484e
diff --git a/scripts/check_funasr_website_static.py b/scripts/check_funasr_website_static.pyindex a152ee070..560531134 100755--- a/scripts/check_funasr_website_static.py+++ b/scripts/check_funasr_website_static.py@@ -4,12 +4,15 @@from __future__ import annotationsimport argparse+from concurrent.futures import ThreadPoolExecutorfrom html.parser import HTMLParserimport reimport sysimport timeimport urllib.requestfrom dataclasses import dataclass+from urllib.parse import urljoin, urlparse+import xml.etree.ElementTree as ETBASE_URL = "https://www.funasr.com"@@ -485,6 +488,104 @@ def extract_images(html: str) -> set[str]:return _collect_visible_content(html).images+class _DirectoryNavigationCollector(HTMLParser):+ def __init__(self) -> None:+ super().__init__()+ self.links: list[str] = []+ self._depth = 0++ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ tag = tag.lower()+ attributes = {name.lower(): value for name, value in attrs}+ if self._depth:+ if tag == "a" and attributes.get("href"):+ self.links.append(attributes["href"])+ if tag not in _VOID_TAGS:+ self._depth += 1+ elif tag == "div" and "nav-links" in (attributes.get("class") or "").split():+ self._depth = 1++ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ if not self._depth or tag.lower() != "a":+ return+ attributes = {name.lower(): value for name, value in attrs}+ if attributes.get("href"):+ self.links.append(attributes["href"])++ def handle_endtag(self, tag: str) -> None:+ if self._depth:+ self._depth -= 1+++def extract_directory_links(html: str) -> list[str]:+ parser = _DirectoryNavigationCollector()+ parser.feed(html)+ parser.close()+ return parser.links+++def validate_navigation(pages: dict[str, str]) -> list[str]:+ failures: list[str] = []+ for url, html in pages.items():+ links = extract_directory_links(html)+ if not links:+ continue+ path = urlparse(url).path+ is_english = path == "/en" or path.startswith("/en/")+ expected = "/en/donors.html" if is_english else "/donors.html"+ wrong_language = "/donors.html" if is_english else "/en/donors.html"+ if is_english:+ alternate_path = path.removeprefix("/en") or "/"+ else:+ alternate_path = f"/en{path}" if path != "/" else "/en/"+ directory_links = []+ for link in links:+ target = urlparse(urljoin(url, link))+ is_language_toggle = (+ target.netloc == "www.funasr.com" and target.path == alternate_path+ )+ is_github_action = target.netloc == "github.com"+ if not is_language_toggle and not is_github_action:+ directory_links.append(link)+ if expected not in links:+ failures.append(f"{url}: directory navigation missing `{expected}`")+ elif not directory_links or directory_links[-1] != expected:+ failures.append(f"{url}: `{expected}` must be the last directory link")+ if links.count(expected) > 1:+ failures.append(+ f"{url}: directory navigation contains duplicate `{expected}`"+ )+ if wrong_language in links:+ failures.append(+ f"{url}: directory navigation contains wrong-language "+ f"`{wrong_language}`"+ )+ return failures+++def extract_sitemap_page_urls(sitemap: str) -> list[str]:+ root = ET.fromstring(sitemap)+ urls: list[str] = []+ seen: set[str] = set()+ for element in root.iter():+ if element.tag.rsplit("}", 1)[-1] != "loc" or not element.text:+ continue+ url = element.text.strip()+ parsed = urlparse(url)+ is_page = parsed.path.endswith("/") or parsed.path.endswith(".html")+ if (+ parsed.scheme == "https"+ and parsed.netloc == "www.funasr.com"+ and not parsed.query+ and not parsed.fragment+ and is_page+ and url not in seen+ ):+ seen.add(url)+ urls.append(url)+ return urls++def validate_pages(pages: dict[str, str]) -> list[str]:failures: list[str] = []for url, contract in PAGE_CONTRACTS.items():@@ -560,6 +661,17 @@ def fetch_pages(timeout: float, retries: int = 3) -> dict[str, str]:return pages+def fetch_navigation_pages(timeout: float, retries: int = 3) -> dict[str, str]:+ sitemap = _fetch_url(f"{BASE_URL}/sitemap.xml", timeout=timeout, retries=retries)+ urls = extract_sitemap_page_urls(sitemap)++ def fetch(url: str) -> tuple[str, str]:+ return url, _fetch_url(url, timeout=timeout, retries=retries)++ with ThreadPoolExecutor(max_workers=min(8, max(1, len(urls)))) as executor:+ return dict(executor.map(fetch, urls))++def fetch_assets(timeout: float, retries: int = 3) -> dict[str, bytes]:assets: dict[str, bytes] = {}for url in STATIC_ASSET_CONTRACTS:@@ -573,7 +685,17 @@ def main(argv: list[str] | None = None) -> int:parser.add_argument("--retries", type=int, default=3)args = parser.parse_args(argv)- failures = validate_pages(fetch_pages(timeout=args.timeout, retries=args.retries))+ navigation_pages = fetch_navigation_pages(+ timeout=args.timeout, retries=args.retries+ )+ pages = {+ url: navigation_pages.get(url)+ or _fetch_url(url, timeout=args.timeout, retries=args.retries)+ for url in PAGE_CONTRACTS+ }+ all_navigation_pages = navigation_pages | pages+ failures = validate_pages(pages)+ failures.extend(validate_navigation(all_navigation_pages))failures.extend(validate_assets(fetch_assets(timeout=args.timeout, retries=args.retries)))@@ -585,7 +707,9 @@ def main(argv: list[str] | None = None) -> int:print("funasr.com static contract passed for "- f"{len(PAGE_CONTRACTS)} pages and {len(STATIC_ASSET_CONTRACTS)} assets"+ f"{len(PAGE_CONTRACTS)} page contracts, "+ f"{len(all_navigation_pages)} navigation pages, and "+ f"{len(STATIC_ASSET_CONTRACTS)} assets")return 0diff --git a/tests/test_check_funasr_website_static.py b/tests/test_check_funasr_website_static.pyindex d0ff34557..f53d7211d 100644--- a/tests/test_check_funasr_website_static.py+++ b/tests/test_check_funasr_website_static.py@@ -657,13 +657,17 @@ def test_website_contract_requires_visible_donor_usage_copy():checker = _load_module()pages = {url: "ok" for url in checker.PAGE_CONTRACTS}- pages["https://www.funasr.com/donors.html"] = """+ pages[+ "https://www.funasr.com/donors.html"+ ] = """<head><meta name="description" content="捐赠资金用于社区基础设施建设,包括购买和维护服务器,以及购买、续费和维护 www.funasr.com 域名。"></head><body>FunASR 社区功德榜</body>"""- pages["https://www.funasr.com/en/donors.html"] = """+ pages[+ "https://www.funasr.com/en/donors.html"+ ] = """<head><meta name="description" content="Donations fund community infrastructure, including server purchase and maintenance, plus the purchase, renewal, and maintenance of the www.funasr.com domain."></head>@@ -718,3 +722,79 @@ def test_fetch_pages_retries_transient_url_errors(monkeypatch):("https://www.funasr.com/example.html", 3),("https://www.funasr.com/example.html", 3),]+++def test_navigation_contract_requires_donors_as_last_directory_link():+ checker = _load_module()+ pages = {+ "https://www.funasr.com/blog/ok.html": """+ <nav><div class="nav-links">+ <a href="/blog/">技术博客</a>+ <a href="/vs-whisper.html">对比 Whisper</a>+ <a href="/donors.html">功德榜</a>+ <a href="/en/blog/ok.html">EN</a>+ <a class="nav-btn" href="https://github.com/modelscope/FunASR">GitHub</a>+ </div></nav>+ """,+ "https://www.funasr.com/en/blog/missing.html": """+ <nav><div class="nav-links">+ <a href="/en/blog/">Blog</a>+ <a href="/en/vs-whisper.html">vs Whisper</a>+ </div></nav>+ """,+ "https://www.funasr.com/blog/not-last.html": """+ <nav><div class="nav-links">+ <a href="/donors.html">功德榜</a>+ <a href="/blog/">技术博客</a>+ </div></nav>+ """,+ "https://www.funasr.com/en/blog/wrong-language.html": """+ <nav><div class="nav-links">+ <a href="/en/blog/">Blog</a>+ <a href="/donors.html">功德榜</a>+ </div></nav>+ """,+ "https://www.funasr.com/plain.html": "<main>No directory navigation</main>",+ }++ failures = checker.validate_navigation(pages)++ assert not any("ok.html" in failure for failure in failures)+ assert any(+ "missing.html" in failure and "missing `/en/donors.html`" in failure+ for failure in failures+ )+ assert any(+ "not-last.html" in failure and "must be the last directory link" in failure+ for failure in failures+ )+ assert any(+ "wrong-language.html" in failure and "missing `/en/donors.html`" in failure+ for failure in failures+ )+ assert any(+ "wrong-language.html" in failure+ and "contains wrong-language `/donors.html`" in failure+ for failure in failures+ )+ assert not any("plain.html" in failure for failure in failures)+++def test_extract_sitemap_page_urls_keeps_unique_same_origin_html_pages():+ checker = _load_module()+ sitemap = """<?xml version="1.0" encoding="UTF-8"?>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url><loc>https://www.funasr.com/</loc></url>+ <url><loc>https://www.funasr.com/blog/</loc></url>+ <url><loc>https://www.funasr.com/blog/guide.html</loc></url>+ <url><loc>https://www.funasr.com/blog/guide.html</loc></url>+ <url><loc>https://www.funasr.com/logo.png</loc></url>+ <url><loc>https://example.com/foreign.html</loc></url>+ </urlset>+ """++ assert checker.extract_sitemap_page_urls(sitemap) == [+ "https://www.funasr.com/",+ "https://www.funasr.com/blog/",+ "https://www.funasr.com/blog/guide.html",+ ]