Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

test: guard donor navigation across public pages (#3421)

zhifu gaoefee7346484e
diff --git a/scripts/check_funasr_website_static.py b/scripts/check_funasr_website_static.py
index a152ee070..560531134 100755
--- a/scripts/check_funasr_website_static.py
+++ b/scripts/check_funasr_website_static.py
@@ -4,12 +4,15 @@
from __future__ import annotations
import argparse
+from concurrent.futures import ThreadPoolExecutor
from html.parser import HTMLParser
import re
import sys
import time
import urllib.request
from dataclasses import dataclass
+from urllib.parse import urljoin, urlparse
+import xml.etree.ElementTree as ET
BASE_URL = "https://www.funasr.com"
@@ -485,6 +488,104 @@ def extract_images(html: str) -> set[str]:
return _collect_visible_content(html).images
+class _DirectoryNavigationCollector(HTMLParser):
+ def __init__(self) -> None:
+ super().__init__()
+ self.links: list[str] = []
+ self._depth = 0
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ tag = tag.lower()
+ attributes = {name.lower(): value for name, value in attrs}
+ if self._depth:
+ if tag == "a" and attributes.get("href"):
+ self.links.append(attributes["href"])
+ if tag not in _VOID_TAGS:
+ self._depth += 1
+ elif tag == "div" and "nav-links" in (attributes.get("class") or "").split():
+ self._depth = 1
+
+ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if not self._depth or tag.lower() != "a":
+ return
+ attributes = {name.lower(): value for name, value in attrs}
+ if attributes.get("href"):
+ self.links.append(attributes["href"])
+
+ def handle_endtag(self, tag: str) -> None:
+ if self._depth:
+ self._depth -= 1
+
+
+def extract_directory_links(html: str) -> list[str]:
+ parser = _DirectoryNavigationCollector()
+ parser.feed(html)
+ parser.close()
+ return parser.links
+
+
+def validate_navigation(pages: dict[str, str]) -> list[str]:
+ failures: list[str] = []
+ for url, html in pages.items():
+ links = extract_directory_links(html)
+ if not links:
+ continue
+ path = urlparse(url).path
+ is_english = path == "/en" or path.startswith("/en/")
+ expected = "/en/donors.html" if is_english else "/donors.html"
+ wrong_language = "/donors.html" if is_english else "/en/donors.html"
+ if is_english:
+ alternate_path = path.removeprefix("/en") or "/"
+ else:
+ alternate_path = f"/en{path}" if path != "/" else "/en/"
+ directory_links = []
+ for link in links:
+ target = urlparse(urljoin(url, link))
+ is_language_toggle = (
+ target.netloc == "www.funasr.com" and target.path == alternate_path
+ )
+ is_github_action = target.netloc == "github.com"
+ if not is_language_toggle and not is_github_action:
+ directory_links.append(link)
+ if expected not in links:
+ failures.append(f"{url}: directory navigation missing `{expected}`")
+ elif not directory_links or directory_links[-1] != expected:
+ failures.append(f"{url}: `{expected}` must be the last directory link")
+ if links.count(expected) > 1:
+ failures.append(
+ f"{url}: directory navigation contains duplicate `{expected}`"
+ )
+ if wrong_language in links:
+ failures.append(
+ f"{url}: directory navigation contains wrong-language "
+ f"`{wrong_language}`"
+ )
+ return failures
+
+
+def extract_sitemap_page_urls(sitemap: str) -> list[str]:
+ root = ET.fromstring(sitemap)
+ urls: list[str] = []
+ seen: set[str] = set()
+ for element in root.iter():
+ if element.tag.rsplit("}", 1)[-1] != "loc" or not element.text:
+ continue
+ url = element.text.strip()
+ parsed = urlparse(url)
+ is_page = parsed.path.endswith("/") or parsed.path.endswith(".html")
+ if (
+ parsed.scheme == "https"
+ and parsed.netloc == "www.funasr.com"
+ and not parsed.query
+ and not parsed.fragment
+ and is_page
+ and url not in seen
+ ):
+ seen.add(url)
+ urls.append(url)
+ return urls
+
+
def validate_pages(pages: dict[str, str]) -> list[str]:
failures: list[str] = []
for url, contract in PAGE_CONTRACTS.items():
@@ -560,6 +661,17 @@ def fetch_pages(timeout: float, retries: int = 3) -> dict[str, str]:
return pages
+def fetch_navigation_pages(timeout: float, retries: int = 3) -> dict[str, str]:
+ sitemap = _fetch_url(f"{BASE_URL}/sitemap.xml", timeout=timeout, retries=retries)
+ urls = extract_sitemap_page_urls(sitemap)
+
+ def fetch(url: str) -> tuple[str, str]:
+ return url, _fetch_url(url, timeout=timeout, retries=retries)
+
+ with ThreadPoolExecutor(max_workers=min(8, max(1, len(urls)))) as executor:
+ return dict(executor.map(fetch, urls))
+
+
def fetch_assets(timeout: float, retries: int = 3) -> dict[str, bytes]:
assets: dict[str, bytes] = {}
for url in STATIC_ASSET_CONTRACTS:
@@ -573,7 +685,17 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--retries", type=int, default=3)
args = parser.parse_args(argv)
- failures = validate_pages(fetch_pages(timeout=args.timeout, retries=args.retries))
+ navigation_pages = fetch_navigation_pages(
+ timeout=args.timeout, retries=args.retries
+ )
+ pages = {
+ url: navigation_pages.get(url)
+ or _fetch_url(url, timeout=args.timeout, retries=args.retries)
+ for url in PAGE_CONTRACTS
+ }
+ all_navigation_pages = navigation_pages | pages
+ failures = validate_pages(pages)
+ failures.extend(validate_navigation(all_navigation_pages))
failures.extend(
validate_assets(fetch_assets(timeout=args.timeout, retries=args.retries))
)
@@ -585,7 +707,9 @@ def main(argv: list[str] | None = None) -> int:
print(
"funasr.com static contract passed for "
- f"{len(PAGE_CONTRACTS)} pages and {len(STATIC_ASSET_CONTRACTS)} assets"
+ f"{len(PAGE_CONTRACTS)} page contracts, "
+ f"{len(all_navigation_pages)} navigation pages, and "
+ f"{len(STATIC_ASSET_CONTRACTS)} assets"
)
return 0
diff --git a/tests/test_check_funasr_website_static.py b/tests/test_check_funasr_website_static.py
index d0ff34557..f53d7211d 100644
--- a/tests/test_check_funasr_website_static.py
+++ b/tests/test_check_funasr_website_static.py
@@ -657,13 +657,17 @@ def test_website_contract_requires_visible_donor_usage_copy():
checker = _load_module()
pages = {url: "ok" for url in checker.PAGE_CONTRACTS}
- pages["https://www.funasr.com/donors.html"] = """
+ pages[
+ "https://www.funasr.com/donors.html"
+ ] = """
<head>
<meta name="description" content="捐赠资金用于社区基础设施建设,包括购买和维护服务器,以及购买、续费和维护 www.funasr.com 域名。">
</head>
<body>FunASR 社区功德榜</body>
"""
- pages["https://www.funasr.com/en/donors.html"] = """
+ pages[
+ "https://www.funasr.com/en/donors.html"
+ ] = """
<head>
<meta name="description" content="Donations fund community infrastructure, including server purchase and maintenance, plus the purchase, renewal, and maintenance of the www.funasr.com domain.">
</head>
@@ -718,3 +722,79 @@ def test_fetch_pages_retries_transient_url_errors(monkeypatch):
("https://www.funasr.com/example.html", 3),
("https://www.funasr.com/example.html", 3),
]
+
+
+def test_navigation_contract_requires_donors_as_last_directory_link():
+ checker = _load_module()
+ pages = {
+ "https://www.funasr.com/blog/ok.html": """
+ <nav><div class="nav-links">
+ <a href="/blog/">技术博客</a>
+ <a href="/vs-whisper.html">对比 Whisper</a>
+ <a href="/donors.html">功德榜</a>
+ <a href="/en/blog/ok.html">EN</a>
+ <a class="nav-btn" href="https://github.com/modelscope/FunASR">GitHub</a>
+ </div></nav>
+ """,
+ "https://www.funasr.com/en/blog/missing.html": """
+ <nav><div class="nav-links">
+ <a href="/en/blog/">Blog</a>
+ <a href="/en/vs-whisper.html">vs Whisper</a>
+ </div></nav>
+ """,
+ "https://www.funasr.com/blog/not-last.html": """
+ <nav><div class="nav-links">
+ <a href="/donors.html">功德榜</a>
+ <a href="/blog/">技术博客</a>
+ </div></nav>
+ """,
+ "https://www.funasr.com/en/blog/wrong-language.html": """
+ <nav><div class="nav-links">
+ <a href="/en/blog/">Blog</a>
+ <a href="/donors.html">功德榜</a>
+ </div></nav>
+ """,
+ "https://www.funasr.com/plain.html": "<main>No directory navigation</main>",
+ }
+
+ failures = checker.validate_navigation(pages)
+
+ assert not any("ok.html" in failure for failure in failures)
+ assert any(
+ "missing.html" in failure and "missing `/en/donors.html`" in failure
+ for failure in failures
+ )
+ assert any(
+ "not-last.html" in failure and "must be the last directory link" in failure
+ for failure in failures
+ )
+ assert any(
+ "wrong-language.html" in failure and "missing `/en/donors.html`" in failure
+ for failure in failures
+ )
+ assert any(
+ "wrong-language.html" in failure
+ and "contains wrong-language `/donors.html`" in failure
+ for failure in failures
+ )
+ assert not any("plain.html" in failure for failure in failures)
+
+
+def test_extract_sitemap_page_urls_keeps_unique_same_origin_html_pages():
+ checker = _load_module()
+ sitemap = """<?xml version="1.0" encoding="UTF-8"?>
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url><loc>https://www.funasr.com/</loc></url>
+ <url><loc>https://www.funasr.com/blog/</loc></url>
+ <url><loc>https://www.funasr.com/blog/guide.html</loc></url>
+ <url><loc>https://www.funasr.com/blog/guide.html</loc></url>
+ <url><loc>https://www.funasr.com/logo.png</loc></url>
+ <url><loc>https://example.com/foreign.html</loc></url>
+ </urlset>
+ """
+
+ assert checker.extract_sitemap_page_urls(sitemap) == [
+ "https://www.funasr.com/",
+ "https://www.funasr.com/blog/",
+ "https://www.funasr.com/blog/guide.html",
+ ]