Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
test: guard donor navigation across public pages (#3422)
zhifu gao5f0e67fec6c2
diff --git a/scripts/check_funasr_website_static.py b/scripts/check_funasr_website_static.pyindex 560531134..43ba0b46f 100755--- a/scripts/check_funasr_website_static.py+++ b/scripts/check_funasr_website_static.py@@ -16,6 +16,9 @@ import xml.etree.ElementTree as ETBASE_URL = "https://www.funasr.com"+DONOR_PAGE_URLS = frozenset(+ (f"{BASE_URL}/donors.html", f"{BASE_URL}/en/donors.html")+)@dataclass(frozen=True)@@ -488,59 +491,120 @@ def extract_images(html: str) -> set[str]:return _collect_visible_content(html).images+@dataclass(frozen=True)+class DirectoryLink:+ href: str+ label: str+++@dataclass+class _NavigationElementState:+ tag: str+ hidden: bool+ link: str | None = None+ text: list[str] | None = None++class _DirectoryNavigationCollector(HTMLParser):def __init__(self) -> None:super().__init__()- self.links: list[str] = []- self._depth = 0+ self.found_navigation = False+ self.links: list[DirectoryLink] = []+ self._stack: list[_NavigationElementState] = []++ @property+ def _parent_hidden(self) -> bool:+ return bool(self._stack and self._stack[-1].hidden)++ def _finish_link(self, state: _NavigationElementState) -> None:+ if state.link is None or state.text is None:+ return+ label = re.sub(r"\s+", " ", " ".join(state.text)).strip()+ if label:+ self.links.append(DirectoryLink(state.link, label))def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:tag = tag.lower()attributes = {name.lower(): value for name, value in attrs}- if self._depth:- if tag == "a" and attributes.get("href"):- self.links.append(attributes["href"])- if tag not in _VOID_TAGS:- self._depth += 1- elif tag == "div" and "nav-links" in (attributes.get("class") or "").split():- self._depth = 1+ if not self._stack:+ is_navigation = (+ tag == "div"+ and "nav-links" in (attributes.get("class") or "").split()+ )+ if not is_navigation:+ return+ self.found_navigation = True+ self._stack.append(+ _NavigationElementState(+ tag,+ _VisibleContentCollector._has_hidden_attribute(attributes),+ )+ )+ return++ hidden = (+ self._parent_hidden+ or tag in _HIDDEN_TAGS+ or _VisibleContentCollector._has_hidden_attribute(attributes)+ )+ if tag not in _VOID_TAGS:+ link = attributes.get("href") if tag == "a" and not hidden else None+ self._stack.append(+ _NavigationElementState(+ tag,+ hidden,+ link=link,+ text=[] if link is not None else None,+ )+ )def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:- if not self._depth or tag.lower() != "a":- return- attributes = {name.lower(): value for name, value in attrs}- if attributes.get("href"):- self.links.append(attributes["href"])+ returndef handle_endtag(self, tag: str) -> None:- if self._depth:- self._depth -= 1+ tag = tag.lower()+ for index in range(len(self._stack) - 1, -1, -1):+ if self._stack[index].tag == tag:+ for state in reversed(self._stack[index:]):+ self._finish_link(state)+ del self._stack[index:]+ return++ def handle_data(self, data: str) -> None:+ if self._parent_hidden or not data.strip():+ return+ for state in reversed(self._stack):+ if state.link is not None and state.text is not None:+ state.text.append(data)+ return-def extract_directory_links(html: str) -> list[str]:+def extract_directory_links(html: str) -> tuple[bool, list[DirectoryLink]]:parser = _DirectoryNavigationCollector()parser.feed(html)parser.close()- return parser.links+ return parser.found_navigation, parser.linksdef validate_navigation(pages: dict[str, str]) -> list[str]:failures: list[str] = []for url, html in pages.items():- links = extract_directory_links(html)- if not links:+ found_navigation, navigation_links = extract_directory_links(html)+ if not found_navigation:continue+ links = [link.href for link in navigation_links]path = urlparse(url).pathis_english = path == "/en" or path.startswith("/en/")expected = "/en/donors.html" if is_english else "/donors.html"+ expected_labels = ("Thanks", "Donors") if is_english else ("功德榜",)wrong_language = "/donors.html" if is_english else "/en/donors.html"if is_english:- alternate_path = path.removeprefix("/en") or "/"+ alternate_path = path[len("/en") :] or "/"else:alternate_path = f"/en{path}" if path != "/" else "/en/"- directory_links = []- for link in links:- target = urlparse(urljoin(url, link))+ directory_links: list[DirectoryLink] = []+ for link in navigation_links:+ target = urlparse(urljoin(url, link.href))is_language_toggle = (target.netloc == "www.funasr.com" and target.path == alternate_path)@@ -548,8 +612,19 @@ def validate_navigation(pages: dict[str, str]) -> list[str]:if not is_language_toggle and not is_github_action:directory_links.append(link)if expected not in links:- failures.append(f"{url}: directory navigation missing `{expected}`")- elif not directory_links or directory_links[-1] != expected:+ failures.append(+ f"{url}: directory navigation missing `{expected}`; link must be visible"+ )+ else:+ expected_links = [link for link in navigation_links if link.href == expected]+ if not any(link.label in expected_labels for link in expected_links):+ label_description = "` or `".join(expected_labels)+ failures.append(+ f"{url}: `{expected}` must use visible label `{label_description}`"+ )+ if expected in links and (+ not directory_links or directory_links[-1].href != expected+ ):failures.append(f"{url}: `{expected}` must be the last directory link")if links.count(expected) > 1:failures.append(@@ -634,11 +709,15 @@ def validate_assets(assets: dict[str, bytes]) -> list[str]:return failures-def _fetch_bytes(url: str, timeout: float, retries: int) -> bytes:+def _fetch_bytes(+ url: str, timeout: float, retries: int, require_exact_url: bool = False+) -> bytes:last_error: Exception | None = Nonefor attempt in range(retries + 1):try:with urllib.request.urlopen(url, timeout=timeout) as response:+ if require_exact_url and response.geturl() != url:+ raise RuntimeError(f"{url} redirected to {response.geturl()}")return response.read()except Exception as exc:last_error = exc@@ -648,8 +727,15 @@ def _fetch_bytes(url: str, timeout: float, retries: int) -> bytes:raise last_error-def _fetch_url(url: str, timeout: float, retries: int) -> str:- return _fetch_bytes(url, timeout=timeout, retries=retries).decode(+def _fetch_url(+ url: str, timeout: float, retries: int, require_exact_url: bool = False+) -> str:+ return _fetch_bytes(+ url,+ timeout=timeout,+ retries=retries,+ require_exact_url=require_exact_url,+ ).decode("utf-8", errors="replace")@@ -657,7 +743,12 @@ def _fetch_url(url: str, timeout: float, retries: int) -> str:def fetch_pages(timeout: float, retries: int = 3) -> dict[str, str]:pages: dict[str, str] = {}for url in PAGE_CONTRACTS:- pages[url] = _fetch_url(url, timeout=timeout, retries=retries)+ pages[url] = _fetch_url(+ url,+ timeout=timeout,+ retries=retries,+ require_exact_url=url in DONOR_PAGE_URLS,+ )return pages@@ -688,12 +779,21 @@ def main(argv: list[str] | None = None) -> int:navigation_pages = fetch_navigation_pages(timeout=args.timeout, retries=args.retries)- pages = {- url: navigation_pages.get(url)- or _fetch_url(url, timeout=args.timeout, retries=args.retries)- for url in PAGE_CONTRACTS- }- all_navigation_pages = navigation_pages | pages+ pages = {}+ for url in PAGE_CONTRACTS:+ if url in DONOR_PAGE_URLS:+ pages[url] = _fetch_url(+ url,+ timeout=args.timeout,+ retries=args.retries,+ require_exact_url=True,+ )+ else:+ pages[url] = navigation_pages.get(url) or _fetch_url(+ url, timeout=args.timeout, retries=args.retries+ )+ all_navigation_pages = navigation_pages.copy()+ all_navigation_pages.update(pages)failures = validate_pages(pages)failures.extend(validate_navigation(all_navigation_pages))failures.extend(diff --git a/tests/test_check_funasr_website_static.py b/tests/test_check_funasr_website_static.pyindex f53d7211d..ccfaf7fa0 100644--- a/tests/test_check_funasr_website_static.py+++ b/tests/test_check_funasr_website_static.py@@ -3,6 +3,8 @@ import sysimport urllib.errorfrom pathlib import Path+import pytest+ROOT = Path(__file__).resolve().parents[1]SCRIPT = ROOT / "scripts" / "check_funasr_website_static.py"@@ -780,6 +782,84 @@ def test_navigation_contract_requires_donors_as_last_directory_link():assert not any("plain.html" in failure for failure in failures)+def test_navigation_contract_requires_visible_correctly_labeled_donor_link():+ checker = _load_module()+ pages = {+ "https://www.funasr.com/blog/hidden.html": """+ <nav><div class="nav-links">+ <a href="/blog/">技术博客</a>+ <a hidden href="/donors.html">功德榜</a>+ </div></nav>+ """,+ "https://www.funasr.com/blog/wrong-label.html": """+ <nav><div class="nav-links">+ <a href="/blog/">技术博客</a>+ <a href="/donors.html">社区</a>+ </div></nav>+ """,+ "https://www.funasr.com/blog/hidden-label.html": """+ <nav><div class="nav-links">+ <a href="/blog/">技术博客</a>+ <a href="/donors.html"><template>功德榜</template></a>+ </div></nav>+ """,+ "https://www.funasr.com/en/blog/empty.html": """+ <nav><div class="nav-links"></div></nav>+ """,+ }++ failures = checker.validate_navigation(pages)++ assert any(+ "hidden.html" in failure+ and "missing `/donors.html`; link must be visible" in failure+ for failure in failures+ )+ assert any(+ "wrong-label.html" in failure+ and "must use visible label `功德榜`" in failure+ for failure in failures+ )+ assert any(+ "hidden-label.html" in failure+ and "missing `/donors.html`; link must be visible" in failure+ for failure in failures+ )+ assert any(+ "empty.html" in failure+ and "missing `/en/donors.html`; link must be visible" in failure+ for failure in failures+ )+++def test_fetch_pages_rejects_redirected_donor_page(monkeypatch):+ checker = _load_module()+ donor_url = "https://www.funasr.com/donors.html"++ class FakeResponse:+ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc, tb):+ return False++ def read(self):+ return b"redirected donor copy"++ def geturl(self):+ return "https://www.funasr.com/community.html"++ monkeypatch.setattr(checker.urllib.request, "urlopen", lambda url, timeout: FakeResponse())+ monkeypatch.setattr(+ checker,+ "PAGE_CONTRACTS",+ {donor_url: checker.PageContract(required=("donor copy",))},+ )++ with pytest.raises(RuntimeError, match="redirected"):+ checker.fetch_pages(timeout=3, retries=0)++def test_extract_sitemap_page_urls_keeps_unique_same_origin_html_pages():checker = _load_module()sitemap = """<?xml version="1.0" encoding="UTF-8"?>