Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: parse Yahoo's newer result layout in web search (#12980)
oobabooga8fa3fd8235dc
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex ef4b28f2610..38b8407e7a8 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -16951,6 +16951,49 @@ def _resolve_engine_tiers(text_engines) -> list:return resolved+def _class_token(name: str) -> str:+ return f"contains(concat(' ', normalize-space(@class), ' '), ' {name} ')"+++# Each section.algo leaves a div unclosed, so lxml nests every later result inside it: match the+# nearest section, and take the next s-desc before the next title (a descendant search takes them all).+_YAHOO_SECTION_TITLES = (+ f"//a[{_class_token('s-title')}][ancestor::section[1][{_class_token('algo')}]]"+)+_YAHOO_SECTION_SNIPPET = f"following::*[self::p[{_class_token('s-desc')}] or self::a[{_class_token('s-title')}]][1][self::p]"+++def _install_yahoo_layout_parser(text_engines) -> None:+ """Parse Yahoo's ``section.algo`` pages, which ddgs 9.8.0-9.16.0 (``div.relsrch`` only) read as+ empty. ddgs builds engines from this registry by name; ``relsrch`` pages keep ddgs's parser."""+ yahoo = (text_engines or {}).get("yahoo")+ if not isinstance(yahoo, type) or getattr(yahoo, "_parses_section_layout", False):+ return++ class _Yahoo(yahoo):+ _parses_section_layout = True++ def extract_results(self, html_text):+ results = super().extract_results(html_text)+ if results:+ return results+ tree = self.extract_tree(self.pre_process_html(html_text))+ for link in tree.xpath(_YAHOO_SECTION_TITLES):+ # TextResult strips tags and collapses whitespace on assignment.+ result = self.result_type()+ result.title = link.get("aria-label") or "".join(+ link.xpath(f".//text()[not(ancestor::span[{_class_token('title-url')}])]")+ )+ result.href = link.get("href") or ""+ snippet = link.xpath(_YAHOO_SECTION_SNIPPET)+ if snippet:+ result.body = snippet[0].xpath("string()")+ results.append(result)+ return results++ text_engines["yahoo"] = _Yahoo++def _image_search_or_none(subjects: list, timeout, cancel_event, website_policy) -> "str | None":"""``_image_search`` that reports a failure as None instead of raising. Every caller sits inside``_web_search``'s own ``except``, which would turn a raise into Search failed: ... and throw@@ -17094,7 +17137,9 @@ def _web_search(from ddgs import DDGSfrom ddgs.engines import ENGINES- engine_tiers = _resolve_engine_tiers(ENGINES.get("text", {}))+ text_engines = ENGINES.get("text") or {}+ _install_yahoo_layout_parser(text_engines)+ engine_tiers = _resolve_engine_tiers(text_engines)if not engine_tiers:raise RuntimeError("no approved search engine is available.")# reset after setup to keep the primary budget; the earlier deadline bounds fallback.diff --git a/studio/backend/tests/test_web_search_yahoo_layout.py b/studio/backend/tests/test_web_search_yahoo_layout.pynew file mode 100644index 00000000000..b98cf5aaf11--- /dev/null+++ b/studio/backend/tests/test_web_search_yahoo_layout.py@@ -0,0 +1,182 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Yahoo's section layout, which ddgs parses to nothing, must reach the model as results."""++from unittest.mock import MagicMock++import pytest++from ddgs.base import BaseSearchEngine+from ddgs.engines import ENGINES+from ddgs.engines.yahoo import Yahoo++from core.inference import tools+++def _redirect(url):+ return (+ "https://r.search.yahoo.com/_ylt=Awr;_ylu=Y29s/RV=2/RE=1792609327/RO=10/RU="+ + url.replace(":", "%3a").replace("/", "%2f")+ + "/RK=2/RS=abc-"+ )+++def _section(+ kind,+ url,+ title,+ desc = None,+):+ # Trimmed from a live page: grp-talgo-ext is never closed, so lxml nests every later section here.+ text = (+ f'<div class="grp grp-talgo-ext"><div class="compText"><p class="s-desc lh-20">'+ f'<span class="s-time">Jul 23, 2025 · </span>{desc}</p></div>'+ if desc+ else ""+ )+ return (+ f'<section class="dd algo s-algo {kind}"><div class="grp grp-talgo"><div class="compTitle p-r">'+ f'<h3 class="title d-ib mt-42"><a class="s-title fz-m" href="{_redirect(url)}" aria-label="{title}">'+ f'<span class="title-url"><span class="fw-500">Site</span>{url}</span>{title}</a></h3>'+ f'<div><a class="thmb algo-favicon" href="{_redirect(url)}"><img src="x.png"/></a></div>'+ f"</div></div>{text}</section>"+ )+++SECTION_LAYOUT = (+ '<html><body><section class="reg searchCenterMiddle">'+ + _section(+ "algo-sr richAlgo",+ "https://docs.python.org/3/library/asyncio.html",+ "asyncio: Asynchronous I/O",+ "asyncio is a library to write <b>concurrent</b> code.",+ )+ + _section(+ "imageThumbnail image small Sr",+ "https://www.datacamp.com/tutorial/python-async",+ "Python Async Programming - DataCamp",+ "Speed up your code with async.",+ )+ # A titled block that is not a result: it must not become one.+ + '<section class="dd AdTop"><div><h3><a class="s-title" href="https://ads.example/x">'+ + "Sponsored</a></h3></div></section>"+ + _section(+ "videoThumbnail richAlgo", "https://m.youtube.com/watch?v=Qb9s3UiMSTA", "Asyncio in Python"+ )+ + _section(+ "algo-sr Sr",+ "https://realpython.com/async-io-python/",+ "Python's asyncio: A Hands-On Walkthrough",+ "In this tutorial, you'll learn how asyncio works.",+ )+ + "</section></body></html>"+)++RELSRCH_LAYOUT = (+ '<html><body><ol><li><div class="dd algo algo-sr relsrch Sr"><div class="compTitle">'+ f'<div class="Title"><h3>GeeksforGeeks asyncio</h3><a href="{_redirect("https://www.geeksforgeeks.org/asyncio/")}">x</a></div>'+ '</div><div class="compText"><div class="Text">Older layout body.</div></div></div></li></ol></body></html>'+)++NO_MATCH = (+ '<html><body><section class="reg searchCenterMiddle"><p>We did not find results for: zzqxv. '+ "Check spelling or type a new query.</p></section></body></html>"+)+++def yahoo():+ text_engines = dict(ENGINES["text"])+ tools._install_yahoo_layout_parser(text_engines)+ return text_engines["yahoo"].__new__(text_engines["yahoo"])+++def _parse(engine, html_text):+ return [+ (r.title, r.href, r.body)+ for r in engine.post_extract_results(engine.extract_results(html_text))+ ]+++def test_ddgs_alone_finds_nothing_in_the_section_layout():+ assert Yahoo.__new__(Yahoo).extract_results(SECTION_LAYOUT) == []+++def test_every_organic_section_becomes_a_result_with_its_own_snippet(yahoo):+ assert _parse(yahoo, SECTION_LAYOUT) == [+ (+ "asyncio: Asynchronous I/O",+ "https://docs.python.org/3/library/asyncio.html",+ "Jul 23, 2025 · asyncio is a library to write concurrent code.",+ ),+ (+ "Python Async Programming - DataCamp",+ "https://www.datacamp.com/tutorial/python-async",+ "Jul 23, 2025 · Speed up your code with async.",+ ),+ # No snippet on the page: the next result's must not be borrowed.+ ("Asyncio in Python", "https://m.youtube.com/watch?v=Qb9s3UiMSTA", ""),+ (+ "Python's asyncio: A Hands-On Walkthrough",+ "https://realpython.com/async-io-python/",+ "Jul 23, 2025 · In this tutorial, you'll learn how asyncio works.",+ ),+ ]+++def test_the_older_layout_still_goes_through_ddgs(yahoo):+ assert _parse(yahoo, RELSRCH_LAYOUT) == [+ ("GeeksforGeeks asyncio", "https://www.geeksforgeeks.org/asyncio/", "Older layout body.")+ ]+++def test_an_honest_no_match_page_stays_empty(yahoo):+ assert _parse(yahoo, NO_MATCH) == []+++def test_install_is_idempotent_and_ignores_registry_stubs():+ text_engines = dict(ENGINES["text"])+ tools._install_yahoo_layout_parser(text_engines)+ installed = text_engines["yahoo"]+ tools._install_yahoo_layout_parser(text_engines)+ assert text_engines["yahoo"] is installed++ stub = object()+ stubbed = {"yahoo": stub}+ tools._install_yahoo_layout_parser(stubbed)+ assert stubbed["yahoo"] is stub+ tools._install_yahoo_layout_parser(None)+++def _offline_init(self, *args, **kwargs):+ self.http_client = MagicMock()+++def test_web_search_returns_yahoo_section_results_when_every_other_engine_is_blocked(monkeypatch):+ """The reported failure: only Yahoo answers, in the layout ddgs cannot read."""+ from ddgs.ddgs import DDGS++ if callable(getattr(DDGS, "_get_network_client", None)):+ monkeypatch.setattr(DDGS, "_get_network_client", lambda self: None)+ monkeypatch.setattr(tools, "_wikipedia_search", lambda *args: [])+ for cls in ENGINES["text"].values():+ # A real client cannot be built on the py<3.10 pin (see test_web_search_tiers); Google's+ # build_payload still sets a cookie on it.+ monkeypatch.setattr(cls, "__init__", _offline_init, raising = False)+ # _web_search swaps its Yahoo subclass into the shared registry; put the entry back afterwards.+ monkeypatch.setitem(ENGINES["text"], "yahoo", ENGINES["text"]["yahoo"])+ contacted = []++ def request(engine, *args, **kwargs):+ contacted.append(engine.name)+ return SECTION_LAYOUT if engine.name == "yahoo" else None # None is how ddgs reports a 429++ monkeypatch.setattr(BaseSearchEngine, "request", request)++ result = tools.execute_tool("web_search", {"query": "python asyncio tutorial"})++ assert "yahoo" in contacted+ assert "URL: https://docs.python.org/3/library/asyncio.html" in result+ assert "URL: https://realpython.com/async-io-python/" in result+ assert "Wikipedia-only" not in result