An open source, non-profit web search engine
Add mwmbl source
Daoud Clarke8b8a7337d9e0
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 47a538a..88775ee 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -277,7 +277,7 @@ CURRENT_AGREEMENT_VERSIONS = {}# Super Search-SUPER_SEARCH_MONTHLY_LIMIT = 10+SUPER_SEARCH_MONTHLY_LIMIT = 100SUPER_SEARCH_TOP_K = 10 # promote sources in top-K seen so far for crawlingSUPER_SEARCH_DEADLINE_SECONDS = 10.0SUPER_SEARCH_PER_SOURCE_TIMEOUT = 2.0diff --git a/mwmbl/tinysearchengine/super_search_sources/__init__.py b/mwmbl/tinysearchengine/super_search_sources/__init__.pyindex 4cf5cd9..e0e28df 100644--- a/mwmbl/tinysearchengine/super_search_sources/__init__.py+++ b/mwmbl/tinysearchengine/super_search_sources/__init__.py@@ -8,10 +8,12 @@ orchestrator.from mwmbl.tinysearchengine.super_search_sources.arxiv import search as search_arxivfrom mwmbl.tinysearchengine.super_search_sources.github import search as search_githubfrom mwmbl.tinysearchengine.super_search_sources.hn import search as search_hn+from mwmbl.tinysearchengine.super_search_sources.mwmbl_index import search as search_mwmblfrom mwmbl.tinysearchengine.super_search_sources.pypi import search as search_pypifrom mwmbl.tinysearchengine.super_search_sources.stackexchange import search as search_stackexchangeSOURCES = {+ "mwmbl": search_mwmbl,"hn": search_hn,"github": search_github,"stackexchange": search_stackexchange,diff --git a/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py b/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.pynew file mode 100644index 0000000..2d8c54a--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py@@ -0,0 +1,19 @@+"""Mwmbl index + Wikipedia via the standard search ranker."""+import asyncio+import logging++import httpx++from mwmbl.search_setup import ranker+from mwmbl.tinysearchengine.indexer import Document++logger = logging.getLogger(__name__)+++async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Document]:+ try:+ results = await asyncio.to_thread(ranker.search, query, [])+ return results[:limit]+ except Exception as e:+ logger.info("mwmbl index source failed: %s", e)+ return []