Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add mwmbl source

Daoud Clarke8b8a7337d9e0
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index 47a538a..88775ee 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -277,7 +277,7 @@ CURRENT_AGREEMENT_VERSIONS = {
}
# Super Search
-SUPER_SEARCH_MONTHLY_LIMIT = 10
+SUPER_SEARCH_MONTHLY_LIMIT = 100
SUPER_SEARCH_TOP_K = 10 # promote sources in top-K seen so far for crawling
SUPER_SEARCH_DEADLINE_SECONDS = 10.0
SUPER_SEARCH_PER_SOURCE_TIMEOUT = 2.0
diff --git a/mwmbl/tinysearchengine/super_search_sources/__init__.py b/mwmbl/tinysearchengine/super_search_sources/__init__.py
index 4cf5cd9..e0e28df 100644
--- a/mwmbl/tinysearchengine/super_search_sources/__init__.py
+++ b/mwmbl/tinysearchengine/super_search_sources/__init__.py
@@ -8,10 +8,12 @@ orchestrator.
from mwmbl.tinysearchengine.super_search_sources.arxiv import search as search_arxiv
from mwmbl.tinysearchengine.super_search_sources.github import search as search_github
from mwmbl.tinysearchengine.super_search_sources.hn import search as search_hn
+from mwmbl.tinysearchengine.super_search_sources.mwmbl_index import search as search_mwmbl
from mwmbl.tinysearchengine.super_search_sources.pypi import search as search_pypi
from mwmbl.tinysearchengine.super_search_sources.stackexchange import search as search_stackexchange
SOURCES = {
+ "mwmbl": search_mwmbl,
"hn": search_hn,
"github": search_github,
"stackexchange": search_stackexchange,
diff --git a/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py b/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py
new file mode 100644
index 0000000..2d8c54a
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py
@@ -0,0 +1,19 @@
+"""Mwmbl index + Wikipedia via the standard search ranker."""
+import asyncio
+import logging
+
+import httpx
+
+from mwmbl.search_setup import ranker
+from mwmbl.tinysearchengine.indexer import Document
+
+logger = logging.getLogger(__name__)
+
+
+async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Document]:
+ try:
+ results = await asyncio.to_thread(ranker.search, query, [])
+ return results[:limit]
+ except Exception as e:
+ logger.info("mwmbl index source failed: %s", e)
+ return []