Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Use the blacklist snapshot for the URL queue in the web workers

Daoud Clarke346389402298
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index edad557..64b0c2c 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -29,16 +29,25 @@ def get_curated_domains() -> set[str]:
redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)
-queued_batches = RedisURLQueue(redis, get_curated_domains)
+
+# Pull the blacklist snapshot in at worker startup rather than letting the first query
+# pay for the ~11 MB Redis read. Subsequent refreshes happen on a daemon thread.
+snapshot_blacklist = get_snapshot_blacklist()
+
+# Hand the snapshot to the URL queue too. Left to its own default it would build a
+# get_default_blacklist_provider(), and POST /crawler/batches/new calls get_batch() ->
+# is_domain_blacklisted(), which would download and parse the remote blocklists inside a
+# request handler and pin ~156 MB of domain strings in each of the (cpu_count * 2 + 1)
+# gunicorn workers - the cost the snapshot exists to avoid. The processes that legitimately
+# need the full provider (update_urls, update_batches, the standalone crawler in crawl.py,
+# whose Redis has no published snapshot) construct their own queue and are unaffected.
+queued_batches = RedisURLQueue(redis, get_curated_domains, blacklist_provider=snapshot_blacklist)
+
completer = Completer()
index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
-# Pull the blacklist snapshot in at worker startup rather than letting the first query
-# pay for the ~11 MB Redis read. Subsequent refreshes happen on a daemon thread.
-get_snapshot_blacklist()
-
ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))
# Diversity is applied by the wrapping MMRRanker, which demotes (rather than drops)
# same-domain / near-duplicate results. Unwrap to disable diversity.