An open source, non-profit web search engine
Use the blacklist snapshot for the URL queue in the web workers
Daoud Clarke346389402298
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex edad557..64b0c2c 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -29,16 +29,25 @@ def get_curated_domains() -> set[str]:redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)-queued_batches = RedisURLQueue(redis, get_curated_domains)++# Pull the blacklist snapshot in at worker startup rather than letting the first query+# pay for the ~11 MB Redis read. Subsequent refreshes happen on a daemon thread.+snapshot_blacklist = get_snapshot_blacklist()++# Hand the snapshot to the URL queue too. Left to its own default it would build a+# get_default_blacklist_provider(), and POST /crawler/batches/new calls get_batch() ->+# is_domain_blacklisted(), which would download and parse the remote blocklists inside a+# request handler and pin ~156 MB of domain strings in each of the (cpu_count * 2 + 1)+# gunicorn workers - the cost the snapshot exists to avoid. The processes that legitimately+# need the full provider (update_urls, update_batches, the standalone crawler in crawl.py,+# whose Redis has no published snapshot) construct their own queue and are unaffected.+queued_batches = RedisURLQueue(redis, get_curated_domains, blacklist_provider=snapshot_blacklist)+completer = Completer()index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()-# Pull the blacklist snapshot in at worker startup rather than letting the first query-# pay for the ~11 MB Redis read. Subsequent refreshes happen on a daemon thread.-get_snapshot_blacklist()-ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))# Diversity is applied by the wrapping MMRRanker, which demotes (rather than drops)# same-domain / near-duplicate results. Unwrap to disable diversity.