An open source, non-profit web search engine
Format the codebase with ruff and fix what the linter found
Daoud Clarke6cd5fb4d516f
diff --git a/analyse/add_term_info.py b/analyse/add_term_info.pyindex c1030c2..a68d75e 100644--- a/analyse/add_term_info.py+++ b/analyse/add_term_info.py@@ -3,17 +3,15 @@ Investigate adding term information to the database.How much extra space will it take?"""-import os+from pathlib import Pathfrom random import Randomimport numpy as npfrom scipy.stats import sem--from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _trim_items_to_page-from zstandard import ZstdCompressor+from mwmbl.tinysearchengine.indexer import Document, TinyIndex, _trim_items_to_pagefrom mwmbl.utils import add_term_inforandom = Random(1)@@ -47,5 +45,5 @@ def run():print("New sizes mean", np.mean(new_sizes), sem(new_sizes))-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/analyse/analyse_crawled_domains.py b/analyse/analyse_crawled_domains.pyindex 28410bf..397b9e7 100644--- a/analyse/analyse_crawled_domains.py+++ b/analyse/analyse_crawled_domains.py@@ -1,14 +1,15 @@"""See how many unique URLs and root domains we have crawled."""+import globimport gzipimport json-from collections import defaultdict, Counter+from collections import Counter, defaultdictfrom urllib.parse import urlparsefrom mwmbl.crawler import HashedBatch-from mwmbl.indexer import CRAWL_GLOB, MWMBL_DATA_DIR+from mwmbl.indexer import CRAWL_GLOBdef get_urls():@@ -29,8 +30,8 @@ def analyse_urls(urls):url_set[url].append(user)parsed_url = urlparse(url)- path = parsed_url.path.strip('/')- if path == '':+ path = parsed_url.path.strip("/")+ if path == "":domains.add(parsed_url.netloc)count = sum(len(x) for x in url_set.values())@@ -52,6 +53,5 @@ def run():analyse_urls(urls)-if __name__ == '__main__':+if __name__ == "__main__":run()-diff --git a/analyse/analyse_crawled_links.py b/analyse/analyse_crawled_links.pyindex 2de43e2..d12d670 100644--- a/analyse/analyse_crawled_links.py+++ b/analyse/analyse_crawled_links.py@@ -5,7 +5,6 @@ from collections import Counterfrom urllib.parse import urlparseimport django-from django.conf import settingsdjango.setup()@@ -15,14 +14,11 @@ from mwmbl.crawler.batch import HashedBatchCRAWL_GLOB = "./devdata/batches/**/2024-05-25/**/*.json.gz"--def get_urls():for path in glob.glob(CRAWL_GLOB, recursive=True):print("Getting data from path", path)data = json.load(gzip.open(path))batch = HashedBatch.parse_obj(data)- user = batch.user_id_hashfor item in batch.items:if item.content is not None:yield item.url, item.content.links, item.content.extra_links@@ -51,5 +47,6 @@ def run():print("Link counts", link_counts.most_common())print("Extra counts", extra_counts.most_common())+if __name__ == "__main__":run()diff --git a/analyse/decompress_errors.py b/analyse/decompress_errors.pyindex 231c3cf..9a1370c 100644--- a/analyse/decompress_errors.py+++ b/analyse/decompress_errors.py@@ -1,6 +1,7 @@"""Create some data, compress it, randomly corrupt it, and then decompress it."""+from collections import defaultdictfrom random import Random@@ -14,7 +15,7 @@ def test_compress_decompress():for i in range(10000):compressor = ZstdCompressor()decompressor = ZstdDecompressor()- random_string = (''.join(random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(4096))).encode('utf8')+ random_string = ("".join(random.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(4096))).encode("utf8")compressed_data = compressor.compress(random_string)# Corrupt the data@@ -27,7 +28,7 @@ def test_compress_decompress():corrupted_data = bytes(compressed_data_array)try:- decompressed_data = decompressor.decompress(corrupted_data)+ decompressor.decompress(corrupted_data)except Exception as e:error_type_counts[f"{type(e)}: {str(e)}"] += 1continuediff --git a/analyse/estimate_unique_urls.py b/analyse/estimate_unique_urls.pyindex dd38c5f..feea102 100644--- a/analyse/estimate_unique_urls.py+++ b/analyse/estimate_unique_urls.py@@ -2,18 +2,14 @@Estimate the number of unique URLs in the index by fittinga Binomial distribution to the data."""+from collections import Counterfrom pathlib import Pathfrom random import Random-import numpy as npfrom pydistinct.ensemble_estimators import median_estimator-from pydistinct.stats_estimators import bootstrap_estimator, goodmans_estimator, smoothed_jackknife_estimator, \- horvitz_thompson_estimator-from scipy.optimize import curve_fit, differential_evolution-from scipy.stats import binom, poisson, betabinom-from mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.indexer import Document, TinyIndexrandom = Random(1)diff --git a/analyse/export_top_domains.py b/analyse/export_top_domains.pyindex b9b4479..de8cac2 100644--- a/analyse/export_top_domains.py+++ b/analyse/export_top_domains.py@@ -1,13 +1,13 @@import json-from mwmbl.indexer import TOP_DOMAINS_JSON_PATHfrom mwmbl.hn_top_domains_filtered import DOMAINS+from mwmbl.indexer import TOP_DOMAINS_JSON_PATHdef export_top_domains_to_json():- with open(TOP_DOMAINS_JSON_PATH, 'w') as output_file:+ with open(TOP_DOMAINS_JSON_PATH, "w") as output_file:json.dump(DOMAINS, output_file, indent=2)-if __name__ == '__main__':+if __name__ == "__main__":export_top_domains_to_json()diff --git a/analyse/export_urls.py b/analyse/export_urls.pyindex 39ba98e..113a9ab 100644--- a/analyse/export_urls.py+++ b/analyse/export_urls.py@@ -1,10 +1,11 @@"""Export the list of unique URLs to a SQLite file for analysis/evaluation."""+import sqlite3-from mwmbl.indexer import URLS_PATHfrom mwmbl.app import get_config_and_index+from mwmbl.indexer import URLS_PATHdef create_database():@@ -32,10 +33,13 @@ def run():with sqlite3.connect(URLS_PATH) as connection:for url_batch in url_batches:parameters = [(url,) for url in url_batch]- connection.executemany("""+ connection.executemany(+ """INSERT OR IGNORE INTO urls VALUES (?)- """, parameters)+ """,+ parameters,+ )-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/analyse/index_concurrency.py b/analyse/index_concurrency.pyindex 25d4cac..2663c0c 100644--- a/analyse/index_concurrency.py+++ b/analyse/index_concurrency.py@@ -1,11 +1,10 @@-import multiprocessingimport osimport stringfrom multiprocessing.pool import ThreadPoolfrom pathlib import Pathfrom random import Random-from mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.indexer import Document, TinyIndexrandom = Random(2)@@ -16,8 +15,9 @@ def random_string():DOCUMENTS = [- Document(title=f"Document {i}", url=f"https://something.com/{i}.html", extract=random_string(), score=i) for- i in range(1000)]+ Document(title=f"Document {i}", url=f"https://something.com/{i}.html", extract=random_string(), score=i)+ for i in range(1000)+]PAGES = [random.randint(0, 10240000) for _ in range(20)]@@ -33,13 +33,13 @@ try:except FileExistsError:pass-indexer = TinyIndex(Document, str(INDEX_PATH), 'r')+indexer = TinyIndex(Document, str(INDEX_PATH), "r")indexer.__enter__()def read_page(page_index: int):try:- page = indexer.get_page(page_index)+ indexer.get_page(page_index)except Exception as e:print(e)return@@ -56,5 +56,5 @@ def index_reading_writing_multithreading():print("End")-if __name__ == '__main__':+if __name__ == "__main__":index_reading_writing_multithreading()diff --git a/analyse/index_url_count.py b/analyse/index_url_count.pyindex dcb7245..cc6ace5 100644--- a/analyse/index_url_count.py+++ b/analyse/index_url_count.py@@ -1,12 +1,13 @@"""Count unique URLs in the index."""-from mwmbl.tinysearchengine import TinyIndex, Document++from mwmbl.tinysearchengine import Document, TinyIndexdef run():urls = set()- with TinyIndex(Document, 'data/index.tinysearch') as index:+ with TinyIndex(Document, "data/index.tinysearch") as index:for i in range(index.num_pages):print("Page", i)page = index.get_page(i)@@ -15,5 +16,5 @@ def run():print("URLs", len(urls))-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/analyse/load_test.py b/analyse/load_test.pyindex 82d3bcb..63b7cf6 100644--- a/analyse/load_test.py+++ b/analyse/load_test.py@@ -8,7 +8,7 @@ import requestsNUM_THREADS = 5-QUERIES_PATH = Path(__file__).parent.parent / 'devdata' / 'rankeval-2024-06' / 'queries.csv'+QUERIES_PATH = Path(__file__).parent.parent / "devdata" / "rankeval-2024-06" / "queries.csv"QUERY_URL = "http://localhost:5000/?q={query}"@@ -24,7 +24,9 @@ def run():times = pool.map(run_query, query_sample)print("Num threads\tNum queries\tMean seconds\tStd seconds\tMin seconds\tMax seconds")- print(f"{NUM_THREADS}\t{len(query_sample)}\t{np.mean(times):.4f}\t{np.std(times):.4f}\t{np.min(times):.4f}\t{np.max(times):.4f}")+ print(+ f"{NUM_THREADS}\t{len(query_sample)}\t{np.mean(times):.4f}\t{np.std(times):.4f}\t{np.min(times):.4f}\t{np.max(times):.4f}"+ )def run_query(query):@@ -36,6 +38,5 @@ def run_query(query):return (end_time - start_time).total_seconds()-if __name__ == '__main__':+if __name__ == "__main__":run()-diff --git a/analyse/recent_batches.py b/analyse/recent_batches.pyindex 460caba..5010092 100644--- a/analyse/recent_batches.py+++ b/analyse/recent_batches.py@@ -1,4 +1,3 @@"""Analyse recent batches looking for duplicates."""-diff --git a/analyse/record_historical_batches.py b/analyse/record_historical_batches.pyindex c482e49..f0420f4 100644--- a/analyse/record_historical_batches.py+++ b/analyse/record_historical_batches.py@@ -1,6 +1,7 @@"""See how many unique URLs and root domains we have crawled."""+import globimport gzipimport json@@ -9,7 +10,6 @@ import requestsfrom mwmbl.indexer import CRAWL_GLOB-API_ENDPOINT = "http://95.216.215.29/batches/historical"@@ -25,31 +25,29 @@ def get_batches():def convert_item(item):return {- 'url': item['url'],- 'status': 200,- 'timestamp': item['timestamp'],- 'content': {- 'title': item['title'],- 'extract': item['extract'],- 'links': item['links'],- }+ "url": item["url"],+ "status": 200,+ "timestamp": item["timestamp"],+ "content": {+ "title": item["title"],+ "extract": item["extract"],+ "links": item["links"],+ },}-def run():total_batches = total_num_batches()batches = get_batches()for i, hashed_batch in enumerate(batches):new_batch = {- 'user_id_hash': hashed_batch['user_id_hash'],- 'timestamp': hashed_batch['timestamp'],- 'items': [convert_item(item) for item in hashed_batch['items']]+ "user_id_hash": hashed_batch["user_id_hash"],+ "timestamp": hashed_batch["timestamp"],+ "items": [convert_item(item) for item in hashed_batch["items"]],}response = requests.post(API_ENDPOINT, json=new_batch)print(f"Response {i} of {total_batches}", response)-if __name__ == '__main__':+if __name__ == "__main__":run()-diff --git a/analyse/send_batch.py b/analyse/send_batch.pyindex 6d52d41..3564f34 100644--- a/analyse/send_batch.py+++ b/analyse/send_batch.py@@ -1,27 +1,33 @@"""Send a batch to a running instance."""+import requestsfrom mwmbl.crawler import Batch, Item, ItemContent--URL = 'http://localhost:5000/crawler/batches/'+URL = "http://localhost:5000/crawler/batches/"def run():- batch = Batch(user_id='test_user_id111111111111111111111111', items=[Item(- url='https://www.theguardian.com/stage/2007/nov/18/theatre',- content=ItemContent(- title='A nation in search of the new black | Theatre | The Guardian',- extract="Topic-stuffed and talk-filled, Kwame Kwei-Armah's new play proves that issue-driven drama is (despite reports of its death) still being written and staged…",- links=[]),- timestamp=123456,- status=200,- )])+ batch = Batch(+ user_id="test_user_id111111111111111111111111",+ items=[+ Item(+ url="https://www.theguardian.com/stage/2007/nov/18/theatre",+ content=ItemContent(+ title="A nation in search of the new black | Theatre | The Guardian",+ extract="Topic-stuffed and talk-filled, Kwame Kwei-Armah's new play proves that issue-driven drama is (despite reports of its death) still being written and staged…",+ links=[],+ ),+ timestamp=123456,+ status=200,+ )+ ],+ )result = requests.post(URL, data=batch.json())print("Result", result.content)-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/analyse/stress_test_request.py b/analyse/stress_test_request.pyindex 9eb2ff1..8d8ce79 100644--- a/analyse/stress_test_request.py+++ b/analyse/stress_test_request.py@@ -1,6 +1,7 @@"""Create a bunch of requests to the back end to stress test it."""+import multiprocessingfrom random import Random@@ -13,7 +14,7 @@ def random_query(query):url = f"http://localhost:5000/api/v1/search/?s={query}"print(url)response = requests.get(url)- data = response.json()+ response.json()# print(data)@@ -25,5 +26,5 @@ def run():print("End")-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/analyse/tfidf.py b/analyse/tfidf.pyindex 9838a3a..d31449d 100644--- a/analyse/tfidf.py+++ b/analyse/tfidf.py@@ -1,6 +1,7 @@"""Estimate word and document frequencies from a sample of the index."""+import jsonimport osfrom collections import Counter@@ -30,9 +31,9 @@ def get_frequencies():for term, count in document_counts.most_common(200):print(f"{term}: {count}")- with open(DOCUMENT_COUNTS_PATH, 'w') as f:+ with open(DOCUMENT_COUNTS_PATH, "w") as f:json.dump(document_counts, f, indent=2)-if __name__ == '__main__':+if __name__ == "__main__":get_frequencies()diff --git a/analyse/update_urls.py b/analyse/update_urls.pyindex f26c804..5efcc3b 100644--- a/analyse/update_urls.py+++ b/analyse/update_urls.py@@ -22,5 +22,5 @@ def run_update_urls_on_fixed_batches():print("Total time:", total_time)-if __name__ == '__main__':+if __name__ == "__main__":run_update_urls_on_fixed_batches()diff --git a/analyse/wiki_stats.py b/analyse/wiki_stats.pyindex 5e30c22..86b7408 100644--- a/analyse/wiki_stats.py+++ b/analyse/wiki_stats.py@@ -1,10 +1,11 @@"""Download Wikipedia statistics, aggregate and store in a file."""+import gzipimport jsonfrom collections import defaultdict-from datetime import datetime, date, timedelta+from datetime import date, datetime, timedeltafrom pathlib import Pathfrom random import Randomfrom typing import Iterable@@ -12,8 +13,10 @@ from typing import Iterableimport requestsfrom joblib import Memory-WIKI_STATS_URL_FORMAT = ("https://dumps.wikimedia.org/other/pageviews/"- "{year}/{year}-{month:02d}/pageviews-{year}{month:02d}{day:02d}-{hour:02d}0000.gz")+WIKI_STATS_URL_FORMAT = (+ "https://dumps.wikimedia.org/other/pageviews/"+ "{year}/{year}-{month:02d}/pageviews-{year}{month:02d}{day:02d}-{hour:02d}0000.gz"+)OUTPUT_PATH = Path(__file__).parent.parent / "mwmbl" / "resources" / "wiki_stats.json"@@ -34,7 +37,9 @@ def get_wiki_stats_urls(n: int = 10):seen_hours.add(hours_ago)url_date = max_date - timedelta(hours=hours_ago)- url = WIKI_STATS_URL_FORMAT.format(year=url_date.year, month=url_date.month, day=url_date.day, hour=url_date.hour)+ url = WIKI_STATS_URL_FORMAT.format(+ year=url_date.year, month=url_date.month, day=url_date.day, hour=url_date.hour+ )yield url@@ -75,10 +80,3 @@ def run():if __name__ == "__main__":run()-------diff --git a/manage.py b/manage.pyindex 91ff951..5c7d5f9 100755--- a/manage.py+++ b/manage.py@@ -1,12 +1,13 @@#!/usr/bin/env python"""Django's command-line utility for administrative tasks."""+import osimport sysdef main():"""Run administrative tasks."""- os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mwmbl.settings_dev')+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")try:from django.core.management import execute_from_command_lineexcept ImportError as exc:@@ -18,5 +19,5 @@ def main():execute_from_command_line(sys.argv)-if __name__ == '__main__':+if __name__ == "__main__":main()diff --git a/mwmbl/admin.py b/mwmbl/admin.pyindex 13aa3c1..75a9b1f 100644--- a/mwmbl/admin.py+++ b/mwmbl/admin.py@@ -3,7 +3,17 @@ from django.contrib import adminfrom django.contrib.admin import ModelAdminfrom django.contrib.auth.admin import UserAdmin-from mwmbl.models import MwmblUser, OldIndex, Curation, FlagCuration, DomainSubmission, ApiKey, MarketingConsent, UserBilling, generate_api_key+from mwmbl.models import (+ ApiKey,+ Curation,+ DomainSubmission,+ FlagCuration,+ MarketingConsent,+ MwmblUser,+ OldIndex,+ UserBilling,+ generate_api_key,+)class ApiKeyForm(forms.ModelForm):@@ -50,6 +60,8 @@ admin.site.register(Curation, ModelAdmin)admin.site.register(OldIndex, ModelAdmin)admin.site.register(FlagCuration, ModelAdmin)admin.site.register(DomainSubmission, ModelAdmin)++class MarketingConsentAdmin(ModelAdmin):list_display = ("user", "source", "opted_in", "timestamp")list_filter = ("source", "opted_in")@@ -57,7 +69,13 @@ class MarketingConsentAdmin(ModelAdmin):class UserBillingAdmin(ModelAdmin):- list_display = ("user", "max_monthly_spend_cents", "polar_customer_id", "current_period_end", "cancel_at_period_end")+ list_display = (+ "user",+ "max_monthly_spend_cents",+ "polar_customer_id",+ "current_period_end",+ "cancel_at_period_end",+ )readonly_fields = ("polar_customer_id", "polar_subscription_id", "current_period_end")search_fields = ("user__username", "user__email", "polar_customer_id")diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.pyindex f84a86e..ca8db66 100644--- a/mwmbl/admin_views.py+++ b/mwmbl/admin_views.py@@ -11,6 +11,7 @@ The page is read-only, and deliberately cheap: the snapshot's size comes from STpage load never pulls the ~11 MB blob out of Redis, and the queue is sampled withSRANDMEMBER so looking at it cannot consume it."""+import osfrom datetime import datetime, timedeltafrom logging import getLogger@@ -25,7 +26,10 @@ from mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEYfrom mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer import blacklist_snapshot, purge_queuefrom mwmbl.indexer.blacklist_snapshot import (- HASH_DTYPE, SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY, get_snapshot_blacklist,+ HASH_DTYPE,+ SNAPSHOT_KEY,+ SNAPSHOT_VERSION_KEY,+ get_snapshot_blacklist,)from mwmbl.indexer.purge_queue import MAX_QUEUE_SIZE, PURGE_QUEUE_KEY, peek_purge_queue@@ -132,11 +136,13 @@ def _task_status() -> list[dict]:statuses = []for task_name in task_names:last_completed = CompletedTask.objects.filter(task_name=task_name).order_by("-run_at").first()- statuses.append({- "name": task_name,- "task": pending.get(task_name),- "last_completed": last_completed,- })+ statuses.append(+ {+ "name": task_name,+ "task": pending.get(task_name),+ "last_completed": last_completed,+ }+ )return statusesdiff --git a/mwmbl/api.py b/mwmbl/api.pyindex 06c4c46..ee33d78 100644--- a/mwmbl/api.py+++ b/mwmbl/api.py@@ -19,15 +19,15 @@ typically at /api/v1/platform/token/pair, /api/v1/platform/token/refresh, etc.from ninja import Fieldfrom ninja_extra import NinjaExtraAPI, api_controller, http_postfrom ninja_extra.permissions import AllowAny-from ninja_jwt.controller import TokenVerificationController, TokenObtainPairController+from ninja_jwt.controller import TokenObtainPairController, TokenVerificationControllerfrom ninja_jwt.schema import TokenObtainPairInputSchemafrom ninja_jwt.schema_control import SchemaControlfrom ninja_jwt.settings import api_settings--from mwmbl.exceptions import InvalidRequestfrom scalar_ninja import ScalarViewerfrom scalar_ninja.scalar_ninja import AgentConfig+from mwmbl.exceptions import InvalidRequest+_schema = SchemaControl(api_settings)@@ -59,6 +59,7 @@ class MwmblTokenController(TokenVerificationController, TokenObtainPairControlleuser_token.check_user_authentication_rule()return user_token.to_response_schema()+api = NinjaExtraAPI(title="Mwmbl API",version="1.0.0",@@ -137,9 +138,9 @@ def register_routers(ranker, batch_cache, queued_batches):This is called from urls.py after Django app setup is complete, so thatdependencies (ranker, batch_cache, queued_batches) are available."""+ import mwmbl.crawler.app as crawler_moduleimport mwmbl.tinysearchengine.search as search_moduleimport mwmbl.tinysearchengine.super_search as super_search_module- import mwmbl.crawler.app as crawler_modulefrom mwmbl.platform.api import router as platform_router# Initialise routers that depend on runtime objectsdiff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex 2d5a48f..b152a64 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -1,4 +1,3 @@-from multiprocessing import Process, Queuefrom pathlib import Pathfrom django.apps import AppConfig@@ -19,7 +18,8 @@ def create_index(index_name, num_pages, rebuild_on_mismatch=False):re-fetching."""# Imports here to avoid AppRegistryNotReady exception- from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE+ from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex+index_path = Path(settings.DATA_PATH) / index_nametry:existing_index = TinyIndex(item_factory=Document, index_path=index_path)@@ -47,8 +47,10 @@ def create_index(index_name, num_pages, rebuild_on_mismatch=False):if existing_index.page_size == PAGE_SIZE and existing_index.num_pages == num_pages:return- message = (f"Existing index page sizes ({existing_index.page_size}) or number of pages "- f"({existing_index.num_pages}) do not match")+ message = (+ f"Existing index page sizes ({existing_index.page_size}) or number of pages "+ f"({existing_index.num_pages}) do not match"+ )if not rebuild_on_mismatch:raise ValueError(message)print(f"{message} - rebuilding {index_path}")@@ -56,9 +58,9 @@ def create_index(index_name, num_pages, rebuild_on_mismatch=False):def _create_index_file(index_path, num_pages):- from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE- TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=num_pages,- page_size=PAGE_SIZE)+ from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex++ TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=num_pages, page_size=PAGE_SIZE)def _replace_index_file(index_path, num_pages):@@ -93,11 +95,11 @@ class MwmblConfig(AppConfig):# boot after a size change pays for the whole file up front - ~17s for the 15 GB# production cache at the ~0.9 GB/s measured locally, which is what the healthcheck# grace in app.json has to cover.- create_index(settings.EXTERNAL_CACHE_INDEX_NAME, settings.EXTERNAL_CACHE_NUM_PAGES,- rebuild_on_mismatch=True)+ create_index(settings.EXTERNAL_CACHE_INDEX_NAME, settings.EXTERNAL_CACHE_NUM_PAGES, rebuild_on_mismatch=True)if settings.HAS_DATABASE:create_index_db()import mwmbl.signals # noqa: F401 - connects the post_save receivers+self._schedule_background_tasks()# Cache key guarding the scheduling critical section below. Held just long@@ -113,6 +115,7 @@ class MwmblConfig(AppConfig):"""import asyncioimport logging+log = logging.getLogger(__name__)# Under bare `uvicorn` (as opposed to gunicorn+uvicorn-workers), apps.ready()@@ -137,15 +140,20 @@ class MwmblConfig(AppConfig):# the rest skip immediately, and the lock's TTL simply bounds how long# a crashed winner blocks a retry by a later-starting worker.from django.core.cache import cache+if not cache.add(MwmblConfig._SCHEDULE_LOCK_KEY, "1", timeout=MwmblConfig._SCHEDULE_LOCK_TIMEOUT):log.info("Skipping background task scheduling: another worker is already handling it.")returntry:from background_task.models import Task+from mwmbl.background import (- purge_blacklisted_from_queue, refresh_blacklist_snapshot, report_usage_to_polar,- retrain_domain_moderation_model, sync_search_counts,+ purge_blacklisted_from_queue,+ refresh_blacklist_snapshot,+ report_usage_to_polar,+ retrain_domain_moderation_model,+ sync_search_counts,)SYNC_TASK = "mwmbl.background.sync_search_counts"@@ -171,13 +179,11 @@ class MwmblConfig(AppConfig):# a one-off rebuild under the same task name (mwmbl.signals), and one of those# sitting in the queue at deploy time must not suppress the periodic task.if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, repeat__gt=0).exists():- refresh_blacklist_snapshot(- repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)+ refresh_blacklist_snapshot(repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)# Drain the queue of blacklisted documents that retrieval has filtered outif not Task.objects.filter(task_name=BLACKLIST_PURGE_TASK).exists():- purge_blacklisted_from_queue(- repeat=settings.BLACKLIST_PURGE_INTERVAL_SECONDS, repeat_until=None)+ purge_blacklisted_from_queue(repeat=settings.BLACKLIST_PURGE_INTERVAL_SECONDS, repeat_until=None)# Retrain the moderation suggester monthly. Scheduled a month out rather than# schedule=0: unlike the blacklist snapshot there is nothing useful about@@ -185,7 +191,9 @@ class MwmblConfig(AppConfig):if not Task.objects.filter(task_name=MODERATION_RETRAIN_TASK).exists():retrain_domain_moderation_model(schedule=settings.MODERATION_RETRAIN_INTERVAL_SECONDS,- repeat=settings.MODERATION_RETRAIN_INTERVAL_SECONDS, repeat_until=None)+ repeat=settings.MODERATION_RETRAIN_INTERVAL_SECONDS,+ repeat_until=None,+ )except Exception:# Don't prevent startup if background task scheduling failsdiff --git a/mwmbl/asgi.py b/mwmbl/asgi.pyindex 73088a9..e30c754 100644--- a/mwmbl/asgi.py+++ b/mwmbl/asgi.py@@ -11,6 +11,6 @@ import osfrom django.core.asgi import get_asgi_application-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mwmbl.settings_dev')+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")application = get_asgi_application()diff --git a/mwmbl/auth.py b/mwmbl/auth.pyindex 18aee75..929fdb2 100644--- a/mwmbl/auth.py+++ b/mwmbl/auth.py@@ -21,6 +21,7 @@ class UsernameOrEmailBackend:def authenticate(self, request, username=None, password=None, **kwargs):from django.contrib.auth import get_user_model+UserModel = get_user_model()if not username or not password:@@ -46,6 +47,7 @@ class UsernameOrEmailBackend:def get_user(self, user_id):from django.contrib.auth import get_user_model+UserModel = get_user_model()try:return UserModel.objects.get(pk=user_id)diff --git a/mwmbl/background.py b/mwmbl/background.pyindex eaf8687..5a83282 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -7,10 +7,11 @@ Also contains Django Background Tasks for periodic maintenance:- refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against- purge_blacklisted_from_queue: removes retrieval-filtered documents from the index"""+import loggingimport sysfrom datetime import datetime, timedelta, timezone-from logging import getLogger, basicConfig+from logging import basicConfig, getLoggerfrom pathlib import Pathfrom time import sleep@@ -23,7 +24,7 @@ from redis import Redisfrom mwmbl import pricingfrom mwmbl.crawler.stats import StatsManager-from mwmbl.indexer import index_batches, historical+from mwmbl.indexer import historical, index_batchesfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist, refresh_snapshotfrom mwmbl.indexer.purge_blacklisted import purge_documents@@ -105,6 +106,7 @@ def copy_indexes_continuously():# Periodic quota maintenance tasks (Django Background Tasks)# ---------------------------------------------------------------------------+@background(schedule=0)def sync_search_counts():"""@@ -141,7 +143,9 @@ def sync_search_counts():month = int(parts[4])count = cache.get(key, default=0)UsageBucket.objects.update_or_create(- user_id=user_id, year=year, month=month,+ user_id=user_id,+ year=year,+ month=month,defaults={"count": count},)except Exception:@@ -152,6 +156,7 @@ def sync_search_counts():# Blacklisted-domain maintenance (Django Background Tasks)# ---------------------------------------------------------------------------+@background(schedule=0)def refresh_blacklist_snapshot():"""@@ -179,14 +184,18 @@ def purge_blacklisted_from_queue():blacklist = get_snapshot_blacklist()index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME- with TinyIndex(Document, str(index_path), 'w') as index:+ with TinyIndex(Document, str(index_path), "w") as index:removed_by_domain = purge_documents(index, documents, blacklist.is_domain_blacklisted)num_removed = sum(removed_by_domain.values())stats_manager.record_blacklisted_removed(num_removed)- logger.info("Purged %d documents from the index across %d domains; %d still queued",- num_removed, len(removed_by_domain), queue_size())+ logger.info(+ "Purged %d documents from the index across %d domains; %d still queued",+ num_removed,+ len(removed_by_domain),+ queue_size(),+ )@background(schedule=0)@@ -216,11 +225,13 @@ def report_usage_to_polar():if delta <= 0:continue- events.append({- "name": "search_request",- "external_customer_id": str(bucket.user.id),- "metadata": {"quantity": delta},- })+ events.append(+ {+ "name": "search_request",+ "external_customer_id": str(bucket.user.id),+ "metadata": {"quantity": delta},+ }+ )bucket.reported_overage = total_overagebuckets_to_update.append(bucket)@@ -241,6 +252,7 @@ def report_usage_to_polar():# Domain moderation suggestions (Django Background Tasks)# ---------------------------------------------------------------------------+@background(schedule=0)def enrich_domain_submission(domain: str):"""Crawl a submitted domain and store the suggestion the moderation queue will read.@@ -253,11 +265,14 @@ def enrich_domain_submission(domain: str):a domain whose evidence is still fresh is left alone. That matters because 615 of 6,949distinct submitted domains have been submitted more than once."""- cutoff = datetime.now(timezone.utc) - timedelta(- days=settings.MODERATION_EVIDENCE_MAX_AGE_DAYS)+ cutoff = datetime.now(timezone.utc) - timedelta(days=settings.MODERATION_EVIDENCE_MAX_AGE_DAYS)existing = DomainEvidence.objects.filter(domain=domain).first()- if (existing is not None and existing.state == DomainEvidence.State.READY- and existing.fetched_at and existing.fetched_at > cutoff):+ if (+ existing is not None+ and existing.state == DomainEvidence.State.READY+ and existing.fetched_at+ and existing.fetched_at > cutoff+ ):logger.info("Evidence for %s is still fresh, skipping crawl", domain)return@@ -278,8 +293,7 @@ def enrich_domain_submission(domain: str):with transaction.atomic():evidence = store_evidence(domain, crawl)refresh_suggestion(evidence)- logger.info("Enriched %s: %s (%.2f)", domain, evidence.suggested_action,- evidence.confidence or 0.0)+ logger.info("Enriched %s: %s (%.2f)", domain, evidence.suggested_action, evidence.confidence or 0.0)@background(schedule=0)@@ -291,11 +305,8 @@ def rescore_pending_submissions():- the page text is already stored - so the whole backlog re-scores in seconds."""reset_model_cache()- pending_domains = set(DomainSubmission.objects- .filter(status="PENDING")- .values_list("name", flat=True))- queryset = DomainEvidence.objects.filter(domain__in=pending_domains,- state=DomainEvidence.State.READY)+ pending_domains = set(DomainSubmission.objects.filter(status="PENDING").values_list("name", flat=True))+ queryset = DomainEvidence.objects.filter(domain__in=pending_domains, state=DomainEvidence.State.READY)rescored = 0for evidence in queryset.iterator():diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.pyindex 0705a01..12fe7df 100644--- a/mwmbl/count_urls.py+++ b/mwmbl/count_urls.py@@ -1,6 +1,6 @@import osfrom collections import Counter-from datetime import date, timedelta, datetime+from datetime import date, datetime, timedeltafrom logging import getLoggerfrom pathlib import Pathfrom random import Random@@ -36,7 +36,7 @@ def count_urls_continuously():start_time = datetime.utcnow()count_urls()end_time = datetime.utcnow()- total_time = (end_time - start_time)+ total_time = end_time - start_timetime_remaining = 60 * 60 * 24 - total_time.total_seconds()logger.info(f"Counting took {total_time}. Sleeping for {timedelta(seconds=time_remaining)}.")sleep(time_remaining)@@ -72,10 +72,14 @@ def count_urls():num_results_estimate = int(total_docs / PAGE_PROPORTION_TO_SAMPLE)url_count_estimate = smoothed_jackknife_estimator(attributes=dict(url_counts.items()), n_pop=num_results_estimate)- domain_count_estimate = smoothed_jackknife_estimator(attributes=dict(domain_counts.items()), n_pop=num_results_estimate)+ domain_count_estimate = smoothed_jackknife_estimator(+ attributes=dict(domain_counts.items()), n_pop=num_results_estimate+ )- logger.info(f"Estimated {url_count_estimate} unique URLs, {domain_count_estimate} unique domains, "- f"and {num_results_estimate} results in the index.")+ logger.info(+ f"Estimated {url_count_estimate} unique URLs, {domain_count_estimate} unique domains, "+ f"and {num_results_estimate} results in the index."+ )redis = get_redis()@@ -135,6 +139,7 @@ def _get_count(redis, count_dict, key, date_i):if __name__ == "__main__":# configure loggingimport logging+logging.basicConfig(level=logging.INFO)count_urls()diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex bf6a67d..6515cb3 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -29,13 +29,13 @@ print("Mwmbl crawling statistics: https://mwmbl.org/stats")django.setup()from mwmbl.crawler.batch import HashedBatch, Result, Results-from mwmbl.crawler.env_vars import CRAWLER_WORKERS, CRAWL_DELAY_SECONDS, MWMBL_API_KEY, MWMBL_CONTACT_INFO-from mwmbl.crawler.retrieve import crawl_batch, crawl_url, CRAWLER_VERSION, USER_AGENT+from mwmbl.crawler.env_vars import CRAWL_DELAY_SECONDS, CRAWLER_WORKERS, MWMBL_API_KEY, MWMBL_CONTACT_INFO+from mwmbl.crawler.retrieve import CRAWLER_VERSION, USER_AGENT, crawl_urlfrom mwmbl.indexer.index_batches import index_batches, index_pagesfrom mwmbl.indexer.update_urls import record_urls_in_databasefrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.redis_url_queue import RedisURLQueue-from mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.indexer import Document, TinyIndexfrom mwmbl.tinysearchengine.rank import score_resultfrom mwmbl.tokenizer import tokenize@@ -73,7 +73,7 @@ def validate_environment():"""Validate required environment variables for crawler operation."""if not MWMBL_API_KEY.strip():raise ValueError("An environment variable MWMBL_API_KEY must be set to run the crawler")-+if MWMBL_CONTACT_INFO == "[email protected]":raise ValueError("MWMBL_CONTACT_INFO must be set to your email or website URL. "@@ -85,25 +85,25 @@ def validate_environment():class Crawler:"""Main crawler class that manages Redis connections, URL queues, and crawling operations.-+This class encapsulates all the crawler functionality including:- Redis connection management- URL queue operations- Batch processing- Indexing operations"""-+def __init__(self, redis_url=None):"""Initialize the crawler with Redis connection.-+Args:redis_url: Optional Redis URL. If not provided, uses settings.REDIS_URL"""self.redis_url = redis_url or settings.REDIS_URLself._redis = Noneself._url_queue = None-+@propertydef redis(self):"""Lazy initialization of Redis connection."""@@ -114,22 +114,22 @@ class Crawler:health_check_interval=30,)return self._redis-+@propertydef url_queue(self):"""Lazy initialization of URL queue."""if self._url_queue is None:self._url_queue = RedisURLQueue(self.redis, _fetch_curated_domains)return self._url_queue-+def check_redis(self):"""Check Redis connection health."""try:self.redis.ping()logger.debug("Redis ping successful")- except ConnectionError as e:+ except ConnectionError:raise SystemExit(f"Cannot reach Redis at {self.redis_url}. Make sure your Redis server is running.")-+def process_batch(self):"""Process a single batch of URLs by crawling them sequentially with rate limiting.@@ -158,19 +158,21 @@ class Crawler:result = crawl_url(url, self.redis)results.append(result)logger.debug("Result", result)-+js_timestamp = int(time.time() * 1000)- batch = HashedBatch.parse_obj({- "user_id_hash": user_id,- "timestamp": js_timestamp,- "items": results,- })+ batch = HashedBatch.parse_obj(+ {+ "user_id_hash": user_id,+ "timestamp": js_timestamp,+ "items": results,+ }+ )record_urls_in_database([batch], self.url_queue)# Push the batch into the Redis queuebatch_json = batch.json()self.redis.rpush(BATCH_QUEUE_KEY, batch_json)-+def run_indexing(self):"""Process completed crawl batches and integrate results into the search index.@@ -194,14 +196,14 @@ class Crawler:logger.info("No more batches to index. Sleeping for 10 seconds.")time.sleep(10)return-+logger.info(f"Got {len(batch_jsons)} batches to index")batches = [HashedBatch.parse_raw(b) for b in batch_jsons]term_new_doc_count = index_batches(batches, index_path)logger.info(f"Indexed, top terms to sync: {term_new_doc_count.most_common(10)}")remote_index = RemoteIndex()- with TinyIndex(Document, index_path, 'w') as local_index:+ with TinyIndex(Document, index_path, "w") as local_index:for term, count in term_new_doc_count.most_common(100):logger.info(f"Syncing term {term} with {count} new local items")remote_items = remote_index.retrieve(term)@@ -220,14 +222,21 @@ class Crawler:new_high_score = max_local_score < min_remote_scoreif new_high_score:- result_items = [Result(url=doc.url, title=doc.title, extract=doc.extract,- score=doc.score, term=doc.term, state=doc.state,- last_crawled=doc.last_crawled) for doc in new_items]+ result_items = [+ Result(+ url=doc.url,+ title=doc.title,+ extract=doc.extract,+ score=doc.score,+ term=doc.term,+ state=doc.state,+ last_crawled=doc.last_crawled,+ )+ for doc in new_items+ ]results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)logger.info(f"Posting {len(result_items)} results")- response = requests.post(- "https://api.mwmbl.org/api/v1/crawler/results", json=results.dict()- )+ response = requests.post("https://api.mwmbl.org/api/v1/crawler/results", json=results.dict())logger.info(f"Response: {response.text}")response.raise_for_status()@@ -243,7 +252,7 @@ class Crawler:new_page_content = local_index.get_page(page_index)logger.info(f"Page content: {new_page_content}")-+def process_batch_continuously(self):"""Continuously process batches with error handling."""while True:@@ -253,7 +262,7 @@ class Crawler:except Exception as err:logger.exception(f"Error processing batch: '{err}'")time.sleep(10)-+def run_indexing_continuously(self):"""Continuously run indexing with error handling."""while True:@@ -263,11 +272,11 @@ class Crawler:except Exception as err:logger.exception(f"Error running indexing: '{err}'")time.sleep(10)-+def run(self):"""Main entry point for the crawler.-+Starts multiple worker processes for batch processing and one indexing process.Monitors processes and restarts them if they crash."""@@ -308,13 +317,17 @@ class Crawler:for crash_time, exit_code, pid in index_crash_history:crash_details.append(f" - {crash_time.isoformat()}: pid={pid}, exit_code={exit_code}")- error_msg = (f"Index process crashed {len(index_crash_history)} times in the last hour "- f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details))+ error_msg = (+ f"Index process crashed {len(index_crash_history)} times in the last hour "+ f"(threshold: 5). Recent crashes:\n" + "\n".join(crash_details)+ )logger.error(error_msg)raise RuntimeError(error_msg)- logger.warning(f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "- f"Crash count in last hour: {len(index_crash_history)}")+ logger.warning(+ f"Indexing process [pid={pid}] died with exit code {exit_code}, respawning. "+ f"Crash count in last hour: {len(index_crash_history)}"+ )index_process = Process(target=self.run_indexing_continuously)index_process.start()@@ -331,6 +344,7 @@ class Crawler:# Global instance for backward compatibility_default_crawler = None+def get_default_crawler():"""Get the default crawler instance."""global _default_crawler@@ -338,19 +352,23 @@ def get_default_crawler():_default_crawler = Crawler()return _default_crawler+# Backward compatibility functionsdef check_redis():"""Check Redis connection health (backward compatibility)."""return get_default_crawler().check_redis()+def process_batch():"""Process a single batch (backward compatibility)."""return get_default_crawler().process_batch()+def run_indexing():"""Run indexing (backward compatibility)."""return get_default_crawler().run_indexing()+def run():"""Main entry point (backward compatibility)."""return get_default_crawler().run()diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 836d8ff..f101fef 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -1,9 +1,9 @@import gzipimport hashlibimport json-from logging import getLoggerimport os-from datetime import datetime, timezone, date+from datetime import date, datetime, timezone+from logging import getLoggerfrom queue import Emptyfrom typing import Unionfrom uuid import uuid4@@ -15,30 +15,41 @@ from ninja import NinjaAPI, Router, Schemafrom ninja.errors import HttpErrorfrom redis import Redis-from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequest, HashedDataset+from mwmbl.crawler.batch import (+ Batch,+ DatasetRequest,+ Error,+ HashedBatch,+ HashedDataset,+ NewBatchRequest,+ PostResultsResponse,+ Results,+)from mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.database import Database-from mwmbl.exceptions import InvalidRequestfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.index_batches import index_documents-from mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus+from mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabasefrom mwmbl.models import ApiKeyfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.settings import (- ENDPOINT_URL,- KEY_ID,APPLICATION_KEY,BUCKET_NAME,+ DATE_REGEX,+ ENDPOINT_URL,+ FILE_NAME_SUFFIX,+ KEY_ID,MAX_BATCH_SIZE,- USER_ID_LENGTH,- VERSION,PUBLIC_URL_PREFIX,PUBLIC_USER_ID_LENGTH,- FILE_NAME_SUFFIX,- DATE_REGEX)+ USER_ID_LENGTH,+ VERSION,+)from mwmbl.tinysearchengine.indexer import Document-stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True))+stats_manager = StatsManager(+ Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)+)logger = getLogger(__name__)@@ -47,8 +58,9 @@ router = Router(tags=["Crawler"])def get_bucket(name):- s3 = boto3.resource('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,- aws_secret_access_key=APPLICATION_KEY)+ s3 = boto3.resource(+ "s3", endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID, aws_secret_access_key=APPLICATION_KEY+ )return s3.Object(BUCKET_NAME, name)@@ -72,8 +84,8 @@ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object# See discussion here: https://stackoverflow.com/a/13484764uid = str(uuid4())[:8]- filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'- data = gzip.compress(model_object.json().encode('utf8'))+ filename = f"1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz"+ data = gzip.compress(model_object.json().encode("utf8"))upload(data, filename)return filename@@ -82,7 +94,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch"""Register all crawler routes on the given router or API instance."""@r.post(- '/batches/',+ "/batches/",summary="Submit a crawl batch",description=("Deprecated - the new crawler uses the /results/ endpoint.\n\n"@@ -95,14 +107,16 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch)def post_batch(request, batch: Batch):if len(batch.items) > MAX_BATCH_SIZE:- return r.create_response(request, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}", status=400)+ return r.create_response(+ request, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}", status=400+ )if len(batch.user_id) != USER_ID_LENGTH:return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)if len(batch.items) == 0:return {- 'status': 'ok',+ "status": "ok",}user_id_hash = _get_user_id_hash(batch)@@ -110,9 +124,13 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchurls = [item.url for item in batch.items]invalid_urls = queued_batches.check_user_crawled_urls(user_id_hash, urls)if invalid_urls:- return r.create_response(request, f"The following URLs were not assigned to the user for crawling:"- f" {invalid_urls}. To suggest a domain to crawl, please visit "- f"https://mwmbl.org/app/domain-submissions/new", status=400)+ return r.create_response(+ request,+ f"The following URLs were not assigned to the user for crawling:"+ f" {invalid_urls}. To suggest a domain to crawl, please visit "+ f"https://mwmbl.org/app/domain-submissions/new",+ status=400,+ )# Using an approach from https://stackoverflow.com/a/30476450now = datetime.now(timezone.utc)@@ -126,7 +144,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchglobal last_batchlast_batch = hashed_batch- batch_url = f'{PUBLIC_URL_PREFIX}{filename}'+ batch_url = f"{PUBLIC_URL_PREFIX}{filename}"batch_cache.store(hashed_batch, batch_url)# Record the batch as being local so that we don't retrieve it again when the server restarts@@ -137,13 +155,13 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchindex_db.record_batches(infos)return {- 'status': 'ok',- 'public_user_id': user_id_hash,- 'url': batch_url,+ "status": "ok",+ "public_user_id": user_id_hash,+ "url": batch_url,}@r.post(- '/batches/new',+ "/batches/new",summary="Request URLs to crawl",description=("Deprecated - crawlers should now determine their own batches.\n\n"@@ -162,7 +180,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchreturn urls@r.get(- '/batches/{date_str}/users/{public_user_id}',+ "/batches/{date_str}/users/{public_user_id}",summary="List batch IDs for a user on a date",description=("Retrieve the list of batch IDs submitted by a specific user on a given date. "@@ -173,11 +191,11 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchdef get_batches_for_date_and_user(request, date_str, public_user_id):check_date_str(date_str)check_public_user_id(public_user_id)- prefix = f'1/{VERSION}/{date_str}/1/{public_user_id}/'+ prefix = f"1/{VERSION}/{date_str}/1/{public_user_id}/"return get_batch_ids_for_prefix(prefix)@r.get(- '/batches/{date_str}/users/{public_user_id}/batch/{batch_id}',+ "/batches/{date_str}/users/{public_user_id}/batch/{batch_id}",summary="Get a specific batch",description=("Retrieve the full content of a specific crawl batch from object storage. "@@ -189,12 +207,12 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchurl = get_batch_url(batch_id, date_str, public_user_id)data = json.loads(gzip.decompress(requests.get(url).content))return {- 'url': url,- 'batch': data,+ "url": url,+ "batch": data,}@r.get(- '/latest-batch',+ "/latest-batch",summary="Get the latest batch",description=("Return the most recently submitted crawl batch held in memory. "@@ -205,7 +223,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchreturn [] if last_batch is None else [last_batch]@r.get(- '/batches/{date_str}/users',+ "/batches/{date_str}/users",summary="List crawlers active on a date",description=("Return the list of public user ID hashes (SHA3-256) for all crawlers that submitted "@@ -214,11 +232,11 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch)def get_user_id_hashes_for_date(request, date_str: str):check_date_str(date_str)- prefix = f'1/{VERSION}/{date_str}/1/'+ prefix = f"1/{VERSION}/{date_str}/1/"return get_subfolders(prefix)@r.get(- '/stats',+ "/stats",summary="Crawler statistics",description=("Return aggregate statistics about the Mwmbl crawler network, including the number "@@ -230,14 +248,12 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchreturn stats_manager.get_stats()@r.get(- '/',+ "/",summary="Health check",- description="Returns `{\"status\": \"ok\"}` if the crawler API is running.",+ description='Returns `{"status": "ok"}` if the crawler API is running.',)def status(request):- return {- 'status': 'ok'- }+ return {"status": "ok"}class CuratedDomain(Schema):name: str@@ -246,7 +262,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchdomains: list[CuratedDomain]@r.get(- '/curated-domains',+ "/curated-domains",summary="Get curated domains for crawling",description=("Retrieve the list of approved domains curated by mwmbl users. "@@ -256,12 +272,11 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch)def get_curated_domains_endpoint(request) -> CuratedDomainsResponse:from mwmbl.curated_domains import get_curated_domains- return CuratedDomainsResponse(- domains=[CuratedDomain(name=d) for d in sorted(get_curated_domains())]- )++ return CuratedDomainsResponse(domains=[CuratedDomain(name=d) for d in sorted(get_curated_domains())])@r.post(- '/results',+ "/results",response={200: PostResultsResponse, 400: Error, 401: Error},summary="Submit indexed results",description=(@@ -279,10 +294,14 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchreturn 401, {"message": "API key required. Pass it in the X-API-Key header."}key_hash = hashlib.sha256(raw_key.encode()).hexdigest()- api_key = ApiKey.objects.filter(- key=key_hash,- scopes__contains=[ApiKey.Scope.CRAWL],- ).select_related("user").first()+ api_key = (+ ApiKey.objects.filter(+ key=key_hash,+ scopes__contains=[ApiKey.Scope.CRAWL],+ )+ .select_related("user")+ .first()+ )if api_key is None:return 401, {"message": "Invalid API key or insufficient scope (crawl scope required)."}@@ -294,11 +313,15 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchif result.last_crawled is not None and result.last_crawled > now_ts:return 400, {"message": f"last_crawled timestamp is in the future for URL: {result.url}"}last_crawled = result.last_crawled if result.last_crawled is not None else now_ts- documents.append(Document(- url=result.url, title=result.title, extract=result.extract,- user_ids=[api_key.user.id],- last_crawled=last_crawled,- ))+ documents.append(+ Document(+ url=result.url,+ title=result.title,+ extract=result.extract,+ user_ids=[api_key.user.id],+ last_crawled=last_crawled,+ )+ )index_path = f"{settings.DATA_PATH}/{settings.INDEX_NAME}"index_documents(documents, index_path)@@ -308,12 +331,12 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchstats_manager.record_results(results, api_key.user.username)return {- 'status': 'ok',- 'url': f'{PUBLIC_URL_PREFIX}{filename}',+ "status": "ok",+ "url": f"{PUBLIC_URL_PREFIX}{filename}",}@r.post(- '/dataset',+ "/dataset",summary="Submit Firefox extension dataset",description=("Submit a dataset of search interactions collected by the Mwmbl Firefox extension. "@@ -335,7 +358,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchtimestamp=dataset.timestamp,extensionVersion=dataset.extensionVersion,queryDataset=dataset.queryDataset,- searchResults=dataset.searchResults+ searchResults=dataset.searchResults,)now = datetime.now(timezone.utc)@@ -345,9 +368,9 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchstats_manager.record_dataset(hashed_dataset)return {- 'status': 'ok',- 'public_user_id': user_id_hash,- 'url': f'{PUBLIC_URL_PREFIX}{filename}',+ "status": "ok",+ "public_user_id": user_id_hash,+ "url": f"{PUBLIC_URL_PREFIX}{filename}",}@@ -364,7 +387,7 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiodef _get_user_id_hash(batch: Union[Batch, NewBatchRequest, DatasetRequest]):- return hashlib.sha3_256(batch.user_id.encode('utf8')).hexdigest()+ return hashlib.sha3_256(batch.user_id.encode("utf8")).hexdigest()def check_public_user_id(public_user_id):@@ -375,25 +398,26 @@ def check_public_user_id(public_user_id):def get_batch_url(batch_id, date_str, public_user_id):check_date_str(date_str)check_public_user_id(public_user_id)- url = f'{PUBLIC_URL_PREFIX}1/{VERSION}/{date_str}/1/{public_user_id}/{batch_id}{FILE_NAME_SUFFIX}'+ url = f"{PUBLIC_URL_PREFIX}1/{VERSION}/{date_str}/1/{public_user_id}/{batch_id}{FILE_NAME_SUFFIX}"return urldef get_batch_id_from_file_name(file_name: str):assert file_name.endswith(FILE_NAME_SUFFIX)- return file_name[:-len(FILE_NAME_SUFFIX)]+ return file_name[: -len(FILE_NAME_SUFFIX)]def get_batch_ids_for_prefix(prefix):filenames = get_batches_for_prefix(prefix)- filename_endings = sorted(filename.rsplit('/', 1)[1] for filename in filenames)- results = {'batch_ids': [get_batch_id_from_file_name(name) for name in filename_endings]}+ filename_endings = sorted(filename.rsplit("/", 1)[1] for filename in filenames)+ results = {"batch_ids": [get_batch_id_from_file_name(name) for name in filename_endings]}return resultsdef get_batches_for_prefix(prefix):- s3 = boto3.resource('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,- aws_secret_access_key=APPLICATION_KEY)+ s3 = boto3.resource(+ "s3", endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID, aws_secret_access_key=APPLICATION_KEY+ )bucket = s3.Bucket(BUCKET_NAME)items = bucket.objects.filter(Prefix=prefix)filenames = [item.key for item in items]@@ -402,23 +426,22 @@ def get_batches_for_prefix(prefix):def check_date_str(date_str):if not DATE_REGEX.match(date_str):- raise HttpError(400, f"Incorrect date format, should be YYYY-MM-DD")+ raise HttpError(400, "Incorrect date format, should be YYYY-MM-DD")def get_subfolders(prefix):- client = boto3.client('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,- aws_secret_access_key=APPLICATION_KEY)- items = client.list_objects(Bucket=BUCKET_NAME,- Prefix=prefix,- Delimiter='/')- item_keys = [item['Prefix'][len(prefix):].strip('/') for item in items['CommonPrefixes']]+ client = boto3.client(+ "s3", endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID, aws_secret_access_key=APPLICATION_KEY+ )+ items = client.list_objects(Bucket=BUCKET_NAME, Prefix=prefix, Delimiter="/")+ item_keys = [item["Prefix"][len(prefix) :].strip("/") for item in items["CommonPrefixes"]]return item_keysdef get_batches_for_date(date_str):check_date_str(date_str)- prefix = f'1/{VERSION}/{date_str}/1/'- cache_filename = prefix + 'batches.json.gz'+ prefix = f"1/{VERSION}/{date_str}/1/"+ cache_filename = prefix + "batches.json.gz"cache_url = PUBLIC_URL_PREFIX + cache_filenametry:cached_batches = json.loads(gzip.decompress(requests.get(cache_url).content))@@ -428,10 +451,10 @@ def get_batches_for_date(date_str):passbatches = get_batches_for_prefix(prefix)- result = {'batch_urls': [f'{PUBLIC_URL_PREFIX}{batch}' for batch in sorted(batches)]}+ result = {"batch_urls": [f"{PUBLIC_URL_PREFIX}{batch}" for batch in sorted(batches)]}if date_str != str(date.today()):# Don't cache data from today since it may change- data = gzip.compress(json.dumps(result).encode('utf8'))+ data = gzip.compress(json.dumps(result).encode("utf8"))upload(data, cache_filename)print(f"Cached batches for {date_str} in {PUBLIC_URL_PREFIX}{cache_filename}")print(f"Returning {len(result['batch_urls'])} batches for {date_str}")diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex 7c40251..45fa655 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -1,6 +1,6 @@from typing import Optional-from ninja import Schema, Field+from ninja import Field, Schemaclass Link(Schema):@@ -131,8 +131,7 @@ class NewBatchRequest(Schema):user_id: str = Field(description=(- "The crawler's private user ID (a 64-character hex string). "- "Used to assign URLs to this specific crawler."+ "The crawler's private user ID (a 64-character hex string). Used to assign URLs to this specific crawler."),example="a" * 64,)diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.pyindex 7b08b2d..83d1235 100644--- a/mwmbl/crawler/domains.py+++ b/mwmbl/crawler/domains.py@@ -2,6 +2,7 @@Record which source domains link to destination domains. Each source domain is a bloom filter containing sets ofdestination domains."""+from itertools import islicefrom logging import getLogger@@ -10,17 +11,16 @@ from pybloomfilter import BloomFilterfrom mwmbl.hn_top_domains_filtered import DOMAINS-logger = getLogger(__name__)DOMAIN_GROUPS = [- ('github.com', 10),- ('en.wikipedia.org', 10),- ('news.ycombinator.com', 10),- ('lemmy.ml', 2),- ('mastodon.social', 2),- ('top', 5),- ('other', 1),+ ("github.com", 10),+ ("en.wikipedia.org", 10),+ ("news.ycombinator.com", 10),+ ("lemmy.ml", 2),+ ("mastodon.social", 2),+ ("top", 5),+ ("other", 1),]TOP_DOMAINS = set(islice(DOMAINS, 4000))@@ -31,8 +31,12 @@ def get_bloom_filter(domain_group: str) -> BloomFilter:try:bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group))except FileNotFoundError:- bloom_filter = BloomFilter(settings.NUM_DOMAINS_IN_BLOOM_FILTER, 1e-6,- settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group), perm=0o666)+ bloom_filter = BloomFilter(+ settings.NUM_DOMAINS_IN_BLOOM_FILTER,+ 1e-6,+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group),+ perm=0o666,+ )return bloom_filter@@ -52,9 +56,9 @@ class DomainLinkDatabase:if source in DOMAIN_GROUPS:domain_group = sourceelif source in TOP_DOMAINS:- domain_group = 'top'+ domain_group = "top"elif source in OTHER_DOMAINS:- domain_group = 'other'+ domain_group = "other"else:# This is a URL that we don't care aboutreturndiff --git a/mwmbl/crawler/env_vars.py b/mwmbl/crawler/env_vars.pyindex 26320fc..556e49e 100644--- a/mwmbl/crawler/env_vars.py+++ b/mwmbl/crawler/env_vars.py@@ -4,7 +4,6 @@ Environment variables configuration for the crawler module.import os-# Crawler worker configurationCRAWLER_WORKERS = int(os.environ.get("CRAWLER_WORKERS", "2"))CRAWL_THREADS = int(os.environ.get("CRAWL_THREADS", "20"))diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 7083d17..7b1bf46 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -5,34 +5,42 @@ from logging import getLoggerfrom multiprocessing.pool import ThreadPoolfrom ssl import SSLCertVerificationErrorfrom typing import List, Optional-from urllib.parse import urlparse, urlunsplit, urljoin+from urllib.parse import urljoin, urlparse, urlunsplitfrom urllib.robotparser import RobotFileParserimport requests-from mwmbl.justext import core, utilsfrom redis import Redisfrom requests import ReadTimeout-from urllib3.exceptions import NewConnectionError, MaxRetryError+from urllib3.exceptions import MaxRetryError, NewConnectionErrorfrom mwmbl.crawler.env_vars import MWMBL_CONTACT_INFOfrom mwmbl.crawler.ssrf import UnsafeURLError, validate_url+from mwmbl.justext import core, utilsfrom mwmbl.justext.core import html_to_domfrom mwmbl.justext.paragraph import Paragraph-# UnsafeURLError subclasses ValueError, so it is already covered here, but list# it explicitly for clarity: an SSRF-blocked URL is skipped like any other bad URL.-ALLOWED_EXCEPTIONS = (ValueError, UnsafeURLError, ConnectionError, ReadTimeout, TimeoutError,- OSError, NewConnectionError, MaxRetryError, SSLCertVerificationError)--POST_BATCH_URL = '/api/v1/crawler/batches/'-POST_NEW_BATCH_URL = '/api/v1/crawler/batches/new'+ALLOWED_EXCEPTIONS = (+ ValueError,+ UnsafeURLError,+ ConnectionError,+ ReadTimeout,+ TimeoutError,+ OSError,+ NewConnectionError,+ MaxRetryError,+ SSLCertVerificationError,+)++POST_BATCH_URL = "/api/v1/crawler/batches/"+POST_NEW_BATCH_URL = "/api/v1/crawler/batches/new"TIMEOUT_SECONDS = 3MAX_REDIRECTS = 5-MAX_FETCH_SIZE = 1024*1024+MAX_FETCH_SIZE = 1024 * 1024MAX_URL_LENGTH = 150-BAD_URL_REGEX = re.compile(r'\/\/localhost\b|\.jpg$|\.png$|\.js$|\.gz$|\.zip$|\.pdf$|\.bz2$|\.ipynb$|\.py$')+BAD_URL_REGEX = re.compile(r"\/\/localhost\b|\.jpg$|\.png$|\.js$|\.gz$|\.zip$|\.pdf$|\.bz2$|\.ipynb$|\.py$")MAX_NEW_LINKS = 50MAX_EXTRA_LINKS = 50NUM_TITLE_CHARS = 65@@ -41,8 +49,8 @@ NUM_EXTRACT_CHARS = 155# or whose text is mostly link anchors (nav/breadcrumbs rather than prose).MIN_FALLBACK_PARAGRAPH_CHARS = 20MAX_FALLBACK_PARAGRAPH_LINK_DENSITY = 0.5-DEFAULT_ENCODING = 'utf8'-DEFAULT_ENC_ERRORS = 'replace'+DEFAULT_ENCODING = "utf8"+DEFAULT_ENC_ERRORS = "replace"MAX_SITE_URLS = 100CRAWLER_VERSION: str = "0.2.0"USER_AGENT = f"mwmbl/{CRAWLER_VERSION} (https://github.com/mwmbl/mwmbl/ contact {MWMBL_CONTACT_INFO})"@@ -70,8 +78,7 @@ def fetch(url):headers = {"User-Agent": USER_AGENT}for _ in range(MAX_REDIRECTS + 1):validate_url(url)- r = requests.get(url, stream=True, timeout=TIMEOUT_SECONDS,- headers=headers, allow_redirects=False)+ r = requests.get(url, stream=True, timeout=TIMEOUT_SECONDS, headers=headers, allow_redirects=False)if r.is_redirect and r.next is not None:r.close()@@ -84,7 +91,7 @@ def fetch(url):content = b""for chunk in r.iter_content(1024):if time.time() - start > TIMEOUT_SECONDS:- raise ValueError('Timeout reached')+ raise ValueError("Timeout reached")content += chunk@@ -106,7 +113,7 @@ def robots_allowed(url: str, redis: Redis) -> bool:return Falsedomain = parsed_url.netloc- robots_url = urlunsplit((parsed_url.scheme, parsed_url.netloc, 'robots.txt', '', ''))+ robots_url = urlunsplit((parsed_url.scheme, parsed_url.netloc, "robots.txt", "", ""))cached_content = _get_robots_from_cache(redis, domain)if cached_content is not None:@@ -130,7 +137,7 @@ def robots_allowed(url: str, redis: Redis) -> bool:return Truedecoded = None- for encoding in ['utf-8', 'iso-8859-1']:+ for encoding in ["utf-8", "iso-8859-1"]:try:decoded = content.decode(encoding).splitlines()break@@ -141,13 +148,13 @@ def robots_allowed(url: str, redis: Redis) -> bool:logger.info(f"Unable to decode robots file {robots_url}")_cache_robots_content(redis, domain, [], error=True)return True-+parse_robots = RobotFileParser()parse_robots.parse(decoded)allowed = parse_robots.can_fetch(USER_AGENT, url)-+_cache_robots_content(redis, domain, decoded, error=False)-+logger.debug(f"Robots allowed for {url}: {allowed}")return allowed@@ -157,13 +164,13 @@ def _get_robots_from_cache(redis: Redis, domain: str) -> Optional[List[str]]:cached = redis.get(cache_key)if cached is None:return None-+try:data = json.loads(cached)- if time.time() > data['expires_at']:+ if time.time() > data["expires_at"]:redis.delete(cache_key)return None- return data['content']+ return data["content"]except (json.JSONDecodeError, KeyError) as e:logger.warning(f"Invalid cache entry for {cache_key}: {e}, removing")redis.delete(cache_key)@@ -174,13 +181,9 @@ def _cache_robots_content(redis: Redis, domain: str, content: List[str], error:cache_key = f"robots:{domain}"now = time.time()ttl = ROBOTS_CACHE_ERROR_TTL_SECONDS if error else ROBOTS_CACHE_TTL_SECONDS-- cache_data = {- 'content': content,- 'cached_at': int(now),- 'expires_at': int(now + ttl)- }-++ cache_data = {"content": content, "cached_at": int(now), "expires_at": int(now + ttl)}+redis.setex(cache_key, ttl, json.dumps(cache_data))logger.debug(f"Cached robots.txt for {domain}, expires in {ttl}s")@@ -228,7 +231,7 @@ def get_new_links(paragraphs: list[Paragraph], current_url):if resolved is None:logger.debug(f"Bad URL: {link}")continue- if paragraph.class_type == 'good':+ if paragraph.class_type == "good":if len(new_links) < MAX_NEW_LINKS:new_links.add(resolved)else:@@ -253,7 +256,7 @@ def extract_from_html_text(html_text: str) -> str:continueextract += " " + paragraph.text.strip()if len(extract) > NUM_EXTRACT_CHARS:- extract = extract[:NUM_EXTRACT_CHARS - 1] + "…"+ extract = extract[: NUM_EXTRACT_CHARS - 1] + "…"breakreturn extract.strip()@@ -304,7 +307,7 @@ def get_first_paragraph(dom) -> str:def _truncate(text: str, limit: int) -> str:- return text[:limit - 1] + '…' if len(text) > limit else text+ return text[: limit - 1] + "…" if len(text) > limit else textdef crawl_url(url, redis: Redis):@@ -313,14 +316,14 @@ def crawl_url(url, redis: Redis):allowed = robots_allowed(url, redis)if not allowed:return {- 'url': url,- 'status': None,- 'timestamp': js_timestamp,- 'content': None,- 'error': {- 'name': 'RobotsDenied',- 'message': 'Robots do not allow this URL',- }+ "url": url,+ "status": None,+ "timestamp": js_timestamp,+ "content": None,+ "error": {+ "name": "RobotsDenied",+ "message": "Robots do not allow this URL",+ },}try:@@ -328,47 +331,47 @@ def crawl_url(url, redis: Redis):except ALLOWED_EXCEPTIONS as e:logger.debug(f"Exception crawling URl {url}: {e}")return {- 'url': url,- 'status': None,- 'timestamp': js_timestamp,- 'content': None,- 'error': {- 'name': 'AbortError',- 'message': str(e),- }+ "url": url,+ "status": None,+ "timestamp": js_timestamp,+ "content": None,+ "error": {+ "name": "AbortError",+ "message": str(e),+ },}if len(content) == 0:return {- 'url': url,- 'resolved_url': resolved_url,- 'status': status_code,- 'timestamp': js_timestamp,- 'content': None,- 'error': {- 'name': 'NoResponseText',- 'message': 'No response found',- }+ "url": url,+ "resolved_url": resolved_url,+ "status": status_code,+ "timestamp": js_timestamp,+ "content": None,+ "error": {+ "name": "NoResponseText",+ "message": "No response found",+ },}- content = re.sub(rb'[\x00-\x08\x0b\x0c\x0e-\x1f]', b'', content)+ content = re.sub(rb"[\x00-\x08\x0b\x0c\x0e-\x1f]", b"", content)try:dom = html_to_dom(content, DEFAULT_ENCODING, None, DEFAULT_ENC_ERRORS)except Exception as e:logger.exception(f"Error parsing dom: {url}")return {- 'url': url,- 'resolved_url': resolved_url,- 'status': status_code,- 'timestamp': js_timestamp,- 'content': None,- 'error': {- 'name': e.__class__.__name__,- 'message': str(e),- }+ "url": url,+ "resolved_url": resolved_url,+ "status": status_code,+ "timestamp": js_timestamp,+ "content": None,+ "error": {+ "name": e.__class__.__name__,+ "message": str(e),+ },}-+title_element = dom.xpath("//title")title = ""if len(title_element) > 0:@@ -377,7 +380,7 @@ def crawl_url(url, redis: Redis):title = title_text.strip()if len(title) > NUM_TITLE_CHARS:- title = title[:NUM_TITLE_CHARS - 1] + '…'+ title = title[: NUM_TITLE_CHARS - 1] + "…"try:paragraphs = core.justext_from_dom(dom, utils.get_stoplist("English"))@@ -385,15 +388,15 @@ def crawl_url(url, redis: Redis):bad_bytes = sorted({b for b in content if b < 0x20 and b not in (0x09, 0x0A, 0x0D)})logger.exception("Error parsing paragraphs - offending control bytes: %s", bad_bytes)return {- 'url': url,- 'resolved_url': resolved_url,- 'status': status_code,- 'timestamp': js_timestamp,- 'content': None,- 'error': {- 'name': e.__class__.__name__,- 'message': str(e),- }+ "url": url,+ "resolved_url": resolved_url,+ "status": status_code,+ "timestamp": js_timestamp,+ "content": None,+ "error": {+ "name": e.__class__.__name__,+ "message": str(e),+ },}new_links, extra_links = get_new_links(paragraphs, url)@@ -407,13 +410,13 @@ def crawl_url(url, redis: Redis):logger.debug(f"Got new links {new_links}")logger.debug(f"Got extra links {extra_links}")- extract = ''+ extract = ""for paragraph in paragraphs:- if paragraph.class_type != 'good':+ if paragraph.class_type != "good":continue- extract += ' ' + paragraph.text.strip()+ extract += " " + paragraph.text.strip()if len(extract) > NUM_EXTRACT_CHARS:- extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'+ extract = extract[: NUM_EXTRACT_CHARS - 1] + "…"break# justext finds no body content for JS-first pages (e.g. Discord, SPAs) and@@ -430,17 +433,17 @@ def crawl_url(url, redis: Redis):extract = _truncate(fallback, NUM_EXTRACT_CHARS)return {- 'url': url,- 'resolved_url': resolved_url,- 'status': status_code,- 'timestamp': js_timestamp,- 'content': {- 'title': title,- 'extract': extract,- 'links': sorted(new_links),- 'extra_links': sorted(extra_links),- },- 'error': None+ "url": url,+ "resolved_url": resolved_url,+ "status": status_code,+ "timestamp": js_timestamp,+ "content": {+ "title": title,+ "extract": extract,+ "links": sorted(new_links),+ "extra_links": sorted(extra_links),+ },+ "error": None,}@@ -448,6 +451,3 @@ def crawl_batch(batch, num_threads, redis: Redis):with ThreadPool(num_threads) as pool:result = pool.map(lambda url: crawl_url(url, redis), batch)return result---diff --git a/mwmbl/crawler/ssrf.py b/mwmbl/crawler/ssrf.pyindex c418d71..1b88015 100644--- a/mwmbl/crawler/ssrf.py+++ b/mwmbl/crawler/ssrf.py@@ -12,6 +12,7 @@ transport adapters and is left as a future hardening step. Callers that followHTTP redirects must re-validate each hop, since a public URL can 3xx to aninternal one."""+import ipaddressimport socketfrom urllib.parse import urlparse@@ -29,12 +30,7 @@ def _ip_is_safe(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:ip = ip.ipv4_mappedreturn not (- ip.is_private- or ip.is_loopback- or ip.is_link_local- or ip.is_reserved- or ip.is_multicast- or ip.is_unspecified+ ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified)diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 92c6e04..8dd6df7 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -3,18 +3,15 @@ from datetime import datetime, timedeltafrom glob import globfrom itertools import islicefrom logging import getLogger-from pathlib import Pathfrom urllib.parse import urlparseimport django-from django.conf import settingsfrom pydantic import BaseModelfrom redis import Redisfrom mwmbl.count_urls import get_counts, get_domain_result_countfrom mwmbl.crawler.batch import HashedBatch, Resultsfrom mwmbl.crawler.urls import URLDatabase-from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.update_urls import get_datetime_from_timestamplogger = getLogger(__name__)@@ -288,10 +285,10 @@ class StatsManager:def record_dataset(self, hashed_dataset) -> None:"""Record dataset statistics from a dataset submission."""from datetime import datetime-+# Parse the date from the datasettry:- dataset_date = datetime.strptime(hashed_dataset.date, '%Y-%m-%d').date()+ dataset_date = datetime.strptime(hashed_dataset.date, "%Y-%m-%d").date()except ValueError:# If date parsing fails, use current datedataset_date = datetime.utcnow().date()@@ -320,14 +317,15 @@ def get_test_batches():yield HashedBatch.parse_raw(gzip_file.read())-if __name__ == '__main__':+if __name__ == "__main__":django.setup()- redis = Redis(host='localhost', port=6379, decode_responses=True)+ redis = Redis(host="localhost", port=6379, decode_responses=True)stats = StatsManager(redis)batches = get_test_batches()start = datetime.now()processed = 0import logging+logging.basicConfig(level=logging.INFO)for batch in islice(batches, 10000):if len(batch.items) <= 2:@@ -337,4 +335,4 @@ if __name__ == '__main__':total_time = (datetime.now() - start).total_seconds()print("Processed", processed)print("Total time", total_time)- print("Time per batch", total_time/processed)+ print("Time per batch", total_time / processed)diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex 5638d22..4ef6b3f 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -1,19 +1,16 @@"""Database storing info on URLs"""+from dataclasses import dataclassfrom datetime import datetime, timedelta, timezonefrom enum import Enumfrom logging import getLogger-from pathlib import Pathfrom typing import Optionalfrom django.conf import settings-from psycopg2.extras import execute_valuesfrom pybloomfilter import BloomFilter-from mwmbl.utils import batch-# URL database configuration constants# Client has one hour to crawl a URL that has been assigned to them, or it will be reassignedREASSIGN_MIN_HOURS = 5@@ -30,17 +27,24 @@ class URLStatus(Enum):"""URL state update is idempotent and can only progress forwards."""- NEW = 0 # One user has identified this URL- QUEUED = 5 # The URL has been queued for crawling- ASSIGNED = 10 # The crawler has given the URL to a user to crawl- ERROR_TIMEOUT = 20 # Timeout while retrieving- ERROR_404 = 30 # 404 response- ERROR_OTHER = 40 # Some other error++ NEW = 0 # One user has identified this URL+ QUEUED = 5 # The URL has been queued for crawling+ ASSIGNED = 10 # The crawler has given the URL to a user to crawl+ ERROR_TIMEOUT = 20 # Timeout while retrieving+ ERROR_404 = 30 # 404 response+ ERROR_OTHER = 40 # Some other errorERROR_ROBOTS_DENIED = 50 # Robots disallow this page- CRAWLED = 100 # At least one user has crawled the URL+ CRAWLED = 100 # At least one user has crawled the URL-CRAWLED_STATUSES = {URLStatus.CRAWLED, URLStatus.ERROR_TIMEOUT, URLStatus.ERROR_404, URLStatus.ERROR_OTHER, URLStatus.ERROR_ROBOTS_DENIED}+CRAWLED_STATUSES = {+ URLStatus.CRAWLED,+ URLStatus.ERROR_TIMEOUT,+ URLStatus.ERROR_404,+ URLStatus.ERROR_OTHER,+ URLStatus.ERROR_ROBOTS_DENIED,+}@dataclassdiff --git a/mwmbl/curated_domains.py b/mwmbl/curated_domains.pyindex a28593b..14b08fb 100644--- a/mwmbl/curated_domains.py+++ b/mwmbl/curated_domains.py@@ -10,6 +10,7 @@ This lives outside search_setup because search_setup imports blacklist_snapshot,blacklist code needs to import this. The curated set is only ever read where a blacklist is*constructed* (build_snapshot, CombinedBlacklistProvider), never per query."""+from django.conf import settingsfrom django.core.cache import cache@@ -27,6 +28,6 @@ def get_curated_domains() -> set[str]:curated_domains = cache.get(CURATED_DOMAINS_CACHE_KEY)if curated_domains is None:- curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))+ curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list("name", flat=True))cache.set(CURATED_DOMAINS_CACHE_KEY, curated_domains, CURATED_DOMAINS_CACHE_TIMEOUT)return curated_domainsdiff --git a/mwmbl/database.py b/mwmbl/database.pyindex bcf2684..8514dfd 100644--- a/mwmbl/database.py+++ b/mwmbl/database.py@@ -1,6 +1,5 @@-from psycopg2 import connect-from django.conf import settings+from psycopg2 import connectdef _get_database_url() -> str:@@ -8,16 +7,16 @@ def _get_database_url() -> str:Derive a psycopg2 DSN from Django's DATABASES['default'] setting.Supports both a pre-built URL (via dj_database_url) and a dict of individual keys."""- db = settings.DATABASES['default']+ db = settings.DATABASES["default"]# dj_database_url stores the original URL in the 'URL' key when available- if 'URL' in db:- return db['URL']+ if "URL" in db:+ return db["URL"]# Fall back to constructing a DSN from individual keys- host = db.get('HOST', 'localhost')- port = db.get('PORT', '5432')- name = db.get('NAME', '')- user = db.get('USER', '')- password = db.get('PASSWORD', '')+ host = db.get("HOST", "localhost")+ port = db.get("PORT", "5432")+ name = db.get("NAME", "")+ user = db.get("USER", "")+ password = db.get("PASSWORD", "")return f"postgresql://{user}:{password}@{host}:{port}/{name}"diff --git a/mwmbl/evaluation/api.py b/mwmbl/evaluation/api.pyindex 3fe42f5..0e1837a 100644--- a/mwmbl/evaluation/api.py+++ b/mwmbl/evaluation/api.py@@ -1,12 +1,14 @@-from ninja import File, UploadedFile, Router-from ninja_jwt.authentication import JWTAuth-from django.utils import timezonefrom typing import List+from django.utils import timezone+from ninja import File, Router, UploadedFile+from ninja_jwt.authentication import JWTAuth++from mwmbl.exceptions import InvalidRequestfrom mwmbl.models import WasmEvaluationJob-from .schemas import WasmSubmissionResponse, EvaluationJobResponse, EvaluationResultsResponse++from .schemas import EvaluationJobResponse, EvaluationResultsResponse, WasmSubmissionResponsefrom .wasm_validator import WasmValidator-from mwmbl.exceptions import InvalidRequestrouter = Router(tags=["Evaluation"])@@ -19,7 +21,7 @@ def check_email_verified(request):@router.post(- '/submit',+ "/submit",response=WasmSubmissionResponse,auth=JWTAuth(),summary="Submit a WASM ranking function",@@ -36,31 +38,25 @@ def submit_wasm(request, file: UploadedFile = File(...)):if file.size > 10 * 1024 * 1024: # 10MB limitraise InvalidRequest("File too large. Maximum size is 10MB.")- if not file.name.endswith('.wasm'):+ if not file.name.endswith(".wasm"):raise InvalidRequest("File must have .wasm extension.")wasm_bytes = file.read()validation_result = WasmValidator.validate_wasm_file(wasm_bytes)- if not validation_result['valid']:+ if not validation_result["valid"]:raise InvalidRequest(f"WASM validation failed: {validation_result['error']}")- job = WasmEvaluationJob.objects.create(- user=request.user,- wasm_file=wasm_bytes,- status='VALIDATED'- )+ job = WasmEvaluationJob.objects.create(user=request.user, wasm_file=wasm_bytes, status="VALIDATED")return WasmSubmissionResponse(- job_id=job.id,- status=job.status,- message="WASM file validated and job created successfully"+ job_id=job.id, status=job.status, message="WASM file validated and job created successfully")@router.post(- '/run/{job_id}',+ "/run/{job_id}",response=EvaluationResultsResponse,auth=JWTAuth(),summary="Run an evaluation job",@@ -79,44 +75,36 @@ def run_evaluation(request, job_id: int):except WasmEvaluationJob.DoesNotExist:raise InvalidRequest("Evaluation job not found", status=404)- if job.status not in ['VALIDATED', 'FAILED']:+ if job.status not in ["VALIDATED", "FAILED"]:raise InvalidRequest(f"Job cannot be run. Current status: {job.status}")- job.status = 'RUNNING'+ job.status = "RUNNING"job.save()try:mock_results = {"ndcg_score": 0.75,"queries_evaluated": 10,- "message": "Mock evaluation completed successfully"+ "message": "Mock evaluation completed successfully",}- job.status = 'COMPLETED'+ job.status = "COMPLETED"job.results = mock_resultsjob.completed_at = timezone.now()job.save()- return EvaluationResultsResponse(- job_id=job.id,- status=job.status,- results=job.results- )+ return EvaluationResultsResponse(job_id=job.id, status=job.status, results=job.results)except Exception as e:- job.status = 'FAILED'+ job.status = "FAILED"job.error_message = str(e)job.save()- return EvaluationResultsResponse(- job_id=job.id,- status=job.status,- error_message=job.error_message- )+ return EvaluationResultsResponse(job_id=job.id, status=job.status, error_message=job.error_message)@router.get(- '/results/{job_id}',+ "/results/{job_id}",response=EvaluationResultsResponse,auth=JWTAuth(),summary="Get evaluation results",@@ -136,15 +124,12 @@ def get_evaluation_results(request, job_id: int):raise InvalidRequest("Evaluation job not found", status=404)return EvaluationResultsResponse(- job_id=job.id,- status=job.status,- results=job.results,- error_message=job.error_message+ job_id=job.id, status=job.status, results=job.results, error_message=job.error_message)@router.get(- '/jobs',+ "/jobs",response=List[EvaluationJobResponse],auth=JWTAuth(),summary="List evaluation jobs",@@ -157,7 +142,7 @@ def get_evaluation_results(request, job_id: int):def list_evaluation_jobs(request):check_email_verified(request)- jobs = WasmEvaluationJob.objects.filter(user=request.user).order_by('-created_at')+ jobs = WasmEvaluationJob.objects.filter(user=request.user).order_by("-created_at")return [EvaluationJobResponse(@@ -166,7 +151,7 @@ def list_evaluation_jobs(request):created_at=job.created_at,completed_at=job.completed_at,results=job.results,- error_message=job.error_message+ error_message=job.error_message,)for job in jobs]diff --git a/mwmbl/evaluation/schemas.py b/mwmbl/evaluation/schemas.pyindex 6d35bf5..2cfbe5b 100644--- a/mwmbl/evaluation/schemas.py+++ b/mwmbl/evaluation/schemas.py@@ -1,6 +1,7 @@-from ninja import Schema, Field-from typing import Optionalfrom datetime import datetime+from typing import Optional++from ninja import Field, Schemaclass WasmSubmissionResponse(Schema):@@ -28,9 +29,7 @@ class EvaluationJobResponse(Schema):example=42,)status: str = Field(- description=(- "Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."- ),+ description=("Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."),example="COMPLETED",)created_at: datetime = Field(@@ -65,9 +64,7 @@ class EvaluationResultsResponse(Schema):example=42,)status: str = Field(- description=(- "Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."- ),+ description=("Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."),example="COMPLETED",)results: Optional[dict] = Field(diff --git a/mwmbl/evaluation/wasm_validator.py b/mwmbl/evaluation/wasm_validator.pyindex bdf7dfb..b6afd9d 100644--- a/mwmbl/evaluation/wasm_validator.py+++ b/mwmbl/evaluation/wasm_validator.py@@ -1,5 +1,6 @@+from typing import Any, Dict+import wasmtime-from typing import Dict, Anyclass WasmValidator:@@ -10,36 +11,29 @@ class WasmValidator:# Create WASM engine and moduleengine = wasmtime.Engine()module = wasmtime.Module(engine, wasm_bytes)-+# Get exports to check for required functionsstore = wasmtime.Store(engine)instance = wasmtime.Instance(store, module, [])exports = instance.exports(store)-+# Check for basic required exports (simplified validation)- required_exports = ['memory'] # At minimum, we need memory export-+ required_exports = ["memory"] # At minimum, we need memory export+missing_exports = []for export_name in required_exports:if export_name not in exports:missing_exports.append(export_name)-+if missing_exports:return {"valid": False,"error": f"Missing required exports: {missing_exports}",- "message": "WASM module validation failed"+ "message": "WASM module validation failed",}-+# Basic functionality test - just check if we can create the module- return {- "valid": True,- "message": "WASM module validation successful"- }-+ return {"valid": True, "message": "WASM module validation successful"}+except Exception as e:- return {- "valid": False,- "error": str(e),- "message": "WASM module validation failed"- }+ return {"valid": False, "error": str(e), "message": "WASM module validation failed"}diff --git a/mwmbl/format.py b/mwmbl/format.pyindex 9fcfcce..ce34372 100644--- a/mwmbl/format.py+++ b/mwmbl/format.py@@ -1,45 +1,121 @@import refrom mwmbl.tinysearchengine.indexer import DocumentState-from mwmbl.tokenizer import tokenize, clean_unicode-+from mwmbl.tokenizer import clean_unicode, tokenizeDOCUMENT_SOURCES = {- DocumentState.FROM_GOOGLE: 'google',- DocumentState.FROM_USER: 'user',- DocumentState.FROM_WIKI: 'wikipedia',- DocumentState.ORGANIC_APPROVED: 'mwmbl',- DocumentState.FROM_GOOGLE_APPROVED: 'google',- DocumentState.FROM_USER_APPROVED: 'user',- DocumentState.FROM_WIKI_APPROVED: 'wikipedia',+ DocumentState.FROM_GOOGLE: "google",+ DocumentState.FROM_USER: "user",+ DocumentState.FROM_WIKI: "wikipedia",+ DocumentState.ORGANIC_APPROVED: "mwmbl",+ DocumentState.FROM_GOOGLE_APPROVED: "google",+ DocumentState.FROM_USER_APPROVED: "user",+ DocumentState.FROM_WIKI_APPROVED: "wikipedia",}HIGHLIGHT_STOPWORDS = {# Articles & Determiners- "a", "an", "the", "this", "that", "these", "those", "each", "every", "some", "any",+ "a",+ "an",+ "the",+ "this",+ "that",+ "these",+ "those",+ "each",+ "every",+ "some",+ "any",# Prepositions- "to", "in", "on", "at", "by", "for", "with", "about", "against", "between",- "into", "through", "during", "before", "after", "above", "below", "from",- "up", "down", "of", "off", "over", "under",+ "to",+ "in",+ "on",+ "at",+ "by",+ "for",+ "with",+ "about",+ "against",+ "between",+ "into",+ "through",+ "during",+ "before",+ "after",+ "above",+ "below",+ "from",+ "up",+ "down",+ "of",+ "off",+ "over",+ "under",# Conjunctions- "and", "but", "or", "nor", "for", "yet", "so", "although", "because", "since", "unless",+ "and",+ "but",+ "or",+ "nor",+ "for",+ "yet",+ "so",+ "although",+ "because",+ "since",+ "unless",# Common Verbs & Pronouns- "is", "am", "are", "was", "were", "be", "been", "being", "have", "has", "had",- "do", "does", "did", "i", "me", "my", "you", "your", "he", "him", "his",- "she", "her", "it", "its", "we", "us", "our", "they", "them", "their",+ "is",+ "am",+ "are",+ "was",+ "were",+ "be",+ "been",+ "being",+ "have",+ "has",+ "had",+ "do",+ "does",+ "did",+ "i",+ "me",+ "my",+ "you",+ "your",+ "he",+ "him",+ "his",+ "she",+ "her",+ "it",+ "its",+ "we",+ "us",+ "our",+ "they",+ "them",+ "their",# Interrogatives (usually noise in technical queries)- "how", "what", "which", "who", "whom", "where", "when", "why"+ "how",+ "what",+ "which",+ "who",+ "whom",+ "where",+ "when",+ "why",}def get_document_source(state: DocumentState):- return DOCUMENT_SOURCES.get(state, 'mwmbl')+ return DOCUMENT_SOURCES.get(state, "mwmbl")def format_result_with_pattern(pattern, result):formatted_result = {}- for content_type, content_raw in [('title', result.title), ('extract', result.extract)]:+ for content_type, content_raw in [("title", result.title), ("extract", result.extract)]:content = clean_unicode(content_raw) if content_raw else ""matches = re.finditer(pattern, content, re.IGNORECASE)all_spans = [0] + sum((list(m.span()) for m in matches), []) + [len(content)]@@ -49,24 +125,25 @@ def format_result_with_pattern(pattern, result):start = all_spans[i]end = all_spans[i + 1]if end - start > 0:- content_result.append({'value': content[start:end], 'is_bold': is_bold})+ content_result.append({"value": content[start:end], "is_bold": is_bold})formatted_result[content_type] = content_result- formatted_result['url'] = result.url- formatted_result['source'] = get_document_source(result.state)+ formatted_result["url"] = result.url+ formatted_result["source"] = get_document_source(result.state)return formatted_resultdef get_query_regex(terms, is_complete: bool, use_word_boundaries: bool):if not terms:- return ''+ return ""- word_sep = r'\b' if use_word_boundaries else ''+ word_sep = r"\b" if use_word_boundaries else ""if is_complete:- term_patterns = [rf'{word_sep}{re.escape(term)}{word_sep}' for term in terms]+ term_patterns = [rf"{word_sep}{re.escape(term)}{word_sep}" for term in terms]else:- term_patterns = [rf'{word_sep}{re.escape(term)}{word_sep}' for term in terms[:-1]] + [- rf'{word_sep}{re.escape(terms[-1])}']- pattern = '|'.join(term_patterns)+ term_patterns = [rf"{word_sep}{re.escape(term)}{word_sep}" for term in terms[:-1]] + [+ rf"{word_sep}{re.escape(terms[-1])}"+ ]+ pattern = "|".join(term_patterns)return pattern@@ -82,16 +159,16 @@ def _extract_highlights(segments: list[dict]) -> list[str]:phrases = []current: list[str] = []for seg in segments:- if seg['is_bold']:- current.append(seg['value'])- elif current and not seg['value'].strip():- current.append(seg['value'])+ if seg["is_bold"]:+ current.append(seg["value"])+ elif current and not seg["value"].strip():+ current.append(seg["value"])else:if current:- phrases.append(''.join(current).strip())+ phrases.append("".join(current).strip())current = []if current:- phrases.append(''.join(current).strip())+ phrases.append("".join(current).strip())unique = set(phrases)return sorted(unique, key=len, reverse=True)@@ -102,14 +179,14 @@ def format_result_v2(result, position: int, query: str) -> dict:filtered_tokens = [t for t in tokens if t not in HIGHLIGHT_STOPWORDS]pattern = get_query_regex(filtered_tokens, True, True)v1 = format_result_with_pattern(pattern, result)- title = ''.join(seg['value'] for seg in v1['title'])- content = ''.join(seg['value'] for seg in v1['extract'])+ title = "".join(seg["value"] for seg in v1["title"])+ content = "".join(seg["value"] for seg in v1["extract"])return {- 'url': result.url,- 'title': title,- 'title_highlights': _extract_highlights(v1['title']),- 'content': content,- 'content_highlights': _extract_highlights(v1['extract']),- 'engine': get_document_source(result.state),- 'score': 1.0 / position,+ "url": result.url,+ "title": title,+ "title_highlights": _extract_highlights(v1["title"]),+ "content": content,+ "content_highlights": _extract_highlights(v1["extract"]),+ "engine": get_document_source(result.state),+ "score": 1.0 / position,}diff --git a/mwmbl/hn_top_domains_filtered.py b/mwmbl/hn_top_domains_filtered.pyindex dc39af7..e657176 100644--- a/mwmbl/hn_top_domains_filtered.py+++ b/mwmbl/hn_top_domains_filtered.py@@ -1,7859 +1,7860 @@-DOMAINS = {'blog.samaltman.com': 0.9906157038365982,-'paulgraham.com': 0.9900792629020208,-'blog.rust-lang.org': 0.9890328852259848,-'blog.ycombinator.com': 0.9883892917770221,-'ciechanow.ski': 0.9880439204029129,-'blog.mozilla.org': 0.9862851554386893,-'stripe.com': 0.9861322763760931,-'danluu.com': 0.9861047192293313,-'www.kalzumeus.com': 0.9850478020093707,-'signal.org': 0.9850307268861681,-'antirez.com': 0.9848258724546234,-'www.mozilla.org': 0.9848038469599045,-'jacquesmattheij.com': 0.9844261150401603,-'www.chris-granger.com': 0.9842666557846631,-'www.apple.com': 0.9836460584625046,-'letsencrypt.org': 0.9835295202763794,-'www.teslamotors.com': 0.9831965330638592,-'hacks.mozilla.org': 0.982862363147374,-'lore.kernel.org': 0.9822733167137851,-'frame.work': 0.9821010345456243,-'www.righto.com': 0.9819349251846876,-'www.troyhunt.com': 0.9814966052145125,-'www.sec.gov': 0.9814725718254595,-'www.tesla.com': 0.9810577172203161,-'keybase.io': 0.9808967126687541,-'www.blender.org': 0.9807600197929631,-'www.spacex.com': 0.9807226511637516,-'www.susanjfowler.com': 0.9805638280990112,-'www.eff.org': 0.9805327186535832,-'blog.golang.org': 0.9800638229086632,-'github.com': 0.9793809676855824,-'www.bloomberg.com': 0.9791747058863844,-'matrix.org': 0.9788189361330681,-'openai.com': 0.9781666403563757,-'www.nytimes.com': 0.9780992933501785,-'www.sublimetext.com': 0.9779900989360693,-'mtlynch.io': 0.9777886722980439,-'www.reuters.com': 0.9776899139875952,-'blog.pinboard.in': 0.9776638411070188,-'whispersystems.org': 0.9776203868069737,-'fabiensanglard.net': 0.9774982104911971,-'drewdevault.com': 0.9774354534940557,-'blog.interviewing.io': 0.9772072688889477,-'github.blog': 0.9770459926321585,-'calpaterson.com': 0.9768950731213937,-'rachelbythebay.com': 0.9768404166128064,-'bellard.org': 0.9767955350707707,-'twitter.com': 0.976728011118412,-'training.kalzumeus.com': 0.9766770275278582,-'www.npr.org': 0.976513349382135,-'jvns.ca': 0.9765007166886884,-'www.daemonology.net': 0.9763913310944178,-'ifixit.org': 0.9762565420103206,-'www.bbc.com': 0.9761524214719092,-'karpathy.github.io': 0.9759182558450648,-'tonsky.me': 0.9759061232371565,-'www.buzzfeednews.com': 0.975834047407925,-'deepmind.com': 0.9757513978763775,-'www.postgresql.org': 0.9756297562580343,-'krebsonsecurity.com': 0.9755023613643372,-'overreacted.io': 0.9754149079228597,-'stratechery.com': 0.9754126620517041,-'brave.com': 0.9752551626581946,-'lwn.net': 0.9751973038824019,-'about.gitlab.com': 0.9751871932295729,-'www.stephendiehl.com': 0.9751552345204195,-'techcrunch.com': 0.9751406961249492,-'raganwald.posterous.com': 0.9750289679638235,-'blog.discordapp.com': 0.9750045998850335,-'bugs.chromium.org': 0.9749443016112311,-'www.marco.org': 0.9749044929470128,-'words.steveklabnik.com': 0.9748959016378708,-'www.backblaze.com': 0.9744710342430156,-'robert.ocallahan.org': 0.9744576196125118,-'blog.cryptographyengineering.com': 0.9744312947315787,-'citizenlab.ca': 0.9742406063151361,-'www.propublica.org': 0.9742125433042081,-'blog.benjojo.co.uk': 0.9741593610657648,-'justine.lol': 0.9740984398105378,-'lists.zx2c4.com': 0.9739788216838905,-'googleblog.blogspot.com': 0.9739234991924623,-'www.supremecourt.gov': 0.9738392227472342,-'idlewords.com': 0.9737410870758311,-'randomascii.wordpress.com': 0.97368784410819,-'jepsen.io': 0.9736187109986643,-'zachholman.com': 0.9735522719786461,-'githubengineering.com': 0.9734850428577687,-'tinyprojects.dev': 0.9733692809444601,-'www.quantamagazine.org': 0.9731098630624817,-'aws.amazon.com': 0.9730261011769831,-'www.washingtonpost.com': 0.9727186917658874,-'www.tbray.org': 0.9727073933211833,-'sqlite.org': 0.9725225769916973,-'sivers.org': 0.9725186799112472,-'inkscape.org': 0.9725022016010344,-'www.deepsouthventures.com': 0.972453385720152,-'blog.openai.com': 0.9723220955611721,-'courses.csail.mit.edu': 0.9722700145319921,-'www.theguardian.com': 0.9722441587612369,-'dcurt.is': 0.9721843131015032,-'blog.atom.io': 0.9720527700753158,-'www.newyorker.com': 0.9720054386788728,-'lists.gnu.org': 0.9719999849041815,-'www.ccc.de': 0.9719484596362211,-'googleprojectzero.blogspot.com': 0.9719076672640107,-'blog.cloudflare.com': 0.9718848285343656,-'jeffhuang.com': 0.9718720207664465,-'duckduckgo.com': 0.9718309347264379,-'www.sqlite.org': 0.9717513581610722,-'marc.info': 0.9716364945361756,-'www.anandtech.com': 0.9715269935352944,-'setosa.io': 0.9714750377943311,-'gogameguru.com': 0.9714728059591474,-'foundation.mozilla.org': 0.9714509129021823,-'patrickcollison.com': 0.9713258071934422,-'daringfireball.net': 0.9711795273015573,-'www.codinghorror.com': 0.971168931371747,-'raphlinus.github.io': 0.9711321390363085,-'blog.chriszacharias.com': 0.9711291613244216,-'facebook.github.io': 0.9711262368674779,-'code.facebook.com': 0.9711146723366064,-'blog.priceonomics.com': 0.9710225579333793,-'petapixel.com': 0.9709592479735079,-'aphyr.com': 0.9709163111780117,-'blog.jupyter.org': 0.970807531784939,-'www.vice.com': 0.9707422391855615,-'josephg.com': 0.9707173697331065,-'groups.google.com': 0.9707047476201209,-'www.candyjapan.com': 0.9706864280847017,-'www.bbc.co.uk': 0.9706752756956991,-'www.raspberrypi.org': 0.9706652392429661,-'paulbuchheit.blogspot.com': 0.9706297170684384,-'blog.triplebyte.com': 0.9706184118205334,-'web.eecs.utk.edu': 0.9705843602166302,-'www.foundationdb.org': 0.970531828365254,-'factorio.com': 0.9705079220882863,-'brew.sh': 0.9702877485101176,-'arstechnica.com': 0.9702828920003089,-'jlongster.com': 0.970220590502143,-'blog.dustinkirkland.com': 0.9702178011253874,-'daniel.haxx.se': 0.9701944690304003,-'rethinkdb.com': 0.9701899521411085,-'blog.linode.com': 0.9700603805951686,-'andrewkelley.me': 0.9699936860995682,-'theintercept.com': 0.9698660114939796,-'morepypy.blogspot.com': 0.9698368234837993,-'lkml.org': 0.9698185224636935,-'www.cbc.ca': 0.9697944363631238,-'www.justinobeirne.com': 0.9697393718499845,-'www.pine64.org': 0.9696643433350861,-'www.thirtythreeforty.net': 0.9696053743544627,-'www.aclu.org': 0.9696051245989374,-'robertheaton.com': 0.9695217351343498,-'www.justice.gov': 0.96946427848646,-'startupclass.samaltman.com': 0.9694186083625935,-'www.atlasobscura.com': 0.9693830118062187,-'blog.archive.org': 0.9692692877978696,-'wozniak.ca': 0.9692591742384942,-'shkspr.mobi': 0.9690681057241932,-'wikileaks.org': 0.9688043568238527,-'lucumr.pocoo.org': 0.9687437893479095,-'www.defmacro.org': 0.9685745716212145,-'cloudplatform.googleblog.com': 0.9685595849759793,-'www.wsj.com': 0.9685329922823139,-'www.gatesnotes.com': 0.9685175797648193,-'lapcatsoftware.com': 0.9684021881186297,-'www.bunniestudios.com': 0.9683871579382967,-'puri.sm': 0.9681483253888264,-'madned.substack.com': 0.9680061059600782,-'www.openculture.com': 0.9679136786699476,-'google.github.io': 0.9679127919179034,-'spreadprivacy.com': 0.9678904674640757,-'37signals.com': 0.9677339661772352,-'thedaywefightback.org': 0.96772024509243,-'asia.nikkei.com': 0.9676721486940052,-'blog.pragmaticengineer.com': 0.9676540840884938,-'brandur.org': 0.9675999497340669,-'worrydream.com': 0.9675574587791994,-'www.gimp.org': 0.9675475521140265,-'www.gabrielweinberg.com': 0.9675439099356123,-'tyler.io': 0.9675272832002738,-'www.theatlantic.com': 0.9674601161371825,-'deno.land': 0.9674479589983649,-'www.economist.com': 0.9674323307648195,-'sheddingbikes.com': 0.9672694753671274,-'pud.com': 0.9671820049226395,-'lavabit.com': 0.9671393498011488,-'blog.bradfieldcs.com': 0.9671289002274366,-'www.smithsonianmag.com': 0.9670852442662822,-'elm-lang.org': 0.9670684273054104,-'themacro.com': 0.9670447125421023,-'security.googleblog.com': 0.9669238650508819,-'www.voxelquest.com': 0.9669230619143929,-'www.gnu.org': 0.966899856247157,-'mail.mozilla.org': 0.9668731777544588,-'www.jeffgeerling.com': 0.9668579896794777,-'stallman.org': 0.9668266183513317,-'edwardsnowden.substack.com': 0.9666584767798317,-'www.latimes.com': 0.9666263838504232,-'blog.torproject.org': 0.9666181158060496,-'bugzilla.mozilla.org': 0.966563230800294,-'developer.apple.com': 0.9665153046671373,-'www.nature.com': 0.9665034870295038,-'blog.docker.com': 0.9664589097197918,-'www.cockroachlabs.com': 0.9664546158920483,-'earthly.dev': 0.9664394437158831,-'nullprogram.com': 0.9664053052413792,-'code.visualstudio.com': 0.9664051613640476,-'teachyourselfcs.com': 0.96638716524213,-'immersivemath.com': 0.9663756953881175,-'fly.io': 0.9663690278982495,-'blog.system76.com': 0.966297694451117,-'torrentfreak.com': 0.9662949409422105,-'swift.org': 0.9662794829073483,-'plausible.io': 0.9661981619745931,-'blog.timescale.com': 0.9660867756775726,-'elixir-lang.org': 0.9660464292856342,-'einaregilsson.com': 0.9660162058065089,-'blog.servo.org': 0.9658921829300279,-'tsaoutofourpants.wordpress.com': 0.9658298664203504,-'www.brendangregg.com': 0.9657781891262812,-'blog.jessfraz.com': 0.965647862426781,-'www.gwern.net': 0.9656005597335781,-'techblog.netflix.com': 0.9654696251760763,-'www.videolan.org': 0.9654507268439513,-'blogs.windows.com': 0.9653807535564003,-'arp242.net': 0.965353872976349,-'mjg59.dreamwidth.org': 0.9653138636026742,-'www.stavros.io': 0.9651634551351548,-'www.expeditedssl.com': 0.9651401658177345,-'matt.might.net': 0.9651059400349459,-'flif.info': 0.9650927452914915,-'apenwarr.ca': 0.9650906673961581,-'googleonlinesecurity.blogspot.com': 0.9650850292429668,-'priceonomics.com': 0.9650537614148901,-'www.humbledmba.com': 0.9649972831093808,-'www.unrealengine.com': 0.964977222983089,-'people.xiph.org': 0.9648830246858617,-'betterexplained.com': 0.9648763473852692,-'al3x.net': 0.9647293502855223,-'notch.tumblr.com': 0.9646905626593433,-'deno.com': 0.9646806401289668,-'www.wired.com': 0.9646357841756316,-'git.kernel.org': 0.9645834348149406,-'www.djangoproject.com': 0.9645683143391122,-'www.ruby-lang.org': 0.964565023525263,-'www.catonmat.net': 0.9645567860107777,-'blog.plover.com': 0.9645429951505131,-'ocw.mit.edu': 0.9644857262463964,-'steveblank.com': 0.9644558764763906,-'blog.haschek.at': 0.9643961053958355,-'old.reddit.com': 0.9643838762037145,-'blog.jgc.org': 0.9643196349091965,-'utcc.utoronto.ca': 0.9643144411744943,-'blog.getbootstrap.com': 0.964223508260427,-'protonmail.com': 0.9641434352105049,-'stackoverflow.blog': 0.964131467903275,-'www.feynmanlectures.caltech.edu': 0.9640844147082678,-'www.nationalgeographic.com': 0.9640002724660541,-'thume.ca': 0.9639172934049174,-'www.ftc.gov': 0.9638662117372075,-'blog.google': 0.9638278246372475,-'talks.golang.org': 0.9638081833676005,-'identity.mozilla.com': 0.9637913542538376,-'www.antipope.org': 0.9637681454362576,-'blog.coinbase.com': 0.9637227588096539,-'freedom.press': 0.9637055279129609,-'research.googleblog.com': 0.9636957954323193,-'sneak.berlin': 0.9636104943866459,-'ferd.ca': 0.9636081542021665,-'system76.com': 0.9636004595237442,-'jsomers.net': 0.9635822620240366,-'nautil.us': 0.9635511482835416,-'blog.fogus.me': 0.963547608254157,-'probablydance.com': 0.9635177117293328,-'blog.stephenwolfram.com': 0.9633882383623975,-'blog.42floors.com': 0.9633188247961246,-'www.thoughtcrime.org': 0.9631942686290309,-'www.winehq.org': 0.9631344924353825,-'www.benkuhn.net': 0.9631304906372505,-'status.cloud.google.com': 0.9630969501387503,-'dev.stephendiehl.com': 0.9630485789318193,-'www.thedrive.com': 0.9629761123330604,-'explainshell.com': 0.9629455061466635,-'krita.org': 0.9629305982536617,-'www.paulgraham.com': 0.96284858742987,-'www.google.com': 0.9626443067571239,-'blogs.msdn.microsoft.com': 0.9626200437043355,-'www.techdirt.com': 0.9625942195899927,-'homakov.blogspot.com': 0.9625907627104796,-'gist.github.com': 0.9625295427025308,-'lists.freebsd.org': 0.9624953857245481,-'www.axios.com': 0.9624629027024122,-'triplebyte.com': 0.9623309816301445,-'blog.filippo.io': 0.9621095196217451,-'mattstoller.substack.com': 0.9620940895652949,-'weblog.rubyonrails.org': 0.9620546173159494,-'viewsourcecode.org': 0.9620510951208904,-'gcc.gnu.org': 0.9619503003058159,-'tjcx.me': 0.9619235394931922,-'www.scottaaronson.com': 0.9618863749512754,-'www.mattkeeter.com': 0.9618540812590697,-'cloud.google.com': 0.9618480240586683,-'edweissman.com': 0.9616816249147265,-'www.citylab.com': 0.961667739783419,-'ziglang.org': 0.9616247712590222,-'ffmpeg.org': 0.9615920017638592,-'www.tablix.org': 0.9615841584158416,-'opensource.googleblog.com': 0.9615491317254445,-'www.lettersofnote.com': 0.9615449446825298,-'blog.pushbullet.com': 0.9614414831416516,-'www.jacquesmattheij.com': 0.9613892326733957,-'zwischenzugs.com': 0.9613828940987429,-'www.reddit.com': 0.9613787026817021,-'blogs.valvesoftware.com': 0.9613624503908588,-'markosaric.com': 0.9613621062764696,-'os.phil-opp.com': 0.9613360087194533,-'blogs.dropbox.com': 0.961291183608417,-'docs.google.com': 0.9612568870731552,-'juliareda.eu': 0.9612194124785559,-'uspirg.org': 0.9611978470349601,-'fasterthanli.me': 0.961115384552988,-'ipfs.io': 0.9610605476989421,-'queue.acm.org': 0.9610281321922659,-'www.evanmiller.org': 0.9609644541353691,-'ejohn.org': 0.9609035373663662,-'releases.ubuntu.com': 0.9608845798797817,-'support.google.com': 0.9608695426684296,-'lcamtuf.coredump.cx': 0.9608422980804076,-'mango.pdf.zone': 0.9607785943599466,-'godotengine.org': 0.9607731456425479,-'justgetflux.com': 0.960753420730463,-'bradfrost.com': 0.9607430819229773,-'nim-lang.org': 0.9607343905204542,-'simonwillison.net': 0.9606872824809413,-'www.blog.google': 0.96065581868309,-'www.sciencemag.org': 0.9605214304889523,-'byorgey.wordpress.com': 0.9604978104194218,-'www.groklaw.net': 0.9604516891162848,-'geon.github.io': 0.9603711026188184,-'lithub.com': 0.9603590822727315,-'fiber.google.com': 0.9602031412872665,-'dave.cheney.net': 0.9601949227730773,-'www.scientificamerican.com': 0.9601944955422137,-'beepb00p.xyz': 0.9601931949641657,-'blog.fastmail.com': 0.9601797872297155,-'seclists.org': 0.9601735085911461,-'www.hillelwayne.com': 0.9600285295350695,-'blogs.scientificamerican.com': 0.9599933529455126,-'www.filfre.net': 0.9599628009632043,-'www.oculus.com': 0.959958903358201,-'graydon2.dreamwidth.org': 0.9599484289082291,-'www.jonathanturner.org': 0.9599080310891523,-'jameshfisher.com': 0.9598899186695382,-'julialang.org': 0.9598652009213735,-'blogs.harvard.edu': 0.9598532297709538,-'academictorrents.com': 0.9598305702264962,-'staltz.com': 0.9598086566619791,-'www.openbsd.org': 0.9597593869835838,-'psyche.co': 0.9597358328264424,-'blog.rongarret.info': 0.9597212936777375,-'www.redblobgames.com': 0.9597207704543808,-'dtrace.org': 0.9596738647309077,-'boingboing.net': 0.9596494107644848,-'www.schneier.com': 0.9595794118397106,-'support.mozilla.org': 0.9595011501832328,-'arxiv.org': 0.959473232072393,-'twobithistory.org': 0.9594688834229274,-'tttthis.com': 0.9594411193380538,-'www.windows93.net': 0.9594238637098613,-'med.stanford.edu': 0.9594108760218417,-'ascii.textfiles.com': 0.959256899162529,-'spectrum.ieee.org': 0.9591657750656283,-'www.aaronsw.com': 0.959152603211572,-'blog.chromium.org': 0.9590740081078435,-'daltoncaldwell.com': 0.9588971457836526,-'jeffe.cs.illinois.edu': 0.9588078449509829,-'hookrace.net': 0.9587652481709775,-'lessig.tumblr.com': 0.9587508585934718,-'blog.garrytan.com': 0.958672955025337,-'leahneukirchen.org': 0.9586676171256273,-'journal.stuffwithstuff.com': 0.9586488857012637,-'status.slack.com': 0.9586248004018981,-'www.hanselman.com': 0.958563907251422,-'www.theverge.com': 0.9585530026858043,-'prog21.dadgum.com': 0.9584999221959226,-'www.craigmurray.org.uk': 0.9584843626546714,-'motherboard.vice.com': 0.9584647236016686,-'awesomekling.github.io': 0.9584478246123319,-'acko.net': 0.9584223182205939,-'www.cnbc.com': 0.9583609005406114,-'misc-stuff.terraaeon.com': 0.9582853790359502,-'blog.robertelder.org': 0.9581077255960667,-'www.patrick-breyer.de': 0.958008658008658,-'mainichi.jp': 0.9579948206173503,-'notepad-plus-plus.org': 0.9579831679547762,-'dolphin-emu.org': 0.957980480121585,-'www.adriancourreges.com': 0.9577752697052809,-'www.anishathalye.com': 0.9577244298214679,-'martin.kleppmann.com': 0.9576931939838363,-'developers.google.com': 0.9576890556647433,-'code.google.com': 0.9576138371902363,-'support.apple.com': 0.9575695045037443,-'buy.louisck.net': 0.9575368691358187,-'sandstorm.io': 0.9575210697630335,-'blog.fogcreek.com': 0.9575063092775307,-'panic.com': 0.9575015340393305,-'svelte.dev': 0.957467967080876,-'justinkan.com': 0.9574047166110535,-'www.ianbicking.org': 0.9573945069975465,-'firstlook.org': 0.9573366845702125,-'foundersatwork.posthaven.com': 0.9572959138256529,-'mail.python.org': 0.9572072777215898,-'acoup.blog': 0.9571594417654118,-'jacobian.org': 0.9571381159110411,-'hakibenita.com': 0.9568994480645705,-'mothership.sg': 0.9568299369913541,-'www.vitavonni.de': 0.9568005963591861,-'palant.de': 0.9567983422438127,-'benhoyt.com': 0.9567603051598327,-'david.li': 0.9567049710339931,-'web.stanford.edu': 0.9566872891659536,-'a16z.com': 0.9565808474917423,-'www.battleforthenet.com': 0.9565732116764964,-'uncrunched.com': 0.9565198150211067,-'www.debian.org': 0.9564674281841146,-'nullsweep.com': 0.9564159156215992,-'wiki.postgresql.org': 0.9563944999840933,-'www.snellman.net': 0.9563712102508779,-'www.craigkerstiens.com': 0.9563664501753555,-'wikimediafoundation.org': 0.9563345661162456,-'eater.net': 0.9563202100645842,-'microsoft.github.io': 0.9563042502322844,-'redditblog.com': 0.9562631224182452,-'www.aaronkharris.com': 0.9561555239477143,-'blogs.microsoft.com': 0.9560820272756707,-'qz.com': 0.955949644460612,-'tailscale.com': 0.9559251595739959,-'codewords.recurse.com': 0.9558536488585196,-'bost.ocks.org': 0.9557437711557101,-'gmailblog.blogspot.com': 0.9557079840056077,-'norvig.com': 0.9556798812425891,-'pluralistic.net': 0.9556745944240528,-'www.simplethread.com': 0.9556668873107272,-'nltimes.nl': 0.9555494259023241,-'newsroom.intel.com': 0.9554793469031995,-'news.mit.edu': 0.9554200420895869,-'jitsi.org': 0.9554166636194328,-'www.cs.cmu.edu': 0.955323422003546,-'article.gmane.org': 0.9553116742480078,-'joearms.github.io': 0.9552623762850195,-'mzucker.github.io': 0.9552342582549446,-'qntm.org': 0.9552256231956235,-'iterm2.com': 0.955216811415582,-'blog.github.com': 0.9552099253702252,-'engineering.shopify.com': 0.9551782438728066,-'geoff.greer.fm': 0.9551520599196336,-'reactjs.org': 0.9551193524754155,-'www.citusdata.com': 0.9551065831945164,-'blog.dropbox.com': 0.9550988360514683,-'pastebin.com': 0.9550820324377278,-'www.fast.ai': 0.9550621528031076,-'blog.twitter.com': 0.9550007700333996,-'marco.org': 0.9549900576536702,-'begriffs.com': 0.9549733491222114,-'lkml.iu.edu': 0.9549524012989699,-'eclecticlight.co': 0.9549360423325072,-'v8project.blogspot.com': 0.9549013081514219,-'sam.zeloof.xyz': 0.9548519610162581,-'www.facebook.com': 0.954738504671582,-'cfenollosa.com': 0.954730634374589,-'stevelosh.com': 0.9547082859524895,-'mosh.mit.edu': 0.9546865955088134,-'archive.org': 0.954683235332995,-'marctenbosch.com': 0.9546562578949468,-'www.dw.com': 0.954642614537768,-'thehustle.co': 0.9546290284301826,-'phabricator.wikimedia.org': 0.9546112568993993,-'twitter.github.com': 0.9545898933522629,-'brendaneich.com': 0.9545746174697838,-'42floors.com': 0.9545517561468556,-'blog.alexmaccaw.com': 0.9545454626799591,-'martinfowler.com': 0.9545379389229811,-'freedom-to-tinker.com': 0.9544719860088577,-'www.slate.com': 0.9544534258667388,-'www.iphoneincanada.ca': 0.9544449857023617,-'bugs.debian.org': 0.9543851860127158,-'ir.tesla.com': 0.9543767275771423,-'blog.asmartbear.com': 0.9543610682085355,-'greenwald.substack.com': 0.9543135677806216,-'gameprogrammingpatterns.com': 0.9543089003699947,-'blogs.msdn.com': 0.9543003666109717,-'www.meteor.com': 0.9542891479234723,-'www.nasa.gov': 0.9542472672213591,-'petitions.whitehouse.gov': 0.9542172936354238,-'segment.com': 0.9542063188154968,-'lexi-lambda.github.io': 0.9541906249333629,-'www.goodnewsnetwork.org': 0.9541839952530666,-'www.crypto101.io': 0.954146742893107,-'hintjens.com': 0.9541384067709378,-'changelog.complete.org': 0.9541382845549411,-'tech.mit.edu': 0.9541192032072701,-'webkit.org': 0.9540750203597949,-'blog.emacsen.net': 0.9540459808061132,-'instagram-engineering.tumblr.com': 0.9540392857685359,-'boringtechnology.club': 0.9540365522550216,-'swombat.com': 0.9540149579111236,-'berthub.eu': 0.9539835716288688,-'erikbern.com': 0.9539560598971636,-'www.theregister.com': 0.9538737853031798,-'blog.reddit.com': 0.9538429471349307,-'learnpythonthehardway.org': 0.9537983780986163,-'dynomight.net': 0.953604369679408,-'danshipper.com': 0.9535908420696627,-'status.aws.amazon.com': 0.9535221631672808,-'www.kennethreitz.org': 0.9535129582723397,-'www.jefftk.com': 0.9534058135420997,-'idiallo.com': 0.9533596892670664,-'developer.mozilla.org': 0.9532377447812391,-'discourse.mozilla.org': 0.9531776168886109,-'www.vox.com': 0.953171184497987,-'blog.burntsushi.net': 0.9531628061816667,-'www.roguelazer.com': 0.9531341112752366,-'beej.us': 0.9531338211286129,-'www.destroyallsoftware.com': 0.9530554229381429,-'eev.ee': 0.9530405280939158,-'konklone.com': 0.9530176239519549,-'www.twilio.com': 0.9530110571482041,-'www.outsideonline.com': 0.9530080955571504,-'quuxplusone.github.io': 0.9528999786913215,-'www.chrisstucchio.com': 0.9528639692666151,-'themarkup.org': 0.9528465949993555,-'www.nobelprize.org': 0.9528281612185788,-'samcurry.net': 0.9528119859873818,-'phoboslab.org': 0.9527996714263891,-'media.ccc.de': 0.9527965238893377,-'tom.preston-werner.com': 0.9527727795158389,-'iism.org': 0.9527632645423888,-'blog.joinmastodon.org': 0.9527602941375386,-'www.accc.gov.au': 0.9527263269590833,-'www.ncbi.nlm.nih.gov': 0.9527198508675186,-'ianstormtaylor.com': 0.952712496504424,-'www.imperialviolet.org': 0.9526934873020364,-'ridiculousfish.com': 0.9526563705263446,-'developers.googleblog.com': 0.9526246790332386,-'backreaction.blogspot.com': 0.9525612446013791,-'www.theparisreview.org': 0.9525261097376244,-'blog.jquery.com': 0.9524629435660441,-'www.michaeleisen.org': 0.9524499448559609,-'news.berkeley.edu': 0.9524491052294287,-'travisdowns.github.io': 0.9523196619971284,-'blog.nuclino.com': 0.9523119161198071,-'news.stanford.edu': 0.9522754316114468,-'www.universityofcalifornia.edu': 0.9522373906378331,-'gravitational.com': 0.9522355620834344,-'zserge.com': 0.9521909899856118,-'observablehq.com': 0.9521499301321013,-'mathoverflow.net': 0.9521475886704486,-'www.joyent.com': 0.9520628241406525,-'blog.kevmod.com': 0.9520548060547861,-'blog.heroku.com': 0.9520355160215065,-'www.strongtowns.org': 0.952027603849832,-'nextjs.org': 0.9520143111821525,-'www.icij.org': 0.9519656123992113,-'www.ifixit.com': 0.9519637451973145,-'www.windytan.com': 0.9518643147043334,-'www.servethehome.com': 0.9518529509789444,-'grumpy.website': 0.9517679506064377,-'bitwarden.com': 0.9517316136273904,-'gitlab.com': 0.9515782210258252,-'distill.pub': 0.9515137257464145,-'craftinginterpreters.com': 0.9514779102710061,-'lord.io': 0.9514586580765187,-'www.eso.org': 0.9514502937076248,-'www.codersnotes.com': 0.9514270198661366,-'thereader.mitpress.mit.edu': 0.9513838603840151,-'blog.regehr.org': 0.9513649990290555,-'googleresearch.blogspot.com': 0.9513477926783215,-'highscalability.com': 0.9513429766551695,-'toddwschneider.com': 0.9513421190975475,-'slatestarcodex.com': 0.9511855687467806,-'www.cloudflarestatus.com': 0.9511829060902408,-'www.cnx-software.com': 0.9511778260754474,-'www.statnews.com': 0.9511048711614947,-'youtube-eng.blogspot.com': 0.9510868926645011,-'news.yale.edu': 0.9510347346201274,-'reneweconomy.com.au': 0.9509936100187831,-'www.jwz.org': 0.950973914048226,-'ncase.me': 0.9509706871520447,-'www.metzdowd.com': 0.9509697689890211,-'edu.mkrecny.com': 0.9508994617990429,-'medium.com': 0.9508522487424811,-'mastermind.atavist.com': 0.9508126598566774,-'www.laphamsquarterly.org': 0.9507738399701613,-'telegram.org': 0.9507726262466255,-'therecord.media': 0.9507691852400171,-'www.salon.com': 0.9507685396993281,-'www.howacarworks.com': 0.9507437871644115,-'web.mit.edu': 0.9507345330973588,-'www.zdnet.com': 0.9507029832966206,-'paulstamatiou.com': 0.9506812073068672,-'eloquentjavascript.net': 0.9506744929567004,-'nathanmarz.com': 0.9506394532348129,-'en.wikipedia.org': 0.9506273469081818,-'explained.ai': 0.9506171121716123,-'golang.org': 0.9505833588437288,-'99percentinvisible.org': 0.9505598730904006,-'restoreprivacy.com': 0.9505353262581112,-'ycombinator.com': 0.9505016273288341,-'www.joshwcomeau.com': 0.950477087355932,-'easydns.com': 0.9504253361110822,-'lists.w3.org': 0.9504081521659986,-'nvidianews.nvidia.com': 0.9504026864806336,-'cyberlaw.stanford.edu': 0.9503746350188895,-'www.redhat.com': 0.9503564767146361,-'blog.racket-lang.org': 0.9503359424455089,-'www.ctrl.blog': 0.9503095809400977,-'bjk5.com': 0.9502965197985591,-'refactoringui.com': 0.9502851812628592,-'reactos.org': 0.9502772626866873,-'matthias-endler.de': 0.9502126168099793,-'blog.usejournal.com': 0.9501925685645481,-'www.deprocrastination.co': 0.9501876199581508,-'blog.llvm.org': 0.9501759187745217,-'blog.keras.io': 0.9501685115979439,-'markodenic.com': 0.9501517264182752,-'tirania.org': 0.9501284983515588,-'www.copetti.org': 0.9500923230891157,-'www.interviewcake.com': 0.9500659008199689,-'newatlas.com': 0.9500230591731798,-'blog.okcupid.com': 0.9499974432299237,-'smallstep.com': 0.949970101995039,-'act.eff.org': 0.9499506284870781,-'www.extremetech.com': 0.9498989577556406,-'kottke.org': 0.9498599050938643,-'aeon.co': 0.9498533547972355,-'blog.printf.net': 0.949841911956293,-'www.johndcook.com': 0.9497830834287447,-'community.letsencrypt.org': 0.9497776958809281,-'www.hashicorp.com': 0.9497516160457014,-'thewaltdisneycompany.com': 0.9495508540625482,-'www.haiku-os.org': 0.9495281464383918,-'scotthelme.co.uk': 0.949509887884973,-'www.gamasutra.com': 0.9494506172378354,-'cdixon.org': 0.9494179897832785,-'azeria-labs.com': 0.9493868918991253,-'stevehanov.ca': 0.949386070425381,-'calebporzio.com': 0.9493736951983298,-'blog.erlang.org': 0.9493443327606497,-'blog.backblaze.com': 0.9493279590675145,-'nymag.com': 0.9493193709909513,-'lists.debian.org': 0.9492598455856424,-'blog.mailgun.com': 0.9492541788226916,-'www.microsoft.com': 0.9492199020476939,-'tokio.rs': 0.9492084252828289,-'woodgears.ca': 0.949182590244603,-'www.zotero.org': 0.9491798215496959,-'zeit.co': 0.9491636525275446,-'tech.dropbox.com': 0.9491502994308182,-'pages.cs.wisc.edu': 0.9491309277173726,-'engineering.instagram.com': 0.9491288216402667,-'www.realworldtech.com': 0.9491223130451418,-'www.mattcutts.com': 0.9490815088845603,-'www.thecut.com': 0.9489293872710988,-'wiki.mozilla.org': 0.9489007109681905,-'theunderstatement.com': 0.9488791842136701,-'donmelton.com': 0.9488523200756159,-'learncodethehardway.org': 0.9487806920147928,-'element.io': 0.9487707078398451,-'thehill.com': 0.948742537996995,-'wattenberger.com': 0.9487142290281894,-'raganwald.com': 0.94870117403287,-'nytimes.com': 0.9487002268786773,-'www.danshapiro.com': 0.9486602162771469,-'amasci.com': 0.9484567715181271,-'steve-yegge.blogspot.com': 0.9484502644034212,-'blog.instapaper.com': 0.9483924654208046,-'www.pathsensitive.com': 0.9483799539691706,-'sachagreif.com': 0.9482981085006165,-'daedtech.com': 0.948282297833387,-'www.phoenixframework.org': 0.948180038678699,-'avc.com': 0.9481175720932656,-'jasmcole.com': 0.9480827978886457,-'lemire.me': 0.9480710409382858,-'ycombinator.posterous.com': 0.9480447746714599,-'crystal-lang.org': 0.9480289688481155,-'www.flightradar24.com': 0.9480106923973552,-'www.lofibucket.com': 0.9479892761394102,-'www.agner.org': 0.9479708121651846,-'www.mercurynews.com': 0.947909748405247,-'peteris.rocks': 0.947883344813371,-'www.haneycodes.net': 0.9478734625701677,-'blog.royalsloth.eu': 0.9478558802047412,-'jeremykun.com': 0.9478499398633942,-'netmarketshare.com': 0.9478165755279099,-'streaming.media.ccc.de': 0.9477448566079527,-'scattered-thoughts.net': 0.9477334018972962,-'theoutline.com': 0.9476703233752298,-'www.tedunangst.com': 0.9476236926963729,-'siberiantimes.com': 0.9476162638549342,-'avherald.com': 0.947493333568265,-'hackaday.com': 0.9474690019349598,-'www.hackerfactor.com': 0.9474384569226819,-'doist.com': 0.9473521795738291,-'ai.googleblog.com': 0.9473483560369622,-'macwright.com': 0.9473244768842966,-'www.cake.co': 0.9473057089030983,-'blog.ptsecurity.com': 0.9472729024375095,-'harpers.org': 0.9472208781730334,-'research.swtch.com': 0.9471779558478995,-'landing.google.com': 0.9471768097472296,-'stackoverflow.com': 0.9471708004823144,-'www.khanacademy.org': 0.9470872439554527,-'www.malwaretech.com': 0.9470832155302483,-'pippinbarr.github.io': 0.9470597529838243,-'www.chronicle.com': 0.9470439916157474,-'www.michaelnielsen.org': 0.9470172835353264,-'yahoo.tumblr.com': 0.9470082148918799,-'threadreaderapp.com': 0.9470043666496185,-'lcamtuf.blogspot.com': 0.9469939359995508,-'www.lowtechmagazine.com': 0.9469818679809128,-'thorstenball.com': 0.9469695919624959,-'copy.sh': 0.946952731836535,-'datagubbe.se': 0.9468822401332838,-'thehftguy.com': 0.9468555549213109,-'www.amazon.com': 0.9468350213972545,-'fishshell.com': 0.9468195302562247,-'lists.torproject.org': 0.9467852152182991,-'www.wbur.org': 0.9467718490726582,-'blog.inkdrop.info': 0.9467189844358732,-'retool.com': 0.9467024959584281,-'e360.yale.edu': 0.9466990573296785,-'magcius.github.io': 0.9466830296864568,-'www.aboutamazon.com': 0.9466566508253851,-'howonlee.github.io': 0.9466352935788864,-'www.randsinrepose.com': 0.9466262113364665,-'www.inkandswitch.com': 0.9466108105187082,-'bhorowitz.com': 0.9466022453520991,-'keenen.xyz': 0.9465855123017971,-'www.forbes.com': 0.9463974275879253,-'www.zachtronics.com': 0.9463917451738472,-'engineering.fb.com': 0.9463604397235058,-'taibbi.substack.com': 0.946310654934769,-'blog.elementary.io': 0.9462655485137151,-'blog.waymo.com': 0.9461989396383733,-'www.factorio.com': 0.9461844645059527,-'ghostbin.com': 0.9461717385490003,-'blog.cubeofm.com': 0.9461649703706937,-'privacyinternational.org': 0.9461573725205245,-'mondaynote.com': 0.9461425333335384,-'chrome.blogspot.com': 0.9461275447307622,-'www.swissinfo.ch': 0.9460542777095045,-'fs.blog': 0.9460214401678857,-'rosenzweig.io': 0.9459592348864171,-'thehackerblog.com': 0.9459179773558213,-'palant.info': 0.9459138777363664,-'course.fast.ai': 0.9459131445844281,-'www.python.org': 0.9459056139143512,-'www.samsung.com': 0.9459051315675879,-'www.niemanlab.org': 0.9458995158159859,-'perens.com': 0.9458876688022819,-'www-cs-students.stanford.edu': 0.9458541576360181,-'www.matuzo.at': 0.9458150903467117,-'yosefk.com': 0.9457825651296167,-'samsaffron.com': 0.9457319055612109,-'www.cs.usfca.edu': 0.9457009533921606,-'www.the-tls.co.uk': 0.9456885441587526,-'erlang.org': 0.9456437311862155,-'home.cern': 0.945571703124721,-'www.joelonsoftware.com': 0.945567200214926,-'scottlocklin.wordpress.com': 0.9455154840830812,-'blog.zoom.us': 0.9454631310396542,-'clojure.org': 0.9453997954968069,-'nest.com': 0.9453765377092626,-'pi-hole.net': 0.9453523308356795,-'erik-engheim.medium.com': 0.9453523308356795,-'projecteuler.net': 0.9453380897239767,-# 'web.archive.org': 0.945336089251807,-'coreos.com': 0.945323515243916,-'slack.engineering': 0.9453141410084411,-'jenkins.io': 0.9452507411562461,-'libcello.org': 0.9452471895570808,-'www.ietf.org': 0.9452216766608553,-'vivaldi.com': 0.9452169051506946,-'blog.acolyer.org': 0.9452136927944143,-'opensource.apple.com': 0.945156165859438,-'www.devever.net': 0.9451481294849068,-'judiciary.house.gov': 0.9451436080311264,-'cstack.github.io': 0.9451357466063348,-'neocities.org': 0.945112946847744,-'www.vanityfair.com': 0.9451005472578717,-'heidiroizen.tumblr.com': 0.945076890496307,-'www.newsweek.com': 0.9450607070750251,-'fosdem.org': 0.9450470405160809,-'lists.llvm.org': 0.9450101340465437,-'swannodette.github.io': 0.9450060465106352,-'www.forbrukerradet.no': 0.944980556159889,-'www.alexkras.com': 0.944800412259009,-'yehudakatz.com': 0.9447946105109732,-'www.politico.eu': 0.9447851995448193,-'research.microsoft.com': 0.944725678463696,-'matklad.github.io': 0.9447162628284546,-'www.fieggen.com': 0.9446507411542784,-'fermatslibrary.com': 0.9446378294996843,-'llvm.org': 0.9446335834779859,-'www.codeofhonor.com': 0.9446318312955025,-'www.sandimetz.com': 0.9445834519299251,-'beautifulracket.com': 0.944575515895676,-'solar.lowtechmagazine.com': 0.9445367689612141,-'www.rollingstone.com': 0.9444958767732993,-'www.theglobeandmail.com': 0.9444675154775829,-'crypto.stanford.edu': 0.9444289674480927,-'www.ucsf.edu': 0.9443956140487278,-'mailman.stanford.edu': 0.9443461149709853,-'games.greggman.com': 0.9442484049862852,-'robrhinehart.com': 0.9442240528350656,-'evertpot.com': 0.9441892143713023,-'ryancarson.com': 0.944157143243096,-'www.oculusvr.com': 0.944136637962384,-'news.microsoft.com': 0.9441300159133728,-'reasonablypolymorphic.com': 0.9440992853388425,-'www.nybooks.com': 0.944043771273854,-'blog.malwarebytes.com': 0.9440378261380166,-'sourcehut.org': 0.9440365818314045,-'antonz.org': 0.9439630271519353,-'weavesilk.com': 0.9439624519736566,-'jakevdp.github.io': 0.9439416004808824,-'www.githubstatus.com': 0.9439219010573966,-'thenextweb.com': 0.9438984254861107,-'squareup.com': 0.9438684150241483,-'www.businessinsider.com': 0.9438381542606629,-'www.teamten.com': 0.9438162837927613,-'minimaxir.com': 0.94381424365665,-'www.sparkfun.com': 0.9437861060710178,-'www.seattletimes.com': 0.9437565154559231,-'int10h.org': 0.943749694541571,-'cpbotha.net': 0.9437489993962991,-'cslibrary.stanford.edu': 0.9437134939561775,-'mosaicscience.com': 0.943712263624496,-'www.cjr.org': 0.9436996745182413,-'lareviewofbooks.org': 0.9436930841143853,-'www.theregister.co.uk': 0.9436788261536825,-'www.phoronix.com': 0.9436747415359684,-'blog.discord.com': 0.9436725616861676,-'doc.rust-lang.org': 0.9436705513749973,-'onezero.medium.com': 0.9436010567522964,-'fuchsia.googlesource.com': 0.9435751703627357,-'www.tomanthony.co.uk': 0.9435582795515635,-'blog.ethereum.org': 0.943554185163966,-'rhaas.blogspot.com': 0.9435179337782055,-'www.historytoday.com': 0.9434724777695044,-'veekaybee.github.io': 0.9434651701356596,-'www.michaelburge.us': 0.9433630281363066,-'www.hackerschool.com': 0.9433243309324416,-'507movements.com': 0.9433217341059063,-'www.bmj.com': 0.9433027312455573,-'blogs.wsj.com': 0.9432673977773712,-'leginfo.legislature.ca.gov': 0.9432572372586395,-'people.csail.mit.edu': 0.9430404662547742,-'nibblestew.blogspot.com': 0.9429934909491315,-'backchannel.com': 0.942970399762578,-'www.openssl.org': 0.9429668863010962,-'www.openwall.com': 0.9429455499515123,-'www.petapixel.com': 0.9428687830757905,-'jeff-vogel.blogspot.com': 0.9427633325613003,-'roslyn.codeplex.com': 0.9427563891114413,-'forums.dropbox.com': 0.9427243466398204,-'answers.microsoft.com': 0.9426879968589384,-'blog.jazzychad.net': 0.9426788682750873,-'www.atrium.co': 0.9426550434185653,-'www.ycombinator.com': 0.9425591138591901,-'statmodeling.stat.columbia.edu': 0.9425542863968303,-'cacm.acm.org': 0.9425392273023008,-'firstround.com': 0.9424999923251012,-'blog.erratasec.com': 0.9423971981109275,-'colah.github.io': 0.942392249847614,-'online.wsj.com': 0.9423603972359367,-'blog.paralleluniverse.co': 0.9423524400607514,-'boz.com': 0.9422809938913371,-'dmitry.gr': 0.942260189831035,-'www.rethinkdb.com': 0.9422304687024133,-'www.linuxjournal.com': 0.9421550256435562,-'www.bottomupcs.com': 0.9421232771919728,-'commoncog.com': 0.9420976962954309,-'status.github.com': 0.9419938569642178,-'www.drmaciver.com': 0.9419852322184868,-'imgur.com': 0.9419771536808862,-'privacylog.blogspot.com': 0.9418867600921542,-'lawsofux.com': 0.9418843340175082,-'missing.csail.mit.edu': 0.9418514250128648,-'mitpress.mit.edu': 0.9418492695998034,-'blog.wesleyac.com': 0.9418306497508919,-'sanctum.geek.nz': 0.9418252921101978,-'rarehistoricalphotos.com': 0.9418058693265607,-'blog.patreon.com': 0.9416385088594889,-'stephaniehurlburt.com': 0.9416294852135288,-'www.ssh.com': 0.9416225784673645,-'sahillavingia.com': 0.9415810451498678,-'justinmeiners.github.io': 0.9415524612629268,-'www.bleepingcomputer.com': 0.9415420860759846,-'thejh.net': 0.941539543862244,-'codingnest.com': 0.941522096507017,-'www.motorauthority.com': 0.9415012015037275,-'kevinlynagh.com': 0.9414829971920764,-'www.home-assistant.io': 0.941436958446535,-'spritesmods.com': 0.9413587273840005,-'www.esa.int': 0.9412959233934257,-'obsproject.com': 0.9412832929782082,-'www.1843magazine.com': 0.9412677053641687,-'therealkatie.net': 0.9412648438023827,-'christine.website': 0.9412231716105919,-'aadrake.com': 0.941209497556143,-'zapier.com': 0.9411980033904498,-'diegobasch.com': 0.9411801630225842,-'lichess.org': 0.9411613867773386,-'www.collectorsweekly.com': 0.9411605509869836,-'www.spiegel.de': 0.9411529028868376,-'kev.inburke.com': 0.9410868672076137,-'matt.sh': 0.9410639090369376,-'turkeyblocks.org': 0.9410266902131287,-'translate.google.com': 0.9410059430133028,-'well.blogs.nytimes.com': 0.9410008856098844,-'vitalik.ca': 0.9409975246700096,-'www.independent.co.uk': 0.9409567204051296,-'www.themarshallproject.org': 0.9409445330421784,-'www.usenix.org': 0.9409235997678326,-'blog.docker.io': 0.9408971471442287,-'kristoff.it': 0.9408956155328897,-'radio.garden': 0.9408954856512393,-'tholman.com': 0.9408838927997484,-'gregoryszorc.com': 0.9408696765619068,-'phys.org': 0.9408448137648736,-'www.prisma.io': 0.9408393039096592,-'aprilzero.com': 0.9408291999272044,-'blog.parse.com': 0.9408215317556151,-'borncity.com': 0.9407974930873888,-'theconversation.com': 0.9407949039397295,-'www.cl.cam.ac.uk': 0.9407721538543924,-'grumpygamer.com': 0.9407621103834799,-'ml.berkeley.edu': 0.9407575811882065,-'gizmodo.com': 0.9406627597554199,-'rsms.me': 0.9406458076838878,-'bitquabit.com': 0.9406214706399899,-'www.avc.com': 0.9406135247252191,-'iamnotaprogrammer.com': 0.940590780285089,-'chrishateswriting.com': 0.940590780285089,-'newsroom.fb.com': 0.9405869291621605,-'saagarjha.com': 0.9405291077737054,-'supabase.io': 0.9404661433287882,-'me.veekun.com': 0.9404573019098256,-'250bpm.com': 0.9404292658187561,-'blog.gregbrockman.com': 0.9403842788606359,-'www.mono-project.com': 0.9403717514626122,-'www.minimallyminimal.com': 0.9403321431642234,-'9to5mac.com': 0.9402902474697965,-'char.gd': 0.9402697487347172,-'plumshell.com': 0.9402496652921678,-'nixos.org': 0.9402384786300783,-'underjord.io': 0.9402384786300781,-'landoflisp.com': 0.940191169313915,-'www.tnhh.net': 0.9401820210600681,-'rapgenius.com': 0.9401741065194024,-'blog.floydhub.com': 0.9401390082080993,-'aws.typepad.com': 0.9401029943851995,-'leafletjs.com': 0.9400456080747939,-'fonts.google.com': 0.9400428820587936,-'matthewearl.github.io': 0.9400368182082475,-'www.kqed.org': 0.9400004457307097,-'timharford.com': 0.9399850780559476,-'blog.innerht.ml': 0.9399767635106161,-'blog.mecheye.net': 0.9399503933564736,-'about.sourcegraph.com': 0.9399124106162126,-'www.wi-fi.org': 0.9398176506266921,-'www.4chan.org': 0.9397638081665932,-'www.thedailybeast.com': 0.9397280815769313,-'www.ipp.mpg.de': 0.9397137553292056,-'pythonsweetness.tumblr.com': 0.9397041924803161,-'arstechnica.co.uk': 0.9396691911012381,-'news.railstutorial.org': 0.9395578781950391,-'terrytao.wordpress.com': 0.9395312291825673,-'www.inputmag.com': 0.9394997769231928,-'labs.detectify.com': 0.9394881401388651,-'undeadly.org': 0.9394661374968044,-'tech.channable.com': 0.939456208198243,-'mobile.nytimes.com': 0.9394237175327378,-'falkvinge.net': 0.9393964166317929,-'www.omniref.com': 0.9393438893288835,-'notes.unwieldy.net': 0.9392345223233198,-'blog.newsblur.com': 0.9391842678491475,-'aturon.github.io': 0.9391810647723153,-'mattwarren.org': 0.9391805858225434,-'www.fastcompany.com': 0.939180509863255,-'reason.com': 0.9391096246428814,-'ramiro.org': 0.9390819120838915,-'www.nsa.gov': 0.9390773999685927,-'www.cs.cornell.edu': 0.9390402992969881,-'www.japantimes.co.jp': 0.9389848040411876,-'www.twitlonger.com': 0.9389760828355987,-'nginx.com': 0.9389702817402761,-'www.humblebundle.com': 0.9389598199049259,-'publicdomainreview.org': 0.9389382795628163,-'blog.nuclearsecrecy.com': 0.9388477073828405,-'pubmed.ncbi.nlm.nih.gov': 0.9388458315114857,-'qotoqot.com': 0.9388402176228176,-'pytorch.org': 0.93884021461205,-'jalammar.github.io': 0.9387918236299037,-'www.ubuntu.com': 0.9387702819164052,-'docs.python.org': 0.9387127698679522,-'google-opensource.blogspot.com': 0.9386938623019518,-'www.muckrock.com': 0.9386705784507212,-'dropbox.tech': 0.9386587283618043,-'www.jasondavies.com': 0.938652854203873,-'kite.com': 0.9386407930634363,-'media.netflix.com': 0.9386280658984226,-'www.indiehackers.com': 0.9386153918054686,-'blog.detectify.com': 0.9385636423663651,-'sixcolors.com': 0.9385558509083574,-'blog.qt.io': 0.9385531275962717,-'www.quirksmode.org': 0.938550073671435,-'community.namecheap.com': 0.9384479268346089,-'thecodist.com': 0.9383878278372055,-'www.hongkongfp.com': 0.9383791221566731,-'www.stephenfry.com': 0.9383772523821312,-'coffeescript.org': 0.9383760425896429,-'major.io': 0.9383578478495119,-'typesetinthefuture.com': 0.9383193833277421,-'www.nayuki.io': 0.9383068163327775,-'www.hscott.net': 0.9382713522578224,-'freebsdfoundation.blogspot.com': 0.9382576426660746,-'eng.uber.com': 0.93823010979631,-'www.cis.upenn.edu': 0.9382279776034698,-'open.nytimes.com': 0.9382271017946164,-'pyvideo.org': 0.9381709441441788,-'bugs.launchpad.net': 0.9381396092893388,-'ai.google': 0.9381254410971094,-'michael.stapelberg.ch': 0.9380985374502866,-'v8.dev': 0.9380973536567017,-'digiday.com': 0.9379742396004388,-'commandcenter.blogspot.com': 0.9379620301227771,-'tools.ietf.org': 0.9379500242941622,-'thewalrus.ca': 0.9378381775859617,-'discussions.apple.com': 0.9377827205296552,-'ignorethecode.net': 0.937717324851746,-'thread.gmane.org': 0.9376901941531085,-'sellercentral.amazon.com': 0.9376857021855104,-'www.sciencealert.com': 0.9376774589797282,-'slackhq.com': 0.937668726802223,-'blog.datawrapper.de': 0.9376182617912265,-'seriouspony.com': 0.9376182617912265,-'musicforprogramming.net': 0.9376182617912265,-'meowni.ca': 0.9375988271796821,-'www.redox-os.org': 0.9375879483491127,-'matt-welsh.blogspot.com': 0.937504055669681,-'minnie.tuhs.org': 0.9374529581205222,-'rondam.blogspot.com': 0.9374449372004028,-'warpspire.com': 0.9374088502294052,-'locusmag.com': 0.9373920717203438,-'www.bloombergview.com': 0.9373542498215184,-'www.damninteresting.com': 0.9373300477725874,-'joeyh.name': 0.9373215784915493,-'www.newstatesman.com': 0.9373076981929569,-'www.freebsd.org': 0.9372715325620732,-'andregarzia.com': 0.9372609577852429,-'caddyserver.com': 0.9372293681509105,-'kukuruku.co': 0.9372291284737027,-'www.jbkempf.com': 0.9372233784657422,-'pharo.org': 0.9371878563371911,-'netblocks.org': 0.9371746434498315,-'chromium.googlesource.com': 0.9371719999296655,-'phrack.org': 0.9371521351298083,-'www.codelitt.com': 0.9371442586175533,-'www.baekdal.com': 0.9371414507989151,-'www.ianvisits.co.uk': 0.9371147927406074,-'www.megaprocessor.com': 0.937090322173857,-'www.yosefk.com': 0.9370290915888351,-'www.mail-archive.com': 0.936986491855952,-'abe-winter.github.io': 0.9369860158978973,-'www.privateinternetaccess.com': 0.9369851095051764,-'nginx.org': 0.9369470816071664,-'www.intel.com': 0.9369378525632501,-'www.meltingasphalt.com': 0.9369168990825746,-'juliacomputing.com': 0.9368655840942366,-'rein.pk': 0.936849859435914,-'store.steampowered.com': 0.9368264728223703,-'www.osnews.com': 0.9368126058230493,-'ourworldindata.org': 0.9367969636489132,-'blogs.law.harvard.edu': 0.93674988182828,-'www-cs-faculty.stanford.edu': 0.9367219385122602,-'blog.joda.org': 0.9367157779314322,-'blog.stephsmith.io': 0.9366996246387541,-'www.linusakesson.net': 0.9366993761963279,-'blog.stackoverflow.com': 0.9366671617187088,-'learn.shayhowe.com': 0.9366612381135532,-'www.chrisfenton.com': 0.9366359092132852,-'alexdanco.com': 0.9366140932238274,-'bert.org': 0.9365314575663983,-'www.buyforlifeproducts.com': 0.9365183246073299,-'slate.com': 0.936478665114466,-'adventofcode.com': 0.9364182659002771,-'reuters.com': 0.9364022987065517,-'wordsandbuttons.online': 0.9364000242256861,-'blog.trailofbits.com': 0.9363648191551225,-'wwws.whitehouse.gov': 0.9362358726756497,-'www.saminiir.com': 0.9362027194085935,-'buttondown.email': 0.9361904266453563,-'www.polygon.com': 0.9361786245135907,-'ryanwaggoner.com': 0.9361726319454605,-'media.defense.gov': 0.9361328144489018,-'meta.wikimedia.org': 0.9361317597043629,-'journals.plos.org': 0.9360648852842034,-'users.ece.utexas.edu': 0.936037069978365,-'www.wired.co.uk': 0.9360115157816417,-'code.fb.com': 0.9359956830531816,-'blog.apnic.net': 0.9359552179738138,-'www.craftinginterpreters.com': 0.9359520862311609,-'blog.agilebits.com': 0.9359127041563384,-'ar.al': 0.935800217022854,-'tomdale.net': 0.9357863410079088,-'doublepulsar.com': 0.9357738329674509,-'www.keyvalues.com': 0.9357728930152032,-'android-developers.googleblog.com': 0.9357673687378363,-'www.w3.org': 0.9357529019651287,-'googlecloudplatform.blogspot.com': 0.9357511057780838,-'gravityandlevity.wordpress.com': 0.9357231633007936,-'zoekeating.tumblr.com': 0.9357009188423131,-'www.startupschool.org': 0.935616713926276,-'manishearth.github.io': 0.9355839880400915,-'android.googlesource.com': 0.9355592628654659,-'thegradient.pub': 0.9355328767434344,-'orgmode.org': 0.9355234169604703,-'www.wsws.org': 0.9355201201831335,-'citizenlab.org': 0.9355108358152496,-'blog.notdot.net': 0.9355042170349265,-'eduardo.intermeta.com.br': 0.9354939284774484,-'joreteg.com': 0.9354927179012053,-'issues.apache.org': 0.9354858969424931,-'www.hopkinsmedicine.org': 0.9354732723093431,-'stopa.io': 0.9354640998209997,-'rigsomelight.com': 0.9354524106623595,-'docs.djangoproject.com': 0.9354103729663484,-'americanaffairsjournal.org': 0.9353862576701931,-'forums.aws.amazon.com': 0.9353840941845297,-'jsfiddle.net': 0.9353603756027571,-'www.metalevel.at': 0.9353562467278554,-'hexatomium.github.io': 0.9353495280479229,-'andreasgal.com': 0.9353197730789111,-'c9x.me': 0.9352853909343191,-'danieltenner.com': 0.9352765782885712,-'danielcompton.net': 0.9352733622613952,-'netflixtechblog.com': 0.9352444459107085,-'www.vidarholen.net': 0.9352427951268865,-'themargins.substack.com': 0.9351811093645357,-'eli.thegreenplace.net': 0.9351789153416379,-'cybernews.com': 0.9351782885205434,-'blogs.smithsonianmag.com': 0.9351577172122721,-'rambo.codes': 0.9351531025036611,-'hackermonthly.com': 0.9351416375508997,-'www.sifive.com': 0.9351248366429326,-'plus.maths.org': 0.9351218966133573,-'incoherency.co.uk': 0.9351026897171553,-'www.erlang.org': 0.9350952451519492,-'circleci.com': 0.9350778094676155,-'catern.com': 0.9350750533895246,-'www.penny-arcade.com': 0.935065899223226,-'howistart.org': 0.935059575032339,-'www.middleeasteye.net': 0.9350521639498252,-'unlikekinds.com': 0.9350373828027304,-'www.videogameschronicle.com': 0.9350222208625469,-'www.roadandtrack.com': 0.9350012079006418,-'hyperallergic.com': 0.9349862159226987,-'blog.level3.com': 0.9349610044277377,-'www.os2museum.com': 0.934946992149697,-'soranews24.com': 0.9349080141492474,-'patshaughnessy.net': 0.9348951953108801,-'bill.harding.blog': 0.9348850338635601,-'blog.wikimedia.org': 0.9348790533709054,-'blog.plan99.net': 0.9348288512702018,-'www.gq.com': 0.9348208606560191,-'www.occrp.org': 0.9347939860561864,-'chronicle.com': 0.9347573157010687,-'mailarchive.ietf.org': 0.9347044422954542,-'devonzuegel.com': 0.9346592212655933,-'www.ben-evans.com': 0.9346544305688902,-'www.mediumdifficulty.com': 0.9345920431557654,-'www.fsf.org': 0.9345910501167647,-'learn.sparkfun.com': 0.9345230614646949,-'www.cs.princeton.edu': 0.934519791347039,-'dlang.org': 0.9345196775723191,-'www.templeos.org': 0.9345190524769148,-'code.blender.org': 0.9345112418088564,-'www.singhkays.com': 0.9344753142105167,-'www.repubblica.it': 0.9344751927302707,-'www.aerofs.com': 0.9344708443467806,-'www.lrb.co.uk': 0.9344236297924859,-'eng.lyft.com': 0.9344184442002427,-'jakearchibald.com': 0.9343998635925788,-'blog.youtube': 0.9343948680470999,-'richg42.blogspot.com': 0.9343774893527464,-'np.reddit.com': 0.9343550039538729,-'blog.easydns.org': 0.9343498007033786,-'caseyhandmer.wordpress.com': 0.9343353091084834,-'xiph.org': 0.9343228401668484,-'www.newscientist.com': 0.9343192570672714,-'www.nicolasbize.com': 0.9343100575008362,-'lineageos.org': 0.9342978511086933,-'briancallahan.net': 0.9342859603892616,-'blog.ltse.com': 0.9342402900090527,-'www.bryanbraun.com': 0.9341978241526933,-'blog.thea.codes': 0.9341759010883351,-'laphamsquarterly.org': 0.9341164747404341,-'code.mixpanel.com': 0.9340744683300655,-'fi.google.com': 0.9340569727587175,-'web.law.duke.edu': 0.9340401302760873,-'www.technologyreview.com': 0.93403386212399,-'iamtrask.github.io': 0.9340084148579416,-'news.ncsu.edu': 0.9340051509637882,-'www.acsh.org': 0.9339402739014211,-'repl.it': 0.9339084167158871,-'roscidus.com': 0.9339046191083922,-'blog.ghost.org': 0.9338973081214549,-'www.icann.org': 0.9338815927929048,-'digest.bps.org.uk': 0.9338601133658677,-'www.nj.com': 0.9338599954559099,-'www.90percentofeverything.com': 0.9338469537566946,-'lars.ingebrigtsen.no': 0.9338290703630162,-'harmful.cat-v.org': 0.933806937838211,-'www.ecma-international.org': 0.9337395167070882,-'engineering.riotgames.com': 0.9337282192512538,-'www.lexaloffle.com': 0.9337178769327638,-'www.noaa.gov': 0.9337158126136492,-'en.greatfire.org': 0.933688333932158,-'www.techcrunch.com': 0.9336781278522835,-'research.fb.com': 0.9336749976938089,-'karl-voit.at': 0.9336349938839054,-'fair.org': 0.9336252434606269,-'www.askamathematician.com': 0.9336136547935397,-'unixism.net': 0.9335805282449469,-'www.rockpapershotgun.com': 0.9335525383720096,-'developers.redhat.com': 0.9335410437326863,-'www.realdanlyons.com': 0.9335250444350301,-'play0ad.com': 0.9334866458886125,-'jwz.livejournal.com': 0.933468431827225,-'tedium.co': 0.9334408912081676,-'bitcoin.org': 0.9333591671165428,-'loup-vaillant.fr': 0.933343126525129,-'getpolarized.io': 0.9333167489840644,-'thewire.in': 0.9333119721322196,-'melmagazine.com': 0.9331968311771961,-'sambleckley.com': 0.9331955922865014,-'www.inc.com': 0.9331571334144225,-'david.heinemeierhansson.com': 0.9331531237247338,-'secondbreakfast.co': 0.9331357922232552,-'www.asahi.com': 0.933135633505878,-'devblogs.microsoft.com': 0.9331213750483387,-'www.nintendo.co.jp': 0.9331114844804387,-'annoying.technology': 0.9330709402145108,-'www.kiro7.com': 0.9330656946067081,-'bits.debian.org': 0.933059049771486,-'www.markshuttleworth.com': 0.9330005151597153,-'anders.unix.se': 0.9329923290063773,-'blogs.sciencemag.org': 0.9329560797100107,-'blog.bitbucket.org': 0.9329522711475849,-'blog.kubernetes.io': 0.9329204663072305,-'devdocs.io': 0.932893331029596,-'justinjackson.ca': 0.9327982068928226,-'corte.si': 0.9327949868127001,-'kotaku.com': 0.9327897653993101,-'cdn.ca9.uscourts.gov': 0.9327301316906936,-'forum.dlang.org': 0.9326997809060901,-'spacecraft.ssl.umd.edu': 0.9326765708381659,-'www.sciencedirect.com': 0.9326581830227065,-'neuralnetworksanddeeplearning.com': 0.9326459352221996,-'www.politico.com': 0.9326420745727587,-'notamonadtutorial.com': 0.9326372939715099,-'news.umich.edu': 0.9325964202141381,-'duck.co': 0.9325821384299628,-'lg.io': 0.9325797024877297,-'vorpus.org': 0.9325422497012895,-'objective-see.com': 0.9324963202009166,-'www.erlang-solutions.com': 0.9324852099306608,-'alexgaynor.net': 0.9324583238847478,-'sub.garrytan.com': 0.9324345058249506,-'www.dropbox.com': 0.9324087214510501,-'xkcd.com': 0.932407776611293,-'blog.codinghorror.com': 0.9323885522938681,-'wingolog.org': 0.9323392944843699,-'www.pipelinedb.com': 0.9322802131775214,-'iquantny.tumblr.com': 0.932275204317245,-'hbr.org': 0.9322550416701433,-'www.washington.edu': 0.9321755936890619,-'blog.nelhage.com': 0.9321436277497521,-'tonyarcieri.com': 0.932131262902639,-'www.geoffreylitt.com': 0.9321152273514123,-'viniciusvacanti.com': 0.9320235111465629,-'www.magicleap.com': 0.9319299806186524,-'www.igvita.com': 0.9318778756566067,-'chrisdone.com': 0.9318776390307953,-'stackshare.io': 0.9318592007853939,-'latacora.singles': 0.9318477865160556,-'www.dell.com': 0.9318413237056871,-'djhworld.github.io': 0.9317862165963432,-'www.infoq.com': 0.9317836684869253,-'papersplease.org': 0.931751319080821,-'thepiratebay.org': 0.9316799771304791,-'calnewport.com': 0.9316572134183647,-'krausefx.com': 0.9316559307678887,-'blog.circleci.com': 0.9316255482754808,-'blog.cognitect.com': 0.9315784532891852,-'www.epicgames.com': 0.9315429629794932,-'scopeblog.stanford.edu': 0.9315377292068189,-'www.masswerk.at': 0.9314679327889084,-'0x65.dev': 0.9314351318900396,-'www.engadget.com': 0.9314186236786943,-'dangerousminds.net': 0.93133503311817,-'www.businessweek.com': 0.9313339649208416,-'meta.slashdot.org': 0.9313051774463162,-'john-millikin.com': 0.9312674134931619,-'www.raptitude.com': 0.9312570267721416,-'www.mersenne.org': 0.9312544294826364,-'blog.reverberate.org': 0.931247035488847,-'alexisohanian.com': 0.9311960984991072,-'electrek.co': 0.9311696023394275,-'www.jeanhsu.com': 0.9311256778507919,-'svelte.technology': 0.9311186925197594,-'programmers.stackexchange.com': 0.9311091436248506,-'discuss.ocaml.org': 0.9311072488914112,-'nedbatchelder.com': 0.9310954334270662,-'www.haskellforall.com': 0.931051756515657,-'retrocomputing.stackexchange.com': 0.9310144532795315,-'catonmat.net': 0.9309769810532454,-'vole.wtf': 0.9309344769161616,-'www.scribd.com': 0.9309198232076064,-'yurichev.com': 0.9309143601885577,-'joel.is': 0.9309094060954536,-'yager.io': 0.9308009815388928,-'boilingsteam.com': 0.9307989777220242,-'gamasutra.com': 0.9307558317808844,-'www.bigmessowires.com': 0.9307114429921958,-'fail0verflow.com': 0.930710252209961,-'news.harvard.edu': 0.9306854591781405,-'beagleboard.org': 0.9306733479517327,-'www.floridatoday.com': 0.9306710239151541,-'www.popehat.com': 0.9306350599333171,-'blog.tutorspree.com': 0.9306202167888316,-'daeken.com': 0.9305668550552411,-'techstartups.com': 0.9305594478826466,-'www.wikitribune.com': 0.930515310124751,-'www.chess.com': 0.9304928063593388,-'newsoffice.mit.edu': 0.9304895970361445,-'wired.com': 0.9304827864643992,-'sealedabstract.com': 0.9304819216359799,-'0x46.net': 0.9304806845007771,-'unsplash.com': 0.9304637690166982,-'my.opera.com': 0.930450954749869,-'nerds.airbnb.com': 0.930435238682839,-'longreads.com': 0.9304235614345844,-'www.dispatch.com': 0.9303745139221324,-'www.fda.gov': 0.9303439075870598,-'lists.centos.org': 0.9303434013425956,-'adgrok.com': 0.9303388809532392,-'thinkingthrough.substack.com': 0.9303337259943262,-'mcfunley.com': 0.9303298670383908,-'beginners.re': 0.9302874468948247,-'khan.github.io': 0.9302341812757136,-'joshworth.com': 0.9302242175504744,-'lamport.azurewebsites.net': 0.9302036598082833,-'calv.info': 0.9301915155599901,-'jdk.java.net': 0.9301739684537851,-'keepassxc.org': 0.9301626315930274,-'thepiratebay.se': 0.9301506979843667,-'metatalk.metafilter.com': 0.9301352170901589,-'gregable.com': 0.9301219028430462,-'newrepublic.com': 0.9301135242301953,-'cryptome.org': 0.9301123136352331,-'www.smashcompany.com': 0.9301070862923108,-'mixergy.com': 0.9301017907321477,-'blog.bolt.io': 0.9300384028733197,-'lesswrong.com': 0.9299875078740503,-'www.recode.net': 0.9299803603149057,-'baremetrics.com': 0.9299780878801276,-'idea.popcount.org': 0.92997751068095,-'blog.nodejs.org': 0.9299713179632104,-'googlecode.blogspot.com': 0.9299671011900217,-'dustincurtis.com': 0.929962639691764,-'capnproto.org': 0.9299597749554257,-'samy.pl': 0.9299580709168783,-'fivethirtyeight.com': 0.9299521782157305,-'babeljs.io': 0.9299138678024598,-'venturebeat.com': 0.9299090359763643,-'bartoszmilewski.com': 0.9298782796519226,-'interactivepython.org': 0.9298650433817537,-'dbmsmusings.blogspot.com': 0.9298460930808272,-'feross.org': 0.9298143975383142,-'www.nateberkopec.com': 0.9297897921141977,-'www.bnnbloomberg.ca': 0.9297228959508879,-'sourcegraph.com': 0.9296937397028914,-'www.visualstudio.com': 0.9296781764315996,-'www.dampfkraft.com': 0.9296648658804626,-'sockpuppet.org': 0.9296582232650001,-'vuejs.org': 0.9296438971766601,-'mozillagfx.wordpress.com': 0.929620568345932,-'www.tomshardware.com': 0.9295421933042515,-'go.googlesource.com': 0.929539952433035,-'every.to': 0.9295113508234609,-'community.signalusers.org': 0.9295104306484039,-'features.propublica.org': 0.9294651901563945,-'trigger.io': 0.9294592741801949,-'blog.wolfram.com': 0.9294394447255451,-'science.nasa.gov': 0.9294364753474607,-'www.globalcitizen.org': 0.9294355607283274,-'brennan.io': 0.9294094367716018,-'www.princeton.edu': 0.9293972895467508,-'www.typescriptlang.org': 0.9293477418572129,-'blog.hboeck.de': 0.9293102195703717,-'www.kickstarter.com': 0.9293062867407036,-'blog.siftscience.com': 0.9293015387633619,-'blog.jetbrains.com': 0.9292407139248418,-'mag.newsweek.com': 0.9291970802919708,-'libcom.org': 0.9291864757522856,-'pudding.cool': 0.9291654244837007,-'www.geek.com': 0.9291554612391715,-'zedshaw.com': 0.9291438333180465,-'mailman.nginx.org': 0.9291339657438804,-'www.techspot.com': 0.9291295247017188,-'www.popularmechanics.com': 0.9291147919612897,-'www.nbcnews.com': 0.9290863046432193,-'acesounderglass.com': 0.929082768087054,-'stitcher.io': 0.9290511337189973,-'www.texasmonthly.com': 0.92902453382746,-'mattmaroon.com': 0.9290233630671553,-'www.gabrielgambetta.com': 0.9290195296996202,-'m.dw.com': 0.9290040778772607,-'www.mit.edu': 0.9289941184727176,-'tratt.net': 0.9289824042121754,-'machinelearning.apple.com': 0.928981486544923,-'blog.posterous.com': 0.9289604924288977,-'oxide.computer': 0.9289445169614982,-'trackchanges.postlight.com': 0.928879705271154,-'codahale.com': 0.9288692041231931,-'emberjs.com': 0.9288495718673251,-'applehelpwriter.com': 0.9287812041116006,-'shaunlebron.github.io': 0.9286838540725274,-'open.buffer.com': 0.9286596873950792,-'www.expatsoftware.com': 0.9286078722400491,-'edgedb.com': 0.928573715100738,-'etherpad.com': 0.9285692478417551,-'muratbuffalo.blogspot.com': 0.9285656789226785,-'fgiesen.wordpress.com': 0.9285630732312407,-'www.firebase.com': 0.9285546878582209,-'blog.thestartuptoolkit.com': 0.9285487410065263,-'cloudirregular.substack.com': 0.9285463604002121,-'flynn.io': 0.9285393525237848,-'www.eevblog.com': 0.9285287752743465,-'www.balena.io': 0.9284888543895996,-'rt.com': 0.9284735976773327,-'www.bell-labs.com': 0.9284678162678174,-'www.persuasion.community': 0.9283945188643842,-'eugeneyan.com': 0.9283782094499546,-'80000hours.org': 0.9283725313298165,-'www.zerotier.com': 0.9283630467998711,-'residentcontrarian.substack.com': 0.9283528817911175,-'httptoolkit.tech': 0.9283475144274776,-'www.ekioh.com': 0.9283450473725626,-'www.cia.gov': 0.9283207863801844,-'ryanleecarson.tumblr.com': 0.9283074648928308,-'nickcraver.com': 0.9282875328539527,-'syonyk.blogspot.com': 0.9282602083788998,-'www.simonsfoundation.org': 0.9282557678633827,-'www.artsy.net': 0.9281354567126978,-'tryretool.com': 0.9281338223307144,-'www.bloombergquint.com': 0.928121311983667,-'johncarlosbaez.wordpress.com': 0.9281129934414499,-'vas3k.com': 0.9280973443374919,-'100r.co': 0.9280948851000741,-'blogs.adobe.com': 0.928088150929166,-'www.motherjones.com': 0.9280843595076367,-'www.nasdaq.com': 0.9280594095034941,-'scarybeastsecurity.blogspot.com': 0.9280588046711343,-'bitcointalk.org': 0.9280388108667238,-'hackershelf.com': 0.9280341162118197,-'news.nationalgeographic.com': 0.9279960640653149,-'myactivity.google.com': 0.9279454537052139,-'blog.nightly.mozilla.org': 0.9279411885747856,-'slashcrypto.org': 0.9279346210995543,-'www.phoboslab.org': 0.927902020777888,-'tosdr.org': 0.9278899901888288,-'www.fiercebiotech.com': 0.9278829786533511,-'pomax.github.io': 0.9278608982168398,-'www.haskell.org': 0.9278608403233632,-'tribunemag.co.uk': 0.927831772588806,-'alexeymk.com': 0.9278193647918856,-'www.cs.vu.nl': 0.9277449579009552,-'news.cnet.com': 0.927730426058379,-'www.caltech.edu': 0.9277282839323142,-'www.azarask.in': 0.9276539782249613,-'gigaom.com': 0.9276147331753969,-'pdos.csail.mit.edu': 0.9276091373570645,-'www.virtualbox.org': 0.9275378961204167,-'increment.com': 0.9275213097235798,-'blog.janestreet.com': 0.9274833694141223,-'www.0x65.dev': 0.9274613287943144,-'textslashplain.com': 0.9274390770834591,-'www.planetary.org': 0.927429526861741,-'allthingsd.com': 0.9274270826887906,-'clojure.com': 0.9274173307178728,-'git.postgresql.org': 0.9274089095546911,-'www.howtogeek.com': 0.927406503902944,-'vr.google.com': 0.9273952095808383,-'blog.kapeli.com': 0.9273878300515732,-'quickdraw.withgoogle.com': 0.9273611719298221,-'erkaman.github.io': 0.9273454992466085,-'discussions.agilebits.com': 0.9273075851865854,-'fauna.com': 0.9272945460775945,-'popehat.com': 0.9272692801582506,-'blog.sandstorm.io': 0.9272550707158864,-'www.openvim.com': 0.9272549340276682,-'www.panic.com': 0.9272306781608166,-'waymo.com': 0.9271995570219662,-'blog.eladgil.com': 0.9271995137279543,-'uk.reuters.com': 0.9271613384786328,-'www.openstreetmap.org': 0.9270974234890762,-'projekte.sueddeutsche.de': 0.9270885496912139,-'whytheluckystiff.net': 0.9270441393847589,-'the-paper-trail.org': 0.9269567323760859,-'medium.dave-bailey.com': 0.9269393622635275,-'www.kadavy.net': 0.9269319992417328,-'symbo1ics.com': 0.9269313741433315,-'erikrood.com': 0.926847335423336,-'text.npr.org': 0.9268424682157039,-'www.gnome.org': 0.9268089290588881,-'www.arp242.net': 0.9267978785624127,-'nvlabs.github.io': 0.9267181712194288,-'craigkerstiens.com': 0.9267131513833771,-'www.evanjones.ca': 0.9266986776879761,-'johnsalvatier.org': 0.926681783824641,-'notes.eatonphil.com': 0.9266798244317584,-'cityobservatory.org': 0.9266470517985763,-'www.autocar.co.uk': 0.9266248470761401,-'store.google.com': 0.9265910046118487,-'blog.alinelerner.com': 0.9265889786185729,-'a.singlediv.com': 0.9265847984290865,-'www.city-journal.org': 0.926568347566864,-'blog.bugsnag.com': 0.9265163617009136,-'blog.steveklabnik.com': 0.9265012336320916,-'www.ox.ac.uk': 0.9264790742628204,-'www.jgc.org': 0.92646604158067,-'maraoz.com': 0.9264393162283295,-'www.somethingsimilar.com': 0.9264338569528214,-'electronupdate.blogspot.com': 0.9264036418816388,-'productforums.google.com': 0.9264016849144228,-'macro.ycombinator.com': 0.9263799756741565,-'damienkatz.net': 0.9263697789667,-'www.moserware.com': 0.9263363409676512,-'nikic.github.io': 0.9263232996382066,-'www.elastic.co': 0.9263050573116817,-'www.cs.utexas.edu': 0.9263034743689128,-'www.bhorowitz.com': 0.9262984080290757,-'tenthousandmeters.com': 0.9262841230837943,-'about.fb.com': 0.9262484440409288,-'www.fmwconcepts.com': 0.9262450791824973,-'recraigslist.com': 0.9262425641046296,-'www.codingjohnson.com': 0.9262281542120302,-'www.aftenposten.no': 0.9262270297289266,-'www-math.mit.edu': 0.9261881534042337,-'blog.1password.com': 0.9261268579416305,-'neugierig.org': 0.9261248188625917,-'01.org': 0.9261246392123041,-'jalopnik.com': 0.9261214539229355,-'www.mnot.net': 0.9261212790512424,-'jmoiron.net': 0.9260837453092551,-'bjoern.brembs.net': 0.9260528919272174,-'en.community.dell.com': 0.9260523403380299,-'yieldthought.com': 0.9260498267973828,-'news.gallup.com': 0.9260444722659747,-'www.quora.com': 0.9260005301179841,-'www.boingboing.net': 0.9259933447507237,-'plusbryan.com': 0.9259774485563826,-'webcache.googleusercontent.com': 0.9259579793050731,-'rameznaam.com': 0.9259525603398252,-'dev.windows.com': 0.9259474200369756,-'www.cnn.com': 0.9259450168290682,-'ij.org': 0.9259017125754639,-'hackingdistributed.com': 0.9258935189335176,-'aosabook.org': 0.9258707865599977,-'webtransparency.cs.princeton.edu': 0.9258302497498668,-'nikcub.appspot.com': 0.9257984690273875,-'www.fcc.gov': 0.9257766215669876,-'multithreaded.stitchfix.com': 0.9256971609820355,-'www.ybrikman.com': 0.9256541091635649,-'www.neh.gov': 0.925636711625259,-'abc.xyz': 0.9256328360448464,-'thecorrespondent.com': 0.9256254843843928,-'www.techempower.com': 0.9256081793071045,-'letsencrypt.status.io': 0.9255938806311487,-'grail.cs.washington.edu': 0.925560977684432,-'syncthing.net': 0.9255454895835606,-'torontolife.com': 0.9255329261936903,-'www.notion.so': 0.9255238074027249,-'www.empiricalzeal.com': 0.925521405860939,-'www.brown.edu': 0.925521405860939,-'dustri.org': 0.9255123842761099,-'restofworld.org': 0.9254846646984972,-'meta.stackoverflow.com': 0.9254699602796586,-'www.mikeash.com': 0.9254652180552938,-'emacsair.me': 0.9254649535069708,-'fossil-scm.org': 0.9254641195091267,-'drive.google.com': 0.9254499381167396,-'rauchg.com': 0.9254066104836953,-'www.oregonlive.com': 0.9254001048984968,-'madebyevan.com': 0.9253946331608134,-'bitbucket.org': 0.925357050068107,-'justinvincent.com': 0.9253388274119266,-'shape-of-code.coding-guidelines.com': 0.9253230426310416,-'www.journals.uchicago.edu': 0.9252909862687759,-'www.hodinkee.com': 0.925286129490443,-'blog.getprismatic.com': 0.9252627177795818,-'android-developers.blogspot.com': 0.9252451042228654,-'jeremyaboyd.com': 0.9252351792387165,-'mozilla.github.io': 0.9252103780415603,-'www.pcgamer.com': 0.9251974859644215,-'tutanota.com': 0.9251840115849959,-'review.chicagobooth.edu': 0.9251560984825087,-'programmingisterrible.com': 0.9251358332003701,-'zainamro.com': 0.9251300763149733,-'www.locusmag.com': 0.9251291781651696,-'waxy.org': 0.9250938835073874,-'bentilly.blogspot.com': 0.9250916663441209,-'hisham.hm': 0.925069607765931,-'www.foundersatwork.com': 0.9250661963070413,-'neil.computer': 0.9250487557545186,-'youmightnotneedjquery.com': 0.9250308951241109,-'sparktoro.com': 0.9250076259146361,-'supporters.eff.org': 0.9250027991144959,-'www.youtube.com': 0.9250024575604987,-'www.django-rest-framework.org': 0.9249975958434261,-'fuse.wikichip.org': 0.9249946532155069,-'alexanderell.is': 0.9249832648395125,-'www.jetbrains.com': 0.9249775253793479,-'rootsofprogress.org': 0.9249775157472907,-'blog.jonlu.ca': 0.9249720526798552,-'grafana.com': 0.9249363529284085,-'joinpeertube.org': 0.92492611085508,-'trac.torproject.org': 0.92492568982512,-'www.kapwing.com': 0.9249078774898674,-'jgthms.com': 0.9249053279219519,-'www.lighterra.com': 0.9248749337230386,-'www.bzarg.com': 0.9248749337230386,-'itep.org': 0.924864446165763,-'xuanji.appspot.com': 0.924862527414928,-'dood.al': 0.9248541691533322,-'seriot.ch': 0.924849625532851,-'go-review.googlesource.com': 0.9248263431754091,-'benjamincongdon.me': 0.924814411910629,-'coderwall.com': 0.9247619851237905,-'www.betabeat.com': 0.9247390428437294,-'cryto.net': 0.924723051999456,-'blogs.bl.uk': 0.9247163678317087,-'datatracker.ietf.org': 0.924674483012171,-'www.b-list.org': 0.9246677557779804,-'apple.stackexchange.com': 0.924658393133553,-'www.mozilla.com': 0.9246560951725633,-'www.mailpile.is': 0.9246491366569076,-'see.stanford.edu': 0.9246092475112012,-'www.chiark.greenend.org.uk': 0.9246053483860566,-'groups.csail.mit.edu': 0.9245841594300752,-'www.arcfn.com': 0.9245434848298174,-'urbit.org': 0.9245310249759199,-'www.docker.com': 0.9245172226554722,-'eprint.iacr.org': 0.9245057684882811,-'dl.acm.org': 0.9244842820223327,-'blog.sessionstack.com': 0.9244816364620212,-'www.breck-mckye.com': 0.9244526176174389,-'www.aosabook.org': 0.9244287224439842,-'prospect.org': 0.9243862527888093,-'micro-editor.github.io': 0.9243497443248759,-'banksimple.com': 0.9243293845078053,-'practicaltypography.com': 0.9243063840022637,-'blogmaverick.com': 0.9242840626877411,-'instagram-engineering.com': 0.9242395677129099,-'theartofmachinery.com': 0.9242293335123167,-'www.cpushack.com': 0.9241635632263332,-'guzey.com': 0.9241634452596086,-'morrick.me': 0.9241492448464671,-'chanind.github.io': 0.9241272337118139,-'blog.emojipedia.org': 0.9241070688292589,-'arjunsreedharan.org': 0.9240265801481968,-'superuser.com': 0.9240150880034484,-'zulko.github.io': 0.9240082509797493,-'blog.fosketts.net': 0.923996721917042,-'darkpatterns.org': 0.9239687269035174,-'www.ulisp.com': 0.9239409936091142,-'herbsutter.com': 0.9238650642442959,-'research.google.com': 0.9238529070048137,-'www.darkcoding.net': 0.9238452017940412,-'latacora.micro.blog': 0.9237958672976048,-'www.drdobbs.com': 0.9237812616938089,-'www.hollywoodreporter.com': 0.9237805856737081,-'www.nginx.com': 0.9237795637254299,-'thestartupfoundry.com': 0.9237690021040942,-'blog.fastmail.fm': 0.923729095971477,-'wit.ai': 0.9237161044120704,-'www.devttys0.com': 0.923715140193862,-'www.qubes-os.org': 0.9237100158767054,-'www.robinsloan.com': 0.9236869500281771,-'timetobleed.com': 0.9236836576824262,-'blog.jitbit.com': 0.9236712552599629,-'gowers.wordpress.com': 0.9236675842265081,-'www.pnas.org': 0.9236623882869138,-'mrale.ph': 0.9236621608796006,-'play.freeciv.org': 0.9235744210893093,-'www.blackhat.com': 0.9235710982881401,-'laforge.gnumonks.org': 0.9234795250933899,-'marker.medium.com': 0.9234758009863355,-'www.sfchronicle.com': 0.9234493747928338,-'tutorial.math.lamar.edu': 0.9234097897078604,-'www.holovaty.com': 0.9233603823514369,-'www.cringely.com': 0.9232863734060375,-'www.folklore.org': 0.9232666029542594,-'apnews.com': 0.9232599354911131,-'devblogs.nvidia.com': 0.9231813399979636,-'www.linuxvoice.com': 0.9231576608319696,-'papers.ssrn.com': 0.9231181748415175,-'brianschrader.com': 0.9231068216939985,-'www.kreci.net': 0.923106349521949,-'www.nngroup.com': 0.9230948271379453,-'funcall.blogspot.com': 0.9230679265670205,-'www.jasonshen.com': 0.9230677977899461,-'lists.freedesktop.org': 0.9230222479984594,-'nodejs.org': 0.92301963271793,-'www.graalvm.org': 0.9230158730158731,-'blog.mozilla.com': 0.923006927671048,-'www.dailydot.com': 0.9230068718762484,-'lbrandy.com': 0.9229742803918614,-'erratasec.blogspot.com': 0.9229733794911794,-'www.chemistryworld.com': 0.9229661370816306,-'www.solipsys.co.uk': 0.9229220876704523,-'layerci.com': 0.9229159114633821,-'www.stellar.org': 0.9229142185891028,-'etherscan.io': 0.9228765015817684,-'www.macrumors.com': 0.922864948439789,-'vim-adventures.com': 0.9228321400159109,-'mailinator.blogspot.com': 0.9228075138073343,-'www.macleans.ca': 0.9227372105459439,-'blog.appcanary.com': 0.9227212708074342,-'www.stallman.org': 0.9227212708074342,-'briancarper.net': 0.9227017702961716,-'joehewitt.com': 0.9226894917334,-'sfconservancy.org': 0.9226892388952699,-'mariovittone.com': 0.9226475279106858,-'500hats.com': 0.9226369106547709,-'blog.tailwindcss.com': 0.9226012568165967,-'dilbert.com': 0.9226004268083031,-'mewo2.com': 0.9225891352707621,-'blog.getpebble.com': 0.9225467649630955,-'preshing.com': 0.922540762544208,-'www.medievalists.net': 0.9225218713450589,-'daily.jstor.org': 0.9225038762181162,-'www.lynalden.com': 0.922503215089193,-'hindenburgresearch.com': 0.9225025964121197,-'joindiaspora.com': 0.9224924773526838,-'chadfowler.com': 0.922481090652384,-'git.sr.ht': 0.9224628516537063,-'twitter.github.io': 0.9224447323362635,-'www.rackspace.com': 0.92242477632422,-'www.science.org': 0.9224216244734532,-'www.esquire.com': 0.9224172218178369,-'augustl.com': 0.9224131314072078,-'news.cornell.edu': 0.9223803694889144,-'paulrouget.com': 0.9223675768999002,-'joelgrus.com': 0.9222919914652804,-'electronjs.org': 0.9222709958305552,-'williamedwardscoder.tumblr.com': 0.9222429833639271,-'fastmail.blog': 0.9222426931980966,-'trixter.oldskool.org': 0.9222240643067603,-'palladiummag.com': 0.9221161388490351,-'www.dwmkerr.com': 0.9220928814524835,-'www.blueorigin.com': 0.9220497550322969,-'blogs.perl.org': 0.922041527915435,-'peter.bourgon.org': 0.9220345737360726,-'boats.gitlab.io': 0.9220270248890161,-'news.usc.edu': 0.9220177070210377,-'www.media.uzh.ch': 0.9220147439869917,-'rubini.us': 0.9220102711604998,-'ognjen.io': 0.9219863609200528,-'www.dragonflybsd.org': 0.9219689458962362,-'time.com': 0.9219350046913536,-'www.vagrantup.com': 0.9219259642840062,-'reclaimthenet.org': 0.921903148362967,-'blogs.lse.ac.uk': 0.921869679768062,-'threader.app': 0.9218497168940121,-'pgdash.io': 0.9217440948884412,-'www.namecheap.com': 0.9217409250741742,-'www.mtgox.com': 0.9217171114471724,-'blog.filepicker.io': 0.9217169081929824,-'mullvad.net': 0.9216909513671212,-'m.signalvnoise.com': 0.9216188131338918,-'gamerant.com': 0.9215933437650176,-'www.computerhistory.org': 0.9215810214981082,-'www.bennettnotes.com': 0.9215678399791286,-'codegolf.stackexchange.com': 0.9215396697671643,-'www.amnesty.org': 0.9214886045349302,-'quip.com': 0.9214846662604561,-'www.balsamiq.com': 0.9214784555047167,-'www.conwaylife.com': 0.9214510848919855,-'quoteinvestigator.com': 0.9214277867192284,-'www.stephanboyer.com': 0.9214274950425619,-'www.ibm.com': 0.9214034975155869,-'kkovacs.eu': 0.9213904944480831,-'www.cs.virginia.edu': 0.9213138750457075,-'blog.heapanalytics.com': 0.9212598460858725,-'www.khronos.org': 0.9212580247777246,-'infotrope.net': 0.9212298613344002,-'r6.ca': 0.9212165287852896,-'elemental.medium.com': 0.9211754093730133,-'standardnotes.org': 0.9211590022701598,-'www.thanassis.space': 0.9211469562862422,-'www.essential.com': 0.9211382113821138,-'blog.checkpoint.com': 0.9210707601354348,-'firebase.googleblog.com': 0.9210648800435821,-'theamericanscholar.org': 0.9210499166083035,-'mxb.dev': 0.9210070283654466,-'arrow.apache.org': 0.9209592797673791,-'moderncrypto.org': 0.9209488760965047,-'edition.cnn.com': 0.9209449218392389,-'penny-arcade.com': 0.9208903657692026,-'www.gresearch.co.uk': 0.9208877326699497,-'news.rapgenius.com': 0.9208723565849319,-'duartes.org': 0.9208704095563033,-'access.redhat.com': 0.9208698327943827,-'dealbook.nytimes.com': 0.9208660000281566,-'www.robinwieruch.de': 0.9208515295663545,-'blog.viktomas.com': 0.9208241650543216,-'www.thenation.com': 0.9208156841051072,-'evilmartians.com': 0.9207891062516377,-'www.unixsheikh.com': 0.9207398153686466,-'david.weebly.com': 0.9207376999076698,-'lanyrd.com': 0.9207234118522407,-'www.slowboring.com': 0.9206921783803773,-'unix.stackexchange.com': 0.9206801573666252,-'seldo.com': 0.9206416671015177,-'www.elischiff.com': 0.920635133939363,-'www.arrl.org': 0.9206234339791557,-'www.spacetelescope.org': 0.9205965394020418,-'writing.markchristian.org': 0.9205569205569206,-'elaineou.com': 0.9205345242508725,-'blog.lastpass.com': 0.9205320290125674,-'netzpolitik.org': 0.9205194544865914,-'ghuntley.com': 0.9205047884758858,-'knightcolumbia.org': 0.9205029147967208,-'www.comparitech.com': 0.9204765637809545,-'scanlime.org': 0.9204328320221652,-'www.sfgate.com': 0.920422056660529,-'eklitzke.org': 0.9203820542379546,-'www.scala-lang.org': 0.9203673159816037,-'people.eecs.berkeley.edu': 0.9203504139456036,-'bits.blogs.nytimes.com': 0.9203453783143634,-'floooh.github.io': 0.9203399579895484,-'obscuritory.com': 0.9203310173661203,-'habitatchronicles.com': 0.9203279953622738,-'www.giftrocket.com': 0.9203035040887337,-'www.debugbear.com': 0.9202964328943275,-'makezine.com': 0.920278418856363,-'www.slideshare.net': 0.920271947086287,-'sandlab.cs.uchicago.edu': 0.9202302631578948,-'timkadlec.com': 0.9202082523474097,-'momentjs.com': 0.9201811740344046,-'www.euractiv.com': 0.9201333156342562,-'www.daniellemorrill.com': 0.9201158212493131,-'writings.stephenwolfram.com': 0.9200560602651936,-'www.gkogan.co': 0.9200121929029601,-'needwant.com': 0.9199907538489269,-'www.medialens.org': 0.9199880352359148,-'blog.wepay.com': 0.9199344757712936,-'maps.google.com': 0.9199182118968083,-'www.dustmop.io': 0.9199009083402148,-'www.comp.nus.edu.sg': 0.9198995227263196,-'wiki.ubuntu.com': 0.9198686814026275,-'remotebase.io': 0.9198347107438016,-'www.37signals.com': 0.9198326817403155,-'alumni.media.mit.edu': 0.9198312121350687,-'www.smashingmagazine.com': 0.9197923257499647,-'kylehalladay.com': 0.9197075896809243,-'fortune.com': 0.9196957257034758,-'fosterelli.co': 0.9196146019753699,-'www.braveclojure.com': 0.9196013394110684,-'anvaka.github.io': 0.919572990689544,-'nomasters.io': 0.9195688225538972,-'reasonstobecheerful.world': 0.9195521506144941,-'thonyc.wordpress.com': 0.9195493663014705,-'x264dev.multimedia.cx': 0.9195307216192476,-'www.androidpolice.com': 0.9194976713702179,-'www.ribbonfarm.com': 0.919494859687766,-'workplace.stackexchange.com': 0.9194815293298146,-'jcs.org': 0.9194811525194942,-'www.codusoperandi.com': 0.9194731766129004,-'behind-the-enemy-lines.blogspot.com': 0.9194452174142858,-'peternixey.com': 0.9194135955492282,-'www.theplayerstribune.com': 0.9194126818562519,-'basho.com': 0.9194014734604079,-'www.curiousgnu.com': 0.9193682460515379,-'m.imgur.com': 0.9193616517608175,-'gafferongames.com': 0.9193033055945021,-'www.idontplaydarts.com': 0.9192963504549827,-'www.forrestthewoods.com': 0.9192741785228133,-'earthquake.usgs.gov': 0.9192641098431819,-'blog.bitpay.com': 0.9192625409487661,-'www.brookings.edu': 0.9192507068377601,-'18f.gsa.gov': 0.9192284715908775,-'www.minecraftforum.net': 0.9192112937034953,-'www.lyft.com': 0.9191683431386107,-'hhvm.com': 0.9191453265158583,-'blog.wolfire.com': 0.9191450229699666,-'www.usatoday.com': 0.9191347594941367,-'nbviewer.ipython.org': 0.9191131949298217,-'reddit.com': 0.9190800965203674,-'www.eenewseurope.com': 0.9190688625693296,-'www.smh.com.au': 0.9190104812043022,-'www.figma.com': 0.9189780967661552,-'www.yubico.com': 0.9189671558387075,-'larrysanger.org': 0.9189574986097159,-'www.cs.berkeley.edu': 0.9189369697206728,-'www.qualys.com': 0.9188953942561601,-'www.charlieharrington.com': 0.9188769529436779,-'casual-effects.com': 0.91886846135451,-'alexkrupp.typepad.com': 0.9188063903443455,-'blog.macromates.com': 0.9187968191722226,-'beza1e1.tuxen.de': 0.9187934225268114,-'www.marketwatch.com': 0.9187035877780302,-'dragan.rocks': 0.9186972641760346,-'scottmeyers.blogspot.com': 0.9186922599750114,-'brson.github.io': 0.9186844191523297,-'kevq.uk': 0.9186712741984764,-'tuhdo.github.io': 0.9186408903885764,-'blog.ploeh.dk': 0.918640800603575,-'www.spoon-tamago.com': 0.918580162432909,-'www.maizure.org': 0.9185797770662466,-'californiaglobe.com': 0.9185558354324097,-'yarchive.net': 0.9185157064745997,-'helpx.adobe.com': 0.918505551895513,-'sourceforge.net': 0.9184879275921072,-'gail.com': 0.9184873949579831,-'snikt.net': 0.9184873949579831,-'blog.plaid.com': 0.9184773866383011,-'ebb.org': 0.9184174897915893,-'insomnia.rest': 0.9184173422143894,-'www.biorxiv.org': 0.9184172443974019,-'cmdrtaco.net': 0.91840628090816,-'tom7.org': 0.9183916897269802,-'taler.net': 0.9183916897269802,-'docs.rs': 0.9183858344184437,-'www.ioccc.org': 0.9183852068279685,-'sites.google.com': 0.9183471189460477,-'domainnamewire.com': 0.918330600390645,-'dafoster.net': 0.9183054021034863,-'spreadsheets.google.com': 0.9182933889184417,-'www.abc.net.au': 0.9182274259544088,-'www.boringcompany.com': 0.918212478920742,-'www.bankofengland.co.uk': 0.9181974227639322,-'blog.ntpsec.org': 0.9181951887885417,-'huyenchip.com': 0.9181777870982438,-'pganalyze.com': 0.9181448902734191,-'startupschool.org': 0.9181360100179967,-'bl.ocks.org': 0.9180840436489085,-'codeamigo.dev': 0.9180588577925552,-'www.lesswrong.com': 0.9179747514962872,-'blog.x.company': 0.9179518350159086,-'inews.co.uk': 0.9179395572482176,-'www.talyarkoni.org': 0.9179286455845682,-'amazon.com': 0.917918074063271,-'pijul.org': 0.9179029713542893,-'www.thegatesnotes.com': 0.917896287388944,-'blogs.reuters.com': 0.9178784513377969,-'bartongeorge.io': 0.9178137271960294,-'arslan.io': 0.9178051685037625,-'bytecodealliance.org': 0.917754880834866,-'journal.paul.querna.org': 0.9177358613483362,-'jasonlbaptiste.com': 0.9177192783299879,-'www.sharelatex.com': 0.9176170787075834,-'www.jessesquires.com': 0.9176086328709556,-'iquilezles.org': 0.9175867575102823,-'yeokhengmeng.com': 0.9175790835471582,-'software.rajivprab.com': 0.9175657081772484,-'news.utexas.edu': 0.917544773761157,-'www.fbi.gov': 0.9175354761400789,-'sirupsen.com': 0.9175322740193886,-'recode.net': 0.9175305114277775,-'www.studyfinds.org': 0.9175005975481785,-'rdegges.com': 0.9174949072553962,-'cemerick.com': 0.9174490483219848,-'cr.openjdk.java.net': 0.9174209932539029,-'mapbox.com': 0.9174119653310046,-'www.lihaoyi.com': 0.917356761960145,-'www.ams.org': 0.9173555467413949,-'golem.ph.utexas.edu': 0.9173444265784106,-'www.oilshell.org': 0.9173296046923102,-'www.darktable.org': 0.9173295566709162,-'www.madinamerica.com': 0.9173266085137967,-'blog.scrapinghub.com': 0.9173116907321415,-'www.sheshbabu.com': 0.9172725950029555,-'nickdrozd.github.io': 0.9172699096194682,-'diveintomark.org': 0.9172600663351143,-'modernfarmer.com': 0.9172547499030632,-'news.ucsc.edu': 0.9172085056879004,-'blogs.hbr.org': 0.9171843582580473,-'opinionator.blogs.nytimes.com': 0.9171826095986618,-'www.rtl-sdr.com': 0.9171698455400066,-'byuu.org': 0.9171665984622892,-'bugzilla.gnome.org': 0.9171648163962425,-'www.sfwa.org': 0.9171648163962425,-'www.spinellis.gr': 0.9171319512518645,-'0pointer.net': 0.9171049662814222,-'eff.org': 0.917104278613718,-'learnxinyminutes.com': 0.9170978082207938,-'9to5google.com': 0.917097241243007,-'www.cs.columbia.edu': 0.9170952792848854,-'www.fourmilab.ch': 0.9170935424089711,-'hynek.me': 0.9170881237147662,-'www.sarahmei.com': 0.9170796812591611,-'blog.piekniewski.info': 0.9170369069517826,-'cb.vu': 0.9169587328599824,-'math.stackexchange.com': 0.916937205478264,-'news.bloomberglaw.com': 0.9169353599665077,-'blog.darknedgy.net': 0.9169312054506662,-'telegra.ph': 0.9168505250691076,-'sniperinmahwah.wordpress.com': 0.9168254460707557,-'lists.swift.org': 0.9167939569922435,-'adit.io': 0.9167855816882253,-'steamcommunity.com': 0.9167584498934136,-'www.buzzfeed.com': 0.9167426632611669,-'www.privacyinternational.org': 0.9167293554781794,-'www.thurrott.com': 0.9167249467906359,-'www.cbsnews.com': 0.9167194231229859,-'socket.io': 0.9166902296520829,-'maryrosecook.com': 0.9166809726360262,-'tomayko.com': 0.9166652528198918,-'if50.substack.com': 0.9166328442343356,-'phenomena.nationalgeographic.com': 0.9166322627474873,-'www.netmeister.org': 0.9166258322005095,-'transition.fcc.gov': 0.9165981621015247,-'playground.tensorflow.org': 0.9165966237016168,-'www.macworld.com': 0.916575278474562,-'www.mahdiyusuf.com': 0.9165626020992179,-'nplusonemag.com': 0.9165533544728419,-'astralcodexten.substack.com': 0.9165406144271586,-'rjlipton.wordpress.com': 0.9164585931916936,-'kunle.app': 0.9164517870748075,-'cheatsheetseries.owasp.org': 0.9164513350559862,-'blogs.janestreet.com': 0.916405195738336,-'www.chromium.org': 0.9163955502793474,-'aht.seriouseats.com': 0.9163820584014595,-'arcadeblogger.com': 0.9162907826609931,-'lock.cmpxchg8b.com': 0.9162763903656581,-'codewithoutrules.com': 0.9162515056697728,-'blog.roboflow.ai': 0.9162348877374784,-'jolla.com': 0.9162125492359456,-'philipwalton.github.io': 0.9162125492359456,-'www.textfiles.com': 0.9161483132429212,-'landshark.io': 0.9160843328162562,-'www.syfy.com': 0.9160823277886534,-'www.a1k0n.net': 0.9160533176276322,-'thirtybees.com': 0.916017316017316,-'wp.josh.com': 0.9159844747688354,-'community.rapid7.com': 0.9159729296552184,-'variety.com': 0.9159645406845424,-'www.petekeen.net': 0.9159458498387062,-'pragprog.com': 0.9158994769642369,-'addons.mozilla.org': 0.9158845258968069,-'blog.abevoelker.com': 0.9158703069575684,-'uxdesign.cc': 0.915853392301296,-'tomblomfield.com': 0.91585125248835,-'www.platformer.news': 0.9158262569189228,-'airbnb.design': 0.9158145365560538,-'www.php.net': 0.9157901084713228,-'blakemasters.tumblr.com': 0.915774364872077,-'forum.bitcoin.org': 0.9157050312564461,-'victorzhou.com': 0.9156996812390578,-'www.newcriterion.com': 0.9156866313823816,-'blog.sagemath.com': 0.9156636337212226,-'security-center.intel.com': 0.9156401908652926,-'secret.club': 0.9156139932226358,-'codebutler.com': 0.9156072900794726,-'gfx-rs.github.io': 0.9155911821045295,-'open.blogs.nytimes.com': 0.9155276050596644,-'lambda-the-ultimate.org': 0.9155112419708785,-'www.universetoday.com': 0.9155071362428421,-'vc.gg': 0.9155052264808362,-'www.bostonglobe.com': 0.9154711894014951,-'slides.com': 0.915438865746552,-'openid.net': 0.9154218308800682,-'sizovs.net': 0.9154120001317235,-'minimalmac.com': 0.9154065991271528,-'pandodaily.com': 0.9153369387847924,-'www.ft.com': 0.9153363776443395,-'money.cnn.com': 0.9153197950917878,-'blogs.technet.com': 0.9152916278406317,-'tynan.com': 0.9152784227275973,-'antipolygraph.org': 0.9152601338015678,-'www.bing.com': 0.9152555063347714,-'dev.lemmy.ml': 0.9152501219202281,-'ai.facebook.com': 0.9151989015144473,-'www.spectator.co.uk': 0.9151910833402672,-'bollu.github.io': 0.915178606993226,-'cognitivemedium.com': 0.9151678416414023,-'www.nih.gov': 0.9151430151901143,-'www.cyanogenmod.org': 0.9151094600863039,-'codepen.io': 0.9150991029481277,-'www.media.mit.edu': 0.9150898590392416,-'ianthehenry.com': 0.9150718483883374,-'sloanreview.mit.edu': 0.9150432572484367,-'speakerdeck.com': 0.9150398484887767,-'en.lichess.org': 0.9150022514278866,-'www.datagenetics.com': 0.9150012436762652,-'people.kernel.org': 0.9149910445768656,-'ma.tt': 0.9149514110201404,-'people.freebsd.org': 0.9149417112190068,-'measuringmeasures.com': 0.9149271022386467,-'www.catb.org': 0.9149270125747468,-'defn.io': 0.9148816198946629,-'www.fastly.com': 0.914873343624634,-'www.eetimes.com': 0.9148540001117531,-'www.newrepublic.com': 0.9146889391211901,-'www.chicagomaroon.com': 0.9146877748460862,-'blog.openbloc.fr': 0.9146877748460862,-'blog.moertel.com': 0.9146704350942687,-'nathanbarry.com': 0.9146551473361769,-'bombich.com': 0.914654069788301,-'mojang.com': 0.9146416740750689,-'www.pgbovine.net': 0.914639262393428,-'www.bicycling.com': 0.9146271639157105,-'www.zemanta.com': 0.9146271639157105,-'www.perell.com': 0.914609307465025,-'mjtsai.com': 0.9145962250738603,-'www.softwarebyrob.com': 0.9145691937602046,-'fusion.net': 0.914564049263131,-'blogs.forbes.com': 0.914542164424382,-'beta.observablehq.com': 0.9145307006213452,-'www.zedshaw.com': 0.9145061280161968,-'apps.npr.org': 0.9144928315823566,-'lists.gnupg.org': 0.9144675565289525,-'gleam.run': 0.9144324984811291,-'www.thestar.com': 0.9143957132161558,-'www.militarytimes.com': 0.9143764543930346,-'blog.memsql.com': 0.9143635601136089,-'cr.yp.to': 0.9143445920251045,-'thepointsguy.com': 0.9143303279494647,-'dinosaurpictures.org': 0.9143080414361231,-'quillette.com': 0.9142948614738509,-'ticki.github.io': 0.914276003611421,-'neil.fraser.name': 0.9142481924113248,-'www3.nhk.or.jp': 0.9142479994563674,-'www.muppetlabs.com': 0.9142412765407236,-'blogs.gnome.org': 0.9142360167665817,-'readonlymemory.vg': 0.9142351900972591,-'news.osu.edu': 0.9142251289181204,-'www.cgl.uwaterloo.ca': 0.9142062865582451,-'www.fossil-scm.org': 0.9142014076198545,-'hub.jhu.edu': 0.9141963295065768,-'raw.github.com': 0.9141853922096216,-'www.sebastianmarshall.com': 0.9141316785617728,-'physicstoday.scitation.org': 0.9141308856828069,-'www.edge.org': 0.9140771607780698,-'blog.bugreplay.com': 0.914047971434993,-'www.medrxiv.org': 0.9139991128038881,-'permalink.gmane.org': 0.9139842285208853,-'www.notechmagazine.com': 0.9139648152317646,-'blog.ipfs.io': 0.9139402591654684,-'ninjasandrobots.com': 0.9139402119313542,-'blog.johnnovak.net': 0.9139144009903326,-'tldr.sh': 0.9138931911912991,-'internals.rust-lang.org': 0.9138888662706784,-'formidable.com': 0.9138769104260843,-'thenounproject.com': 0.9138746646560476,-'blog.earbits.com': 0.9138048995152518,-'gamehistory.org': 0.9137927637033101,-'www.influxdata.com': 0.913761562076886,-'m.washingtonpost.com': 0.9137051021098685,-'pythonspeed.com': 0.9137019640744012,-'observer.com': 0.9136945429590685,-'mastodon.social': 0.9136829141356647,-'whatever.scalzi.com': 0.9136722362354969,-'rietta.com': 0.9136487147921646,-'cushionapp.com': 0.9136267087023814,-'www.cantorsparadise.com': 0.9135854914860095,-'www.intelligence.senate.gov': 0.9135819523129652,-'blog.nullspace.io': 0.9134635980748237,-'keleshev.com': 0.9134557014620313,-'www.freethink.com': 0.913438665510595,-'singularityhub.com': 0.9134225553478318,-'www.greenteapress.com': 0.9134066096671126,-'hazlitt.net': 0.9134026214327062,-'www.zainrizvi.io': 0.913399178678123,-'www.welcometothejungle.co': 0.9133545417162321,-'joeduffyblog.com': 0.9133528454968176,-'tbbuck.com': 0.913350699282715,-'blog.fawny.org': 0.9133445360688772,-'cyclingtips.com': 0.9133347976741121,-'www.f-secure.com': 0.913329909915946,-'www.tweag.io': 0.9133006601895646,-'edri.org': 0.9132989780194611,-'brooker.co.za': 0.913269877818758,-'learnyouahaskell.com': 0.9132658585002156,-'dmitrysoshnikov.com': 0.9132544931220842,-'frankgroeneveld.nl': 0.9132379248658319,-'inconvergent.net': 0.9132264356820329,-'kennethreitz.org': 0.913224092229413,-'editorconfig.org': 0.9131943780375904,-'atom.io': 0.9131691418622392,-'mattgemmell.com': 0.9131626341114434,-'www.fastcoexist.com': 0.9131427628529503,-'atap.google.com': 0.9130249616995455,-'esbuild.github.io': 0.9130044843049328,-'blog.koehntopp.info': 0.9129670374057173,-'europa.eu': 0.9129663426547358,-'htmx.org': 0.912955421768738,-'www.markozivanovic.com': 0.912917338189758,-'number-none.com': 0.9129109262881329,-'sequoia-pgp.org': 0.9128836629372516,-'tanelpoder.com': 0.9128836629372516,-'www.nvidia.com': 0.9128613184383916,-'www.prospectmagazine.co.uk': 0.9128286921650719,-'bouk.co': 0.9128166963356691,-'www.espn.com': 0.9127399070748246,-'www.courtlistener.com': 0.912716064588026,-'lav.io': 0.9126813150141094,-'landave.io': 0.9126647018297601,-'blog.phusion.nl': 0.912650575320633,-'thezvi.wordpress.com': 0.9126499253271542,-'nickjanetakis.com': 0.9126192502614521,-'stateofjs.com': 0.9126077344903017,-'hakim.se': 0.9126077344903017,-'blog.datomic.com': 0.9125894031840723,-'docs.racket-lang.org': 0.912488235007798,-'jamie-wong.com': 0.9124775693621691,-'mocko.org.uk': 0.9124699553919461,-'www.telegraph.co.uk': 0.9124491411165716,-'ag.ny.gov': 0.9124354337525871,-'billwadge.wordpress.com': 0.9124293382023613,-'www.matasano.com': 0.9124000351892007,-'okigiveup.net': 0.9123862469871238,-'logicmag.io': 0.9123859846898301,-'andrewchen.co': 0.9123448304091328,-'www.airpair.com': 0.9123437098236648,-'cowlark.com': 0.9123304746459454,-'www.pbs.org': 0.9123204159144807,-'semianalysis.com': 0.912313946546377,-'kk.org': 0.9123097221722443,-'log.nadim.cc': 0.9122965641952984,-'www.collaborativefund.com': 0.9122890867111917,-'www.ventusky.com': 0.912268380036006,-'www.nintendo.com': 0.9122375993317341,-'www.worldofindie.co.uk': 0.9122336940304385,-'jeroenjanssens.com': 0.9122336940304385,-'libreboot.org': 0.9122148556890368,-'blog.roboflow.com': 0.9121810251169891,-'www.msn.com': 0.9121613167344725,-'debugger.medium.com': 0.9121472531001502,-'www.pagetable.com': 0.9121460025296508,-'www.eejournal.com': 0.9121137219392826,-'blog.sanctum.geek.nz': 0.91211049118521,-'blog.elcomsoft.com': 0.9120714505588577,-'www.judegomila.com': 0.9120548967590474,-'cirw.in': 0.9119924579917503,-'corner.squareup.com': 0.9119829497403208,-'flightfox.com': 0.911934167943058,-'tip.golang.org': 0.9119192289295014,-'www.nzherald.co.nz': 0.9119181580628049,-'www.wezm.net': 0.911916042600115,-'crontab.guru': 0.9119103202923069,-'degoes.net': 0.9119022003821793,-'investors.modernatx.com': 0.9118967098234642,-'linustechtips.com': 0.9118886368935806,-'blog.tanyakhovanova.com': 0.911882662223509,-'amandapeyton.com': 0.9118468718524512,-'nypost.com': 0.9118217286764665,-'forumserver.twoplustwo.com': 0.9118191670930907,-'belkadan.com': 0.9117907816852344,-'nicolasgallagher.com': 0.9117507194110998,-'leowid.com': 0.9117097398796832,-'blog.izs.me': 0.9117080886266495,-'www.electrospaces.net': 0.9117024618532766,-'granta.com': 0.9116811248127611,-'blog.darksky.net': 0.911673049778484,-'lettersofnote.com': 0.9116636714221488,-'www.webkit.org': 0.9115845605402175,-'shipilev.net': 0.9115598314538631,-'www.nanex.net': 0.911539637720508,-'blog.whatsapp.com': 0.9115335907580295,-'leetcode.com': 0.9115051248737622,-'www.numfocus.org': 0.9115022081675324,-'onlinelibrary.wiley.com': 0.9114852476099333,-'www.tomscott.com': 0.9114829485455537,-'trmm.net': 0.9114358511601164,-'www.marketplace.org': 0.9113899909762714,-'www.codecademy.com': 0.911385022367728,-'news.vice.com': 0.9113079103997874,-'bit-player.org': 0.9112589244732427,-'postmake.io': 0.9112553086495423,-'journal.standardnotes.org': 0.9112172605590297,-'semantic-ui.com': 0.9111938765485481,-'blog.dshr.org': 0.9111782499363091,-'www.rogerebert.com': 0.9111691070786543,-'web.dev': 0.9111397506487405,-'bravenewgeek.com': 0.9111366238548788,-'benjiweber.co.uk': 0.9111133306489915,-'www.theinformation.com': 0.9111027630703883,-'jmmv.dev': 0.9110762745053466,-'www.nycresistor.com': 0.9110654315146017,-'bloomberg.github.io': 0.9110576284941153,-'glench.com': 0.9110421711549385,-'blog.kicksend.com': 0.9109887018713544,-'www.godaddy.com': 0.9109502513393147,-'neverworkintheory.org': 0.910947601730565,-'thebillfold.com': 0.9109433900278253,-'www.superhighway98.com': 0.9109274563820018,-'tylerneylon.com': 0.9109274563820018,-'gankro.github.io': 0.9109257416809919,-'thetruesize.com': 0.9108778939911146,-'zipcpu.com': 0.9108728406864637,-'www.cs.uni.edu': 0.910867930832846,-'www.whitehouse.gov': 0.9108673306488233,-'bradleytaunt.com': 0.9108617254932104,-'scripting.com': 0.9108577372151384,-'www.cybertec-postgresql.com': 0.9108466966632626,-'www.wcvb.com': 0.9107985466943099,-'www.dslreports.com': 0.9107964183891935,-'justi.cz': 0.9107667989469016,-'finance.yahoo.com': 0.9107655094145327,-'blog.data.gov.sg': 0.910764849456426,-'www.gutenberg.org': 0.9107167501810286,-'www.sciencedaily.com': 0.9106989174929755,-'blog.airbnb.com': 0.9106859675193685,-'www.sendwithus.com': 0.9106524760338102,-'chrismm.com': 0.9106400698754001,-'stevecorona.com': 0.9106182479757787,-'swanson.github.com': 0.910602900714108,-'eager.io': 0.9105915511196,-'www.insidesources.com': 0.9105882590524582,-'chrisseaton.com': 0.9105861690903986,-'cstheory.stackexchange.com': 0.9105756115852348,-'www.behind-the-enemy-lines.com': 0.9105542219435652,-'www.commondreams.org': 0.9105378564647107,-'tech.marksblogg.com': 0.91050910005871,-'blog.bitquabit.com': 0.9104538788044407,-'utkusen.com': 0.9104251525129977,-'lab.hakim.se': 0.9104228721342874,-'calacanis.com': 0.9104018184523797,-'blog.metaobject.com': 0.9103873178112856,-'blogs.discovermagazine.com': 0.9103541727829514,-'flutter.io': 0.9103411596660869,-'www.chriskrycho.com': 0.9103411596660869,-'www.washingtonmonthly.com': 0.9103284229318426,-'nora.codes': 0.9103242591900224,-'blog.professorbeekums.com': 0.9103190381466665,-'yro.slashdot.org': 0.9103086060368418,-'noncombatant.org': 0.9102799785330806,-'www.irrlicht3d.org': 0.9102781052287677,-'www.analog.com': 0.9102635806004643,-'bret.appspot.com': 0.9102570367702293,-'byuu.net': 0.9102490436100149,-'learntemail.sam.today': 0.9102460058415499,-'discuss.joelonsoftware.com': 0.9102271772174491,-'news.samsung.com': 0.9102218695714285,-'hamberg.no': 0.9101858671649673,-'culture.pl': 0.9101776992266171,-'godbolt.org': 0.9101759214909476,-'www.usgamer.net': 0.9101501554941474,-'openmymind.net': 0.9101212855439798,-'www.pgcon.org': 0.9101151487041538,-'blog.insightdatascience.com': 0.9101089594835049,-'blog.mapbox.com': 0.9100815042120244,-'heap.io': 0.9100756276392993,-'www.openttd.org': 0.910035809777446,-'www.gazehawk.com': 0.9100190649304827,-'blog.headius.com': 0.9100084850499096,-'jquerymobile.com': 0.9099865878820667,-'strlen.com': 0.9099808992547239,-'bellmar.medium.com': 0.9099735750567878,-'designfestival.com': 0.9099647154391645,-'blog.kowalczyk.info': 0.909946343995861,-'natewienert.com': 0.9099163091164866,-'blog.iron.io': 0.9099140278683325,-'www.optaplanner.org': 0.9098736627179512,-'blog.directededge.com': 0.9098673466271318,-'www.acm.org': 0.9098204575391435,-'postgres.heroku.com': 0.9098008938162723,-'netflix.github.io': 0.9097922949542527,-'www.upguard.com': 0.9097914092921688,-'dopeboy.github.io': 0.9097690974580366,-'www.panix.com': 0.909767441860465,-'www.arangodb.com': 0.9097488818296668,-'www.ctvnews.ca': 0.9097251357478893,-'superorganizers.substack.com': 0.9097041628130547,-'tips.ariyh.com': 0.9096731009765122,-'henrikwarne.com': 0.9096514841360387,-'web.media.mit.edu': 0.909634017448133,-'www.math.columbia.edu': 0.9096124004726523,-'sorbet.org': 0.9095992544268405,-'www.whattofix.com': 0.9095812789302404,-'lukeplant.me.uk': 0.90953747228724,-'blog.quora.com': 0.9095278802240517,-'areallybadidea.com': 0.909495363691211,-'www.gsb.stanford.edu': 0.9094535928364683,-'rednuht.org': 0.9094069027053294,-'cappuccino.org': 0.9093755871303836,-'blog.wallaroolabs.com': 0.9093612256561168,-'resobscura.blogspot.com': 0.9093403026878701,-'story.californiasunday.com': 0.9093105225503313,-'xato.net': 0.9092732945858745,-'thinkfaster.co': 0.9092409956171542,-'blog.algorithmia.com': 0.9091584066166702,-'n-o-d-e.net': 0.9091578728445029,-'www.talkingquickly.co.uk': 0.9091487127436086,-'blamcast.net': 0.9091487127436086,-'www.opendemocracy.net': 0.9091030079275115,-'www.pixelbeat.org': 0.9091017229185366,-'blog.2ndquadrant.com': 0.9090827165871289,-'klinger.io': 0.9090715471566316,-'invisibleup.com': 0.9090623220718969,-'www.mattgreer.org': 0.909028070014661,-'www.georgehart.com': 0.9090164458580612,-'gsuiteupdates.googleblog.com': 0.9090066640642879,-'programming.guide': 0.9090056285178236,-'www.revealnews.org': 0.9089822305000187,-'www.android.com': 0.9089431208516167,-'renderman.pixar.com': 0.9089238890253597,-'swtch.com': 0.9088645421539494,-'akkartik.name': 0.9088447989502915,-'ma.ttias.be': 0.9088072198608288,-'blog.freecodecamp.com': 0.9087962710873405,-'cognitiontoday.com': 0.9087409016865039,-'data.triplebyte.com': 0.9087226720990824,-'unix4lyfe.org': 0.9087151101306639,-'coconut-lang.org': 0.9087151101306639,-'discourse.nixos.org': 0.908662900188324,-'macwright.org': 0.908628352520185,-'www.varnish-cache.org': 0.9086150127257842,-'blog.lyft.com': 0.9085807144025437,-'martinrue.com': 0.9085670862102617,-'bittersoutherner.com': 0.9085029987755043,-'www.somebits.com': 0.9084860999243902,-'certsimple.com': 0.9084684115974558,-'www.calvertjournal.com': 0.9084635529608529,-'www.pentestpartners.com': 0.9084628705092087,-'wiki.gentoo.org': 0.9084338073440062,-'00f.net': 0.9083970094532444,-'www.weforum.org': 0.9083913811504614,-'evanmiller.org': 0.9083652366260793,-'www.geekwire.com': 0.9083231743190484,-'news.yahoo.com': 0.9083103718085543,-'books.google.com': 0.9082816718436784,-'georgestocker.com': 0.9082719861623312,-'www.yacoset.com': 0.9082542163067842,-'spencerfry.com': 0.9082262546419613,-'citusdata.com': 0.9082125231004685,-'www.thenewatlantis.com': 0.9081731363479266,-'j2kun.svbtle.com': 0.9081723260580993,-'www.bothsidesofthetable.com': 0.9081408393460154,-'www.thetimes.co.uk': 0.9081193478982206,-'corecursive.com': 0.90809594719532,-'www.wildml.com': 0.9080846001869393,-'smallcultfollowing.com': 0.9080827648147168,-'www.theartnewspaper.com': 0.9080723328672826,-'cdixon.posterous.com': 0.9080713598958169,-'www.readwriteweb.com': 0.9080580806277859,-'www.hsph.harvard.edu': 0.9080327405123425,-'www.state.gov': 0.908024664450505,-'planscope.io': 0.907995411751062,-'www.allthingsdistributed.com': 0.9079938413302127,-'www.boston.com': 0.9079342359331115,-'blog.kentcdodds.com': 0.9079295161962991,-'www.brianstorti.com': 0.9079295161962991,-'scala-lang.org': 0.9079245106804908,-'austinvernon.eth.link': 0.9078822412155745,-'pindancing.blogspot.com': 0.9078822412155745,-'www.residentadvisor.net': 0.9078629004665392,-'www.documentcloud.org': 0.9078579855885078,-'googledocs.blogspot.com': 0.9078489828529417,-'www.cdc.gov': 0.9078393365891005,-'www.howardism.org': 0.9078286053408602,-'fsharpforfunandprofit.com': 0.9078054396581823,-'semiaccurate.com': 0.907755821837259,-'www.hetzner.com': 0.9077473776003605,-'aiweirdness.com': 0.9077065010884221,-'www.bit-101.com': 0.9077018983499333,-'www.41latitude.com': 0.9077018983499333,-'www.amnh.org': 0.9076987359242361,-'infolab.stanford.edu': 0.9076647094055436,-'stories.californiasunday.com': 0.9076472906657365,-'pxlnv.com': 0.9076015494709667,-'batsov.com': 0.9075988217886798,-'www.gamedev.net': 0.9075987374715156,-'diff.wikimedia.org': 0.9075899793873731,-'rust-analyzer.github.io': 0.9075741551452703,-'www.mondaynote.com': 0.9075302424153898,-'www.gatesfoundation.org': 0.9075167456992206,-'meta.ath0.com': 0.9075048785901592,-'radicle.xyz': 0.9074969769110395,-'www.gigamonkeys.com': 0.9074847578759488,-'thetyee.ca': 0.9074067773987987,-'www.ucdavis.edu': 0.9073906291556996,-'annehelen.substack.com': 0.9073906291556995,-'www.undeadly.org': 0.9073802859341632,-'teenage.engineering': 0.9073802859341632,-'approximatelycorrect.com': 0.9073547174968317,-'console.dev': 0.9073543457497613,-'www.qemu.org': 0.9073039659002148,-'chadaustin.me': 0.907285573667598,-'javascript.info': 0.9072810118016286,-'crawshaw.io': 0.9072713221321198,-'anandtech.com': 0.9072582150847296,-'minireference.com': 0.9072555275680507,-'0x00sec.org': 0.9072265870419588,-'jtaby.com': 0.9072263761646095,-'hackernoon.com': 0.9072112942336243,-'aem1k.com': 0.9071770334928229,-'www.theatlanticwire.com': 0.9071768956517269,-'voice.mozilla.org': 0.9071764184712818,-'www.tennessean.com': 0.9071055766821053,-'graphicallinearalgebra.net': 0.9070506542876967,-'jpmens.net': 0.9070445763259152,-'www.eugenewei.com': 0.907025376171475,-'blog.packagecloud.io': 0.9070193601765509,-'zef.me': 0.9070189250200639,-'blog.makezine.com': 0.907016130343466,-'yannesposito.com': 0.9069508537217187,-'ubuntu.com': 0.9069426319310011,-'apolloinrealtime.org': 0.9069028892973577,-'leontrolski.github.io': 0.9069028892973576,-'divinations.substack.com': 0.9068786540871128,-'nicole.express': 0.9068478175005559,-'www.cnet.com': 0.9068461959910697,-'docs.browserless.io': 0.9068163026535472,-'joshondesign.com': 0.906770998121308,-'humanparts.medium.com': 0.906760680978119,-'aviationweek.com': 0.9067465504035637,-'philip.greenspun.com': 0.9067439389898158,-'huonw.github.io': 0.9066784039500629,-'ariadne.space': 0.90667782006653,-'blog.sia.tech': 0.9066295330975407,-'www.cato.org': 0.9065844637081601,-'blog.hipchat.com': 0.9065819832684133,-'ruben.verborgh.org': 0.9065777308846613,-'www.straightdope.com': 0.9064957391706991,-'www.joshbarczak.com': 0.9064865112228077,-'www.aarongreenspan.com': 0.9064448754833254,-'james-iry.blogspot.com': 0.9064382639201024,-'changelog.com': 0.9064361162234437,-'jdlm.info': 0.9064121095346035,-'www.csail.mit.edu': 0.9064063607235981,-'blog.cleancoder.com': 0.9063898821262703,-'kyrobeshay.com': 0.9063750617475119,-'js1k.com': 0.9063646805868381,-'socketsite.com': 0.9063204907590037,-'mobile.reuters.com': 0.9063074578535042,-'www.spinics.net': 0.9062822710883404,-'www.h-online.com': 0.9062819868634641,-'humanwhocodes.com': 0.9062381797427127,-'angular.io': 0.9062269960472951,-'www.digitalcameraworld.com': 0.906205656831627,-'www.amd.com': 0.9061918994722282,-'noahpinion.substack.com': 0.9061897254478186,-'scienceblogs.com': 0.9061724154956764,-'forge.medium.com': 0.9061379433537395,-'folklore.org': 0.9061180195674341,-'www.jonathan-cook.net': 0.906117767771791,-'randomwire.com': 0.9060987415295255,-'blog.crunchydata.com': 0.9060970567080769,-'www.timeshighereducation.co.uk': 0.9060939645760167,-'gds.blog.gov.uk': 0.9060670828649058,-'www.mihaileric.com': 0.9060240156895187,-'seatgeek.com': 0.9059982211064962,-'www.fanfiction.net': 0.9059866760601158,-'msrc-blog.microsoft.com': 0.9059631330906113,-'thecounter.org': 0.9059275361638426,-'www.haproxy.com': 0.905913947863991,-'movingfulcrum.com': 0.9058292135077338,-'onstartups.com': 0.9057830668516696,-'noyb.eu': 0.9057682181268583,-'millionshort.com': 0.9057645582241304,-'blog.sketchapp.com': 0.905760488137004,-'www.tillett.info': 0.9057526849990746,-'community.ui.com': 0.9057461537102033,-'adnauseam.io': 0.9057337220602527,-'intoli.com': 0.9056718941128913,-'www.jsoftware.com': 0.9056617340683936,-'www.oreilly.com': 0.9056506837308099,-'www.themacro.com': 0.905632500892492,-'www.chicagotribune.com': 0.9055936474876807,-'developer.squareup.com': 0.9055714474003402,-'pixelambacht.nl': 0.9055660472352755,-'lambdaschool.com': 0.9055457567735196,-'consumerist.com': 0.9055167142071894,-'lisp-univ-etc.blogspot.com': 0.905513555501623,-'www.reaktor.com': 0.9055111900863794,-'mars.jpl.nasa.gov': 0.9054761782517211,-'blog.rescuetime.com': 0.9054099923733353,-'dustycloud.org': 0.9053737646709764,-'the.exa.website': 0.9053731049780873,-'www.lareviewofbooks.org': 0.905357669166342,-'adventures.michaelfbryan.com': 0.9053432603320545,-'bigtechnology.substack.com': 0.9053275639886895,-'mashable.com': 0.9052624670823227,-'blog.xamarin.com': 0.9052537100443511,-'www.micahlerner.com': 0.905230894226323,-'software.intel.com': 0.9052122500296891,-'canonical.org': 0.9051821187623978,-'www.scmp.com': 0.9051295599614926,-'grantland.com': 0.9050923790278581,-'metarabbit.wordpress.com': 0.9050923774541669,-'education.github.com': 0.9050362272771562,-'www.thelancet.com': 0.905021932673904,-'www.nextplatform.com': 0.9050024178186049,-'home-assistant.io': 0.904995020548573,-'varnish-cache.org': 0.9049950205485728,-'blog.netlab.360.com': 0.9049426199082281,-'m.nautil.us': 0.9049309527012956,-'www.wikileaks.org': 0.9049234922760648,-'www.cell.com': 0.9049048617612142,-'cheapskatesguide.org': 0.9048941298503079,-'engineering.tumblr.com': 0.9048881466624363,-'w3techs.com': 0.9048639649614062,-'literaryreview.co.uk': 0.9048564834641546,-'explorabl.es': 0.9048086359175662,-'blog.felipe.rs': 0.9048079619547811,-'bitsup.blogspot.com': 0.9047993533025517,-'unherd.com': 0.9047865592498257,-'garry.posterous.com': 0.9047577664612345,-'worthdoingbadly.com': 0.9047131145041029,-'rob.conery.io': 0.9046723461377625,-'io9.com': 0.9046401640309223,-'nolanlawson.com': 0.9046163101493089,-'www.soundguys.com': 0.9046093349153457,-'www.starterstory.com': 0.9045594173223862,-'blog.jse.li': 0.9045467732931323,-'knowingless.com': 0.9045071158772837,-'decrypt.co': 0.9044787320123616,-'www.euronews.com': 0.9044514611283094,-'www.mattwarren.org': 0.9044367882909422,-'www.nirsoft.net': 0.9044100097952366,-'www.atlassian.com': 0.9043691389685379,-'exploringjs.com': 0.9043580712164743,-'www.recurse.com': 0.9043507328694445,-'www.legacy.com': 0.9043392504930966,-'docs.fcc.gov': 0.9043351544730283,-'channel9.msdn.com': 0.904284603417082,-'blog.ksplice.com': 0.9042701981747182,-'www.valvesoftware.com': 0.9042649125194085,-'blog.netbsd.org': 0.904253667752545,-'hyper.sh': 0.9042360279323705,-'blog.waleedkhan.name': 0.9042360279323705,-'computerhistory.org': 0.904233366454177,-'sugru.com': 0.9041590518924904,-'feefighters.com': 0.9041509269810387,-'dejanseo.com.au': 0.9040498577824194,-'www.ianww.com': 0.9040340548815767,-'blog.cr.yp.to': 0.9040072201081354,-'trac.nginx.org': 0.9039603960396039,-'kentnguyen.com': 0.903957880171432,-'jonathanturley.org': 0.9039575565206787,-'snyk.io': 0.9039222826692356,-'radar.oreilly.com': 0.9038574383300398,-'blog.sentry.io': 0.9037725052372675,-'pluto.jhuapl.edu': 0.9037348670913641,-'calendar.perfplanet.com': 0.903704307455864,-'graphics.stanford.edu': 0.903685214892998,-'singhkays.com': 0.9036579185992312,-'tinyapps.org': 0.9036401962155587,-'motherjones.com': 0.9036341732104931,-'blog.evjang.com': 0.903604799270119,-'www.lightbluetouchpaper.org': 0.9035449087212127,-'blog.arduino.cc': 0.9035388791817426,-'www.psychnewsdaily.com': 0.9035015745269135,-'gnu.org': 0.9035002162125149,-'pchiusano.github.io': 0.9034726569621144,-'hypercritical.co': 0.9034345666188365,-'0a.io': 0.9034135558240236,-'www.erowid.org': 0.9034051526175023,-'www.inklestudios.com': 0.9033800880640788,-'blog.soylent.com': 0.9033111216571915,-'www.fpcomplete.com': 0.9032736546035143,-'dmitryfrank.com': 0.9032706083767721,-'www.red-lang.org': 0.9032700232946759,-'www.parallella.org': 0.9032534786832978,-'habr.com': 0.903232673933909,-'www.musictech.net': 0.9031936127744511,-'josephwalla.com': 0.9031899305005244,-'hsivonen.fi': 0.9031735375682458,-'thoughtcatalog.com': 0.9031627630658119,-'www.scylladb.com': 0.9031607915889202,-'www.dreamhost.com': 0.9031378273716731,-'www.stochasticlifestyle.com': 0.9031310549200046,-'vaibhavsagar.com': 0.9031083985391525,-'searchengineland.com': 0.9030654738013855,-'www.nrk.no': 0.9030651285810807,-'blog.app.net': 0.903029680517345,-'mail.openjdk.java.net': 0.9030249968896539,-'sciencebulletin.org': 0.9029854747384259,-'www.paperplanes.de': 0.9029834819452879,-'research.facebook.com': 0.9029781526945183,-'jessenoller.com': 0.9029740692225192,-'blog.hirelite.com': 0.9029584423609692,-'www.bemmu.com': 0.9029304388261952,-'www.cam.ac.uk': 0.9029214208630774,-'adguard.com': 0.9028940843182978,-'www.protocol.com': 0.9028937484894913,-'redislabs.com': 0.9028875063757023,-'www.coding2learn.org': 0.9028312810037816,-'www.tonywright.com': 0.9027993696652322,-'writing.kemitchell.com': 0.902794255100101,-'support.microsoft.com': 0.9027883582257822,-'voxeu.org': 0.9027732002536809,-'math.ucr.edu': 0.9027587862028252,-'gekk.info': 0.9026852677733105,-'www.dwitter.net': 0.9026795513672158,-'nakedsecurity.sophos.com': 0.9026433950295091,-'www.tomdispatch.com': 0.9026213839350323,-'www.erichgrunewald.com': 0.9025809741606529,-'everydayastronaut.com': 0.9025525737662192,-'meta.stackexchange.com': 0.9025397509956542,-'dashbit.co': 0.9025100393249755,-'kroah.com': 0.9025051904560609,-'www.terrybisson.com': 0.902478233349385,-'www.whatsapp.com': 0.9024176929829565,-'ploum.net': 0.9024105508563706,-'explosion.ai': 0.9024029708934638,-'blog.repl.it': 0.9024021393458092,-'neurosciencenews.com': 0.9023952036309971,-'www.supermemo.com': 0.9023729956597971,-'www.diyphotography.net': 0.902367985362208,-'kcsrk.info': 0.902365879005221,-'security.stackexchange.com': 0.9023603426250738,-'developer.android.com': 0.9023376538238216,-'blog.rjmetrics.com': 0.9023127105964952,-'kakoune.org': 0.902272263107977,-'www.saurik.com': 0.9022715997521633,-'thetoolsweneed.com': 0.902259888208641,-'atthis.link': 0.9022431859413726,-'hg.python.org': 0.9021998686737284,-'www.softwarepreservation.org': 0.9021625352128104,-'www.kleinbottle.com': 0.9021190716448031,-'interrupt.memfault.com': 0.9020903788101069,-'dspace.mit.edu': 0.9020251184851814,-'philipwalton.com': 0.9020062837472365,-'www.foxbusiness.com': 0.9019313178215171,-'jrsinclair.com': 0.9019295383152887,-'www.space.com': 0.9019268835645871,-'sachin.posterous.com': 0.9018811076664313,-'khn.org': 0.901879302131479,-'www.pigdog.org': 0.901878363289037,-'zackkanter.com': 0.9018673516010077,-'www.informationdiet.com': 0.9018348908028704,-'giantrobotlasers.com': 0.9017894645998003,-'incolumitas.com': 0.9017770940926804,-'jeffkreeftmeijer.com': 0.9017611553399866,-'bostonreview.net': 0.9017262602947111,-'techreport.com': 0.901717492257879,-'www.baldurbjarnason.com': 0.9017151044394945,-'www.sicpers.info': 0.9017134966165734,-'blog.sfgate.com': 0.9016737116837992,-'write.as': 0.9016704517485878,-'www.aljazeera.com': 0.9016695698160121,-'fr.slideshare.net': 0.9016442121790157,-'everything2.com': 0.9016361459958601,-'typicalprogrammer.com': 0.9015788203543227,-'blog.scaleway.com': 0.9015776579003968,-'rethrick.com': 0.9015770143955718,-'inference-review.com': 0.9015413236700288,-'thedorkweb.substack.com': 0.9015280289106077,-'blog.23andme.com': 0.9015240824972036,-'www.mikealche.com': 0.9015168230317929,-'hashcat.net': 0.9015094649406228,-'madebyloren.com': 0.9014851889844163,-'www.blockchain.com': 0.901483519454099,-'www.computerworld.com': 0.9014820166011037,-'cen.acs.org': 0.901480955026061,-'lauren.vortex.com': 0.9014724896133568,-'oreilly.com': 0.901407801663114,-'blog.netflix.com': 0.9013957945143364,-'commoncrawl.org': 0.9013596330544786,-'www.svds.com': 0.9013257789593407,-'blog.goodstuff.im': 0.9013257789593407,-'www.lhup.edu': 0.9013191983110588,-'www.nist.gov': 0.9013044488448885,-'jasonevanish.com': 0.9012724811591895,-'nextcity.org': 0.9012682451009019,-'nuitka.net': 0.9012059293191188,-'dataswamp.org': 0.9011871538800621,-'www.reactos.org': 0.9011785225363149,-'limpet.net': 0.9011626533660011,-'nextjournal.com': 0.9011584604272797,-'sinews.siam.org': 0.9011379162640358,-'blog.zactownsend.com': 0.9011275964672605,-'m.guardiannews.com': 0.9011260854499006,-'mazzo.li': 0.9011141262538125,-'news.developer.nvidia.com': 0.9011001587785914,-'spideroak.com': 0.901093458914619,-'www.startribune.com': 0.9010814985872827,-'www.gitbook.com': 0.901071077131811,-'www.brusselstimes.com': 0.9010360879334309,-'blog.instagram.com': 0.9010324035185349,-'benedikt-bitterli.me': 0.9009740364816841,-'fedoramagazine.org': 0.9009664628347622,-'www.rte.ie': 0.9009526229516198,-'f-droid.org': 0.9008856641605046,-'blog.dwolla.com': 0.9008628892479406,-'news.trust.org': 0.9008305066882848,-'about.bnef.com': 0.9007424598766725,-'txti.es': 0.9007231608685488,-'samizdat.mines.edu': 0.9007085795324656,-'www.murilopereira.com': 0.9007078611650351,-'bitcoincharts.com': 0.9006914299992029,-'material.io': 0.9006442988271292,-'openradar.appspot.com': 0.9006237134629408,-'blog.rapportive.com': 0.9006217110635638,-'m-cacm.acm.org': 0.9005996462860704,-'blog.halide.cam': 0.9005839293957032,-'www.billthelizard.com': 0.9005839293957032,-'news.sciencemag.org': 0.9005769469610003,-'www.edmunds.com': 0.900563544204038,-'wiki.c2.com': 0.9005588568752737,-'thinkvitamin.com': 0.9005542231541395,-'ipython.org': 0.900531415561299,-'panelbear.com': 0.9005250466106081,-'stuartsierra.com': 0.9004498846216558,-'clojurescript.org': 0.900443619125688,-'maps.org': 0.9004208529632491,-'www.dextronet.com': 0.9004014616042372,-'blog.digitalocean.com': 0.9003805295727227,-'www.happyassassin.net': 0.9003455623339318,-'blockchain.info': 0.9003439746043133,-'realm.io': 0.9003295505994889,-'unixsheikh.com': 0.9003091480910362,-'zyan.scripts.mit.edu': 0.9002640065392239,-'til.simonwillison.net': 0.9002517190240483,-'fingerprintjs.com': 0.9002278669918841,-'cs-blog.khanacademy.org': 0.9001663145026005,-'eugene.kaspersky.com': 0.9000762462446584,-'www.bay12games.com': 0.9000684308983997,-'ginzametrics.com': 0.9000536631561354,-'blog.simpleanalytics.com': 0.9000166910943967,-'viewfromthewing.com': 0.9000166910943967,-'liam-on-linux.livejournal.com': 0.9000090039034558,-'softwaremaniacs.org': 0.9000031026474421,-'blog.mariadb.org': 0.8999936863202533,-'levels.io': 0.899970180177946,-'craigmod.com': 0.8999690838572385,-'news.spotify.com': 0.8999500411279416,-'forums.developer.apple.com': 0.8999431500814804,-'www.overcomingbias.com': 0.8999429014404717,-'users.rust-lang.org': 0.8999355740659959,-'scrimba.com': 0.8999254626887211,-'www.sbnation.com': 0.8998688532688766,-'benswann.com': 0.89986648239432,-'www.thelocal.dk': 0.8998638303612118,-'pitchfork.com': 0.8998590033239674,-'wekeroad.com': 0.899851220900418,-'animalnewyork.com': 0.8998375739356855,-'beets.io': 0.899813377379441,-'www.ibiblio.org': 0.8997974933028597,-'kernelnewbies.org': 0.899793879009676,-'www.fullstackreact.com': 0.8997933884297521,-'ricostacruz.com': 0.8997816625141879,-'www.digitalocean.com': 0.8997784555641042,-'www.maps.org': 0.8997736250289394,-'pointersgonewild.com': 0.89975961274635,-'512pixels.net': 0.8997410869841632,-'blog.wilshipley.com': 0.899720920102866,-'devlog.hexops.com': 0.8997114101311218,-'blogs.nvidia.com': 0.899693028608083,-'www.sacbee.com': 0.8996905748131163,-'www.publicbooks.org': 0.8996781736043378,-'hforsten.com': 0.8996749694919105,-'okmij.org': 0.899647639382427,-'www.dataquest.io': 0.8996134651823179,-'www.nocsdegree.com': 0.8995491494660874,-'pandas.pydata.org': 0.8995416268122183,-'doctorow.medium.com': 0.8995206430959863,-'blog.ezyang.com': 0.8995116025900058,-'cs.gmu.edu': 0.8995004521044683,-'weblog.raganwald.com': 0.8994722109240123,-'finance.fortune.cnn.com': 0.8994585795276846,-'pedestrianobservations.com': 0.8994403946669056,-'dangrover.com': 0.8994295897503085,-'nostarch.com': 0.8994103848538745,-'foreignpolicy.com': 0.8994068719520655,-'benchmarksgame-team.pages.debian.net': 0.8993756538078823,-'kvark.github.io': 0.8993354696336303,-'www.scummvm.org': 0.8993014570629587,-'www.jpl.nasa.gov': 0.8992820116565705,-'theweek.com': 0.8992433138556716,-'status.heroku.com': 0.8992414299043588,-'tech.grammarly.com': 0.8992224733195932,-'www.mensjournal.com': 0.8992186596239703,-'braythwayt.com': 0.8992151653414675,-'clayallsopp.com': 0.8991984891927328,-'blog.japaric.io': 0.8991683991683992,-'www.linuxuprising.com': 0.8991634552022689,-'www.cambridge.org': 0.8991459176334309,-'pando.com': 0.8991426686559758,-'nextcloud.com': 0.8991240467738062,-'blog.luap.info': 0.8991106158419904,-'goto.ucsd.edu': 0.8990952211205248,-'layervault.tumblr.com': 0.8990927364776206,-'www.preetamnath.com': 0.8990436000092332,-'www.loper-os.org': 0.8990428163259621,-'beta.latimes.com': 0.8990201091709796,-'status.linode.com': 0.8990191396542296,-'www.yahoo.com': 0.8990120070182801,-'www.mining.com': 0.8989843218989275,-'thestartuptoolkit.com': 0.8989617943814647,-'exple.tive.org': 0.8989583647726666,-'www.percona.com': 0.8989398873032174,-'tug.org': 0.8988675834004226,-'azure.microsoft.com': 0.8988119185957827,-'badassjs.com': 0.8987895474430552,-'florentcrivello.com': 0.8987782517021887,-'joelonsoftware.com': 0.8987694677031264,-'www.cntraveler.com': 0.8987494824459783,-'postmasculine.com': 0.8986927094085853,-'blade.nagaokaut.ac.jp': 0.8986856729161738,-'blog.demofox.org': 0.8986624269542334,-'www.zeit.de': 0.8986585350988878,-'louwrentius.com': 0.8986513425823265,-'www.jeffwofford.com': 0.8986246802875617,-'lists.canonical.org': 0.8985997713186075,-'blog.aboutamazon.com': 0.898569865878112,-'discourse.haskell.org': 0.8985468763312833,-'marginalrevolution.com': 0.8985222766062069,-'www.southampton.ac.uk': 0.8985031338241491,-'wccftech.com': 0.898464128485399,-'pages.nist.gov': 0.8984293193717278,-'blog.jcoglan.com': 0.8984277156180708,-'today.duke.edu': 0.898426650553988,-'blog.us.playstation.com': 0.8984266505539878,-'technet.microsoft.com': 0.8984178818074816,-'moultano.wordpress.com': 0.8983087054446331,-'forums.swift.org': 0.8983030929609286,-'pyfound.blogspot.com': 0.8982450409736414,-'sebastianraschka.com': 0.8982066640921482,-'www.michaelfogleman.com': 0.8982013504259682,-'www.confluent.io': 0.8981951121390044,-'www.maa.org': 0.8981932917194397,-'serverfault.com': 0.898185336292522,-'fitzgeraldnick.com': 0.8981340410428447,-'www.2600.com': 0.8981193270212353,-'rustyshelf.org': 0.898115004185606,-'svnweb.freebsd.org': 0.8981026654387363,-'www.argmin.net': 0.8980921557949308,-'www.theawl.com': 0.8980833367511953,-'keygen.sh': 0.8980696445122388,-'www.freightwaves.com': 0.8980574109093967,-'developers.facebook.com': 0.8980338313592694,-'www.randomhacks.net': 0.8980285406407745,-'susam.in': 0.8980164962202184,-'exercism.io': 0.8980136526006044,-'trekhleb.dev': 0.8980055706761685,-'securelist.com': 0.8979897593608089,-'air.mozilla.org': 0.897889243419127,-'yahooeng.tumblr.com': 0.8978817471988295,-'techrights.org': 0.8978658400805657,-'franciskim.co': 0.8978653593888846,-'ergoemacs.org': 0.8978533245623609,-'www.scs.stanford.edu': 0.8978490165530042,-'www.currentaffairs.org': 0.8978420868913102,-'arc.aiaa.org': 0.897839844709445,-'youtube.googleblog.com': 0.8978353788604845,-'www.pulitzer.org': 0.8977920724736956,-'computer.rip': 0.8977456785440727,-'blog.dgraph.io': 0.8977112410728876,-'blog.fastforwardlabs.com': 0.8977025051645369,-'www.bridgewater.com': 0.8976967291200032,-'mycroft.ai': 0.8976933371034872,-'open.salon.com': 0.8976917672314974,-'blog.longreads.com': 0.8976472678414861,-'lists.ubuntu.com': 0.897645348573024,-'nvie.com': 0.8976386294805203,-'adamdrake.com': 0.8976376774058287,-'www.cordcuttersnews.com': 0.8975937131370894,-'www.kayak.com': 0.8975688924284162,-'aiexperiments.withgoogle.com': 0.8975573917462177,-'gustedt.wordpress.com': 0.8975532085064689,-'www.adobe.com': 0.8975379400846643,-'miter.mit.edu': 0.8975316961912847,-'blog.statuspage.io': 0.8975269760160564,-'hyperpolyglot.org': 0.8975160197448814,-'www.scaleway.com': 0.8974907733168768,-'bennettfeely.com': 0.8974804776492202,-'joshwcomeau.com': 0.8974719675010521,-'www.ktvu.com': 0.8974664720007207,-'www.tomsguide.com': 0.897458826143719,-'www.jitbit.com': 0.8973926361586634,-'time.is': 0.8973510461083218,-'tinyclouds.org': 0.8973083698372893,-'gurjeet.singh.im': 0.8973075039780294,-'blog.mailgun.net': 0.8972959119527889,-'harry.me': 0.8972525836598453,-'archpaper.com': 0.8972054834238723,-'yoric.github.io': 0.897152918128832,-'community.atlassian.com': 0.8971438675267877,-'news.ycombinator.com': 0.8971152540222578,-'aviation.stackexchange.com': 0.8970966595562585,-'scholars-stage.blogspot.com': 0.8970933350643355,-'durmonski.com': 0.897088311771032,-'blog.zorinaq.com': 0.8970730785733094,-'blog.peepcode.com': 0.8970173769216392,-'kivy.org': 0.8969978306759151,-'www.crowdtilt.com': 0.8969827660481152,-'mathematica.stackexchange.com': 0.8969710211525342,-'www.wyden.senate.gov': 0.8969688474969271,-'watsi.org': 0.8969438303261918,-'igor-blue.github.io': 0.8968776201906612,-'medium.economist.com': 0.8968759451432184,-'www.wolframalpha.com': 0.8968667656586023,-'highlyscalable.wordpress.com': 0.8968525375322202,-'in.reuters.com': 0.8967974687633106,-'www.nethack.org': 0.8967951042087746,-'tidbits.com': 0.8967894768726519,-'theappendix.net': 0.8967463599409581,-'tech.fortune.cnn.com': 0.8967131583500147,-'www.fossjobs.net': 0.8966914843070399,-'blog.robinhood.com': 0.8966536394829941,-'www.realtimerendering.com': 0.8966481963801871,-'blog.theincredibleholk.org': 0.8966222436790824,-'www.ee.ryerson.ca': 0.8965601151332392,-'databricks.com': 0.8965596021498984,-'www.defensenews.com': 0.8965587750067616,-'rcoh.me': 0.896554024229452,-'mijailovic.net': 0.8965535170148166,-'gallium.inria.fr': 0.8965486747498262,-'blog.optimizely.com': 0.896542105323571,-'blog.replit.com': 0.8965338803932761,-'learn.adafruit.com': 0.8965131842089722,-'joelrunyon.com': 0.8964677519262857,-'blog.frizk.net': 0.8964121097275121,-'alarmingdevelopment.org': 0.8964094023716221,-'robots.thoughtbot.com': 0.8963990601096038,-'www.includeos.org': 0.8963879787342307,-'mrcoles.com': 0.8963879787342307,-'c2.com': 0.8963178119940578,-'www.whonix.org': 0.8963115990850473,-'fivebooks.com': 0.8963105340034757,-'reprog.wordpress.com': 0.8963054521368881,-'www.wilfred.me.uk': 0.8962945507182206,-'whydoeseverythingsuck.com': 0.8962216827001023,-'www.pcworld.com': 0.8962205591216735,-'www.bamsoftware.com': 0.8961964199305918,-'www.cs.unm.edu': 0.8961816656276443,-'www.idris-lang.org': 0.8961766972485017,-'www.raspberrypi.com': 0.8961620944863709,-'effectiviology.com': 0.8961424475553789,-'asciiflow.com': 0.8961224916388418,-'philosophically.com': 0.896116599326938,-'www.alexa.com': 0.8961037898681212,-'algorithmicbotany.org': 0.8960953021444669,-'yasoob.me': 0.8960526720715563,-'varun.ca': 0.896004489709373,-'medievalbooks.nl': 0.8959983974915444,-'www.kgw.com': 0.895989268551949,-'www.junglecoder.com': 0.8959808787419773,-'insideintercom.io': 0.8959693504237235,-'www.html5rocks.com': 0.8959295211749472,-'www.arxiv-vanity.com': 0.8959180319048996,-'micahflee.com': 0.8958958043171017,-'www.nccgroup.trust': 0.8958587838339611,-'c0de517e.blogspot.com': 0.8958503147286802,-'www.foreignpolicy.com': 0.8958091411993182,-'ashtonkemerling.com': 0.8957869981475534,-'www.amusingplanet.com': 0.8957821987841423,-'lea.verou.me': 0.895732027114529,-'nationalinterest.org': 0.8957204150819327,-'lists.webkit.org': 0.8956698653136842,-'opkode.com': 0.8956558833770757,-'www-formal.stanford.edu': 0.89562877673962,-'www.michaelgeist.ca': 0.8956008548612578,-'theory.stanford.edu': 0.8955882277276088,-'i.stanford.edu': 0.8955669355788611,-'exstreamist.com': 0.8955604388524536,-'citeseerx.ist.psu.edu': 0.8955488558889958,-'bugzilla.redhat.com': 0.8955216901346179,-'www.cmu.edu': 0.8954764227994559,-'0xax.gitbooks.io': 0.8954552987943115,-'jasoncrawford.org': 0.8954419392195877,-'www.lego.com': 0.8954257332135742,-'discourse.julialang.org': 0.8954216385277134,-'www.open-std.org': 0.8953539493751335,-'www.indiatoday.in': 0.8953429983961907,-'www.winestockwebdesign.com': 0.8953288508285574,-'mobiforge.com': 0.8953182171062833,-'www.regulations.gov': 0.895303818266352,-'www.salesforce.com': 0.8952981434270118,-'www.trackinghappiness.com': 0.8952717597076212,-'blackhole12.blogspot.com': 0.8952521384621974,-'blog.infinitenegativeutility.com': 0.8952329903252306,-'www.opera.com': 0.8952211159380423,-'blogs.balsamiq.com': 0.8952154811855707,-'treyhunner.com': 0.8952154811855707,-'constructionphysics.substack.com': 0.8951823466408914,-'beyondloom.com': 0.8951761937264585,-'www.lennyrachitsky.com': 0.8951749721627695,-'haskell.cs.yale.edu': 0.8951742276618677,-'raymii.org': 0.8951548310801417,-'llogiq.github.io': 0.8951452226380562,-'blog.amirkhella.com': 0.8950945164652512,-'www.mooreds.com': 0.8950941927209954,-'www.hudsandguis.com': 0.895021645021645,-'flask.pocoo.org': 0.89499976188154,-'www.dol.gov': 0.8949918028432319,-'wololo.net': 0.8949102546729428,-'dashes.com': 0.894908282336013,-'mortoray.com': 0.8948856187046399,-'www.mediawiki.org': 0.8948851399600354,-'developerblog.redhat.com': 0.8948826280677145,-'www.banterly.net': 0.8948384778162753,-'www.justin.tv': 0.8948269726237952,-'www.koreaherald.com': 0.8947866914467383,-'two-wrongs.com': 0.8947866914467383,-'www.3ders.org': 0.8947587499828596,-'millcomputing.com': 0.894753411664112,-'remysharp.com': 0.894728785876003,-'magenta.tensorflow.org': 0.8947055951377291,-'www.techradar.com': 0.8947023301779632,-'www.pushing-pixels.org': 0.8946997760143556,-'sagemath.blogspot.com': 0.8946383392529572,-'mymodernmet.com': 0.8946360917364401,-'jamesmcm.github.io': 0.8946346573429327,-'designyoutrust.com': 0.8945718808483918,-'boards.4chan.org': 0.8945658314381272,-'blog.framebase.io': 0.8945652173913043,-'openwrt.org': 0.8945648303938399,-'blog.codecombat.com': 0.8945386717554477,-'www.informit.com': 0.8945265591993962,-'www.evilsocket.net': 0.894519291668392,-'www.governing.com': 0.8945150467351556,-'www.masteringemacs.org': 0.894486414286821,-'tomassetti.me': 0.8944815712239113,-'gun.io': 0.8944737983455031,-'www.netlify.com': 0.8944433597452348,-'www.ntsb.gov': 0.8944261516293643,-'www.cs.kent.ac.uk': 0.8943581679658725,-'www.kernel.org': 0.8943363945858583,-'www.flyertalk.com': 0.8943293407889205,-'www.unlimitednovelty.com': 0.8943013001071519,-'math.andrej.com': 0.8942887015711656,-'healeycodes.com': 0.8942310355364101,-'blog.jmdawson.co.uk': 0.894210852992969,-'metaoptimize.com': 0.8942001103153372,-'mynorthwest.com': 0.894171800552717,-'asciinema.org': 0.8941695108843633,-'pydanny.com': 0.8941547728795499,-'warisboring.com': 0.8940977556210705,-'blog.serverfault.com': 0.8940908548836543,-'narratively.com': 0.8940876544990076,-'freakonomics.com': 0.8940592156715558,-'www.joindiaspora.com': 0.8940510018896056,-'www.caixinglobal.com': 0.8940493754864369,-'guccifer2.wordpress.com': 0.8940429649134846,-'www.freelists.org': 0.894038057617328,-'devblog.bu.mp': 0.8940332414895527,-'mislav.uniqpath.com': 0.894024162032998,-'www.heydonworks.com': 0.8939995146383073,-'glyph.twistedmatrix.com': 0.8939986046314382,-'paulirish.com': 0.8939908517980819,-'a1k0n.net': 0.8939890710382514,-'www.vldb.org': 0.893980099250682,-'www.squeakland.org': 0.8939334940011854,-'america.aljazeera.com': 0.8939301873013243,-'www.epi.org': 0.8939255228575806,-'tryolabs.com': 0.8938904483824683,-'willcrichton.net': 0.8938879993790215,-'news.lenovo.com': 0.8938868214729069,-'paulbernal.wordpress.com': 0.8938730853391685,-'www.psmag.com': 0.8938507630159833,-'meiert.com': 0.8938177244811635,-'www.ghacks.net': 0.8938115780468081,-'www.shamusyoung.com': 0.8938032619934929,-'zeptobars.com': 0.893764616861537,-'0x0f0f0f.github.io': 0.893764616861537,-'cm.bell-labs.com': 0.8937058283807041,-'www.fastcolabs.com': 0.8936928790642708,-'aging.nautil.us': 0.8936740461671302,-'hackerone.com': 0.8936672524995389,-'storage.courtlistener.com': 0.893640350877193,-'danlebrero.com': 0.893623066742338,-'mapzen.com': 0.8936063685534883,-'bernsteinbear.com': 0.8935751450877738,-'austinmann.com': 0.8935501465273382,-'www.vinaysahni.com': 0.8935481547848754,-'burakkanber.com': 0.8935378844831082,-'mollyrocket.com': 0.8935354120568186,-'feeding.cloud.geek.nz': 0.8935244667500523,-'cliffle.com': 0.8934905766791413,-'www.mikeperham.com': 0.8934692786771169,-'www.sequoiacap.com': 0.8934381407790684,-'www.thestranger.com': 0.8934172181855746,-'python.org': 0.8933810789765779,-'www.datacenterdynamics.com': 0.893374339023219,-'www.nongnu.org': 0.8933522846439509,-'blog.pkh.me': 0.8933285357815479,-'greenteapress.com': 0.8933064903929183,-'debarghyadas.com': 0.8933044930066333,-'jashkenas.github.com': 0.8933044930066333,-'jborichevskiy.com': 0.8932713559026754,-'www.saturdayeveningpost.com': 0.8932203537854467,-'www.engineyard.com': 0.8931369083416167,-'jeremymikkola.com': 0.8931346519747027,-'chriseidhof.tumblr.com': 0.893107113349865,-'www.the-scientist.com': 0.8931050183937893,-'heterodoxacademy.org': 0.8930676522754398,-'daveceddia.com': 0.8930562709346127,-'eighty-twenty.org': 0.8929575630252592,-'itvision.altervista.org': 0.8928942922265454,-'pogue.blogs.nytimes.com': 0.892885991203199,-'science.sciencemag.org': 0.8928844657775074,-'www.citypages.com': 0.8928735399686778,-'www.houstonchronicle.com': 0.8928722277760239,-'www.perl.com': 0.8928505576437601,-'www.mcclatchydc.com': 0.892841447269041,-'www.over-yonder.net': 0.8928301959201767,-'donatstudios.com': 0.8928301959201767,-'ucsdnews.ucsd.edu': 0.8928192288277865,-'www.johnmyleswhite.com': 0.8928134392551893,-'vorner.github.io': 0.8928096417930931,-'nikic.github.com': 0.8928090193925269,-'samsclass.info': 0.8927237027135478,-'markmaunder.com': 0.8926951731429821,-'tauday.com': 0.8926800567850355,-'riscv.org': 0.8926238746067623,-'blog.blazingdb.com': 0.8926082454543174,-'garrett.damore.org': 0.892590826266076,-'www.ilikebigbits.com': 0.8925607594811928,-'mediagoblin.org': 0.8925318168757916,-'keithclark.co.uk': 0.8925217211946057,-'reviews.llvm.org': 0.8925217211946057,-'www.census.gov': 0.8925111327414155,-'www.carbonbrief.org': 0.8925108441158517,-'1000memories.com': 0.8924866571941482,-'tommorris.org': 0.8924689354696166,-'csswizardry.com': 0.8924685362435004,-'simonmar.github.io': 0.8924611973392461,-'t3x.org': 0.8924420140543968,-'alpinelinux.org': 0.8924276788770722,-'news.blizzard.com': 0.8924063008658305,-'lowendmac.com': 0.892380727879953,-'jakubmarian.com': 0.8923725482586792,-'smackdown.blogsblogsblogs.com': 0.8923365014823826,-'www.gamefromscratch.com': 0.8922886748833608,-'www.history.com': 0.892284563335802,-'www.aps.org': 0.8922636239768308,-'bitbashing.io': 0.8922618659350428,-'thecreativeindependent.com': 0.8922525349154249,-'abcnews.go.com': 0.8922472028702633,-'lilianweng.github.io': 0.8922138047648334,-'filecoin.io': 0.8921882836554222,-'kristerw.blogspot.com': 0.8921827368433277,-'www.archdaily.com': 0.8921786726941157,-'critter.blog': 0.8921605201397375,-'blog.uber.com': 0.8921471734712938,-'blog.darklang.com': 0.8921367030876631,-'youteam.io': 0.8921269513814725,-'developer.oculus.com': 0.8921230866971346,-'www.airbnb.com': 0.8921210442927137,-'www.caitlingreen.org': 0.8921201754031437,-'stuffin.space': 0.8921201754031437,-'www.logicmatters.net': 0.8921008427968538,-'blogs.atlassian.com': 0.8920598317674283,-'www.coindesk.com': 0.8920474723589183,-'blog.checklyhq.com': 0.8920323362989564,-'www.reifyworks.com': 0.8920283981813807,-'www.tensorflow.org': 0.8919880973555573,-'medicalxpress.com': 0.8919405231029118,-'lists.wikimedia.org': 0.8919231093801725,-'www.cabinetmagazine.org': 0.8919061013347211,-'vez.mrsk.me': 0.8918998921036764,-'tenderlovemaking.com': 0.8918462339278702,-'timdettmers.com': 0.8918366190595837,-'jpkoning.blogspot.com': 0.8918251594506603,-'www.timesonline.co.uk': 0.8917973535504273,-'pdfs.semanticscholar.org': 0.8917971680021309,-'www.trustwave.com': 0.8917680987246398,-'ambrevar.xyz': 0.8917657796004286,-'www.theroot.com': 0.8917542491581687,-'tylertringas.com': 0.8917090979583869,-'www.ncsc.gov.uk': 0.8916672988164659,-'globenewswire.com': 0.8916669637212149,-'www.applevis.com': 0.8916621885801436,-'www.metafilter.com': 0.8916619189169765,-'erickhun.com': 0.8916502534520422,-'blog.wearewizards.io': 0.8916476923173318,-'cpj.org': 0.8916369716407726,-'boyter.org': 0.891577544859255,-'m.facebook.com': 0.8915642265642092,-'www.calnewport.com': 0.8915632480334919,-'www.profgalloway.com': 0.8915466752832707,-'mentallandscape.com': 0.8915429443685636,-'m.theatlantic.com': 0.891523653721213,-'hackertarget.com': 0.8915152085122275,-'www.pyret.org': 0.8914988814317674,-'www.obdev.at': 0.8914980426595512,-'dash.harvard.edu': 0.8914784869901488,-'engineeringblog.yelp.com': 0.8914654287085163,-'www.mercurial-scm.org': 0.8914323327846669,-'datanitro.com': 0.8914234373706179,-'www.huffingtonpost.in': 0.8914121923586303,-'fangpenlin.com': 0.891404181051816,-'discovermagazine.com': 0.8914023623580617,-'streetlifesolutions.blogspot.com': 0.8914002514918445,-'blog.documentfoundation.org': 0.8913787017013197,-'dotsies.org': 0.8913773796192609,-'tails.boum.org': 0.8913724994898894,-'randfishkin.com': 0.8912627828106103,-'tomasp.net': 0.8911820823224039,-'www.taiwannews.com.tw': 0.8911580981808408,-'modern-sql.com': 0.891151176073313,-'carlos.bueno.org': 0.8911385230462937,-'www.kansascity.com': 0.8910854933759852,-'english.stackexchange.com': 0.8910824248780256,-'cdm.link': 0.8910804711300865,-'nighthacks.com': 0.8910750341672654,-'blog.vrypan.net': 0.8910540018083172,-'nyxt.atlas.engineer': 0.8910326186239637,-'blog.evernote.com': 0.8909942969284109,-'news.bbc.co.uk': 0.89099404579875,-'securitylab.github.com': 0.8909843257796506,-'pjreddie.com': 0.8909657104816301,-'fury.com': 0.8909653233291642,-'blog.skylight.io': 0.8909427640646718,-'kickstarter.engineering': 0.8909427640646718,-'www.darpa.mil': 0.8909203771658175,-'psyarxiv.com': 0.8909128838846586,-'spottedsun.com': 0.89089016985033,-'matt.aimonetti.net': 0.8907810868277886,-'news.dartlang.org': 0.8907684232451024,-'musescore.org': 0.8907008278994514,-'www.apa.org': 0.8906903176590568,-'www.flightglobal.com': 0.8906785542989906,-'www.theatlanticcities.com': 0.8906598207125368,-'search.cpan.org': 0.8906114753639363,-'www.p01.org': 0.8906084333939481,-'blog.dantup.com': 0.8906084333939481,-'thedailywtf.com': 0.8905424089852495,-'e.foundation': 0.8905340551839913,-'cookieplmonster.github.io': 0.8905252719934627,-'news.php.net': 0.8905175241100567,-'toroid.org': 0.8905145635041263,-'www.kk.org': 0.8904878920768752,-'forum.sublimetext.com': 0.890452944414247,-'praxis.fortelabs.co': 0.890452944414247,-'leostartsup.com': 0.8904132589825239,-'web.cecs.pdx.edu': 0.890412029578113,-'www.ofbrooklyn.com': 0.8903954802259887,-'www.revk.uk': 0.8903240899854906,-'rawkes.com': 0.8903199760734138,-'furbo.org': 0.8903142798666888,-'nickoneill.com': 0.8902859706962722,-'compilers.iecc.com': 0.8902788378277958,-'www.climatechangenews.com': 0.8902788378277958,-'avichal.com': 0.8902714932126697,-'www.healthcaredive.com': 0.8902714932126697,-'lucasfcosta.com': 0.8902709782517574,-'stevecoast.com': 0.8902478204961621,-'dariusforoux.com': 0.8902302723631834,-'msdn.microsoft.com': 0.8901788121850751,-'www.nzz.ch': 0.8901568608802078,-'trac.webkit.org': 0.8901554708541648,-'www.waze.com': 0.8901472253680635,-'www.liip.ch': 0.8901472253680635,-'ericlippert.com': 0.8901127718368207,-'matpalm.com': 0.8901030282480019,-'roshfu.com': 0.8900305511261404,-'adeshpande3.github.io': 0.8900188928475846,-'www.factmag.com': 0.8900054823385649,-'grafana.org': 0.8899942321734828,-'www.divio.com': 0.889923339928156,-'www.law.cornell.edu': 0.8899173371280885,-'www.cs.huji.ac.il': 0.8899161000679049,-'gothamist.com': 0.8899111843846533,-'www.ajc.com': 0.8899040092884533,-'mathpix.com': 0.8899039505478837,-'www.vulture.com': 0.8899034479557729,-'bair.berkeley.edu': 0.8898667877223271,-'www.goldsborough.me': 0.8897894354492999,-'arkadiyt.com': 0.8897865060368211,-'flowingdata.com': 0.8897859193800971,-'grist.org': 0.8897796023630788,-'clusterhq.com': 0.889770984328511,-'postmarketos.org': 0.8897648371785116,-'monkey.org': 0.8897626687452419,-'www.coderholic.com': 0.8897409044066217,-'blog.silktide.com': 0.8897409044066216,-'docs.house.gov': 0.88972591966882,-'colab.research.google.com': 0.8897224911816017,-'www.nhs.uk': 0.8896805121516772,-'insideclimatenews.org': 0.8896793812620282,-'torvalds-family.blogspot.com': 0.8896665476254435,-'www.holloway.com': 0.8896662494413392,-'www.airspacemag.com': 0.8896507942259744,-'grsecurity.net': 0.889646047305093,-'www.kevinrooke.com': 0.8896372799856362,-'engineering.gusto.com': 0.8896147691691174,-'blog.christianperone.com': 0.8896015309843164,-'www.badscience.net': 0.889589692438837,-'www.jerf.org': 0.889547641907146,-'vfoley.xyz': 0.8895216400911161,-'www.dangrossman.info': 0.8895193470455725,-'blog.timac.org': 0.8895027671338473,-'www.nme.com': 0.8894941034343402,-'withblue.ink': 0.8894380029474798,-'www.modernperlbooks.com': 0.8894320825157749,-'guix.gnu.org': 0.8894056608506781,-'vasilishynkarenka.com': 0.8894015273075921,-'chrome.google.com': 0.8893920540006104,-'startupboy.com': 0.8893837967279286,-'thesoundingline.com': 0.8893544871283143,-'blog.kotowicz.net': 0.8893507542723823,-'www.ubuntuvibes.com': 0.8893113229443974,-'www.tindie.com': 0.8892920494549288,-'www.falstad.com': 0.8892263646054737,-'alisdair.mcdiarmid.org': 0.8892079775948455,-'www.brennancenter.org': 0.8892079775948455,-'theanarchistlibrary.org': 0.8891911407644053,-'www.cftc.gov': 0.8891651323421873,-'lisp-journey.gitlab.io': 0.8891428571428571,-'www.namepros.com': 0.8891410371836007,-'eu.usatoday.com': 0.8890682694542983,-'blog.canonical.com': 0.8890615085402753,-'sahandsaba.com': 0.88904655549824,-'blog.rstudio.org': 0.8890367675160065,-'neilkakkar.com': 0.8890307804355818,-'news.nationalpost.com': 0.8890115573593319,-'www.snopes.com': 0.8889924719688463,-'www.brainpickings.org': 0.8889686116306911,-'mitmproxy.org': 0.8889639851594255,-'blog.balancedpayments.com': 0.888958456392269,-'breakingdefense.com': 0.8889416410999423,-'markets.businessinsider.com': 0.8889221685107183,-'jerrygamblin.com': 0.8889200836058827,-'angularjs.blogspot.com': 0.8888770420833161,-'www.shopify.com': 0.8888551514369234,-'www.helloerik.com': 0.8888428887518348,-'languagelog.ldc.upenn.edu': 0.8888040353901259,-'diogomonica.com': 0.888789480686917,-'oversight.house.gov': 0.8887677241378522,-'blog.yossarian.net': 0.8887466744525089,-'tim.blog': 0.8887179571174896,-'blog.marcocantu.com': 0.8887010787021495,-'www.stilldrinking.org': 0.8886969078588848,-'plentyoffish.wordpress.com': 0.8886858465608289,-'troubles.md': 0.8886733614347104,-'8bitworkshop.com': 0.8886716751051769,-'plato.stanford.edu': 0.8886226501737253,-'blog.watsi.org': 0.8886099090504368,-'www.vatican.va': 0.8886001974662521,-'www.gridsagegames.com': 0.8885999017945436,-'blog.wakemate.com': 0.8885718128809637,-'www.guardian.co.uk': 0.8885578778095446,-'www.eastbaytimes.com': 0.8885533867034545,-'insights.ubuntu.com': 0.8885369737806736,-'www.uber.com': 0.888502360897741,-'hharnisc.github.io': 0.8884780095069668,-'blog.eyas.sh': 0.8884641430053347,-'adlrocha.substack.com': 0.8884402828343574,-'www.agwa.name': 0.8883681305045957,-'lexfridman.com': 0.8883669315681367,-'www.hp.com': 0.8883541180846748,-'chargen.matasano.com': 0.888350093124338,-'docs.github.com': 0.888346612251366,-'blogs.esa.int': 0.8883268060163584,-'hackage.haskell.org': 0.8882716929118732,-'www.askthepilot.com': 0.8882206989905654,-'investor.fb.com': 0.8881776846453903,-'hackertourism.com': 0.8881603547360667,-'varianceexplained.org': 0.8881303906523573,-'www.stanford.edu': 0.8881280238738776,-'googleappengine.blogspot.com': 0.8881149640019546,-'blog.spark.io': 0.8880887280047254,-'danielmiessler.com': 0.8880772991848912,-'blog.cloudant.com': 0.8880720742931111,-'help.netflix.com': 0.888032506658475,-'www.consumer.ftc.gov': 0.8880113570637971,-'thechangelog.com': 0.8880056295154011,-'mrdoob.com': 0.8879829721117504,-'www.saveur.com': 0.8879678355351808,-'www.moderntreasury.com': 0.8879300370907698,-'blogs.suntimes.com': 0.8879083159007929,-'staff.tumblr.com': 0.8878771427989084,-'archive.wired.com': 0.8878447305921355,-'www.ironicsans.com': 0.8878289779246122,-'blog.capitalandgrowth.org': 0.8878130504973006,-'www.thisiscolossal.com': 0.8878105455839786,-'benchling.engineering': 0.8878049780860752,-'www.symmetrymagazine.org': 0.8877876402671063,-'www.feld.com': 0.8877667891871323,-'everynoise.com': 0.8877314814814815,-'input.fontbureau.com': 0.8877209761748434,-'qiao.github.io': 0.8877030948951334,-'factordaily.com': 0.887680355039973,-'ecc-comp.blogspot.com': 0.887651452722378,-'addyosmani.com': 0.8876225519241614,-'reallifemag.com': 0.8876055526528979,-'niedzejkob.p4.team': 0.8875742399340828,-'www.cs.purdue.edu': 0.887560497701347,-'vercel.com': 0.8875553045329022,-'blog.sesse.net': 0.887545005792023,-'mynoise.net': 0.8875240702669522,-'nevalalee.wordpress.com': 0.8874709976798144,-'www.justingarrison.com': 0.8874550740684668,-'www.cse.buffalo.edu': 0.887431083352139,-'jamesgolick.com': 0.8874059947114532,-'gnunet.org': 0.887393102678099,-'www.tampabay.com': 0.8873751025878571,-'www.jide.com': 0.887368855995013,-'producthabits.com': 0.8873611172314394,-'www.fourhourworkweek.com': 0.8873408966020546,-'refer.ly': 0.8873384440828381,-'www.aei.org': 0.8873351640879104,-'www.mentalfloss.com': 0.8873305275256577,-'izbicki.me': 0.8873178158351525,-'mathiasbynens.be': 0.8872781506210743,-'www.pcjs.org': 0.8872494045912365,-'arcan-fe.com': 0.887245072438425,-'www.americamagazine.org': 0.8872386352495559,-'mailman.nanog.org': 0.8872223978170161,-'bluejekyll.github.io': 0.8872093023255814,-'www.mysliderule.com': 0.8872093023255814,-'blogs.oracle.com': 0.8871952202335835,-'leonardofed.io': 0.8871763351184384,-'www.sourcetrail.com': 0.8871714606492734,-'www.linuxfromscratch.org': 0.8871714606492734,-'bariweiss.substack.com': 0.8871587544964651,-'moot.it': 0.8871570315823085,-'popula.com': 0.887128977917166,-'press.web.cern.ch': 0.887082617470916,-'ownyourbits.com': 0.8870779976717112,-'getstream.io': 0.8870262897116983,-'scpd.stanford.edu': 0.8870121302239351,-'dcgross.com': 0.8868939399089152,-'unsupervisedmethods.com': 0.8868826572298285,-'www.willmcgugan.com': 0.8868766525394289,-'wiki.tcl-lang.org': 0.8868748835508599,-'blog.disqus.com': 0.8868414318888773,-'blogs.technet.microsoft.com': 0.8868323608446661,-'tromp.github.io': 0.8867756037157464,-'blog.ircmaxell.com': 0.8867580271160181,-'mux.com': 0.8867579106919085,-'bugs.mysql.com': 0.8867572809726417,-'www.irishtimes.com': 0.8867543138710346,-'dpc.pw': 0.8867103394982209,-'dusted.codes': 0.8866966751368195,-'quanticdev.com': 0.8866903311266173,-'www.neowin.net': 0.8866503890472428,-'chrislord.net': 0.886641393239182,-'blog.otoro.net': 0.8866334244061989,-'www.thenational.scot': 0.8866271172090046,-'www.inquirer.com': 0.8865967718863557,-'modelviewculture.com': 0.8865845558030554,-'info.iet.unipi.it': 0.8865497076023392,-'www.storehouse.co': 0.886548015153011,-'www.anfractuosity.com': 0.8865113264892915,-'danwang.co': 0.8864808931374831,-'www.researchgate.net': 0.8864516461055575,-'www.doxsey.net': 0.8864376154183435,-'bugs.ruby-lang.org': 0.8864368986388406,-'whyy.org': 0.8864168618266979,-'itch.io': 0.8864120842802148,-'amigalove.com': 0.8863992170741539,-'people.mozilla.com': 0.8863952990075262,-'medium.freecodecamp.com': 0.8863811065750393,-'blog.doist.com': 0.8863606429850415,-'www.swyx.io': 0.8863599205692547,-'www.eidel.io': 0.886301279533414,-'senrigan.io': 0.8862368459608536,-'www.pvk.ca': 0.8861969908332605,-'www.ccs.neu.edu': 0.8861380071332806,-'cturt.github.io': 0.8861102984083253,-'www.lysator.liu.se': 0.8860844730714155,-'philcalcado.com': 0.8859388762978412,-'lists.dragonflybsd.org': 0.8859388762978412,-'microkerneldude.wordpress.com': 0.8859288068767922,-'www.theringer.com': 0.8858962060628648,-'bzg.fr': 0.8858719540548398,-'www.crypto.com': 0.8858494594971499,-'fibery.io': 0.8858217047605946,-'kimonote.com': 0.885805964130514,-'news.softpedia.com': 0.8857837864929501,-'successfulsoftware.net': 0.8857784923477241,-'blog.intercom.com': 0.8857763495575585,-'www.embeddedrelated.com': 0.8857676106438698,-'www.psychologytoday.com': 0.8857585049003665,-'haveibeenpwned.com': 0.8856992747056773,-'blog.ryotak.me': 0.8856929365305167,-'blog.piston.rs': 0.8856929365305167,-'joshmitteldorf.scienceblog.com': 0.8855962693776582,-'www.reviews.com': 0.8855218005991683,-'svpow.com': 0.8855122462369361,-'blog.nahurst.com': 0.8855122462369361,-'engineering.kablamo.com.au': 0.8854939252840581,-'blog.adafruit.com': 0.8854467055810114,-'earth.nullschool.net': 0.8854306983539961,-'inventwithpython.com': 0.885393853053717,-'chalkdustmagazine.com': 0.8853197575419666,-'blog.objectmentor.com': 0.8852948669873847,-'www.aholme.co.uk': 0.8852815790604109,-'www.dkriesel.com': 0.8852756720656075,-'vanemden.wordpress.com': 0.8852604927216077,-'investor.google.com': 0.8852562818553033,-'pipeline.corante.com': 0.8852387066889292,-'action.eff.org': 0.8852266245403357,-'blog.twitch.tv': 0.8852196608142338,-'www.nand2tetris.org': 0.8852071005917159,-'gpfault.net': 0.8851652261576463,-'meshedinsights.com': 0.8851628232067584,-'probcomp.csail.mit.edu': 0.8851438011782188,-'missiveapp.com': 0.8851280394688669,-'blog.sendgrid.com': 0.8851277655579799,-'mitadmissions.org': 0.8851045058363526,-'info.meteor.com': 0.8850985466953342,-'thewirecutter.com': 0.8850944093158075,-'www.aboveavalon.com': 0.8850896030144491,-'haxe.org': 0.8850332564592883,-'www.snarky.ca': 0.8850298847276258,-'blog.mapillary.com': 0.8849756787053948,-'www.alternet.org': 0.8849651154014133,-'martinheinz.dev': 0.884952570972269,-'www.stevebenjamins.com': 0.8849347568208777,-'derwiki.tumblr.com': 0.8849077586898806,-'www.kuro5hin.org': 0.8848948828327496,-'www.archlinux.org': 0.8848794541675421,-'www.bennjordan.com': 0.8848672501882364,-'tromey.com': 0.8848672501882364,-'medlifestyle.news': 0.8848240975158286,-'hackpad.com': 0.8848240112405613,-'zinascii.com': 0.8848232963497987,-'heredragonsabound.blogspot.com': 0.8848232963497987,-'pycon.blogspot.com': 0.8847972407140463,-'thegenesisblock.com': 0.8847466884910953,-'jayriverlong.github.io': 0.8846854627897014,-'rkoutnik.com': 0.8846701041470997,-'moalquraishi.wordpress.com': 0.8846503798740485,-'blog.angularjs.org': 0.8846476203665491,-'learnyousomeerlang.com': 0.8846416196747935,-'blog.rubymotion.com': 0.8846052536362908,-'www.berkshirehathaway.com': 0.8845734385329344,-'bigstory.ap.org': 0.8845644349212194,-'www.dji.com': 0.8845531462931538,-'homes.cs.washington.edu': 0.884536530239123,-'momjian.us': 0.8845249482528319,-'anderspitman.net': 0.8845238095238095,-'ubiquity.acm.org': 0.8845123969111347,-'archive.vector.org.uk': 0.8844858003347821,-'bitcoinfoundation.org': 0.8844841517497576,-'esr.ibiblio.org': 0.884465525984111,-'www.caranddriver.com': 0.8844455469996317,-'www.bristol.ac.uk': 0.8844312836410105,-'www.keacher.com': 0.8844188041447092,-'www.planet.com': 0.8843849199350312,-'www.braintreepayments.com': 0.8843684120647333,-'www.topic.com': 0.8843258553540223,-'www.lyx.org': 0.8843101335952557,-'hal.inria.fr': 0.884278551246099,-'blog.powerdns.com': 0.8842750467710181,-'www.discovermeteor.com': 0.8842406817313907,-'www.statista.com': 0.884226458408693,-'diveintohtml5.org': 0.884224840703763,-'shkrobius.livejournal.com': 0.8842248407037628,-'danwin.com': 0.8842240470222248,-'thingist.com': 0.8841700109782021,-'sf.curbed.com': 0.8841684697837997,-'www.artima.com': 0.8840774930170289,-'www.gregegan.net': 0.8840639202320028,-'veed.io': 0.8840295485188058,-'www.kguttag.com': 0.8840223725080164,-'www.passwordstore.org': 0.8839871084196906,-'www.time.com': 0.8839776340050455,-'patft.uspto.gov': 0.8839759157295332,-'brikis98.blogspot.com': 0.8839185792660761,-'luckypeach.com': 0.8839078718615478,-'www.256kilobytes.com': 0.8839065999741562,-'kamalmarhubi.com': 0.8838909108920787,-'www.levels.fyi': 0.8838529264451486,-'www.grubstreet.com': 0.8838363484488945,-'kubernetes.io': 0.8838168079942494,-'privpapers.ssrn.com': 0.8837957120914789,-'finledger.com': 0.8837957120914789,-'ircv3.net': 0.8837369391764082,-'webtorrent.io': 0.8836930455635492,-'animagraffs.com': 0.8836872834930155,-'www.coinbase.com': 0.8836702182178287,-'jobs.apple.com': 0.8836631610037271,-'lifehacker.com': 0.88361719985225,-'blog.mongodb.org': 0.883606109678017,-'lobe.ai': 0.8835655842394832,-'blogs.ams.org': 0.8835451368221627,-'www.psychologicalscience.org': 0.8834931862797779,-'www.autonews.com': 0.8834926699108666,-'google-styleguide.googlecode.com': 0.8834824574967941,-'venomousporridge.com': 0.8834619351905811,-'www.bennorthrop.com': 0.8834521090229532,-'blog.lusis.org': 0.8834395848799916,-'www.defcon.org': 0.8834198543134106,-'railstips.org': 0.8834172428920269,-'sitn.hms.harvard.edu': 0.8834134615384616,-'www.johnwdefeo.com': 0.8834118239436308,-'www.metabase.com': 0.8833568908825655,-'jordansmith.io': 0.8833386064130131,-'blog.brokep.com': 0.8833292707180577,-'davnicwil.com': 0.883290787723497,-'daverupert.com': 0.8832817007531839,-'www.internalpointers.com': 0.8832731648616126,-'blog.scottnonnenberg.com': 0.8832680173548925,-'webmasters.googleblog.com': 0.8832159517897961,-'ivanceras.github.io': 0.8831795673307369,-'www.inference.vc': 0.8831370240507179,-'white.ucc.asn.au': 0.883132530120482,-'dis.4chan.org': 0.8831325301204819,-'mta.openssl.org': 0.8831205226464401,-'blog.sourcerer.io': 0.8831047861347778,-'callumprentice.github.io': 0.8830898580872119,-'www.techtransparencyproject.org': 0.8830469812134585,-'blog.adobe.com': 0.883040341397881,-'agfundernews.com': 0.8830325567122732,-'www.centauri-dreams.org': 0.8829955449207836,-'documentcloud.github.com': 0.8829915560916767,-'rust-lang.github.io': 0.8829915560916767,-'portswigger.net': 0.8829877514947376,-'tagide.com': 0.8829745187840874,-'creativecommons.org': 0.8829668579998763,-'flak.tedunangst.com': 0.8829644456502317,-'ferrous-systems.com': 0.8829609951535142,-'shapecatcher.com': 0.8829477368055727,-'solovyov.net': 0.8829399933646876,-'blog.pmarca.com': 0.8829123044832685,-'linusakesson.net': 0.8829080057671795,-'www.benfrederickson.com': 0.8828991573081899,-'maps.stamen.com': 0.8828934647050843,-'techcommunity.microsoft.com': 0.8828546011216626,-'blog.pyston.org': 0.8828536336243845,-'blog.freshdesk.com': 0.8828536336243845,-'www.radionz.co.nz': 0.8828494689784901,-'translate.googleusercontent.com': 0.882847058520687,-'stephenmann.io': 0.8828104499622798,-'www.berkeleyside.com': 0.8828101152681945,-'engineering.zenpayroll.com': 0.8827759389076459,-'googlefiberblog.blogspot.com': 0.8827390835242562,-'nadiaeghbal.com': 0.8827255975365081,-'chartio.com': 0.882704073135325,-'slidetocode.com': 0.8826822967578939,-'mic.com': 0.8826760832749148,-'ayende.com': 0.8826726588726063,-'sixrevisions.com': 0.8826715498136648,-'www.ligo.caltech.edu': 0.8826619298426813,-'www.insider.com': 0.8826223554956923,-'grahamc.com': 0.8825699143811585,-'www.sbcl.org': 0.882563699339324,-'www-m.cnn.com': 0.8825634181290516,-'advances.sciencemag.org': 0.8825504380314206,-'codingvc.com': 0.8825483021452769,-'blog.buoyant.io': 0.8825343736402197,-'tools.google.com': 0.8825286182855726,-'www.thisamericanlife.org': 0.8825206320161334,-'www.livescience.com': 0.8825187990685426,-'blog.udacity.com': 0.88248770629827,-'www.mint.com': 0.8824829228307717,-'www.quantstart.com': 0.8824744656067158,-'i3wm.org': 0.882461673497088,-'lugassy.net': 0.882461673497088,-'renato.athaydes.com': 0.8824351262415586,-'john.ankarstrom.se': 0.8824318706073884,-'deis.com': 0.882398256845465,-'blog.travis-ci.com': 0.8823494388558837,-'akshat.posterous.com': 0.882348785313154,-'blog.danieljanus.pl': 0.882305963719186,-'www.lambdassociates.org': 0.882291549111701,-'www.paypal-engineering.com': 0.8822900726859743,-'macdaddy.io': 0.8822875933955345,-'homepages.inf.ed.ac.uk': 0.882263750625994,-'events.google.com': 0.8822491129403885,-'isocpp.org': 0.882201264152672,-'www.secjuice.com': 0.8821793252738344,-'www.martinvigo.com': 0.8821771335964247,-'osblog.stephenmarz.com': 0.8821656713562592,-'business.financialpost.com': 0.8821599714093322,-'thetech.com': 0.8821385176184691,-'berkshirehathaway.com': 0.882138517618469,-'harvardlawreview.org': 0.8821355848021567,-'leafo.net': 0.8821225145186178,-'curia.europa.eu': 0.882094739471072,-'ckeditor.com': 0.8820783917249119,-'www.topgear.com': 0.8820554259493689,-'thebaffler.com': 0.8820445386842816,-'wilwheaton.net': 0.8820435463334623,-'andyadams.org': 0.8820328528134829,-'applidium.com': 0.8820326538487594,-'techspecs.blog': 0.8819755098672539,-'cafbit.com': 0.8819755098672539,-'community.spotify.com': 0.8819640976966642,-'jacobinmag.com': 0.8819625349712696,-'www.indiegogo.com': 0.881946367637731,-'hueniverse.com': 0.8819393022613744,-'tent.io': 0.8819180061465067,-'blog.caranddriver.com': 0.8818992762989476,-'www.avweb.com': 0.8818262516609122,-'kieranhealy.org': 0.8817976429098323,-'webrtchacks.com': 0.8817875203232487,-'www.penzba.co.uk': 0.8817713693955629,-'www.onbeing.org': 0.8817382469287067,-'www.makeartwithpython.com': 0.8817227794803292,-'storagemojo.com': 0.8816983007282381,-'rentzsch.tumblr.com': 0.8816906014355783,-'www.chrisnorstrom.com': 0.881678989119816,-'mir.aculo.us': 0.8816552237217568,-'automatetheboringstuff.com': 0.8815884675997907,-'doc.cat-v.org': 0.8815699328665041,-'skilldrick.github.io': 0.8815628815628815,-'www.azulsystems.com': 0.8815545999084712,-'googledevelopers.blogspot.com': 0.8815293986804532,-'shopify.engineering': 0.8815078480198789,-'www.windy.com': 0.8814945016309822,-'gen.medium.com': 0.8814868144618807,-'mygengo.com': 0.8814826810568291,-'microship.com': 0.8814701304564254,-'jroweboy.github.io': 0.881441670329882,-'www.isro.gov.in': 0.8814323797659261,-'amitness.com': 0.8814233401080115,-'journal.neilgaiman.com': 0.8813000878314081,-'blog.longnow.org': 0.8812928039894358,-'developers.lyst.com': 0.881286326309607,-'threads2.scripting.com': 0.8812146696862522,-'www.intego.com': 0.8812141651552359,-'abc7news.com': 0.8812015459094202,-'ac.els-cdn.com': 0.8811985270068678,-'zwischenzugs.wordpress.com': 0.8811838816065725,-'ooni.torproject.org': 0.8811640746997138,-'rodneybrooks.com': 0.8811464380886621,-'mentalfloss.com': 0.8811323569439224,-'www.elie.net': 0.8810667382996152,-'www.eecs.berkeley.edu': 0.881035190029909,-'surjan.substack.com': 0.8810020145474288,-'metrics.torproject.org': 0.880999086515737,-'www.scss.tcd.ie': 0.8809783370810794,-'www.zdziarski.com': 0.8809560496711293,-'blog.lizzie.io': 0.8809270824147091,-'www.nntp.perl.org': 0.8809140700339988,-'www.skorks.com': 0.8809070694481402,-'www.lastweekinaws.com': 0.880898941183668,-'projectgus.com': 0.8808911111787601,-'togelius.blogspot.com': 0.8808652592717593,-'www.thedailymuse.com': 0.880861164752333,-'blog.mailchimp.com': 0.8808342908333803,-'www.seas.harvard.edu': 0.8808287698175389,-'kernelmag.dailydot.com': 0.8807834100859487,-'www.audubon.org': 0.8807603148427023,-'juretriglav.si': 0.8807541586341653,-'www.ias.edu': 0.8806635755411035,-'www.startupsopensourced.com': 0.8806386086637549,-'scott.a16z.com': 0.880594330778045,-'www.webdistortion.com': 0.8805671581809865,-'coinbase.com': 0.8805536570003393,-'my.matterport.com': 0.8805531859476551,-'world.hey.com': 0.8805499381704851,-'freenetproject.org': 0.8805418719211823,-'ez.substack.com': 0.8805347456971939,-'www.cryptologie.net': 0.880520955636949,-'www.ewherry.com': 0.8804202710724481,-'www.aaronklein.com': 0.8804202710724481,-'www.metachris.com': 0.8803487337679655,-'www.info.ucl.ac.be': 0.8803332113342701,-'www.maritime-executive.com': 0.8802634318319039,-'nationalpost.com': 0.8802495762864062,-'stu.mp': 0.8802212225414684,-'pcwalton.github.io': 0.8801825251719878,-'www.lensrentals.com': 0.8801530918564299,-'programmingzen.com': 0.8801076222388025,-'applied-data.science': 0.8800802934751344,-'www.2uo.de': 0.8800661016526783,-'www.americanscientist.org': 0.8800620422490765,-'vault.fbi.gov': 0.8800567908408685,-'www.mymodernmet.com': 0.880048242675238,-'www.stripes.com': 0.8799620726032865,-'aralbalkan.com': 0.8799619967876905,-'www.washingtonian.com': 0.8799468909598039,-'jamey.thesharps.us': 0.8799345914752841,-'visualwebsiteoptimizer.com': 0.8799281242616495,-'blogs.uw.edu': 0.879925783709813,-'blog.sqlizer.io': 0.8799186267288787,-'workshop.chromeexperiments.com': 0.8799186267288787,-'www.jamieweb.net': 0.8799112718655394,-'royal.pingdom.com': 0.8798976823547263,-'oag.ca.gov': 0.8798952586106802,-'www.askamanager.org': 0.8798868933154113,-'briarproject.org': 0.8798570681672088,-'everythingsysadmin.com': 0.8798435781150284,-'serokell.io': 0.8798171504945399,-'mmcgrana.github.com': 0.8797647395096098,-'ruslanspivak.com': 0.8797628893146511,-'wesmckinney.com': 0.8797444121875875,-'blog.mattbierner.com': 0.8797205044550149,-'mit.edu': 0.8796628612944964,-'www.asimovinstitute.org': 0.8796512842876961,-'press.stripe.com': 0.8796512842876961,-'technicshistory.wordpress.com': 0.8796210680092151,-'common-lisp.net': 0.8796152653799312,-'www.u-tokyo.ac.jp': 0.8796017299329183,-'www.dailykos.com': 0.8795999531727511,-'www.pagerduty.com': 0.8795889070772409,-'austinkleon.com': 0.8795887664062606,-'blog.last.fm': 0.8795848845621085,-'blog.clearbit.com': 0.8795162875960222,-'blog.thinkst.com': 0.8794805645736704,-'hivelogic.com': 0.87947663783247,-'hechingerreport.org': 0.8794286991256227,-'www.fortressofdoors.com': 0.8794194991885873,-'docs.docker.com': 0.8794114856568306,-'kushagragour.in': 0.879405287790302,-'ultimatehackingkeyboard.com': 0.8793982349555475,-'developer.ibm.com': 0.8793941898423207,-'rushter.com': 0.879388928994475,-'www.tarsnap.com': 0.8793327066997211,-'www.wtsp.com': 0.8793263869529092,-'brohrer.github.io': 0.8793263869529092,-'elementary.io': 0.8793165731880934,-'destevez.net': 0.8793048124903404,-'www.gentoo.org': 0.879269905778943,-'nbviewer.jupyter.org': 0.879269721586867,-'blog.angular.io': 0.879268923349318,-'www.techrepublic.com': 0.8792309439284567,-'www.virgin.com': 0.8792219903286642,-'blog.getspool.com': 0.8792029887920298,-'www.lucidchart.com': 0.8791838979662947,-'blog.readability.com': 0.8791595821198683,-'narrative.ly': 0.8791473124131408,-'davidgerard.co.uk': 0.879133623229212,-'www.tldp.org': 0.8791060662671288,-'www.chillingeffects.org': 0.8790911678663772,-'mattbruenig.com': 0.8790858191942302,-'blog.coursera.org': 0.8790756346109743,-'engineering.khanacademy.org': 0.8790360280222413,-'aurellem.org': 0.8790251342720007,-'www8.hp.com': 0.8789932584616722,-'www.mrmoneymustache.com': 0.8789801725348548,-'themetricsystem.rjmetrics.com': 0.8789799435157434,-'popey.com': 0.8789713929868309,-'xlinux.nist.gov': 0.8789408088929164,-'ben-evans.com': 0.8789322960390088,-'www.datapointed.net': 0.8789114224720275,-'www.net.in.tum.de': 0.8788892296044377,-'stratos.seas.harvard.edu': 0.8788892296044377,-'www.smbc-comics.com': 0.878826000548314,-'blog.gittip.com': 0.8788213217536158,-'www.cloudflare.com': 0.8788059374287456,-'rmurphey.com': 0.8787968488987687,-'www.tuhs.org': 0.8787968488987687,-'blog.openstreetmap.org': 0.8787897231673689,-'www.istartedsomething.com': 0.8787889297497098,-'ilyabirman.net': 0.8787732444342291,-'www.github.com': 0.8787717233938399,-'amix.dk': 0.8787529849051614,-'www.insidehighered.com': 0.8787455322695712,-'actu.epfl.ch': 0.8787380577275826,-'www.buyforlife.com': 0.8787356591099948,-'staysaasy.com': 0.8786604306758078,-'www.okayhq.com': 0.8786525461110833,-'engineering.mixpanel.com': 0.878632488020514,-'www.netresec.com': 0.8786288106631085,-'jott.live': 0.8786251117226983,-'www.health.harvard.edu': 0.8786179702092969,-'plot.ly': 0.8785945787217043,-'www.edwardtufte.com': 0.8785389757836423,-'pzemtsov.github.io': 0.8785192835169581,-'grack.com': 0.878506371619462,-'www.aeonmagazine.com': 0.878499144479296,-'lowlevelbits.org': 0.8784722367844,-'blog.dotcloud.com': 0.8784660513882162,-'robwalling.com': 0.8784461152882206,-'luajit.org': 0.8784380131632007,-'mccormick.cx': 0.8784273467707013,-'www.usehaystack.io': 0.8784273467707013,-'stedolan.github.io': 0.8784164208081071,-'supermemo.guru': 0.8784156694332751,-'jackschaedler.github.io': 0.8783768191039716,-'mgba.io': 0.87835268885913,-'forum.nim-lang.org': 0.8783083943148878,-'sympa.inria.fr': 0.8783048144562394,-'lists.nongnu.org': 0.8783041478638947,-'lispcookbook.github.io': 0.8782936010037641,-'ds9a.nl': 0.8782772345442594,-'thinkpol.ca': 0.8782726019439232,-'itunes.apple.com': 0.8782672365580176,-'www.coursera.org': 0.8782670165903833,-'www.nrc.nl': 0.8782340201533003,-'www.faa.gov': 0.8781736885382665,-'artsandculture.google.com': 0.8781425787274354,-'smerity.com': 0.8781378214625035,-'www.omaha.com': 0.8781181772597978,-'www.wweek.com': 0.8781063159893508,-'www.stroustrup.com': 0.8780938693821273,-'sacks.substack.com': 0.8780908854852072,-'www.heraldsun.com.au': 0.8780821695385574,-'wiki.alopex.li': 0.8780784699174555,-'www.dartlang.org': 0.8780551661146612,-'bugs.php.net': 0.878032279338754,-'www.cyclingnews.com': 0.8779874213836478,-'www.minix3.org': 0.8779874213836478,-'www.cryptomuseum.com': 0.8779834887038765,-'blog.sendhub.com': 0.8779734555774865,-'web.williams.edu': 0.8779619274642066,-'sourceware.org': 0.8779595104655152,-'blog.revolutionanalytics.com': 0.8779228861519864,-'blog.feabhas.com': 0.877905135054773,-'rubenerd.com': 0.8778819789027571,-'longnow.org': 0.8778804834133449,-'www.kevinslin.com': 0.8778337531486146,-'testing.googleblog.com': 0.8778159226448956,-'merbist.com': 0.8778006932561861,-'academic.oup.com': 0.8777955202058847,-'www.romancortes.com': 0.8777775637794132,-'ottverse.com': 0.8777664608276228,-'panopticlick.eff.org': 0.8777635360894769,-'www.presslabs.com': 0.8777093280795232,-'www.archaeology.org': 0.8777019277787409,-'www.qt.io': 0.8776867090078414,-'www.rust-lang.org': 0.8776640621382866,-'komonews.com': 0.8776519468194085,-'petewarden.typepad.com': 0.877650269187265,-'redecentralize.org': 0.8776475140888633,-'spencerdailey.com': 0.8776383740955995,-'ryanbigg.com': 0.8776375837266788,-'codeascraft.com': 0.8776350192149127,-'davidsimon.com': 0.8776245936362499,-'nodered.org': 0.8775609145071156,-'releases.llvm.org': 0.8775465677232488,-'blog.npmjs.org': 0.8775321171093747,-'nothings.org': 0.8775219068062873,-'www.minecraft.net': 0.8775149658488136,-'www.nber.org': 0.8774912434535365,-'docs.keydb.dev': 0.8774559444971969,-'s3.amazonaws.com': 0.8774127148800118,-'www.startuplessonslearned.com': 0.8773991243759531,-'lostinjit.blogspot.com': 0.8773967208056322,-'unity3d.com': 0.8773730560374686,-'rtpg.co': 0.8773704171934261,-'meteuphoric.wordpress.com': 0.8773701347020316,-'c9.io': 0.8772877729098074,-'davidad.github.io': 0.8772551751572703,-'www.cs.tufts.edu': 0.8772142384084949,-'patents.google.com': 0.8771976309077076,-'carlschwan.eu': 0.8771903919813752,-'www.disneyresearch.com': 0.8771847497268329,-'kilianvalkhof.com': 0.8771650377417883,-'easylang.online': 0.8771572953832314,-'simonschreibt.de': 0.87714784842864,-'answers.onstartups.com': 0.8771043002987845,-'casetext.com': 0.8770972548772688,-'thekidshouldseethis.com': 0.8770970221777706,-'blog.bittorrent.com': 0.8770863340546824,-'www.phrack.org': 0.8770711101073438,-'blog.deepgram.com': 0.8770646346636887,-'grooveshark.com': 0.8770646346636887,-'www.stat.cmu.edu': 0.8770412624073355,-'matthewrayfield.com': 0.8770299470017662,-'blog.merovius.de': 0.8769629784425906,-'jordan-wright.github.io': 0.8769261031957445,-'www.michael-noll.com': 0.876870919176529,-'www.detroitnews.com': 0.876865566511843,-'www.chromestatus.com': 0.8768237137831741,-'spacy.io': 0.8767936477263165,-'blog.theoldreader.com': 0.8767904369417293,-'larrysummers.com': 0.8767904369417293,-'devo.ps': 0.8767685151051207,-'jackg.org': 0.8767448498103232,-'www.leancrew.com': 0.8767166755854882,-'blog.smartdec.net': 0.876696975867677,-'aaronparecki.com': 0.8766851750370125,-'eandt.theiet.org': 0.8766793394897466,-'www.lennysnewsletter.com': 0.8766464219782513,-'blog.factual.com': 0.8766406692255209,-'archlinux.me': 0.8765956164893757,-'cglab.ca': 0.8765746912638428,-'meteor.com': 0.8765739666365363,-'blog.jpalardy.com': 0.8765005864209046,-'amitkapila16.blogspot.com': 0.8764939037562264,-'en.yna.co.kr': 0.8764898808598593,-'linux.slashdot.org': 0.8764680273690748,-'www.trevormckendrick.com': 0.8764571075800416,-'cmcenroe.me': 0.8764460213566705,-'robertovitillo.com': 0.8764453376914981,-'metacademy.org': 0.8764331210191083,-'lethain.com': 0.8764062511281379,-'www.news.com.au': 0.876395323072595,-'www.severe-weather.eu': 0.8763844603500262,-'www.federalreserve.gov': 0.8763690497257808,-'passcode.csmonitor.com': 0.8763573796830383,-'chrismorgan.info': 0.8763466464048606,-'www.kurzweilai.net': 0.8763277380739884,-'www.federalregister.gov': 0.8763029852387336,-'carolchen.me': 0.8762960630948078,-'scholar.princeton.edu': 0.8762929484619637,-'limi.net': 0.8762759528715298,-'sijmen.ruwhof.net': 0.8761846124088934,-'winaero.com': 0.8761846124088934,-'www.techquila.co.in': 0.8761680078483304,-'www.theamericanconservative.com': 0.876167214287758,-'faculty.washington.edu': 0.8761620699999401,-'marijnhaverbeke.nl': 0.876139575410831,-'swizec.com': 0.8761258858184888,-'www.thebestpageintheuniverse.net': 0.876117496807152,-'www.epsilontheory.com': 0.8761124613754698,-'www.exurbe.com': 0.8761030084533606,-'www.motortrend.com': 0.8760487843348314,-'blog.seattlepi.com': 0.8759960447121663,-'www.enlightenment.org': 0.8759950518819543,-'www.seriouseats.com': 0.8759880790728855,-'appleinsider.com': 0.8759788729692606,-'stilldrinking.org': 0.8759642287018524,-'www.holistics.io': 0.8759580367004831,-'www.dannyguo.com': 0.8759212674654512,-'letsworkshop.com': 0.8759027598733787,-'www.androidauthority.com': 0.8758978540777702,-'visalist.io': 0.8758718179278925,-'www.makelinux.net': 0.8758439962639901,-'avikdas.com': 0.8758367116558062,-'www.linkedin.com': 0.8757903442097913,-'wiki.freebsd.org': 0.8757706173362011,-'www.tandfonline.com': 0.8757698347226249,-'www.pouet.net': 0.8757326866058377,-'blocksandfiles.com': 0.8756986319076105,-'www.docracy.com': 0.8756913215448502,-'www.useit.com': 0.8756472373396756,-'www.sapiens.org': 0.8756019742944366,-'www.ccn.com': 0.8755951918062195,-'www.mattblodgett.com': 0.8755948781981208,-'www.france24.com': 0.8755846335815239,-'www.zacharyburt.com': 0.8755387620665042,-'www.congress.gov': 0.8755256407121746,-'fab.cba.mit.edu': 0.8755115130516663,-'newsroom.mastercard.com': 0.8754825096495608,-'youexec.com': 0.8754809119780987,-'pragmaticpineapple.com': 0.8754604704436907,-'ruder.io': 0.8754604704436906,-'www.cpomagazine.com': 0.8754507502798615,-'fontsinuse.com': 0.8754438392621019,-'www.uscis.gov': 0.8754248448695789,-'youtube.com': 0.8754200542585205,-'movio.co': 0.8754182523171723,-'www.justwatch.com': 0.8754182523171723,-'www.seeedstudio.com': 0.8754129248508049,-'electrospaces.blogspot.com': 0.8754013788662113,-'journal.dedasys.com': 0.8753840466834827,-'css-tricks.com': 0.8753766177219949,-'googleenterprise.blogspot.com': 0.8753419411917504,-'radian.org': 0.8753349762136738,-'futurism.com': 0.8752759686541421,-'brandon.invergo.net': 0.875259685377418,-'blogs.bmj.com': 0.8752587898620211,-'cs.stanford.edu': 0.8752572315765397,-'www.law.nyu.edu': 0.8752557472640224,-'madebymike.com.au': 0.8752557472640223,-'www.seomoz.org': 0.8752420123288777,-'blog.invisiblethings.org': 0.8752416201881567,-'www.greghendershott.com': 0.8752189373052796,-'frankchimero.com': 0.8752104961969256,-'www.dodgycoder.net': 0.8752104961969256,-'txt.fyi': 0.8751961676198574,-'www.cisa.gov': 0.8751827659203362,-'www.failory.com': 0.8751736016299931,-'jamanetwork.com': 0.8751682514227755,-'www.objc.io': 0.8751399221981017,-'duo.com': 0.875136824237271,-'gs.statcounter.com': 0.8751360927111078,-'futurice.com': 0.8751180437027989,-'sifted.eu': 0.8751010675262672,-'laravel-news.com': 0.8750758486776293,-'archagon.net': 0.8750478337445297,-'tinyletter.com': 0.8750075762218684,-'blog.scoutapp.com': 0.875005539344818,-'tomorrowcorporation.com': 0.8750053569816645,-'blogs.opera.com': 0.8749949083342682,-'m.sfgate.com': 0.8749578015755867,-'blog.scottlogic.com': 0.8749532936904543,-'www.vincit.fi': 0.8749482978568669,-'www.hiringlab.org': 0.8749298773541315,-'adioso.com': 0.8749298773541315,-'technology.blog.gov.uk': 0.8748916088022032,-'intelligence.org': 0.874874784745018,-'musicmachinery.com': 0.874874767784566,-'www.spaceref.com': 0.8748480825746138,-'raisingfutureadults.blogspot.com': 0.8748200301964755,-'fantasticanachronism.com': 0.8747762139154207,-'calculist.org': 0.8747762139154207,-'lizthedeveloper.com': 0.8747656784427211,-'elliotjaystocks.com': 0.8747563598722315,-'www.julian.com': 0.8747501335716515,-'garbagecollected.org': 0.8747266158827341,-'timdaub.github.io': 0.8746432384113364,-'www.good.is': 0.8745679480603733,-'www.elidedbranches.com': 0.8745591524175645,-'people.gnome.org': 0.8745420458214501,-'goteleport.com': 0.8745075530736341,-'rome.ro': 0.8744782906989623,-'molily.de': 0.8744677514854211,-'www.douglas-self.com': 0.8744450764890996,-'www.rubymotion.com': 0.874400769900326,-'www.shubhro.com': 0.8743836350202087,-'blog.jim-nielsen.com': 0.8743167857862288,-'www.deseret.com': 0.8743016225209655,-'www.dallasnews.com': 0.8742922264773864,-'purecss.io': 0.874271743877152,-'lowercasecapital.com': 0.874271743877152,-'blog.getpaint.net': 0.874271743877152,-'google.com': 0.8742545289746798,-'betabeat.com': 0.874238729033988,-'dawn.cs.stanford.edu': 0.8742346653711291,-'futureofcoding.org': 0.8741693203974006,-'www.ocregister.com': 0.87415339402842,-'materialize.com': 0.8741062320515165,-'teddziuba.com': 0.8740975697255843,-'www.thelocal.de': 0.8740894106328552,-'webaim.org': 0.8740351003371842,-'www.firstthings.com': 0.8739883726292156,-'www.yodaiken.com': 0.8739612387765462,-'www.goodcheapandfast.com': 0.87394184329453,-'fastcompression.blogspot.com': 0.8739308258375799,-'perl6advent.wordpress.com': 0.8739142963181532,-'www.shirky.com': 0.873897843474887,-'www.baltimoresun.com': 0.873865725506546,-'www.greaterwrong.com': 0.8738154479524215,-'bastian.rieck.me': 0.8738148485847019,-'toastytech.com': 0.8738123717532574,-'raw.githubusercontent.com': 0.8738090863060529,-'www.wealthsimple.com': 0.8737884336019249,-'sanfrancisco.cbslocal.com': 0.8737831715062644,-'blog.palantir.com': 0.8736979166666666,-'learngitbranching.js.org': 0.8736979166666666,-'arr.am': 0.8736979166666666,-'i.blackhat.com': 0.8736899056814162,-'lolware.net': 0.873689463390705,-'sherylcanter.com': 0.8736870447303569,-'andrewgelman.com': 0.8736823027610198,-'www.stat.columbia.edu': 0.8736693395358077,-'pouchdb.com': 0.8736594278511989,-'www.compose.io': 0.8736483544069877,-'mleverything.substack.com': 0.8736159033763847,-'work.qz.com': 0.8736112922383825,-'lucasmreis.github.io': 0.8736055963750089,-'news.itsfoss.com': 0.8736019609008807,-'wiki.nikitavoloboev.xyz': 0.8735941094898686,-'developer.nvidia.com': 0.873569853920636,-'antigonejournal.com': 0.8734640566717642,-'spectator.us': 0.8734631581578451,-'www.businesswire.com': 0.8734611836531004,-'stressgrid.com': 0.87343280677026,-'pgbovine.net': 0.8734068690708945,-'forums.tigsource.com': 0.8733979453959029,-'thenewstack.io': 0.8733971641405377,-'blog.adamsmith.cc': 0.8733885158313304,-'www.gallup.com': 0.8733177322868378,-'www.institutionalinvestor.com': 0.873174323391906,-'allendowney.blogspot.com': 0.8731681454070807,-'www.lowrisc.org': 0.873141850865328,-'jshakespeare.com': 0.8731286306597288,-'www.nasaspaceflight.com': 0.8731126994667253,-'divan.github.io': 0.873106828361247,-'emacsconf.org': 0.873106828361247,-'www.scottporad.com': 0.8731061096250692,-'cam.ly': 0.8731039371172824,-'en.wikibooks.org': 0.8730999086484691,-'peak5390.wordpress.com': 0.8730838690655434,-'www.boredpanda.com': 0.8730796077702506,-'adam.heroku.com': 0.8730645256354151,-'transportevolved.com': 0.8730606539065758,-'jamesfriend.com.au': 0.8730468292001098,-'sethrobertson.github.io': 0.8730063430759272,-'kennethreitz.com': 0.872966789255468,-'nigeltao.github.io': 0.8728857599922942,-'www.damtp.cam.ac.uk': 0.8728857599922942,-'blog.chewxy.com': 0.872876574828518,-'savannah.gnu.org': 0.8728754513879475,-'www.sfexaminer.com': 0.872849372895737,-'engineering.pinterest.com': 0.8727923546520051,-'hitchdev.com': 0.8727791664778902,-'www.airlineratings.com': 0.8727662828334155,-'dt.deviantart.com': 0.8727436710427438,-'blog.dustincurtis.com': 0.8727333641616176,-'andymatuschak.org': 0.8727040209571875,-'ccc.de': 0.8727034120734908,-'www.cs.bham.ac.uk': 0.8726465691109567,-'www.robertputt.co.uk': 0.8726417417469949,-'www.miamiherald.com': 0.8725798697059735,-'blog.fitbit.com': 0.8725623923381794,-'adrianchadd.blogspot.com': 0.8725623923381794,-'developer.amazon.com': 0.8725538657953651,-'blog.codeship.com': 0.8725534393008655,-'www.attn.com': 0.8725475818041069,-'paleotronic.com': 0.8725451217616845,-'www.cdixon.org': 0.8725371731911513,-'cuddly-octo-palm-tree.com': 0.8725361366622865,-'www.flownet.com': 0.8725345822755496,-'www.sacrideo.us': 0.8725261893403989,-'newyorker.com': 0.8725219622684063,-'marmelab.com': 0.8725088072483861,-'help.twitter.com': 0.8724643326336655,-'browser.geekbench.com': 0.8724643326336655,-'blairreeves.me': 0.8724601687001106,-'thesquareplanet.com': 0.872441172969379,-'thesephist.com': 0.8724167847627841,-'www.freenas.org': 0.8723746839925903,-'kwokchain.com': 0.872332880121703,-'bostinnovation.com': 0.8723328354214289,-'love2d.org': 0.8723183994033862,-'en.people.cn': 0.872305563134482,-'blog.sucuri.net': 0.8722868151723697,-'tink.uk': 0.87225641869954,-'developer.palm.com': 0.8722310220579464,-'inessential.com': 0.8722306989053726,-'latimesblogs.latimes.com': 0.8722297510115247,-'startuplessonslearned.blogspot.com': 0.8721812850079301,-'blog.qt.digia.com': 0.8721548264341702,-'cssdeck.com': 0.8721274930836064,-'medium.freecodecamp.org': 0.8720631693223321,-'www.philly.com': 0.8720354776247603,-'nat.org': 0.8719937064288694,-'www.architecturaldigest.com': 0.871986927054336,-'adactio.com': 0.8719856028794241,-'anvil.works': 0.8719506746013774,-'accidentallyquadratic.tumblr.com': 0.8719251801589324,-'arduino.cc': 0.8718637797679766,-'blog.embed.ly': 0.8718082655702977,-'anyvite.com': 0.8718025672263069,-'www.eater.com': 0.871748228284988,-'moxie.org': 0.8717373817367493,-'www.jacquescorbytuech.com': 0.8717101335367482,-'n8n.io': 0.8716898518049913,-'commonsware.com': 0.871674162722329,-'www.snoyman.com': 0.8716637912464283,-'blog.patricktriest.com': 0.87161890252714,-'leaf.dragonflybsd.org': 0.8716189025271399,-'www.gamedeveloper.com': 0.8716189025271399,-'www.decisionsciencenews.com': 0.8716189025271399,-'primitivetechnology.wordpress.com': 0.8716187818992139,-'www.marinetraffic.com': 0.8716117229392578,-'www.dallasobserver.com': 0.8716085858849908,-'www.consumeraffairs.com': 0.871556493468071,-'english.aljazeera.net': 0.8715487596633263,-'eng.paxos.com': 0.871523178807947,-'scobleizer.com': 0.8715026146236272,-'endler.dev': 0.871459748976304,-'www.assemblyai.com': 0.871459748976304,-'spaceweatherarchive.com': 0.8714580347584702,-'www.todayifoundout.com': 0.871448243747315,-'www.itprotoday.com': 0.8714363693988826,-'physics.stackexchange.com': 0.8714320610312986,-'www.lifespan.io': 0.8714320342847328,-'pspdfkit.com': 0.8714289642359972,-'www.slackware.com': 0.8714078524585236,-'economix.blogs.nytimes.com': 0.8713770585361653,-'imba.io': 0.8713614361461072,-'blog.wizsec.jp': 0.8713527851458885,-'neal.fun': 0.8713508609752882,-'shift.newco.co': 0.8713309282655746,-'m50d.github.io': 0.8712974347864724,-'c.learncodethehardway.org': 0.8712925320326951,-'hosted.ap.org': 0.8712897256940902,-'www.onebigfluke.com': 0.8712867098269154,-'getmagicnow.com': 0.8712820888223272,-'projects.thestar.com': 0.8712523001988615,-'www.ralfj.de': 0.8712321679229643,-'www.vantage.sh': 0.871207076528529,-'www.linuxfoundation.org': 0.8711369679250435,-'openclassroom.stanford.edu': 0.8711347101538987,-'usefyi.com': 0.8711273706889674,-'www.bingocardcreator.com': 0.8711151425360408,-'blog.robenkleene.com': 0.8711151425360408,-'mako.cc': 0.8711144740115356,-'blog.frontapp.com': 0.8710729457593172,-'virtuallyfun.com': 0.8710620996646494,-'www.csmonitor.com': 0.8710243224833312,-'www.nintendo.co.uk': 0.8710106382978724,-'codon.com': 0.8709949725685586,-'www.sixthtone.com': 0.870972915708058,-'statweb.stanford.edu': 0.8709617791625288,-'www.selectedintelligence.com': 0.8709463428439262,-'www.elasticsearch.org': 0.8709352679197123,-'math.mit.edu': 0.870930298916698,-'dwheeler.com': 0.870930298916698,-'kim.com': 0.8709205575405985,-'www.nbcsandiego.com': 0.8708959915522153,-'paragonie.com': 0.8708785801404334,-'bostonherald.com': 0.8708776881584194,-'mathwithbaddrawings.com': 0.8708565076256994,-'www.openshot.org': 0.8708388814913449,-'bugs.python.org': 0.8708284581145338,-'blog.aerofs.com': 0.8708080233174353,-'www.chicagomag.com': 0.8707562016036294,-'www.250bpm.com': 0.8707223328636163,-'datagenetics.com': 0.8706854622162113,-'blogs.embarcadero.com': 0.870674156132316,-'www.ispreview.co.uk': 0.8706343121704596,-'www.twistlock.com': 0.8706274964354553,-'www.lukew.com': 0.8705263186498488,-'news.efinancialcareers.com': 0.8705226226571788,-'www.travelandleisure.com': 0.8705014801192988,-'blog.okturtles.com': 0.8704939919893191,-'ilsr.org': 0.8704939919893191,-'www.usb.org': 0.8704796759014012,-'dish.andrewsullivan.com': 0.8704773071162746,-'likewise.am': 0.8704734916123844,-'blog.idonethis.com': 0.8704583890982127,-'blog.talosintelligence.com': 0.8704526519003233,-'googlesystem.blogspot.com': 0.8704373858850157,-'garrysub.posterous.com': 0.8704372803921456,-'insinuator.net': 0.8704372803921456,-'hypothes.is': 0.8704167324964841,-'cdt.org': 0.8704025091193442,-'apod.nasa.gov': 0.8703900488888665,-'herman.bearblog.dev': 0.8703870042603329,-'archiveteam.org': 0.8703718279113771,-'seths.blog': 0.8703521175616137,-'blog.internetcases.com': 0.8703208556149733,-'olifro.st': 0.870316353450948,-'www.commerce.gov': 0.8703148755143479,-'cosmosmagazine.com': 0.8703108288261813,-'onemileatatime.com': 0.8703062566192509,-'blog.cardiogr.am': 0.8702780879767176,-'www.nsf.gov': 0.8702217832404014,-'michaelfeathers.silvrback.com': 0.8702081612825068,-'cse.buffalo.edu': 0.8701754328744842,-'www.cardiff.ac.uk': 0.8701754328744842,-'stuartkhall.com': 0.8701745691078596,-'www.ablogtowatch.com': 0.8701496552122142,-'kripken.github.io': 0.8701188385936537,-'www.submarinecablemap.com': 0.8700961746199228,-'blog.pandora.com': 0.8700581031783425,-'undark.org': 0.8700047089370274,-'indiestatik.com': 0.8699989528159702,-'www.ldeming.com': 0.8699989528159702,-'www.shadertoy.com': 0.8699979474186827,-'blog.forecast.io': 0.8699840133880231,-'www.thumbtack.com': 0.8699483761483322,-'neopythonic.blogspot.com': 0.8699126451206446,-'blog.yoshuawuyts.com': 0.8699103901809758,-'meaningness.com': 0.8698796561423923,-'www.climatecentral.org': 0.8698732077608144,-'phasenoise.livejournal.com': 0.8698692075920079,-'solarianprogrammer.com': 0.8698554554648813,-'www.allenpike.com': 0.869850220276853,-'prometheus.io': 0.8698088579434151,-'www.zocalopublicsquare.org': 0.8698034656609108,-'jsonresume.org': 0.8697986577181208,-'ts.data61.csiro.au': 0.8697986577181208,-'downloadsquad.switched.com': 0.8697589511534172,-'chris-said.io': 0.8697498026681764,-'www.eurekalert.org': 0.8697479182812706,-'rise.cs.berkeley.edu': 0.8697428562146504,-'nav.al': 0.8696282036759441,-'www.contextis.com': 0.8696220176972772,-'alpha.app.net': 0.8695846584298926,-'simon.shimmerproject.org': 0.8695798593001582,-'blog.andyet.com': 0.8695385830681973,-'harvardmagazine.com': 0.8695167508879433,-'montrealgazette.com': 0.8695147149611512,-'www.ossblog.org': 0.8694501584181511,-'www.snazz.xyz': 0.8694481830417228,-'securethoughts.com': 0.8694070114575367,-'www.dcrainmaker.com': 0.8694035765566716,-'static.googleusercontent.com': 0.869401063536612,-'www.cyberscoop.com': 0.8693847613582892,-'engineering.imvu.com': 0.8693718520967112,-'cs.nyu.edu': 0.869371790902293,-'github.hubspot.com': 0.8693590367674685,-'zenodo.org': 0.8693126875353983,-'www.nczonline.net': 0.8693117689767528,-'awealthofcommonsense.com': 0.8692734628001552,-'jcp.org': 0.8692722371967655,-'www.tabletmag.com': 0.869255146058989,-'www.spritzinc.com': 0.8692337048510355,-'www.pl-enthusiast.net': 0.8692303984396749,-'blog.rtwilson.com': 0.8692255308065057,-'parislemon.com': 0.8692052486630297,-'www.perimeterx.com': 0.869171185060124,-'blog.ndepend.com': 0.8691625738297151,-'www.gregreda.com': 0.8691455800391642,-'www.cooper.com': 0.8691426419439985,-'newsroom.ibm.com': 0.869132356539317,-'www.core77.com': 0.8691279702829294,-'paraschopra.com': 0.869122539211249,-'openspace.sfmoma.org': 0.8691093062023977,-'blog.stephenmarz.com': 0.8691082953129042,-'blog.workshape.io': 0.8690653365846199,-'ngrok.com': 0.8690653365846199,-'code.flickr.net': 0.8690543850923815,-'uk.pcmag.com': 0.8690031200561459,-'superpowered.com': 0.868989779225976,-'hunterwalk.medium.com': 0.8689812194478416,-'www.cs.tau.ac.il': 0.8689490020361462,-'littlemaninmyhead.wordpress.com': 0.8689216938948838,-'www.excamera.com': 0.8689189189189189,-'bjorn.tipling.com': 0.8689189189189189,-'people.idsia.ch': 0.8688594113666208,-'www.olark.com': 0.8688594113666208,-'blog.sigplan.org': 0.8688223104414097,-'www.volokh.com': 0.8687744918859344,-'betterprogramming.pub': 0.8687724684006338,-'www.consumerreports.org': 0.8687417291110767,-'spin.atomicobject.com': 0.8687241428724921,-'opencv.org': 0.8687019086527302,-'evincarofautumn.blogspot.com': 0.8687019086527302,-'blog.imgtec.com': 0.868681889755656,-'pauladamsmith.com': 0.86866369377526,-'neosmart.net': 0.8686249309907671,-'reprints.longform.org': 0.8686085856898127,-'www.xda-developers.com': 0.8685957498546732,-'reefpoints.dockyard.com': 0.8685213588812543,-'benguild.com': 0.8684664223004914,-'www.naughtycomputer.uk': 0.8684379703659447,-'www.kb.cert.org': 0.8684268393607657,-'www.insidescience.org': 0.868400521356853,-'motorola-blog.blogspot.com': 0.8683887736567877,-'op-co.de': 0.8683853459972863,-'pvk.ca': 0.8683840412350319,-'roadsandkingdoms.com': 0.8683501222568618,-'www.netbsd.org': 0.8682992634857859,-'lite.cnn.io': 0.8682859224905237,-'redeye.firstround.com': 0.8682663428825431,-'forum.xda-developers.com': 0.8682587280913919,-'posthog.com': 0.8682495044484383,-'ftp.iza.org': 0.868245075106375,-'craphound.com': 0.8682258194103606,-'leancrew.com': 0.8682221213291426,-'www.rijksmuseum.nl': 0.8682073993067194,-'blog.newsweek.com': 0.8681920308533846,-'www.hipmunk.com': 0.8681741501104386,-'fuchsia.dev': 0.8681336649460718,-'infrequently.org': 0.8680443914407565,-'www.nationalreview.com': 0.8680234764373578,-'www.futilitycloset.com': 0.8680090490995264,-'blogs.apache.org': 0.8679900476857304,-'www.hayabusa2.jaxa.jp': 0.8679812862496842,-'www.lenovo.com': 0.867894246896281,-'malisper.me': 0.8678702820090507,-'people.debian.org': 0.8678402113450426,-'blog.jwhitham.org': 0.8678305149745539,-'hai.stanford.edu': 0.8677844697267021,-'hubicka.blogspot.com': 0.8677805560584517,-'bemmu.posterous.com': 0.8677440846959065,-'www.gizmodo.com.au': 0.8677203766504066,-'meyerweb.com': 0.8676812093352608,-'www.novaspivack.com': 0.8676756246000905,-'www.theamericanscholar.org': 0.8676692275096932,-'www.pjrc.com': 0.8676602643652892,-'googleforwork.blogspot.com': 0.8676558691706335,-'get.slack.help': 0.8676558691706335,-'mwi.usma.edu': 0.8676236271344514,-'blogs.zoho.com': 0.8676226844533295,-'www.aaronstannard.com': 0.8676196165257838,-'www.messynessychic.com': 0.867572253713323,-'uk.businessinsider.com': 0.8675716785772413,-'www.defectivebydesign.org': 0.867569136323548,-'bash.org': 0.8675094991983033,-'venturehacks.com': 0.8675094413288004,-'www.thepublicdiscourse.com': 0.8674973517990539,-'browser.engineering': 0.8674925582906508,-'www.peachesnstink.com': 0.8674597307626667,-'www.ics.uci.edu': 0.8674482943120847,-'www.cs.utah.edu': 0.8674397196409499,-'www.inkstonenews.com': 0.8674036054993348,-'www.privacytools.io': 0.8673748578488399,-'brianknapp.me': 0.8673589537571962,-'www.mediaite.com': 0.867339960916398,-'www.theinquirer.net': 0.8673345006090067,-'www.dhruvonmath.com': 0.867331172642827,-'lucb1e.com': 0.8672987335530669,-'www.hetzner.de': 0.8672930834878059,-'blogs.nature.com': 0.8672878554675312,-'newsfeed.time.com': 0.8672878554675312,-'davemart.in': 0.8672750734639297,-'cryptopals.com': 0.8672750734639297,-'paperswithcode.com': 0.8672082397249838,-'blog.hipmunk.com': 0.8671864413349992,-'citizen428.net': 0.8671759938812859,-'shitpost.plover.com': 0.8671759938812859,-'developer.github.com': 0.8671757822917692,-'www.lispcast.com': 0.86717529476292,-'www.webofstories.com': 0.8671647243985804,-'blog.asana.com': 0.8671537327631247,-'vvvvalvalval.github.io': 0.8671420654436023,-'news.northeastern.edu': 0.8671175664071602,-'www.digitalmars.com': 0.8671085676412111,-'www.mysk.blog': 0.8671085676412111,-'www.janestreet.com': 0.8671085676412111,-'francescodilorenzo.com': 0.8671085676412111,-'www.binghamton.edu': 0.8671020117333935,-'measuringmeasures.blogspot.com': 0.8671015509192971,-'blog.businessofsoftware.org': 0.8670987836087559,-'www.dougengelbart.org': 0.8670867654048934,-'tkainrad.dev': 0.8670687156162842,-'www.machinelearningplus.com': 0.8670617301641067,-'200ok.ch': 0.8670503631928145,-'quizlet.com': 0.8669907032962705,-'www.singlelunch.com': 0.8669898915969994,-'chairnerd.seatgeek.com': 0.8669768798830123,-'www.govtech.com': 0.8669681804482694,-'news.mongabay.com': 0.8669281682982936,-'erikmcclure.com': 0.8669236451292578,-'aelkus.github.io': 0.8669193008503213,-'www.pyimagesearch.com': 0.8669169539583714,-'www.thejournal.ie': 0.8669063286580795,-'reaktor.com': 0.8669062156202597,-'christophermeiklejohn.com': 0.8668972800697652,-'globaleconomicanalysis.blogspot.com': 0.8668972800697652,-'reasonml.github.io': 0.8668408053269024,-'www.csiro.au': 0.8668352643092083,-'www.maxmasnick.com': 0.8668020425330096,-'www.bitmatica.com': 0.8667582417582418,-'bbs.archlinux.org': 0.8667582417582418,-'bookofhook.blogspot.com': 0.8667582417582418,-'rappers.mdaniels.com.s3-website-us-east-1.amazonaws.com': 0.8667582417582418,-'milapneupane.com.np': 0.8667582417582418,-'lists.randombit.net': 0.8667581999829783,-'tyrrrz.me': 0.8667581999829783,-'research.checkpoint.com': 0.8667536287753966,-'avdi.org': 0.8667515410500497,-'tuxphones.com': 0.8667177933546236,-'wolfstreet.com': 0.8667059036925511,-'www.danielmiessler.com': 0.8666932411921048,-'klarasystems.com': 0.8666904333167735,-'blog.leahhanson.us': 0.8666699206546021,-'lua-users.org': 0.8666542828163675,-'blog.tjll.net': 0.866647835447029,-'massivelyop.com': 0.8665522518442965,-'kuow.org': 0.8665345404342943,-'www.chinafile.com': 0.8665345404342943,-'www.allaboutcircuits.com': 0.8665332702488167,-'www.physorg.com': 0.8665324818274975,-'www.altdevblogaday.com': 0.8665315510980502,-'codeahoy.com': 0.8664780210212021,-'perl.plover.com': 0.8664701135603498,-'www.wpri.com': 0.8664576956212064,-'www.brianchristner.io': 0.8664469380333186,-'srconstantin.wordpress.com': 0.8664373069106099,-'yeoman.io': 0.8663968845512826,-'pixelastic.github.io': 0.8663968845512826,-'www.structuredprocrastination.com': 0.8663817900462257,-'www.galactanet.com': 0.8663440641046102,-'alexmuir.com': 0.8663020356625262,-'www.cs.washington.edu': 0.866214599696902,-'gapingvoid.com': 0.866214599696902,-'doisinkidney.com': 0.8662068965517241,-'jeffknupp.com': 0.866196580751142,-'weblogs.asp.net': 0.8661868876649387,-'www.eclipse.org': 0.8661815854172948,-'www.twitter.com': 0.8661688444094342,-'www.gmo.com': 0.8660899350618185,-'h14s.p5r.org': 0.8660899350618185,-'www.privatdozent.co': 0.8660524023972933,-'conorpp.com': 0.8660524023972933,-'dev-blog.apollodata.com': 0.8660261935749697,-'www.bof.nl': 0.8660220994475138,-'www.potaroo.net': 0.8660076718528887,-'www.disneyanimation.com': 0.8659912587210011,-'perspectives.mvdirona.com': 0.8659285828108513,-'digiconomist.net': 0.8659107168851126,-'futhark-lang.org': 0.8658910633267001,-'blog.stephanbehnke.com': 0.8658778264013361,-'www.themillions.com': 0.8658146270171843,-'www.charlespetzold.com': 0.8657610099215972,-'dev.opera.com': 0.8657226869029186,-'www.contrast.ie': 0.8657095900925946,-'threatpost.com': 0.8656950945183971,-'diginomica.com': 0.8656889877908728,-'www.iro.umontreal.ca': 0.865673655242536,-'goodui.org': 0.8656691714697559,-'kevinmahoney.co.uk': 0.8656053786488649,-'den.dev': 0.8655508375322314,-'blog.danlew.net': 0.8655009175546347,-'insidermag.net': 0.8654928424374734,-'eviltrout.com': 0.8654668848568644,-'bartwronski.com': 0.8653908212138248,-'asserttrue.blogspot.com': 0.8653811565244239,-'laughtonelectronics.com': 0.8653462441760643,-'www.toomas.net': 0.8653462441760643,-'www.peoplespolicyproject.org': 0.8653036598492724,-'www.kpcb.com': 0.865280579032659,-'engineering.stanford.edu': 0.8652651824846332,-'www.mcsweeneys.net': 0.8652414926783502,-'www.weeklystandard.com': 0.8652325002095317,-'scratch.mit.edu': 0.8652264286658259,-'therideshareguy.com': 0.8651696061186824,-'hyperboleandahalf.blogspot.com': 0.8651696061186824,-'evhead.com': 0.8651553668857692,-'praeclarum.org': 0.8651529057353353,-'riowang.blogspot.com': 0.8651391622821514,-'www.rs-online.com': 0.86513559976471,-'blog.linuxmint.com': 0.8651351630976869,-'blog.soylent.me': 0.8651258007804876,-'cabel.me': 0.8650904033379694,-'wildbit.com': 0.8650667180545558,-'www.huffingtonpost.com': 0.86501806564583,-'code.nasa.gov': 0.8649895088434025,-'zenhabits.net': 0.8649439383347666,-'dl.dropbox.com': 0.8649350287137462,-'devblog.avdi.org': 0.8648862222006666,-'blog.phpfog.com': 0.8648344406451537,-'www.thetransportpolitic.com': 0.8648252523162882,-'forestry.io': 0.8648063640773068,-'www.dabeaz.com': 0.8647748270793637,-'www.brightball.com': 0.8647346279381389,-'style.org': 0.8647346279381389,-'www.elezea.com': 0.8647230821512976,-'www.aarp.org': 0.864718332495695,-'themillions.com': 0.8646781650668347,-'saastr.quora.com': 0.8646637505280962,-'ottawacitizen.com': 0.8646581352452648,-'mkremins.github.io': 0.8646201560571354,-'edge.org': 0.8646127068735515,-'micropython.org': 0.8645605097088179,-'www.metamorphblog.com': 0.8645587566462556,-'blog.jez.io': 0.8645309820862669,-'aatishb.com': 0.8644462651203315,-'www.celinehh.com': 0.8644462651203315,-'davidyat.es': 0.8644462651203315,-'code.joejag.com': 0.8644356837498463,-'www.washingtonexaminer.com': 0.864373005099176,-'www.resetera.com': 0.8643497804356822,-'quantum.country': 0.8643112341108541,-'savingjournalism.substack.com': 0.8642759030508148,-'ahungry.com': 0.8642759030508148,-'gamozolabs.github.io': 0.8642719268946735,-'jamespanderson.tumblr.com': 0.8642680643637904,-'elementaryos.org': 0.8642208492393608,-'www.debuggex.com': 0.8642032789469397,-'www.planetizen.com': 0.8641967719680171,-'www.eweek.com': 0.864172798382722,-'tatiyants.com': 0.8641038586159879,-'discover.dc.nihr.ac.uk': 0.8640971396521386,-'web.cs.ucdavis.edu': 0.8640703175243611,-'xmpp.org': 0.8640502790628004,-'chriskohlhepp.wordpress.com': 0.8640478440979216,-'mystartuphas30daystolive.tumblr.com': 0.8640252337365474,-'rhelblog.redhat.com': 0.8640126486430193,-'www.electricitymap.org': 0.8639997125170845,-'americasmarkets.usatoday.com': 0.8639946073685961,-'projects.propublica.org': 0.8639224455926199,-'e-estonia.com': 0.8639219016557921,-'blog.basho.com': 0.8639181305846296,-'ungleich.ch': 0.863900787970001,-'squid314.livejournal.com': 0.8638782213420932,-'www.cassandracapital.net': 0.8638782213420932,-'aiyprojects.withgoogle.com': 0.8638467210374888,-'bsdimp.blogspot.com': 0.8638147888998915,-'jwt.io': 0.8638117466836764,-'www.esri.com': 0.8638089133245348,-'www.gaborcselle.com': 0.8637930884909581,-'venam.nixers.net': 0.8637807788335469,-'www.7pace.com': 0.8637364419671971,-'www.joshmcguigan.com': 0.8637220704617038,-'promarket.org': 0.8636701430867854,-'tav.espians.com': 0.8636677390061345,-'www.datastax.com': 0.8636570251868989,-'www.cmyr.net': 0.8636232605564672,-'mainisusuallyafunction.blogspot.com': 0.8636109472670038,-'www.hisutton.com': 0.8635963039783531,-'lsub.org': 0.8635700664090139,-'alpblog.heroku.com': 0.8635306327438487,-'gpuopen.com': 0.8635262100242561,-'nvd.nist.gov': 0.8634973542011698,-'english.elpais.com': 0.8634616688985162,-'www.craigslist.org': 0.863445932794428,-'nostalgebraist.tumblr.com': 0.8634392689015953,-'www.hakaimagazine.com': 0.8634375812585431,-'marcan.st': 0.8634342519721849,-'wiki.debian.org': 0.8633915097045765,-'php.net': 0.8633877833122457,-'benjaminreinhardt.com': 0.8633802816901408,-'blog.pnas.org': 0.86336052660942,-'www.adityaravishankar.com': 0.86336052660942,-'mobile.twitter.com': 0.8633312362141495,-'www.alistapart.com': 0.863319649821934,-'msgpack.org': 0.8633096782467932,-'keionline.org': 0.8633096782467932,-'www.logicalclocks.com': 0.8632631941253387,-'svedic.org': 0.8632309919400734,-'newsroom.unsw.edu.au': 0.8632254714408825,-'maxwellito.github.io': 0.8632001968821827,-'www.commandlinefu.com': 0.8631417710298782,-'www.clockworkpi.com': 0.8631140242937411,-'www.typography.com': 0.8630969520774667,-'alumni.stanford.edu': 0.8630912751881218,-'www.sigarch.org': 0.8630849989436014,-'www.countbayesie.com': 0.8630846691596683,-'blog.echen.me': 0.8630803489219858,-'groovehq.com': 0.863062038855598,-'calteches.library.caltech.edu': 0.863062038855598,-'www.rochester.edu': 0.8630082813677696,-'blog.bjrn.se': 0.8629931779073361,-'lawandcrime.com': 0.8629659867085913,-'betashop.com': 0.862958801056521,-'www.ericsink.com': 0.8629424736221599,-'engineering.linkedin.com': 0.8629291758709224,-'cleantechnica.com': 0.8629063952275409,-'meltingasphalt.com': 0.8628160615079228,-'amiunique.org': 0.8628087618596978,-'bvckup2.com': 0.8628005657708628,-'ieftimov.com': 0.8627657236548011,-'ewanvalentine.io': 0.8627020659905763,-'secalerts.co': 0.8627018556487716,-'www.patrick-wied.at': 0.8626866200547734,-'www.pitt.edu': 0.8626432815245133,-'hothardware.com': 0.8626338662999138,-'www.appleinsider.com': 0.8626260620914765,-'hillelwayne.com': 0.8626238486818072,-'gridstylesheets.org': 0.8626062322946175,-'sciencex.com': 0.8626007339716192,-'developers.slashdot.org': 0.86255044240195,-'blog.confluent.io': 0.8624968367523541,-'glandium.org': 0.8624955638682134,-'arandomurl.com': 0.8624894125730017,-'blog.nugget.one': 0.8624812956348196,-'www.rd.com': 0.8624301242273775,-'slifty.com': 0.8624113475177305,-'www.lighttable.com': 0.8624113475177305,-'payments.amazon.com': 0.8624113475177305,-'www.lurklurk.org': 0.862401566562782,-'derstandard.at': 0.862401566562782,-'workstuff.tumblr.com': 0.8623257751778021,-'www.lvh.io': 0.8623241846862385,-'rs.io': 0.8623129582427793,-'eslint.org': 0.862298463525373,-'www.cucumbertown.com': 0.8622891062009898,-'jquery.com': 0.8622837910960226,-'z.cash': 0.8622770331746279,-'www.ottawacitizen.com': 0.8622229843756593,-'nanonets.com': 0.8621765227456938,-'www.crunchgear.com': 0.8621559229318876,-'mattruby.substack.com': 0.8621556881643411,-'mbuffett.com': 0.8621556881643411,-'cybergibbons.com': 0.8621443416178289,-'www.lua.org': 0.8621439982277107,-'monodraw.helftone.com': 0.8621272923907152,-'vicki.substack.com': 0.8621255550009437,-'www.davidgcohen.com': 0.8621062295375266,-'jfrog.com': 0.8620661061833018,-'joostdevblog.blogspot.com': 0.8620661061833018,-'johnkary.net': 0.8620331171141896,-'jakobgreenfeld.com': 0.8620331171141896,-'bosker.wordpress.com': 0.8619844782836177,-'encosia.com': 0.8619710029888306,-'martinkool.com': 0.8619319552747774,-'use-the-index-luke.com': 0.8619185059392996,-'siteanalytics.compete.com': 0.8619154305011097,-'madewithlove.be': 0.8619031112487354,-'blog.digg.com': 0.8619019650013046,-'wildlyinaccurate.com': 0.8618675915764481,-'news.agu.org': 0.8618622951257247,-'www.hnsearch.com': 0.8618535032580247,-'bbc.com': 0.8618329035365963,-'thevaluable.dev': 0.8618329035365963,-'cve.mitre.org': 0.8618136566919955,-'www.r2d3.us': 0.8618128425706462,-'www.wsbtv.com': 0.8617435427240897,-'wiki.php.net': 0.8617039994341269,-'regex101.com': 0.861640779434757,-'www.fastcodesign.com': 0.8616335728873852,-'waitbutwhy.com': 0.8616048195074264,-'wrapbootstrap.com': 0.8615996055432654,-'www.oracle.com': 0.8615524291467332,-'angrystaffofficer.com': 0.8615283953190601,-'pythonprogramming.net': 0.8615283953190601,-'codemirror.net': 0.8615237207033272,-'erthalion.info': 0.8615038163918213,-'news.wsu.edu': 0.8615031765236854,-'jmtd.net': 0.8614607909822787,-'www.businessinsider.com.au': 0.8614418925622166,-'www.keithschwarz.com': 0.8614202461594263,-'blog.testdouble.com': 0.8614030589984097,-'rationalconspiracy.com': 0.8613615814645382,-'michaelochurch.wordpress.com': 0.8613601391266915,-'blogs.berkeley.edu': 0.8613538103460412,-'blog.superfeedr.com': 0.8613308060874829,-'hn-books.com': 0.8613200444920518,-'techsolidarity.org': 0.8613153663268618,-'annualletter.gatesfoundation.org': 0.8613153663268618,-'www.ardanlabs.com': 0.8612708268948052,-'www.mapbox.com': 0.8612700375538782,-'protonvpn.com': 0.8612360355266332,-'www.winrumors.com': 0.8611989655340138,-'www.arcticstartup.com': 0.8611695091473522,-'blogs.akamai.com': 0.8611395195993665,-'penntoday.upenn.edu': 0.8611371780676135,-'www.howitactuallyworks.com': 0.8611264027197298,-'www.theage.com.au': 0.8611116664910229,-'blogs.computerworld.com': 0.8610945749876592,-'news.artnet.com': 0.8610390836029311,-'yongfook.com': 0.8610327646216055,-'ask.metafilter.com': 0.8610302755520367,-'blog.zencoder.com': 0.8609595040886218,-'labs.spotify.com': 0.8609558167629312,-'ashfurrow.com': 0.8609452045938362,-'rachelandrew.co.uk': 0.8609370906301821,-'xem.github.io': 0.860936923468393,-'natureofcode.com': 0.8609280600555332,-'www.daedtech.com': 0.8609275762383382,-'jnd.org': 0.8609065111890492,-'cryptoslate.com': 0.8608950862608346,-'googlepolicyeurope.blogspot.com': 0.8608494490780891,-'electricliterature.com': 0.8608434109180494,-'www.cs.toronto.edu': 0.8608065788416125,-'mtgox.com': 0.8608060590690018,-'laughingsquid.com': 0.8607982963555786,-'nickdesaulniers.github.io': 0.8607761141579771,-'news.netcraft.com': 0.8607587653123538,-'www.mattfriz.com': 0.8607469264593318,-'blogs.computerworlduk.com': 0.8606535841782695,-'weblog.jamisbuck.org': 0.8606463446127512,-'particletree.com': 0.860619018859172,-'www.loopinsight.com': 0.8605648031182908,-'blog.qualys.com': 0.86055640956746,-'moz.com': 0.860495291636908,-'vimeo.com': 0.8604892664547054,-'www.chalmers.se': 0.860431654676259,-'homepage.ntlworld.com': 0.8604242343472561,-'www.jpost.com': 0.8604154523480785,-'www.governor.ny.gov': 0.8603434360710707,-'www.teenvogue.com': 0.8603394119266393,-'blog.bufferapp.com': 0.8603306431458172,-'pharr.org': 0.8603247905476864,-'www.cforcoding.com': 0.8603197271193488,-'mixpanel.com': 0.8603064014189074,-'bits-please.blogspot.com': 0.860301774508791,-'waypoint.vice.com': 0.8602882797773976,-'elpais.com': 0.8602682993832683,-'www.thevintagenews.com': 0.8602498276023521,-'bdickason.com': 0.8602305475504323,-'immunant.com': 0.8601526618229086,-'madhadron.com': 0.8601439808675682,-'www.dwheeler.com': 0.8600331290988332,-'www.vim.org': 0.8600002647579501,-'blog.prototypr.io': 0.859998158168068,-'webpolicy.org': 0.8599719205993361,-'coding.smashingmagazine.com': 0.8599291407790329,-'wizardzines.com': 0.8598885971042833,-'www.itnews.com.au': 0.8598724037037537,-'www.codethinked.com': 0.8598208830600297,-'www.technologynetworks.com': 0.8598085045605989,-'abclocal.go.com': 0.8598024771455696,-'blog.wolframalpha.com': 0.8597915197356469,-'blog.juxt.pro': 0.8597891002608672,-'newsroom.tiktok.com': 0.8597628867114886,-'defector.com': 0.8597558066643536,-'apps.apple.com': 0.8597457842210768,-'www.jamesyu.org': 0.8597029099342595,-'www.dailytech.com': 0.8596655487230563,-'elie.net': 0.8596315170135848,-'linear.app': 0.8596237337192475,-'clerk.house.gov': 0.8596237337192475,-'keepachangelog.com': 0.8596075354545292,-'featherweightmusings.blogspot.com': 0.8596075354545292,-'eng.localytics.com': 0.8595938002959398,-'emacsrocks.com': 0.8595762853230908,-'dassur.ma': 0.8595420928346226,-'www.sightline.org': 0.859531902937568,-'www.gamesradar.com': 0.8595242144121686,-'sysdig.com': 0.8595202139290916,-'www.cloudkick.com': 0.8595183443817482,-'devopsu.com': 0.859493942235387,-'believermag.com': 0.8594892467285535,-'unhandledexpression.com': 0.8594741803219609,-'efficiencyiseverything.com': 0.8594569005504344,-'www.therobotreport.com': 0.859433089374823,-'www.howtographql.com': 0.8594202898550725,-'archive.today': 0.8593643247719568,-'www.jacobinmag.com': 0.8593589358679861,-'progrium.com': 0.8593468157466189,-'redfin.engineering': 0.8593466877552149,-'www.zillow.com': 0.8593304013971459,-'www.nexedi.com': 0.8593210096380903,-'madaidans-insecurities.github.io': 0.8593210096380903,-'home.treasury.gov': 0.8592910289545252,-'scholar.harvard.edu': 0.8592733512209211,-'opensourceconnections.com': 0.8592682967350698,-'bsdly.blogspot.com': 0.8592666292050983,-'aras-p.info': 0.8592595047254012,-'battlepenguin.com': 0.8592397001892514,-'solokeys.com': 0.8592264014510466,-'www.alexkyte.me': 0.8591861061610623,-'www.digitalmusicnews.com': 0.8591609711742595,-'kenneth.io': 0.8591232018072773,-'mywiki.wooledge.org': 0.8590993783573315,-'epic.org': 0.8590931061006938,-'behavioralscientist.org': 0.8590931061006938,-'jack.minardi.org': 0.8590798567329014,-'zeptobars.ru': 0.8590350573746124,-'bramcohen.com': 0.8590132997607233,-'frankforce.com': 0.8590074352869126,-'erniemiller.org': 0.8590074352869125,-'robservatory.com': 0.8589982689488246,-'engineering.squarespace.com': 0.8589982689488246,-'www.artofmanliness.com': 0.8589698957080373,-'blogs.bing.com': 0.8589523074975127,-'www.world-nuclear-news.org': 0.8589473343420543,-'openmirage.org': 0.8589305193408414,-'prism-break.org': 0.8589248369205337,-'slab.com': 0.8589172698916326,-'monkeylearn.com': 0.8589030969473112,-'journals.sagepub.com': 0.8588825279469426,-'metaredux.com': 0.8588307379036221,-'www.eurogamer.net': 0.8588239617965691,-'www.wisdomination.com': 0.8587259126214145,-'src.chromium.org': 0.8586820383432011,-'www.tudelft.nl': 0.8586674667651873,-'blog.project-insanity.org': 0.8586633233128,-'hadihariri.com': 0.8586515656051527,-'blog.xot.nl': 0.8586508044813143,-'www.nypost.com': 0.8586366736605365,-'andrewchenblog.com': 0.8586246895566572,-'jordan-wright.com': 0.8586221711446911,-'journal.burningman.org': 0.858622171144691,-'blog.tensorflow.org': 0.8586139292072289,-'googlewebmastercentral.blogspot.com': 0.8586030875489932,-'standblog.org': 0.858600583090379,-'ant.design': 0.858600583090379,-'overapi.com': 0.8585933703160669,-'www.suntimes.com': 0.8585797957168473,-'thecable.foreignpolicy.com': 0.8585540700891416,-'earther.gizmodo.com': 0.8584865305709988,-'www.bestinclass.dk': 0.8584597924729722,-'hn.algolia.com': 0.8584545053239319,-'gwillem.gitlab.io': 0.8584541318402437,-'elbenshira.com': 0.8584541318402437,-'bankunderground.co.uk': 0.8583675490979057,-'www.crowdsupply.com': 0.8583166788787248,-'lists.linuxfoundation.org': 0.8583007929896874,-'mikehadlow.blogspot.com': 0.8583007929896874,-'prideout.net': 0.8582618132398327,-'joinup.ec.europa.eu': 0.8582375872301639,-'coderoom.wordpress.com': 0.8582225134679472,-'andrewoneverything.com': 0.8582199611092117,-'peterlyons.com': 0.8581871345029239,-'facebook.com': 0.8581759828028657,-'www.vogue.com': 0.8581630406503709,-'future.a16z.com': 0.8581407364644609,-'www.thedotpost.com': 0.8581294866962557,-'davidwalsh.name': 0.85807314827283,-'www.heise.de': 0.8580470280497292,-'regex.info': 0.8580444001760147,-'blog.jayway.com': 0.8580313921739998,-'www.discovermagazine.com': 0.8580086852563162,-'www.gamespot.com': 0.858006803378577,-'www.mattmazur.com': 0.8579544667792158,-'0x80.pl': 0.8579258153670124,-'pine64.com': 0.8579245737044574,-'www.gog.com': 0.8579147324679443,-'jdh.hamkins.org': 0.857912419834021,-'openjdk.java.net': 0.8578856841040577,-'code.djangoproject.com': 0.8578761757124823,-'airbnb.io': 0.8578759796916386,-'www.neatorama.com': 0.8578511503394977,-'intridea.com': 0.8578447662759853,-'techchannel.att.com': 0.8578445745748121,-'mule.substack.com': 0.8578445745748121,-'www.fastmailstatus.com': 0.857833367118916,-'nas.sr': 0.8578228253764041,-'cloudblogs.microsoft.com': 0.8578026799771654,-'danielbmarkham.com': 0.8577690138809076,-'blogs.kde.org': 0.8577593606755378,-'graphics.latimes.com': 0.8577592314786884,-'www.wordfence.com': 0.8577572254294423,-'quariety.com': 0.8577565674618444,-'www.ritholtz.com': 0.8577423661740396,-'jasonadriaan.com': 0.8576809443893051,-'joelcalifa.com': 0.8576809443893051,-'www.cs.umd.edu': 0.857661758811538,-'zacharyvoase.com': 0.857656134895176,-'www.decisionproblem.com': 0.8576347888613035,-'steamdb.info': 0.8576311826444905,-'blog.chef.io': 0.8576130873561624,-'techraptor.net': 0.8575933683258181,-'www.nanog.org': 0.857579845060974,-'www.cs.yale.edu': 0.8575769410337604,-'blogs.skype.com': 0.8575392141865088,-'igoro.com': 0.8575204710620608,-'www.math.toronto.edu': 0.8574671949652505,-'tether.to': 0.8574671949652504,-'adrianmejia.com': 0.8573817628591522,-'adblockplus.org': 0.8573641935296868,-'techxplore.com': 0.8573012751875033,-'professional-troublemaker.com': 0.8572977062348858,-'idealab.talkingpointsmemo.com': 0.8572223861972004,-'nesslabs.com': 0.8572196165500797,-'www.nationalgeographic.co.uk': 0.8571954684569548,-'iridakos.com': 0.8571705482234938,-'theagilepanda.com': 0.8571428571428571,-'togototo.wordpress.com': 0.8571428571428571,-'blog.skyliner.io': 0.8571354628699216,-'eu.techcrunch.com': 0.8571248597503665,-'datastudio.google.com': 0.8571140921734792,-'www.django-cms.org': 0.8570965178487575,-'blog.path.com': 0.8570914140075269,-'www.villagevoice.com': 0.857081275011955,-'somehowmanage.com': 0.857055205499945,-'www.mnn.com': 0.8570470364584489,-'abovethelaw.com': 0.8570463120033517,-'blog.logrocket.com': 0.8570182021331599,-'www.apress.com': 0.8569721488001009,-'www.news.cornell.edu': 0.8569706892788174,-'www.curbed.com': 0.8569597747993827,-'digitizor.com': 0.856955455369854,-'blog.lareviewofbooks.org': 0.8569521195010459,-'www.cambus.net': 0.8569380680580816,-'blog.rize.io': 0.8568954207368967,-'2ality.com': 0.8568509445047684,-'ux.stackexchange.com': 0.856847117644151,-'www.thedomains.com': 0.8568397055760405,-'timesofindia.indiatimes.com': 0.856835710146828,-'examine.com': 0.8568332988347195,-'speirs.org': 0.8568142257362032,-'learnbyexample.github.io': 0.8568010189328112,-'cs.brown.edu': 0.8567908824604603,-'www.dnalounge.com': 0.8567908041212858,-'www.nbcbayarea.com': 0.8567028634763063,-'macperformanceguide.com': 0.8566332090796122,-'social.techcrunch.com': 0.856633209079612,-'gocardless.com': 0.8566325894779986,-'www.driverless.id': 0.8565431663151448,-'simonsarris.substack.com': 0.856507155170115,-'blog.l0cal.com': 0.856499398164963,-'poy.time.com': 0.8564993981649629,-'labs.ripe.net': 0.8564206791243404,-'britishlibrary.typepad.co.uk': 0.8564116348023619,-'directorsblog.nih.gov': 0.8563781052354208,-'www.trustedsec.com': 0.8563276469242246,-'www.nicholaswindsorhoward.com': 0.8563276469242246,-'wefunder.com': 0.8563033480182963,-'grpc.io': 0.8563005637859097,-'www.innovationaus.com': 0.8562719072138797,-'algorithmwatch.org': 0.856263282232775,-'kylewritescode.com': 0.8562626989641361,-'www.iquilezles.org': 0.8562567878094767,-'www.doc.ic.ac.uk': 0.8561753420541418,-'www.r-bloggers.com': 0.8561384095933265,-'www.kxan.com': 0.8561304599253935,-'blog.shodan.io': 0.856101606479241,-'blog.couchdb.org': 0.8560857595989197,-'randsinrepose.com': 0.8560683987937139,-'blogs.igalia.com': 0.8560658162982266,-'erikhoel.substack.com': 0.8560552035555758,-'www.columbia.edu': 0.8560502120089583,-'adambard.com': 0.8560134134615376,-'xania.org': 0.8559909491087049,-'pinboard.in': 0.8559460897276601,-'lobste.rs': 0.8559156074776079,-'interestingengineering.com': 0.8558912746106632,-'www.dilbert.com': 0.855888464977386,-'jcdav.is': 0.8558808822556393,-'zwbetz.com': 0.8558715995865966,-'npr.org': 0.8558692421991085,-'compellingsciencefiction.com': 0.8558593149932564,-'headmelted.com': 0.8558229432148337,-'blog.itlater.com': 0.8558223030664946,-'www.joystiq.com': 0.8557770050631482,-'blog.loadzero.com': 0.8557371228445998,-'www.exploringbinary.com': 0.8557365123930964,-'www.reason.com': 0.8557121404409874,-'towardsdatascience.com': 0.8556587797122474,-'ostif.org': 0.855654761904762,-'articles.latimes.com': 0.855649649030829,-'www.cs.ucr.edu': 0.8556340456009993,-'www.transportenvironment.org': 0.8556309211595626,-'www.sciencenews.org': 0.8556293530573256,-'blog.portswigger.net': 0.855625404687521,-'dshipper.posterous.com': 0.8556180321566556,-'abopen.com': 0.8555419933626652,-'www.twitch.tv': 0.8555327933571086,-'www.realclearscience.com': 0.8555038318613801,-'www.llnl.gov': 0.8554731239960308,-'brandmark.io': 0.8554590545897166,-'www.eflorenzano.com': 0.8554573688896427,-'play.google.com': 0.8554165451041797,-'www.ethanzuckerman.com': 0.8553991433840825,-'lunduke.substack.com': 0.8553770673931268,-'www.montulli.org': 0.8553552610577579,-'www.dvorak.org': 0.8553507205142731,-'pythoninsider.blogspot.com': 0.855339221122335,-'octopart.com': 0.8553291642797942,-'ithare.com': 0.8553097001258215,-'isc.sans.edu': 0.8552998663643834,-'doi.org': 0.8552929122812928,-'www.starling-software.com': 0.855240263021274,-'sedimental.org': 0.8552238805970149,-'www.nitrokey.com': 0.8552078417856235,-'www.fool.com': 0.8551227533080648,-'ntietz.com': 0.8551173351343933,-'thepointmag.com': 0.8551125688254727,-'www.symantec.com': 0.8551104394486714,-'highline.huffingtonpost.com': 0.8550936296505218,-'blog.adacore.com': 0.8550882114429865,-'techland.time.com': 0.8550381807790381,-'www-03.ibm.com': 0.855023796089594,-'blog.davidtate.org': 0.8550204472368161,-'blog.gitlab.org': 0.8549666512282963,-'blog.dijit.sh': 0.8549645308164788,-'boredzo.org': 0.8549332224976328,-'www.bishopfox.com': 0.8549242562625317,-'blog.sproutcore.com': 0.8548755303332917,-'www.nakedcapitalism.com': 0.8548518987841709,-'hackaday.io': 0.8548507362141421,-'www.nccgroup.com': 0.8548006867421194,-'docs.microsoft.com': 0.8547764550021021,-'www.kansas.com': 0.8547447271228046,-'www.myfonts.com': 0.8547443058333942,-'www.databreaches.net': 0.8547388607511365,-'design.google': 0.8546474649453593,-'www.bvp.com': 0.8546145212136451,-'eriwen.com': 0.854565525324503,-'flightaware.com': 0.854535067640072,-'us2.campaign-archive1.com': 0.854531163801279,-'engineering.skroutz.gr': 0.8544744689256348,-'arcentry.com': 0.8544398366164802,-'redmonk.com': 0.8544165230751263,-'jesseduffield.com': 0.8543827046117866,-'www.foodandwine.com': 0.8543802950724103,-'www.jacobsschool.ucsd.edu': 0.8543584327039565,-'www.vandenoever.info': 0.8543584327039565,-'tubetime.us': 0.8543543543543544,-'xenbits.xen.org': 0.85435049306025,-'blogs.sun.com': 0.8543496525648886,-'nofilmschool.com': 0.8543378482464506,-'joi.ito.com': 0.8543316742759597,-'blog.garlicsim.org': 0.8543018539678975,-'akr.am': 0.854281488649609,-'www.nypl.org': 0.8542056110271361,-'blog.opendns.com': 0.8542056110271361,-'shmuplations.com': 0.8542031037522354,-'blog.statwing.com': 0.8542005291800069,-'m.bbc.com': 0.8541979742875379,-'boston.conman.org': 0.8541978420073731,-'lilium.com': 0.8541923613616496,-'phk.freebsd.dk': 0.8541720175820052,-'underthehood.meltwater.com': 0.8541376157274252,-'www.airliners.net': 0.8541362938673078,-'www.gobankingrates.com': 0.8541353383458646,-'maxplanck.nautil.us': 0.8540985777081339,-'leandromoreira.com.br': 0.8540707480810653,-'24ways.org': 0.8540629407199574,-'www.federaltimes.com': 0.8540125885303304,-'blog.trello.com': 0.8540087030052186,-'www.slash7.com': 0.8539974953863674,-'www.projectrho.com': 0.8539630717568143,-'yahoopolicy.tumblr.com': 0.8539187064781893,-'pencil.evolus.vn': 0.8539187064781893,-'getkerf.wordpress.com': 0.8539187064781893,-'newfoodeconomy.org': 0.853885872320658,-'www.geekosystem.com': 0.8538783892446405,-'bengtan.com': 0.8538510433274764,-'dustin.github.com': 0.8538486125176874,-'jasonlefkowitz.net': 0.8538441285736149,-'teamtreehouse.com': 0.8538268240476388,-'blog.quarkslab.com': 0.853807684540482,-'www.artnome.com': 0.8537866111759274,-'blog.chron.com': 0.8537460072706317,-'blog.close.io': 0.8537188598181367,-'lonesysadmin.net': 0.8536953242835595,-'www.lemonde.fr': 0.8536772759118291,-'www.guernicamag.com': 0.8536142442449905,-'codeascraft.etsy.com': 0.8536109251434499,-'blog.oleganza.com': 0.8535993869171182,-'www.poetryfoundation.org': 0.8535856365759876,-'archive.computerhistory.org': 0.8535651705660577,-'www.philzimmermann.com': 0.8535638214444478,-'www.sqlalchemy.org': 0.8535347372406801,-'www.rubyinside.com': 0.8535314579904915,-'rjbs.manxome.org': 0.8534743202416919,-'www.itl.nist.gov': 0.8534657928784027,-'lab.simurai.com': 0.8534578819472406,-'www.shortcutfoo.com': 0.8534538731861366,-'www.asymco.com': 0.8534535683227724,-'johnmathews.eu': 0.8534414067561554,-'www.msnbc.msn.com': 0.8534205452840495,-'www.cs.brown.edu': 0.8534113013764459,-'blog.vivekhaldar.com': 0.8534010422454464,-'www.t3.org': 0.8533802695967294,-'coloradosun.com': 0.8533683929679808,-'www.gamesindustry.biz': 0.8533666288339562,-'www.benjoffe.com': 0.8533494797004003,-'windows.microsoft.com': 0.8533490913172264,-'codeblog.jonskeet.uk': 0.8533155100042978,-'mattdesl.svbtle.com': 0.8532886204160572,-'wilshipley.com': 0.8532666831748213,-'lunduke.com': 0.8532546707850448,-'blog.documentcloud.org': 0.8532526475037822,-'pratt.duke.edu': 0.8532505235078635,-'forum.palemoon.org': 0.8532499119645807,-'ring.cx': 0.8532409206090049,-'www.healthimaging.com': 0.8532409206090049,-'blog.lawnstarter.com': 0.853234538226177,-'www.drorpoleg.com': 0.853234538226177,-'www.abajournal.com': 0.8532334026694913,-'volokh.com': 0.8532246743179585,-'www.akitasoftware.com': 0.8532127948822967,-'blog.algolia.com': 0.8531662937825892,-'cgit.freedesktop.org': 0.8531345095634363,-'creativegood.com': 0.8531247497260012,-'www.bizjournals.com': 0.8530952665861291,-'krypt.co': 0.85304727815771,-'david-peter.de': 0.8530425774087601,-'www.bitcoin.org': 0.853030303030303,-'www.markdotto.com': 0.853030303030303,-'donsbot.wordpress.com': 0.8530217272624623,-'joyent.com': 0.8530153570981687,-'googleplusplatform.blogspot.com': 0.8530153000371625,-'modern-products.tumblr.com': 0.8530153000371625,-'hashicorp.com': 0.8529426622739795,-'www.tuaw.com': 0.8529238643536575,-'jessepollak.me': 0.8528766579532876,-'www.spectrum.ieee.org': 0.8528722385076482,-'evasi0n.com': 0.8527890991537469,-'continuum.io': 0.8527523717085685,-'inconshreveable.com': 0.8527291982231584,-'washingtonmonthly.com': 0.8527169016435672,-'www.go350.com': 0.8526967137013025,-'newhumanist.org.uk': 0.8526876181950579,-'www.thelocal.se': 0.8526535411240088,-'arielelkin.github.io': 0.8526249122103083,-'ldpreload.com': 0.8526076535424412,-'blog.valerieaurora.org': 0.8526007849075418,-'tympanus.net': 0.8526005626429696,-'www.pewresearch.org': 0.8525869052062965,-'gcaptain.com': 0.8525025061695275,-'www.fireeye.com': 0.8524966902771423,-'mkdev.me': 0.8524961199751095,-'thoughtmaybe.com': 0.8524858010387742,-'blog.lukaszolejnik.com': 0.8524707074061916,-'getmondo.co.uk': 0.852437604863552,-'open.bufferapp.com': 0.852434953761713,-'alumni.berkeley.edu': 0.8524296840496752,-'forums.macrumors.com': 0.8523683544548385,-'alastairreid.github.io': 0.8523644066969532,-'www.scottberkun.com': 0.852359208523592,-'ontwik.com': 0.8523482955385304,-'git-scm.com': 0.8523082962459579,-'news.un.org': 0.8522696996619386,-'www.adafruit.com': 0.8522616020934412,-'www.mathjax.org': 0.8522588583128298,-'www.chinalawblog.com': 0.8522208346251817,-'yahoolabs.tumblr.com': 0.8522048661669432,-'www.dw.de': 0.8521590754317577,-'www.udacity.com': 0.8521539143105872,-'dianeravitch.net': 0.8521341463414633,-'www.narrator.ai': 0.852097863618245,-'broadbandnow.com': 0.852097863618245,-'genius.com': 0.8520940394549934,-'theaircurrent.com': 0.8519939439576096,-'www.bytecellar.com': 0.8519428423381916,-'www.rifters.com': 0.8519032600490041,-'www.math.upenn.edu': 0.8518649443865483,-'algorithmsoup.wordpress.com': 0.8518190799236097,-'ellegriffin.substack.com': 0.8518190799236097,-'mikewest.org': 0.8517824188629416,-'loyaltylobby.com': 0.8516821355411338,-'blog.hut8labs.com': 0.8516819571865444,-'www.metmuseum.org': 0.8516671986689323,-'www-users.cs.umn.edu': 0.8516590199664924,-'undo.io': 0.8516245163512932,-'news.discovery.com': 0.8515880430654605,-'hulry.com': 0.8515702882943715,-'www.greenheartgames.com': 0.8515702882943715,-'ie.microsoft.com': 0.8515602032387396,-'www.thehindu.com': 0.8515477928389076,-'www.ably.io': 0.8515404721157463,-'chrispenner.ca': 0.8514969974322367,-'www.logodesignlove.com': 0.8514733938627516,-'people.ischool.berkeley.edu': 0.8514695464544928,-'timothycrosley.github.io': 0.8514548238897397,-'www.neocomputer.org': 0.8514548238897397,-'m.motherjones.com': 0.8514463981383273,-'ro-che.info': 0.8514172067988788,-'www.ofnumbers.com': 0.8514160013087799,-'rdist.root.org': 0.8514077817668315,-'blog.alexellis.io': 0.8514045841413571,-'www.notebookcheck.net': 0.8513578715363501,-'new.livestream.com': 0.8513411202985242,-'www.sonyaellenmann.com': 0.8513367975602456,-'builtin.com': 0.8513357366412594,-'thecritic.co.uk': 0.8513136478418425,-'bitworking.org': 0.851299858555184,-'iwantmyname.com': 0.8512894868376749,-'www.circleid.com': 0.851254578211912,-'woboq.com': 0.8512444088430177,-'www.instructables.com': 0.8512139095333348,-'markmail.org': 0.8512136385469307,-'da-data.blogspot.com': 0.8511940319698232,-'www.inverse.com': 0.8511748736762975,-'robinhood.com': 0.8511710385891778,-'blog.tojicode.com': 0.8511685296886505,-'usa.streetsblog.org': 0.8511588204743716,-'sanderknape.com': 0.851058381116051,-'technical.ly': 0.8510465053531708,-'www.trek10.com': 0.8510328925328215,-'theprint.in': 0.8510207129082807,-'000fff.org': 0.8509998804696824,-'journal.frontiersin.org': 0.8509377801906453,-'on.wsj.com': 0.850920176035645,-'sircmpwn.github.io': 0.8508709080357458,-'www.networkworld.com': 0.8508447666287313,-'apps.bostonglobe.com': 0.8508308486054927,-'pfrazee.hashbase.io': 0.8508225328373329,-'www.commentarymagazine.com': 0.8508100679431329,-'petewarden.com': 0.8507826894953255,-'www.exeter.ac.uk': 0.8507809015713883,-'www.statwing.com': 0.8507692307692307,-'www.unenvironment.org': 0.8507692307692307,-'www.paintcodeapp.com': 0.8507513412377187,-'andreschweighofer.com': 0.8507309300531638,-'forums.unrealengine.com': 0.8507306931858649,-'www.wikidata.org': 0.8507275608332324,-'draculatheme.com': 0.8507209915723823,-'hongkongfp.com': 0.8507154665320323,-'psmag.com': 0.8507120935262249,-'blog.flightcaster.com': 0.850683949182937,-'vancouversun.com': 0.8506812814664019,-'www.cc.gatech.edu': 0.8506656522986229,-'news.wisc.edu': 0.8506650572190997,-'madebymany.com': 0.850661432792337,-'www.afp.com': 0.8506381100610537,-'cudl.lib.cam.ac.uk': 0.8505963477950123,-'www.huyng.com': 0.8505963477950121,-'www.jeremymorgan.com': 0.8505908966433915,-'thereformedbroker.com': 0.8505772906521217,-'bigocheatsheet.com': 0.8505365958446607,-'matduggan.com': 0.850500461698309,-'deis.io': 0.8504704364122011,-'poolp.org': 0.8504003265009309,-'alteredqualia.com': 0.8503991615824809,-'product.hubspot.com': 0.8503749082035804,-'www.erlang-factory.com': 0.8503149886076794,-'paranoia.dubfire.net': 0.850273893946855,-'concurrencyfreaks.blogspot.com': 0.850256626665339,-'www.cato-at-liberty.org': 0.8502310137099974,-'databasearchitects.blogspot.com': 0.8502084998761508,-'www.netflixprize.com': 0.8501934998584227,-'www.utilitydive.com': 0.8501445783462607,-'www.olioapps.com': 0.850137018218358,-'rcrowley.org': 0.850105596274167,-'kentonv.github.io': 0.8500969474584005,-'hastebin.com': 0.8500573664501953,-'www.dotnetfoundation.org': 0.850018648725371,-'mattmazur.com': 0.8499955866420006,-'breakoutstartups.substack.com': 0.8499877911408882,-'www.platformonomics.com': 0.8499799037669202,-'www.groovehq.com': 0.8499412891580047,-'alf.nu': 0.8499136585227424,-'antoinevastel.com': 0.8498761519134193,-'liveweave.com': 0.8498511311102708,-'causal.app': 0.8498452012383901,-'liw.fi': 0.8498452012383901,-'www.paypal.com': 0.8497695268466062,-'redis.io': 0.8497424748729502,-'collectiveidea.com': 0.8496851714802387,-'felleisen.org': 0.8496847068727177,-'blog.nobugware.com': 0.8496603118181073,-'www.windowscentral.com': 0.8496154708571939,-'www.xobni.com': 0.8495968094932279,-'outage.report': 0.8495880396284066,-'www.justsecurity.org': 0.849566454817199,-'blog.rfox.eu': 0.8495467369935737,-'way-cooler.org': 0.8495467369935737,-'adamierymenko.com': 0.849538361390885,-'mesosphere.com': 0.8495051866724436,-'munchies.vice.com': 0.8495023041501157,-'meatfighter.com': 0.8495018599221447,-'dsc.discovery.com': 0.8494918847267826,-'sciencenordic.com': 0.8494773842761618,-'uxmovement.com': 0.8494740695810459,-'death.andgravity.com': 0.8494550556394679,-'www.idlewords.com': 0.8493746001892146,-'staffeng.com': 0.8493531997757849,-'godottutorials.pro': 0.8493096657310281,-'earthobservatory.nasa.gov': 0.8493083274285178,-'www.gladwell.com': 0.8492393112893071,-'www.cgpgrey.com': 0.8492221809584313,-'www.dailymail.co.uk': 0.8492114987186158,-'source.opennews.org': 0.8492058945166755,-'mikecanex.wordpress.com': 0.849180109488917,-'unsharpen.com': 0.8491592756377232,-'haseebq.com': 0.8491526060447331,-'blog.diasporafoundation.org': 0.849136592576036,-'www.pugetsystems.com': 0.8491094043930865,-'roadmap.sh': 0.8491002596731841,-'www.rawstory.com': 0.8490683893365794,-'www.nylas.com': 0.8490240168878735,-'www.tunetheweb.com': 0.8489936415581292,-'www.subtraction.com': 0.8489862226571359,-'adam.blog.heroku.com': 0.8489616756127251,-'www.gdcvault.com': 0.8489549021982677,-'n26.com': 0.8489269016604418,-'www.passportindex.org': 0.8489269016604418,-'www.sfweekly.com': 0.8489155778996055,-'wiki.csswg.org': 0.8489096573208723,-'remoteok.io': 0.8488395634361555,-'www.guru3d.com': 0.8488079037622022,-'www.wisdom.weizmann.ac.il': 0.8487791265218381,-'fred.stlouisfed.org': 0.8487687545118902,-'www.quantcast.com': 0.8487674782234258,-'www.dspguide.com': 0.8487415764723351,-'monitor.firefox.com': 0.8487326723774654,-'www.manchester.ac.uk': 0.8487326070292353,-'www.ecowatch.com': 0.8487187193792909,-'exponents.co': 0.8487052872629832,-'ubere.ng': 0.8486844576029813,-'lists.cs.uiuc.edu': 0.8486469141688284,-'www.erasmatazz.com': 0.8485843687777063,-'prestonbyrne.com': 0.8485821394042593,-'fakesteve.blogspot.com': 0.8485803980992321,-'andyet.net': 0.8485375978388094,-'blog.getfirebug.com': 0.8485293903506373,-'joshtronic.com': 0.8485255486310321,-'ngm.nationalgeographic.com': 0.8484980049334815,-'www.cat-bus.com': 0.8484880761635857,-'oos.moxiecode.com': 0.8484880761635857,-'dlo.me': 0.8484699284663549,-'www.openbazaar.org': 0.8484375,-'tumbledry.org': 0.8484375,-'boris-marinov.github.io': 0.8484375,-'www.pcmag.com': 0.8484334813622002,-'wsj.com': 0.8484234837206328,-'ssd.eff.org': 0.8484099350915519,-'www.worldometers.info': 0.8483883403559179,-'www.denverpost.com': 0.8483741516385197,-'blog.teambox.com': 0.8482961926347361,-'www.staff.science.uu.nl': 0.8482766434239408,-'chromium-review.googlesource.com': 0.8482702404753355,-'www.antoarts.com': 0.8482404035930796,-'www.redcode.nl': 0.8482404035930796,-'www.morningstar.com': 0.8482320069770474,-'www.nomachetejuggling.com': 0.8481585934322239,-'influxdata.com': 0.8481227202820945,-'ericholscher.com': 0.8481140078411193,-'www.nfx.com': 0.8481012921108911,-'wellpreparedmind.wordpress.com': 0.8480622757690688,-'kara.allthingsd.com': 0.8480371905314535,-'taptaptap.com': 0.8480285039672184,-'rhizome.org': 0.8479990358005195,-'c-faq.com': 0.8479835930179948,-'lighttable.com': 0.8478713990382907,-'www.ocweekly.com': 0.8478713990382906,-'www.yesmagazine.org': 0.8478423076321161,-'boston.cbslocal.com': 0.8477838372324837,-'sethvargo.com': 0.847722900935325,-'www.teamliquid.net': 0.8477100468877423,-'kn100.me': 0.8476871371630261,-'carsonified.com': 0.8476327362582359,-'makecode.com': 0.8476290144215641,-'www.leejamesrobinson.com': 0.8476290144215641,-'journals.lww.com': 0.8476245506194432,-'philfreo.com': 0.8476129588207604,-'insights.som.yale.edu': 0.8475823333113045,-'fredwu.me': 0.8475685309018646,-'www.desertsun.com': 0.8475476379514038,-'blog.noctua-software.com': 0.8475078292773957,-'engineering.twitter.com': 0.8474442201925468,-'elazzabi.com': 0.847403765595545,-'camendesign.com': 0.8473892733773356,-'spaceflightnow.com': 0.8473844409998718,-'www.cs.uic.edu': 0.8473371233913075,-'www.lpalmieri.com': 0.8473049751829188,-'www.call-with-current-continuation.org': 0.8472612335852765,-'pub.gajendra.net': 0.8472440944881889,-'write.privacytools.io': 0.8471964059191153,-'blog.gitprime.com': 0.8471964059191153,-'deepstreamhub.com': 0.8471964059191153,-'fosspost.org': 0.8471934496110164,-'www.staradvertiser.com': 0.8471904258144719,-'daggle.com': 0.8471392026866065,-'text.sourcegraph.com': 0.8471325439607902,-'graphics.reuters.com': 0.8470805937710428,-'www.pulumi.com': 0.8470801636194034,-'deepstream.io': 0.8470226785546888,-'shop.oreilly.com': 0.8469734259689878,-'www.rebol.com': 0.8469659113684828,-'nylas.com': 0.8469368267588653,-'sethgodin.typepad.com': 0.8469363395112108,-'oilprice.com': 0.8469313973547731,-'www.itsnicethat.com': 0.8469219290040122,-'www.theblackvault.com': 0.8468972040529525,-'jboriss.wordpress.com': 0.8468972040529525,-'jeena.net': 0.8468879391324495,-'www.mixnode.com': 0.8468817868572549,-'danco.substack.com': 0.8468470187947723,-'dyno-might.github.io': 0.8468364833167144,-'extremelearning.com.au': 0.8467447980712459,-'news.crunchbase.com': 0.8467020525586185,-'the-japan-news.com': 0.8466850596011346,-'www.tor.com': 0.8466742350710085,-'graphics.pixar.com': 0.8466553705876948,-'jackmott.github.io': 0.846643484398615,-'www.deekit.com': 0.8466140516084654,-'gkbrk.com': 0.8466121179458332,-'washingtonexaminer.com': 0.846542334844308,-'www.jayhuang.org': 0.8465047427289335,-'lunchbag.ca': 0.8464844037983796,-'www.vanschneider.com': 0.8464718715474767,-'www.seobook.com': 0.8464370807656654,-'lfe.io': 0.8464044390292613,-'www.unitedstatescourts.org': 0.8463713866332889,-'www.listennotes.com': 0.8463577921119287,-'techblog.livingsocial.com': 0.8463371185563517,-'blogoscoped.com': 0.8463347353586499,-'www.density.io': 0.8463125089091263,-'dbasch.posterous.com': 0.8462528298095997,-'martiancraft.com': 0.8462422366153325,-'blog.meldium.com': 0.8461981486766226,-'blog.codeclimate.com': 0.8461758267783162,-'www.abtinforouzandeh.com': 0.8461646472697207,-'rachelnabors.com': 0.8461583302534714,-'zencoder.com': 0.8461094533062384,-'www.itworld.com': 0.8460477806046857,-'blog.ipinfo.io': 0.846031746031746,-'www.popsci.com': 0.8460295003872216,-'www.autoweek.com': 0.8460205561195527,-'bbot.org': 0.8459877925136515,-'crookedtimber.org': 0.8459783522151496,-'www.cultofmac.com': 0.8459180081962092,-'en.wikisource.org': 0.8458715274833618,-'www.kaggle.com': 0.8458458101824604,-'www.europarl.europa.eu': 0.8458356717488885,-'benchmarksgame.alioth.debian.org': 0.8458109252610799,-'librepcb.org': 0.8457869634340223,-'www.nationalobserver.com': 0.8457869634340223,-'skitch.com': 0.8457433438672782,-'www.bfilipek.com': 0.8457240963150365,-'scottaaronson.com': 0.8456937780186609,-'fee.org': 0.8456866411435615,-'www.doomworld.com': 0.8456828105532045,-'librivox.org': 0.845661772928326,-'www.crispymtn.com': 0.845661772928326,-'www.nydailynews.com': 0.8455856575577351,-'brilliantmaps.com': 0.8455577844591506,-'tiamat.tsotech.com': 0.8455576461751069,-'npf.io': 0.8455538969123215,-'jonathan.bergknoff.com': 0.8455538969123215,-'belitsoft.com': 0.8455477879801876,-'blog.thecybershadow.net': 0.8455414012738853,-'blog.privacytools.io': 0.8455414012738853,-'hardmath123.github.io': 0.8455383067302876,-'elifesciences.org': 0.8455333018767387,-'www.nemil.com': 0.845471203066534,-'web.eecs.umich.edu': 0.8454557317112926,-'500hats.typepad.com': 0.8454234389310107,-'jsonfeed.org': 0.8454122902628886,-'opendatastructures.org': 0.8454122902628886,-'tex.stackexchange.com': 0.8454020068893373,-'yifan.lu': 0.8453802898253059,-'www.sierraclub.org': 0.8453802898253059,-'axisofeval.blogspot.com': 0.845371248915586,-'www.lbatalha.com': 0.8453195049851959,-'theferrett.livejournal.com': 0.8452950558213717,-'libmill.org': 0.8452950558213717,-'www.epa.gov': 0.8452050900524932,-'tricycle.org': 0.8451979459749552,-'gking.harvard.edu': 0.8451920342583992,-'seattletimes.nwsource.com': 0.8451827104383299,-'killscreen.com': 0.8451534193795461,-'www.pixeltrue.com': 0.8451429290751707,-'www.randalolson.com': 0.8451252770638041,-'en.blog.wordpress.com': 0.8450869071274824,-'blog.thijsalkema.de': 0.845084400536627,-'www.zephoria.org': 0.8450267106265884,-'paulromer.net': 0.8450058741046084,-'blog.gtk.org': 0.8449976566750211,-'blog.flattr.net': 0.8449113529737324,-'seedmagazine.com': 0.8448457357890011,-'www.rt.com': 0.8448330221761664,-'technosophos.com': 0.8448317661673463,-'monzo.com': 0.844824966969565,-'www.imperial.ac.uk': 0.8448169716366182,-'andreyf.tumblr.com': 0.8448123793732512,-'renesd.blogspot.com': 0.8448109932557698,-'www.secretgeek.net': 0.8447678779550777,-'alistapart.com': 0.8447602813793536,-'facebook.design': 0.8446854923268982,-'sachachua.com': 0.8446808335848872,-'www.codesections.com': 0.8446790832287159,-'davidwong.fr': 0.844658979317956,-'blogs.office.com': 0.8446378605952511,-'root.cern.ch': 0.8446152758619639,-'indieweb.org': 0.8446147837430709,-'blog.reemer.com': 0.8446120410396897,-'www.change.org': 0.844610318859491,-'www.mrspeaker.net': 0.8446002192794787,-'www.sun-sentinel.com': 0.8445943201382862,-'gorban.org': 0.844551282051282,-'www.bgr.com': 0.8445425484905292,-'coinmarketcap.com': 0.8445032134621009,-'0day.work': 0.8444799253517218,-'blog.realkinetic.com': 0.8444710920604793,-'m.bbc.co.uk': 0.844465510423491,-'earther.com': 0.8444605232524242,-'link.springer.com': 0.8444514328769025,-'positech.co.uk': 0.8444262068957591,-'www.onthemedia.org': 0.8444066696037038,-'www.cleveralgorithms.com': 0.8444062415001815,-'mailchi.mp': 0.8443098020754962,-'www.pixelapse.com': 0.8443017656500802,-'berniesanders.com': 0.8443017656500802,-'a.wholelottanothing.org': 0.8442952471954357,-'events.ccc.de': 0.8442791650208269,-'www.chicagobusiness.com': 0.8442628765380391,-'nsl.com': 0.844239240640813,-'srcco.de': 0.844239240640813,-'testpilot.firefox.com': 0.8442195629693428,-'noahveltman.com': 0.8441367921512271,-'jamiequint.com': 0.8441367921512271,-'reverb.com': 0.8441367921512271,-'www.pri.org': 0.8441332017534635,-'www.trustnodes.com': 0.8441221359793255,-'blogs.mathworks.com': 0.8440748728050913,-'www.cncf.io': 0.8440600291289487,-'opensource.com': 0.844058190643116,-'about.digg.com': 0.8440360228264422,-'news.brown.edu': 0.8440209497481723,-'www.forth.com': 0.8440090068688062,-'blockstream.com': 0.843993466124853,-'getaether.net': 0.8439867218274119,-'www.hooktheory.com': 0.8439867218274119,-'irssi.org': 0.8439867218274119,-'jemma.dev': 0.8439780808263159,-'gittup.org': 0.8439780808263159,-'www.dropboxforum.com': 0.8439738032049753,-'blog.neocities.org': 0.8439723488535399,-'www.aopa.org': 0.8439413757138091,-'sappingattention.blogspot.com': 0.8439256178572813,-'math-blog.com': 0.843908007189824,-'meetup.hackernewsdc.org': 0.8438982906033671,-'www.ibtimes.sg': 0.8438980758876966,-'curiousrat.com': 0.8438980758876966,-'www.norvig.com': 0.8438704067825865,-'michaelnielsen.org': 0.8438489596573671,-'www.ponylang.io': 0.843843768902312,-'www.noisebridge.net': 0.843843768902312,-'apps.fcc.gov': 0.8438354432862912,-'www.eenews.net': 0.8437777995615074,-'www.dhs.gov': 0.8437568931274587,-'www.uu.se': 0.8437434035754188,-'blog.streak.com': 0.8437430051067691,-'bluishcoder.co.nz': 0.8436990841434638,-'trello.com': 0.8436685239481871,-'prolost.com': 0.8436572621196788,-'corgibytes.com': 0.843643069430385,-'cbloomrants.blogspot.com': 0.8436415581159902,-'grouptalent.com': 0.8436259181114076,-'news.sky.com': 0.8435750425055707,-'dankaminsky.com': 0.8435552716799266,-'blog.kaspersky.com': 0.8435542294083143,-'blog.rubbingalcoholic.com': 0.8435483870967742,-'paul.copplest.one': 0.8435464913857005,-'andrewhy.de': 0.8435425442649045,-'www.techfounder.net': 0.8435095761996304,-'spinroot.com': 0.8435095761996304,-'garethrees.org': 0.8435072199487013,-'blog.gedrap.me': 0.8434993251797909,-'oceans.nautil.us': 0.8434897723987279,-'www.autonlab.org': 0.8434766589029354,-'kallithea-scm.org': 0.8433870123342087,-'www.getrevue.co': 0.8433604592288586,-'www.whatsonweibo.com': 0.8433258307054031,-'tass.com': 0.843318150706682,-'paste.lisp.org': 0.8433142035934179,-'www.rollcall.com': 0.8433069334851104,-'reference.wolfram.com': 0.8432544830728409,-'opendotdotdot.blogspot.com': 0.8432276309668896,-'kevin.burke.dev': 0.8432276309668896,-'decisiondata.org': 0.8432276309668896,-'xenaproject.wordpress.com': 0.8432246085096772,-'benbernardblog.com': 0.8432189502435823,-'usefathom.com': 0.8432043711877606,-'www.pcpro.co.uk': 0.8431913701641113,-'progit.org': 0.8431775195017707,-'i.imgur.com': 0.8431547290624056,-'developer.okta.com': 0.8431277870072948,-'blog.medium.com': 0.8431066839897218,-'ffii.org': 0.8430969026172707,-'byrslf.co': 0.8430850673452497,-'sigops.org': 0.8430850673452497,-'blog.whoishiring.io': 0.8430812250163859,-'www.mcvuk.com': 0.8430765289027752,-'www.google.pl': 0.843042071197411,-'blog.untogether.co.uk': 0.843042071197411,-'www.uh.edu': 0.8430266348842538,-'fable.io': 0.8429358535210576,-'www.hizook.com': 0.8429332432339712,-'www.ovh.com': 0.8429034962421136,-'www.tedinski.com': 0.8428702241660795,-'entrepreneur.venturebeat.com': 0.8428673093425917,-'www.c-spanvideo.org': 0.8428673045430252,-'www.historyextra.com': 0.8428269409461818,-'www.defenseone.com': 0.8428181514777605,-'volument.com': 0.8428160789962235,-'www.imaging-resource.com': 0.8428098585582268,-'popupcity.net': 0.8427876823338736,-'www.skyandtelescope.com': 0.8427597458791004,-'www.viva64.com': 0.8426735071847921,-'existentialtype.wordpress.com': 0.8426727373097719,-'www.ikea.com': 0.8426410914015982,-'earnestcapital.com': 0.8426131831456453,-'jmarbach.com': 0.8425928309863708,-'chrisharrison.net': 0.8425324675324676,-'www.jsonline.com': 0.8425180713598075,-'parenting.nytimes.com': 0.8425153384707086,-'tesla-cdn.thron.com': 0.8424060866368837,-'www.neuroscientistnews.com': 0.8424034611022545,-'www.wondersofphysics.com': 0.842390899623404,-'www.lanl.gov': 0.8423253496487528,-'skeptics.stackexchange.com': 0.8423210116773696,-'www.rfc-editor.org': 0.8422724006868584,-'www.industrialempathy.com': 0.8422674053917267,-'performancejs.com': 0.8422235444245709,-'infiniteundo.com': 0.8422087232147684,-'www.heroku.com': 0.8422087232147684,-'blog.lse.epita.fr': 0.8421833530068131,-'lacker.io': 0.8421778420117882,-'timeline.com': 0.8421633036155716,-'wiki.blender.org': 0.8420940430794808,-'www.freakonomics.com': 0.8420407955613295,-'blog.fugue.co': 0.8420361620441877,-'mediadecoder.blogs.nytimes.com': 0.8419912813368542,-'www.irs.gov': 0.8419451366935324,-'www.golem.de': 0.8419385789535662,-'pingcap.com': 0.8419161509277815,-'john.onolan.org': 0.8418365869145599,-'blog.mocality.co.ke': 0.8417877979157168,-'status.coinbase.com': 0.8417606283759475,-'www.attejuvonen.fi': 0.8417377670517942,-'serversforhackers.com': 0.8417376768816427,-'work.caltech.edu': 0.8417166624718854,-'fuelyourcoding.com': 0.841709160679554,-'dyn.com': 0.8416196735446702,-'blog.tabini.ca': 0.8415950088038011,-'ieeexplore.ieee.org': 0.8415919239489138,-'www.israel21c.org': 0.8415492534155486,-'rubyonrails.org': 0.8415351989418526,-'www.designweek.co.uk': 0.8415188999809792,-'collison.ie': 0.8415120389596948,-'www.evilmadscientist.com': 0.8414856456254624,-'muddylemon.com': 0.8414700703965355,-'agateau.com': 0.8414619982013722,-'blog.miguelgrinberg.com': 0.8414611167027505,-'nmap.org': 0.8414349271123382,-'www.theblockcrypto.com': 0.8414118140696276,-'www.haaretz.com': 0.8413639156892987,-'codecapsule.com': 0.8413373129936518,-'ethz.ch': 0.84131835872283,-'blog.bloomca.me': 0.8413144015517519,-'blog.snapeda.com': 0.8413079629673079,-'evonomics.com': 0.8413074174618784,-'www2.macleans.ca': 0.8412847684511929,-'composition.al': 0.841279463048803,-'js13kgames.com': 0.8412786874884526,-'lists.basho.com': 0.8412692653556527,-'people.redhat.com': 0.8412692653556527,-'oceanservice.noaa.gov': 0.8412678170260773,-'sciencenorway.no': 0.8412438625204582,-'speaking.io': 0.8412438625204582,-'www.exceptionnotfound.net': 0.8412220203835995,-'www.2ality.com': 0.8411429398560686,-'www.eecs.harvard.edu': 0.841119342861029,-'bldgblog.blogspot.com': 0.8411187765808322,-'thisismynext.com': 0.8410701280879982,-'unchartedterritories.tomaspueyo.com': 0.8410441941601818,-'www.skilled.io': 0.8410441941601818,-'tenfourfox.blogspot.com': 0.8410170033847231,-'thebreakfastpost.com': 0.8410159372229994,-'blog.bumblebeelabs.com': 0.8409937866242836,-'mmikowski.github.io': 0.8409753773821584,-'paultyma.blogspot.com': 0.8409535720717558,-'materialize.io': 0.8409502033267174,-'www.ncdc.noaa.gov': 0.840942212674096,-'www.haystacksoftware.com': 0.8409422126740959,-'techiedelight.quora.com': 0.8409422126740959,-'blog.modernmechanix.com': 0.8408958299079167,-'www.math3ma.com': 0.8408873008599574,-'retractionwatch.com': 0.8408852959781978,-'www.appleoutsider.com': 0.8408203431401681,-'www.dreamsongs.com': 0.8408038061650092,-'blogs.cisco.com': 0.8407980680062469,-'www.research.ibm.com': 0.8407779430526359,-'gohugo.io': 0.8407728272201884,-'nicolodavis.com': 0.8407632567723468,-'www.andreykurenkov.com': 0.8407450453082168,-'conversionxl.com': 0.8407343949968893,-'www.qemu-advent-calendar.org': 0.8407224958949097,-'enfranchisedmind.com': 0.8407038581523874,-'ozkatz.github.com': 0.8406930647478468,-'ahrefs.com': 0.8406628331444651,-'paulmck.livejournal.com': 0.8406466273816793,-'spf13.com': 0.8406252111185971,-'adamschwartz.co': 0.8406212712895198,-'scikit-learn.org': 0.8406196399338859,-'www.lessonsoffailure.com': 0.8406081585039813,-'mathenchant.wordpress.com': 0.8406026505628841,-'charlesleifer.com': 0.8405838838741042,-'zeux.io': 0.8405743435687888,-'bicycledutch.wordpress.com': 0.8405743435687888,-'ai.stanford.edu': 0.8405537358599618,-'iximiuz.com': 0.8405441526018748,-'kerkour.com': 0.8405182576561551,-'www.racked.com': 0.8404972480623266,-'news.morningstar.com': 0.8404838510660001,-'hfet.org': 0.8404775873220368,-'newcome.wordpress.com': 0.8404605263157895,-'enterprisecraftsmanship.com': 0.8404567783699838,-'dailynous.com': 0.8404286033548175,-'usersknow.blogspot.com': 0.8402966516996352,-'www.sandiegouniontribune.com': 0.8402646673456328,-'www.playframework.com': 0.8402476003167625,-'betterhumans.coach.me': 0.840238289918768,-'thomaspark.co': 0.8402340830446674,-'deadlockempire.github.io': 0.8401976935749588,-'mimimatthews.com': 0.8401762295218094,-'chris.beams.io': 0.8401743605753639,-'www.whoismcafee.com': 0.8401743605753639,-'www.stlouisfed.org': 0.8401103027746665,-'fredrikdeboer.com': 0.8400945631732266,-'blog.cerebralab.com': 0.8400945631732265,-'weworkremotely.com': 0.8400536676760625,-'english.yonhapnews.co.kr': 0.840015150270215,-'blog.ffwll.ch': 0.8399806882990191,-'www.kaspersky.com': 0.8399734036718564,-'webmasters.stackexchange.com': 0.8399733386768045,-'taoofmac.com': 0.8399516851214591,-'webapps.stackexchange.com': 0.8399426081446101,-'nenadseo.com': 0.839939548717707,-'our.status.im': 0.8399358470807224,-'justinhileman.info': 0.839916636052546,-'www.financemagnates.com': 0.83991429571072,-'srlabs.de': 0.8399137310646634,-'emacsredux.com': 0.8399137310646634,-'www.colorforth.com': 0.8398830755348644,-'tech.trivago.com': 0.8398706352344396,-'rusty.ozlabs.org': 0.8398201966467941,-'www.unionleader.com': 0.8398201966467941,-'tomverbeure.github.io': 0.8397956166500623,-'www.huckmag.com': 0.8397408630849317,-'www.3quarksdaily.com': 0.8397300193715044,-'snarky.ca': 0.8397285354135811,-'basicattentiontoken.org': 0.8396694214876033,-'0patch.blogspot.com': 0.8396569050806407,-'www.stackfield.com': 0.8396561419057332,-'laktek.com': 0.8395904864603353,-'www.elasticsearch.com': 0.8395809014444943,-'thomashunter.name': 0.8395739505236344,-'tapbots.com': 0.8395257149612579,-'nlp.stanford.edu': 0.8395127943859381,-'shime.sh': 0.8395015090274327,-'news.consumerreports.org': 0.8395015090274327,-'www.alwaysrightinstitute.com': 0.8395015090274327,-'plover.net': 0.8394877101221607,-'oleb.net': 0.8394619474972026,-'www.mymoneyblog.com': 0.8394511334746066,-'blog.clarifai.com': 0.8394429821320054,-'technodrone.blogspot.com': 0.8394039735099338,-'labs.earthpeople.se': 0.8394039735099338,-'blog.greweb.fr': 0.8394025195959639,-'truthout.org': 0.8393907588504428,-'www.rowetel.com': 0.8393890552964602,-'acha.ninja': 0.8393871342567361,-'blog.carlmjohnson.net': 0.8393408901985292,-'programming-journal.org': 0.8393406966484743,-'www.news18.com': 0.8392713736775148,-'sam.bluwiki.com': 0.8392570453444638,-'www.monbiot.com': 0.8392563174673074,-'rawgit.com': 0.8392483684869478,-'www.foxla.com': 0.8392193702289447,-'dvt.name': 0.839218830660763,-'blog.facebook.com': 0.8392045415231093,-'freddiedeboer.substack.com': 0.8391840290729782,-'developer.marvel.com': 0.8391376451077944,-'www.lowendtalk.com': 0.8391326019880078,-'docs.celeryproject.org': 0.8391326019880078,-'blog.trendmicro.com': 0.8391258095534089,-'poststatus.com': 0.8390975027931495,-'blockgeeks.com': 0.8390450127680085,-'itnext.io': 0.8390022169180034,-'blog.existentialize.com': 0.8389858546214449,-'semiengineering.com': 0.8389442107551947,-'www.digibarn.com': 0.8389114239700469,-'mikemcquaid.com': 0.8388881112064441,-'hackflow.com': 0.838888111206444,-'lists.opensuse.org': 0.8388704318936877,-'weblog.rogueamoeba.com': 0.8387522024258429,-'pubs.acs.org': 0.838739869285439,-'www.webdesignmuseum.org': 0.8387125258530213,-'www.garron.blog': 0.8387125258530211,-'instadeq.com': 0.8387105083280972,-'www.brisbanetimes.com.au': 0.8386591269613846,-'leanpub.com': 0.8386465804494829,-'fendrich.se': 0.8386023294509152,-'dl.dropboxusercontent.com': 0.8385995778248918,-'lettier.github.io': 0.8385983762141392,-'www.edn.com': 0.8384880352277337,-'www.synthtopia.com': 0.8384691370060022,-'www.michaelbromley.co.uk': 0.8384548675731243,-'www.refsmmat.com': 0.8384548675731243,-'jonbischke.com': 0.8384341839739798,-'warzel.substack.com': 0.8384049720140551,-'blogs.kqed.org': 0.8383873467097201,-'www.xconomy.com': 0.8383614008234231,-'www.timeslive.co.za': 0.838354450918394,-'keithp.com': 0.8383347979461153,-'psycnet.apa.org': 0.8383190430040641,-'blog.online.net': 0.8383007137970923,-'www.atariarchives.org': 0.8382952495330802,-'www.fivethirtyeight.com': 0.8382374049516083,-'www.wearethemighty.com': 0.8381915632322696,-'www.arstechnica.com': 0.8381868933365827,-'heatst.com': 0.8381531996802586,-'blog.wekeroad.com': 0.838151852850549,-'www.dnainfo.com': 0.8381308827626469,-'wufoo.com': 0.8381135600270289,-'www.zompist.com': 0.8381135600270289,-'www.nbr.co.nz': 0.8380751982509018,-'tomtunguz.com': 0.8380729812475479,-'www.scripting.com': 0.8380697700961443,-'blog.twilio.com': 0.8380613391841799,-'www.nhplace.com': 0.8380324040052766,-'pypi.python.org': 0.8380184467463045,-'www.the-digital-reader.com': 0.837988405339907,-'www.vpri.org': 0.8379878140892437,-'www.bailis.org': 0.8379777722457122,-'www.maximise.dk': 0.8379766121747607,-'99designs.com': 0.8379580943514392,-'www.crummy.com': 0.8379490514739079,-'www.ibtimes.co.uk': 0.8379434303523721,-'vmware.github.io': 0.8379411472179317,-'www.6sqft.com': 0.8379298178385524,-'symkat.com': 0.8379298178385524,-'survata.com': 0.8378674714439094,-'www.greenwichtime.com': 0.8378674714439094,-'www.leandomainsearch.com': 0.8378176898959185,-'penguindreams.org': 0.8377993988882627,-'www.cs.ox.ac.uk': 0.8377993988882627,-'windowsteamblog.com': 0.8377936722512888,-'www.more-magic.net': 0.8377744800755205,-'wundergraph.com': 0.8377666240744057,-'www.stereogum.com': 0.8377124380298397,-'media.defcon.org': 0.8376483665038642,-'andrewsullivan.theatlantic.com': 0.8376455603446734,-'physics.aps.org': 0.8376321975113651,-'www.addedbytes.com': 0.8376314906854643,-'theconversation.edu.au': 0.8376217948668653,-'carwoo.com': 0.8376024057546919,-'www.swiftbysundell.com': 0.8375837630694467,-'bigthink.com': 0.837574858033942,-'www.nltk.org': 0.8375336447475157,-'pron.github.io': 0.8375247229364985,-'atlas.cern': 0.8375209380234506,-'www.nyse.com': 0.8375209380234506,-'icyphox.sh': 0.8374905510071093,-'hypothesis.works': 0.8374147657218561,-'jacobobryant.com': 0.8373876028525398,-'blogs.zappos.com': 0.8373722614685953,-'blog.joellehman.com': 0.8373621224652893,-'conal.net': 0.8373554518624804,-'www.dadhacker.com': 0.837354651092982,-'stars.chromeexperiments.com': 0.8372729777719929,-'blog.thousandeyes.com': 0.8372663541460196,-'www.drbunsen.org': 0.8372619145116595,-'blog.tinfoilsecurity.com': 0.8372571688925584,-'www.landley.net': 0.837248322147651,-'jjbeshara.com': 0.837248322147651,-'www.searchenginejournal.com': 0.8372190177985843,-'samgentle.com': 0.8371607111617112,-'www.mywinamp.com': 0.8371394760092047,-'www.acm.uiuc.edu': 0.8371394760092047,-'blogs.sfweekly.com': 0.8371151654308293,-'theaviationist.com': 0.8370871690994972,-'www.mailchimp.com': 0.8370705380744307,-'incubator.apache.org': 0.8370705380744307,-'www.mikealrogers.com': 0.8370700724965489,-'www.unofficialgoogledatascience.com': 0.8370621601099476,-'www.wildbunny.co.uk': 0.8370316066816674,-'mega.co.nz': 0.8370228944247635,-'vimcasts.org': 0.8370122300530931,-'home.web.cern.ch': 0.8370107312738333,-'news.virginia.edu': 0.8370064211588852,-'blog.zawodny.com': 0.8370064211588852,-'www.mdpi.com': 0.8370060708284108,-'blog.mousereeve.com': 0.8370038016134355,-'hackernewsbooks.com': 0.8369747899159664,-'detexify.kirelabs.org': 0.8369455416288776,-'blip.tv': 0.8369060266959822,-'goodstuff.im': 0.8369038418527343,-'opensignal.com': 0.8368950022832518,-'the-pastry-box-project.net': 0.836851759413345,-'parkerhiggins.net': 0.8368481032461187,-'www.roughtype.com': 0.8367664523195075,-'www.ecosia.org': 0.8367628729886891,-'dougmccune.com': 0.8367628729886891,-'lambdaisland.com': 0.8367557134451551,-'blog.8thlight.com': 0.8367523868737199,-'mml-book.github.io': 0.8367337334597409,-'akaptur.com': 0.8367337334597409,-'www.appneta.com': 0.8367099124724148,-'eos.org': 0.836703598387807,-'www.eia.gov': 0.836692477143149,-'www.freesoftwaremagazine.com': 0.8366844191693678,-'blog.isquaredsoftware.com': 0.8366450083184942,-'www.effectgames.com': 0.8366086074590122,-'danbirken.com': 0.8366082851176284,-'www.georgesaines.com': 0.8365836924813026,-'projects.nytimes.com': 0.8365185959005205,-'cilium.io': 0.8364900150066928,-'luisant.ca': 0.8364627688697146,-'kde.org': 0.8364506685642159,-'gengo.ai': 0.8364249578414841,-'gigamonkeys.com': 0.836424957841484,-'ask.slashdot.org': 0.8364157461593826,-'www.inf.ethz.ch': 0.8364037125832664,-'bufr.tumblr.com': 0.8363012314554059,-'go.bloomberg.com': 0.8362895418561958,-'www.mv-voice.com': 0.8362735866170948,-'www.visual6502.org': 0.8362643122145562,-'codeofrob.com': 0.8362576149569876,-'www.ludumdare.com': 0.8362566416598247,-'www.bradford-delong.com': 0.8361844302416513,-'sidsite.com': 0.8361486486486487,-'persquaremile.com': 0.8361417911687711,-'boss.blogs.nytimes.com': 0.8361416007600664,-'pathdependent.com': 0.8361365464427095,-'www.marketplacepulse.com': 0.8361110009915448,-'javascriptweblog.wordpress.com': 0.8361100871908929,-'blog.steve.fi': 0.8360870917809617,-'www.stefankendall.com': 0.8360443900320628,-'machinelearningmastery.com': 0.8360325276933736,-'tomcritchlow.com': 0.8360317433744368,-'www.vqronline.org': 0.8359910491447964,-'pmarchive.com': 0.8359774363850249,-'blog.exodusintel.com': 0.8359613952379756,-'icelandmonitor.mbl.is': 0.8359181324679799,-'learnui.design': 0.8359165073352933,-'www.wisdomandwonder.com': 0.8359037098645399,-'gnupg.org': 0.8359037098645399,-'www.otherhand.org': 0.8359037098645399,-'andyswan.com': 0.8358651631273738,-'vlaaad.github.io': 0.8358605537700576,-'unikernel.org': 0.8358276165245331,-'forums.adobe.com': 0.8358049697804542,-'theappeal.org': 0.8358049697804542,-'people.mozilla.org': 0.8357546553679396,-'www.multivax.com': 0.8357546553679396,-'www.hilarymason.com': 0.8357367418987102,-'metro.co.uk': 0.8357169083210791,-'www.fxguide.com': 0.8356995496213405,-'mail.gnome.org': 0.8356930267331182,-'www.ripe.net': 0.8356373665440678,-'polygraph.cool': 0.8355932203389831,-'newsinteractives.cbc.ca': 0.8355673184633825,-'oonwoye.com': 0.8355085503926205,-'www.centernetworks.com': 0.83550554167365,-'blog.kenperlin.com': 0.8354986352909005,-'codelabs.developers.google.com': 0.8354986352909004,-'ericasadun.com': 0.8354812545307914,-'voyager.jpl.nasa.gov': 0.8354770866238536,-'glinden.blogspot.com': 0.8354248687848085,-'www.jeffknupp.com': 0.8354226452254534,-'0xax.blogspot.com': 0.8354177064677306,-'home.comcast.net': 0.8354135974937222,-'www.scrapingbee.com': 0.8353947775852016,-'mail-index.netbsd.org': 0.8353813194329008,-'mikeindustries.com': 0.835369856283584,-'failory.com': 0.8353532155275709,-'www.ritsumei.ac.jp': 0.835322744014879,-'typing.io': 0.8353140916808149,-'blog.britishmuseum.org': 0.835306193677583,-'mailbox.org': 0.8352951727923229,-'hackety.org': 0.8352886313345826,-'invisible-island.net': 0.8352861612933625,-'blog.minio.io': 0.8352821552252798,-'www.etymonline.com': 0.8352819967060172,-'founderdating.com': 0.8352750374933166,-'users.softlab.ece.ntua.gr': 0.8352418158169491,-'read.acloud.guru': 0.8352400454348197,-'julian.digital': 0.8352206968521787,-'ben.balter.com': 0.8352023832422831,-'osc.universityofcalifornia.edu': 0.8351585723230902,-'futureoflife.org': 0.8351040617937822,-'www.multicians.org': 0.8350943317132041,-'allthatsinteresting.com': 0.8350698686955953,-'machinethink.net': 0.8350562425214455,-'blogs.denverpost.com': 0.8350421047806545,-'mishadoff.com': 0.8350340136054422,-'getfirebug.com': 0.8350340136054422,-'www.dxomark.com': 0.8349344540359945,-'www.mycpu.org': 0.8349193509952372,-'www.raphkoster.com': 0.8349185464697736,-'bps-research-digest.blogspot.com': 0.8349065337932573,-'cs229.stanford.edu': 0.834894479023565,-'sadanand-singh.github.io': 0.834894479023565,-'blog.doordash.com': 0.834891053529973,-'paul.kinlan.me': 0.834891053529973,-'sliderulemuseum.com': 0.8347529812606473,-'www.hpcwire.com': 0.8347353682368306,-'flaviocopes.com': 0.8347098860524725,-'nocomplexity.com': 0.8346402286365225,-'designbygravity.wordpress.com': 0.8346002566095803,-'what-if.xkcd.com': 0.8345955143488654,-'www.healthcareitnews.com': 0.834479735234925,-'turingpi.com': 0.834470989761092,-'pfrazee.github.io': 0.834470989761092,-'csrc.nist.gov': 0.8344705397477052,-'taskwarrior.org': 0.8344665233968941,-'grepular.com': 0.8344590445370506,-'nuclearsecrecy.com': 0.8344590445370506,-'maxkle.in': 0.8344294161546372,-'www.csd.uwo.ca': 0.8344273945101189,-'alanhollis.com': 0.8343851939878961,-'www.splitgraph.com': 0.8343399060316927,-'www.instapainting.com': 0.834303164841344,-'taxfoundation.org': 0.834303164841344,-'delta.chat': 0.8342867534842825,-'appft1.uspto.gov': 0.8342760797997661,-'leerob.io': 0.8342207211363054,-'chrisshort.net': 0.8342193690645514,-'blog.venanti.us': 0.8342064686003738,-'news.cnrs.fr': 0.8341880341880342,-'500px.com': 0.8341880341880342,-'krugman.blogs.nytimes.com': 0.8341820699049042,-'fearlesssalarynegotiation.com': 0.8341803150678754,-'www.frankmcsherry.org': 0.8341803150678754,-'nts.strzibny.name': 0.8341550107611373,-'developers.memsql.com': 0.8341526449980673,-'sr.ht': 0.8341334625060556,-'ebiquity.umbc.edu': 0.8340723188782359,-'news.ubc.ca': 0.8340723188782359,-'blog.wired.com': 0.8340173587394528,-'www.dailyherald.com': 0.8340105184549977,-'artur-martsinkovskyi.github.io': 0.8339880299244583,-'felixge.de': 0.8339880299244583,-'blog.ayoungprogrammer.com': 0.833983000311883,-'inhabitat.com': 0.833970800806039,-'www.lifehacker.com.au': 0.8339654277683552,-'science.howstuffworks.com': 0.8338999781361612,-'cancer.nautil.us': 0.8338999781361612,-'appcubby.com': 0.8338994925108514,-'www.math.utah.edu': 0.8338684428078268,-'tooslowexception.com': 0.8338684428078268,-'www.patrickcraig.co.uk': 0.8338645320250127,-'perfectionkills.com': 0.8338619239102345,-'ncona.com': 0.8337668211914532,-'blogs.aws.amazon.com': 0.8337525489654461,-'www.cnil.fr': 0.83372186106167,-'guidovranken.wordpress.com': 0.8337106909166525,-'littlebitofcode.com': 0.8337106909166525,-'crpgaddict.blogspot.com': 0.8337028204720323,-'zerodha.tech': 0.833697181290741,-'medicine.wustl.edu': 0.8336839373019279,-'blog.socialcops.com': 0.8336751280750007,-'ifttt.com': 0.8336678541410665,-'blog.scalyr.com': 0.8335986795297188,-'www.psypost.org': 0.8335783172647908,-'www.abcactionnews.com': 0.8335754162799685,-'one.google.com': 0.8335754162799685,-'blog.marksweep.com': 0.8335690009897165,-'basecamp.com': 0.8334897559746374,-'www.rdegges.com': 0.8334643316971914,-'michael.otacoo.com': 0.8334458765497461,-'toucharcade.com': 0.8334457758738516,-'adage.com': 0.8334270713794664,-'getbootstrap.com': 0.8334213784102003,-'blog.klipse.tech': 0.8334108661664996,-'classics.mit.edu': 0.8334006037794777,-'dev.to': 0.8333163721881794,-'deadline.com': 0.833286267441488,-'www.expressvpn.com': 0.8332852925186139,-'zverovich.net': 0.8332852925186139,-'www.bnl.gov': 0.8332751437457405,-'discuss.linuxcontainers.org': 0.8332368170670513,-'codesketch.com': 0.8332263874970826,-'www.leveragedsellout.com': 0.8331869545898674,-'www.imore.com': 0.8331827749737907,-'people.inf.ethz.ch': 0.8331734250731533,-'blog.smartbear.com': 0.8331510235912066,-'ryanflorence.com': 0.8331396140981399,-'stephanpastis.wordpress.com': 0.8331396140981399,-'static1.squarespace.com': 0.8331073989685633,-'createdigitalmusic.com': 0.8330970061467121,-'bbc.co.uk': 0.8330970061467121,-'www.nhm.ac.uk': 0.8330761814047353,-'press.princeton.edu': 0.8330608346746611,-'blog.hartleybrody.com': 0.8329386129816697,-'futureboy.us': 0.8329182087040424,-'www.windowslatest.com': 0.8329104232060686,-'ewontfix.com': 0.8328938411174854,-'developer.yahoo.com': 0.8328635993581618,-'jng.imagine27.com': 0.8328607138329057,-'gogs.io': 0.8328565779067133,-'geoffboeing.com': 0.8328409513000423,-'www.phillymag.com': 0.8328409513000423,-'graphthinking.blogspot.com': 0.8328383114974501,-'blog.authy.com': 0.8327906592352505,-'earthengine.google.com': 0.8327019997862043,-'adamwathan.me': 0.8326583014571802,-'distributionhacks.com': 0.8326485954892083,-'www.thinkster.io': 0.832628817491836,-'terminusdb.com': 0.832628817491836,-'www.technoblogy.com': 0.8326183396075918,-'30hourjobs.com': 0.8326092279817636,-'infoproc.blogspot.com': 0.8326089175295298,-'www.ftrain.com': 0.8326051014052062,-'foundation.zurb.com': 0.8326012313075719,-'blog.viraptor.info': 0.8325776839853443,-'bennesvig.com': 0.8325751054908778,-'www.expressionsofchange.org': 0.832572579886013,-'mrl.nyu.edu': 0.8325111338579982,-'blog.urfix.com': 0.8325111338579982,-'lists.x.org': 0.8324697754749568,-'huggingface.co': 0.8324625696123126,-'zeuxcg.org': 0.8324088201420566,-'vmssoftware.com': 0.8323629518973035,-'jimplush.com': 0.8323629518973035,-'thereboot.com': 0.8323490243960362,-'antimatter15.com': 0.8323430057057912,-'japhr.blogspot.com': 0.8323430057057912,-'blog.davidecoppola.com': 0.8323083996896996,-'www.kotaku.com.au': 0.832301047102968,-'3perf.com': 0.8322876151248816,-'ryxcommar.com': 0.8322876151248816,-'www.mysql.com': 0.8322876151248816,-'scitechdaily.com': 0.8322404618162444,-'rjzaworski.com': 0.8321799307958477,-'www.dolthub.com': 0.8321769413258827,-'www.jperla.com': 0.8321740485435664,-'roberto.open-lab.com': 0.8321725984943663,-'www.tofugu.com': 0.8321694358359643,-'www.preprints.org': 0.8321312662446396,-'ntrs.nasa.gov': 0.8321306733695456,-'blog.kaggle.com': 0.8321136466040312,-'getcarina.com': 0.8321067502232443,-'thebarentsobserver.com': 0.8320962403151547,-'www.improbable.com': 0.8320824782702279,-'type.method.ac': 0.8320667644923518,-'www.aleph.se': 0.8320667644923518,-'lists.macosforge.org': 0.8320667644923518,-'zguide.zeromq.org': 0.8320667644923518,-'onethingwell.org': 0.8320539832149718,-'www.gittip.com': 0.8320539832149718,-'electronics.stackexchange.com': 0.8320387304939284,-'www.newsobserver.com': 0.8319542731572184,-'voat.co': 0.8319340573214568,-'www.viemu.com': 0.8319229644593481,-'www.embeddeduse.com': 0.8318890814558059,-'ideolalia.com': 0.831795173528973,-'andrewdumont.me': 0.8317884623663075,-'sriramk.com': 0.8317884108541596,-'www.chenhuijing.com': 0.8317790598787409,-'www.yegor256.com': 0.8317697748343722,-'www.foxnews.com': 0.8317694725922751,-'www.cs.nott.ac.uk': 0.8317650230135005,-'www.trustedreviews.com': 0.8317627240153616,-'greatergood.berkeley.edu': 0.8316786529650598,-'mitxela.com': 0.8315323559022839,-'mail.haskell.org': 0.8315275148175146,-'cooking.stackexchange.com': 0.8315230832695719,-'www.apnews.com': 0.831500326107284,-'www.belfasttelegraph.co.uk': 0.8314967853302809,-'zackshapiro.com': 0.8314155294643537,-'www.collectspace.com': 0.831412959256786,-'www.alchemistowl.org': 0.8313334448247652,-'www.interfluidity.com': 0.8313143760797148,-'mithril.js.org': 0.8313043478260869,-'sunboxlabs.com': 0.8313043478260869,-'css.csail.mit.edu': 0.83128825969837,-'mattermark.com': 0.8312791828318407,-'ballotpedia.org': 0.8312622600459071,-'r2c.dev': 0.8312622600459071,-'goaccess.io': 0.8312325519112657,-'www.michaelwest.com.au': 0.8312257534350801,-'redsymbol.net': 0.8312144135803776,-'journals.aps.org': 0.8312128897645714,-'blog.gingerlime.com': 0.8312016352070194,-'bugsplat.info': 0.8311332389568854,-'projects.aljazeera.com': 0.8310960202717635,-'jlouisramblings.blogspot.com': 0.8310960202717635,-'www.datastuff.tech': 0.8310748264197387,-'www.cliffc.org': 0.8310367707450238,-'www.merriam-webster.com': 0.8310169102957669,-'washingtonpost.com': 0.831005263332199,-'www.leapmotion.com': 0.8309966909104267,-'dom.as': 0.8309526362308279,-'dotnet.github.io': 0.8309273722835752,-'rickcarlino.com': 0.8309273722835752,-'p-cos.blogspot.com': 0.8309273722835752,-'www.electronicsweekly.com': 0.8308205769875683,-'blog.mythic-beasts.com': 0.8308141567715057,-'blog.codiqa.com': 0.830787883150054,-'www.urbandictionary.com': 0.8307663266042938,-'blog.ericgoldman.org': 0.8307656053312806,-'thatcomputerscientist.com': 0.8307366610510188,-'shankman.com': 0.830715532286213,-'blog.uservoice.com': 0.830715532286213,-'shafyy.com': 0.8306912467549404,-'www.23andme.com': 0.8306912467549404,-'thebulletin.org': 0.8306726473588931,-'sekao.net': 0.8306717904781525,-'om.co': 0.8306518197608382,-'reacttraining.com': 0.8306279342292152,-'phili.pe': 0.8306279342292152,-'kamerontanseli.ghost.io': 0.8306279342292152,-'www.deccanherald.com': 0.830617757462069,-'recipes.hypotheses.org': 0.8306075161940402,-'www.scotthyoung.com': 0.8304676761207413,-'countercomplex.blogspot.com': 0.8304599554901336,-'blog.kraken.com': 0.8304102297264507,-'ello.co': 0.8303950986221522,-'asiatimes.com': 0.8303938402412525,-'blog.tindie.com': 0.8303900100100166,-'www.plagiarismtoday.com': 0.8303836069407874,-'moviecode.tumblr.com': 0.8303351870840227,-'www.the74million.org': 0.8303274336465168,-'analytics.usa.gov': 0.8303274336465168,-'blog.jpl-consulting.com': 0.8303274336465168,-'archive.nytimes.com': 0.8303195537893885,-'blogs.zdnet.com': 0.8302477853461946,-'www.gosquared.com': 0.8302108084053713,-'www.unite.ai': 0.8301971117330208,-'pastie.org': 0.8301429578448887,-'www.bluetooth.com': 0.830122591943958,-'heartbeat.fritz.ai': 0.8300819862294267,-'blog.liw.fi': 0.8300810779081896,-'gilesbowkett.blogspot.com': 0.8300257142091928,-'placesjournal.org': 0.8299577016504602,-'interfacelab.com': 0.8299038213762403,-'programming-motherfucker.com': 0.8298839065342439,-'www.thespacereview.com': 0.8298526044878857,-'www.zippia.com': 0.8298245614035087,-'alumnit.ca': 0.8298220650253975,-'pne.people.si.umich.edu': 0.8297232221938442,-'blog.scienceexchange.com': 0.8296882900501562,-'danielallendeutsch.com': 0.8296786018063418,-'veyronb.wordpress.com': 0.8296786018063417,-'www.learningclojure.com': 0.829663604832329,-'android.stackexchange.com': 0.8296351227828602,-'neo4j.com': 0.8296343605300973,-'www.insentricity.com': 0.8295390134146845,-'johnhawks.net': 0.8295371897944813,-'gist.io': 0.829536713446392,-'www.irishexaminer.com': 0.8295100407790094,-'blog.runnable.com': 0.8294913771537328,-'www.dmv.ca.gov': 0.8294663396318804,-'technology.stitchfix.com': 0.8293836920149785,-'anyasq.com': 0.8293440267345259,-'eigenfoo.xyz': 0.8293416074439832,-'sunlightfoundation.com': 0.8293282465454183,-'developer.rackspace.com': 0.829323262325674,-'rwmj.wordpress.com': 0.8293063560783351,-'www.makegameswith.us': 0.8292959190283576,-'wesleytansey.com': 0.8292534470294121,-'startupljackson.com': 0.8292534470294121,-'www.xprize.org': 0.8292534470294121,-'www.mongodb.org': 0.829231224668068,-'www.drive.com.au': 0.8292253521126759,-'crate.io': 0.8291626909612831,-'voice.kadira.io': 0.8291212925208525,-'blog.drewolson.org': 0.8291191739941484,-'slog.thestranger.com': 0.8291191739941484,-'remarkedly.com': 0.8291054023142576,-'artem.krylysov.com': 0.8290877591855479,-'www.collabora.com': 0.8290844858349273,-'www.handelsblatt.com': 0.8290824405762608,-'www.dailycamera.com': 0.8289847063149993,-'jozefg.bitbucket.org': 0.8289755956167212,-'baus.net': 0.8289755956167212,-'www.davisr.me': 0.8289669222429755,-'ghc.haskell.org': 0.8289669222429755,-'akkshaya.blog': 0.8289669222429755,-'www.boristhebrave.com': 0.8289241622574957,-'ootbcomp.com': 0.8289241622574957,-'www.puzzlescript.net': 0.8289241622574957,-'mattjibson.com': 0.828841660397834,-'knowledge.wharton.upenn.edu': 0.8288260425307535,-'simpleflying.com': 0.8288152386804701,-'houseoflawandorder.com': 0.828808793096398,-'abstractions.nautil.us': 0.8288076574107112,-'www.scalescale.com': 0.8288034171529194,-'www.ostechnix.com': 0.8288034171529194,-'www.londonreconnections.com': 0.8288034171529193,-'sandro.volpee.de': 0.8287903687677997,-'www.reghardware.com': 0.8287468808977058,-'www.shellcheck.net': 0.8287381145882802,-'m.youtube.com': 0.8287237652975437,-'www.freep.com': 0.828700914495102,-'www.cs.hmc.edu': 0.8286794342224705,-'www.post-gazette.com': 0.8286406919749909,-'ecorner.stanford.edu': 0.8285837667040196,-'olimex.wordpress.com': 0.8285738879971565,-'englishrussia.com': 0.8285720380368012,-'2ton.com.au': 0.8285684110847549,-'debugandrelease.blogspot.com': 0.8285505526982948,-'news.err.ee': 0.8285383023940318,-'academictimes.com': 0.8285283281973459,-'beyondgrep.com': 0.8285017969437128,-'www.lesspenguiny.com': 0.8285017969437128,-'tinyhack.com': 0.8285017969437128,-'www.mailgun.com': 0.8284973918318256,-'homepage.divms.uiowa.edu': 0.8284973918318256,-'grimoire.ca': 0.82849280304493,-'unicornfree.com': 0.8284746725029384,-'www.afr.com': 0.8284438035910985,-'www.routefifty.com': 0.8284215095159949,-'www.nintendolife.com': 0.8284071908632212,-'www.wolfram.com': 0.8283831975537147,-'www.rfi.fr': 0.8282942879074087,-'talkweb.eu': 0.8282723807936619,-'drdobbs.com': 0.8282680775958894,-'elidourado.com': 0.8282580762478886,-'www.math.uwaterloo.ca': 0.8282472101761524,-'www.industryweek.com': 0.8282421484626746,-'tipjoys2cents.blogspot.com': 0.8282133103461377,-'www.prerender.cloud': 0.8282133103461377,-'www.thezdi.com': 0.8282032202463764,-'www.ableton.com': 0.8281936977442643,-'kylerush.net': 0.8281936977442643,-'hbswk.hbs.edu': 0.8281850054053886,-'www.hopesandfears.com': 0.8281077808394027,-'mappingignorance.org': 0.8281015489846159,-'code.jsoftware.com': 0.8281015489846159,-'thefrailestthing.com': 0.828083220780241,-'blog.gopheracademy.com': 0.8280826702520344,-'threedots.tech': 0.8280375017895765,-'robnapier.net': 0.8280259986658036,-'blogs.nasa.gov': 0.8280019811960149,-'blog.indutny.com': 0.8279912351566244,-'cloudonaut.io': 0.8279487990535627,-'boston.craigslist.org': 0.827928286178696,-'www.techprowd.com': 0.827928286178696,-'smalldata.tech': 0.8279186302269647,-'notebook.drmaciver.com': 0.8278936902761703,-'www.getpostman.com': 0.8278936902761703,-'newcriterion.com': 0.82785314452846,-'www.nationalaffairs.com': 0.8278141715678232,-'roarmag.org': 0.8278111419376111,-'blog.ezliu.com': 0.82779905891824,-'engineering.heroku.com': 0.82779905891824,-'sulami.github.io': 0.8277548514011955,-'techdirt.com': 0.8277129992323573,-'www.automobilemag.com': 0.827708703374778,-'lihautan.com': 0.8275741663396405,-'aschmann.net': 0.8275741663396404,-'blog.apps.npr.org': 0.8275741663396404,-'ionicframework.com': 0.8275048014166265,-'google-engtools.blogspot.com': 0.8274021352313168,-'blog.davidsingleton.org': 0.8274021352313168,-'blog.jessitron.com': 0.8273903856669967,-'www.orau.org': 0.8272273741633687,-'digesttime.com': 0.8272248458013906,-'kswizz.com': 0.8272248458013906,-'wmpoweruser.com': 0.8272247723472747,-'nimbleindustries.io': 0.8272187229694837,-'jolieodell.wordpress.com': 0.8272099836896732,-'www.bookforum.com': 0.8271999089538162,-'genode.org': 0.8271844999040995,-'decorrespondent.nl': 0.82714602901724,-'theorangeduck.com': 0.8271367134087778,-'blog.redfin.com': 0.8271155479281936,-'threema.ch': 0.8270778121654588,-'blog.gaborcselle.com': 0.8270675104289051,-'www.eastbayexpress.com': 0.8270623051198946,-'www.nilsjuenemann.de': 0.8270167583183482,-'www.ausgamers.com': 0.8269871316556712,-'www.nickkolenda.com': 0.8269846287715326,-'freakonomics.blogs.nytimes.com': 0.8269510780284808,-'datadome.co': 0.8269501507198006,-'www.matusiak.eu': 0.8269501507198006,-'marksaroufim.substack.com': 0.8269362550900476,-'kernel.org': 0.8269362550900476,-'foaas.com': 0.8268624447770888,-'www.quiverquant.com': 0.8268223600931086,-'www.pennmedicine.org': 0.8267857142857142,-'panamapapers.icij.org': 0.8267786682199767,-'rare-technologies.com': 0.8267627529251013,-'www.wusa9.com': 0.826717577802525,-'rustwasm.github.io': 0.8266828193880259,-'exupero.org': 0.8266797238058068,-'nodesource.com': 0.8266756851479922,-'tryapl.org': 0.8266364460873337,-'insights.sei.cmu.edu': 0.8266146306841327,-'zeynep.substack.com': 0.8265133053820785,-'adamcaudill.com': 0.8265133053820785,-'flexport.engineering': 0.8265065205596224,-'theneurosphere.com': 0.8264888595136076,-'news.rice.edu': 0.8264758779932855,-'blog.ninlabs.com': 0.8264758497316637,-'blog.snapchat.com': 0.8264601098042835,-'twistedoakstudios.com': 0.8264580233843033,-'qvault.io': 0.8264470673072151,-'zachaysan.tumblr.com': 0.826417360607573,-'investigations.nbcnews.com': 0.8264128301922102,-'ourmachinery.com': 0.8263907059824316,-'www.fs.blog': 0.8263508838430117,-'www.linuxatemyram.com': 0.8263506013533878,-'blog.kraih.com': 0.8263506013533877,-'blog.nyaruka.com': 0.8263442628141022,-'jeffdechambeau.com': 0.8263442628141021,-'opensoul.org': 0.8263238944048071,-'www.cortes.design': 0.8263165312463789,-'www.pawfal.org': 0.8263046319865304,-'blog.typesafe.com': 0.8262863127448572,-'blog.andrewcantino.com': 0.8262623863759415,-'www.boeing.com': 0.8262623863759415,-'www.ted.com': 0.8262579048280261,-'blog.custora.com': 0.8262292214279253,-'www.naftaliharris.com': 0.8261791079056829,-'javascriptkicks.com': 0.8261791079056829,-'the-toast.net': 0.8261672942679482,-'www.cs.rit.edu': 0.8261648745519713,-'jliszka.github.io': 0.8261587974287634,-'m.economictimes.com': 0.8261099137432224,-'vessenes.com': 0.8260709719869568,-'www.ineteconomics.org': 0.826029111542426,-'blogs.unity3d.com': 0.8260137334026378,-'0fps.net': 0.8260060814624561,-'justcramer.com': 0.8260060814624561,-'www.nest.com': 0.8260056123053906,-'qualiacomputing.com': 0.8259333270861942,-'improbable.io': 0.8259007499406081,-'blog.dilbert.com': 0.8258693250944908,-'standpointmag.co.uk': 0.8258674439632754,-'nelsonslog.wordpress.com': 0.8258527827648116,-'www.pixelmator.com': 0.8258442158455263,-'www.c2.com': 0.8258100134548192,-'www.heraldscotland.com': 0.8257871536416392,-'yuzu-emu.org': 0.8257634254559968,-'www.physicscentral.com': 0.8257504892739511,-'www.overclock3d.net': 0.8257367279398189,-'www.crainsnewyork.com': 0.8257322176521333,-'www.dtc.umn.edu': 0.8256884706720989,-'livefreeordichotomize.com': 0.8256884706720989,-'graphics.wsj.com': 0.8256623456620759,-'lostechies.com': 0.8256485195117381,-'weblog.bocoup.com': 0.8256267961489369,-'www.rechargenews.com': 0.8256135856550654,-'weatherspark.com': 0.8256135856550654,-'wiki.apache.org': 0.8256007740509997,-'ceylon-lang.org': 0.8256007740509996,-'androidrookies.com': 0.8255966724327264,-'nwn.blogs.com': 0.8255570752578437,-'nathany.com': 0.8255395683453237,-'www.deepdotweb.com': 0.8255335258309922,-'jonathanischwartz.wordpress.com': 0.8254547893724321,-'www.pitheringabout.com': 0.8254520771980598,-'steve.yegge.googlepages.com': 0.8254511675613948,-'www.useronboard.com': 0.8253939552044646,-'www.kjmagnetics.com': 0.8253701708120639,-'ftp.openbsd.org': 0.8253198620399047,-'www.bogleheads.org': 0.8253198620399047,-'v5.chriskrycho.com': 0.8253173977170429,-'jordanmechner.com': 0.8253173977170429,-'fare.tunes.org': 0.8253125699065577,-'www.dangermouse.net': 0.8252739167424069,-'pdf.yt': 0.82525521399989,-'robbreport.com': 0.8252506344809964,-'insideevs.com': 0.8252342355806511,-'www.logolounge.com': 0.8252252252252251,-'ftp.arl.army.mil': 0.8252200375406061,-'www.linux.com': 0.8252089691299165,-'www.csoonline.com': 0.8251776577952398,-'catapult.co': 0.8251768351081376,-'blog.webkid.io': 0.8251768351081376,-'www.uwgb.edu': 0.8251526412738834,-'girinstud.io': 0.8251526412738834,-'adilmoujahid.com': 0.8251526412738834,-'www.rntz.net': 0.8251450579360111,-'www.smore.com': 0.8251318689390142,-'news.rthk.hk': 0.8251206340702621,-'www.c-span.org': 0.825114692730766,-'m.stopa.io': 0.8250423585067803,-'www.opb.org': 0.8250415548188467,-'artofmanliness.com': 0.825002300673692,-'opendylan.org': 0.825001402919603,-'tailwindcss.com': 0.8249955991764671,-'youtube-creators.googleblog.com': 0.8249955991764671,-'www.plosone.org': 0.824994642431906,-'la.curbed.com': 0.8249814854575737,-'gfycat.com': 0.8249792022510777,-'pressfreedomfoundation.org': 0.8249170036751421,-'www.tested.com': 0.8249099846980117,-'blog.gruntwork.io': 0.8248984430563132,-'www.innoq.com': 0.8248966445207588,-'granolashotgun.com': 0.8248948672452628,-'articles.moneycentral.msn.com': 0.8248521762231498,-'www.bu.edu': 0.8247468947911291,-'mozillalabs.com': 0.8247395166074101,-'thinkgrowth.org': 0.8247377515142629,-'soledadpenades.com': 0.8247300709391115,-'snips.ai': 0.8247300709391115,-'insightdataengineering.com': 0.8247228989993084,-'altdevblogaday.com': 0.8247202423028511,-'bugs.webkit.org': 0.8246896410441447,-'www.serpentine.com': 0.8246896410441446,-'crimereads.com': 0.8246832358182735,-'www.chadfowler.com': 0.8246492171080979,-'flow.microsoft.com': 0.8245931283905966,-'muut.com': 0.8245391490982966,-'deque.blog': 0.8245391490982966,-'www.nearform.com': 0.8245170505651781,-'www.us-cert.gov': 0.82449055422799,-'regexcrossword.com': 0.8244429611663081,-'chris-lamb.co.uk': 0.824442961166308,-'unretrofied.com': 0.8244331063039999,-'www.rsaconference.com': 0.8244327243272294,-'www.nhregister.com': 0.8244082580729432,-'cryoshon.co': 0.8244082580729432,-'www.dpmms.cam.ac.uk': 0.8243716227838693,-'www.patreon.com': 0.8243522520227725,-'salon.thefamily.co': 0.8243182801287786,-'www.leakedsource.com': 0.8242753623188406,-'www.clicktorelease.com': 0.824270209428051,-'www.chipworks.com': 0.824267686738341,-'www.hardkernel.com': 0.824267686738341,-'news.slashdot.org': 0.8242672968534664,-'developer.atlassian.com': 0.8242608280205023,-'www.top500.org': 0.8242567265672777,-'technicshistory.com': 0.8242340010424594,-'segment.io': 0.8242217790014779,-'mitsloan.mit.edu': 0.8242217790014779,-'thebuild.com': 0.8242049759936478,-'thistooshallgrow.com': 0.8241685536402895,-'alliance.nautil.us': 0.8241562516385773,-'www.slashgeek.net': 0.8241560987787111,-'www.zeroequalsfalse.press': 0.8241315599752687,-'poincare101.blogspot.com': 0.8240928369315027,-'blog.zarfhome.com': 0.8240852612726679,-'www.ryanpickren.com': 0.8240852612726679,-'design.canonical.com': 0.8240707368381324,-'www.theengineer.co.uk': 0.8240609124764349,-'the-magazine.org': 0.8239745714397421,-'dennybritz.com': 0.8239663447195944,-'ben.casnocha.com': 0.8239663447195944,-'www.alexirpan.com': 0.8239445529818444,-'www.siliconvalleywatcher.com': 0.8239203009772859,-'chill.com': 0.8239075286809479,-'kanoki.org': 0.8238968676022924,-'knowablemagazine.org': 0.8238968676022923,-'www.fun-mooc.fr': 0.8238950612054784,-'www.seas.upenn.edu': 0.823854197776325,-'longbets.org': 0.8238504937824159,-'papers.nips.cc': 0.8238165389819486,-'dyske.com': 0.8237859610530819,-'i.gizmodo.com': 0.8237724868134527,-'www.kth.se': 0.8237610739927905,-'www.tawheedkader.com': 0.8237541123759499,-'www.gov.uk': 0.8237306525116473,-'chsasank.github.io': 0.8237179544778666,-'plasmasturm.org': 0.8236804766955088,-'blog.backtype.com': 0.8236678521506691,-'feaneron.com': 0.8236399179105635,-'tech.backtype.com': 0.8236399179105635,-'www.annashipman.co.uk': 0.8236363636363635,-'sive.rs': 0.8235995425409803,-'keen.io': 0.8235778391019275,-'wensing.tumblr.com': 0.8234963171167604,-'www1.informatik.uni-erlangen.de': 0.8234356896394766,-'www.ewanleith.com': 0.8233853975881945,-'tech.puredanger.com': 0.8233619779868596,-'freshman.tech': 0.823356964994172,-'weakdh.org': 0.8233151183970855,-'boundary.com': 0.8233084145427331,-'taitems.tumblr.com': 0.8232972027089648,-'www2.eecs.berkeley.edu': 0.823294055901832,-'arvindn.livejournal.com': 0.8232708263906926,-'pcmonk.me': 0.8232633368501505,-'www.wfla.com': 0.8232633368501505,-'blog.adamnash.com': 0.8232369516538023,-'english.martinvarsavsky.net': 0.8232086570141259,-'www.usv.com': 0.8231252352764534,-'federalnewsnetwork.com': 0.8231197830462293,-'www.theatlas.com': 0.8231091015701053,-'www.riknieu.com': 0.8231091015701053,-'www.iterm2.com': 0.8231091015701053,-'www.itwire.com': 0.8231086108233077,-'www.standalone-sysadmin.com': 0.8231078625216017,-'www.autoblog.com': 0.8230993668082046,-'pv-magazine-usa.com': 0.8230980746736871,-'blog.andrewvc.com': 0.8230893291673793,-'www.mappingthejourney.com': 0.8230893291673792,-'blogs.s-osg.org': 0.8230893291673792,-'www.erica.biz': 0.8230405980636032,-'digitallife.today.msnbc.msn.com': 0.822992700729927,-'humantransit.org': 0.822978047436078,-'anaxi.com': 0.8229457720254757,-'bedrocklinux.org': 0.8229141632870333,-'sina.is': 0.8229141632870332,-'www.joachim-breitner.de': 0.8229052530564972,-'o.canada.com': 0.8228104673631248,-'michaelrbernste.in': 0.8228054773077562,-'status.digitalocean.com': 0.8227743399626116,-'www.computerworlduk.com': 0.8227710906704893,-'tech.newstatesman.com': 0.8227353406126373,-'abc7chicago.com': 0.8227318903356561,-'bytes.yingw787.com': 0.8226344289193692,-'www.mikekarnj.com': 0.822628692450413,-'blogarithms.github.io': 0.8226216439253611,-'designinstruct.com': 0.8226216439253611,-'fsecurify.com': 0.8225830523375955,-'deepsource.io': 0.8225746395058767,-'n-gate.com': 0.8225612808865542,-'www.25hoursaday.com': 0.8225607212300213,-'global.handelsblatt.com': 0.8225208403926864,-'pragli.com': 0.8225208403926864,-'manuel.bernhardt.io': 0.8225011685480498,-'blog.jooq.org': 0.8224250424416801,-'www.portfolio.com': 0.8224243429262191,-'web.nvd.nist.gov': 0.8224221426834478,-'www.hn-books.com': 0.8223443223443224,-'felixpalmer.github.io': 0.8223443223443223,-'weev.livejournal.com': 0.8223443223443223,-'www.harpersbazaar.com': 0.822307206378181,-'snapframework.com': 0.822307206378181,-'www.hussmanfunds.com': 0.822307206378181,-'benlog.com': 0.8222776118268347,-'www.fluentcpp.com': 0.8222033505775027,-'www.toptal.com': 0.8221835594409774,-'www.openssh.com': 0.8221823092541788,-'blog.avast.com': 0.8221738853904804,-'www.timesofisrael.com': 0.822154529331881,-'www.tomdalling.com': 0.8221220478981254,-'matloff.wordpress.com': 0.8221023761491743,-'www.stuff.co.nz': 0.8220743929973073,-'heather.cs.ucdavis.edu': 0.8220584263105031,-'theincidentaleconomist.com': 0.8220326740109917,-'radiapp.com': 0.8220183486238531,-'blog.ramen.is': 0.8219861931408673,-'geti2p.net': 0.8219861931408673,-'bgr.com': 0.8218680813488343,-'hey.com': 0.821866426984301,-'33bits.org': 0.8218275173853028,-'www.pressherald.com': 0.8218104832596405,-'www.staycaffeinated.com': 0.8217905775466726,-'www.igorkromin.net': 0.8217742285549731,-'zackarymorris.tumblr.com': 0.8217716107859097,-'seattletimes.com': 0.8217659740535453,-'news.northwestern.edu': 0.8217641342619865,-'canny.io': 0.8217480223498653,-'www.modernhealthcare.com': 0.8217164092599815,-'about.google': 0.8217164092599815,-'www.ifweassume.com': 0.8216730309542566,-'mobile.abc.net.au': 0.8216657761831399,-'blog.readme.io': 0.8216640179430849,-'www.bschaatsbergen.com': 0.8216640179430849,-'bendyworks.com': 0.8216400745937131,-'randywestergren.com': 0.8215693482108614,-'www.crn.com.au': 0.8215655177172531,-'yinwang0.wordpress.com': 0.8215439095876336,-'substack.net': 0.8215269428046106,-'www.toothycat.net': 0.8214683749692366,-'vega.github.io': 0.8214578695182713,-'pixelscommander.com': 0.8214578695182713,-'www.thinq.co.uk': 0.8214038185099727,-'www.huffingtonpost.com.au': 0.8213863864440609,-'www.depesz.com': 0.8213734717251845,-'www.w2lessons.com': 0.8213669213667488,-'www.mostlymaths.net': 0.8213616628443653,-'putanumonit.com': 0.8213391337481347,-'www.executeprogram.com': 0.8213391337481347,-'www.computing.co.uk': 0.8213046552380585,-'eblong.com': 0.8212918616104282,-'asylum.madhouse-project.org': 0.8212918616104282,-'www.granolashotgun.com': 0.8212918616104282,-'www.redditblog.com': 0.8212710455497659,-'www.causal.app': 0.821219763084809,-'valerieaurora.org': 0.8212075934194915,-'www.owasp.org': 0.8211882087756882,-'petition.parliament.uk': 0.8211640347006609,-'engineering.columbia.edu': 0.821142906452761,-'fastml.com': 0.8211297895691131,-'nerderati.com': 0.8211239168679134,-'www.nutsvolts.com': 0.8211239168679134,-'vine.co': 0.821112630700933,-'www.grobmeier.de': 0.8211091773217756,-'html.adobe.com': 0.8211013298757308,-'coursetro.com': 0.8210338632520735,-'www.rleonardi.com': 0.8210332103321032,-'make.wordpress.org': 0.8210118109358497,-'iq.opengenus.org': 0.8209763562067588,-'tech.slashdot.org': 0.820954477342236,-'www.stanforddaily.com': 0.8209221578417347,-'www.mapd.com': 0.8208973498293703,-'campaignstops.blogs.nytimes.com': 0.8208298386512924,-'www.theblaze.com': 0.8208160487478949,-'codingkaiser.blog': 0.8208103239764403,-'cushychicken.github.io': 0.8208101603109114,-'techblog.badoo.com': 0.8207887125985329,-'arcanesentiment.blogspot.com': 0.8207861411585611,-'www.lawfareblog.com': 0.8207748819367903,-'taskandpurpose.com': 0.8207638339548695,-'programmingdesignsystems.com': 0.8207024029574862,-'www.robots.ox.ac.uk': 0.8206399949682861,-'us.pycon.org': 0.820630666964755,-'market.android.com': 0.8206208283828789,-'blog.kirigin.com': 0.8205932101360961,-'www.pragprog.com': 0.820579551832526,-'blog.perfectaudience.com': 0.8205466327449291,-'mreinhold.org': 0.820546632744929,-'www.thoughtworks.com': 0.8205459808249582,-'blog.frankel.ch': 0.8205171785173707,-'philipnilsson.github.io': 0.8205088545817216,-'www.deathandtaxesmag.com': 0.8203995498364007,-'7news.com.au': 0.8203793607701599,-'keminglabs.com': 0.8203793607701598,-'research.securitum.com': 0.8203793607701598,-'blog.nanonets.com': 0.8203793607701598,-'www.roguelynn.com': 0.8203703703703703,-'zenhack.net': 0.8203703703703703,-'corpgov.law.harvard.edu': 0.8203690845488739,-'thelongandshort.org': 0.8203387274635711,-'bramcohen.livejournal.com': 0.820330588464305,-'www.nyu.edu': 0.82033015672031,-'howtosplitanatom.com': 0.8203248926942794,-'losangeles.cbslocal.com': 0.8202840702703775,-'www.freebsdfoundation.org': 0.8202742664017543,-'pqina.nl': 0.8202180020912505,-'ramblemaps.com': 0.820186718359544,-'blog.httpwatch.com': 0.8201707118020548,-'aroberge.blogspot.com': 0.8201707118020547,-'joanwestenberg.medium.com': 0.8201707118020546,-'www.texastribune.org': 0.820127140783887,-'vagosec.org': 0.8201145209512104,-'blog.michielborkent.nl': 0.8200731194854671,-'www.desmogblog.com': 0.8200731194854671,-'informationarchitects.net': 0.8200687812178955,-'www.mpg.de': 0.8200224577197879,-'devcenter.heroku.com': 0.8200215315397114,-'blog.hackerrank.com': 0.8199539191124647,-'www.mailboxapp.com': 0.8199115962474446,-'libregraphicsworld.org': 0.8198809873858414,-'www.netmarketshare.com': 0.8198634237702345,-'www.honeycomb.io': 0.8198446827660495,-'wwwnc.cdc.gov': 0.8197995558341037,-'davidhauser.com': 0.8197995558341036,-'www.philanthropy.com': 0.8197755193144188,-'www.armstrongeconomics.com': 0.8197658321751845,-'www.techpowerup.com': 0.8197581725949339,-'fourweekmba.com': 0.8197401870662347,-'runwayml.com': 0.8197062084117198,-'cs.uwaterloo.ca': 0.8197032651831856,-'jeffarchibald.ca': 0.8196469792108286,-'monograph.io': 0.8196424057593232,-'jaredforsyth.com': 0.8196424057593232,-'rework.withgoogle.com': 0.8196231659693813,-'www.butwhatfor.com': 0.8195815182778377,-'webglfundamentals.org': 0.8195389645544945,-'skepchick.org': 0.8195389645544945,-'t37.net': 0.8195018817046663,-'blockstack.org': 0.8195018817046663,-'www.ticalc.org': 0.8195018248001847,-'konghq.com': 0.8194820842106035,-'cseweb.ucsd.edu': 0.8194210882358477,-'www.orlandosentinel.com': 0.8194144516611968,-'ansuz.sooke.bc.ca': 0.8194061161585031,-'platform.uno': 0.8193808271817485,-'svs.gsfc.nasa.gov': 0.8193808271817484,-'i-kh.net': 0.819366852886406,-'www.t3x.org': 0.8193537789546194,-'thinkingagriculture.io': 0.8193537789546194,-'www.colorado.edu': 0.8192948515552042,-'philipkiely.com': 0.8192790918819292,-'theultralinx.com': 0.8192615833622455,-'the-witness.net': 0.8192531710522758,-'www.woot.com': 0.8192133344078468,-'blog.crowdtilt.com': 0.8192133344078468,-'docs.python-guide.org': 0.8192031083028783,-'blog.indextank.com': 0.819186830384332,-'bfi.uchicago.edu': 0.8191137736578458,-'theirrelevantinvestor.com': 0.8190345082551709,-'herman.asia': 0.8190298507462687,-'quartzy.qz.com': 0.8189961869849737,-'www.peterbe.com': 0.8189793251113663,-'shakycode.com': 0.8189532189180286,-'jeffreykegler.github.io': 0.8189309404959616,-'programmer.97things.oreilly.com': 0.8189307834762161,-'spectrum.chat': 0.8189034014413787,-'alugy.com': 0.8189014793607566,-'www.modern.ie': 0.8188594320310705,-'ilovetypography.com': 0.8188542303813234,-'parametric.press': 0.8188376402212918,-'blog.staffjoy.com': 0.8187965320542665,-'grahamcluley.com': 0.8187959495636524,-'benediktmeurer.de': 0.8187751653221643,-'www.loggly.com': 0.818750224724795,-'typophile.com': 0.8187110947432705,-'themorningnews.com': 0.8187110947432705,-'blog.estimote.com': 0.8187092985024379,-'www.nola.com': 0.8187092491447127,-'blog.fanout.io': 0.8187064944602657,-'theoreticalminimum.com': 0.8186915887850467,-'snap.stanford.edu': 0.8186423752137325,-'serhack.me': 0.8186312306298296,-'digital.nhs.uk': 0.8186026992823302,-'investor.yahoo.net': 0.8185979779164885,-'www.gamingonlinux.com': 0.818574619301555,-'interactions.acm.org': 0.8185585358755773,-'htmlpreview.github.io': 0.8185193011459522,-'codeberg.org': 0.8185193011459521,-'www.arm.com': 0.8184839406823776,-'www.fedscoop.com': 0.8184720810751717,-'video.google.com': 0.8184638061300447,-'eyeondesign.aiga.org': 0.8184578256865039,-'vihart.com': 0.8184565521589099,-'history.nasa.gov': 0.8184538842053415,-'www.webmonkey.com': 0.8184424171038894,-'www.pacifict.com': 0.8184091497388042,-'www.anaconda.com': 0.8184074729195554,-'jamesclear.com': 0.8183928113741262,-'images.guide': 0.8183520599250935,-'sandofsky.com': 0.818333099426741,-'tessrinearson.com': 0.8182949341417263,-'mysqlserverteam.com': 0.8182817858828835,-'www.virology.ws': 0.8182801242351887,-'transparencyreport.google.com': 0.8182801242351887,-'b.lesseverything.com': 0.8182544712962565,-'forklog.media': 0.8182524048349719,-'mega.nz': 0.8182293546514141,-'postcards.blogs.fortune.cnn.com': 0.8181713824074244,-'kevinrose.com': 0.8181713824074243,-'www.sankalpjonna.com': 0.8181713824074243,-'nick.typepad.com': 0.8181333642385573,-'mathbabe.org': 0.8180898841225986,-'hacksoflife.blogspot.com': 0.8180862740536983,-'nickmchardy.com': 0.8180611629393052,-'www.spectrumnews.org': 0.8180586243621752,-'www.fictiv.com': 0.8180206950485711,-'www.riskbasedsecurity.com': 0.8179622866340774,-'juliagalef.com': 0.8178989768184225,-'wheels.blogs.nytimes.com': 0.8178917680568788,-'medium.learningbyshipping.com': 0.8178382028024892,-'travel.stackexchange.com': 0.8178270639852792,-'developer.yahoo.net': 0.8177152660164336,-'www.lostbookofsales.com': 0.8177143703947889,-'blog.rainforestqa.com': 0.8176913530816636,-'www.zeldman.com': 0.8176675642405572,-'www.shogan.co.uk': 0.8176447363707375,-'blog.friendfeed.com': 0.8176243006331482,-'tech.instacart.com': 0.8176187126924949,-'www.well-typed.com': 0.8176129455942598,-'madebynathan.com': 0.8175850766542176,-'www.dam.brown.edu': 0.8175673958400209,-'www.electrive.com': 0.8175415128966471,-'open.bekk.no': 0.8174977146526387,-'launch.is': 0.8174957585928975,-'mindyourdecisions.com': 0.8174283379047065,-'avc.blogs.com': 0.8174249012744215,-'underscore.io': 0.8174038787709598,-'beakerbrowser.com': 0.8173898378490387,-'tibleiz.net': 0.8173670622724108,-'learnlayout.com': 0.8173662532870822,-'statescoop.com': 0.8173342969093829,-'pcottle.github.io': 0.8173258003766477,-'mkaz.blog': 0.8173258003766477,-'davmac.wordpress.com': 0.817306487680897,-'khalilstemmler.com': 0.817286266376623,-'mobile.slate.com': 0.817274377154757,-'pepijndevos.nl': 0.8172690940621161,-'en.arguman.org': 0.8172440600460996,-'dougseven.com': 0.8172346051319319,-'www.hsj.co.uk': 0.8172346051319318,-'www.washingtoncitypaper.com': 0.8172227432631624,-'www.stefanjudis.com': 0.8172227432631624,-'remotehabits.com': 0.8170840840435508,-'pawelurbanek.com': 0.8170758370632349,-'www.linode.com': 0.8170518536342932,-'mattrogish.com': 0.8170279345181233,-'www.dataorienteddesign.com': 0.8170279345181233,-'vtorosyan.github.io': 0.8170279345181232,-'www.adapteva.com': 0.8170168040129737,-'waldo.jaquith.org': 0.8170168040129737,-'www.sentient.ai': 0.8169811320754717,-'cheeaun.com': 0.8169811320754716,-'adam.chlipala.net': 0.8169734054322659,-'www.fastmail.com': 0.8169005980617469,-'law.duke.edu': 0.8169005980617468,-'uglyduck.ca': 0.8167837603451541,-'labs.sentinelone.com': 0.8167337370996636,-'whoisnnamdi.com': 0.8166875449806325,-'eagain.net': 0.8166739998002963,-'news.columbia.edu': 0.8166739998002963,-'bibwild.wordpress.com': 0.8166660149109494,-'docs.fedoraproject.org': 0.8166606678373952,-'branchfree.org': 0.8166606678373952,-'www.howmanypeopleareinspacerightnow.com': 0.8166606678373952,-'devhints.io': 0.8166486478801723,-'http.cat': 0.8166351606805292,-'www.aspi.org.au': 0.8166351606805292,-'lalo.li': 0.8166351606805292,-'forum.linode.com': 0.8166342281215091,-'www.cyanogenmod.com': 0.8166342281215091,-'www.math.brown.edu': 0.8166342281215091,-'embracethered.com': 0.8166342281215091,-'insights.stackoverflow.com': 0.8166112476989804,-'codemanship.co.uk': 0.816610401586209,-'bombthrower.com': 0.8165653679484848,-'open.dgraph.io': 0.8165613796925617,-'dlewis.net': 0.8165199873064167,-'www.infoworld.com': 0.8165194544092945,-'simonsfoundation.org': 0.8164711754414633,-'blog.evanweaver.com': 0.8164587254389714,-'webdocs.cs.ualberta.ca': 0.8164587254389714,-'runnable.com': 0.8164587254389714,-'scienceblog.com': 0.816442930863004,-'theinvisiblethings.blogspot.com': 0.8164118858386505,-'caterina.net': 0.8163684777302764,-'www.cockeyed.com': 0.8163493619189814,-'garry.tv': 0.8163458866047911,-'tech.mattmillman.com': 0.8163458866047911,-'pavelfatin.com': 0.8163391912303228,-'blog.testmunk.com': 0.8163326429746816,-'righteousruminations.blogspot.com': 0.8163302805339576,-'playnice.ly': 0.8163302805339576,-'thedriven.io': 0.8162915628964128,-'liorpachter.wordpress.com': 0.8162914041012925,-'rr-project.org': 0.8162878787878787,-'vis4.net': 0.8162878787878787,-'purelyfunctional.tv': 0.8162705368577864,-'cachestocaches.com': 0.816247477572823,-'www.novelr.com': 0.8162474775728229,-'www.myasmine.com': 0.8162289080621473,-'www.hanshq.net': 0.8162230743510456,-'urbankchoze.blogspot.com': 0.8160645118710772,-'powazek.com': 0.8160565844295424,-'www.mjt.me.uk': 0.8160235722456507,-'www.loc.gov': 0.8160198829440672,-'www.businessinsider.fr': 0.8160110236421557,-'benmetcalfe.com': 0.816002952282756,-'cloudant.com': 0.8159956502678918,-'buddycloud.com': 0.8159660889619967,-'devinhelton.com': 0.8159603862822139,-'blog.onedrive.com': 0.8159603862822139,-'hackoftheday.securitytube.net': 0.8159603862822139,-'internetintel.oracle.com': 0.8159603862822139,-'news.rpi.edu': 0.8159324993062729,-'kyso.io': 0.8157856217718672,-'blog.cryptoaustralia.org.au': 0.8157807107640108,-'www.pixelmonkey.org': 0.8157110214426208,-'gbracha.blogspot.com': 0.8156837220936173,-'pandoralive.info': 0.8156756405717832,-'www.lessmilk.com': 0.8156734149395104,-'blog.everyblock.com': 0.815665920751951,-'moyix.blogspot.com': 0.815665920751951,-'blog.keep.network': 0.815665920751951,-'www.dpreview.com': 0.8156119793953887,-'blog.calbucci.com': 0.8156110920088071,-'kowainik.github.io': 0.8156055311358869,-'blogs.discovery.com': 0.8156055311358867,-'flic.kr': 0.8155522718024564,-'ledgersmbdev.blogspot.com': 0.8155503540881356,-'www.bostonmagazine.com': 0.8155484449427585,-'www.haydenplanetarium.org': 0.815539827164826,-'lukesmith.xyz': 0.815539827164826,-'blog.djy.io': 0.815539827164826,-'www.treasury.gov': 0.8154535511908948,-'abovethecrowd.com': 0.815439760100364,-'yogthos.net': 0.8154331327745159,-'www.adn.com': 0.8153813333062851,-'www.eveonline.com': 0.8153685129167869,-'blog.metamorphium.com': 0.8153509023913048,-'dannyvankooten.com': 0.8152380952380952,-'www.creativereview.co.uk': 0.8152232350076929,-'blog.hasura.io': 0.8152023189276763,-'artsbeat.blogs.nytimes.com': 0.8151914351876436,-'dosync.posterous.com': 0.8151891543888505,-'openreview.net': 0.8151568208322646,-'blog.serverdensity.com': 0.8151341964704665,-'manybutfinite.com': 0.8151251257545465,-'mars.nasa.gov': 0.8150920162802555,-'www.rbth.com': 0.8150730163714182,-'www.fefe.de': 0.8150624797096998,-'mcla.ug': 0.8150272592774294,-'chem.tufts.edu': 0.8150272592774294,-'code.jquery.com': 0.8149967373525299,-'tantek.com': 0.8149534354896242,-'www.grahamcluley.com': 0.8149532812632393,-'query.nytimes.com': 0.814942807315685,-'greig.cc': 0.8149045173924782,-'www.xaprb.com': 0.8149019506115348,-'badssl.com': 0.814885496183206,-'bbirnbaum.com': 0.8148706344450946,-'officialandroid.blogspot.com': 0.8147398134191081,-'ruudvanasseldonk.com': 0.8147309156410547,-'www.softwareatscale.dev': 0.8146603203498196,-'heap.engineering': 0.8146603203498196,-'videlalvaro.github.io': 0.8146574608017414,-'norswap.com': 0.8146006935103474,-'reactionwheel.net': 0.8145996500684587,-'bitslog.wordpress.com': 0.8145723063065177,-'rasterize.io': 0.8145327073791039,-'adlervermillion.com': 0.8145327073791038,-'timeless.judofyr.net': 0.81453154875717,-'haskell.org': 0.8145294619307533,-'githut.info': 0.8145279229959786,-'www.gadgette.com': 0.8145279229959786,-'frantic.im': 0.8145279229959786,-'www.creativeapplications.net': 0.8145169207397831,-'googlecommerce.blogspot.com': 0.8144066006321738,-'mikeash.com': 0.8144025820168224,-'wecode.wepay.com': 0.8143827137547628,-'www.cs.ucdavis.edu': 0.8143765574898738,-'data.heapanalytics.com': 0.8143226776111164,-'blog.mixergy.com': 0.8143177092759811,-'www.sudo.ws': 0.8142855202937566,-'simplystatistics.org': 0.814273938401625,-'pbpython.com': 0.81426991310501,-'www.andrew.cmu.edu': 0.814256651162313,-'www.techinasia.com': 0.8142474634536047,-'www.scirra.com': 0.8142144867968754,-'mrmacintosh.com': 0.8142070774434265,-'omarabid.com': 0.8142001608362681,-'db.cs.cmu.edu': 0.8141839403388533,-'reactkungfu.com': 0.8141839403388533,-'paulbridger.com': 0.8141723199425726,-'openbazaar.org': 0.8141723199425726,-'www.globalpost.com': 0.8141091737204239,-'david-smith.org': 0.8140528179043202,-'www.philipotoole.com': 0.8140231382115423,-'20bits.com': 0.8139880326617631,-'www.kickscondor.com': 0.813983802425237,-'blog.zdsmith.com': 0.813920131333506,-'www.iclarified.com': 0.8139139706368135,-'www.x-plane.com': 0.8138853765840854,-'mirage.io': 0.8138853765840854,-'blog.expensify.com': 0.8138812059202145,-'www.iso.org': 0.8138437732374225,-'cryptoservices.github.io': 0.8138386793877036,-'hedgehogreview.com': 0.8138372159261252,-'www.simple.com': 0.8138043652042445,-'ftp.cs.ucla.edu': 0.8137212614300459,-'blog.laptopmag.com': 0.8137088732053118,-'cliffmass.blogspot.com': 0.8136763300597577,-'kubuszok.com': 0.8136519481838238,-'whatblag.com': 0.813578419801053,-'insight.kellogg.northwestern.edu': 0.8135739399890168,-'skerritt.blog': 0.8135660771798076,-'www.shacknews.com': 0.8135400926338027,-'www.hcn.org': 0.8134966474909895,-'www.desmoinesregister.com': 0.8134966474909894,-'www.intelligentliving.co': 0.81349213300375,-'www.huffpost.com': 0.8134799974360535,-'nextbigfuture.com': 0.8134623168672729,-'www.arneswinnen.net': 0.8134615384615383,-'www.bellingcat.com': 0.8134267051774864,-'www.zverovich.net': 0.8133918753159222,-'blog.delicious.com': 0.8133799785061123,-'hackthesystem.com': 0.8133799785061122,-'blog.comae.io': 0.8133732585740193,-'altdevblogaday.org': 0.8133173831188817,-'w2.eff.org': 0.8132979261023464,-'news.fastcompany.com': 0.8132816312546639,-'blog.dominodatalab.com': 0.8132112268692228,-'www.burlingtonfreepress.com': 0.8131021194605009,-'betterwebtype.com': 0.8130859124005918,-'tech.yipit.com': 0.8130859124005917,-'www.capitolhillseattle.com': 0.8130859124005917,-'flyosity.com': 0.8130320794442989,-'juffalow.com': 0.8130140383642517,-'www.alexstjohn.com': 0.8130023652493408,-'awardwinningfjords.com': 0.8129597143337574,-'humanbioscience.org': 0.8129551673058468,-'programmersatwork.wordpress.com': 0.8129191502536695,-'www.gao.gov': 0.8129073172130272,-'www.sci-news.com': 0.8129054883394705,-'www.teslarati.com': 0.8128722926988359,-'www.creativedeletion.com': 0.8128708612164178,-'hushkit.net': 0.8128656957185499,-'www.sketch.com': 0.8127222898502466,-'0xfe.blogspot.com': 0.8127222898502466,-'www.governmentattic.org': 0.8127166548773207,-'www.iflscience.com': 0.8127104066471765,-'www.bmwblog.com': 0.8127009540598622,-'thephd.github.io': 0.8126088980839451,-'papercups.io': 0.8125137829701504,-'deeplearning4j.org': 0.8125126453331284,-'www.kaiostech.com': 0.8125018296149806,-'www.kdab.com': 0.8124430720535263,-'communities.intel.com': 0.8123620795777623,-'www.knowablemagazine.org': 0.8123571766847992,-'sci-hub.tw': 0.8122620383047426,-'flink.apache.org': 0.8122287873617684,-'www.mobilecrunch.com': 0.8120912628571249,-'blog.liangzan.net': 0.8120625991864463,-'paulcavallaro.com': 0.8120625991864463,-'www.freebsdnews.net': 0.8120155038759689,-'www.cs.dartmouth.edu': 0.8120120009628422,-'slashdot.org': 0.8120103261102206,-'beta.theglobeandmail.com': 0.8119800311478199,-'www.franzoni.eu': 0.8119518701628052,-'www.unz.com': 0.8119446655347015,-'launch.gust.com': 0.8119242878514453,-'mattlakeman.org': 0.8119242878514452,-'us.battle.net': 0.8119135744478586,-'vitess.io': 0.8119046881511972,-'www.parsonsmatt.org': 0.8118880860804719,-'www.sciam.com': 0.8118415645513156,-'blog.inklingmarkets.com': 0.8118015537952591,-'darknetdiaries.com': 0.8117495396709252,-'blog.mixpanel.com': 0.8117419261805633,-'blog.kippt.com': 0.8117273158415922,-'danielrapp.github.io': 0.8117273158415922,-'semantic-domain.blogspot.com': 0.8117250261807765,-'blog.jayfields.com': 0.8117106131262737,-'dronegarageblog.wordpress.com': 0.8116970637651796,-'m.cnet.com': 0.8116871715030578,-'bionic.fullstory.com': 0.8116504854368931,-'blog.reybango.com': 0.8116192339099034,-'baesystemsai.blogspot.com': 0.8116098514216743,-'tasvideos.org': 0.811592254952584,-'www.nathancunn.com': 0.8115765934419358,-'theguardian.com': 0.8115466314959758,-'www.bls.gov': 0.8114586352368218,-'www.slashgear.com': 0.8114130074807454,-'get21stnight.com': 0.8113811919256128,-'wadefoster.net': 0.8113380585952003,-'backstage.soundcloud.com': 0.8113341783615344,-'blog.mongohq.com': 0.8113084373038951,-'blog.samwhited.com': 0.8113018683008136,-'lukethomas.com': 0.8112900298646131,-'johnmurray.io': 0.811284046692607,-'www.scpr.org': 0.8111931888951548,-'serverless.com': 0.8111853186864186,-'www.mintpressnews.com': 0.8111566169738434,-'dev.hubspot.com': 0.8111491117505176,-'uxdesign.smashingmagazine.com': 0.8110875951026609,-'www.dcscience.net': 0.8110637872870197,-'www.badlogicgames.com': 0.8110531473954321,-'www.imdb.com': 0.8109585365005275,-'www.kevinwilliampang.com': 0.810916179337232,-'benjaminmayo.co.uk': 0.8109161793372319,-'www.tonymacx86.com': 0.8109161793372319,-'news.firedoglake.com': 0.8109161793372319,-'gitorious.org': 0.810892038104006,-'www.indystar.com': 0.810892038104006,-'www.nextbigwhat.com': 0.810871453085769,-'vectorized.io': 0.8108646859647678,-'www.fakesteve.net': 0.8108603160330469,-'theness.com': 0.8108582902726974,-'blog.zenkit.com': 0.8108296140970532,-'consequenceofsound.net': 0.8108001038237775,-'dealbook.blogs.nytimes.com': 0.810782053555595,-'raccoon.onyxbits.de': 0.8107413934848161,-'www.linaro.org': 0.8107386685587079,-'insights.hpe.com': 0.8107316248294252,-'webflow.com': 0.8107174188583165,-'insidesearch.blogspot.com': 0.8107157266121297,-'www.hhs.gov': 0.8107124152561004,-'blog.folyo.me': 0.8107124152561003,-'blog.playcanvas.com': 0.8107042145283994,-'oldblog.antirez.com': 0.8106726118909099,-'computinged.wordpress.com': 0.8106676355919645,-'www.beekeeperstudio.io': 0.8106491457950028,-'www.meetup.com': 0.8106205529212165,-'capiche.com': 0.8105820355692206,-'www.cbp.gov': 0.810546875,-'stackexchange.com': 0.8105386561469894,-'kapeli.com': 0.8105257538297364,-'blog.sigfpe.com': 0.8105035346007113,-'nianticlabs.com': 0.8104898882641829,-'econlog.econlib.org': 0.8104880381821702,-'bitsofco.de': 0.8104598795589895,-'blog.gyrosco.pe': 0.8104246070876013,-'blog.n01se.net': 0.8104076949009623,-'mozakai.blogspot.com': 0.8104076949009623,-'iamwil.posterous.com': 0.8103954830327119,-'www.minyanville.com': 0.8103877873546691,-'jcjc-dev.com': 0.8103364623358701,-'decryptmedia.com': 0.8103177363469551,-'alexvermeer.com': 0.8103141664126979,-'reddragdiva.dreamwidth.org': 0.8103141664126978,-'anarchogeek.com': 0.8103141664126978,-'truthonthemarket.com': 0.810302799232476,-'numenta.com': 0.8102602204436639,-'economictimes.indiatimes.com': 0.8102491144464327,-'www.perceptualedge.com': 0.8102043958285661,-'www.garykessler.net': 0.8101761252446185,-'loopj.com': 0.8101761252446184,-'newteevee.com': 0.8101251805166595,-'morningconsult.com': 0.8101075221559366,-'mrmrs.io': 0.810089337360651,-'www.springboard.com': 0.8100885519221457,-'www.computinghistory.org.uk': 0.8100885519221456,-'ownlocal.com': 0.810072359778719,-'wellcomecollection.org': 0.810072359778719,-'owasp.org': 0.810072359778719,-'linuxreviews.org': 0.8100650707969348,-'monadical.com': 0.8100469597795646,-'gamesfromwithin.com': 0.810042091205529,-'www.geeksforgeeks.org': 0.8100336615814749,-'www.universal-sci.com': 0.810032793083701,-'esoteric.codes': 0.8099527252346412,-'www.petermichaud.com': 0.8099337587386388,-'www.freedom-to-tinker.com': 0.8099164380793796,-'www.laptopmag.com': 0.809846769190124,-'www.3news.co.nz': 0.8098352603019351,-'sel4.systems': 0.8098186580704161,-'www.accountingtoday.com': 0.8098186580704161,-'host.madison.com': 0.8097889409717836,-'www.nokia.com': 0.8097879208159734,-'insanecoding.blogspot.com': 0.8097776259075163,-'kdenlive.org': 0.8097696203436352,-'www.bitstamp.net': 0.8097578373340156,-'stonemaiergames.com': 0.809735836328836,-'ceronman.com': 0.809735818851372,-'airmail.news': 0.809735818851372,-'nickwignall.com': 0.8097127006730107,-'www.develop-online.net': 0.8096968183358267,-'brunocalza.me': 0.809690221614063,-'gadgetopia.com': 0.8096902216140629,-'seekingalpha.com': 0.8096704138265437,-'www.jofreeman.com': 0.8095931882025065,-'muckandbrass.com': 0.8095931882025065,-'nobelprize.org': 0.8095392106610299,-'www.vg247.com': 0.8095156620373399,-'chetansurpur.com': 0.8094671848396732,-'haxiomic.github.io': 0.8094631273918713,-'deislabs.io': 0.8094631273918712,-'jmc.stanford.edu': 0.8094565130568238,-'philpapers.org': 0.8094356549031609,-'youtube-global.blogspot.com': 0.8094272365570714,-'rubygems.org': 0.8093966911744022,-'www.laquadrature.net': 0.8093644943397358,-'dheera.net': 0.80935652930136,-'use.perl.org': 0.8093278422294106,-'dannorth.net': 0.8093207850460444,-'orbitalindex.com': 0.8093095444635583,-'historyofyesterday.com': 0.8092882173909588,-'www.citymetric.com': 0.809287154904464,-'www.codecommit.com': 0.8092420697380182,-'danlec.com': 0.8092306399056751,-'endlessparentheses.com': 0.8092053968063221,-'www.vrfinal.com': 0.809205396806322,-'www.the-american-interest.com': 0.8091888493515856,-'cycle-gap.blogspot.com': 0.8091399291892002,-'elinux.org': 0.8091062649443477,-'agtb.wordpress.com': 0.8091062649443477,-'addyosmani.github.com': 0.8090929406134104,-'www.immadsnewworld.com': 0.8090591990898237,-'interplayoflight.wordpress.com': 0.8090551181102362,-'kylecordes.com': 0.8090551181102361,-'www.ndss-symposium.org': 0.8090548860707052,-'www.grantland.com': 0.808953355501278,-'accu.org': 0.8089301368051649,-'americans.org': 0.8089046493675752,-'languagengine.co': 0.8088694832763874,-'www.al-monitor.com': 0.8088667083388086,-'www.marksimonson.com': 0.8088667083388086,-'danilocampos.com': 0.8088667083388086,-'money.yahoo.com': 0.8088648827181377,-'www.mathpages.com': 0.8088432529287259,-'nakamotoinstitute.org': 0.8088145420012367,-'blog.ionelmc.ro': 0.8087651491972426,-'www.strawpoll.me': 0.8087480632308696,-'www.theweek.co.uk': 0.8087282943708363,-'www.morningbrew.com': 0.8086875731228074,-'ksj.mit.edu': 0.8086785009861932,-'ozar.me': 0.808655053694382,-'scheme2006.cs.uchicago.edu': 0.8085933953841323,-'9fans.net': 0.8085933953841322,-'pythonhosted.org': 0.8085610188346061,-'lite.cnn.com': 0.8085535287872542,-'www.fark.com': 0.808501385570164,-'www.techendo.co': 0.808501106941333,-'downlode.org': 0.8084934128064108,-'upstart.me': 0.8084934128064108,-'www.extendslogic.com': 0.8084934128064108,-'theundefeated.com': 0.8084934128064108,-'blog.appsignal.com': 0.8084690154513154,-'www.informationarchitects.jp': 0.8084214332325731,-'www.intrade.com': 0.8083885146980836,-'lukasz.langa.pl': 0.8083885146980836,-'www.energy.gov': 0.8083804906443909,-'talkpython.fm': 0.8083525540804629,-'yourbasic.org': 0.8083074906530393,-'www.sachinagarwal.com': 0.8083003952569169,-'openshift.redhat.com': 0.8082657786610448,-'tech.adroll.com': 0.8082443572105479,-'paleofuture.gizmodo.com': 0.8082365397238809,-'tgceec.tumblr.com': 0.808209532747702,-'bowery.io': 0.8081978171238972,-'blog.sourcing.io': 0.8081838340004474,-'www.9to5mac.com': 0.8081802724018405,-'bostinno.com': 0.8081755973845505,-'jatan.space': 0.8081563211953493,-'www-inst.eecs.berkeley.edu': 0.8081204961840847,-'evrone.com': 0.8080872755977898,-'labs.domipheus.com': 0.8080351711070707,-'www.thrillist.com': 0.8079971777160608,-'www.gizmag.com': 0.8079969849430149,-'landley.net': 0.807958662672192,-'danweinreb.org': 0.8078992257179908,-'tyk.io': 0.807867976487508,-'mymicrocontroller.com': 0.8078630033918272,-'www.samharris.org': 0.8078422754090425,-'epetitions.direct.gov.uk': 0.8078284811604556,-'blog.tildeloop.com': 0.8078284811604556,-'discuss.emberjs.com': 0.8078241273575438,-'blog.superuser.com': 0.8077665343961988,-'oliveremberton.com': 0.807743372835657,-'vpnpro.com': 0.8077402589309914,-'www.lastwordonnothing.com': 0.8076974958304752,-'stevenf.com': 0.8076913832175415,-'www.timeanddate.com': 0.8076143611045982,-'cornellsun.com': 0.8075830492659702,-'letoverlambda.com': 0.8075830492659702,-'snapcraft.io': 0.8075475712589127,-'www.observer.com': 0.8074847637579465,-'unit42.paloaltonetworks.com': 0.8074697991147913,-'www.fastcocreate.com': 0.8074665167071722,-'mixmax.com': 0.8074624950006974,-'blog.goodaudience.com': 0.8074490566695363,-'wikitravel.org': 0.8074371698943806,-'elzr.com': 0.8074371698943806,-'gerger.co': 0.8074101861989343,-'developer.amazonwebservices.com': 0.8073933592944413,-'blog.postmarkapp.com': 0.8073171397946091,-'adereth.github.io': 0.8073171397946091,-'www.gamepro.com': 0.8073017117935154,-'www.sencha.com': 0.8072902381119292,-'christianheilmann.com': 0.8072902381119292,-'mg.co.za': 0.8072653978420726,-'python-history.blogspot.com': 0.8072408370446265,-'www.warhistoryonline.com': 0.807215873070334,-'theatlantic.com': 0.8072127719392547,-'sakurity.com': 0.8071790086703337,-'blog.yalabot.com': 0.8071726013094649,-'softwareengineering.stackexchange.com': 0.8071626259596408,-'vnhacker.blogspot.com': 0.8071372108129627,-'www.securityweek.com': 0.8070728244513765,-'www.myce.com': 0.8070371527879234,-'progressiveapp.store': 0.8070260217041482,-'teamblog.supportbee.com': 0.8070260217041482,-'arogozhnikov.github.io': 0.8070036402591226,-'mechanical-sympathy.blogspot.com': 0.8069161416177959,-'strangelyconsistent.org': 0.8069045196582395,-'www.themarysue.com': 0.8068976247535697,-'www.attendly.com': 0.8068580186201768,-'www.snell-pym.org.uk': 0.8068326941714653,-'tools.cisco.com': 0.8068196782315848,-'influxdb.com': 0.8068173283593989,-'worksinprogress.co': 0.8067896834805538,-'www.scripps.edu': 0.8067572864866587,-'dissenter.firedoglake.com': 0.806652591404526,-'crackstation.net': 0.8066141922329747,-'www.afar.com': 0.8065122291007617,-'hasura.io': 0.8064237491553132,-'www.cs.nyu.edu': 0.8063872255489021,-'www.mtholyoke.edu': 0.8063872255489021,-'blog.startifact.com': 0.8063862184736535,-'www.colourlovers.com': 0.8063834547609555,-'www.gettingmoreawesome.com': 0.8063764986530164,-'www.themorningnews.org': 0.8063013263733552,-'hardcoresoftware.learningbyshipping.com': 0.8062797912662047,-'bramlambrecht.com': 0.8062668907413534,-'www.spacesafetymagazine.com': 0.8062339137045579,-'newscenter.berkeley.edu': 0.8062219609407214,-'suckless.org': 0.8062207668887819,-'www.staticgen.com': 0.8062025100696604,-'www.eecs.umich.edu': 0.8062025100696603,-'www.thefader.com': 0.8061953047624655,-'www.independent.ie': 0.806109251382116,-'pivotallabs.com': 0.8060806747033802,-'www.opsian.com': 0.8060621471224426,-'www.taigeair.com': 0.8060621471224426,-'talkingpointsmemo.com': 0.8060591184167385,-'williampross.com': 0.8060455947863886,-'popmotion.io': 0.8060424719294661,-'robb.weblaws.org': 0.806042471929466,-'globaldev.co.uk': 0.806011328959103,-'pencerw.com': 0.806,-'michelenasti.com': 0.8059842345365967,-'sourcesort.com': 0.8059044933660855,-'www.monash.edu': 0.8059044933660855,-'haydenjames.io': 0.8058768485324959,-'stjepang.github.io': 0.805873630277718,-'blog.webb.page': 0.805873630277718,-'tcfast.com': 0.8058521366516195,-'blog.figma.com': 0.8058473182210699,-'blog.fikesfarm.com': 0.8058387400788318,-'www.telecompaper.com': 0.8058387400788318,-'mikegerwitz.com': 0.8058387400788318,-'micheleincalifornia.blogspot.com': 0.8058373166902851,-'cyber.harvard.edu': 0.8058229699826737,-'content.time.com': 0.8057897338858303,-'catless.ncl.ac.uk': 0.8057745286288139,-'www.techflash.com': 0.8057273188584126,-'omis.me': 0.805697502622664,-'anildash.com': 0.8056599652743452,-'www.authy.com': 0.8056112224448897,-'bitterwinter.org': 0.8056112224448897,-'news.illinois.edu': 0.8055525618659666,-'trilema.com': 0.8055277480254573,-'nshipster.com': 0.8055021399615895,-'uppy.io': 0.8054909385894538,-'probmods.org': 0.805478770001181,-'labs.umbrella.com': 0.805478770001181,-'www.lawyersgunsmoneyblog.com': 0.8054692502161738,-'paulspontifications.blogspot.com': 0.8054688521990506,-'wiki.python.org': 0.8054642788538674,-'metaphysicaldeveloper.wordpress.com': 0.8054364779428177,-'tech.okcupid.com': 0.805426569995148,-'leejo.github.io': 0.8054207149429934,-'scholar.google.com': 0.8053870344669396,-'jseliger.com': 0.8053643358322942,-'www.openrightsgroup.org': 0.8053456370357672,-'zendev.com': 0.8053227485814811,-'deepsense.ai': 0.8053174136210413,-'googlepublicpolicy.blogspot.com': 0.8052966184857965,-'reading.supply': 0.8052571979264512,-'www.fullstackpython.com': 0.8052571979264512,-'making.duolingo.com': 0.8052499518418211,-'cloudrail.com': 0.8052345044507019,-'crt.sh': 0.8052208835341366,-'charity.wtf': 0.8052128017402501,-'gettingreal.37signals.com': 0.8052119053726816,-'products.office.com': 0.805195056791058,-'blog.spiderlabs.com': 0.8051909890788225,-'javascript.christmas': 0.8051778217183054,-'www.ruby-forum.com': 0.8051778217183053,-'webkul.github.io': 0.8051216600975734,-'crypt.codemancers.com': 0.8050979962669772,-'aomedia.org': 0.805084051401519,-'www.physics-astronomy.org': 0.8050038695306542,-'dothemath.ucsd.edu': 0.8049264940426683,-'www.av8n.com': 0.8049169449005759,-'signalsandthreads.com': 0.8049169449005759,-'www.apptorium.com': 0.8048961063441398,-'smarterware.org': 0.8048459862314374,-'www.playstation.com': 0.8048413390083552,-'toys.lerdorf.com': 0.8048289738430583,-'blog.handcraft.com': 0.8047698525990352,-'www.alex-ionescu.com': 0.8047611301824745,-'team.adzerk.com': 0.8047527420132489,-'www.trottercashion.com': 0.8047172511803042,-'www.dylanpaulus.com': 0.8047172511803041,-'shiroyasha.io': 0.8046842108009492,-'sunlightlabs.com': 0.8046689741257063,-'johnbender.us': 0.8046551925330852,-'www.orionmagazine.org': 0.8046409236882213,-'500ish.com': 0.8045980649418749,-'mbostock.github.com': 0.8045837440928983,-'www.eclogiselle.com': 0.8045630694400487,-'j00ru.vexillium.org': 0.8045036919905235,-'www.partow.net': 0.8045036919905235,-'www.zpub.com': 0.8045036919905235,-'www.hq.nasa.gov': 0.8043661879315178,-'news.usni.org': 0.804355836696792,-'charemza.name': 0.8043090754458252,-'gondor.io': 0.8043052663473409,-'gamelab.mit.edu': 0.8042844920724259,-'starcraft.com': 0.8042844920724259,-'sorentwo.com': 0.804261705269134,-'thebrowser.com': 0.8042503564660758,-'ericsink.com': 0.804229811533429,-'blogs.loc.gov': 0.8041942737849643,-'sslmate.com': 0.8041845438238772,-'googlereader.blogspot.com': 0.8041760226307137,-'pingcap.github.io': 0.8041760226307136,-'blog.nearlyfreespeech.net': 0.8041648746111758,-'blog.peer5.com': 0.8041599344447704,-'understandlegacycode.com': 0.8041241866706869,-'www3.imperial.ac.uk': 0.8040897254741548,-'ovid.github.io': 0.8040060267347496,-'perell.com': 0.8039678014067241,-'marketingland.com': 0.8039660516562604,-'maierfelix.github.io': 0.803954084248265,-'www.plainsite.org': 0.803949941652702,-'www.fabcapo.com': 0.8039204604178055,-'www.osohq.com': 0.8039193847705445,-'marc.merlins.org': 0.8038831339196976,-'satyajit.ranjeev.in': 0.8038831339196976,-'kirindave.tumblr.com': 0.8038831339196976,-'transcend.io': 0.8038268126556799,-'pozorvlak.livejournal.com': 0.8038248807746377,-'elweb.co': 0.8038124805380112,-'www.arqbackup.com': 0.8037337901625023,-'www.turnkeylinux.org': 0.803725088601539,-'www.box.com': 0.8037180799167778,-'preciousplastic.com': 0.8036437246963563,-'www.stonetemple.com': 0.8036218680934559,-'oeis.org': 0.8036018584338912,-'www.bath.ac.uk': 0.8036016394623412,-'services.google.com': 0.803590777077633,-'theodi.org': 0.8035882804728938,-'saucelabs.com': 0.8035344497963749,-'greydanus.github.io': 0.8035295033581054,-'www.educative.io': 0.8035052925700027,-'librarianshipwreck.wordpress.com': 0.8034837043428301,-'sway.com': 0.803480126236027,-'datacolada.org': 0.803472812615791,-'convox.com': 0.8033973749313138,-'www.tokbox.com': 0.8033848854367407,-'review.firstround.com': 0.8033799488939632,-'www.netmagazine.com': 0.8033798192718756,-'www.stanfordalumni.org': 0.8033701218466093,-'www.fox5ny.com': 0.8033480309814771,-'phx.corporate-ir.net': 0.8033227542756777,-'christophjanz.blogspot.com': 0.8033072428565003,-'grugq.github.io': 0.8032867126150643,-'gist.githubusercontent.com': 0.8032867126150643,-'www.welivesecurity.com': 0.8032782303750765,-'www.linfo.org': 0.8032479251672556,-'www.vikingcodeschool.com': 0.8032454361054767,-'talsraviv.com': 0.8032454361054766,-'wyss.harvard.edu': 0.8032452848408873,-'brainspl.at': 0.8032452848408873,-'simblob.blogspot.com': 0.8032046395373721,-'sefsar.com': 0.8031660842268623,-'www.anmolsarma.in': 0.803136984148784,-'www.semiaccurate.com': 0.8031240460985013,-'www.mikekohn.net': 0.8031074147500573,-'m.wsj.com': 0.8030996898658181,-'doc-ok.org': 0.8030754588314308,-'duncandavidson.com': 0.8030754588314308,-'www.redbullstratos.com': 0.8030754588314308,-'lodev.org': 0.8030754588314308,-'www.sqreen.io': 0.8030754588314308,-'weburbanist.com': 0.8030719460684711,-'www.homelandsecuritynewswire.com': 0.8030383963381544,-'programming.oreilly.com': 0.8029986620998482,-'www.yuiblog.com': 0.8029870560713339,-'mgalligan.com': 0.802938284099674,-'startupsanonymous.com': 0.802896517417948,-'genomebiology.biomedcentral.com': 0.8028333073906616,-'www.commentary.org': 0.8028283532567575,-'paidcontent.org': 0.8027935252684901,-'www.entrepreneur.com': 0.8027573121471558,-'opensource.google.com': 0.8027457335537694,-'gus-massa.blogspot.com': 0.8027457335537693,-'attractivechaos.wordpress.com': 0.8027428934082615,-'timothyfitz.wordpress.com': 0.8027428934082615,-'news.voyage.auto': 0.8027425585809037,-'blog.safia.rocks': 0.802725147613407,-'pagely.com': 0.802725147613407,-'pedestrianobservations.wordpress.com': 0.8027251476134069,-'www.huckmagazine.com': 0.8027158674008132,-'amplitude.com': 0.8027111401704623,-'academicvc.com': 0.8026691214318213,-'zurb.com': 0.802669002882095,-'www.fogcreek.com': 0.8026427258093481,-'pubs.aeaweb.org': 0.8025918806108637,-'ochronus.online': 0.8025797396414792,-'mukulrathi.co.uk': 0.8025366605137177,-'www.tinfoilsecurity.com': 0.8025110355010865,-'blog.intercom.io': 0.802438081407932,-'www.counterpointresearch.com': 0.802349177742619,-'caines.ca': 0.8023472216796856,-'bitmessage.org': 0.8023472216796855,-'fare.livejournal.com': 0.802344964463485,-'sameroom.io': 0.8023231254672252,-'grapheneos.org': 0.8022611036781359,-'ryanhoover.me': 0.8022215636421977,-'ghost.org': 0.8021756677926977,-'ably.com': 0.8021756677926974,-'dgraph.io': 0.8021756677926974,-'www.pcg-random.org': 0.8021607965447277,-'www.electronicdesign.com': 0.802117143232502,-'blog.benjamin-encz.de': 0.8021070483963555,-'blog.apollographql.com': 0.8021070483963553,-'interaktiv.morgenpost.de': 0.8021068976672623,-'www.neogaf.com': 0.8020794141131092,-'jasongullickson.posterous.com': 0.802070055097611,-'www.lvguowei.me': 0.8020465972489538,-'theapeiron.co.uk': 0.8020408163265306,-'insideairbnb.com': 0.8020408163265306,-'www.business-standard.com': 0.8020316557325065,-'blog.chiariglione.org': 0.8020183701272139,-'news.byu.edu': 0.8019177911188085,-'ico.org.uk': 0.8019103627766435,-'www.asktog.com': 0.8018750459311643,-'www.thelightphone.com': 0.8018513951254583,-'weblog.muledesign.com': 0.8018281461815683,-'functional.christmas': 0.8017833933313524,-'dendory.net': 0.8017566800074669,-'css-irl.info': 0.8016782621735088,-'m.spiegel.de': 0.8016676297112596,-'blog.rubybestpractices.com': 0.8016652051765923,-'buzzsumo.com': 0.8016526731161184,-'edinburghhacklab.com': 0.8016359918200409,-'www.uc.edu': 0.8016359918200409,-'deprogrammaticaipsum.com': 0.8016284628618231,-'blog.teamtreehouse.com': 0.8016183213966954,-'www.poynter.org': 0.8015928533864275,-'blog.thirdyearmba.com': 0.8015659486074044,-'cmr.github.io': 0.8015566134648262,-'www.hackingwithswift.com': 0.8015552594719607,-'www.43folders.com': 0.801514257843383,-'www.whispersystems.org': 0.801509946546816,-'www.bravenewgeek.com': 0.8014600688018096,-'blog.siliconpublishing.com': 0.8014553154641547,-'psiloveyou.xyz': 0.8014114620819677,-'social.msdn.microsoft.com': 0.8014114620819677,-'thenewinquiry.com': 0.8014082764643022,-'www.grc.com': 0.8014045918631542,-'yle.fi': 0.8013958725561011,-'www.buildzoom.com': 0.8013406095826441,-'www.kmjn.org': 0.8013048425718535,-'www.miaminewtimes.com': 0.8012606895008024,-'pipedream.com': 0.8012606895008023,-'xoofx.com': 0.8012606895008023,-'9elements.com': 0.8012357442611142,-'www.safetyresearch.net': 0.8012295081967213,-'memcached.org': 0.8012295081967213,-'letterstoanewdeveloper.com': 0.801226878568775,-'viewfromthewing.boardingarea.com': 0.8012126203272316,-'nanxiao.me': 0.8011961282535711,-'ferrucc.io': 0.8011506248520655,-'blog.mrmeyer.com': 0.8011434789943767,-'www.dailycal.org': 0.8011077312639568,-'tldp.org': 0.8011077312639568,-'www.nbcphiladelphia.com': 0.8010985913944949,-'ww2.kqed.org': 0.8010518269035773,-'nitter.net': 0.8010512756640218,-'www.mathieupassenaud.fr': 0.8010059231728947,-'uploadvr.com': 0.8009858227111218,-'lincolnloop.com': 0.800978810280933,-'gaming.icrontic.com': 0.8009712545655284,-'www.techvibes.com': 0.8009550432546098,-'www.news.gatech.edu': 0.8009318982607027,-'leastauthority.com': 0.8008308575748613,-'find.xyz': 0.8008120556386431,-'rhodesmill.org': 0.8007877241325981,-'authzed.com': 0.8007485330706217,-'www.agweb.com': 0.8007485330706217,-'allegro.tech': 0.8007425067937589,-'nex-3.com': 0.8007357953179356,-'macaw.co': 0.8007357953179356,-'nic.ferrier.me.uk': 0.8007109685993726,-'www.planetscale.com': 0.8007054295824444,-'arshadchowdhury.com': 0.8006990158865002,-'www.adbusters.org': 0.8006901908979573,-'techno-anthropology.blogspot.com': 0.8006290310742641,-'noteflakes.com': 0.8006120189176902,-'blog.ellenchisa.com': 0.8006120189176901,-'ajaxian.com': 0.8005654734164377,-'getchip.com': 0.8004935213803819,-'www.dutchnews.nl': 0.8004918240352497,-'www.efavdb.com': 0.8004115226337449,-'mabez.dev': 0.8004115226337449,-'www.azul.com': 0.80040857295631,-'www.bonappetit.com': 0.8003933326639253,-'microconf.com': 0.8003448118609695,-'mxb.at': 0.8003448118609695,-'rancher.com': 0.8003102115898207,-'antoniocangiano.com': 0.8002939423538775,-'www.modernretail.co': 0.8002715044965122,-'gigamonkeys.wordpress.com': 0.8001954409713545,-'blog.veitheller.de': 0.8001954409713545,-'www.thehistoryblog.com': 0.8001885843714048,-'www.javacodegeeks.com': 0.80016657689339,-'blog.stevensanderson.com': 0.8001363248010883,-'neilmitchell.blogspot.com': 0.8001226738254366,-'www.foundingfuel.com': 0.800121664181143,-'simpleprogrammer.com': 0.8001130650615119,-'eclipse.org': 0.8000856208636091,-'www.acs.org': 0.8000105725594235,-'sigusr2.net': 0.8000105725594235}-+DOMAINS = {+ "blog.samaltman.com": 0.9906157038365982,+ "paulgraham.com": 0.9900792629020208,+ "blog.rust-lang.org": 0.9890328852259848,+ "blog.ycombinator.com": 0.9883892917770221,+ "ciechanow.ski": 0.9880439204029129,+ "blog.mozilla.org": 0.9862851554386893,+ "stripe.com": 0.9861322763760931,+ "danluu.com": 0.9861047192293313,+ "www.kalzumeus.com": 0.9850478020093707,+ "signal.org": 0.9850307268861681,+ "antirez.com": 0.9848258724546234,+ "www.mozilla.org": 0.9848038469599045,+ "jacquesmattheij.com": 0.9844261150401603,+ "www.chris-granger.com": 0.9842666557846631,+ "www.apple.com": 0.9836460584625046,+ "letsencrypt.org": 0.9835295202763794,+ "www.teslamotors.com": 0.9831965330638592,+ "hacks.mozilla.org": 0.982862363147374,+ "lore.kernel.org": 0.9822733167137851,+ "frame.work": 0.9821010345456243,+ "www.righto.com": 0.9819349251846876,+ "www.troyhunt.com": 0.9814966052145125,+ "www.sec.gov": 0.9814725718254595,+ "www.tesla.com": 0.9810577172203161,+ "keybase.io": 0.9808967126687541,+ "www.blender.org": 0.9807600197929631,+ "www.spacex.com": 0.9807226511637516,+ "www.susanjfowler.com": 0.9805638280990112,+ "www.eff.org": 0.9805327186535832,+ "blog.golang.org": 0.9800638229086632,+ "github.com": 0.9793809676855824,+ "www.bloomberg.com": 0.9791747058863844,+ "matrix.org": 0.9788189361330681,+ "openai.com": 0.9781666403563757,+ "www.nytimes.com": 0.9780992933501785,+ "www.sublimetext.com": 0.9779900989360693,+ "mtlynch.io": 0.9777886722980439,+ "www.reuters.com": 0.9776899139875952,+ "blog.pinboard.in": 0.9776638411070188,+ "whispersystems.org": 0.9776203868069737,+ "fabiensanglard.net": 0.9774982104911971,+ "drewdevault.com": 0.9774354534940557,+ "blog.interviewing.io": 0.9772072688889477,+ "github.blog": 0.9770459926321585,+ "calpaterson.com": 0.9768950731213937,+ "rachelbythebay.com": 0.9768404166128064,+ "bellard.org": 0.9767955350707707,+ "twitter.com": 0.976728011118412,+ "training.kalzumeus.com": 0.9766770275278582,+ "www.npr.org": 0.976513349382135,+ "jvns.ca": 0.9765007166886884,+ "www.daemonology.net": 0.9763913310944178,+ "ifixit.org": 0.9762565420103206,+ "www.bbc.com": 0.9761524214719092,+ "karpathy.github.io": 0.9759182558450648,+ "tonsky.me": 0.9759061232371565,+ "www.buzzfeednews.com": 0.975834047407925,+ "deepmind.com": 0.9757513978763775,+ "www.postgresql.org": 0.9756297562580343,+ "krebsonsecurity.com": 0.9755023613643372,+ "overreacted.io": 0.9754149079228597,+ "stratechery.com": 0.9754126620517041,+ "brave.com": 0.9752551626581946,+ "lwn.net": 0.9751973038824019,+ "about.gitlab.com": 0.9751871932295729,+ "www.stephendiehl.com": 0.9751552345204195,+ "techcrunch.com": 0.9751406961249492,+ "raganwald.posterous.com": 0.9750289679638235,+ "blog.discordapp.com": 0.9750045998850335,+ "bugs.chromium.org": 0.9749443016112311,+ "www.marco.org": 0.9749044929470128,+ "words.steveklabnik.com": 0.9748959016378708,+ "www.backblaze.com": 0.9744710342430156,+ "robert.ocallahan.org": 0.9744576196125118,+ "blog.cryptographyengineering.com": 0.9744312947315787,+ "citizenlab.ca": 0.9742406063151361,+ "www.propublica.org": 0.9742125433042081,+ "blog.benjojo.co.uk": 0.9741593610657648,+ "justine.lol": 0.9740984398105378,+ "lists.zx2c4.com": 0.9739788216838905,+ "googleblog.blogspot.com": 0.9739234991924623,+ "www.supremecourt.gov": 0.9738392227472342,+ "idlewords.com": 0.9737410870758311,+ "randomascii.wordpress.com": 0.97368784410819,+ "jepsen.io": 0.9736187109986643,+ "zachholman.com": 0.9735522719786461,+ "githubengineering.com": 0.9734850428577687,+ "tinyprojects.dev": 0.9733692809444601,+ "www.quantamagazine.org": 0.9731098630624817,+ "aws.amazon.com": 0.9730261011769831,+ "www.washingtonpost.com": 0.9727186917658874,+ "www.tbray.org": 0.9727073933211833,+ "sqlite.org": 0.9725225769916973,+ "sivers.org": 0.9725186799112472,+ "inkscape.org": 0.9725022016010344,+ "www.deepsouthventures.com": 0.972453385720152,+ "blog.openai.com": 0.9723220955611721,+ "courses.csail.mit.edu": 0.9722700145319921,+ "www.theguardian.com": 0.9722441587612369,+ "dcurt.is": 0.9721843131015032,+ "blog.atom.io": 0.9720527700753158,+ "www.newyorker.com": 0.9720054386788728,+ "lists.gnu.org": 0.9719999849041815,+ "www.ccc.de": 0.9719484596362211,+ "googleprojectzero.blogspot.com": 0.9719076672640107,+ "blog.cloudflare.com": 0.9718848285343656,+ "jeffhuang.com": 0.9718720207664465,+ "duckduckgo.com": 0.9718309347264379,+ "www.sqlite.org": 0.9717513581610722,+ "marc.info": 0.9716364945361756,+ "www.anandtech.com": 0.9715269935352944,+ "setosa.io": 0.9714750377943311,+ "gogameguru.com": 0.9714728059591474,+ "foundation.mozilla.org": 0.9714509129021823,+ "patrickcollison.com": 0.9713258071934422,+ "daringfireball.net": 0.9711795273015573,+ "www.codinghorror.com": 0.971168931371747,+ "raphlinus.github.io": 0.9711321390363085,+ "blog.chriszacharias.com": 0.9711291613244216,+ "facebook.github.io": 0.9711262368674779,+ "code.facebook.com": 0.9711146723366064,+ "blog.priceonomics.com": 0.9710225579333793,+ "petapixel.com": 0.9709592479735079,+ "aphyr.com": 0.9709163111780117,+ "blog.jupyter.org": 0.970807531784939,+ "www.vice.com": 0.9707422391855615,+ "josephg.com": 0.9707173697331065,+ "groups.google.com": 0.9707047476201209,+ "www.candyjapan.com": 0.9706864280847017,+ "www.bbc.co.uk": 0.9706752756956991,+ "www.raspberrypi.org": 0.9706652392429661,+ "paulbuchheit.blogspot.com": 0.9706297170684384,+ "blog.triplebyte.com": 0.9706184118205334,+ "web.eecs.utk.edu": 0.9705843602166302,+ "www.foundationdb.org": 0.970531828365254,+ "factorio.com": 0.9705079220882863,+ "brew.sh": 0.9702877485101176,+ "arstechnica.com": 0.9702828920003089,+ "jlongster.com": 0.970220590502143,+ "blog.dustinkirkland.com": 0.9702178011253874,+ "daniel.haxx.se": 0.9701944690304003,+ "rethinkdb.com": 0.9701899521411085,+ "blog.linode.com": 0.9700603805951686,+ "andrewkelley.me": 0.9699936860995682,+ "theintercept.com": 0.9698660114939796,+ "morepypy.blogspot.com": 0.9698368234837993,+ "lkml.org": 0.9698185224636935,+ "www.cbc.ca": 0.9697944363631238,+ "www.justinobeirne.com": 0.9697393718499845,+ "www.pine64.org": 0.9696643433350861,+ "www.thirtythreeforty.net": 0.9696053743544627,+ "www.aclu.org": 0.9696051245989374,+ "robertheaton.com": 0.9695217351343498,+ "www.justice.gov": 0.96946427848646,+ "startupclass.samaltman.com": 0.9694186083625935,+ "www.atlasobscura.com": 0.9693830118062187,+ "blog.archive.org": 0.9692692877978696,+ "wozniak.ca": 0.9692591742384942,+ "shkspr.mobi": 0.9690681057241932,+ "wikileaks.org": 0.9688043568238527,+ "lucumr.pocoo.org": 0.9687437893479095,+ "www.defmacro.org": 0.9685745716212145,+ "cloudplatform.googleblog.com": 0.9685595849759793,+ "www.wsj.com": 0.9685329922823139,+ "www.gatesnotes.com": 0.9685175797648193,+ "lapcatsoftware.com": 0.9684021881186297,+ "www.bunniestudios.com": 0.9683871579382967,+ "puri.sm": 0.9681483253888264,+ "madned.substack.com": 0.9680061059600782,+ "www.openculture.com": 0.9679136786699476,+ "google.github.io": 0.9679127919179034,+ "spreadprivacy.com": 0.9678904674640757,+ "37signals.com": 0.9677339661772352,+ "thedaywefightback.org": 0.96772024509243,+ "asia.nikkei.com": 0.9676721486940052,+ "blog.pragmaticengineer.com": 0.9676540840884938,+ "brandur.org": 0.9675999497340669,+ "worrydream.com": 0.9675574587791994,+ "www.gimp.org": 0.9675475521140265,+ "www.gabrielweinberg.com": 0.9675439099356123,+ "tyler.io": 0.9675272832002738,+ "www.theatlantic.com": 0.9674601161371825,+ "deno.land": 0.9674479589983649,+ "www.economist.com": 0.9674323307648195,+ "sheddingbikes.com": 0.9672694753671274,+ "pud.com": 0.9671820049226395,+ "lavabit.com": 0.9671393498011488,+ "blog.bradfieldcs.com": 0.9671289002274366,+ "www.smithsonianmag.com": 0.9670852442662822,+ "elm-lang.org": 0.9670684273054104,+ "themacro.com": 0.9670447125421023,+ "security.googleblog.com": 0.9669238650508819,+ "www.voxelquest.com": 0.9669230619143929,+ "www.gnu.org": 0.966899856247157,+ "mail.mozilla.org": 0.9668731777544588,+ "www.jeffgeerling.com": 0.9668579896794777,+ "stallman.org": 0.9668266183513317,+ "edwardsnowden.substack.com": 0.9666584767798317,+ "www.latimes.com": 0.9666263838504232,+ "blog.torproject.org": 0.9666181158060496,+ "bugzilla.mozilla.org": 0.966563230800294,+ "developer.apple.com": 0.9665153046671373,+ "www.nature.com": 0.9665034870295038,+ "blog.docker.com": 0.9664589097197918,+ "www.cockroachlabs.com": 0.9664546158920483,+ "earthly.dev": 0.9664394437158831,+ "nullprogram.com": 0.9664053052413792,+ "code.visualstudio.com": 0.9664051613640476,+ "teachyourselfcs.com": 0.96638716524213,+ "immersivemath.com": 0.9663756953881175,+ "fly.io": 0.9663690278982495,+ "blog.system76.com": 0.966297694451117,+ "torrentfreak.com": 0.9662949409422105,+ "swift.org": 0.9662794829073483,+ "plausible.io": 0.9661981619745931,+ "blog.timescale.com": 0.9660867756775726,+ "elixir-lang.org": 0.9660464292856342,+ "einaregilsson.com": 0.9660162058065089,+ "blog.servo.org": 0.9658921829300279,+ "tsaoutofourpants.wordpress.com": 0.9658298664203504,+ "www.brendangregg.com": 0.9657781891262812,+ "blog.jessfraz.com": 0.965647862426781,+ "www.gwern.net": 0.9656005597335781,+ "techblog.netflix.com": 0.9654696251760763,+ "www.videolan.org": 0.9654507268439513,+ "blogs.windows.com": 0.9653807535564003,+ "arp242.net": 0.965353872976349,+ "mjg59.dreamwidth.org": 0.9653138636026742,+ "www.stavros.io": 0.9651634551351548,+ "www.expeditedssl.com": 0.9651401658177345,+ "matt.might.net": 0.9651059400349459,+ "flif.info": 0.9650927452914915,+ "apenwarr.ca": 0.9650906673961581,+ "googleonlinesecurity.blogspot.com": 0.9650850292429668,+ "priceonomics.com": 0.9650537614148901,+ "www.humbledmba.com": 0.9649972831093808,+ "www.unrealengine.com": 0.964977222983089,+ "people.xiph.org": 0.9648830246858617,+ "betterexplained.com": 0.9648763473852692,+ "al3x.net": 0.9647293502855223,+ "notch.tumblr.com": 0.9646905626593433,+ "deno.com": 0.9646806401289668,+ "www.wired.com": 0.9646357841756316,+ "git.kernel.org": 0.9645834348149406,+ "www.djangoproject.com": 0.9645683143391122,+ "www.ruby-lang.org": 0.964565023525263,+ "www.catonmat.net": 0.9645567860107777,+ "blog.plover.com": 0.9645429951505131,+ "ocw.mit.edu": 0.9644857262463964,+ "steveblank.com": 0.9644558764763906,+ "blog.haschek.at": 0.9643961053958355,+ "old.reddit.com": 0.9643838762037145,+ "blog.jgc.org": 0.9643196349091965,+ "utcc.utoronto.ca": 0.9643144411744943,+ "blog.getbootstrap.com": 0.964223508260427,+ "protonmail.com": 0.9641434352105049,+ "stackoverflow.blog": 0.964131467903275,+ "www.feynmanlectures.caltech.edu": 0.9640844147082678,+ "www.nationalgeographic.com": 0.9640002724660541,+ "thume.ca": 0.9639172934049174,+ "www.ftc.gov": 0.9638662117372075,+ "blog.google": 0.9638278246372475,+ "talks.golang.org": 0.9638081833676005,+ "identity.mozilla.com": 0.9637913542538376,+ "www.antipope.org": 0.9637681454362576,+ "blog.coinbase.com": 0.9637227588096539,+ "freedom.press": 0.9637055279129609,+ "research.googleblog.com": 0.9636957954323193,+ "sneak.berlin": 0.9636104943866459,+ "ferd.ca": 0.9636081542021665,+ "system76.com": 0.9636004595237442,+ "jsomers.net": 0.9635822620240366,+ "nautil.us": 0.9635511482835416,+ "blog.fogus.me": 0.963547608254157,+ "probablydance.com": 0.9635177117293328,+ "blog.stephenwolfram.com": 0.9633882383623975,+ "blog.42floors.com": 0.9633188247961246,+ "www.thoughtcrime.org": 0.9631942686290309,+ "www.winehq.org": 0.9631344924353825,+ "www.benkuhn.net": 0.9631304906372505,+ "status.cloud.google.com": 0.9630969501387503,+ "dev.stephendiehl.com": 0.9630485789318193,+ "www.thedrive.com": 0.9629761123330604,+ "explainshell.com": 0.9629455061466635,+ "krita.org": 0.9629305982536617,+ "www.paulgraham.com": 0.96284858742987,+ "www.google.com": 0.9626443067571239,+ "blogs.msdn.microsoft.com": 0.9626200437043355,+ "www.techdirt.com": 0.9625942195899927,+ "homakov.blogspot.com": 0.9625907627104796,+ "gist.github.com": 0.9625295427025308,+ "lists.freebsd.org": 0.9624953857245481,+ "www.axios.com": 0.9624629027024122,+ "triplebyte.com": 0.9623309816301445,+ "blog.filippo.io": 0.9621095196217451,+ "mattstoller.substack.com": 0.9620940895652949,+ "weblog.rubyonrails.org": 0.9620546173159494,+ "viewsourcecode.org": 0.9620510951208904,+ "gcc.gnu.org": 0.9619503003058159,+ "tjcx.me": 0.9619235394931922,+ "www.scottaaronson.com": 0.9618863749512754,+ "www.mattkeeter.com": 0.9618540812590697,+ "cloud.google.com": 0.9618480240586683,+ "edweissman.com": 0.9616816249147265,+ "www.citylab.com": 0.961667739783419,+ "ziglang.org": 0.9616247712590222,+ "ffmpeg.org": 0.9615920017638592,+ "www.tablix.org": 0.9615841584158416,+ "opensource.googleblog.com": 0.9615491317254445,+ "www.lettersofnote.com": 0.9615449446825298,+ "blog.pushbullet.com": 0.9614414831416516,+ "www.jacquesmattheij.com": 0.9613892326733957,+ "zwischenzugs.com": 0.9613828940987429,+ "www.reddit.com": 0.9613787026817021,+ "blogs.valvesoftware.com": 0.9613624503908588,+ "markosaric.com": 0.9613621062764696,+ "os.phil-opp.com": 0.9613360087194533,+ "blogs.dropbox.com": 0.961291183608417,+ "docs.google.com": 0.9612568870731552,+ "juliareda.eu": 0.9612194124785559,+ "uspirg.org": 0.9611978470349601,+ "fasterthanli.me": 0.961115384552988,+ "ipfs.io": 0.9610605476989421,+ "queue.acm.org": 0.9610281321922659,+ "www.evanmiller.org": 0.9609644541353691,+ "ejohn.org": 0.9609035373663662,+ "releases.ubuntu.com": 0.9608845798797817,+ "support.google.com": 0.9608695426684296,+ "lcamtuf.coredump.cx": 0.9608422980804076,+ "mango.pdf.zone": 0.9607785943599466,+ "godotengine.org": 0.9607731456425479,+ "justgetflux.com": 0.960753420730463,+ "bradfrost.com": 0.9607430819229773,+ "nim-lang.org": 0.9607343905204542,+ "simonwillison.net": 0.9606872824809413,+ "www.blog.google": 0.96065581868309,+ "www.sciencemag.org": 0.9605214304889523,+ "byorgey.wordpress.com": 0.9604978104194218,+ "www.groklaw.net": 0.9604516891162848,+ "geon.github.io": 0.9603711026188184,+ "lithub.com": 0.9603590822727315,+ "fiber.google.com": 0.9602031412872665,+ "dave.cheney.net": 0.9601949227730773,+ "www.scientificamerican.com": 0.9601944955422137,+ "beepb00p.xyz": 0.9601931949641657,+ "blog.fastmail.com": 0.9601797872297155,+ "seclists.org": 0.9601735085911461,+ "www.hillelwayne.com": 0.9600285295350695,+ "blogs.scientificamerican.com": 0.9599933529455126,+ "www.filfre.net": 0.9599628009632043,+ "www.oculus.com": 0.959958903358201,+ "graydon2.dreamwidth.org": 0.9599484289082291,+ "www.jonathanturner.org": 0.9599080310891523,+ "jameshfisher.com": 0.9598899186695382,+ "julialang.org": 0.9598652009213735,+ "blogs.harvard.edu": 0.9598532297709538,+ "academictorrents.com": 0.9598305702264962,+ "staltz.com": 0.9598086566619791,+ "www.openbsd.org": 0.9597593869835838,+ "psyche.co": 0.9597358328264424,+ "blog.rongarret.info": 0.9597212936777375,+ "www.redblobgames.com": 0.9597207704543808,+ "dtrace.org": 0.9596738647309077,+ "boingboing.net": 0.9596494107644848,+ "www.schneier.com": 0.9595794118397106,+ "support.mozilla.org": 0.9595011501832328,+ "arxiv.org": 0.959473232072393,+ "twobithistory.org": 0.9594688834229274,+ "tttthis.com": 0.9594411193380538,+ "www.windows93.net": 0.9594238637098613,+ "med.stanford.edu": 0.9594108760218417,+ "ascii.textfiles.com": 0.959256899162529,+ "spectrum.ieee.org": 0.9591657750656283,+ "www.aaronsw.com": 0.959152603211572,+ "blog.chromium.org": 0.9590740081078435,+ "daltoncaldwell.com": 0.9588971457836526,+ "jeffe.cs.illinois.edu": 0.9588078449509829,+ "hookrace.net": 0.9587652481709775,+ "lessig.tumblr.com": 0.9587508585934718,+ "blog.garrytan.com": 0.958672955025337,+ "leahneukirchen.org": 0.9586676171256273,+ "journal.stuffwithstuff.com": 0.9586488857012637,+ "status.slack.com": 0.9586248004018981,+ "www.hanselman.com": 0.958563907251422,+ "www.theverge.com": 0.9585530026858043,+ "prog21.dadgum.com": 0.9584999221959226,+ "www.craigmurray.org.uk": 0.9584843626546714,+ "motherboard.vice.com": 0.9584647236016686,+ "awesomekling.github.io": 0.9584478246123319,+ "acko.net": 0.9584223182205939,+ "www.cnbc.com": 0.9583609005406114,+ "misc-stuff.terraaeon.com": 0.9582853790359502,+ "blog.robertelder.org": 0.9581077255960667,+ "www.patrick-breyer.de": 0.958008658008658,+ "mainichi.jp": 0.9579948206173503,+ "notepad-plus-plus.org": 0.9579831679547762,+ "dolphin-emu.org": 0.957980480121585,+ "www.adriancourreges.com": 0.9577752697052809,+ "www.anishathalye.com": 0.9577244298214679,+ "martin.kleppmann.com": 0.9576931939838363,+ "developers.google.com": 0.9576890556647433,+ "code.google.com": 0.9576138371902363,+ "support.apple.com": 0.9575695045037443,+ "buy.louisck.net": 0.9575368691358187,+ "sandstorm.io": 0.9575210697630335,+ "blog.fogcreek.com": 0.9575063092775307,+ "panic.com": 0.9575015340393305,+ "svelte.dev": 0.957467967080876,+ "justinkan.com": 0.9574047166110535,+ "www.ianbicking.org": 0.9573945069975465,+ "firstlook.org": 0.9573366845702125,+ "foundersatwork.posthaven.com": 0.9572959138256529,+ "mail.python.org": 0.9572072777215898,+ "acoup.blog": 0.9571594417654118,+ "jacobian.org": 0.9571381159110411,+ "hakibenita.com": 0.9568994480645705,+ "mothership.sg": 0.9568299369913541,+ "www.vitavonni.de": 0.9568005963591861,+ "palant.de": 0.9567983422438127,+ "benhoyt.com": 0.9567603051598327,+ "david.li": 0.9567049710339931,+ "web.stanford.edu": 0.9566872891659536,+ "a16z.com": 0.9565808474917423,+ "www.battleforthenet.com": 0.9565732116764964,+ "uncrunched.com": 0.9565198150211067,+ "www.debian.org": 0.9564674281841146,+ "nullsweep.com": 0.9564159156215992,+ "wiki.postgresql.org": 0.9563944999840933,+ "www.snellman.net": 0.9563712102508779,+ "www.craigkerstiens.com": 0.9563664501753555,+ "wikimediafoundation.org": 0.9563345661162456,+ "eater.net": 0.9563202100645842,+ "microsoft.github.io": 0.9563042502322844,+ "redditblog.com": 0.9562631224182452,+ "www.aaronkharris.com": 0.9561555239477143,+ "blogs.microsoft.com": 0.9560820272756707,+ "qz.com": 0.955949644460612,+ "tailscale.com": 0.9559251595739959,+ "codewords.recurse.com": 0.9558536488585196,+ "bost.ocks.org": 0.9557437711557101,+ "gmailblog.blogspot.com": 0.9557079840056077,+ "norvig.com": 0.9556798812425891,+ "pluralistic.net": 0.9556745944240528,+ "www.simplethread.com": 0.9556668873107272,+ "nltimes.nl": 0.9555494259023241,+ "newsroom.intel.com": 0.9554793469031995,+ "news.mit.edu": 0.9554200420895869,+ "jitsi.org": 0.9554166636194328,+ "www.cs.cmu.edu": 0.955323422003546,+ "article.gmane.org": 0.9553116742480078,+ "joearms.github.io": 0.9552623762850195,+ "mzucker.github.io": 0.9552342582549446,+ "qntm.org": 0.9552256231956235,+ "iterm2.com": 0.955216811415582,+ "blog.github.com": 0.9552099253702252,+ "engineering.shopify.com": 0.9551782438728066,+ "geoff.greer.fm": 0.9551520599196336,+ "reactjs.org": 0.9551193524754155,+ "www.citusdata.com": 0.9551065831945164,+ "blog.dropbox.com": 0.9550988360514683,+ "pastebin.com": 0.9550820324377278,+ "www.fast.ai": 0.9550621528031076,+ "blog.twitter.com": 0.9550007700333996,+ "marco.org": 0.9549900576536702,+ "begriffs.com": 0.9549733491222114,+ "lkml.iu.edu": 0.9549524012989699,+ "eclecticlight.co": 0.9549360423325072,+ "v8project.blogspot.com": 0.9549013081514219,+ "sam.zeloof.xyz": 0.9548519610162581,+ "www.facebook.com": 0.954738504671582,+ "cfenollosa.com": 0.954730634374589,+ "stevelosh.com": 0.9547082859524895,+ "mosh.mit.edu": 0.9546865955088134,+ "archive.org": 0.954683235332995,+ "marctenbosch.com": 0.9546562578949468,+ "www.dw.com": 0.954642614537768,+ "thehustle.co": 0.9546290284301826,+ "phabricator.wikimedia.org": 0.9546112568993993,+ "twitter.github.com": 0.9545898933522629,+ "brendaneich.com": 0.9545746174697838,+ "42floors.com": 0.9545517561468556,+ "blog.alexmaccaw.com": 0.9545454626799591,+ "martinfowler.com": 0.9545379389229811,+ "freedom-to-tinker.com": 0.9544719860088577,+ "www.slate.com": 0.9544534258667388,+ "www.iphoneincanada.ca": 0.9544449857023617,+ "bugs.debian.org": 0.9543851860127158,+ "ir.tesla.com": 0.9543767275771423,+ "blog.asmartbear.com": 0.9543610682085355,+ "greenwald.substack.com": 0.9543135677806216,+ "gameprogrammingpatterns.com": 0.9543089003699947,+ "blogs.msdn.com": 0.9543003666109717,+ "www.meteor.com": 0.9542891479234723,+ "www.nasa.gov": 0.9542472672213591,+ "petitions.whitehouse.gov": 0.9542172936354238,+ "segment.com": 0.9542063188154968,+ "lexi-lambda.github.io": 0.9541906249333629,+ "www.goodnewsnetwork.org": 0.9541839952530666,+ "www.crypto101.io": 0.954146742893107,+ "hintjens.com": 0.9541384067709378,+ "changelog.complete.org": 0.9541382845549411,+ "tech.mit.edu": 0.9541192032072701,+ "webkit.org": 0.9540750203597949,+ "blog.emacsen.net": 0.9540459808061132,+ "instagram-engineering.tumblr.com": 0.9540392857685359,+ "boringtechnology.club": 0.9540365522550216,+ "swombat.com": 0.9540149579111236,+ "berthub.eu": 0.9539835716288688,+ "erikbern.com": 0.9539560598971636,+ "www.theregister.com": 0.9538737853031798,+ "blog.reddit.com": 0.9538429471349307,+ "learnpythonthehardway.org": 0.9537983780986163,+ "dynomight.net": 0.953604369679408,+ "danshipper.com": 0.9535908420696627,+ "status.aws.amazon.com": 0.9535221631672808,+ "www.kennethreitz.org": 0.9535129582723397,+ "www.jefftk.com": 0.9534058135420997,+ "idiallo.com": 0.9533596892670664,+ "developer.mozilla.org": 0.9532377447812391,+ "discourse.mozilla.org": 0.9531776168886109,+ "www.vox.com": 0.953171184497987,+ "blog.burntsushi.net": 0.9531628061816667,+ "www.roguelazer.com": 0.9531341112752366,+ "beej.us": 0.9531338211286129,+ "www.destroyallsoftware.com": 0.9530554229381429,+ "eev.ee": 0.9530405280939158,+ "konklone.com": 0.9530176239519549,+ "www.twilio.com": 0.9530110571482041,+ "www.outsideonline.com": 0.9530080955571504,+ "quuxplusone.github.io": 0.9528999786913215,+ "www.chrisstucchio.com": 0.9528639692666151,+ "themarkup.org": 0.9528465949993555,+ "www.nobelprize.org": 0.9528281612185788,+ "samcurry.net": 0.9528119859873818,+ "phoboslab.org": 0.9527996714263891,+ "media.ccc.de": 0.9527965238893377,+ "tom.preston-werner.com": 0.9527727795158389,+ "iism.org": 0.9527632645423888,+ "blog.joinmastodon.org": 0.9527602941375386,+ "www.accc.gov.au": 0.9527263269590833,+ "www.ncbi.nlm.nih.gov": 0.9527198508675186,+ "ianstormtaylor.com": 0.952712496504424,+ "www.imperialviolet.org": 0.9526934873020364,+ "ridiculousfish.com": 0.9526563705263446,+ "developers.googleblog.com": 0.9526246790332386,+ "backreaction.blogspot.com": 0.9525612446013791,+ "www.theparisreview.org": 0.9525261097376244,+ "blog.jquery.com": 0.9524629435660441,+ "www.michaeleisen.org": 0.9524499448559609,+ "news.berkeley.edu": 0.9524491052294287,+ "travisdowns.github.io": 0.9523196619971284,+ "blog.nuclino.com": 0.9523119161198071,+ "news.stanford.edu": 0.9522754316114468,+ "www.universityofcalifornia.edu": 0.9522373906378331,+ "gravitational.com": 0.9522355620834344,+ "zserge.com": 0.9521909899856118,+ "observablehq.com": 0.9521499301321013,+ "mathoverflow.net": 0.9521475886704486,+ "www.joyent.com": 0.9520628241406525,+ "blog.kevmod.com": 0.9520548060547861,+ "blog.heroku.com": 0.9520355160215065,+ "www.strongtowns.org": 0.952027603849832,+ "nextjs.org": 0.9520143111821525,+ "www.icij.org": 0.9519656123992113,+ "www.ifixit.com": 0.9519637451973145,+ "www.windytan.com": 0.9518643147043334,+ "www.servethehome.com": 0.9518529509789444,+ "grumpy.website": 0.9517679506064377,+ "bitwarden.com": 0.9517316136273904,+ "gitlab.com": 0.9515782210258252,+ "distill.pub": 0.9515137257464145,+ "craftinginterpreters.com": 0.9514779102710061,+ "lord.io": 0.9514586580765187,+ "www.eso.org": 0.9514502937076248,+ "www.codersnotes.com": 0.9514270198661366,+ "thereader.mitpress.mit.edu": 0.9513838603840151,+ "blog.regehr.org": 0.9513649990290555,+ "googleresearch.blogspot.com": 0.9513477926783215,+ "highscalability.com": 0.9513429766551695,+ "toddwschneider.com": 0.9513421190975475,+ "slatestarcodex.com": 0.9511855687467806,+ "www.cloudflarestatus.com": 0.9511829060902408,+ "www.cnx-software.com": 0.9511778260754474,+ "www.statnews.com": 0.9511048711614947,+ "youtube-eng.blogspot.com": 0.9510868926645011,+ "news.yale.edu": 0.9510347346201274,+ "reneweconomy.com.au": 0.9509936100187831,+ "www.jwz.org": 0.950973914048226,+ "ncase.me": 0.9509706871520447,+ "www.metzdowd.com": 0.9509697689890211,+ "edu.mkrecny.com": 0.9508994617990429,+ "medium.com": 0.9508522487424811,+ "mastermind.atavist.com": 0.9508126598566774,+ "www.laphamsquarterly.org": 0.9507738399701613,+ "telegram.org": 0.9507726262466255,+ "therecord.media": 0.9507691852400171,+ "www.salon.com": 0.9507685396993281,+ "www.howacarworks.com": 0.9507437871644115,+ "web.mit.edu": 0.9507345330973588,+ "www.zdnet.com": 0.9507029832966206,+ "paulstamatiou.com": 0.9506812073068672,+ "eloquentjavascript.net": 0.9506744929567004,+ "nathanmarz.com": 0.9506394532348129,+ "en.wikipedia.org": 0.9506273469081818,+ "explained.ai": 0.9506171121716123,+ "golang.org": 0.9505833588437288,+ "99percentinvisible.org": 0.9505598730904006,+ "restoreprivacy.com": 0.9505353262581112,+ "ycombinator.com": 0.9505016273288341,+ "www.joshwcomeau.com": 0.950477087355932,+ "easydns.com": 0.9504253361110822,+ "lists.w3.org": 0.9504081521659986,+ "nvidianews.nvidia.com": 0.9504026864806336,+ "cyberlaw.stanford.edu": 0.9503746350188895,+ "www.redhat.com": 0.9503564767146361,+ "blog.racket-lang.org": 0.9503359424455089,+ "www.ctrl.blog": 0.9503095809400977,+ "bjk5.com": 0.9502965197985591,+ "refactoringui.com": 0.9502851812628592,+ "reactos.org": 0.9502772626866873,+ "matthias-endler.de": 0.9502126168099793,+ "blog.usejournal.com": 0.9501925685645481,+ "www.deprocrastination.co": 0.9501876199581508,+ "blog.llvm.org": 0.9501759187745217,+ "blog.keras.io": 0.9501685115979439,+ "markodenic.com": 0.9501517264182752,+ "tirania.org": 0.9501284983515588,+ "www.copetti.org": 0.9500923230891157,+ "www.interviewcake.com": 0.9500659008199689,+ "newatlas.com": 0.9500230591731798,+ "blog.okcupid.com": 0.9499974432299237,+ "smallstep.com": 0.949970101995039,+ "act.eff.org": 0.9499506284870781,+ "www.extremetech.com": 0.9498989577556406,+ "kottke.org": 0.9498599050938643,+ "aeon.co": 0.9498533547972355,+ "blog.printf.net": 0.949841911956293,+ "www.johndcook.com": 0.9497830834287447,+ "community.letsencrypt.org": 0.9497776958809281,+ "www.hashicorp.com": 0.9497516160457014,+ "thewaltdisneycompany.com": 0.9495508540625482,+ "www.haiku-os.org": 0.9495281464383918,+ "scotthelme.co.uk": 0.949509887884973,+ "www.gamasutra.com": 0.9494506172378354,+ "cdixon.org": 0.9494179897832785,+ "azeria-labs.com": 0.9493868918991253,+ "stevehanov.ca": 0.949386070425381,+ "calebporzio.com": 0.9493736951983298,+ "blog.erlang.org": 0.9493443327606497,+ "blog.backblaze.com": 0.9493279590675145,+ "nymag.com": 0.9493193709909513,+ "lists.debian.org": 0.9492598455856424,+ "blog.mailgun.com": 0.9492541788226916,+ "www.microsoft.com": 0.9492199020476939,+ "tokio.rs": 0.9492084252828289,+ "woodgears.ca": 0.949182590244603,+ "www.zotero.org": 0.9491798215496959,+ "zeit.co": 0.9491636525275446,+ "tech.dropbox.com": 0.9491502994308182,+ "pages.cs.wisc.edu": 0.9491309277173726,+ "engineering.instagram.com": 0.9491288216402667,+ "www.realworldtech.com": 0.9491223130451418,+ "www.mattcutts.com": 0.9490815088845603,+ "www.thecut.com": 0.9489293872710988,+ "wiki.mozilla.org": 0.9489007109681905,+ "theunderstatement.com": 0.9488791842136701,+ "donmelton.com": 0.9488523200756159,+ "learncodethehardway.org": 0.9487806920147928,+ "element.io": 0.9487707078398451,+ "thehill.com": 0.948742537996995,+ "wattenberger.com": 0.9487142290281894,+ "raganwald.com": 0.94870117403287,+ "nytimes.com": 0.9487002268786773,+ "www.danshapiro.com": 0.9486602162771469,+ "amasci.com": 0.9484567715181271,+ "steve-yegge.blogspot.com": 0.9484502644034212,+ "blog.instapaper.com": 0.9483924654208046,+ "www.pathsensitive.com": 0.9483799539691706,+ "sachagreif.com": 0.9482981085006165,+ "daedtech.com": 0.948282297833387,+ "www.phoenixframework.org": 0.948180038678699,+ "avc.com": 0.9481175720932656,+ "jasmcole.com": 0.9480827978886457,+ "lemire.me": 0.9480710409382858,+ "ycombinator.posterous.com": 0.9480447746714599,+ "crystal-lang.org": 0.9480289688481155,+ "www.flightradar24.com": 0.9480106923973552,+ "www.lofibucket.com": 0.9479892761394102,+ "www.agner.org": 0.9479708121651846,+ "www.mercurynews.com": 0.947909748405247,+ "peteris.rocks": 0.947883344813371,+ "www.haneycodes.net": 0.9478734625701677,+ "blog.royalsloth.eu": 0.9478558802047412,+ "jeremykun.com": 0.9478499398633942,+ "netmarketshare.com": 0.9478165755279099,+ "streaming.media.ccc.de": 0.9477448566079527,+ "scattered-thoughts.net": 0.9477334018972962,+ "theoutline.com": 0.9476703233752298,+ "www.tedunangst.com": 0.9476236926963729,+ "siberiantimes.com": 0.9476162638549342,+ "avherald.com": 0.947493333568265,+ "hackaday.com": 0.9474690019349598,+ "www.hackerfactor.com": 0.9474384569226819,+ "doist.com": 0.9473521795738291,+ "ai.googleblog.com": 0.9473483560369622,+ "macwright.com": 0.9473244768842966,+ "www.cake.co": 0.9473057089030983,+ "blog.ptsecurity.com": 0.9472729024375095,+ "harpers.org": 0.9472208781730334,+ "research.swtch.com": 0.9471779558478995,+ "landing.google.com": 0.9471768097472296,+ "stackoverflow.com": 0.9471708004823144,+ "www.khanacademy.org": 0.9470872439554527,+ "www.malwaretech.com": 0.9470832155302483,+ "pippinbarr.github.io": 0.9470597529838243,+ "www.chronicle.com": 0.9470439916157474,+ "www.michaelnielsen.org": 0.9470172835353264,+ "yahoo.tumblr.com": 0.9470082148918799,+ "threadreaderapp.com": 0.9470043666496185,+ "lcamtuf.blogspot.com": 0.9469939359995508,+ "www.lowtechmagazine.com": 0.9469818679809128,+ "thorstenball.com": 0.9469695919624959,+ "copy.sh": 0.946952731836535,+ "datagubbe.se": 0.9468822401332838,+ "thehftguy.com": 0.9468555549213109,+ "www.amazon.com": 0.9468350213972545,+ "fishshell.com": 0.9468195302562247,+ "lists.torproject.org": 0.9467852152182991,+ "www.wbur.org": 0.9467718490726582,+ "blog.inkdrop.info": 0.9467189844358732,+ "retool.com": 0.9467024959584281,+ "e360.yale.edu": 0.9466990573296785,+ "magcius.github.io": 0.9466830296864568,+ "www.aboutamazon.com": 0.9466566508253851,+ "howonlee.github.io": 0.9466352935788864,+ "www.randsinrepose.com": 0.9466262113364665,+ "www.inkandswitch.com": 0.9466108105187082,+ "bhorowitz.com": 0.9466022453520991,+ "keenen.xyz": 0.9465855123017971,+ "www.forbes.com": 0.9463974275879253,+ "www.zachtronics.com": 0.9463917451738472,+ "engineering.fb.com": 0.9463604397235058,+ "taibbi.substack.com": 0.946310654934769,+ "blog.elementary.io": 0.9462655485137151,+ "blog.waymo.com": 0.9461989396383733,+ "www.factorio.com": 0.9461844645059527,+ "ghostbin.com": 0.9461717385490003,+ "blog.cubeofm.com": 0.9461649703706937,+ "privacyinternational.org": 0.9461573725205245,+ "mondaynote.com": 0.9461425333335384,+ "chrome.blogspot.com": 0.9461275447307622,+ "www.swissinfo.ch": 0.9460542777095045,+ "fs.blog": 0.9460214401678857,+ "rosenzweig.io": 0.9459592348864171,+ "thehackerblog.com": 0.9459179773558213,+ "palant.info": 0.9459138777363664,+ "course.fast.ai": 0.9459131445844281,+ "www.python.org": 0.9459056139143512,+ "www.samsung.com": 0.9459051315675879,+ "www.niemanlab.org": 0.9458995158159859,+ "perens.com": 0.9458876688022819,+ "www-cs-students.stanford.edu": 0.9458541576360181,+ "www.matuzo.at": 0.9458150903467117,+ "yosefk.com": 0.9457825651296167,+ "samsaffron.com": 0.9457319055612109,+ "www.cs.usfca.edu": 0.9457009533921606,+ "www.the-tls.co.uk": 0.9456885441587526,+ "erlang.org": 0.9456437311862155,+ "home.cern": 0.945571703124721,+ "www.joelonsoftware.com": 0.945567200214926,+ "scottlocklin.wordpress.com": 0.9455154840830812,+ "blog.zoom.us": 0.9454631310396542,+ "clojure.org": 0.9453997954968069,+ "nest.com": 0.9453765377092626,+ "pi-hole.net": 0.9453523308356795,+ "erik-engheim.medium.com": 0.9453523308356795,+ "projecteuler.net": 0.9453380897239767,+ # 'web.archive.org': 0.945336089251807,+ "coreos.com": 0.945323515243916,+ "slack.engineering": 0.9453141410084411,+ "jenkins.io": 0.9452507411562461,+ "libcello.org": 0.9452471895570808,+ "www.ietf.org": 0.9452216766608553,+ "vivaldi.com": 0.9452169051506946,+ "blog.acolyer.org": 0.9452136927944143,+ "opensource.apple.com": 0.945156165859438,+ "www.devever.net": 0.9451481294849068,+ "judiciary.house.gov": 0.9451436080311264,+ "cstack.github.io": 0.9451357466063348,+ "neocities.org": 0.945112946847744,+ "www.vanityfair.com": 0.9451005472578717,+ "heidiroizen.tumblr.com": 0.945076890496307,+ "www.newsweek.com": 0.9450607070750251,+ "fosdem.org": 0.9450470405160809,+ "lists.llvm.org": 0.9450101340465437,+ "swannodette.github.io": 0.9450060465106352,+ "www.forbrukerradet.no": 0.944980556159889,+ "www.alexkras.com": 0.944800412259009,+ "yehudakatz.com": 0.9447946105109732,+ "www.politico.eu": 0.9447851995448193,+ "research.microsoft.com": 0.944725678463696,+ "matklad.github.io": 0.9447162628284546,+ "www.fieggen.com": 0.9446507411542784,+ "fermatslibrary.com": 0.9446378294996843,+ "llvm.org": 0.9446335834779859,+ "www.codeofhonor.com": 0.9446318312955025,+ "www.sandimetz.com": 0.9445834519299251,+ "beautifulracket.com": 0.944575515895676,+ "solar.lowtechmagazine.com": 0.9445367689612141,+ "www.rollingstone.com": 0.9444958767732993,+ "www.theglobeandmail.com": 0.9444675154775829,+ "crypto.stanford.edu": 0.9444289674480927,+ "www.ucsf.edu": 0.9443956140487278,+ "mailman.stanford.edu": 0.9443461149709853,+ "games.greggman.com": 0.9442484049862852,+ "robrhinehart.com": 0.9442240528350656,+ "evertpot.com": 0.9441892143713023,+ "ryancarson.com": 0.944157143243096,+ "www.oculusvr.com": 0.944136637962384,+ "news.microsoft.com": 0.9441300159133728,+ "reasonablypolymorphic.com": 0.9440992853388425,+ "www.nybooks.com": 0.944043771273854,+ "blog.malwarebytes.com": 0.9440378261380166,+ "sourcehut.org": 0.9440365818314045,+ "antonz.org": 0.9439630271519353,+ "weavesilk.com": 0.9439624519736566,+ "jakevdp.github.io": 0.9439416004808824,+ "www.githubstatus.com": 0.9439219010573966,+ "thenextweb.com": 0.9438984254861107,+ "squareup.com": 0.9438684150241483,+ "www.businessinsider.com": 0.9438381542606629,+ "www.teamten.com": 0.9438162837927613,+ "minimaxir.com": 0.94381424365665,+ "www.sparkfun.com": 0.9437861060710178,+ "www.seattletimes.com": 0.9437565154559231,+ "int10h.org": 0.943749694541571,+ "cpbotha.net": 0.9437489993962991,+ "cslibrary.stanford.edu": 0.9437134939561775,+ "mosaicscience.com": 0.943712263624496,+ "www.cjr.org": 0.9436996745182413,+ "lareviewofbooks.org": 0.9436930841143853,+ "www.theregister.co.uk": 0.9436788261536825,+ "www.phoronix.com": 0.9436747415359684,+ "blog.discord.com": 0.9436725616861676,+ "doc.rust-lang.org": 0.9436705513749973,+ "onezero.medium.com": 0.9436010567522964,+ "fuchsia.googlesource.com": 0.9435751703627357,+ "www.tomanthony.co.uk": 0.9435582795515635,+ "blog.ethereum.org": 0.943554185163966,+ "rhaas.blogspot.com": 0.9435179337782055,+ "www.historytoday.com": 0.9434724777695044,+ "veekaybee.github.io": 0.9434651701356596,+ "www.michaelburge.us": 0.9433630281363066,+ "www.hackerschool.com": 0.9433243309324416,+ "507movements.com": 0.9433217341059063,+ "www.bmj.com": 0.9433027312455573,+ "blogs.wsj.com": 0.9432673977773712,+ "leginfo.legislature.ca.gov": 0.9432572372586395,+ "people.csail.mit.edu": 0.9430404662547742,+ "nibblestew.blogspot.com": 0.9429934909491315,+ "backchannel.com": 0.942970399762578,+ "www.openssl.org": 0.9429668863010962,+ "www.openwall.com": 0.9429455499515123,+ "www.petapixel.com": 0.9428687830757905,+ "jeff-vogel.blogspot.com": 0.9427633325613003,+ "roslyn.codeplex.com": 0.9427563891114413,+ "forums.dropbox.com": 0.9427243466398204,+ "answers.microsoft.com": 0.9426879968589384,+ "blog.jazzychad.net": 0.9426788682750873,+ "www.atrium.co": 0.9426550434185653,+ "www.ycombinator.com": 0.9425591138591901,+ "statmodeling.stat.columbia.edu": 0.9425542863968303,+ "cacm.acm.org": 0.9425392273023008,+ "firstround.com": 0.9424999923251012,+ "blog.erratasec.com": 0.9423971981109275,+ "colah.github.io": 0.942392249847614,+ "online.wsj.com": 0.9423603972359367,+ "blog.paralleluniverse.co": 0.9423524400607514,+ "boz.com": 0.9422809938913371,+ "dmitry.gr": 0.942260189831035,+ "www.rethinkdb.com": 0.9422304687024133,+ "www.linuxjournal.com": 0.9421550256435562,+ "www.bottomupcs.com": 0.9421232771919728,+ "commoncog.com": 0.9420976962954309,+ "status.github.com": 0.9419938569642178,+ "www.drmaciver.com": 0.9419852322184868,+ "imgur.com": 0.9419771536808862,+ "privacylog.blogspot.com": 0.9418867600921542,+ "lawsofux.com": 0.9418843340175082,+ "missing.csail.mit.edu": 0.9418514250128648,+ "mitpress.mit.edu": 0.9418492695998034,+ "blog.wesleyac.com": 0.9418306497508919,+ "sanctum.geek.nz": 0.9418252921101978,+ "rarehistoricalphotos.com": 0.9418058693265607,+ "blog.patreon.com": 0.9416385088594889,+ "stephaniehurlburt.com": 0.9416294852135288,+ "www.ssh.com": 0.9416225784673645,+ "sahillavingia.com": 0.9415810451498678,+ "justinmeiners.github.io": 0.9415524612629268,+ "www.bleepingcomputer.com": 0.9415420860759846,+ "thejh.net": 0.941539543862244,+ "codingnest.com": 0.941522096507017,+ "www.motorauthority.com": 0.9415012015037275,+ "kevinlynagh.com": 0.9414829971920764,+ "www.home-assistant.io": 0.941436958446535,+ "spritesmods.com": 0.9413587273840005,+ "www.esa.int": 0.9412959233934257,+ "obsproject.com": 0.9412832929782082,+ "www.1843magazine.com": 0.9412677053641687,+ "therealkatie.net": 0.9412648438023827,+ "christine.website": 0.9412231716105919,+ "aadrake.com": 0.941209497556143,+ "zapier.com": 0.9411980033904498,+ "diegobasch.com": 0.9411801630225842,+ "lichess.org": 0.9411613867773386,+ "www.collectorsweekly.com": 0.9411605509869836,+ "www.spiegel.de": 0.9411529028868376,+ "kev.inburke.com": 0.9410868672076137,+ "matt.sh": 0.9410639090369376,+ "turkeyblocks.org": 0.9410266902131287,+ "translate.google.com": 0.9410059430133028,+ "well.blogs.nytimes.com": 0.9410008856098844,+ "vitalik.ca": 0.9409975246700096,+ "www.independent.co.uk": 0.9409567204051296,+ "www.themarshallproject.org": 0.9409445330421784,+ "www.usenix.org": 0.9409235997678326,+ "blog.docker.io": 0.9408971471442287,+ "kristoff.it": 0.9408956155328897,+ "radio.garden": 0.9408954856512393,+ "tholman.com": 0.9408838927997484,+ "gregoryszorc.com": 0.9408696765619068,+ "phys.org": 0.9408448137648736,+ "www.prisma.io": 0.9408393039096592,+ "aprilzero.com": 0.9408291999272044,+ "blog.parse.com": 0.9408215317556151,+ "borncity.com": 0.9407974930873888,+ "theconversation.com": 0.9407949039397295,+ "www.cl.cam.ac.uk": 0.9407721538543924,+ "grumpygamer.com": 0.9407621103834799,+ "ml.berkeley.edu": 0.9407575811882065,+ "gizmodo.com": 0.9406627597554199,+ "rsms.me": 0.9406458076838878,+ "bitquabit.com": 0.9406214706399899,+ "www.avc.com": 0.9406135247252191,+ "iamnotaprogrammer.com": 0.940590780285089,+ "chrishateswriting.com": 0.940590780285089,+ "newsroom.fb.com": 0.9405869291621605,+ "saagarjha.com": 0.9405291077737054,+ "supabase.io": 0.9404661433287882,+ "me.veekun.com": 0.9404573019098256,+ "250bpm.com": 0.9404292658187561,+ "blog.gregbrockman.com": 0.9403842788606359,+ "www.mono-project.com": 0.9403717514626122,+ "www.minimallyminimal.com": 0.9403321431642234,+ "9to5mac.com": 0.9402902474697965,+ "char.gd": 0.9402697487347172,+ "plumshell.com": 0.9402496652921678,+ "nixos.org": 0.9402384786300783,+ "underjord.io": 0.9402384786300781,+ "landoflisp.com": 0.940191169313915,+ "www.tnhh.net": 0.9401820210600681,+ "rapgenius.com": 0.9401741065194024,+ "blog.floydhub.com": 0.9401390082080993,+ "aws.typepad.com": 0.9401029943851995,+ "leafletjs.com": 0.9400456080747939,+ "fonts.google.com": 0.9400428820587936,+ "matthewearl.github.io": 0.9400368182082475,+ "www.kqed.org": 0.9400004457307097,+ "timharford.com": 0.9399850780559476,+ "blog.innerht.ml": 0.9399767635106161,+ "blog.mecheye.net": 0.9399503933564736,+ "about.sourcegraph.com": 0.9399124106162126,+ "www.wi-fi.org": 0.9398176506266921,+ "www.4chan.org": 0.9397638081665932,+ "www.thedailybeast.com": 0.9397280815769313,+ "www.ipp.mpg.de": 0.9397137553292056,+ "pythonsweetness.tumblr.com": 0.9397041924803161,+ "arstechnica.co.uk": 0.9396691911012381,+ "news.railstutorial.org": 0.9395578781950391,+ "terrytao.wordpress.com": 0.9395312291825673,+ "www.inputmag.com": 0.9394997769231928,+ "labs.detectify.com": 0.9394881401388651,+ "undeadly.org": 0.9394661374968044,+ "tech.channable.com": 0.939456208198243,+ "mobile.nytimes.com": 0.9394237175327378,+ "falkvinge.net": 0.9393964166317929,+ "www.omniref.com": 0.9393438893288835,+ "notes.unwieldy.net": 0.9392345223233198,+ "blog.newsblur.com": 0.9391842678491475,+ "aturon.github.io": 0.9391810647723153,+ "mattwarren.org": 0.9391805858225434,+ "www.fastcompany.com": 0.939180509863255,+ "reason.com": 0.9391096246428814,+ "ramiro.org": 0.9390819120838915,+ "www.nsa.gov": 0.9390773999685927,+ "www.cs.cornell.edu": 0.9390402992969881,+ "www.japantimes.co.jp": 0.9389848040411876,+ "www.twitlonger.com": 0.9389760828355987,+ "nginx.com": 0.9389702817402761,+ "www.humblebundle.com": 0.9389598199049259,+ "publicdomainreview.org": 0.9389382795628163,+ "blog.nuclearsecrecy.com": 0.9388477073828405,+ "pubmed.ncbi.nlm.nih.gov": 0.9388458315114857,+ "qotoqot.com": 0.9388402176228176,+ "pytorch.org": 0.93884021461205,+ "jalammar.github.io": 0.9387918236299037,+ "www.ubuntu.com": 0.9387702819164052,+ "docs.python.org": 0.9387127698679522,+ "google-opensource.blogspot.com": 0.9386938623019518,+ "www.muckrock.com": 0.9386705784507212,+ "dropbox.tech": 0.9386587283618043,+ "www.jasondavies.com": 0.938652854203873,+ "kite.com": 0.9386407930634363,+ "media.netflix.com": 0.9386280658984226,+ "www.indiehackers.com": 0.9386153918054686,+ "blog.detectify.com": 0.9385636423663651,+ "sixcolors.com": 0.9385558509083574,+ "blog.qt.io": 0.9385531275962717,+ "www.quirksmode.org": 0.938550073671435,+ "community.namecheap.com": 0.9384479268346089,+ "thecodist.com": 0.9383878278372055,+ "www.hongkongfp.com": 0.9383791221566731,+ "www.stephenfry.com": 0.9383772523821312,+ "coffeescript.org": 0.9383760425896429,+ "major.io": 0.9383578478495119,+ "typesetinthefuture.com": 0.9383193833277421,+ "www.nayuki.io": 0.9383068163327775,+ "www.hscott.net": 0.9382713522578224,+ "freebsdfoundation.blogspot.com": 0.9382576426660746,+ "eng.uber.com": 0.93823010979631,+ "www.cis.upenn.edu": 0.9382279776034698,+ "open.nytimes.com": 0.9382271017946164,+ "pyvideo.org": 0.9381709441441788,+ "bugs.launchpad.net": 0.9381396092893388,+ "ai.google": 0.9381254410971094,+ "michael.stapelberg.ch": 0.9380985374502866,+ "v8.dev": 0.9380973536567017,+ "digiday.com": 0.9379742396004388,+ "commandcenter.blogspot.com": 0.9379620301227771,+ "tools.ietf.org": 0.9379500242941622,+ "thewalrus.ca": 0.9378381775859617,+ "discussions.apple.com": 0.9377827205296552,+ "ignorethecode.net": 0.937717324851746,+ "thread.gmane.org": 0.9376901941531085,+ "sellercentral.amazon.com": 0.9376857021855104,+ "www.sciencealert.com": 0.9376774589797282,+ "slackhq.com": 0.937668726802223,+ "blog.datawrapper.de": 0.9376182617912265,+ "seriouspony.com": 0.9376182617912265,+ "musicforprogramming.net": 0.9376182617912265,+ "meowni.ca": 0.9375988271796821,+ "www.redox-os.org": 0.9375879483491127,+ "matt-welsh.blogspot.com": 0.937504055669681,+ "minnie.tuhs.org": 0.9374529581205222,+ "rondam.blogspot.com": 0.9374449372004028,+ "warpspire.com": 0.9374088502294052,+ "locusmag.com": 0.9373920717203438,+ "www.bloombergview.com": 0.9373542498215184,+ "www.damninteresting.com": 0.9373300477725874,+ "joeyh.name": 0.9373215784915493,+ "www.newstatesman.com": 0.9373076981929569,+ "www.freebsd.org": 0.9372715325620732,+ "andregarzia.com": 0.9372609577852429,+ "caddyserver.com": 0.9372293681509105,+ "kukuruku.co": 0.9372291284737027,+ "www.jbkempf.com": 0.9372233784657422,+ "pharo.org": 0.9371878563371911,+ "netblocks.org": 0.9371746434498315,+ "chromium.googlesource.com": 0.9371719999296655,+ "phrack.org": 0.9371521351298083,+ "www.codelitt.com": 0.9371442586175533,+ "www.baekdal.com": 0.9371414507989151,+ "www.ianvisits.co.uk": 0.9371147927406074,+ "www.megaprocessor.com": 0.937090322173857,+ "www.yosefk.com": 0.9370290915888351,+ "www.mail-archive.com": 0.936986491855952,+ "abe-winter.github.io": 0.9369860158978973,+ "www.privateinternetaccess.com": 0.9369851095051764,+ "nginx.org": 0.9369470816071664,+ "www.intel.com": 0.9369378525632501,+ "www.meltingasphalt.com": 0.9369168990825746,+ "juliacomputing.com": 0.9368655840942366,+ "rein.pk": 0.936849859435914,+ "store.steampowered.com": 0.9368264728223703,+ "www.osnews.com": 0.9368126058230493,+ "ourworldindata.org": 0.9367969636489132,+ "blogs.law.harvard.edu": 0.93674988182828,+ "www-cs-faculty.stanford.edu": 0.9367219385122602,+ "blog.joda.org": 0.9367157779314322,+ "blog.stephsmith.io": 0.9366996246387541,+ "www.linusakesson.net": 0.9366993761963279,+ "blog.stackoverflow.com": 0.9366671617187088,+ "learn.shayhowe.com": 0.9366612381135532,+ "www.chrisfenton.com": 0.9366359092132852,+ "alexdanco.com": 0.9366140932238274,+ "bert.org": 0.9365314575663983,+ "www.buyforlifeproducts.com": 0.9365183246073299,+ "slate.com": 0.936478665114466,+ "adventofcode.com": 0.9364182659002771,+ "reuters.com": 0.9364022987065517,+ "wordsandbuttons.online": 0.9364000242256861,+ "blog.trailofbits.com": 0.9363648191551225,+ "wwws.whitehouse.gov": 0.9362358726756497,+ "www.saminiir.com": 0.9362027194085935,+ "buttondown.email": 0.9361904266453563,+ "www.polygon.com": 0.9361786245135907,+ "ryanwaggoner.com": 0.9361726319454605,+ "media.defense.gov": 0.9361328144489018,+ "meta.wikimedia.org": 0.9361317597043629,+ "journals.plos.org": 0.9360648852842034,+ "users.ece.utexas.edu": 0.936037069978365,+ "www.wired.co.uk": 0.9360115157816417,+ "code.fb.com": 0.9359956830531816,+ "blog.apnic.net": 0.9359552179738138,+ "www.craftinginterpreters.com": 0.9359520862311609,+ "blog.agilebits.com": 0.9359127041563384,+ "ar.al": 0.935800217022854,+ "tomdale.net": 0.9357863410079088,+ "doublepulsar.com": 0.9357738329674509,+ "www.keyvalues.com": 0.9357728930152032,+ "android-developers.googleblog.com": 0.9357673687378363,+ "www.w3.org": 0.9357529019651287,+ "googlecloudplatform.blogspot.com": 0.9357511057780838,+ "gravityandlevity.wordpress.com": 0.9357231633007936,+ "zoekeating.tumblr.com": 0.9357009188423131,+ "www.startupschool.org": 0.935616713926276,+ "manishearth.github.io": 0.9355839880400915,+ "android.googlesource.com": 0.9355592628654659,+ "thegradient.pub": 0.9355328767434344,+ "orgmode.org": 0.9355234169604703,+ "www.wsws.org": 0.9355201201831335,+ "citizenlab.org": 0.9355108358152496,+ "blog.notdot.net": 0.9355042170349265,+ "eduardo.intermeta.com.br": 0.9354939284774484,+ "joreteg.com": 0.9354927179012053,+ "issues.apache.org": 0.9354858969424931,+ "www.hopkinsmedicine.org": 0.9354732723093431,+ "stopa.io": 0.9354640998209997,+ "rigsomelight.com": 0.9354524106623595,+ "docs.djangoproject.com": 0.9354103729663484,+ "americanaffairsjournal.org": 0.9353862576701931,+ "forums.aws.amazon.com": 0.9353840941845297,+ "jsfiddle.net": 0.9353603756027571,+ "www.metalevel.at": 0.9353562467278554,+ "hexatomium.github.io": 0.9353495280479229,+ "andreasgal.com": 0.9353197730789111,+ "c9x.me": 0.9352853909343191,+ "danieltenner.com": 0.9352765782885712,+ "danielcompton.net": 0.9352733622613952,+ "netflixtechblog.com": 0.9352444459107085,+ "www.vidarholen.net": 0.9352427951268865,+ "themargins.substack.com": 0.9351811093645357,+ "eli.thegreenplace.net": 0.9351789153416379,+ "cybernews.com": 0.9351782885205434,+ "blogs.smithsonianmag.com": 0.9351577172122721,+ "rambo.codes": 0.9351531025036611,+ "hackermonthly.com": 0.9351416375508997,+ "www.sifive.com": 0.9351248366429326,+ "plus.maths.org": 0.9351218966133573,+ "incoherency.co.uk": 0.9351026897171553,+ "www.erlang.org": 0.9350952451519492,+ "circleci.com": 0.9350778094676155,+ "catern.com": 0.9350750533895246,+ "www.penny-arcade.com": 0.935065899223226,+ "howistart.org": 0.935059575032339,+ "www.middleeasteye.net": 0.9350521639498252,+ "unlikekinds.com": 0.9350373828027304,+ "www.videogameschronicle.com": 0.9350222208625469,+ "www.roadandtrack.com": 0.9350012079006418,+ "hyperallergic.com": 0.9349862159226987,+ "blog.level3.com": 0.9349610044277377,+ "www.os2museum.com": 0.934946992149697,+ "soranews24.com": 0.9349080141492474,+ "patshaughnessy.net": 0.9348951953108801,+ "bill.harding.blog": 0.9348850338635601,+ "blog.wikimedia.org": 0.9348790533709054,+ "blog.plan99.net": 0.9348288512702018,+ "www.gq.com": 0.9348208606560191,+ "www.occrp.org": 0.9347939860561864,+ "chronicle.com": 0.9347573157010687,+ "mailarchive.ietf.org": 0.9347044422954542,+ "devonzuegel.com": 0.9346592212655933,+ "www.ben-evans.com": 0.9346544305688902,+ "www.mediumdifficulty.com": 0.9345920431557654,+ "www.fsf.org": 0.9345910501167647,+ "learn.sparkfun.com": 0.9345230614646949,+ "www.cs.princeton.edu": 0.934519791347039,+ "dlang.org": 0.9345196775723191,+ "www.templeos.org": 0.9345190524769148,+ "code.blender.org": 0.9345112418088564,+ "www.singhkays.com": 0.9344753142105167,+ "www.repubblica.it": 0.9344751927302707,+ "www.aerofs.com": 0.9344708443467806,+ "www.lrb.co.uk": 0.9344236297924859,+ "eng.lyft.com": 0.9344184442002427,+ "jakearchibald.com": 0.9343998635925788,+ "blog.youtube": 0.9343948680470999,+ "richg42.blogspot.com": 0.9343774893527464,+ "np.reddit.com": 0.9343550039538729,+ "blog.easydns.org": 0.9343498007033786,+ "caseyhandmer.wordpress.com": 0.9343353091084834,+ "xiph.org": 0.9343228401668484,+ "www.newscientist.com": 0.9343192570672714,+ "www.nicolasbize.com": 0.9343100575008362,+ "lineageos.org": 0.9342978511086933,+ "briancallahan.net": 0.9342859603892616,+ "blog.ltse.com": 0.9342402900090527,+ "www.bryanbraun.com": 0.9341978241526933,+ "blog.thea.codes": 0.9341759010883351,+ "laphamsquarterly.org": 0.9341164747404341,+ "code.mixpanel.com": 0.9340744683300655,+ "fi.google.com": 0.9340569727587175,+ "web.law.duke.edu": 0.9340401302760873,+ "www.technologyreview.com": 0.93403386212399,+ "iamtrask.github.io": 0.9340084148579416,+ "news.ncsu.edu": 0.9340051509637882,+ "www.acsh.org": 0.9339402739014211,+ "repl.it": 0.9339084167158871,+ "roscidus.com": 0.9339046191083922,+ "blog.ghost.org": 0.9338973081214549,+ "www.icann.org": 0.9338815927929048,+ "digest.bps.org.uk": 0.9338601133658677,+ "www.nj.com": 0.9338599954559099,+ "www.90percentofeverything.com": 0.9338469537566946,+ "lars.ingebrigtsen.no": 0.9338290703630162,+ "harmful.cat-v.org": 0.933806937838211,+ "www.ecma-international.org": 0.9337395167070882,+ "engineering.riotgames.com": 0.9337282192512538,+ "www.lexaloffle.com": 0.9337178769327638,+ "www.noaa.gov": 0.9337158126136492,+ "en.greatfire.org": 0.933688333932158,+ "www.techcrunch.com": 0.9336781278522835,+ "research.fb.com": 0.9336749976938089,+ "karl-voit.at": 0.9336349938839054,+ "fair.org": 0.9336252434606269,+ "www.askamathematician.com": 0.9336136547935397,+ "unixism.net": 0.9335805282449469,+ "www.rockpapershotgun.com": 0.9335525383720096,+ "developers.redhat.com": 0.9335410437326863,+ "www.realdanlyons.com": 0.9335250444350301,+ "play0ad.com": 0.9334866458886125,+ "jwz.livejournal.com": 0.933468431827225,+ "tedium.co": 0.9334408912081676,+ "bitcoin.org": 0.9333591671165428,+ "loup-vaillant.fr": 0.933343126525129,+ "getpolarized.io": 0.9333167489840644,+ "thewire.in": 0.9333119721322196,+ "melmagazine.com": 0.9331968311771961,+ "sambleckley.com": 0.9331955922865014,+ "www.inc.com": 0.9331571334144225,+ "david.heinemeierhansson.com": 0.9331531237247338,+ "secondbreakfast.co": 0.9331357922232552,+ "www.asahi.com": 0.933135633505878,+ "devblogs.microsoft.com": 0.9331213750483387,+ "www.nintendo.co.jp": 0.9331114844804387,+ "annoying.technology": 0.9330709402145108,+ "www.kiro7.com": 0.9330656946067081,+ "bits.debian.org": 0.933059049771486,+ "www.markshuttleworth.com": 0.9330005151597153,+ "anders.unix.se": 0.9329923290063773,+ "blogs.sciencemag.org": 0.9329560797100107,+ "blog.bitbucket.org": 0.9329522711475849,+ "blog.kubernetes.io": 0.9329204663072305,+ "devdocs.io": 0.932893331029596,+ "justinjackson.ca": 0.9327982068928226,+ "corte.si": 0.9327949868127001,+ "kotaku.com": 0.9327897653993101,+ "cdn.ca9.uscourts.gov": 0.9327301316906936,+ "forum.dlang.org": 0.9326997809060901,+ "spacecraft.ssl.umd.edu": 0.9326765708381659,+ "www.sciencedirect.com": 0.9326581830227065,+ "neuralnetworksanddeeplearning.com": 0.9326459352221996,+ "www.politico.com": 0.9326420745727587,+ "notamonadtutorial.com": 0.9326372939715099,+ "news.umich.edu": 0.9325964202141381,+ "duck.co": 0.9325821384299628,+ "lg.io": 0.9325797024877297,+ "vorpus.org": 0.9325422497012895,+ "objective-see.com": 0.9324963202009166,+ "www.erlang-solutions.com": 0.9324852099306608,+ "alexgaynor.net": 0.9324583238847478,+ "sub.garrytan.com": 0.9324345058249506,+ "www.dropbox.com": 0.9324087214510501,+ "xkcd.com": 0.932407776611293,+ "blog.codinghorror.com": 0.9323885522938681,+ "wingolog.org": 0.9323392944843699,+ "www.pipelinedb.com": 0.9322802131775214,+ "iquantny.tumblr.com": 0.932275204317245,+ "hbr.org": 0.9322550416701433,+ "www.washington.edu": 0.9321755936890619,+ "blog.nelhage.com": 0.9321436277497521,+ "tonyarcieri.com": 0.932131262902639,+ "www.geoffreylitt.com": 0.9321152273514123,+ "viniciusvacanti.com": 0.9320235111465629,+ "www.magicleap.com": 0.9319299806186524,+ "www.igvita.com": 0.9318778756566067,+ "chrisdone.com": 0.9318776390307953,+ "stackshare.io": 0.9318592007853939,+ "latacora.singles": 0.9318477865160556,+ "www.dell.com": 0.9318413237056871,+ "djhworld.github.io": 0.9317862165963432,+ "www.infoq.com": 0.9317836684869253,+ "papersplease.org": 0.931751319080821,+ "thepiratebay.org": 0.9316799771304791,+ "calnewport.com": 0.9316572134183647,+ "krausefx.com": 0.9316559307678887,+ "blog.circleci.com": 0.9316255482754808,+ "blog.cognitect.com": 0.9315784532891852,+ "www.epicgames.com": 0.9315429629794932,+ "scopeblog.stanford.edu": 0.9315377292068189,+ "www.masswerk.at": 0.9314679327889084,+ "0x65.dev": 0.9314351318900396,+ "www.engadget.com": 0.9314186236786943,+ "dangerousminds.net": 0.93133503311817,+ "www.businessweek.com": 0.9313339649208416,+ "meta.slashdot.org": 0.9313051774463162,+ "john-millikin.com": 0.9312674134931619,+ "www.raptitude.com": 0.9312570267721416,+ "www.mersenne.org": 0.9312544294826364,+ "blog.reverberate.org": 0.931247035488847,+ "alexisohanian.com": 0.9311960984991072,+ "electrek.co": 0.9311696023394275,+ "www.jeanhsu.com": 0.9311256778507919,+ "svelte.technology": 0.9311186925197594,+ "programmers.stackexchange.com": 0.9311091436248506,+ "discuss.ocaml.org": 0.9311072488914112,+ "nedbatchelder.com": 0.9310954334270662,+ "www.haskellforall.com": 0.931051756515657,+ "retrocomputing.stackexchange.com": 0.9310144532795315,+ "catonmat.net": 0.9309769810532454,+ "vole.wtf": 0.9309344769161616,+ "www.scribd.com": 0.9309198232076064,+ "yurichev.com": 0.9309143601885577,+ "joel.is": 0.9309094060954536,+ "yager.io": 0.9308009815388928,+ "boilingsteam.com": 0.9307989777220242,+ "gamasutra.com": 0.9307558317808844,+ "www.bigmessowires.com": 0.9307114429921958,+ "fail0verflow.com": 0.930710252209961,+ "news.harvard.edu": 0.9306854591781405,+ "beagleboard.org": 0.9306733479517327,+ "www.floridatoday.com": 0.9306710239151541,+ "www.popehat.com": 0.9306350599333171,+ "blog.tutorspree.com": 0.9306202167888316,+ "daeken.com": 0.9305668550552411,+ "techstartups.com": 0.9305594478826466,+ "www.wikitribune.com": 0.930515310124751,+ "www.chess.com": 0.9304928063593388,+ "newsoffice.mit.edu": 0.9304895970361445,+ "wired.com": 0.9304827864643992,+ "sealedabstract.com": 0.9304819216359799,+ "0x46.net": 0.9304806845007771,+ "unsplash.com": 0.9304637690166982,+ "my.opera.com": 0.930450954749869,+ "nerds.airbnb.com": 0.930435238682839,+ "longreads.com": 0.9304235614345844,+ "www.dispatch.com": 0.9303745139221324,+ "www.fda.gov": 0.9303439075870598,+ "lists.centos.org": 0.9303434013425956,+ "adgrok.com": 0.9303388809532392,+ "thinkingthrough.substack.com": 0.9303337259943262,+ "mcfunley.com": 0.9303298670383908,+ "beginners.re": 0.9302874468948247,+ "khan.github.io": 0.9302341812757136,+ "joshworth.com": 0.9302242175504744,+ "lamport.azurewebsites.net": 0.9302036598082833,+ "calv.info": 0.9301915155599901,+ "jdk.java.net": 0.9301739684537851,+ "keepassxc.org": 0.9301626315930274,+ "thepiratebay.se": 0.9301506979843667,+ "metatalk.metafilter.com": 0.9301352170901589,+ "gregable.com": 0.9301219028430462,+ "newrepublic.com": 0.9301135242301953,+ "cryptome.org": 0.9301123136352331,+ "www.smashcompany.com": 0.9301070862923108,+ "mixergy.com": 0.9301017907321477,+ "blog.bolt.io": 0.9300384028733197,+ "lesswrong.com": 0.9299875078740503,+ "www.recode.net": 0.9299803603149057,+ "baremetrics.com": 0.9299780878801276,+ "idea.popcount.org": 0.92997751068095,+ "blog.nodejs.org": 0.9299713179632104,+ "googlecode.blogspot.com": 0.9299671011900217,+ "dustincurtis.com": 0.929962639691764,+ "capnproto.org": 0.9299597749554257,+ "samy.pl": 0.9299580709168783,+ "fivethirtyeight.com": 0.9299521782157305,+ "babeljs.io": 0.9299138678024598,+ "venturebeat.com": 0.9299090359763643,+ "bartoszmilewski.com": 0.9298782796519226,+ "interactivepython.org": 0.9298650433817537,+ "dbmsmusings.blogspot.com": 0.9298460930808272,+ "feross.org": 0.9298143975383142,+ "www.nateberkopec.com": 0.9297897921141977,+ "www.bnnbloomberg.ca": 0.9297228959508879,+ "sourcegraph.com": 0.9296937397028914,+ "www.visualstudio.com": 0.9296781764315996,+ "www.dampfkraft.com": 0.9296648658804626,+ "sockpuppet.org": 0.9296582232650001,+ "vuejs.org": 0.9296438971766601,+ "mozillagfx.wordpress.com": 0.929620568345932,+ "www.tomshardware.com": 0.9295421933042515,+ "go.googlesource.com": 0.929539952433035,+ "every.to": 0.9295113508234609,+ "community.signalusers.org": 0.9295104306484039,+ "features.propublica.org": 0.9294651901563945,+ "trigger.io": 0.9294592741801949,+ "blog.wolfram.com": 0.9294394447255451,+ "science.nasa.gov": 0.9294364753474607,+ "www.globalcitizen.org": 0.9294355607283274,+ "brennan.io": 0.9294094367716018,+ "www.princeton.edu": 0.9293972895467508,+ "www.typescriptlang.org": 0.9293477418572129,+ "blog.hboeck.de": 0.9293102195703717,+ "www.kickstarter.com": 0.9293062867407036,+ "blog.siftscience.com": 0.9293015387633619,+ "blog.jetbrains.com": 0.9292407139248418,+ "mag.newsweek.com": 0.9291970802919708,+ "libcom.org": 0.9291864757522856,+ "pudding.cool": 0.9291654244837007,+ "www.geek.com": 0.9291554612391715,+ "zedshaw.com": 0.9291438333180465,+ "mailman.nginx.org": 0.9291339657438804,+ "www.techspot.com": 0.9291295247017188,+ "www.popularmechanics.com": 0.9291147919612897,+ "www.nbcnews.com": 0.9290863046432193,+ "acesounderglass.com": 0.929082768087054,+ "stitcher.io": 0.9290511337189973,+ "www.texasmonthly.com": 0.92902453382746,+ "mattmaroon.com": 0.9290233630671553,+ "www.gabrielgambetta.com": 0.9290195296996202,+ "m.dw.com": 0.9290040778772607,+ "www.mit.edu": 0.9289941184727176,+ "tratt.net": 0.9289824042121754,+ "machinelearning.apple.com": 0.928981486544923,+ "blog.posterous.com": 0.9289604924288977,+ "oxide.computer": 0.9289445169614982,+ "trackchanges.postlight.com": 0.928879705271154,+ "codahale.com": 0.9288692041231931,+ "emberjs.com": 0.9288495718673251,+ "applehelpwriter.com": 0.9287812041116006,+ "shaunlebron.github.io": 0.9286838540725274,+ "open.buffer.com": 0.9286596873950792,+ "www.expatsoftware.com": 0.9286078722400491,+ "edgedb.com": 0.928573715100738,+ "etherpad.com": 0.9285692478417551,+ "muratbuffalo.blogspot.com": 0.9285656789226785,+ "fgiesen.wordpress.com": 0.9285630732312407,+ "www.firebase.com": 0.9285546878582209,+ "blog.thestartuptoolkit.com": 0.9285487410065263,+ "cloudirregular.substack.com": 0.9285463604002121,+ "flynn.io": 0.9285393525237848,+ "www.eevblog.com": 0.9285287752743465,+ "www.balena.io": 0.9284888543895996,+ "rt.com": 0.9284735976773327,+ "www.bell-labs.com": 0.9284678162678174,+ "www.persuasion.community": 0.9283945188643842,+ "eugeneyan.com": 0.9283782094499546,+ "80000hours.org": 0.9283725313298165,+ "www.zerotier.com": 0.9283630467998711,+ "residentcontrarian.substack.com": 0.9283528817911175,+ "httptoolkit.tech": 0.9283475144274776,+ "www.ekioh.com": 0.9283450473725626,+ "www.cia.gov": 0.9283207863801844,+ "ryanleecarson.tumblr.com": 0.9283074648928308,+ "nickcraver.com": 0.9282875328539527,+ "syonyk.blogspot.com": 0.9282602083788998,+ "www.simonsfoundation.org": 0.9282557678633827,+ "www.artsy.net": 0.9281354567126978,+ "tryretool.com": 0.9281338223307144,+ "www.bloombergquint.com": 0.928121311983667,+ "johncarlosbaez.wordpress.com": 0.9281129934414499,+ "vas3k.com": 0.9280973443374919,+ "100r.co": 0.9280948851000741,+ "blogs.adobe.com": 0.928088150929166,+ "www.motherjones.com": 0.9280843595076367,+ "www.nasdaq.com": 0.9280594095034941,+ "scarybeastsecurity.blogspot.com": 0.9280588046711343,+ "bitcointalk.org": 0.9280388108667238,+ "hackershelf.com": 0.9280341162118197,+ "news.nationalgeographic.com": 0.9279960640653149,+ "myactivity.google.com": 0.9279454537052139,+ "blog.nightly.mozilla.org": 0.9279411885747856,+ "slashcrypto.org": 0.9279346210995543,+ "www.phoboslab.org": 0.927902020777888,+ "tosdr.org": 0.9278899901888288,+ "www.fiercebiotech.com": 0.9278829786533511,+ "pomax.github.io": 0.9278608982168398,+ "www.haskell.org": 0.9278608403233632,+ "tribunemag.co.uk": 0.927831772588806,+ "alexeymk.com": 0.9278193647918856,+ "www.cs.vu.nl": 0.9277449579009552,+ "news.cnet.com": 0.927730426058379,+ "www.caltech.edu": 0.9277282839323142,+ "www.azarask.in": 0.9276539782249613,+ "gigaom.com": 0.9276147331753969,+ "pdos.csail.mit.edu": 0.9276091373570645,+ "www.virtualbox.org": 0.9275378961204167,+ "increment.com": 0.9275213097235798,+ "blog.janestreet.com": 0.9274833694141223,+ "www.0x65.dev": 0.9274613287943144,+ "textslashplain.com": 0.9274390770834591,+ "www.planetary.org": 0.927429526861741,+ "allthingsd.com": 0.9274270826887906,+ "clojure.com": 0.9274173307178728,+ "git.postgresql.org": 0.9274089095546911,+ "www.howtogeek.com": 0.927406503902944,+ "vr.google.com": 0.9273952095808383,+ "blog.kapeli.com": 0.9273878300515732,+ "quickdraw.withgoogle.com": 0.9273611719298221,+ "erkaman.github.io": 0.9273454992466085,+ "discussions.agilebits.com": 0.9273075851865854,+ "fauna.com": 0.9272945460775945,+ "popehat.com": 0.9272692801582506,+ "blog.sandstorm.io": 0.9272550707158864,+ "www.openvim.com": 0.9272549340276682,+ "www.panic.com": 0.9272306781608166,+ "waymo.com": 0.9271995570219662,+ "blog.eladgil.com": 0.9271995137279543,+ "uk.reuters.com": 0.9271613384786328,+ "www.openstreetmap.org": 0.9270974234890762,+ "projekte.sueddeutsche.de": 0.9270885496912139,+ "whytheluckystiff.net": 0.9270441393847589,+ "the-paper-trail.org": 0.9269567323760859,+ "medium.dave-bailey.com": 0.9269393622635275,+ "www.kadavy.net": 0.9269319992417328,+ "symbo1ics.com": 0.9269313741433315,+ "erikrood.com": 0.926847335423336,+ "text.npr.org": 0.9268424682157039,+ "www.gnome.org": 0.9268089290588881,+ "www.arp242.net": 0.9267978785624127,+ "nvlabs.github.io": 0.9267181712194288,+ "craigkerstiens.com": 0.9267131513833771,+ "www.evanjones.ca": 0.9266986776879761,+ "johnsalvatier.org": 0.926681783824641,+ "notes.eatonphil.com": 0.9266798244317584,+ "cityobservatory.org": 0.9266470517985763,+ "www.autocar.co.uk": 0.9266248470761401,+ "store.google.com": 0.9265910046118487,+ "blog.alinelerner.com": 0.9265889786185729,+ "a.singlediv.com": 0.9265847984290865,+ "www.city-journal.org": 0.926568347566864,+ "blog.bugsnag.com": 0.9265163617009136,+ "blog.steveklabnik.com": 0.9265012336320916,+ "www.ox.ac.uk": 0.9264790742628204,+ "www.jgc.org": 0.92646604158067,+ "maraoz.com": 0.9264393162283295,+ "www.somethingsimilar.com": 0.9264338569528214,+ "electronupdate.blogspot.com": 0.9264036418816388,+ "productforums.google.com": 0.9264016849144228,+ "macro.ycombinator.com": 0.9263799756741565,+ "damienkatz.net": 0.9263697789667,+ "www.moserware.com": 0.9263363409676512,+ "nikic.github.io": 0.9263232996382066,+ "www.elastic.co": 0.9263050573116817,+ "www.cs.utexas.edu": 0.9263034743689128,+ "www.bhorowitz.com": 0.9262984080290757,+ "tenthousandmeters.com": 0.9262841230837943,+ "about.fb.com": 0.9262484440409288,+ "www.fmwconcepts.com": 0.9262450791824973,+ "recraigslist.com": 0.9262425641046296,+ "www.codingjohnson.com": 0.9262281542120302,+ "www.aftenposten.no": 0.9262270297289266,+ "www-math.mit.edu": 0.9261881534042337,+ "blog.1password.com": 0.9261268579416305,+ "neugierig.org": 0.9261248188625917,+ "01.org": 0.9261246392123041,+ "jalopnik.com": 0.9261214539229355,+ "www.mnot.net": 0.9261212790512424,+ "jmoiron.net": 0.9260837453092551,+ "bjoern.brembs.net": 0.9260528919272174,+ "en.community.dell.com": 0.9260523403380299,+ "yieldthought.com": 0.9260498267973828,+ "news.gallup.com": 0.9260444722659747,+ "www.quora.com": 0.9260005301179841,+ "www.boingboing.net": 0.9259933447507237,+ "plusbryan.com": 0.9259774485563826,+ "webcache.googleusercontent.com": 0.9259579793050731,+ "rameznaam.com": 0.9259525603398252,+ "dev.windows.com": 0.9259474200369756,+ "www.cnn.com": 0.9259450168290682,+ "ij.org": 0.9259017125754639,+ "hackingdistributed.com": 0.9258935189335176,+ "aosabook.org": 0.9258707865599977,+ "webtransparency.cs.princeton.edu": 0.9258302497498668,+ "nikcub.appspot.com": 0.9257984690273875,+ "www.fcc.gov": 0.9257766215669876,+ "multithreaded.stitchfix.com": 0.9256971609820355,+ "www.ybrikman.com": 0.9256541091635649,+ "www.neh.gov": 0.925636711625259,+ "abc.xyz": 0.9256328360448464,+ "thecorrespondent.com": 0.9256254843843928,+ "www.techempower.com": 0.9256081793071045,+ "letsencrypt.status.io": 0.9255938806311487,+ "grail.cs.washington.edu": 0.925560977684432,+ "syncthing.net": 0.9255454895835606,+ "torontolife.com": 0.9255329261936903,+ "www.notion.so": 0.9255238074027249,+ "www.empiricalzeal.com": 0.925521405860939,+ "www.brown.edu": 0.925521405860939,+ "dustri.org": 0.9255123842761099,+ "restofworld.org": 0.9254846646984972,+ "meta.stackoverflow.com": 0.9254699602796586,+ "www.mikeash.com": 0.9254652180552938,+ "emacsair.me": 0.9254649535069708,+ "fossil-scm.org": 0.9254641195091267,+ "drive.google.com": 0.9254499381167396,+ "rauchg.com": 0.9254066104836953,+ "www.oregonlive.com": 0.9254001048984968,+ "madebyevan.com": 0.9253946331608134,+ "bitbucket.org": 0.925357050068107,+ "justinvincent.com": 0.9253388274119266,+ "shape-of-code.coding-guidelines.com": 0.9253230426310416,+ "www.journals.uchicago.edu": 0.9252909862687759,+ "www.hodinkee.com": 0.925286129490443,+ "blog.getprismatic.com": 0.9252627177795818,+ "android-developers.blogspot.com": 0.9252451042228654,+ "jeremyaboyd.com": 0.9252351792387165,+ "mozilla.github.io": 0.9252103780415603,+ "www.pcgamer.com": 0.9251974859644215,+ "tutanota.com": 0.9251840115849959,+ "review.chicagobooth.edu": 0.9251560984825087,+ "programmingisterrible.com": 0.9251358332003701,+ "zainamro.com": 0.9251300763149733,+ "www.locusmag.com": 0.9251291781651696,+ "waxy.org": 0.9250938835073874,+ "bentilly.blogspot.com": 0.9250916663441209,+ "hisham.hm": 0.925069607765931,+ "www.foundersatwork.com": 0.9250661963070413,+ "neil.computer": 0.9250487557545186,+ "youmightnotneedjquery.com": 0.9250308951241109,+ "sparktoro.com": 0.9250076259146361,+ "supporters.eff.org": 0.9250027991144959,+ "www.youtube.com": 0.9250024575604987,+ "www.django-rest-framework.org": 0.9249975958434261,+ "fuse.wikichip.org": 0.9249946532155069,+ "alexanderell.is": 0.9249832648395125,+ "www.jetbrains.com": 0.9249775253793479,+ "rootsofprogress.org": 0.9249775157472907,+ "blog.jonlu.ca": 0.9249720526798552,+ "grafana.com": 0.9249363529284085,+ "joinpeertube.org": 0.92492611085508,+ "trac.torproject.org": 0.92492568982512,+ "www.kapwing.com": 0.9249078774898674,+ "jgthms.com": 0.9249053279219519,+ "www.lighterra.com": 0.9248749337230386,+ "www.bzarg.com": 0.9248749337230386,+ "itep.org": 0.924864446165763,+ "xuanji.appspot.com": 0.924862527414928,+ "dood.al": 0.9248541691533322,+ "seriot.ch": 0.924849625532851,+ "go-review.googlesource.com": 0.9248263431754091,+ "benjamincongdon.me": 0.924814411910629,+ "coderwall.com": 0.9247619851237905,+ "www.betabeat.com": 0.9247390428437294,+ "cryto.net": 0.924723051999456,+ "blogs.bl.uk": 0.9247163678317087,+ "datatracker.ietf.org": 0.924674483012171,+ "www.b-list.org": 0.9246677557779804,+ "apple.stackexchange.com": 0.924658393133553,+ "www.mozilla.com": 0.9246560951725633,+ "www.mailpile.is": 0.9246491366569076,+ "see.stanford.edu": 0.9246092475112012,+ "www.chiark.greenend.org.uk": 0.9246053483860566,+ "groups.csail.mit.edu": 0.9245841594300752,+ "www.arcfn.com": 0.9245434848298174,+ "urbit.org": 0.9245310249759199,+ "www.docker.com": 0.9245172226554722,+ "eprint.iacr.org": 0.9245057684882811,+ "dl.acm.org": 0.9244842820223327,+ "blog.sessionstack.com": 0.9244816364620212,+ "www.breck-mckye.com": 0.9244526176174389,+ "www.aosabook.org": 0.9244287224439842,+ "prospect.org": 0.9243862527888093,+ "micro-editor.github.io": 0.9243497443248759,+ "banksimple.com": 0.9243293845078053,+ "practicaltypography.com": 0.9243063840022637,+ "blogmaverick.com": 0.9242840626877411,+ "instagram-engineering.com": 0.9242395677129099,+ "theartofmachinery.com": 0.9242293335123167,+ "www.cpushack.com": 0.9241635632263332,+ "guzey.com": 0.9241634452596086,+ "morrick.me": 0.9241492448464671,+ "chanind.github.io": 0.9241272337118139,+ "blog.emojipedia.org": 0.9241070688292589,+ "arjunsreedharan.org": 0.9240265801481968,+ "superuser.com": 0.9240150880034484,+ "zulko.github.io": 0.9240082509797493,+ "blog.fosketts.net": 0.923996721917042,+ "darkpatterns.org": 0.9239687269035174,+ "www.ulisp.com": 0.9239409936091142,+ "herbsutter.com": 0.9238650642442959,+ "research.google.com": 0.9238529070048137,+ "www.darkcoding.net": 0.9238452017940412,+ "latacora.micro.blog": 0.9237958672976048,+ "www.drdobbs.com": 0.9237812616938089,+ "www.hollywoodreporter.com": 0.9237805856737081,+ "www.nginx.com": 0.9237795637254299,+ "thestartupfoundry.com": 0.9237690021040942,+ "blog.fastmail.fm": 0.923729095971477,+ "wit.ai": 0.9237161044120704,+ "www.devttys0.com": 0.923715140193862,+ "www.qubes-os.org": 0.9237100158767054,+ "www.robinsloan.com": 0.9236869500281771,+ "timetobleed.com": 0.9236836576824262,+ "blog.jitbit.com": 0.9236712552599629,+ "gowers.wordpress.com": 0.9236675842265081,+ "www.pnas.org": 0.9236623882869138,+ "mrale.ph": 0.9236621608796006,+ "play.freeciv.org": 0.9235744210893093,+ "www.blackhat.com": 0.9235710982881401,+ "laforge.gnumonks.org": 0.9234795250933899,+ "marker.medium.com": 0.9234758009863355,+ "www.sfchronicle.com": 0.9234493747928338,+ "tutorial.math.lamar.edu": 0.9234097897078604,+ "www.holovaty.com": 0.9233603823514369,+ "www.cringely.com": 0.9232863734060375,+ "www.folklore.org": 0.9232666029542594,+ "apnews.com": 0.9232599354911131,+ "devblogs.nvidia.com": 0.9231813399979636,+ "www.linuxvoice.com": 0.9231576608319696,+ "papers.ssrn.com": 0.9231181748415175,+ "brianschrader.com": 0.9231068216939985,+ "www.kreci.net": 0.923106349521949,+ "www.nngroup.com": 0.9230948271379453,+ "funcall.blogspot.com": 0.9230679265670205,+ "www.jasonshen.com": 0.9230677977899461,+ "lists.freedesktop.org": 0.9230222479984594,+ "nodejs.org": 0.92301963271793,+ "www.graalvm.org": 0.9230158730158731,+ "blog.mozilla.com": 0.923006927671048,+ "www.dailydot.com": 0.9230068718762484,+ "lbrandy.com": 0.9229742803918614,+ "erratasec.blogspot.com": 0.9229733794911794,+ "www.chemistryworld.com": 0.9229661370816306,+ "www.solipsys.co.uk": 0.9229220876704523,+ "layerci.com": 0.9229159114633821,+ "www.stellar.org": 0.9229142185891028,+ "etherscan.io": 0.9228765015817684,+ "www.macrumors.com": 0.922864948439789,+ "vim-adventures.com": 0.9228321400159109,+ "mailinator.blogspot.com": 0.9228075138073343,+ "www.macleans.ca": 0.9227372105459439,+ "blog.appcanary.com": 0.9227212708074342,+ "www.stallman.org": 0.9227212708074342,+ "briancarper.net": 0.9227017702961716,+ "joehewitt.com": 0.9226894917334,+ "sfconservancy.org": 0.9226892388952699,+ "mariovittone.com": 0.9226475279106858,+ "500hats.com": 0.9226369106547709,+ "blog.tailwindcss.com": 0.9226012568165967,+ "dilbert.com": 0.9226004268083031,+ "mewo2.com": 0.9225891352707621,+ "blog.getpebble.com": 0.9225467649630955,+ "preshing.com": 0.922540762544208,+ "www.medievalists.net": 0.9225218713450589,+ "daily.jstor.org": 0.9225038762181162,+ "www.lynalden.com": 0.922503215089193,+ "hindenburgresearch.com": 0.9225025964121197,+ "joindiaspora.com": 0.9224924773526838,+ "chadfowler.com": 0.922481090652384,+ "git.sr.ht": 0.9224628516537063,+ "twitter.github.io": 0.9224447323362635,+ "www.rackspace.com": 0.92242477632422,+ "www.science.org": 0.9224216244734532,+ "www.esquire.com": 0.9224172218178369,+ "augustl.com": 0.9224131314072078,+ "news.cornell.edu": 0.9223803694889144,+ "paulrouget.com": 0.9223675768999002,+ "joelgrus.com": 0.9222919914652804,+ "electronjs.org": 0.9222709958305552,+ "williamedwardscoder.tumblr.com": 0.9222429833639271,+ "fastmail.blog": 0.9222426931980966,+ "trixter.oldskool.org": 0.9222240643067603,+ "palladiummag.com": 0.9221161388490351,+ "www.dwmkerr.com": 0.9220928814524835,+ "www.blueorigin.com": 0.9220497550322969,+ "blogs.perl.org": 0.922041527915435,+ "peter.bourgon.org": 0.9220345737360726,+ "boats.gitlab.io": 0.9220270248890161,+ "news.usc.edu": 0.9220177070210377,+ "www.media.uzh.ch": 0.9220147439869917,+ "rubini.us": 0.9220102711604998,+ "ognjen.io": 0.9219863609200528,+ "www.dragonflybsd.org": 0.9219689458962362,+ "time.com": 0.9219350046913536,+ "www.vagrantup.com": 0.9219259642840062,+ "reclaimthenet.org": 0.921903148362967,+ "blogs.lse.ac.uk": 0.921869679768062,+ "threader.app": 0.9218497168940121,+ "pgdash.io": 0.9217440948884412,+ "www.namecheap.com": 0.9217409250741742,+ "www.mtgox.com": 0.9217171114471724,+ "blog.filepicker.io": 0.9217169081929824,+ "mullvad.net": 0.9216909513671212,+ "m.signalvnoise.com": 0.9216188131338918,+ "gamerant.com": 0.9215933437650176,+ "www.computerhistory.org": 0.9215810214981082,+ "www.bennettnotes.com": 0.9215678399791286,+ "codegolf.stackexchange.com": 0.9215396697671643,+ "www.amnesty.org": 0.9214886045349302,+ "quip.com": 0.9214846662604561,+ "www.balsamiq.com": 0.9214784555047167,+ "www.conwaylife.com": 0.9214510848919855,+ "quoteinvestigator.com": 0.9214277867192284,+ "www.stephanboyer.com": 0.9214274950425619,+ "www.ibm.com": 0.9214034975155869,+ "kkovacs.eu": 0.9213904944480831,+ "www.cs.virginia.edu": 0.9213138750457075,+ "blog.heapanalytics.com": 0.9212598460858725,+ "www.khronos.org": 0.9212580247777246,+ "infotrope.net": 0.9212298613344002,+ "r6.ca": 0.9212165287852896,+ "elemental.medium.com": 0.9211754093730133,+ "standardnotes.org": 0.9211590022701598,+ "www.thanassis.space": 0.9211469562862422,+ "www.essential.com": 0.9211382113821138,+ "blog.checkpoint.com": 0.9210707601354348,+ "firebase.googleblog.com": 0.9210648800435821,+ "theamericanscholar.org": 0.9210499166083035,+ "mxb.dev": 0.9210070283654466,+ "arrow.apache.org": 0.9209592797673791,+ "moderncrypto.org": 0.9209488760965047,+ "edition.cnn.com": 0.9209449218392389,+ "penny-arcade.com": 0.9208903657692026,+ "www.gresearch.co.uk": 0.9208877326699497,+ "news.rapgenius.com": 0.9208723565849319,+ "duartes.org": 0.9208704095563033,+ "access.redhat.com": 0.9208698327943827,+ "dealbook.nytimes.com": 0.9208660000281566,+ "www.robinwieruch.de": 0.9208515295663545,+ "blog.viktomas.com": 0.9208241650543216,+ "www.thenation.com": 0.9208156841051072,+ "evilmartians.com": 0.9207891062516377,+ "www.unixsheikh.com": 0.9207398153686466,+ "david.weebly.com": 0.9207376999076698,+ "lanyrd.com": 0.9207234118522407,+ "www.slowboring.com": 0.9206921783803773,+ "unix.stackexchange.com": 0.9206801573666252,+ "seldo.com": 0.9206416671015177,+ "www.elischiff.com": 0.920635133939363,+ "www.arrl.org": 0.9206234339791557,+ "www.spacetelescope.org": 0.9205965394020418,+ "writing.markchristian.org": 0.9205569205569206,+ "elaineou.com": 0.9205345242508725,+ "blog.lastpass.com": 0.9205320290125674,+ "netzpolitik.org": 0.9205194544865914,+ "ghuntley.com": 0.9205047884758858,+ "knightcolumbia.org": 0.9205029147967208,+ "www.comparitech.com": 0.9204765637809545,+ "scanlime.org": 0.9204328320221652,+ "www.sfgate.com": 0.920422056660529,+ "eklitzke.org": 0.9203820542379546,+ "www.scala-lang.org": 0.9203673159816037,+ "people.eecs.berkeley.edu": 0.9203504139456036,+ "bits.blogs.nytimes.com": 0.9203453783143634,+ "floooh.github.io": 0.9203399579895484,+ "obscuritory.com": 0.9203310173661203,+ "habitatchronicles.com": 0.9203279953622738,+ "www.giftrocket.com": 0.9203035040887337,+ "www.debugbear.com": 0.9202964328943275,+ "makezine.com": 0.920278418856363,+ "www.slideshare.net": 0.920271947086287,+ "sandlab.cs.uchicago.edu": 0.9202302631578948,+ "timkadlec.com": 0.9202082523474097,+ "momentjs.com": 0.9201811740344046,+ "www.euractiv.com": 0.9201333156342562,+ "www.daniellemorrill.com": 0.9201158212493131,+ "writings.stephenwolfram.com": 0.9200560602651936,+ "www.gkogan.co": 0.9200121929029601,+ "needwant.com": 0.9199907538489269,+ "www.medialens.org": 0.9199880352359148,+ "blog.wepay.com": 0.9199344757712936,+ "maps.google.com": 0.9199182118968083,+ "www.dustmop.io": 0.9199009083402148,+ "www.comp.nus.edu.sg": 0.9198995227263196,+ "wiki.ubuntu.com": 0.9198686814026275,+ "remotebase.io": 0.9198347107438016,+ "www.37signals.com": 0.9198326817403155,+ "alumni.media.mit.edu": 0.9198312121350687,+ "www.smashingmagazine.com": 0.9197923257499647,+ "kylehalladay.com": 0.9197075896809243,+ "fortune.com": 0.9196957257034758,+ "fosterelli.co": 0.9196146019753699,+ "www.braveclojure.com": 0.9196013394110684,+ "anvaka.github.io": 0.919572990689544,+ "nomasters.io": 0.9195688225538972,+ "reasonstobecheerful.world": 0.9195521506144941,+ "thonyc.wordpress.com": 0.9195493663014705,+ "x264dev.multimedia.cx": 0.9195307216192476,+ "www.androidpolice.com": 0.9194976713702179,+ "www.ribbonfarm.com": 0.919494859687766,+ "workplace.stackexchange.com": 0.9194815293298146,+ "jcs.org": 0.9194811525194942,+ "www.codusoperandi.com": 0.9194731766129004,+ "behind-the-enemy-lines.blogspot.com": 0.9194452174142858,+ "peternixey.com": 0.9194135955492282,+ "www.theplayerstribune.com": 0.9194126818562519,+ "basho.com": 0.9194014734604079,+ "www.curiousgnu.com": 0.9193682460515379,+ "m.imgur.com": 0.9193616517608175,+ "gafferongames.com": 0.9193033055945021,+ "www.idontplaydarts.com": 0.9192963504549827,+ "www.forrestthewoods.com": 0.9192741785228133,+ "earthquake.usgs.gov": 0.9192641098431819,+ "blog.bitpay.com": 0.9192625409487661,+ "www.brookings.edu": 0.9192507068377601,+ "18f.gsa.gov": 0.9192284715908775,+ "www.minecraftforum.net": 0.9192112937034953,+ "www.lyft.com": 0.9191683431386107,+ "hhvm.com": 0.9191453265158583,+ "blog.wolfire.com": 0.9191450229699666,+ "www.usatoday.com": 0.9191347594941367,+ "nbviewer.ipython.org": 0.9191131949298217,+ "reddit.com": 0.9190800965203674,+ "www.eenewseurope.com": 0.9190688625693296,+ "www.smh.com.au": 0.9190104812043022,+ "www.figma.com": 0.9189780967661552,+ "www.yubico.com": 0.9189671558387075,+ "larrysanger.org": 0.9189574986097159,+ "www.cs.berkeley.edu": 0.9189369697206728,+ "www.qualys.com": 0.9188953942561601,+ "www.charlieharrington.com": 0.9188769529436779,+ "casual-effects.com": 0.91886846135451,+ "alexkrupp.typepad.com": 0.9188063903443455,+ "blog.macromates.com": 0.9187968191722226,+ "beza1e1.tuxen.de": 0.9187934225268114,+ "www.marketwatch.com": 0.9187035877780302,+ "dragan.rocks": 0.9186972641760346,+ "scottmeyers.blogspot.com": 0.9186922599750114,+ "brson.github.io": 0.9186844191523297,+ "kevq.uk": 0.9186712741984764,+ "tuhdo.github.io": 0.9186408903885764,+ "blog.ploeh.dk": 0.918640800603575,+ "www.spoon-tamago.com": 0.918580162432909,+ "www.maizure.org": 0.9185797770662466,+ "californiaglobe.com": 0.9185558354324097,+ "yarchive.net": 0.9185157064745997,+ "helpx.adobe.com": 0.918505551895513,+ "sourceforge.net": 0.9184879275921072,+ "gail.com": 0.9184873949579831,+ "snikt.net": 0.9184873949579831,+ "blog.plaid.com": 0.9184773866383011,+ "ebb.org": 0.9184174897915893,+ "insomnia.rest": 0.9184173422143894,+ "www.biorxiv.org": 0.9184172443974019,+ "cmdrtaco.net": 0.91840628090816,+ "tom7.org": 0.9183916897269802,+ "taler.net": 0.9183916897269802,+ "docs.rs": 0.9183858344184437,+ "www.ioccc.org": 0.9183852068279685,+ "sites.google.com": 0.9183471189460477,+ "domainnamewire.com": 0.918330600390645,+ "dafoster.net": 0.9183054021034863,+ "spreadsheets.google.com": 0.9182933889184417,+ "www.abc.net.au": 0.9182274259544088,+ "www.boringcompany.com": 0.918212478920742,+ "www.bankofengland.co.uk": 0.9181974227639322,+ "blog.ntpsec.org": 0.9181951887885417,+ "huyenchip.com": 0.9181777870982438,+ "pganalyze.com": 0.9181448902734191,+ "startupschool.org": 0.9181360100179967,+ "bl.ocks.org": 0.9180840436489085,+ "codeamigo.dev": 0.9180588577925552,+ "www.lesswrong.com": 0.9179747514962872,+ "blog.x.company": 0.9179518350159086,+ "inews.co.uk": 0.9179395572482176,+ "www.talyarkoni.org": 0.9179286455845682,+ "amazon.com": 0.917918074063271,+ "pijul.org": 0.9179029713542893,+ "www.thegatesnotes.com": 0.917896287388944,+ "blogs.reuters.com": 0.9178784513377969,+ "bartongeorge.io": 0.9178137271960294,+ "arslan.io": 0.9178051685037625,+ "bytecodealliance.org": 0.917754880834866,+ "journal.paul.querna.org": 0.9177358613483362,+ "jasonlbaptiste.com": 0.9177192783299879,+ "www.sharelatex.com": 0.9176170787075834,+ "www.jessesquires.com": 0.9176086328709556,+ "iquilezles.org": 0.9175867575102823,+ "yeokhengmeng.com": 0.9175790835471582,+ "software.rajivprab.com": 0.9175657081772484,+ "news.utexas.edu": 0.917544773761157,+ "www.fbi.gov": 0.9175354761400789,+ "sirupsen.com": 0.9175322740193886,+ "recode.net": 0.9175305114277775,+ "www.studyfinds.org": 0.9175005975481785,+ "rdegges.com": 0.9174949072553962,+ "cemerick.com": 0.9174490483219848,+ "cr.openjdk.java.net": 0.9174209932539029,+ "mapbox.com": 0.9174119653310046,+ "www.lihaoyi.com": 0.917356761960145,+ "www.ams.org": 0.9173555467413949,+ "golem.ph.utexas.edu": 0.9173444265784106,+ "www.oilshell.org": 0.9173296046923102,+ "www.darktable.org": 0.9173295566709162,+ "www.madinamerica.com": 0.9173266085137967,+ "blog.scrapinghub.com": 0.9173116907321415,+ "www.sheshbabu.com": 0.9172725950029555,+ "nickdrozd.github.io": 0.9172699096194682,+ "diveintomark.org": 0.9172600663351143,+ "modernfarmer.com": 0.9172547499030632,+ "news.ucsc.edu": 0.9172085056879004,+ "blogs.hbr.org": 0.9171843582580473,+ "opinionator.blogs.nytimes.com": 0.9171826095986618,+ "www.rtl-sdr.com": 0.9171698455400066,+ "byuu.org": 0.9171665984622892,+ "bugzilla.gnome.org": 0.9171648163962425,+ "www.sfwa.org": 0.9171648163962425,+ "www.spinellis.gr": 0.9171319512518645,+ "0pointer.net": 0.9171049662814222,+ "eff.org": 0.917104278613718,+ "learnxinyminutes.com": 0.9170978082207938,+ "9to5google.com": 0.917097241243007,+ "www.cs.columbia.edu": 0.9170952792848854,+ "www.fourmilab.ch": 0.9170935424089711,+ "hynek.me": 0.9170881237147662,+ "www.sarahmei.com": 0.9170796812591611,+ "blog.piekniewski.info": 0.9170369069517826,+ "cb.vu": 0.9169587328599824,+ "math.stackexchange.com": 0.916937205478264,+ "news.bloomberglaw.com": 0.9169353599665077,+ "blog.darknedgy.net": 0.9169312054506662,+ "telegra.ph": 0.9168505250691076,+ "sniperinmahwah.wordpress.com": 0.9168254460707557,+ "lists.swift.org": 0.9167939569922435,+ "adit.io": 0.9167855816882253,+ "steamcommunity.com": 0.9167584498934136,+ "www.buzzfeed.com": 0.9167426632611669,+ "www.privacyinternational.org": 0.9167293554781794,+ "www.thurrott.com": 0.9167249467906359,+ "www.cbsnews.com": 0.9167194231229859,+ "socket.io": 0.9166902296520829,+ "maryrosecook.com": 0.9166809726360262,+ "tomayko.com": 0.9166652528198918,+ "if50.substack.com": 0.9166328442343356,+ "phenomena.nationalgeographic.com": 0.9166322627474873,+ "www.netmeister.org": 0.9166258322005095,+ "transition.fcc.gov": 0.9165981621015247,+ "playground.tensorflow.org": 0.9165966237016168,+ "www.macworld.com": 0.916575278474562,+ "www.mahdiyusuf.com": 0.9165626020992179,+ "nplusonemag.com": 0.9165533544728419,+ "astralcodexten.substack.com": 0.9165406144271586,+ "rjlipton.wordpress.com": 0.9164585931916936,+ "kunle.app": 0.9164517870748075,+ "cheatsheetseries.owasp.org": 0.9164513350559862,+ "blogs.janestreet.com": 0.916405195738336,+ "www.chromium.org": 0.9163955502793474,+ "aht.seriouseats.com": 0.9163820584014595,+ "arcadeblogger.com": 0.9162907826609931,+ "lock.cmpxchg8b.com": 0.9162763903656581,+ "codewithoutrules.com": 0.9162515056697728,+ "blog.roboflow.ai": 0.9162348877374784,+ "jolla.com": 0.9162125492359456,+ "philipwalton.github.io": 0.9162125492359456,+ "www.textfiles.com": 0.9161483132429212,+ "landshark.io": 0.9160843328162562,+ "www.syfy.com": 0.9160823277886534,+ "www.a1k0n.net": 0.9160533176276322,+ "thirtybees.com": 0.916017316017316,+ "wp.josh.com": 0.9159844747688354,+ "community.rapid7.com": 0.9159729296552184,+ "variety.com": 0.9159645406845424,+ "www.petekeen.net": 0.9159458498387062,+ "pragprog.com": 0.9158994769642369,+ "addons.mozilla.org": 0.9158845258968069,+ "blog.abevoelker.com": 0.9158703069575684,+ "uxdesign.cc": 0.915853392301296,+ "tomblomfield.com": 0.91585125248835,+ "www.platformer.news": 0.9158262569189228,+ "airbnb.design": 0.9158145365560538,+ "www.php.net": 0.9157901084713228,+ "blakemasters.tumblr.com": 0.915774364872077,+ "forum.bitcoin.org": 0.9157050312564461,+ "victorzhou.com": 0.9156996812390578,+ "www.newcriterion.com": 0.9156866313823816,+ "blog.sagemath.com": 0.9156636337212226,+ "security-center.intel.com": 0.9156401908652926,+ "secret.club": 0.9156139932226358,+ "codebutler.com": 0.9156072900794726,+ "gfx-rs.github.io": 0.9155911821045295,+ "open.blogs.nytimes.com": 0.9155276050596644,+ "lambda-the-ultimate.org": 0.9155112419708785,+ "www.universetoday.com": 0.9155071362428421,+ "vc.gg": 0.9155052264808362,+ "www.bostonglobe.com": 0.9154711894014951,+ "slides.com": 0.915438865746552,+ "openid.net": 0.9154218308800682,+ "sizovs.net": 0.9154120001317235,+ "minimalmac.com": 0.9154065991271528,+ "pandodaily.com": 0.9153369387847924,+ "www.ft.com": 0.9153363776443395,+ "money.cnn.com": 0.9153197950917878,+ "blogs.technet.com": 0.9152916278406317,+ "tynan.com": 0.9152784227275973,+ "antipolygraph.org": 0.9152601338015678,+ "www.bing.com": 0.9152555063347714,+ "dev.lemmy.ml": 0.9152501219202281,+ "ai.facebook.com": 0.9151989015144473,+ "www.spectator.co.uk": 0.9151910833402672,+ "bollu.github.io": 0.915178606993226,+ "cognitivemedium.com": 0.9151678416414023,+ "www.nih.gov": 0.9151430151901143,+ "www.cyanogenmod.org": 0.9151094600863039,+ "codepen.io": 0.9150991029481277,+ "www.media.mit.edu": 0.9150898590392416,+ "ianthehenry.com": 0.9150718483883374,+ "sloanreview.mit.edu": 0.9150432572484367,+ "speakerdeck.com": 0.9150398484887767,+ "en.lichess.org": 0.9150022514278866,+ "www.datagenetics.com": 0.9150012436762652,+ "people.kernel.org": 0.9149910445768656,+ "ma.tt": 0.9149514110201404,+ "people.freebsd.org": 0.9149417112190068,+ "measuringmeasures.com": 0.9149271022386467,+ "www.catb.org": 0.9149270125747468,+ "defn.io": 0.9148816198946629,+ "www.fastly.com": 0.914873343624634,+ "www.eetimes.com": 0.9148540001117531,+ "www.newrepublic.com": 0.9146889391211901,+ "www.chicagomaroon.com": 0.9146877748460862,+ "blog.openbloc.fr": 0.9146877748460862,+ "blog.moertel.com": 0.9146704350942687,+ "nathanbarry.com": 0.9146551473361769,+ "bombich.com": 0.914654069788301,+ "mojang.com": 0.9146416740750689,+ "www.pgbovine.net": 0.914639262393428,+ "www.bicycling.com": 0.9146271639157105,+ "www.zemanta.com": 0.9146271639157105,+ "www.perell.com": 0.914609307465025,+ "mjtsai.com": 0.9145962250738603,+ "www.softwarebyrob.com": 0.9145691937602046,+ "fusion.net": 0.914564049263131,+ "blogs.forbes.com": 0.914542164424382,+ "beta.observablehq.com": 0.9145307006213452,+ "www.zedshaw.com": 0.9145061280161968,+ "apps.npr.org": 0.9144928315823566,+ "lists.gnupg.org": 0.9144675565289525,+ "gleam.run": 0.9144324984811291,+ "www.thestar.com": 0.9143957132161558,+ "www.militarytimes.com": 0.9143764543930346,+ "blog.memsql.com": 0.9143635601136089,+ "cr.yp.to": 0.9143445920251045,+ "thepointsguy.com": 0.9143303279494647,+ "dinosaurpictures.org": 0.9143080414361231,+ "quillette.com": 0.9142948614738509,+ "ticki.github.io": 0.914276003611421,+ "neil.fraser.name": 0.9142481924113248,+ "www3.nhk.or.jp": 0.9142479994563674,+ "www.muppetlabs.com": 0.9142412765407236,+ "blogs.gnome.org": 0.9142360167665817,+ "readonlymemory.vg": 0.9142351900972591,+ "news.osu.edu": 0.9142251289181204,+ "www.cgl.uwaterloo.ca": 0.9142062865582451,+ "www.fossil-scm.org": 0.9142014076198545,+ "hub.jhu.edu": 0.9141963295065768,+ "raw.github.com": 0.9141853922096216,+ "www.sebastianmarshall.com": 0.9141316785617728,+ "physicstoday.scitation.org": 0.9141308856828069,+ "www.edge.org": 0.9140771607780698,+ "blog.bugreplay.com": 0.914047971434993,+ "www.medrxiv.org": 0.9139991128038881,+ "permalink.gmane.org": 0.9139842285208853,+ "www.notechmagazine.com": 0.9139648152317646,+ "blog.ipfs.io": 0.9139402591654684,+ "ninjasandrobots.com": 0.9139402119313542,+ "blog.johnnovak.net": 0.9139144009903326,+ "tldr.sh": 0.9138931911912991,+ "internals.rust-lang.org": 0.9138888662706784,+ "formidable.com": 0.9138769104260843,+ "thenounproject.com": 0.9138746646560476,+ "blog.earbits.com": 0.9138048995152518,+ "gamehistory.org": 0.9137927637033101,+ "www.influxdata.com": 0.913761562076886,+ "m.washingtonpost.com": 0.9137051021098685,+ "pythonspeed.com": 0.9137019640744012,+ "observer.com": 0.9136945429590685,+ "mastodon.social": 0.9136829141356647,+ "whatever.scalzi.com": 0.9136722362354969,+ "rietta.com": 0.9136487147921646,+ "cushionapp.com": 0.9136267087023814,+ "www.cantorsparadise.com": 0.9135854914860095,+ "www.intelligence.senate.gov": 0.9135819523129652,+ "blog.nullspace.io": 0.9134635980748237,+ "keleshev.com": 0.9134557014620313,+ "www.freethink.com": 0.913438665510595,+ "singularityhub.com": 0.9134225553478318,+ "www.greenteapress.com": 0.9134066096671126,+ "hazlitt.net": 0.9134026214327062,+ "www.zainrizvi.io": 0.913399178678123,+ "www.welcometothejungle.co": 0.9133545417162321,+ "joeduffyblog.com": 0.9133528454968176,+ "tbbuck.com": 0.913350699282715,+ "blog.fawny.org": 0.9133445360688772,+ "cyclingtips.com": 0.9133347976741121,+ "www.f-secure.com": 0.913329909915946,+ "www.tweag.io": 0.9133006601895646,+ "edri.org": 0.9132989780194611,+ "brooker.co.za": 0.913269877818758,+ "learnyouahaskell.com": 0.9132658585002156,+ "dmitrysoshnikov.com": 0.9132544931220842,+ "frankgroeneveld.nl": 0.9132379248658319,+ "inconvergent.net": 0.9132264356820329,+ "kennethreitz.org": 0.913224092229413,+ "editorconfig.org": 0.9131943780375904,+ "atom.io": 0.9131691418622392,+ "mattgemmell.com": 0.9131626341114434,+ "www.fastcoexist.com": 0.9131427628529503,+ "atap.google.com": 0.9130249616995455,+ "esbuild.github.io": 0.9130044843049328,+ "blog.koehntopp.info": 0.9129670374057173,+ "europa.eu": 0.9129663426547358,+ "htmx.org": 0.912955421768738,+ "www.markozivanovic.com": 0.912917338189758,+ "number-none.com": 0.9129109262881329,+ "sequoia-pgp.org": 0.9128836629372516,+ "tanelpoder.com": 0.9128836629372516,+ "www.nvidia.com": 0.9128613184383916,+ "www.prospectmagazine.co.uk": 0.9128286921650719,+ "bouk.co": 0.9128166963356691,+ "www.espn.com": 0.9127399070748246,+ "www.courtlistener.com": 0.912716064588026,+ "lav.io": 0.9126813150141094,+ "landave.io": 0.9126647018297601,+ "blog.phusion.nl": 0.912650575320633,+ "thezvi.wordpress.com": 0.9126499253271542,+ "nickjanetakis.com": 0.9126192502614521,+ "stateofjs.com": 0.9126077344903017,+ "hakim.se": 0.9126077344903017,+ "blog.datomic.com": 0.9125894031840723,+ "docs.racket-lang.org": 0.912488235007798,+ "jamie-wong.com": 0.9124775693621691,+ "mocko.org.uk": 0.9124699553919461,+ "www.telegraph.co.uk": 0.9124491411165716,+ "ag.ny.gov": 0.9124354337525871,+ "billwadge.wordpress.com": 0.9124293382023613,+ "www.matasano.com": 0.9124000351892007,+ "okigiveup.net": 0.9123862469871238,+ "logicmag.io": 0.9123859846898301,+ "andrewchen.co": 0.9123448304091328,+ "www.airpair.com": 0.9123437098236648,+ "cowlark.com": 0.9123304746459454,+ "www.pbs.org": 0.9123204159144807,+ "semianalysis.com": 0.912313946546377,+ "kk.org": 0.9123097221722443,+ "log.nadim.cc": 0.9122965641952984,+ "www.collaborativefund.com": 0.9122890867111917,+ "www.ventusky.com": 0.912268380036006,+ "www.nintendo.com": 0.9122375993317341,+ "www.worldofindie.co.uk": 0.9122336940304385,+ "jeroenjanssens.com": 0.9122336940304385,+ "libreboot.org": 0.9122148556890368,+ "blog.roboflow.com": 0.9121810251169891,+ "www.msn.com": 0.9121613167344725,+ "debugger.medium.com": 0.9121472531001502,+ "www.pagetable.com": 0.9121460025296508,+ "www.eejournal.com": 0.9121137219392826,+ "blog.sanctum.geek.nz": 0.91211049118521,+ "blog.elcomsoft.com": 0.9120714505588577,+ "www.judegomila.com": 0.9120548967590474,+ "cirw.in": 0.9119924579917503,+ "corner.squareup.com": 0.9119829497403208,+ "flightfox.com": 0.911934167943058,+ "tip.golang.org": 0.9119192289295014,+ "www.nzherald.co.nz": 0.9119181580628049,+ "www.wezm.net": 0.911916042600115,+ "crontab.guru": 0.9119103202923069,+ "degoes.net": 0.9119022003821793,+ "investors.modernatx.com": 0.9118967098234642,+ "linustechtips.com": 0.9118886368935806,+ "blog.tanyakhovanova.com": 0.911882662223509,+ "amandapeyton.com": 0.9118468718524512,+ "nypost.com": 0.9118217286764665,+ "forumserver.twoplustwo.com": 0.9118191670930907,+ "belkadan.com": 0.9117907816852344,+ "nicolasgallagher.com": 0.9117507194110998,+ "leowid.com": 0.9117097398796832,+ "blog.izs.me": 0.9117080886266495,+ "www.electrospaces.net": 0.9117024618532766,+ "granta.com": 0.9116811248127611,+ "blog.darksky.net": 0.911673049778484,+ "lettersofnote.com": 0.9116636714221488,+ "www.webkit.org": 0.9115845605402175,+ "shipilev.net": 0.9115598314538631,+ "www.nanex.net": 0.911539637720508,+ "blog.whatsapp.com": 0.9115335907580295,+ "leetcode.com": 0.9115051248737622,+ "www.numfocus.org": 0.9115022081675324,+ "onlinelibrary.wiley.com": 0.9114852476099333,+ "www.tomscott.com": 0.9114829485455537,+ "trmm.net": 0.9114358511601164,+ "www.marketplace.org": 0.9113899909762714,+ "www.codecademy.com": 0.911385022367728,+ "news.vice.com": 0.9113079103997874,+ "bit-player.org": 0.9112589244732427,+ "postmake.io": 0.9112553086495423,+ "journal.standardnotes.org": 0.9112172605590297,+ "semantic-ui.com": 0.9111938765485481,+ "blog.dshr.org": 0.9111782499363091,+ "www.rogerebert.com": 0.9111691070786543,+ "web.dev": 0.9111397506487405,+ "bravenewgeek.com": 0.9111366238548788,+ "benjiweber.co.uk": 0.9111133306489915,+ "www.theinformation.com": 0.9111027630703883,+ "jmmv.dev": 0.9110762745053466,+ "www.nycresistor.com": 0.9110654315146017,+ "bloomberg.github.io": 0.9110576284941153,+ "glench.com": 0.9110421711549385,+ "blog.kicksend.com": 0.9109887018713544,+ "www.godaddy.com": 0.9109502513393147,+ "neverworkintheory.org": 0.910947601730565,+ "thebillfold.com": 0.9109433900278253,+ "www.superhighway98.com": 0.9109274563820018,+ "tylerneylon.com": 0.9109274563820018,+ "gankro.github.io": 0.9109257416809919,+ "thetruesize.com": 0.9108778939911146,+ "zipcpu.com": 0.9108728406864637,+ "www.cs.uni.edu": 0.910867930832846,+ "www.whitehouse.gov": 0.9108673306488233,+ "bradleytaunt.com": 0.9108617254932104,+ "scripting.com": 0.9108577372151384,+ "www.cybertec-postgresql.com": 0.9108466966632626,+ "www.wcvb.com": 0.9107985466943099,+ "www.dslreports.com": 0.9107964183891935,+ "justi.cz": 0.9107667989469016,+ "finance.yahoo.com": 0.9107655094145327,+ "blog.data.gov.sg": 0.910764849456426,+ "www.gutenberg.org": 0.9107167501810286,+ "www.sciencedaily.com": 0.9106989174929755,+ "blog.airbnb.com": 0.9106859675193685,+ "www.sendwithus.com": 0.9106524760338102,+ "chrismm.com": 0.9106400698754001,+ "stevecorona.com": 0.9106182479757787,+ "swanson.github.com": 0.910602900714108,+ "eager.io": 0.9105915511196,+ "www.insidesources.com": 0.9105882590524582,+ "chrisseaton.com": 0.9105861690903986,+ "cstheory.stackexchange.com": 0.9105756115852348,+ "www.behind-the-enemy-lines.com": 0.9105542219435652,+ "www.commondreams.org": 0.9105378564647107,+ "tech.marksblogg.com": 0.91050910005871,+ "blog.bitquabit.com": 0.9104538788044407,+ "utkusen.com": 0.9104251525129977,+ "lab.hakim.se": 0.9104228721342874,+ "calacanis.com": 0.9104018184523797,+ "blog.metaobject.com": 0.9103873178112856,+ "blogs.discovermagazine.com": 0.9103541727829514,+ "flutter.io": 0.9103411596660869,+ "www.chriskrycho.com": 0.9103411596660869,+ "www.washingtonmonthly.com": 0.9103284229318426,+ "nora.codes": 0.9103242591900224,+ "blog.professorbeekums.com": 0.9103190381466665,+ "yro.slashdot.org": 0.9103086060368418,+ "noncombatant.org": 0.9102799785330806,+ "www.irrlicht3d.org": 0.9102781052287677,+ "www.analog.com": 0.9102635806004643,+ "bret.appspot.com": 0.9102570367702293,+ "byuu.net": 0.9102490436100149,+ "learntemail.sam.today": 0.9102460058415499,+ "discuss.joelonsoftware.com": 0.9102271772174491,+ "news.samsung.com": 0.9102218695714285,+ "hamberg.no": 0.9101858671649673,+ "culture.pl": 0.9101776992266171,+ "godbolt.org": 0.9101759214909476,+ "www.usgamer.net": 0.9101501554941474,+ "openmymind.net": 0.9101212855439798,+ "www.pgcon.org": 0.9101151487041538,+ "blog.insightdatascience.com": 0.9101089594835049,+ "blog.mapbox.com": 0.9100815042120244,+ "heap.io": 0.9100756276392993,+ "www.openttd.org": 0.910035809777446,+ "www.gazehawk.com": 0.9100190649304827,+ "blog.headius.com": 0.9100084850499096,+ "jquerymobile.com": 0.9099865878820667,+ "strlen.com": 0.9099808992547239,+ "bellmar.medium.com": 0.9099735750567878,+ "designfestival.com": 0.9099647154391645,+ "blog.kowalczyk.info": 0.909946343995861,+ "natewienert.com": 0.9099163091164866,+ "blog.iron.io": 0.9099140278683325,+ "www.optaplanner.org": 0.9098736627179512,+ "blog.directededge.com": 0.9098673466271318,+ "www.acm.org": 0.9098204575391435,+ "postgres.heroku.com": 0.9098008938162723,+ "netflix.github.io": 0.9097922949542527,+ "www.upguard.com": 0.9097914092921688,+ "dopeboy.github.io": 0.9097690974580366,+ "www.panix.com": 0.909767441860465,+ "www.arangodb.com": 0.9097488818296668,+ "www.ctvnews.ca": 0.9097251357478893,+ "superorganizers.substack.com": 0.9097041628130547,+ "tips.ariyh.com": 0.9096731009765122,+ "henrikwarne.com": 0.9096514841360387,+ "web.media.mit.edu": 0.909634017448133,+ "www.math.columbia.edu": 0.9096124004726523,+ "sorbet.org": 0.9095992544268405,+ "www.whattofix.com": 0.9095812789302404,+ "lukeplant.me.uk": 0.90953747228724,+ "blog.quora.com": 0.9095278802240517,+ "areallybadidea.com": 0.909495363691211,+ "www.gsb.stanford.edu": 0.9094535928364683,+ "rednuht.org": 0.9094069027053294,+ "cappuccino.org": 0.9093755871303836,+ "blog.wallaroolabs.com": 0.9093612256561168,+ "resobscura.blogspot.com": 0.9093403026878701,+ "story.californiasunday.com": 0.9093105225503313,+ "xato.net": 0.9092732945858745,+ "thinkfaster.co": 0.9092409956171542,+ "blog.algorithmia.com": 0.9091584066166702,+ "n-o-d-e.net": 0.9091578728445029,+ "www.talkingquickly.co.uk": 0.9091487127436086,+ "blamcast.net": 0.9091487127436086,+ "www.opendemocracy.net": 0.9091030079275115,+ "www.pixelbeat.org": 0.9091017229185366,+ "blog.2ndquadrant.com": 0.9090827165871289,+ "klinger.io": 0.9090715471566316,+ "invisibleup.com": 0.9090623220718969,+ "www.mattgreer.org": 0.909028070014661,+ "www.georgehart.com": 0.9090164458580612,+ "gsuiteupdates.googleblog.com": 0.9090066640642879,+ "programming.guide": 0.9090056285178236,+ "www.revealnews.org": 0.9089822305000187,+ "www.android.com": 0.9089431208516167,+ "renderman.pixar.com": 0.9089238890253597,+ "swtch.com": 0.9088645421539494,+ "akkartik.name": 0.9088447989502915,+ "ma.ttias.be": 0.9088072198608288,+ "blog.freecodecamp.com": 0.9087962710873405,+ "cognitiontoday.com": 0.9087409016865039,+ "data.triplebyte.com": 0.9087226720990824,+ "unix4lyfe.org": 0.9087151101306639,+ "coconut-lang.org": 0.9087151101306639,+ "discourse.nixos.org": 0.908662900188324,+ "macwright.org": 0.908628352520185,+ "www.varnish-cache.org": 0.9086150127257842,+ "blog.lyft.com": 0.9085807144025437,+ "martinrue.com": 0.9085670862102617,+ "bittersoutherner.com": 0.9085029987755043,+ "www.somebits.com": 0.9084860999243902,+ "certsimple.com": 0.9084684115974558,+ "www.calvertjournal.com": 0.9084635529608529,+ "www.pentestpartners.com": 0.9084628705092087,+ "wiki.gentoo.org": 0.9084338073440062,+ "00f.net": 0.9083970094532444,+ "www.weforum.org": 0.9083913811504614,+ "evanmiller.org": 0.9083652366260793,+ "www.geekwire.com": 0.9083231743190484,+ "news.yahoo.com": 0.9083103718085543,+ "books.google.com": 0.9082816718436784,+ "georgestocker.com": 0.9082719861623312,+ "www.yacoset.com": 0.9082542163067842,+ "spencerfry.com": 0.9082262546419613,+ "citusdata.com": 0.9082125231004685,+ "www.thenewatlantis.com": 0.9081731363479266,+ "j2kun.svbtle.com": 0.9081723260580993,+ "www.bothsidesofthetable.com": 0.9081408393460154,+ "www.thetimes.co.uk": 0.9081193478982206,+ "corecursive.com": 0.90809594719532,+ "www.wildml.com": 0.9080846001869393,+ "smallcultfollowing.com": 0.9080827648147168,+ "www.theartnewspaper.com": 0.9080723328672826,+ "cdixon.posterous.com": 0.9080713598958169,+ "www.readwriteweb.com": 0.9080580806277859,+ "www.hsph.harvard.edu": 0.9080327405123425,+ "www.state.gov": 0.908024664450505,+ "planscope.io": 0.907995411751062,+ "www.allthingsdistributed.com": 0.9079938413302127,+ "www.boston.com": 0.9079342359331115,+ "blog.kentcdodds.com": 0.9079295161962991,+ "www.brianstorti.com": 0.9079295161962991,+ "scala-lang.org": 0.9079245106804908,+ "austinvernon.eth.link": 0.9078822412155745,+ "pindancing.blogspot.com": 0.9078822412155745,+ "www.residentadvisor.net": 0.9078629004665392,+ "www.documentcloud.org": 0.9078579855885078,+ "googledocs.blogspot.com": 0.9078489828529417,+ "www.cdc.gov": 0.9078393365891005,+ "www.howardism.org": 0.9078286053408602,+ "fsharpforfunandprofit.com": 0.9078054396581823,+ "semiaccurate.com": 0.907755821837259,+ "www.hetzner.com": 0.9077473776003605,+ "aiweirdness.com": 0.9077065010884221,+ "www.bit-101.com": 0.9077018983499333,+ "www.41latitude.com": 0.9077018983499333,+ "www.amnh.org": 0.9076987359242361,+ "infolab.stanford.edu": 0.9076647094055436,+ "stories.californiasunday.com": 0.9076472906657365,+ "pxlnv.com": 0.9076015494709667,+ "batsov.com": 0.9075988217886798,+ "www.gamedev.net": 0.9075987374715156,+ "diff.wikimedia.org": 0.9075899793873731,+ "rust-analyzer.github.io": 0.9075741551452703,+ "www.mondaynote.com": 0.9075302424153898,+ "www.gatesfoundation.org": 0.9075167456992206,+ "meta.ath0.com": 0.9075048785901592,+ "radicle.xyz": 0.9074969769110395,+ "www.gigamonkeys.com": 0.9074847578759488,+ "thetyee.ca": 0.9074067773987987,+ "www.ucdavis.edu": 0.9073906291556996,+ "annehelen.substack.com": 0.9073906291556995,+ "www.undeadly.org": 0.9073802859341632,+ "teenage.engineering": 0.9073802859341632,+ "approximatelycorrect.com": 0.9073547174968317,+ "console.dev": 0.9073543457497613,+ "www.qemu.org": 0.9073039659002148,+ "chadaustin.me": 0.907285573667598,+ "javascript.info": 0.9072810118016286,+ "crawshaw.io": 0.9072713221321198,+ "anandtech.com": 0.9072582150847296,+ "minireference.com": 0.9072555275680507,+ "0x00sec.org": 0.9072265870419588,+ "jtaby.com": 0.9072263761646095,+ "hackernoon.com": 0.9072112942336243,+ "aem1k.com": 0.9071770334928229,+ "www.theatlanticwire.com": 0.9071768956517269,+ "voice.mozilla.org": 0.9071764184712818,+ "www.tennessean.com": 0.9071055766821053,+ "graphicallinearalgebra.net": 0.9070506542876967,+ "jpmens.net": 0.9070445763259152,+ "www.eugenewei.com": 0.907025376171475,+ "blog.packagecloud.io": 0.9070193601765509,+ "zef.me": 0.9070189250200639,+ "blog.makezine.com": 0.907016130343466,+ "yannesposito.com": 0.9069508537217187,+ "ubuntu.com": 0.9069426319310011,+ "apolloinrealtime.org": 0.9069028892973577,+ "leontrolski.github.io": 0.9069028892973576,+ "divinations.substack.com": 0.9068786540871128,+ "nicole.express": 0.9068478175005559,+ "www.cnet.com": 0.9068461959910697,+ "docs.browserless.io": 0.9068163026535472,+ "joshondesign.com": 0.906770998121308,+ "humanparts.medium.com": 0.906760680978119,+ "aviationweek.com": 0.9067465504035637,+ "philip.greenspun.com": 0.9067439389898158,+ "huonw.github.io": 0.9066784039500629,+ "ariadne.space": 0.90667782006653,+ "blog.sia.tech": 0.9066295330975407,+ "www.cato.org": 0.9065844637081601,+ "blog.hipchat.com": 0.9065819832684133,+ "ruben.verborgh.org": 0.9065777308846613,+ "www.straightdope.com": 0.9064957391706991,+ "www.joshbarczak.com": 0.9064865112228077,+ "www.aarongreenspan.com": 0.9064448754833254,+ "james-iry.blogspot.com": 0.9064382639201024,+ "changelog.com": 0.9064361162234437,+ "jdlm.info": 0.9064121095346035,+ "www.csail.mit.edu": 0.9064063607235981,+ "blog.cleancoder.com": 0.9063898821262703,+ "kyrobeshay.com": 0.9063750617475119,+ "js1k.com": 0.9063646805868381,+ "socketsite.com": 0.9063204907590037,+ "mobile.reuters.com": 0.9063074578535042,+ "www.spinics.net": 0.9062822710883404,+ "www.h-online.com": 0.9062819868634641,+ "humanwhocodes.com": 0.9062381797427127,+ "angular.io": 0.9062269960472951,+ "www.digitalcameraworld.com": 0.906205656831627,+ "www.amd.com": 0.9061918994722282,+ "noahpinion.substack.com": 0.9061897254478186,+ "scienceblogs.com": 0.9061724154956764,+ "forge.medium.com": 0.9061379433537395,+ "folklore.org": 0.9061180195674341,+ "www.jonathan-cook.net": 0.906117767771791,+ "randomwire.com": 0.9060987415295255,+ "blog.crunchydata.com": 0.9060970567080769,+ "www.timeshighereducation.co.uk": 0.9060939645760167,+ "gds.blog.gov.uk": 0.9060670828649058,+ "www.mihaileric.com": 0.9060240156895187,+ "seatgeek.com": 0.9059982211064962,+ "www.fanfiction.net": 0.9059866760601158,+ "msrc-blog.microsoft.com": 0.9059631330906113,+ "thecounter.org": 0.9059275361638426,+ "www.haproxy.com": 0.905913947863991,+ "movingfulcrum.com": 0.9058292135077338,+ "onstartups.com": 0.9057830668516696,+ "noyb.eu": 0.9057682181268583,+ "millionshort.com": 0.9057645582241304,+ "blog.sketchapp.com": 0.905760488137004,+ "www.tillett.info": 0.9057526849990746,+ "community.ui.com": 0.9057461537102033,+ "adnauseam.io": 0.9057337220602527,+ "intoli.com": 0.9056718941128913,+ "www.jsoftware.com": 0.9056617340683936,+ "www.oreilly.com": 0.9056506837308099,+ "www.themacro.com": 0.905632500892492,+ "www.chicagotribune.com": 0.9055936474876807,+ "developer.squareup.com": 0.9055714474003402,+ "pixelambacht.nl": 0.9055660472352755,+ "lambdaschool.com": 0.9055457567735196,+ "consumerist.com": 0.9055167142071894,+ "lisp-univ-etc.blogspot.com": 0.905513555501623,+ "www.reaktor.com": 0.9055111900863794,+ "mars.jpl.nasa.gov": 0.9054761782517211,+ "blog.rescuetime.com": 0.9054099923733353,+ "dustycloud.org": 0.9053737646709764,+ "the.exa.website": 0.9053731049780873,+ "www.lareviewofbooks.org": 0.905357669166342,+ "adventures.michaelfbryan.com": 0.9053432603320545,+ "bigtechnology.substack.com": 0.9053275639886895,+ "mashable.com": 0.9052624670823227,+ "blog.xamarin.com": 0.9052537100443511,+ "www.micahlerner.com": 0.905230894226323,+ "software.intel.com": 0.9052122500296891,+ "canonical.org": 0.9051821187623978,+ "www.scmp.com": 0.9051295599614926,+ "grantland.com": 0.9050923790278581,+ "metarabbit.wordpress.com": 0.9050923774541669,+ "education.github.com": 0.9050362272771562,+ "www.thelancet.com": 0.905021932673904,+ "www.nextplatform.com": 0.9050024178186049,+ "home-assistant.io": 0.904995020548573,+ "varnish-cache.org": 0.9049950205485728,+ "blog.netlab.360.com": 0.9049426199082281,+ "m.nautil.us": 0.9049309527012956,+ "www.wikileaks.org": 0.9049234922760648,+ "www.cell.com": 0.9049048617612142,+ "cheapskatesguide.org": 0.9048941298503079,+ "engineering.tumblr.com": 0.9048881466624363,+ "w3techs.com": 0.9048639649614062,+ "literaryreview.co.uk": 0.9048564834641546,+ "explorabl.es": 0.9048086359175662,+ "blog.felipe.rs": 0.9048079619547811,+ "bitsup.blogspot.com": 0.9047993533025517,+ "unherd.com": 0.9047865592498257,+ "garry.posterous.com": 0.9047577664612345,+ "worthdoingbadly.com": 0.9047131145041029,+ "rob.conery.io": 0.9046723461377625,+ "io9.com": 0.9046401640309223,+ "nolanlawson.com": 0.9046163101493089,+ "www.soundguys.com": 0.9046093349153457,+ "www.starterstory.com": 0.9045594173223862,+ "blog.jse.li": 0.9045467732931323,+ "knowingless.com": 0.9045071158772837,+ "decrypt.co": 0.9044787320123616,+ "www.euronews.com": 0.9044514611283094,+ "www.mattwarren.org": 0.9044367882909422,+ "www.nirsoft.net": 0.9044100097952366,+ "www.atlassian.com": 0.9043691389685379,+ "exploringjs.com": 0.9043580712164743,+ "www.recurse.com": 0.9043507328694445,+ "www.legacy.com": 0.9043392504930966,+ "docs.fcc.gov": 0.9043351544730283,+ "channel9.msdn.com": 0.904284603417082,+ "blog.ksplice.com": 0.9042701981747182,+ "www.valvesoftware.com": 0.9042649125194085,+ "blog.netbsd.org": 0.904253667752545,+ "hyper.sh": 0.9042360279323705,+ "blog.waleedkhan.name": 0.9042360279323705,+ "computerhistory.org": 0.904233366454177,+ "sugru.com": 0.9041590518924904,+ "feefighters.com": 0.9041509269810387,+ "dejanseo.com.au": 0.9040498577824194,+ "www.ianww.com": 0.9040340548815767,+ "blog.cr.yp.to": 0.9040072201081354,+ "trac.nginx.org": 0.9039603960396039,+ "kentnguyen.com": 0.903957880171432,+ "jonathanturley.org": 0.9039575565206787,+ "snyk.io": 0.9039222826692356,+ "radar.oreilly.com": 0.9038574383300398,+ "blog.sentry.io": 0.9037725052372675,+ "pluto.jhuapl.edu": 0.9037348670913641,+ "calendar.perfplanet.com": 0.903704307455864,+ "graphics.stanford.edu": 0.903685214892998,+ "singhkays.com": 0.9036579185992312,+ "tinyapps.org": 0.9036401962155587,+ "motherjones.com": 0.9036341732104931,+ "blog.evjang.com": 0.903604799270119,+ "www.lightbluetouchpaper.org": 0.9035449087212127,+ "blog.arduino.cc": 0.9035388791817426,+ "www.psychnewsdaily.com": 0.9035015745269135,+ "gnu.org": 0.9035002162125149,+ "pchiusano.github.io": 0.9034726569621144,+ "hypercritical.co": 0.9034345666188365,+ "0a.io": 0.9034135558240236,+ "www.erowid.org": 0.9034051526175023,+ "www.inklestudios.com": 0.9033800880640788,+ "blog.soylent.com": 0.9033111216571915,+ "www.fpcomplete.com": 0.9032736546035143,+ "dmitryfrank.com": 0.9032706083767721,+ "www.red-lang.org": 0.9032700232946759,+ "www.parallella.org": 0.9032534786832978,+ "habr.com": 0.903232673933909,+ "www.musictech.net": 0.9031936127744511,+ "josephwalla.com": 0.9031899305005244,+ "hsivonen.fi": 0.9031735375682458,+ "thoughtcatalog.com": 0.9031627630658119,+ "www.scylladb.com": 0.9031607915889202,+ "www.dreamhost.com": 0.9031378273716731,+ "www.stochasticlifestyle.com": 0.9031310549200046,+ "vaibhavsagar.com": 0.9031083985391525,+ "searchengineland.com": 0.9030654738013855,+ "www.nrk.no": 0.9030651285810807,+ "blog.app.net": 0.903029680517345,+ "mail.openjdk.java.net": 0.9030249968896539,+ "sciencebulletin.org": 0.9029854747384259,+ "www.paperplanes.de": 0.9029834819452879,+ "research.facebook.com": 0.9029781526945183,+ "jessenoller.com": 0.9029740692225192,+ "blog.hirelite.com": 0.9029584423609692,+ "www.bemmu.com": 0.9029304388261952,+ "www.cam.ac.uk": 0.9029214208630774,+ "adguard.com": 0.9028940843182978,+ "www.protocol.com": 0.9028937484894913,+ "redislabs.com": 0.9028875063757023,+ "www.coding2learn.org": 0.9028312810037816,+ "www.tonywright.com": 0.9027993696652322,+ "writing.kemitchell.com": 0.902794255100101,+ "support.microsoft.com": 0.9027883582257822,+ "voxeu.org": 0.9027732002536809,+ "math.ucr.edu": 0.9027587862028252,+ "gekk.info": 0.9026852677733105,+ "www.dwitter.net": 0.9026795513672158,+ "nakedsecurity.sophos.com": 0.9026433950295091,+ "www.tomdispatch.com": 0.9026213839350323,+ "www.erichgrunewald.com": 0.9025809741606529,+ "everydayastronaut.com": 0.9025525737662192,+ "meta.stackexchange.com": 0.9025397509956542,+ "dashbit.co": 0.9025100393249755,+ "kroah.com": 0.9025051904560609,+ "www.terrybisson.com": 0.902478233349385,+ "www.whatsapp.com": 0.9024176929829565,+ "ploum.net": 0.9024105508563706,+ "explosion.ai": 0.9024029708934638,+ "blog.repl.it": 0.9024021393458092,+ "neurosciencenews.com": 0.9023952036309971,+ "www.supermemo.com": 0.9023729956597971,+ "www.diyphotography.net": 0.902367985362208,+ "kcsrk.info": 0.902365879005221,+ "security.stackexchange.com": 0.9023603426250738,+ "developer.android.com": 0.9023376538238216,+ "blog.rjmetrics.com": 0.9023127105964952,+ "kakoune.org": 0.902272263107977,+ "www.saurik.com": 0.9022715997521633,+ "thetoolsweneed.com": 0.902259888208641,+ "atthis.link": 0.9022431859413726,+ "hg.python.org": 0.9021998686737284,+ "www.softwarepreservation.org": 0.9021625352128104,+ "www.kleinbottle.com": 0.9021190716448031,+ "interrupt.memfault.com": 0.9020903788101069,+ "dspace.mit.edu": 0.9020251184851814,+ "philipwalton.com": 0.9020062837472365,+ "www.foxbusiness.com": 0.9019313178215171,+ "jrsinclair.com": 0.9019295383152887,+ "www.space.com": 0.9019268835645871,+ "sachin.posterous.com": 0.9018811076664313,+ "khn.org": 0.901879302131479,+ "www.pigdog.org": 0.901878363289037,+ "zackkanter.com": 0.9018673516010077,+ "www.informationdiet.com": 0.9018348908028704,+ "giantrobotlasers.com": 0.9017894645998003,+ "incolumitas.com": 0.9017770940926804,+ "jeffkreeftmeijer.com": 0.9017611553399866,+ "bostonreview.net": 0.9017262602947111,+ "techreport.com": 0.901717492257879,+ "www.baldurbjarnason.com": 0.9017151044394945,+ "www.sicpers.info": 0.9017134966165734,+ "blog.sfgate.com": 0.9016737116837992,+ "write.as": 0.9016704517485878,+ "www.aljazeera.com": 0.9016695698160121,+ "fr.slideshare.net": 0.9016442121790157,+ "everything2.com": 0.9016361459958601,+ "typicalprogrammer.com": 0.9015788203543227,+ "blog.scaleway.com": 0.9015776579003968,+ "rethrick.com": 0.9015770143955718,+ "inference-review.com": 0.9015413236700288,+ "thedorkweb.substack.com": 0.9015280289106077,+ "blog.23andme.com": 0.9015240824972036,+ "www.mikealche.com": 0.9015168230317929,+ "hashcat.net": 0.9015094649406228,+ "madebyloren.com": 0.9014851889844163,+ "www.blockchain.com": 0.901483519454099,+ "www.computerworld.com": 0.9014820166011037,+ "cen.acs.org": 0.901480955026061,+ "lauren.vortex.com": 0.9014724896133568,+ "oreilly.com": 0.901407801663114,+ "blog.netflix.com": 0.9013957945143364,+ "commoncrawl.org": 0.9013596330544786,+ "www.svds.com": 0.9013257789593407,+ "blog.goodstuff.im": 0.9013257789593407,+ "www.lhup.edu": 0.9013191983110588,+ "www.nist.gov": 0.9013044488448885,+ "jasonevanish.com": 0.9012724811591895,+ "nextcity.org": 0.9012682451009019,+ "nuitka.net": 0.9012059293191188,+ "dataswamp.org": 0.9011871538800621,+ "www.reactos.org": 0.9011785225363149,+ "limpet.net": 0.9011626533660011,+ "nextjournal.com": 0.9011584604272797,+ "sinews.siam.org": 0.9011379162640358,+ "blog.zactownsend.com": 0.9011275964672605,+ "m.guardiannews.com": 0.9011260854499006,+ "mazzo.li": 0.9011141262538125,+ "news.developer.nvidia.com": 0.9011001587785914,+ "spideroak.com": 0.901093458914619,+ "www.startribune.com": 0.9010814985872827,+ "www.gitbook.com": 0.901071077131811,+ "www.brusselstimes.com": 0.9010360879334309,+ "blog.instagram.com": 0.9010324035185349,+ "benedikt-bitterli.me": 0.9009740364816841,+ "fedoramagazine.org": 0.9009664628347622,+ "www.rte.ie": 0.9009526229516198,+ "f-droid.org": 0.9008856641605046,+ "blog.dwolla.com": 0.9008628892479406,+ "news.trust.org": 0.9008305066882848,+ "about.bnef.com": 0.9007424598766725,+ "txti.es": 0.9007231608685488,+ "samizdat.mines.edu": 0.9007085795324656,+ "www.murilopereira.com": 0.9007078611650351,+ "bitcoincharts.com": 0.9006914299992029,+ "material.io": 0.9006442988271292,+ "openradar.appspot.com": 0.9006237134629408,+ "blog.rapportive.com": 0.9006217110635638,+ "m-cacm.acm.org": 0.9005996462860704,+ "blog.halide.cam": 0.9005839293957032,+ "www.billthelizard.com": 0.9005839293957032,+ "news.sciencemag.org": 0.9005769469610003,+ "www.edmunds.com": 0.900563544204038,+ "wiki.c2.com": 0.9005588568752737,+ "thinkvitamin.com": 0.9005542231541395,+ "ipython.org": 0.900531415561299,+ "panelbear.com": 0.9005250466106081,+ "stuartsierra.com": 0.9004498846216558,+ "clojurescript.org": 0.900443619125688,+ "maps.org": 0.9004208529632491,+ "www.dextronet.com": 0.9004014616042372,+ "blog.digitalocean.com": 0.9003805295727227,+ "www.happyassassin.net": 0.9003455623339318,+ "blockchain.info": 0.9003439746043133,+ "realm.io": 0.9003295505994889,+ "unixsheikh.com": 0.9003091480910362,+ "zyan.scripts.mit.edu": 0.9002640065392239,+ "til.simonwillison.net": 0.9002517190240483,+ "fingerprintjs.com": 0.9002278669918841,+ "cs-blog.khanacademy.org": 0.9001663145026005,+ "eugene.kaspersky.com": 0.9000762462446584,+ "www.bay12games.com": 0.9000684308983997,+ "ginzametrics.com": 0.9000536631561354,+ "blog.simpleanalytics.com": 0.9000166910943967,+ "viewfromthewing.com": 0.9000166910943967,+ "liam-on-linux.livejournal.com": 0.9000090039034558,+ "softwaremaniacs.org": 0.9000031026474421,+ "blog.mariadb.org": 0.8999936863202533,+ "levels.io": 0.899970180177946,+ "craigmod.com": 0.8999690838572385,+ "news.spotify.com": 0.8999500411279416,+ "forums.developer.apple.com": 0.8999431500814804,+ "www.overcomingbias.com": 0.8999429014404717,+ "users.rust-lang.org": 0.8999355740659959,+ "scrimba.com": 0.8999254626887211,+ "www.sbnation.com": 0.8998688532688766,+ "benswann.com": 0.89986648239432,+ "www.thelocal.dk": 0.8998638303612118,+ "pitchfork.com": 0.8998590033239674,+ "wekeroad.com": 0.899851220900418,+ "animalnewyork.com": 0.8998375739356855,+ "beets.io": 0.899813377379441,+ "www.ibiblio.org": 0.8997974933028597,+ "kernelnewbies.org": 0.899793879009676,+ "www.fullstackreact.com": 0.8997933884297521,+ "ricostacruz.com": 0.8997816625141879,+ "www.digitalocean.com": 0.8997784555641042,+ "www.maps.org": 0.8997736250289394,+ "pointersgonewild.com": 0.89975961274635,+ "512pixels.net": 0.8997410869841632,+ "blog.wilshipley.com": 0.899720920102866,+ "devlog.hexops.com": 0.8997114101311218,+ "blogs.nvidia.com": 0.899693028608083,+ "www.sacbee.com": 0.8996905748131163,+ "www.publicbooks.org": 0.8996781736043378,+ "hforsten.com": 0.8996749694919105,+ "okmij.org": 0.899647639382427,+ "www.dataquest.io": 0.8996134651823179,+ "www.nocsdegree.com": 0.8995491494660874,+ "pandas.pydata.org": 0.8995416268122183,+ "doctorow.medium.com": 0.8995206430959863,+ "blog.ezyang.com": 0.8995116025900058,+ "cs.gmu.edu": 0.8995004521044683,+ "weblog.raganwald.com": 0.8994722109240123,+ "finance.fortune.cnn.com": 0.8994585795276846,+ "pedestrianobservations.com": 0.8994403946669056,+ "dangrover.com": 0.8994295897503085,+ "nostarch.com": 0.8994103848538745,+ "foreignpolicy.com": 0.8994068719520655,+ "benchmarksgame-team.pages.debian.net": 0.8993756538078823,+ "kvark.github.io": 0.8993354696336303,+ "www.scummvm.org": 0.8993014570629587,+ "www.jpl.nasa.gov": 0.8992820116565705,+ "theweek.com": 0.8992433138556716,+ "status.heroku.com": 0.8992414299043588,+ "tech.grammarly.com": 0.8992224733195932,+ "www.mensjournal.com": 0.8992186596239703,+ "braythwayt.com": 0.8992151653414675,+ "clayallsopp.com": 0.8991984891927328,+ "blog.japaric.io": 0.8991683991683992,+ "www.linuxuprising.com": 0.8991634552022689,+ "www.cambridge.org": 0.8991459176334309,+ "pando.com": 0.8991426686559758,+ "nextcloud.com": 0.8991240467738062,+ "blog.luap.info": 0.8991106158419904,+ "goto.ucsd.edu": 0.8990952211205248,+ "layervault.tumblr.com": 0.8990927364776206,+ "www.preetamnath.com": 0.8990436000092332,+ "www.loper-os.org": 0.8990428163259621,+ "beta.latimes.com": 0.8990201091709796,+ "status.linode.com": 0.8990191396542296,+ "www.yahoo.com": 0.8990120070182801,+ "www.mining.com": 0.8989843218989275,+ "thestartuptoolkit.com": 0.8989617943814647,+ "exple.tive.org": 0.8989583647726666,+ "www.percona.com": 0.8989398873032174,+ "tug.org": 0.8988675834004226,+ "azure.microsoft.com": 0.8988119185957827,+ "badassjs.com": 0.8987895474430552,+ "florentcrivello.com": 0.8987782517021887,+ "joelonsoftware.com": 0.8987694677031264,+ "www.cntraveler.com": 0.8987494824459783,+ "postmasculine.com": 0.8986927094085853,+ "blade.nagaokaut.ac.jp": 0.8986856729161738,+ "blog.demofox.org": 0.8986624269542334,+ "www.zeit.de": 0.8986585350988878,+ "louwrentius.com": 0.8986513425823265,+ "www.jeffwofford.com": 0.8986246802875617,+ "lists.canonical.org": 0.8985997713186075,+ "blog.aboutamazon.com": 0.898569865878112,+ "discourse.haskell.org": 0.8985468763312833,+ "marginalrevolution.com": 0.8985222766062069,+ "www.southampton.ac.uk": 0.8985031338241491,+ "wccftech.com": 0.898464128485399,+ "pages.nist.gov": 0.8984293193717278,+ "blog.jcoglan.com": 0.8984277156180708,+ "today.duke.edu": 0.898426650553988,+ "blog.us.playstation.com": 0.8984266505539878,+ "technet.microsoft.com": 0.8984178818074816,+ "moultano.wordpress.com": 0.8983087054446331,+ "forums.swift.org": 0.8983030929609286,+ "pyfound.blogspot.com": 0.8982450409736414,+ "sebastianraschka.com": 0.8982066640921482,+ "www.michaelfogleman.com": 0.8982013504259682,+ "www.confluent.io": 0.8981951121390044,+ "www.maa.org": 0.8981932917194397,+ "serverfault.com": 0.898185336292522,+ "fitzgeraldnick.com": 0.8981340410428447,+ "www.2600.com": 0.8981193270212353,+ "rustyshelf.org": 0.898115004185606,+ "svnweb.freebsd.org": 0.8981026654387363,+ "www.argmin.net": 0.8980921557949308,+ "www.theawl.com": 0.8980833367511953,+ "keygen.sh": 0.8980696445122388,+ "www.freightwaves.com": 0.8980574109093967,+ "developers.facebook.com": 0.8980338313592694,+ "www.randomhacks.net": 0.8980285406407745,+ "susam.in": 0.8980164962202184,+ "exercism.io": 0.8980136526006044,+ "trekhleb.dev": 0.8980055706761685,+ "securelist.com": 0.8979897593608089,+ "air.mozilla.org": 0.897889243419127,+ "yahooeng.tumblr.com": 0.8978817471988295,+ "techrights.org": 0.8978658400805657,+ "franciskim.co": 0.8978653593888846,+ "ergoemacs.org": 0.8978533245623609,+ "www.scs.stanford.edu": 0.8978490165530042,+ "www.currentaffairs.org": 0.8978420868913102,+ "arc.aiaa.org": 0.897839844709445,+ "youtube.googleblog.com": 0.8978353788604845,+ "www.pulitzer.org": 0.8977920724736956,+ "computer.rip": 0.8977456785440727,+ "blog.dgraph.io": 0.8977112410728876,+ "blog.fastforwardlabs.com": 0.8977025051645369,+ "www.bridgewater.com": 0.8976967291200032,+ "mycroft.ai": 0.8976933371034872,+ "open.salon.com": 0.8976917672314974,+ "blog.longreads.com": 0.8976472678414861,+ "lists.ubuntu.com": 0.897645348573024,+ "nvie.com": 0.8976386294805203,+ "adamdrake.com": 0.8976376774058287,+ "www.cordcuttersnews.com": 0.8975937131370894,+ "www.kayak.com": 0.8975688924284162,+ "aiexperiments.withgoogle.com": 0.8975573917462177,+ "gustedt.wordpress.com": 0.8975532085064689,+ "www.adobe.com": 0.8975379400846643,+ "miter.mit.edu": 0.8975316961912847,+ "blog.statuspage.io": 0.8975269760160564,+ "hyperpolyglot.org": 0.8975160197448814,+ "www.scaleway.com": 0.8974907733168768,+ "bennettfeely.com": 0.8974804776492202,+ "joshwcomeau.com": 0.8974719675010521,+ "www.ktvu.com": 0.8974664720007207,+ "www.tomsguide.com": 0.897458826143719,+ "www.jitbit.com": 0.8973926361586634,+ "time.is": 0.8973510461083218,+ "tinyclouds.org": 0.8973083698372893,+ "gurjeet.singh.im": 0.8973075039780294,+ "blog.mailgun.net": 0.8972959119527889,+ "harry.me": 0.8972525836598453,+ "archpaper.com": 0.8972054834238723,+ "yoric.github.io": 0.897152918128832,+ "community.atlassian.com": 0.8971438675267877,+ "news.ycombinator.com": 0.8971152540222578,+ "aviation.stackexchange.com": 0.8970966595562585,+ "scholars-stage.blogspot.com": 0.8970933350643355,+ "durmonski.com": 0.897088311771032,+ "blog.zorinaq.com": 0.8970730785733094,+ "blog.peepcode.com": 0.8970173769216392,+ "kivy.org": 0.8969978306759151,+ "www.crowdtilt.com": 0.8969827660481152,+ "mathematica.stackexchange.com": 0.8969710211525342,+ "www.wyden.senate.gov": 0.8969688474969271,+ "watsi.org": 0.8969438303261918,+ "igor-blue.github.io": 0.8968776201906612,+ "medium.economist.com": 0.8968759451432184,+ "www.wolframalpha.com": 0.8968667656586023,+ "highlyscalable.wordpress.com": 0.8968525375322202,+ "in.reuters.com": 0.8967974687633106,+ "www.nethack.org": 0.8967951042087746,+ "tidbits.com": 0.8967894768726519,+ "theappendix.net": 0.8967463599409581,+ "tech.fortune.cnn.com": 0.8967131583500147,+ "www.fossjobs.net": 0.8966914843070399,+ "blog.robinhood.com": 0.8966536394829941,+ "www.realtimerendering.com": 0.8966481963801871,+ "blog.theincredibleholk.org": 0.8966222436790824,+ "www.ee.ryerson.ca": 0.8965601151332392,+ "databricks.com": 0.8965596021498984,+ "www.defensenews.com": 0.8965587750067616,+ "rcoh.me": 0.896554024229452,+ "mijailovic.net": 0.8965535170148166,+ "gallium.inria.fr": 0.8965486747498262,+ "blog.optimizely.com": 0.896542105323571,+ "blog.replit.com": 0.8965338803932761,+ "learn.adafruit.com": 0.8965131842089722,+ "joelrunyon.com": 0.8964677519262857,+ "blog.frizk.net": 0.8964121097275121,+ "alarmingdevelopment.org": 0.8964094023716221,+ "robots.thoughtbot.com": 0.8963990601096038,+ "www.includeos.org": 0.8963879787342307,+ "mrcoles.com": 0.8963879787342307,+ "c2.com": 0.8963178119940578,+ "www.whonix.org": 0.8963115990850473,+ "fivebooks.com": 0.8963105340034757,+ "reprog.wordpress.com": 0.8963054521368881,+ "www.wilfred.me.uk": 0.8962945507182206,+ "whydoeseverythingsuck.com": 0.8962216827001023,+ "www.pcworld.com": 0.8962205591216735,+ "www.bamsoftware.com": 0.8961964199305918,+ "www.cs.unm.edu": 0.8961816656276443,+ "www.idris-lang.org": 0.8961766972485017,+ "www.raspberrypi.com": 0.8961620944863709,+ "effectiviology.com": 0.8961424475553789,+ "asciiflow.com": 0.8961224916388418,+ "philosophically.com": 0.896116599326938,+ "www.alexa.com": 0.8961037898681212,+ "algorithmicbotany.org": 0.8960953021444669,+ "yasoob.me": 0.8960526720715563,+ "varun.ca": 0.896004489709373,+ "medievalbooks.nl": 0.8959983974915444,+ "www.kgw.com": 0.895989268551949,+ "www.junglecoder.com": 0.8959808787419773,+ "insideintercom.io": 0.8959693504237235,+ "www.html5rocks.com": 0.8959295211749472,+ "www.arxiv-vanity.com": 0.8959180319048996,+ "micahflee.com": 0.8958958043171017,+ "www.nccgroup.trust": 0.8958587838339611,+ "c0de517e.blogspot.com": 0.8958503147286802,+ "www.foreignpolicy.com": 0.8958091411993182,+ "ashtonkemerling.com": 0.8957869981475534,+ "www.amusingplanet.com": 0.8957821987841423,+ "lea.verou.me": 0.895732027114529,+ "nationalinterest.org": 0.8957204150819327,+ "lists.webkit.org": 0.8956698653136842,+ "opkode.com": 0.8956558833770757,+ "www-formal.stanford.edu": 0.89562877673962,+ "www.michaelgeist.ca": 0.8956008548612578,+ "theory.stanford.edu": 0.8955882277276088,+ "i.stanford.edu": 0.8955669355788611,+ "exstreamist.com": 0.8955604388524536,+ "citeseerx.ist.psu.edu": 0.8955488558889958,+ "bugzilla.redhat.com": 0.8955216901346179,+ "www.cmu.edu": 0.8954764227994559,+ "0xax.gitbooks.io": 0.8954552987943115,+ "jasoncrawford.org": 0.8954419392195877,+ "www.lego.com": 0.8954257332135742,+ "discourse.julialang.org": 0.8954216385277134,+ "www.open-std.org": 0.8953539493751335,+ "www.indiatoday.in": 0.8953429983961907,+ "www.winestockwebdesign.com": 0.8953288508285574,+ "mobiforge.com": 0.8953182171062833,+ "www.regulations.gov": 0.895303818266352,+ "www.salesforce.com": 0.8952981434270118,+ "www.trackinghappiness.com": 0.8952717597076212,+ "blackhole12.blogspot.com": 0.8952521384621974,+ "blog.infinitenegativeutility.com": 0.8952329903252306,+ "www.opera.com": 0.8952211159380423,+ "blogs.balsamiq.com": 0.8952154811855707,+ "treyhunner.com": 0.8952154811855707,+ "constructionphysics.substack.com": 0.8951823466408914,+ "beyondloom.com": 0.8951761937264585,+ "www.lennyrachitsky.com": 0.8951749721627695,+ "haskell.cs.yale.edu": 0.8951742276618677,+ "raymii.org": 0.8951548310801417,+ "llogiq.github.io": 0.8951452226380562,+ "blog.amirkhella.com": 0.8950945164652512,+ "www.mooreds.com": 0.8950941927209954,+ "www.hudsandguis.com": 0.895021645021645,+ "flask.pocoo.org": 0.89499976188154,+ "www.dol.gov": 0.8949918028432319,+ "wololo.net": 0.8949102546729428,+ "dashes.com": 0.894908282336013,+ "mortoray.com": 0.8948856187046399,+ "www.mediawiki.org": 0.8948851399600354,+ "developerblog.redhat.com": 0.8948826280677145,+ "www.banterly.net": 0.8948384778162753,+ "www.justin.tv": 0.8948269726237952,+ "www.koreaherald.com": 0.8947866914467383,+ "two-wrongs.com": 0.8947866914467383,+ "www.3ders.org": 0.8947587499828596,+ "millcomputing.com": 0.894753411664112,+ "remysharp.com": 0.894728785876003,+ "magenta.tensorflow.org": 0.8947055951377291,+ "www.techradar.com": 0.8947023301779632,+ "www.pushing-pixels.org": 0.8946997760143556,+ "sagemath.blogspot.com": 0.8946383392529572,+ "mymodernmet.com": 0.8946360917364401,+ "jamesmcm.github.io": 0.8946346573429327,+ "designyoutrust.com": 0.8945718808483918,+ "boards.4chan.org": 0.8945658314381272,+ "blog.framebase.io": 0.8945652173913043,+ "openwrt.org": 0.8945648303938399,+ "blog.codecombat.com": 0.8945386717554477,+ "www.informit.com": 0.8945265591993962,+ "www.evilsocket.net": 0.894519291668392,+ "www.governing.com": 0.8945150467351556,+ "www.masteringemacs.org": 0.894486414286821,+ "tomassetti.me": 0.8944815712239113,+ "gun.io": 0.8944737983455031,+ "www.netlify.com": 0.8944433597452348,+ "www.ntsb.gov": 0.8944261516293643,+ "www.cs.kent.ac.uk": 0.8943581679658725,+ "www.kernel.org": 0.8943363945858583,+ "www.flyertalk.com": 0.8943293407889205,+ "www.unlimitednovelty.com": 0.8943013001071519,+ "math.andrej.com": 0.8942887015711656,+ "healeycodes.com": 0.8942310355364101,+ "blog.jmdawson.co.uk": 0.894210852992969,+ "metaoptimize.com": 0.8942001103153372,+ "mynorthwest.com": 0.894171800552717,+ "asciinema.org": 0.8941695108843633,+ "pydanny.com": 0.8941547728795499,+ "warisboring.com": 0.8940977556210705,+ "blog.serverfault.com": 0.8940908548836543,+ "narratively.com": 0.8940876544990076,+ "freakonomics.com": 0.8940592156715558,+ "www.joindiaspora.com": 0.8940510018896056,+ "www.caixinglobal.com": 0.8940493754864369,+ "guccifer2.wordpress.com": 0.8940429649134846,+ "www.freelists.org": 0.894038057617328,+ "devblog.bu.mp": 0.8940332414895527,+ "mislav.uniqpath.com": 0.894024162032998,+ "www.heydonworks.com": 0.8939995146383073,+ "glyph.twistedmatrix.com": 0.8939986046314382,+ "paulirish.com": 0.8939908517980819,+ "a1k0n.net": 0.8939890710382514,+ "www.vldb.org": 0.893980099250682,+ "www.squeakland.org": 0.8939334940011854,+ "america.aljazeera.com": 0.8939301873013243,+ "www.epi.org": 0.8939255228575806,+ "tryolabs.com": 0.8938904483824683,+ "willcrichton.net": 0.8938879993790215,+ "news.lenovo.com": 0.8938868214729069,+ "paulbernal.wordpress.com": 0.8938730853391685,+ "www.psmag.com": 0.8938507630159833,+ "meiert.com": 0.8938177244811635,+ "www.ghacks.net": 0.8938115780468081,+ "www.shamusyoung.com": 0.8938032619934929,+ "zeptobars.com": 0.893764616861537,+ "0x0f0f0f.github.io": 0.893764616861537,+ "cm.bell-labs.com": 0.8937058283807041,+ "www.fastcolabs.com": 0.8936928790642708,+ "aging.nautil.us": 0.8936740461671302,+ "hackerone.com": 0.8936672524995389,+ "storage.courtlistener.com": 0.893640350877193,+ "danlebrero.com": 0.893623066742338,+ "mapzen.com": 0.8936063685534883,+ "bernsteinbear.com": 0.8935751450877738,+ "austinmann.com": 0.8935501465273382,+ "www.vinaysahni.com": 0.8935481547848754,+ "burakkanber.com": 0.8935378844831082,+ "mollyrocket.com": 0.8935354120568186,+ "feeding.cloud.geek.nz": 0.8935244667500523,+ "cliffle.com": 0.8934905766791413,+ "www.mikeperham.com": 0.8934692786771169,+ "www.sequoiacap.com": 0.8934381407790684,+ "www.thestranger.com": 0.8934172181855746,+ "python.org": 0.8933810789765779,+ "www.datacenterdynamics.com": 0.893374339023219,+ "www.nongnu.org": 0.8933522846439509,+ "blog.pkh.me": 0.8933285357815479,+ "greenteapress.com": 0.8933064903929183,+ "debarghyadas.com": 0.8933044930066333,+ "jashkenas.github.com": 0.8933044930066333,+ "jborichevskiy.com": 0.8932713559026754,+ "www.saturdayeveningpost.com": 0.8932203537854467,+ "www.engineyard.com": 0.8931369083416167,+ "jeremymikkola.com": 0.8931346519747027,+ "chriseidhof.tumblr.com": 0.893107113349865,+ "www.the-scientist.com": 0.8931050183937893,+ "heterodoxacademy.org": 0.8930676522754398,+ "daveceddia.com": 0.8930562709346127,+ "eighty-twenty.org": 0.8929575630252592,+ "itvision.altervista.org": 0.8928942922265454,+ "pogue.blogs.nytimes.com": 0.892885991203199,+ "science.sciencemag.org": 0.8928844657775074,+ "www.citypages.com": 0.8928735399686778,+ "www.houstonchronicle.com": 0.8928722277760239,+ "www.perl.com": 0.8928505576437601,+ "www.mcclatchydc.com": 0.892841447269041,+ "www.over-yonder.net": 0.8928301959201767,+ "donatstudios.com": 0.8928301959201767,+ "ucsdnews.ucsd.edu": 0.8928192288277865,+ "www.johnmyleswhite.com": 0.8928134392551893,+ "vorner.github.io": 0.8928096417930931,+ "nikic.github.com": 0.8928090193925269,+ "samsclass.info": 0.8927237027135478,+ "markmaunder.com": 0.8926951731429821,+ "tauday.com": 0.8926800567850355,+ "riscv.org": 0.8926238746067623,+ "blog.blazingdb.com": 0.8926082454543174,+ "garrett.damore.org": 0.892590826266076,+ "www.ilikebigbits.com": 0.8925607594811928,+ "mediagoblin.org": 0.8925318168757916,+ "keithclark.co.uk": 0.8925217211946057,+ "reviews.llvm.org": 0.8925217211946057,+ "www.census.gov": 0.8925111327414155,+ "www.carbonbrief.org": 0.8925108441158517,+ "1000memories.com": 0.8924866571941482,+ "tommorris.org": 0.8924689354696166,+ "csswizardry.com": 0.8924685362435004,+ "simonmar.github.io": 0.8924611973392461,+ "t3x.org": 0.8924420140543968,+ "alpinelinux.org": 0.8924276788770722,+ "news.blizzard.com": 0.8924063008658305,+ "lowendmac.com": 0.892380727879953,+ "jakubmarian.com": 0.8923725482586792,+ "smackdown.blogsblogsblogs.com": 0.8923365014823826,+ "www.gamefromscratch.com": 0.8922886748833608,+ "www.history.com": 0.892284563335802,+ "www.aps.org": 0.8922636239768308,+ "bitbashing.io": 0.8922618659350428,+ "thecreativeindependent.com": 0.8922525349154249,+ "abcnews.go.com": 0.8922472028702633,+ "lilianweng.github.io": 0.8922138047648334,+ "filecoin.io": 0.8921882836554222,+ "kristerw.blogspot.com": 0.8921827368433277,+ "www.archdaily.com": 0.8921786726941157,+ "critter.blog": 0.8921605201397375,+ "blog.uber.com": 0.8921471734712938,+ "blog.darklang.com": 0.8921367030876631,+ "youteam.io": 0.8921269513814725,+ "developer.oculus.com": 0.8921230866971346,+ "www.airbnb.com": 0.8921210442927137,+ "www.caitlingreen.org": 0.8921201754031437,+ "stuffin.space": 0.8921201754031437,+ "www.logicmatters.net": 0.8921008427968538,+ "blogs.atlassian.com": 0.8920598317674283,+ "www.coindesk.com": 0.8920474723589183,+ "blog.checklyhq.com": 0.8920323362989564,+ "www.reifyworks.com": 0.8920283981813807,+ "www.tensorflow.org": 0.8919880973555573,+ "medicalxpress.com": 0.8919405231029118,+ "lists.wikimedia.org": 0.8919231093801725,+ "www.cabinetmagazine.org": 0.8919061013347211,+ "vez.mrsk.me": 0.8918998921036764,+ "tenderlovemaking.com": 0.8918462339278702,+ "timdettmers.com": 0.8918366190595837,+ "jpkoning.blogspot.com": 0.8918251594506603,+ "www.timesonline.co.uk": 0.8917973535504273,+ "pdfs.semanticscholar.org": 0.8917971680021309,+ "www.trustwave.com": 0.8917680987246398,+ "ambrevar.xyz": 0.8917657796004286,+ "www.theroot.com": 0.8917542491581687,+ "tylertringas.com": 0.8917090979583869,+ "www.ncsc.gov.uk": 0.8916672988164659,+ "globenewswire.com": 0.8916669637212149,+ "www.applevis.com": 0.8916621885801436,+ "www.metafilter.com": 0.8916619189169765,+ "erickhun.com": 0.8916502534520422,+ "blog.wearewizards.io": 0.8916476923173318,+ "cpj.org": 0.8916369716407726,+ "boyter.org": 0.891577544859255,+ "m.facebook.com": 0.8915642265642092,+ "www.calnewport.com": 0.8915632480334919,+ "www.profgalloway.com": 0.8915466752832707,+ "mentallandscape.com": 0.8915429443685636,+ "m.theatlantic.com": 0.891523653721213,+ "hackertarget.com": 0.8915152085122275,+ "www.pyret.org": 0.8914988814317674,+ "www.obdev.at": 0.8914980426595512,+ "dash.harvard.edu": 0.8914784869901488,+ "engineeringblog.yelp.com": 0.8914654287085163,+ "www.mercurial-scm.org": 0.8914323327846669,+ "datanitro.com": 0.8914234373706179,+ "www.huffingtonpost.in": 0.8914121923586303,+ "fangpenlin.com": 0.891404181051816,+ "discovermagazine.com": 0.8914023623580617,+ "streetlifesolutions.blogspot.com": 0.8914002514918445,+ "blog.documentfoundation.org": 0.8913787017013197,+ "dotsies.org": 0.8913773796192609,+ "tails.boum.org": 0.8913724994898894,+ "randfishkin.com": 0.8912627828106103,+ "tomasp.net": 0.8911820823224039,+ "www.taiwannews.com.tw": 0.8911580981808408,+ "modern-sql.com": 0.891151176073313,+ "carlos.bueno.org": 0.8911385230462937,+ "www.kansascity.com": 0.8910854933759852,+ "english.stackexchange.com": 0.8910824248780256,+ "cdm.link": 0.8910804711300865,+ "nighthacks.com": 0.8910750341672654,+ "blog.vrypan.net": 0.8910540018083172,+ "nyxt.atlas.engineer": 0.8910326186239637,+ "blog.evernote.com": 0.8909942969284109,+ "news.bbc.co.uk": 0.89099404579875,+ "securitylab.github.com": 0.8909843257796506,+ "pjreddie.com": 0.8909657104816301,+ "fury.com": 0.8909653233291642,+ "blog.skylight.io": 0.8909427640646718,+ "kickstarter.engineering": 0.8909427640646718,+ "www.darpa.mil": 0.8909203771658175,+ "psyarxiv.com": 0.8909128838846586,+ "spottedsun.com": 0.89089016985033,+ "matt.aimonetti.net": 0.8907810868277886,+ "news.dartlang.org": 0.8907684232451024,+ "musescore.org": 0.8907008278994514,+ "www.apa.org": 0.8906903176590568,+ "www.flightglobal.com": 0.8906785542989906,+ "www.theatlanticcities.com": 0.8906598207125368,+ "search.cpan.org": 0.8906114753639363,+ "www.p01.org": 0.8906084333939481,+ "blog.dantup.com": 0.8906084333939481,+ "thedailywtf.com": 0.8905424089852495,+ "e.foundation": 0.8905340551839913,+ "cookieplmonster.github.io": 0.8905252719934627,+ "news.php.net": 0.8905175241100567,+ "toroid.org": 0.8905145635041263,+ "www.kk.org": 0.8904878920768752,+ "forum.sublimetext.com": 0.890452944414247,+ "praxis.fortelabs.co": 0.890452944414247,+ "leostartsup.com": 0.8904132589825239,+ "web.cecs.pdx.edu": 0.890412029578113,+ "www.ofbrooklyn.com": 0.8903954802259887,+ "www.revk.uk": 0.8903240899854906,+ "rawkes.com": 0.8903199760734138,+ "furbo.org": 0.8903142798666888,+ "nickoneill.com": 0.8902859706962722,+ "compilers.iecc.com": 0.8902788378277958,+ "www.climatechangenews.com": 0.8902788378277958,+ "avichal.com": 0.8902714932126697,+ "www.healthcaredive.com": 0.8902714932126697,+ "lucasfcosta.com": 0.8902709782517574,+ "stevecoast.com": 0.8902478204961621,+ "dariusforoux.com": 0.8902302723631834,+ "msdn.microsoft.com": 0.8901788121850751,+ "www.nzz.ch": 0.8901568608802078,+ "trac.webkit.org": 0.8901554708541648,+ "www.waze.com": 0.8901472253680635,+ "www.liip.ch": 0.8901472253680635,+ "ericlippert.com": 0.8901127718368207,+ "matpalm.com": 0.8901030282480019,+ "roshfu.com": 0.8900305511261404,+ "adeshpande3.github.io": 0.8900188928475846,+ "www.factmag.com": 0.8900054823385649,+ "grafana.org": 0.8899942321734828,+ "www.divio.com": 0.889923339928156,+ "www.law.cornell.edu": 0.8899173371280885,+ "www.cs.huji.ac.il": 0.8899161000679049,+ "gothamist.com": 0.8899111843846533,+ "www.ajc.com": 0.8899040092884533,+ "mathpix.com": 0.8899039505478837,+ "www.vulture.com": 0.8899034479557729,+ "bair.berkeley.edu": 0.8898667877223271,+ "www.goldsborough.me": 0.8897894354492999,+ "arkadiyt.com": 0.8897865060368211,+ "flowingdata.com": 0.8897859193800971,+ "grist.org": 0.8897796023630788,+ "clusterhq.com": 0.889770984328511,+ "postmarketos.org": 0.8897648371785116,+ "monkey.org": 0.8897626687452419,+ "www.coderholic.com": 0.8897409044066217,+ "blog.silktide.com": 0.8897409044066216,+ "docs.house.gov": 0.88972591966882,+ "colab.research.google.com": 0.8897224911816017,+ "www.nhs.uk": 0.8896805121516772,+ "insideclimatenews.org": 0.8896793812620282,+ "torvalds-family.blogspot.com": 0.8896665476254435,+ "www.holloway.com": 0.8896662494413392,+ "www.airspacemag.com": 0.8896507942259744,+ "grsecurity.net": 0.889646047305093,+ "www.kevinrooke.com": 0.8896372799856362,+ "engineering.gusto.com": 0.8896147691691174,+ "blog.christianperone.com": 0.8896015309843164,+ "www.badscience.net": 0.889589692438837,+ "www.jerf.org": 0.889547641907146,+ "vfoley.xyz": 0.8895216400911161,+ "www.dangrossman.info": 0.8895193470455725,+ "blog.timac.org": 0.8895027671338473,+ "www.nme.com": 0.8894941034343402,+ "withblue.ink": 0.8894380029474798,+ "www.modernperlbooks.com": 0.8894320825157749,+ "guix.gnu.org": 0.8894056608506781,+ "vasilishynkarenka.com": 0.8894015273075921,+ "chrome.google.com": 0.8893920540006104,+ "startupboy.com": 0.8893837967279286,+ "thesoundingline.com": 0.8893544871283143,+ "blog.kotowicz.net": 0.8893507542723823,+ "www.ubuntuvibes.com": 0.8893113229443974,+ "www.tindie.com": 0.8892920494549288,+ "www.falstad.com": 0.8892263646054737,+ "alisdair.mcdiarmid.org": 0.8892079775948455,+ "www.brennancenter.org": 0.8892079775948455,+ "theanarchistlibrary.org": 0.8891911407644053,+ "www.cftc.gov": 0.8891651323421873,+ "lisp-journey.gitlab.io": 0.8891428571428571,+ "www.namepros.com": 0.8891410371836007,+ "eu.usatoday.com": 0.8890682694542983,+ "blog.canonical.com": 0.8890615085402753,+ "sahandsaba.com": 0.88904655549824,+ "blog.rstudio.org": 0.8890367675160065,+ "neilkakkar.com": 0.8890307804355818,+ "news.nationalpost.com": 0.8890115573593319,+ "www.snopes.com": 0.8889924719688463,+ "www.brainpickings.org": 0.8889686116306911,+ "mitmproxy.org": 0.8889639851594255,+ "blog.balancedpayments.com": 0.888958456392269,+ "breakingdefense.com": 0.8889416410999423,+ "markets.businessinsider.com": 0.8889221685107183,+ "jerrygamblin.com": 0.8889200836058827,+ "angularjs.blogspot.com": 0.8888770420833161,+ "www.shopify.com": 0.8888551514369234,+ "www.helloerik.com": 0.8888428887518348,+ "languagelog.ldc.upenn.edu": 0.8888040353901259,+ "diogomonica.com": 0.888789480686917,+ "oversight.house.gov": 0.8887677241378522,+ "blog.yossarian.net": 0.8887466744525089,+ "tim.blog": 0.8887179571174896,+ "blog.marcocantu.com": 0.8887010787021495,+ "www.stilldrinking.org": 0.8886969078588848,+ "plentyoffish.wordpress.com": 0.8886858465608289,+ "troubles.md": 0.8886733614347104,+ "8bitworkshop.com": 0.8886716751051769,+ "plato.stanford.edu": 0.8886226501737253,+ "blog.watsi.org": 0.8886099090504368,+ "www.vatican.va": 0.8886001974662521,+ "www.gridsagegames.com": 0.8885999017945436,+ "blog.wakemate.com": 0.8885718128809637,+ "www.guardian.co.uk": 0.8885578778095446,+ "www.eastbaytimes.com": 0.8885533867034545,+ "insights.ubuntu.com": 0.8885369737806736,+ "www.uber.com": 0.888502360897741,+ "hharnisc.github.io": 0.8884780095069668,+ "blog.eyas.sh": 0.8884641430053347,+ "adlrocha.substack.com": 0.8884402828343574,+ "www.agwa.name": 0.8883681305045957,+ "lexfridman.com": 0.8883669315681367,+ "www.hp.com": 0.8883541180846748,+ "chargen.matasano.com": 0.888350093124338,+ "docs.github.com": 0.888346612251366,+ "blogs.esa.int": 0.8883268060163584,+ "hackage.haskell.org": 0.8882716929118732,+ "www.askthepilot.com": 0.8882206989905654,+ "investor.fb.com": 0.8881776846453903,+ "hackertourism.com": 0.8881603547360667,+ "varianceexplained.org": 0.8881303906523573,+ "www.stanford.edu": 0.8881280238738776,+ "googleappengine.blogspot.com": 0.8881149640019546,+ "blog.spark.io": 0.8880887280047254,+ "danielmiessler.com": 0.8880772991848912,+ "blog.cloudant.com": 0.8880720742931111,+ "help.netflix.com": 0.888032506658475,+ "www.consumer.ftc.gov": 0.8880113570637971,+ "thechangelog.com": 0.8880056295154011,+ "mrdoob.com": 0.8879829721117504,+ "www.saveur.com": 0.8879678355351808,+ "www.moderntreasury.com": 0.8879300370907698,+ "blogs.suntimes.com": 0.8879083159007929,+ "staff.tumblr.com": 0.8878771427989084,+ "archive.wired.com": 0.8878447305921355,+ "www.ironicsans.com": 0.8878289779246122,+ "blog.capitalandgrowth.org": 0.8878130504973006,+ "www.thisiscolossal.com": 0.8878105455839786,+ "benchling.engineering": 0.8878049780860752,+ "www.symmetrymagazine.org": 0.8877876402671063,+ "www.feld.com": 0.8877667891871323,+ "everynoise.com": 0.8877314814814815,+ "input.fontbureau.com": 0.8877209761748434,+ "qiao.github.io": 0.8877030948951334,+ "factordaily.com": 0.887680355039973,+ "ecc-comp.blogspot.com": 0.887651452722378,+ "addyosmani.com": 0.8876225519241614,+ "reallifemag.com": 0.8876055526528979,+ "niedzejkob.p4.team": 0.8875742399340828,+ "www.cs.purdue.edu": 0.887560497701347,+ "vercel.com": 0.8875553045329022,+ "blog.sesse.net": 0.887545005792023,+ "mynoise.net": 0.8875240702669522,+ "nevalalee.wordpress.com": 0.8874709976798144,+ "www.justingarrison.com": 0.8874550740684668,+ "www.cse.buffalo.edu": 0.887431083352139,+ "jamesgolick.com": 0.8874059947114532,+ "gnunet.org": 0.887393102678099,+ "www.tampabay.com": 0.8873751025878571,+ "www.jide.com": 0.887368855995013,+ "producthabits.com": 0.8873611172314394,+ "www.fourhourworkweek.com": 0.8873408966020546,+ "refer.ly": 0.8873384440828381,+ "www.aei.org": 0.8873351640879104,+ "www.mentalfloss.com": 0.8873305275256577,+ "izbicki.me": 0.8873178158351525,+ "mathiasbynens.be": 0.8872781506210743,+ "www.pcjs.org": 0.8872494045912365,+ "arcan-fe.com": 0.887245072438425,+ "www.americamagazine.org": 0.8872386352495559,+ "mailman.nanog.org": 0.8872223978170161,+ "bluejekyll.github.io": 0.8872093023255814,+ "www.mysliderule.com": 0.8872093023255814,+ "blogs.oracle.com": 0.8871952202335835,+ "leonardofed.io": 0.8871763351184384,+ "www.sourcetrail.com": 0.8871714606492734,+ "www.linuxfromscratch.org": 0.8871714606492734,+ "bariweiss.substack.com": 0.8871587544964651,+ "moot.it": 0.8871570315823085,+ "popula.com": 0.887128977917166,+ "press.web.cern.ch": 0.887082617470916,+ "ownyourbits.com": 0.8870779976717112,+ "getstream.io": 0.8870262897116983,+ "scpd.stanford.edu": 0.8870121302239351,+ "dcgross.com": 0.8868939399089152,+ "unsupervisedmethods.com": 0.8868826572298285,+ "www.willmcgugan.com": 0.8868766525394289,+ "wiki.tcl-lang.org": 0.8868748835508599,+ "blog.disqus.com": 0.8868414318888773,+ "blogs.technet.microsoft.com": 0.8868323608446661,+ "tromp.github.io": 0.8867756037157464,+ "blog.ircmaxell.com": 0.8867580271160181,+ "mux.com": 0.8867579106919085,+ "bugs.mysql.com": 0.8867572809726417,+ "www.irishtimes.com": 0.8867543138710346,+ "dpc.pw": 0.8867103394982209,+ "dusted.codes": 0.8866966751368195,+ "quanticdev.com": 0.8866903311266173,+ "www.neowin.net": 0.8866503890472428,+ "chrislord.net": 0.886641393239182,+ "blog.otoro.net": 0.8866334244061989,+ "www.thenational.scot": 0.8866271172090046,+ "www.inquirer.com": 0.8865967718863557,+ "modelviewculture.com": 0.8865845558030554,+ "info.iet.unipi.it": 0.8865497076023392,+ "www.storehouse.co": 0.886548015153011,+ "www.anfractuosity.com": 0.8865113264892915,+ "danwang.co": 0.8864808931374831,+ "www.researchgate.net": 0.8864516461055575,+ "www.doxsey.net": 0.8864376154183435,+ "bugs.ruby-lang.org": 0.8864368986388406,+ "whyy.org": 0.8864168618266979,+ "itch.io": 0.8864120842802148,+ "amigalove.com": 0.8863992170741539,+ "people.mozilla.com": 0.8863952990075262,+ "medium.freecodecamp.com": 0.8863811065750393,+ "blog.doist.com": 0.8863606429850415,+ "www.swyx.io": 0.8863599205692547,+ "www.eidel.io": 0.886301279533414,+ "senrigan.io": 0.8862368459608536,+ "www.pvk.ca": 0.8861969908332605,+ "www.ccs.neu.edu": 0.8861380071332806,+ "cturt.github.io": 0.8861102984083253,+ "www.lysator.liu.se": 0.8860844730714155,+ "philcalcado.com": 0.8859388762978412,+ "lists.dragonflybsd.org": 0.8859388762978412,+ "microkerneldude.wordpress.com": 0.8859288068767922,+ "www.theringer.com": 0.8858962060628648,+ "bzg.fr": 0.8858719540548398,+ "www.crypto.com": 0.8858494594971499,+ "fibery.io": 0.8858217047605946,+ "kimonote.com": 0.885805964130514,+ "news.softpedia.com": 0.8857837864929501,+ "successfulsoftware.net": 0.8857784923477241,+ "blog.intercom.com": 0.8857763495575585,+ "www.embeddedrelated.com": 0.8857676106438698,+ "www.psychologytoday.com": 0.8857585049003665,+ "haveibeenpwned.com": 0.8856992747056773,+ "blog.ryotak.me": 0.8856929365305167,+ "blog.piston.rs": 0.8856929365305167,+ "joshmitteldorf.scienceblog.com": 0.8855962693776582,+ "www.reviews.com": 0.8855218005991683,+ "svpow.com": 0.8855122462369361,+ "blog.nahurst.com": 0.8855122462369361,+ "engineering.kablamo.com.au": 0.8854939252840581,+ "blog.adafruit.com": 0.8854467055810114,+ "earth.nullschool.net": 0.8854306983539961,+ "inventwithpython.com": 0.885393853053717,+ "chalkdustmagazine.com": 0.8853197575419666,+ "blog.objectmentor.com": 0.8852948669873847,+ "www.aholme.co.uk": 0.8852815790604109,+ "www.dkriesel.com": 0.8852756720656075,+ "vanemden.wordpress.com": 0.8852604927216077,+ "investor.google.com": 0.8852562818553033,+ "pipeline.corante.com": 0.8852387066889292,+ "action.eff.org": 0.8852266245403357,+ "blog.twitch.tv": 0.8852196608142338,+ "www.nand2tetris.org": 0.8852071005917159,+ "gpfault.net": 0.8851652261576463,+ "meshedinsights.com": 0.8851628232067584,+ "probcomp.csail.mit.edu": 0.8851438011782188,+ "missiveapp.com": 0.8851280394688669,+ "blog.sendgrid.com": 0.8851277655579799,+ "mitadmissions.org": 0.8851045058363526,+ "info.meteor.com": 0.8850985466953342,+ "thewirecutter.com": 0.8850944093158075,+ "www.aboveavalon.com": 0.8850896030144491,+ "haxe.org": 0.8850332564592883,+ "www.snarky.ca": 0.8850298847276258,+ "blog.mapillary.com": 0.8849756787053948,+ "www.alternet.org": 0.8849651154014133,+ "martinheinz.dev": 0.884952570972269,+ "www.stevebenjamins.com": 0.8849347568208777,+ "derwiki.tumblr.com": 0.8849077586898806,+ "www.kuro5hin.org": 0.8848948828327496,+ "www.archlinux.org": 0.8848794541675421,+ "www.bennjordan.com": 0.8848672501882364,+ "tromey.com": 0.8848672501882364,+ "medlifestyle.news": 0.8848240975158286,+ "hackpad.com": 0.8848240112405613,+ "zinascii.com": 0.8848232963497987,+ "heredragonsabound.blogspot.com": 0.8848232963497987,+ "pycon.blogspot.com": 0.8847972407140463,+ "thegenesisblock.com": 0.8847466884910953,+ "jayriverlong.github.io": 0.8846854627897014,+ "rkoutnik.com": 0.8846701041470997,+ "moalquraishi.wordpress.com": 0.8846503798740485,+ "blog.angularjs.org": 0.8846476203665491,+ "learnyousomeerlang.com": 0.8846416196747935,+ "blog.rubymotion.com": 0.8846052536362908,+ "www.berkshirehathaway.com": 0.8845734385329344,+ "bigstory.ap.org": 0.8845644349212194,+ "www.dji.com": 0.8845531462931538,+ "homes.cs.washington.edu": 0.884536530239123,+ "momjian.us": 0.8845249482528319,+ "anderspitman.net": 0.8845238095238095,+ "ubiquity.acm.org": 0.8845123969111347,+ "archive.vector.org.uk": 0.8844858003347821,+ "bitcoinfoundation.org": 0.8844841517497576,+ "esr.ibiblio.org": 0.884465525984111,+ "www.caranddriver.com": 0.8844455469996317,+ "www.bristol.ac.uk": 0.8844312836410105,+ "www.keacher.com": 0.8844188041447092,+ "www.planet.com": 0.8843849199350312,+ "www.braintreepayments.com": 0.8843684120647333,+ "www.topic.com": 0.8843258553540223,+ "www.lyx.org": 0.8843101335952557,+ "hal.inria.fr": 0.884278551246099,+ "blog.powerdns.com": 0.8842750467710181,+ "www.discovermeteor.com": 0.8842406817313907,+ "www.statista.com": 0.884226458408693,+ "diveintohtml5.org": 0.884224840703763,+ "shkrobius.livejournal.com": 0.8842248407037628,+ "danwin.com": 0.8842240470222248,+ "thingist.com": 0.8841700109782021,+ "sf.curbed.com": 0.8841684697837997,+ "www.artima.com": 0.8840774930170289,+ "www.gregegan.net": 0.8840639202320028,+ "veed.io": 0.8840295485188058,+ "www.kguttag.com": 0.8840223725080164,+ "www.passwordstore.org": 0.8839871084196906,+ "www.time.com": 0.8839776340050455,+ "patft.uspto.gov": 0.8839759157295332,+ "brikis98.blogspot.com": 0.8839185792660761,+ "luckypeach.com": 0.8839078718615478,+ "www.256kilobytes.com": 0.8839065999741562,+ "kamalmarhubi.com": 0.8838909108920787,+ "www.levels.fyi": 0.8838529264451486,+ "www.grubstreet.com": 0.8838363484488945,+ "kubernetes.io": 0.8838168079942494,+ "privpapers.ssrn.com": 0.8837957120914789,+ "finledger.com": 0.8837957120914789,+ "ircv3.net": 0.8837369391764082,+ "webtorrent.io": 0.8836930455635492,+ "animagraffs.com": 0.8836872834930155,+ "www.coinbase.com": 0.8836702182178287,+ "jobs.apple.com": 0.8836631610037271,+ "lifehacker.com": 0.88361719985225,+ "blog.mongodb.org": 0.883606109678017,+ "lobe.ai": 0.8835655842394832,+ "blogs.ams.org": 0.8835451368221627,+ "www.psychologicalscience.org": 0.8834931862797779,+ "www.autonews.com": 0.8834926699108666,+ "google-styleguide.googlecode.com": 0.8834824574967941,+ "venomousporridge.com": 0.8834619351905811,+ "www.bennorthrop.com": 0.8834521090229532,+ "blog.lusis.org": 0.8834395848799916,+ "www.defcon.org": 0.8834198543134106,+ "railstips.org": 0.8834172428920269,+ "sitn.hms.harvard.edu": 0.8834134615384616,+ "www.johnwdefeo.com": 0.8834118239436308,+ "www.metabase.com": 0.8833568908825655,+ "jordansmith.io": 0.8833386064130131,+ "blog.brokep.com": 0.8833292707180577,+ "davnicwil.com": 0.883290787723497,+ "daverupert.com": 0.8832817007531839,+ "www.internalpointers.com": 0.8832731648616126,+ "blog.scottnonnenberg.com": 0.8832680173548925,+ "webmasters.googleblog.com": 0.8832159517897961,+ "ivanceras.github.io": 0.8831795673307369,+ "www.inference.vc": 0.8831370240507179,+ "white.ucc.asn.au": 0.883132530120482,+ "dis.4chan.org": 0.8831325301204819,+ "mta.openssl.org": 0.8831205226464401,+ "blog.sourcerer.io": 0.8831047861347778,+ "callumprentice.github.io": 0.8830898580872119,+ "www.techtransparencyproject.org": 0.8830469812134585,+ "blog.adobe.com": 0.883040341397881,+ "agfundernews.com": 0.8830325567122732,+ "www.centauri-dreams.org": 0.8829955449207836,+ "documentcloud.github.com": 0.8829915560916767,+ "rust-lang.github.io": 0.8829915560916767,+ "portswigger.net": 0.8829877514947376,+ "tagide.com": 0.8829745187840874,+ "creativecommons.org": 0.8829668579998763,+ "flak.tedunangst.com": 0.8829644456502317,+ "ferrous-systems.com": 0.8829609951535142,+ "shapecatcher.com": 0.8829477368055727,+ "solovyov.net": 0.8829399933646876,+ "blog.pmarca.com": 0.8829123044832685,+ "linusakesson.net": 0.8829080057671795,+ "www.benfrederickson.com": 0.8828991573081899,+ "maps.stamen.com": 0.8828934647050843,+ "techcommunity.microsoft.com": 0.8828546011216626,+ "blog.pyston.org": 0.8828536336243845,+ "blog.freshdesk.com": 0.8828536336243845,+ "www.radionz.co.nz": 0.8828494689784901,+ "translate.googleusercontent.com": 0.882847058520687,+ "stephenmann.io": 0.8828104499622798,+ "www.berkeleyside.com": 0.8828101152681945,+ "engineering.zenpayroll.com": 0.8827759389076459,+ "googlefiberblog.blogspot.com": 0.8827390835242562,+ "nadiaeghbal.com": 0.8827255975365081,+ "chartio.com": 0.882704073135325,+ "slidetocode.com": 0.8826822967578939,+ "mic.com": 0.8826760832749148,+ "ayende.com": 0.8826726588726063,+ "sixrevisions.com": 0.8826715498136648,+ "www.ligo.caltech.edu": 0.8826619298426813,+ "www.insider.com": 0.8826223554956923,+ "grahamc.com": 0.8825699143811585,+ "www.sbcl.org": 0.882563699339324,+ "www-m.cnn.com": 0.8825634181290516,+ "advances.sciencemag.org": 0.8825504380314206,+ "codingvc.com": 0.8825483021452769,+ "blog.buoyant.io": 0.8825343736402197,+ "tools.google.com": 0.8825286182855726,+ "www.thisamericanlife.org": 0.8825206320161334,+ "www.livescience.com": 0.8825187990685426,+ "blog.udacity.com": 0.88248770629827,+ "www.mint.com": 0.8824829228307717,+ "www.quantstart.com": 0.8824744656067158,+ "i3wm.org": 0.882461673497088,+ "lugassy.net": 0.882461673497088,+ "renato.athaydes.com": 0.8824351262415586,+ "john.ankarstrom.se": 0.8824318706073884,+ "deis.com": 0.882398256845465,+ "blog.travis-ci.com": 0.8823494388558837,+ "akshat.posterous.com": 0.882348785313154,+ "blog.danieljanus.pl": 0.882305963719186,+ "www.lambdassociates.org": 0.882291549111701,+ "www.paypal-engineering.com": 0.8822900726859743,+ "macdaddy.io": 0.8822875933955345,+ "homepages.inf.ed.ac.uk": 0.882263750625994,+ "events.google.com": 0.8822491129403885,+ "isocpp.org": 0.882201264152672,+ "www.secjuice.com": 0.8821793252738344,+ "www.martinvigo.com": 0.8821771335964247,+ "osblog.stephenmarz.com": 0.8821656713562592,+ "business.financialpost.com": 0.8821599714093322,+ "thetech.com": 0.8821385176184691,+ "berkshirehathaway.com": 0.882138517618469,+ "harvardlawreview.org": 0.8821355848021567,+ "leafo.net": 0.8821225145186178,+ "curia.europa.eu": 0.882094739471072,+ "ckeditor.com": 0.8820783917249119,+ "www.topgear.com": 0.8820554259493689,+ "thebaffler.com": 0.8820445386842816,+ "wilwheaton.net": 0.8820435463334623,+ "andyadams.org": 0.8820328528134829,+ "applidium.com": 0.8820326538487594,+ "techspecs.blog": 0.8819755098672539,+ "cafbit.com": 0.8819755098672539,+ "community.spotify.com": 0.8819640976966642,+ "jacobinmag.com": 0.8819625349712696,+ "www.indiegogo.com": 0.881946367637731,+ "hueniverse.com": 0.8819393022613744,+ "tent.io": 0.8819180061465067,+ "blog.caranddriver.com": 0.8818992762989476,+ "www.avweb.com": 0.8818262516609122,+ "kieranhealy.org": 0.8817976429098323,+ "webrtchacks.com": 0.8817875203232487,+ "www.penzba.co.uk": 0.8817713693955629,+ "www.onbeing.org": 0.8817382469287067,+ "www.makeartwithpython.com": 0.8817227794803292,+ "storagemojo.com": 0.8816983007282381,+ "rentzsch.tumblr.com": 0.8816906014355783,+ "www.chrisnorstrom.com": 0.881678989119816,+ "mir.aculo.us": 0.8816552237217568,+ "automatetheboringstuff.com": 0.8815884675997907,+ "doc.cat-v.org": 0.8815699328665041,+ "skilldrick.github.io": 0.8815628815628815,+ "www.azulsystems.com": 0.8815545999084712,+ "googledevelopers.blogspot.com": 0.8815293986804532,+ "shopify.engineering": 0.8815078480198789,+ "www.windy.com": 0.8814945016309822,+ "gen.medium.com": 0.8814868144618807,+ "mygengo.com": 0.8814826810568291,+ "microship.com": 0.8814701304564254,+ "jroweboy.github.io": 0.881441670329882,+ "www.isro.gov.in": 0.8814323797659261,+ "amitness.com": 0.8814233401080115,+ "journal.neilgaiman.com": 0.8813000878314081,+ "blog.longnow.org": 0.8812928039894358,+ "developers.lyst.com": 0.881286326309607,+ "threads2.scripting.com": 0.8812146696862522,+ "www.intego.com": 0.8812141651552359,+ "abc7news.com": 0.8812015459094202,+ "ac.els-cdn.com": 0.8811985270068678,+ "zwischenzugs.wordpress.com": 0.8811838816065725,+ "ooni.torproject.org": 0.8811640746997138,+ "rodneybrooks.com": 0.8811464380886621,+ "mentalfloss.com": 0.8811323569439224,+ "www.elie.net": 0.8810667382996152,+ "www.eecs.berkeley.edu": 0.881035190029909,+ "surjan.substack.com": 0.8810020145474288,+ "metrics.torproject.org": 0.880999086515737,+ "www.scss.tcd.ie": 0.8809783370810794,+ "www.zdziarski.com": 0.8809560496711293,+ "blog.lizzie.io": 0.8809270824147091,+ "www.nntp.perl.org": 0.8809140700339988,+ "www.skorks.com": 0.8809070694481402,+ "www.lastweekinaws.com": 0.880898941183668,+ "projectgus.com": 0.8808911111787601,+ "togelius.blogspot.com": 0.8808652592717593,+ "www.thedailymuse.com": 0.880861164752333,+ "blog.mailchimp.com": 0.8808342908333803,+ "www.seas.harvard.edu": 0.8808287698175389,+ "kernelmag.dailydot.com": 0.8807834100859487,+ "www.audubon.org": 0.8807603148427023,+ "juretriglav.si": 0.8807541586341653,+ "www.ias.edu": 0.8806635755411035,+ "www.startupsopensourced.com": 0.8806386086637549,+ "scott.a16z.com": 0.880594330778045,+ "www.webdistortion.com": 0.8805671581809865,+ "coinbase.com": 0.8805536570003393,+ "my.matterport.com": 0.8805531859476551,+ "world.hey.com": 0.8805499381704851,+ "freenetproject.org": 0.8805418719211823,+ "ez.substack.com": 0.8805347456971939,+ "www.cryptologie.net": 0.880520955636949,+ "www.ewherry.com": 0.8804202710724481,+ "www.aaronklein.com": 0.8804202710724481,+ "www.metachris.com": 0.8803487337679655,+ "www.info.ucl.ac.be": 0.8803332113342701,+ "www.maritime-executive.com": 0.8802634318319039,+ "nationalpost.com": 0.8802495762864062,+ "stu.mp": 0.8802212225414684,+ "pcwalton.github.io": 0.8801825251719878,+ "www.lensrentals.com": 0.8801530918564299,+ "programmingzen.com": 0.8801076222388025,+ "applied-data.science": 0.8800802934751344,+ "www.2uo.de": 0.8800661016526783,+ "www.americanscientist.org": 0.8800620422490765,+ "vault.fbi.gov": 0.8800567908408685,+ "www.mymodernmet.com": 0.880048242675238,+ "www.stripes.com": 0.8799620726032865,+ "aralbalkan.com": 0.8799619967876905,+ "www.washingtonian.com": 0.8799468909598039,+ "jamey.thesharps.us": 0.8799345914752841,+ "visualwebsiteoptimizer.com": 0.8799281242616495,+ "blogs.uw.edu": 0.879925783709813,+ "blog.sqlizer.io": 0.8799186267288787,+ "workshop.chromeexperiments.com": 0.8799186267288787,+ "www.jamieweb.net": 0.8799112718655394,+ "royal.pingdom.com": 0.8798976823547263,+ "oag.ca.gov": 0.8798952586106802,+ "www.askamanager.org": 0.8798868933154113,+ "briarproject.org": 0.8798570681672088,+ "everythingsysadmin.com": 0.8798435781150284,+ "serokell.io": 0.8798171504945399,+ "mmcgrana.github.com": 0.8797647395096098,+ "ruslanspivak.com": 0.8797628893146511,+ "wesmckinney.com": 0.8797444121875875,+ "blog.mattbierner.com": 0.8797205044550149,+ "mit.edu": 0.8796628612944964,+ "www.asimovinstitute.org": 0.8796512842876961,+ "press.stripe.com": 0.8796512842876961,+ "technicshistory.wordpress.com": 0.8796210680092151,+ "common-lisp.net": 0.8796152653799312,+ "www.u-tokyo.ac.jp": 0.8796017299329183,+ "www.dailykos.com": 0.8795999531727511,+ "www.pagerduty.com": 0.8795889070772409,+ "austinkleon.com": 0.8795887664062606,+ "blog.last.fm": 0.8795848845621085,+ "blog.clearbit.com": 0.8795162875960222,+ "blog.thinkst.com": 0.8794805645736704,+ "hivelogic.com": 0.87947663783247,+ "hechingerreport.org": 0.8794286991256227,+ "www.fortressofdoors.com": 0.8794194991885873,+ "docs.docker.com": 0.8794114856568306,+ "kushagragour.in": 0.879405287790302,+ "ultimatehackingkeyboard.com": 0.8793982349555475,+ "developer.ibm.com": 0.8793941898423207,+ "rushter.com": 0.879388928994475,+ "www.tarsnap.com": 0.8793327066997211,+ "www.wtsp.com": 0.8793263869529092,+ "brohrer.github.io": 0.8793263869529092,+ "elementary.io": 0.8793165731880934,+ "destevez.net": 0.8793048124903404,+ "www.gentoo.org": 0.879269905778943,+ "nbviewer.jupyter.org": 0.879269721586867,+ "blog.angular.io": 0.879268923349318,+ "www.techrepublic.com": 0.8792309439284567,+ "www.virgin.com": 0.8792219903286642,+ "blog.getspool.com": 0.8792029887920298,+ "www.lucidchart.com": 0.8791838979662947,+ "blog.readability.com": 0.8791595821198683,+ "narrative.ly": 0.8791473124131408,+ "davidgerard.co.uk": 0.879133623229212,+ "www.tldp.org": 0.8791060662671288,+ "www.chillingeffects.org": 0.8790911678663772,+ "mattbruenig.com": 0.8790858191942302,+ "blog.coursera.org": 0.8790756346109743,+ "engineering.khanacademy.org": 0.8790360280222413,+ "aurellem.org": 0.8790251342720007,+ "www8.hp.com": 0.8789932584616722,+ "www.mrmoneymustache.com": 0.8789801725348548,+ "themetricsystem.rjmetrics.com": 0.8789799435157434,+ "popey.com": 0.8789713929868309,+ "xlinux.nist.gov": 0.8789408088929164,+ "ben-evans.com": 0.8789322960390088,+ "www.datapointed.net": 0.8789114224720275,+ "www.net.in.tum.de": 0.8788892296044377,+ "stratos.seas.harvard.edu": 0.8788892296044377,+ "www.smbc-comics.com": 0.878826000548314,+ "blog.gittip.com": 0.8788213217536158,+ "www.cloudflare.com": 0.8788059374287456,+ "rmurphey.com": 0.8787968488987687,+ "www.tuhs.org": 0.8787968488987687,+ "blog.openstreetmap.org": 0.8787897231673689,+ "www.istartedsomething.com": 0.8787889297497098,+ "ilyabirman.net": 0.8787732444342291,+ "www.github.com": 0.8787717233938399,+ "amix.dk": 0.8787529849051614,+ "www.insidehighered.com": 0.8787455322695712,+ "actu.epfl.ch": 0.8787380577275826,+ "www.buyforlife.com": 0.8787356591099948,+ "staysaasy.com": 0.8786604306758078,+ "www.okayhq.com": 0.8786525461110833,+ "engineering.mixpanel.com": 0.878632488020514,+ "www.netresec.com": 0.8786288106631085,+ "jott.live": 0.8786251117226983,+ "www.health.harvard.edu": 0.8786179702092969,+ "plot.ly": 0.8785945787217043,+ "www.edwardtufte.com": 0.8785389757836423,+ "pzemtsov.github.io": 0.8785192835169581,+ "grack.com": 0.878506371619462,+ "www.aeonmagazine.com": 0.878499144479296,+ "lowlevelbits.org": 0.8784722367844,+ "blog.dotcloud.com": 0.8784660513882162,+ "robwalling.com": 0.8784461152882206,+ "luajit.org": 0.8784380131632007,+ "mccormick.cx": 0.8784273467707013,+ "www.usehaystack.io": 0.8784273467707013,+ "stedolan.github.io": 0.8784164208081071,+ "supermemo.guru": 0.8784156694332751,+ "jackschaedler.github.io": 0.8783768191039716,+ "mgba.io": 0.87835268885913,+ "forum.nim-lang.org": 0.8783083943148878,+ "sympa.inria.fr": 0.8783048144562394,+ "lists.nongnu.org": 0.8783041478638947,+ "lispcookbook.github.io": 0.8782936010037641,+ "ds9a.nl": 0.8782772345442594,+ "thinkpol.ca": 0.8782726019439232,+ "itunes.apple.com": 0.8782672365580176,+ "www.coursera.org": 0.8782670165903833,+ "www.nrc.nl": 0.8782340201533003,+ "www.faa.gov": 0.8781736885382665,+ "artsandculture.google.com": 0.8781425787274354,+ "smerity.com": 0.8781378214625035,+ "www.omaha.com": 0.8781181772597978,+ "www.wweek.com": 0.8781063159893508,+ "www.stroustrup.com": 0.8780938693821273,+ "sacks.substack.com": 0.8780908854852072,+ "www.heraldsun.com.au": 0.8780821695385574,+ "wiki.alopex.li": 0.8780784699174555,+ "www.dartlang.org": 0.8780551661146612,+ "bugs.php.net": 0.878032279338754,+ "www.cyclingnews.com": 0.8779874213836478,+ "www.minix3.org": 0.8779874213836478,+ "www.cryptomuseum.com": 0.8779834887038765,+ "blog.sendhub.com": 0.8779734555774865,+ "web.williams.edu": 0.8779619274642066,+ "sourceware.org": 0.8779595104655152,+ "blog.revolutionanalytics.com": 0.8779228861519864,+ "blog.feabhas.com": 0.877905135054773,+ "rubenerd.com": 0.8778819789027571,+ "longnow.org": 0.8778804834133449,+ "www.kevinslin.com": 0.8778337531486146,+ "testing.googleblog.com": 0.8778159226448956,+ "merbist.com": 0.8778006932561861,+ "academic.oup.com": 0.8777955202058847,+ "www.romancortes.com": 0.8777775637794132,+ "ottverse.com": 0.8777664608276228,+ "panopticlick.eff.org": 0.8777635360894769,+ "www.presslabs.com": 0.8777093280795232,+ "www.archaeology.org": 0.8777019277787409,+ "www.qt.io": 0.8776867090078414,+ "www.rust-lang.org": 0.8776640621382866,+ "komonews.com": 0.8776519468194085,+ "petewarden.typepad.com": 0.877650269187265,+ "redecentralize.org": 0.8776475140888633,+ "spencerdailey.com": 0.8776383740955995,+ "ryanbigg.com": 0.8776375837266788,+ "codeascraft.com": 0.8776350192149127,+ "davidsimon.com": 0.8776245936362499,+ "nodered.org": 0.8775609145071156,+ "releases.llvm.org": 0.8775465677232488,+ "blog.npmjs.org": 0.8775321171093747,+ "nothings.org": 0.8775219068062873,+ "www.minecraft.net": 0.8775149658488136,+ "www.nber.org": 0.8774912434535365,+ "docs.keydb.dev": 0.8774559444971969,+ "s3.amazonaws.com": 0.8774127148800118,+ "www.startuplessonslearned.com": 0.8773991243759531,+ "lostinjit.blogspot.com": 0.8773967208056322,+ "unity3d.com": 0.8773730560374686,+ "rtpg.co": 0.8773704171934261,+ "meteuphoric.wordpress.com": 0.8773701347020316,+ "c9.io": 0.8772877729098074,+ "davidad.github.io": 0.8772551751572703,+ "www.cs.tufts.edu": 0.8772142384084949,+ "patents.google.com": 0.8771976309077076,+ "carlschwan.eu": 0.8771903919813752,+ "www.disneyresearch.com": 0.8771847497268329,+ "kilianvalkhof.com": 0.8771650377417883,+ "easylang.online": 0.8771572953832314,+ "simonschreibt.de": 0.87714784842864,+ "answers.onstartups.com": 0.8771043002987845,+ "casetext.com": 0.8770972548772688,+ "thekidshouldseethis.com": 0.8770970221777706,+ "blog.bittorrent.com": 0.8770863340546824,+ "www.phrack.org": 0.8770711101073438,+ "blog.deepgram.com": 0.8770646346636887,+ "grooveshark.com": 0.8770646346636887,+ "www.stat.cmu.edu": 0.8770412624073355,+ "matthewrayfield.com": 0.8770299470017662,+ "blog.merovius.de": 0.8769629784425906,+ "jordan-wright.github.io": 0.8769261031957445,+ "www.michael-noll.com": 0.876870919176529,+ "www.detroitnews.com": 0.876865566511843,+ "www.chromestatus.com": 0.8768237137831741,+ "spacy.io": 0.8767936477263165,+ "blog.theoldreader.com": 0.8767904369417293,+ "larrysummers.com": 0.8767904369417293,+ "devo.ps": 0.8767685151051207,+ "jackg.org": 0.8767448498103232,+ "www.leancrew.com": 0.8767166755854882,+ "blog.smartdec.net": 0.876696975867677,+ "aaronparecki.com": 0.8766851750370125,+ "eandt.theiet.org": 0.8766793394897466,+ "www.lennysnewsletter.com": 0.8766464219782513,+ "blog.factual.com": 0.8766406692255209,+ "archlinux.me": 0.8765956164893757,+ "cglab.ca": 0.8765746912638428,+ "meteor.com": 0.8765739666365363,+ "blog.jpalardy.com": 0.8765005864209046,+ "amitkapila16.blogspot.com": 0.8764939037562264,+ "en.yna.co.kr": 0.8764898808598593,+ "linux.slashdot.org": 0.8764680273690748,+ "www.trevormckendrick.com": 0.8764571075800416,+ "cmcenroe.me": 0.8764460213566705,+ "robertovitillo.com": 0.8764453376914981,+ "metacademy.org": 0.8764331210191083,+ "lethain.com": 0.8764062511281379,+ "www.news.com.au": 0.876395323072595,+ "www.severe-weather.eu": 0.8763844603500262,+ "www.federalreserve.gov": 0.8763690497257808,+ "passcode.csmonitor.com": 0.8763573796830383,+ "chrismorgan.info": 0.8763466464048606,+ "www.kurzweilai.net": 0.8763277380739884,+ "www.federalregister.gov": 0.8763029852387336,+ "carolchen.me": 0.8762960630948078,+ "scholar.princeton.edu": 0.8762929484619637,+ "limi.net": 0.8762759528715298,+ "sijmen.ruwhof.net": 0.8761846124088934,+ "winaero.com": 0.8761846124088934,+ "www.techquila.co.in": 0.8761680078483304,+ "www.theamericanconservative.com": 0.876167214287758,+ "faculty.washington.edu": 0.8761620699999401,+ "marijnhaverbeke.nl": 0.876139575410831,+ "swizec.com": 0.8761258858184888,+ "www.thebestpageintheuniverse.net": 0.876117496807152,+ "www.epsilontheory.com": 0.8761124613754698,+ "www.exurbe.com": 0.8761030084533606,+ "www.motortrend.com": 0.8760487843348314,+ "blog.seattlepi.com": 0.8759960447121663,+ "www.enlightenment.org": 0.8759950518819543,+ "www.seriouseats.com": 0.8759880790728855,+ "appleinsider.com": 0.8759788729692606,+ "stilldrinking.org": 0.8759642287018524,+ "www.holistics.io": 0.8759580367004831,+ "www.dannyguo.com": 0.8759212674654512,+ "letsworkshop.com": 0.8759027598733787,+ "www.androidauthority.com": 0.8758978540777702,+ "visalist.io": 0.8758718179278925,+ "www.makelinux.net": 0.8758439962639901,+ "avikdas.com": 0.8758367116558062,+ "www.linkedin.com": 0.8757903442097913,+ "wiki.freebsd.org": 0.8757706173362011,+ "www.tandfonline.com": 0.8757698347226249,+ "www.pouet.net": 0.8757326866058377,+ "blocksandfiles.com": 0.8756986319076105,+ "www.docracy.com": 0.8756913215448502,+ "www.useit.com": 0.8756472373396756,+ "www.sapiens.org": 0.8756019742944366,+ "www.ccn.com": 0.8755951918062195,+ "www.mattblodgett.com": 0.8755948781981208,+ "www.france24.com": 0.8755846335815239,+ "www.zacharyburt.com": 0.8755387620665042,+ "www.congress.gov": 0.8755256407121746,+ "fab.cba.mit.edu": 0.8755115130516663,+ "newsroom.mastercard.com": 0.8754825096495608,+ "youexec.com": 0.8754809119780987,+ "pragmaticpineapple.com": 0.8754604704436907,+ "ruder.io": 0.8754604704436906,+ "www.cpomagazine.com": 0.8754507502798615,+ "fontsinuse.com": 0.8754438392621019,+ "www.uscis.gov": 0.8754248448695789,+ "youtube.com": 0.8754200542585205,+ "movio.co": 0.8754182523171723,+ "www.justwatch.com": 0.8754182523171723,+ "www.seeedstudio.com": 0.8754129248508049,+ "electrospaces.blogspot.com": 0.8754013788662113,+ "journal.dedasys.com": 0.8753840466834827,+ "css-tricks.com": 0.8753766177219949,+ "googleenterprise.blogspot.com": 0.8753419411917504,+ "radian.org": 0.8753349762136738,+ "futurism.com": 0.8752759686541421,+ "brandon.invergo.net": 0.875259685377418,+ "blogs.bmj.com": 0.8752587898620211,+ "cs.stanford.edu": 0.8752572315765397,+ "www.law.nyu.edu": 0.8752557472640224,+ "madebymike.com.au": 0.8752557472640223,+ "www.seomoz.org": 0.8752420123288777,+ "blog.invisiblethings.org": 0.8752416201881567,+ "www.greghendershott.com": 0.8752189373052796,+ "frankchimero.com": 0.8752104961969256,+ "www.dodgycoder.net": 0.8752104961969256,+ "txt.fyi": 0.8751961676198574,+ "www.cisa.gov": 0.8751827659203362,+ "www.failory.com": 0.8751736016299931,+ "jamanetwork.com": 0.8751682514227755,+ "www.objc.io": 0.8751399221981017,+ "duo.com": 0.875136824237271,+ "gs.statcounter.com": 0.8751360927111078,+ "futurice.com": 0.8751180437027989,+ "sifted.eu": 0.8751010675262672,+ "laravel-news.com": 0.8750758486776293,+ "archagon.net": 0.8750478337445297,+ "tinyletter.com": 0.8750075762218684,+ "blog.scoutapp.com": 0.875005539344818,+ "tomorrowcorporation.com": 0.8750053569816645,+ "blogs.opera.com": 0.8749949083342682,+ "m.sfgate.com": 0.8749578015755867,+ "blog.scottlogic.com": 0.8749532936904543,+ "www.vincit.fi": 0.8749482978568669,+ "www.hiringlab.org": 0.8749298773541315,+ "adioso.com": 0.8749298773541315,+ "technology.blog.gov.uk": 0.8748916088022032,+ "intelligence.org": 0.874874784745018,+ "musicmachinery.com": 0.874874767784566,+ "www.spaceref.com": 0.8748480825746138,+ "raisingfutureadults.blogspot.com": 0.8748200301964755,+ "fantasticanachronism.com": 0.8747762139154207,+ "calculist.org": 0.8747762139154207,+ "lizthedeveloper.com": 0.8747656784427211,+ "elliotjaystocks.com": 0.8747563598722315,+ "www.julian.com": 0.8747501335716515,+ "garbagecollected.org": 0.8747266158827341,+ "timdaub.github.io": 0.8746432384113364,+ "www.good.is": 0.8745679480603733,+ "www.elidedbranches.com": 0.8745591524175645,+ "people.gnome.org": 0.8745420458214501,+ "goteleport.com": 0.8745075530736341,+ "rome.ro": 0.8744782906989623,+ "molily.de": 0.8744677514854211,+ "www.douglas-self.com": 0.8744450764890996,+ "www.rubymotion.com": 0.874400769900326,+ "www.shubhro.com": 0.8743836350202087,+ "blog.jim-nielsen.com": 0.8743167857862288,+ "www.deseret.com": 0.8743016225209655,+ "www.dallasnews.com": 0.8742922264773864,+ "purecss.io": 0.874271743877152,+ "lowercasecapital.com": 0.874271743877152,+ "blog.getpaint.net": 0.874271743877152,+ "google.com": 0.8742545289746798,+ "betabeat.com": 0.874238729033988,+ "dawn.cs.stanford.edu": 0.8742346653711291,+ "futureofcoding.org": 0.8741693203974006,+ "www.ocregister.com": 0.87415339402842,+ "materialize.com": 0.8741062320515165,+ "teddziuba.com": 0.8740975697255843,+ "www.thelocal.de": 0.8740894106328552,+ "webaim.org": 0.8740351003371842,+ "www.firstthings.com": 0.8739883726292156,+ "www.yodaiken.com": 0.8739612387765462,+ "www.goodcheapandfast.com": 0.87394184329453,+ "fastcompression.blogspot.com": 0.8739308258375799,+ "perl6advent.wordpress.com": 0.8739142963181532,+ "www.shirky.com": 0.873897843474887,+ "www.baltimoresun.com": 0.873865725506546,+ "www.greaterwrong.com": 0.8738154479524215,+ "bastian.rieck.me": 0.8738148485847019,+ "toastytech.com": 0.8738123717532574,+ "raw.githubusercontent.com": 0.8738090863060529,+ "www.wealthsimple.com": 0.8737884336019249,+ "sanfrancisco.cbslocal.com": 0.8737831715062644,+ "blog.palantir.com": 0.8736979166666666,+ "learngitbranching.js.org": 0.8736979166666666,+ "arr.am": 0.8736979166666666,+ "i.blackhat.com": 0.8736899056814162,+ "lolware.net": 0.873689463390705,+ "sherylcanter.com": 0.8736870447303569,+ "andrewgelman.com": 0.8736823027610198,+ "www.stat.columbia.edu": 0.8736693395358077,+ "pouchdb.com": 0.8736594278511989,+ "www.compose.io": 0.8736483544069877,+ "mleverything.substack.com": 0.8736159033763847,+ "work.qz.com": 0.8736112922383825,+ "lucasmreis.github.io": 0.8736055963750089,+ "news.itsfoss.com": 0.8736019609008807,+ "wiki.nikitavoloboev.xyz": 0.8735941094898686,+ "developer.nvidia.com": 0.873569853920636,+ "antigonejournal.com": 0.8734640566717642,+ "spectator.us": 0.8734631581578451,+ "www.businesswire.com": 0.8734611836531004,+ "stressgrid.com": 0.87343280677026,+ "pgbovine.net": 0.8734068690708945,+ "forums.tigsource.com": 0.8733979453959029,+ "thenewstack.io": 0.8733971641405377,+ "blog.adamsmith.cc": 0.8733885158313304,+ "www.gallup.com": 0.8733177322868378,+ "www.institutionalinvestor.com": 0.873174323391906,+ "allendowney.blogspot.com": 0.8731681454070807,+ "www.lowrisc.org": 0.873141850865328,+ "jshakespeare.com": 0.8731286306597288,+ "www.nasaspaceflight.com": 0.8731126994667253,+ "divan.github.io": 0.873106828361247,+ "emacsconf.org": 0.873106828361247,+ "www.scottporad.com": 0.8731061096250692,+ "cam.ly": 0.8731039371172824,+ "en.wikibooks.org": 0.8730999086484691,+ "peak5390.wordpress.com": 0.8730838690655434,+ "www.boredpanda.com": 0.8730796077702506,+ "adam.heroku.com": 0.8730645256354151,+ "transportevolved.com": 0.8730606539065758,+ "jamesfriend.com.au": 0.8730468292001098,+ "sethrobertson.github.io": 0.8730063430759272,+ "kennethreitz.com": 0.872966789255468,+ "nigeltao.github.io": 0.8728857599922942,+ "www.damtp.cam.ac.uk": 0.8728857599922942,+ "blog.chewxy.com": 0.872876574828518,+ "savannah.gnu.org": 0.8728754513879475,+ "www.sfexaminer.com": 0.872849372895737,+ "engineering.pinterest.com": 0.8727923546520051,+ "hitchdev.com": 0.8727791664778902,+ "www.airlineratings.com": 0.8727662828334155,+ "dt.deviantart.com": 0.8727436710427438,+ "blog.dustincurtis.com": 0.8727333641616176,+ "andymatuschak.org": 0.8727040209571875,+ "ccc.de": 0.8727034120734908,+ "www.cs.bham.ac.uk": 0.8726465691109567,+ "www.robertputt.co.uk": 0.8726417417469949,+ "www.miamiherald.com": 0.8725798697059735,+ "blog.fitbit.com": 0.8725623923381794,+ "adrianchadd.blogspot.com": 0.8725623923381794,+ "developer.amazon.com": 0.8725538657953651,+ "blog.codeship.com": 0.8725534393008655,+ "www.attn.com": 0.8725475818041069,+ "paleotronic.com": 0.8725451217616845,+ "www.cdixon.org": 0.8725371731911513,+ "cuddly-octo-palm-tree.com": 0.8725361366622865,+ "www.flownet.com": 0.8725345822755496,+ "www.sacrideo.us": 0.8725261893403989,+ "newyorker.com": 0.8725219622684063,+ "marmelab.com": 0.8725088072483861,+ "help.twitter.com": 0.8724643326336655,+ "browser.geekbench.com": 0.8724643326336655,+ "blairreeves.me": 0.8724601687001106,+ "thesquareplanet.com": 0.872441172969379,+ "thesephist.com": 0.8724167847627841,+ "www.freenas.org": 0.8723746839925903,+ "kwokchain.com": 0.872332880121703,+ "bostinnovation.com": 0.8723328354214289,+ "love2d.org": 0.8723183994033862,+ "en.people.cn": 0.872305563134482,+ "blog.sucuri.net": 0.8722868151723697,+ "tink.uk": 0.87225641869954,+ "developer.palm.com": 0.8722310220579464,+ "inessential.com": 0.8722306989053726,+ "latimesblogs.latimes.com": 0.8722297510115247,+ "startuplessonslearned.blogspot.com": 0.8721812850079301,+ "blog.qt.digia.com": 0.8721548264341702,+ "cssdeck.com": 0.8721274930836064,+ "medium.freecodecamp.org": 0.8720631693223321,+ "www.philly.com": 0.8720354776247603,+ "nat.org": 0.8719937064288694,+ "www.architecturaldigest.com": 0.871986927054336,+ "adactio.com": 0.8719856028794241,+ "anvil.works": 0.8719506746013774,+ "accidentallyquadratic.tumblr.com": 0.8719251801589324,+ "arduino.cc": 0.8718637797679766,+ "blog.embed.ly": 0.8718082655702977,+ "anyvite.com": 0.8718025672263069,+ "www.eater.com": 0.871748228284988,+ "moxie.org": 0.8717373817367493,+ "www.jacquescorbytuech.com": 0.8717101335367482,+ "n8n.io": 0.8716898518049913,+ "commonsware.com": 0.871674162722329,+ "www.snoyman.com": 0.8716637912464283,+ "blog.patricktriest.com": 0.87161890252714,+ "leaf.dragonflybsd.org": 0.8716189025271399,+ "www.gamedeveloper.com": 0.8716189025271399,+ "www.decisionsciencenews.com": 0.8716189025271399,+ "primitivetechnology.wordpress.com": 0.8716187818992139,+ "www.marinetraffic.com": 0.8716117229392578,+ "www.dallasobserver.com": 0.8716085858849908,+ "www.consumeraffairs.com": 0.871556493468071,+ "english.aljazeera.net": 0.8715487596633263,+ "eng.paxos.com": 0.871523178807947,+ "scobleizer.com": 0.8715026146236272,+ "endler.dev": 0.871459748976304,+ "www.assemblyai.com": 0.871459748976304,+ "spaceweatherarchive.com": 0.8714580347584702,+ "www.todayifoundout.com": 0.871448243747315,+ "www.itprotoday.com": 0.8714363693988826,+ "physics.stackexchange.com": 0.8714320610312986,+ "www.lifespan.io": 0.8714320342847328,+ "pspdfkit.com": 0.8714289642359972,+ "www.slackware.com": 0.8714078524585236,+ "economix.blogs.nytimes.com": 0.8713770585361653,+ "imba.io": 0.8713614361461072,+ "blog.wizsec.jp": 0.8713527851458885,+ "neal.fun": 0.8713508609752882,+ "shift.newco.co": 0.8713309282655746,+ "m50d.github.io": 0.8712974347864724,+ "c.learncodethehardway.org": 0.8712925320326951,+ "hosted.ap.org": 0.8712897256940902,+ "www.onebigfluke.com": 0.8712867098269154,+ "getmagicnow.com": 0.8712820888223272,+ "projects.thestar.com": 0.8712523001988615,+ "www.ralfj.de": 0.8712321679229643,+ "www.vantage.sh": 0.871207076528529,+ "www.linuxfoundation.org": 0.8711369679250435,+ "openclassroom.stanford.edu": 0.8711347101538987,+ "usefyi.com": 0.8711273706889674,+ "www.bingocardcreator.com": 0.8711151425360408,+ "blog.robenkleene.com": 0.8711151425360408,+ "mako.cc": 0.8711144740115356,+ "blog.frontapp.com": 0.8710729457593172,+ "virtuallyfun.com": 0.8710620996646494,+ "www.csmonitor.com": 0.8710243224833312,+ "www.nintendo.co.uk": 0.8710106382978724,+ "codon.com": 0.8709949725685586,+ "www.sixthtone.com": 0.870972915708058,+ "statweb.stanford.edu": 0.8709617791625288,+ "www.selectedintelligence.com": 0.8709463428439262,+ "www.elasticsearch.org": 0.8709352679197123,+ "math.mit.edu": 0.870930298916698,+ "dwheeler.com": 0.870930298916698,+ "kim.com": 0.8709205575405985,+ "www.nbcsandiego.com": 0.8708959915522153,+ "paragonie.com": 0.8708785801404334,+ "bostonherald.com": 0.8708776881584194,+ "mathwithbaddrawings.com": 0.8708565076256994,+ "www.openshot.org": 0.8708388814913449,+ "bugs.python.org": 0.8708284581145338,+ "blog.aerofs.com": 0.8708080233174353,+ "www.chicagomag.com": 0.8707562016036294,+ "www.250bpm.com": 0.8707223328636163,+ "datagenetics.com": 0.8706854622162113,+ "blogs.embarcadero.com": 0.870674156132316,+ "www.ispreview.co.uk": 0.8706343121704596,+ "www.twistlock.com": 0.8706274964354553,+ "www.lukew.com": 0.8705263186498488,+ "news.efinancialcareers.com": 0.8705226226571788,+ "www.travelandleisure.com": 0.8705014801192988,+ "blog.okturtles.com": 0.8704939919893191,+ "ilsr.org": 0.8704939919893191,+ "www.usb.org": 0.8704796759014012,+ "dish.andrewsullivan.com": 0.8704773071162746,+ "likewise.am": 0.8704734916123844,+ "blog.idonethis.com": 0.8704583890982127,+ "blog.talosintelligence.com": 0.8704526519003233,+ "googlesystem.blogspot.com": 0.8704373858850157,+ "garrysub.posterous.com": 0.8704372803921456,+ "insinuator.net": 0.8704372803921456,+ "hypothes.is": 0.8704167324964841,+ "cdt.org": 0.8704025091193442,+ "apod.nasa.gov": 0.8703900488888665,+ "herman.bearblog.dev": 0.8703870042603329,+ "archiveteam.org": 0.8703718279113771,+ "seths.blog": 0.8703521175616137,+ "blog.internetcases.com": 0.8703208556149733,+ "olifro.st": 0.870316353450948,+ "www.commerce.gov": 0.8703148755143479,+ "cosmosmagazine.com": 0.8703108288261813,+ "onemileatatime.com": 0.8703062566192509,+ "blog.cardiogr.am": 0.8702780879767176,+ "www.nsf.gov": 0.8702217832404014,+ "michaelfeathers.silvrback.com": 0.8702081612825068,+ "cse.buffalo.edu": 0.8701754328744842,+ "www.cardiff.ac.uk": 0.8701754328744842,+ "stuartkhall.com": 0.8701745691078596,+ "www.ablogtowatch.com": 0.8701496552122142,+ "kripken.github.io": 0.8701188385936537,+ "www.submarinecablemap.com": 0.8700961746199228,+ "blog.pandora.com": 0.8700581031783425,+ "undark.org": 0.8700047089370274,+ "indiestatik.com": 0.8699989528159702,+ "www.ldeming.com": 0.8699989528159702,+ "www.shadertoy.com": 0.8699979474186827,+ "blog.forecast.io": 0.8699840133880231,+ "www.thumbtack.com": 0.8699483761483322,+ "neopythonic.blogspot.com": 0.8699126451206446,+ "blog.yoshuawuyts.com": 0.8699103901809758,+ "meaningness.com": 0.8698796561423923,+ "www.climatecentral.org": 0.8698732077608144,+ "phasenoise.livejournal.com": 0.8698692075920079,+ "solarianprogrammer.com": 0.8698554554648813,+ "www.allenpike.com": 0.869850220276853,+ "prometheus.io": 0.8698088579434151,+ "www.zocalopublicsquare.org": 0.8698034656609108,+ "jsonresume.org": 0.8697986577181208,+ "ts.data61.csiro.au": 0.8697986577181208,+ "downloadsquad.switched.com": 0.8697589511534172,+ "chris-said.io": 0.8697498026681764,+ "www.eurekalert.org": 0.8697479182812706,+ "rise.cs.berkeley.edu": 0.8697428562146504,+ "nav.al": 0.8696282036759441,+ "www.contextis.com": 0.8696220176972772,+ "alpha.app.net": 0.8695846584298926,+ "simon.shimmerproject.org": 0.8695798593001582,+ "blog.andyet.com": 0.8695385830681973,+ "harvardmagazine.com": 0.8695167508879433,+ "montrealgazette.com": 0.8695147149611512,+ "www.ossblog.org": 0.8694501584181511,+ "www.snazz.xyz": 0.8694481830417228,+ "securethoughts.com": 0.8694070114575367,+ "www.dcrainmaker.com": 0.8694035765566716,+ "static.googleusercontent.com": 0.869401063536612,+ "www.cyberscoop.com": 0.8693847613582892,+ "engineering.imvu.com": 0.8693718520967112,+ "cs.nyu.edu": 0.869371790902293,+ "github.hubspot.com": 0.8693590367674685,+ "zenodo.org": 0.8693126875353983,+ "www.nczonline.net": 0.8693117689767528,+ "awealthofcommonsense.com": 0.8692734628001552,+ "jcp.org": 0.8692722371967655,+ "www.tabletmag.com": 0.869255146058989,+ "www.spritzinc.com": 0.8692337048510355,+ "www.pl-enthusiast.net": 0.8692303984396749,+ "blog.rtwilson.com": 0.8692255308065057,+ "parislemon.com": 0.8692052486630297,+ "www.perimeterx.com": 0.869171185060124,+ "blog.ndepend.com": 0.8691625738297151,+ "www.gregreda.com": 0.8691455800391642,+ "www.cooper.com": 0.8691426419439985,+ "newsroom.ibm.com": 0.869132356539317,+ "www.core77.com": 0.8691279702829294,+ "paraschopra.com": 0.869122539211249,+ "openspace.sfmoma.org": 0.8691093062023977,+ "blog.stephenmarz.com": 0.8691082953129042,+ "blog.workshape.io": 0.8690653365846199,+ "ngrok.com": 0.8690653365846199,+ "code.flickr.net": 0.8690543850923815,+ "uk.pcmag.com": 0.8690031200561459,+ "superpowered.com": 0.868989779225976,+ "hunterwalk.medium.com": 0.8689812194478416,+ "www.cs.tau.ac.il": 0.8689490020361462,+ "littlemaninmyhead.wordpress.com": 0.8689216938948838,+ "www.excamera.com": 0.8689189189189189,+ "bjorn.tipling.com": 0.8689189189189189,+ "people.idsia.ch": 0.8688594113666208,+ "www.olark.com": 0.8688594113666208,+ "blog.sigplan.org": 0.8688223104414097,+ "www.volokh.com": 0.8687744918859344,+ "betterprogramming.pub": 0.8687724684006338,+ "www.consumerreports.org": 0.8687417291110767,+ "spin.atomicobject.com": 0.8687241428724921,+ "opencv.org": 0.8687019086527302,+ "evincarofautumn.blogspot.com": 0.8687019086527302,+ "blog.imgtec.com": 0.868681889755656,+ "pauladamsmith.com": 0.86866369377526,+ "neosmart.net": 0.8686249309907671,+ "reprints.longform.org": 0.8686085856898127,+ "www.xda-developers.com": 0.8685957498546732,+ "reefpoints.dockyard.com": 0.8685213588812543,+ "benguild.com": 0.8684664223004914,+ "www.naughtycomputer.uk": 0.8684379703659447,+ "www.kb.cert.org": 0.8684268393607657,+ "www.insidescience.org": 0.868400521356853,+ "motorola-blog.blogspot.com": 0.8683887736567877,+ "op-co.de": 0.8683853459972863,+ "pvk.ca": 0.8683840412350319,+ "roadsandkingdoms.com": 0.8683501222568618,+ "www.netbsd.org": 0.8682992634857859,+ "lite.cnn.io": 0.8682859224905237,+ "redeye.firstround.com": 0.8682663428825431,+ "forum.xda-developers.com": 0.8682587280913919,+ "posthog.com": 0.8682495044484383,+ "ftp.iza.org": 0.868245075106375,+ "craphound.com": 0.8682258194103606,+ "leancrew.com": 0.8682221213291426,+ "www.rijksmuseum.nl": 0.8682073993067194,+ "blog.newsweek.com": 0.8681920308533846,+ "www.hipmunk.com": 0.8681741501104386,+ "fuchsia.dev": 0.8681336649460718,+ "infrequently.org": 0.8680443914407565,+ "www.nationalreview.com": 0.8680234764373578,+ "www.futilitycloset.com": 0.8680090490995264,+ "blogs.apache.org": 0.8679900476857304,+ "www.hayabusa2.jaxa.jp": 0.8679812862496842,+ "www.lenovo.com": 0.867894246896281,+ "malisper.me": 0.8678702820090507,+ "people.debian.org": 0.8678402113450426,+ "blog.jwhitham.org": 0.8678305149745539,+ "hai.stanford.edu": 0.8677844697267021,+ "hubicka.blogspot.com": 0.8677805560584517,+ "bemmu.posterous.com": 0.8677440846959065,+ "www.gizmodo.com.au": 0.8677203766504066,+ "meyerweb.com": 0.8676812093352608,+ "www.novaspivack.com": 0.8676756246000905,+ "www.theamericanscholar.org": 0.8676692275096932,+ "www.pjrc.com": 0.8676602643652892,+ "googleforwork.blogspot.com": 0.8676558691706335,+ "get.slack.help": 0.8676558691706335,+ "mwi.usma.edu": 0.8676236271344514,+ "blogs.zoho.com": 0.8676226844533295,+ "www.aaronstannard.com": 0.8676196165257838,+ "www.messynessychic.com": 0.867572253713323,+ "uk.businessinsider.com": 0.8675716785772413,+ "www.defectivebydesign.org": 0.867569136323548,+ "bash.org": 0.8675094991983033,+ "venturehacks.com": 0.8675094413288004,+ "www.thepublicdiscourse.com": 0.8674973517990539,+ "browser.engineering": 0.8674925582906508,+ "www.peachesnstink.com": 0.8674597307626667,+ "www.ics.uci.edu": 0.8674482943120847,+ "www.cs.utah.edu": 0.8674397196409499,+ "www.inkstonenews.com": 0.8674036054993348,+ "www.privacytools.io": 0.8673748578488399,+ "brianknapp.me": 0.8673589537571962,+ "www.mediaite.com": 0.867339960916398,+ "www.theinquirer.net": 0.8673345006090067,+ "www.dhruvonmath.com": 0.867331172642827,+ "lucb1e.com": 0.8672987335530669,+ "www.hetzner.de": 0.8672930834878059,+ "blogs.nature.com": 0.8672878554675312,+ "newsfeed.time.com": 0.8672878554675312,+ "davemart.in": 0.8672750734639297,+ "cryptopals.com": 0.8672750734639297,+ "paperswithcode.com": 0.8672082397249838,+ "blog.hipmunk.com": 0.8671864413349992,+ "citizen428.net": 0.8671759938812859,+ "shitpost.plover.com": 0.8671759938812859,+ "developer.github.com": 0.8671757822917692,+ "www.lispcast.com": 0.86717529476292,+ "www.webofstories.com": 0.8671647243985804,+ "blog.asana.com": 0.8671537327631247,+ "vvvvalvalval.github.io": 0.8671420654436023,+ "news.northeastern.edu": 0.8671175664071602,+ "www.digitalmars.com": 0.8671085676412111,+ "www.mysk.blog": 0.8671085676412111,+ "www.janestreet.com": 0.8671085676412111,+ "francescodilorenzo.com": 0.8671085676412111,+ "www.binghamton.edu": 0.8671020117333935,+ "measuringmeasures.blogspot.com": 0.8671015509192971,+ "blog.businessofsoftware.org": 0.8670987836087559,+ "www.dougengelbart.org": 0.8670867654048934,+ "tkainrad.dev": 0.8670687156162842,+ "www.machinelearningplus.com": 0.8670617301641067,+ "200ok.ch": 0.8670503631928145,+ "quizlet.com": 0.8669907032962705,+ "www.singlelunch.com": 0.8669898915969994,+ "chairnerd.seatgeek.com": 0.8669768798830123,+ "www.govtech.com": 0.8669681804482694,+ "news.mongabay.com": 0.8669281682982936,+ "erikmcclure.com": 0.8669236451292578,+ "aelkus.github.io": 0.8669193008503213,+ "www.pyimagesearch.com": 0.8669169539583714,+ "www.thejournal.ie": 0.8669063286580795,+ "reaktor.com": 0.8669062156202597,+ "christophermeiklejohn.com": 0.8668972800697652,+ "globaleconomicanalysis.blogspot.com": 0.8668972800697652,+ "reasonml.github.io": 0.8668408053269024,+ "www.csiro.au": 0.8668352643092083,+ "www.maxmasnick.com": 0.8668020425330096,+ "www.bitmatica.com": 0.8667582417582418,+ "bbs.archlinux.org": 0.8667582417582418,+ "bookofhook.blogspot.com": 0.8667582417582418,+ "rappers.mdaniels.com.s3-website-us-east-1.amazonaws.com": 0.8667582417582418,+ "milapneupane.com.np": 0.8667582417582418,+ "lists.randombit.net": 0.8667581999829783,+ "tyrrrz.me": 0.8667581999829783,+ "research.checkpoint.com": 0.8667536287753966,+ "avdi.org": 0.8667515410500497,+ "tuxphones.com": 0.8667177933546236,+ "wolfstreet.com": 0.8667059036925511,+ "www.danielmiessler.com": 0.8666932411921048,+ "klarasystems.com": 0.8666904333167735,+ "blog.leahhanson.us": 0.8666699206546021,+ "lua-users.org": 0.8666542828163675,+ "blog.tjll.net": 0.866647835447029,+ "massivelyop.com": 0.8665522518442965,+ "kuow.org": 0.8665345404342943,+ "www.chinafile.com": 0.8665345404342943,+ "www.allaboutcircuits.com": 0.8665332702488167,+ "www.physorg.com": 0.8665324818274975,+ "www.altdevblogaday.com": 0.8665315510980502,+ "codeahoy.com": 0.8664780210212021,+ "perl.plover.com": 0.8664701135603498,+ "www.wpri.com": 0.8664576956212064,+ "www.brianchristner.io": 0.8664469380333186,+ "srconstantin.wordpress.com": 0.8664373069106099,+ "yeoman.io": 0.8663968845512826,+ "pixelastic.github.io": 0.8663968845512826,+ "www.structuredprocrastination.com": 0.8663817900462257,+ "www.galactanet.com": 0.8663440641046102,+ "alexmuir.com": 0.8663020356625262,+ "www.cs.washington.edu": 0.866214599696902,+ "gapingvoid.com": 0.866214599696902,+ "doisinkidney.com": 0.8662068965517241,+ "jeffknupp.com": 0.866196580751142,+ "weblogs.asp.net": 0.8661868876649387,+ "www.eclipse.org": 0.8661815854172948,+ "www.twitter.com": 0.8661688444094342,+ "www.gmo.com": 0.8660899350618185,+ "h14s.p5r.org": 0.8660899350618185,+ "www.privatdozent.co": 0.8660524023972933,+ "conorpp.com": 0.8660524023972933,+ "dev-blog.apollodata.com": 0.8660261935749697,+ "www.bof.nl": 0.8660220994475138,+ "www.potaroo.net": 0.8660076718528887,+ "www.disneyanimation.com": 0.8659912587210011,+ "perspectives.mvdirona.com": 0.8659285828108513,+ "digiconomist.net": 0.8659107168851126,+ "futhark-lang.org": 0.8658910633267001,+ "blog.stephanbehnke.com": 0.8658778264013361,+ "www.themillions.com": 0.8658146270171843,+ "www.charlespetzold.com": 0.8657610099215972,+ "dev.opera.com": 0.8657226869029186,+ "www.contrast.ie": 0.8657095900925946,+ "threatpost.com": 0.8656950945183971,+ "diginomica.com": 0.8656889877908728,+ "www.iro.umontreal.ca": 0.865673655242536,+ "goodui.org": 0.8656691714697559,+ "kevinmahoney.co.uk": 0.8656053786488649,+ "den.dev": 0.8655508375322314,+ "blog.danlew.net": 0.8655009175546347,+ "insidermag.net": 0.8654928424374734,+ "eviltrout.com": 0.8654668848568644,+ "bartwronski.com": 0.8653908212138248,+ "asserttrue.blogspot.com": 0.8653811565244239,+ "laughtonelectronics.com": 0.8653462441760643,+ "www.toomas.net": 0.8653462441760643,+ "www.peoplespolicyproject.org": 0.8653036598492724,+ "www.kpcb.com": 0.865280579032659,+ "engineering.stanford.edu": 0.8652651824846332,+ "www.mcsweeneys.net": 0.8652414926783502,+ "www.weeklystandard.com": 0.8652325002095317,+ "scratch.mit.edu": 0.8652264286658259,+ "therideshareguy.com": 0.8651696061186824,+ "hyperboleandahalf.blogspot.com": 0.8651696061186824,+ "evhead.com": 0.8651553668857692,+ "praeclarum.org": 0.8651529057353353,+ "riowang.blogspot.com": 0.8651391622821514,+ "www.rs-online.com": 0.86513559976471,+ "blog.linuxmint.com": 0.8651351630976869,+ "blog.soylent.me": 0.8651258007804876,+ "cabel.me": 0.8650904033379694,+ "wildbit.com": 0.8650667180545558,+ "www.huffingtonpost.com": 0.86501806564583,+ "code.nasa.gov": 0.8649895088434025,+ "zenhabits.net": 0.8649439383347666,+ "dl.dropbox.com": 0.8649350287137462,+ "devblog.avdi.org": 0.8648862222006666,+ "blog.phpfog.com": 0.8648344406451537,+ "www.thetransportpolitic.com": 0.8648252523162882,+ "forestry.io": 0.8648063640773068,+ "www.dabeaz.com": 0.8647748270793637,+ "www.brightball.com": 0.8647346279381389,+ "style.org": 0.8647346279381389,+ "www.elezea.com": 0.8647230821512976,+ "www.aarp.org": 0.864718332495695,+ "themillions.com": 0.8646781650668347,+ "saastr.quora.com": 0.8646637505280962,+ "ottawacitizen.com": 0.8646581352452648,+ "mkremins.github.io": 0.8646201560571354,+ "edge.org": 0.8646127068735515,+ "micropython.org": 0.8645605097088179,+ "www.metamorphblog.com": 0.8645587566462556,+ "blog.jez.io": 0.8645309820862669,+ "aatishb.com": 0.8644462651203315,+ "www.celinehh.com": 0.8644462651203315,+ "davidyat.es": 0.8644462651203315,+ "code.joejag.com": 0.8644356837498463,+ "www.washingtonexaminer.com": 0.864373005099176,+ "www.resetera.com": 0.8643497804356822,+ "quantum.country": 0.8643112341108541,+ "savingjournalism.substack.com": 0.8642759030508148,+ "ahungry.com": 0.8642759030508148,+ "gamozolabs.github.io": 0.8642719268946735,+ "jamespanderson.tumblr.com": 0.8642680643637904,+ "elementaryos.org": 0.8642208492393608,+ "www.debuggex.com": 0.8642032789469397,+ "www.planetizen.com": 0.8641967719680171,+ "www.eweek.com": 0.864172798382722,+ "tatiyants.com": 0.8641038586159879,+ "discover.dc.nihr.ac.uk": 0.8640971396521386,+ "web.cs.ucdavis.edu": 0.8640703175243611,+ "xmpp.org": 0.8640502790628004,+ "chriskohlhepp.wordpress.com": 0.8640478440979216,+ "mystartuphas30daystolive.tumblr.com": 0.8640252337365474,+ "rhelblog.redhat.com": 0.8640126486430193,+ "www.electricitymap.org": 0.8639997125170845,+ "americasmarkets.usatoday.com": 0.8639946073685961,+ "projects.propublica.org": 0.8639224455926199,+ "e-estonia.com": 0.8639219016557921,+ "blog.basho.com": 0.8639181305846296,+ "ungleich.ch": 0.863900787970001,+ "squid314.livejournal.com": 0.8638782213420932,+ "www.cassandracapital.net": 0.8638782213420932,+ "aiyprojects.withgoogle.com": 0.8638467210374888,+ "bsdimp.blogspot.com": 0.8638147888998915,+ "jwt.io": 0.8638117466836764,+ "www.esri.com": 0.8638089133245348,+ "www.gaborcselle.com": 0.8637930884909581,+ "venam.nixers.net": 0.8637807788335469,+ "www.7pace.com": 0.8637364419671971,+ "www.joshmcguigan.com": 0.8637220704617038,+ "promarket.org": 0.8636701430867854,+ "tav.espians.com": 0.8636677390061345,+ "www.datastax.com": 0.8636570251868989,+ "www.cmyr.net": 0.8636232605564672,+ "mainisusuallyafunction.blogspot.com": 0.8636109472670038,+ "www.hisutton.com": 0.8635963039783531,+ "lsub.org": 0.8635700664090139,+ "alpblog.heroku.com": 0.8635306327438487,+ "gpuopen.com": 0.8635262100242561,+ "nvd.nist.gov": 0.8634973542011698,+ "english.elpais.com": 0.8634616688985162,+ "www.craigslist.org": 0.863445932794428,+ "nostalgebraist.tumblr.com": 0.8634392689015953,+ "www.hakaimagazine.com": 0.8634375812585431,+ "marcan.st": 0.8634342519721849,+ "wiki.debian.org": 0.8633915097045765,+ "php.net": 0.8633877833122457,+ "benjaminreinhardt.com": 0.8633802816901408,+ "blog.pnas.org": 0.86336052660942,+ "www.adityaravishankar.com": 0.86336052660942,+ "mobile.twitter.com": 0.8633312362141495,+ "www.alistapart.com": 0.863319649821934,+ "msgpack.org": 0.8633096782467932,+ "keionline.org": 0.8633096782467932,+ "www.logicalclocks.com": 0.8632631941253387,+ "svedic.org": 0.8632309919400734,+ "newsroom.unsw.edu.au": 0.8632254714408825,+ "maxwellito.github.io": 0.8632001968821827,+ "www.commandlinefu.com": 0.8631417710298782,+ "www.clockworkpi.com": 0.8631140242937411,+ "www.typography.com": 0.8630969520774667,+ "alumni.stanford.edu": 0.8630912751881218,+ "www.sigarch.org": 0.8630849989436014,+ "www.countbayesie.com": 0.8630846691596683,+ "blog.echen.me": 0.8630803489219858,+ "groovehq.com": 0.863062038855598,+ "calteches.library.caltech.edu": 0.863062038855598,+ "www.rochester.edu": 0.8630082813677696,+ "blog.bjrn.se": 0.8629931779073361,+ "lawandcrime.com": 0.8629659867085913,+ "betashop.com": 0.862958801056521,+ "www.ericsink.com": 0.8629424736221599,+ "engineering.linkedin.com": 0.8629291758709224,+ "cleantechnica.com": 0.8629063952275409,+ "meltingasphalt.com": 0.8628160615079228,+ "amiunique.org": 0.8628087618596978,+ "bvckup2.com": 0.8628005657708628,+ "ieftimov.com": 0.8627657236548011,+ "ewanvalentine.io": 0.8627020659905763,+ "secalerts.co": 0.8627018556487716,+ "www.patrick-wied.at": 0.8626866200547734,+ "www.pitt.edu": 0.8626432815245133,+ "hothardware.com": 0.8626338662999138,+ "www.appleinsider.com": 0.8626260620914765,+ "hillelwayne.com": 0.8626238486818072,+ "gridstylesheets.org": 0.8626062322946175,+ "sciencex.com": 0.8626007339716192,+ "developers.slashdot.org": 0.86255044240195,+ "blog.confluent.io": 0.8624968367523541,+ "glandium.org": 0.8624955638682134,+ "arandomurl.com": 0.8624894125730017,+ "blog.nugget.one": 0.8624812956348196,+ "www.rd.com": 0.8624301242273775,+ "slifty.com": 0.8624113475177305,+ "www.lighttable.com": 0.8624113475177305,+ "payments.amazon.com": 0.8624113475177305,+ "www.lurklurk.org": 0.862401566562782,+ "derstandard.at": 0.862401566562782,+ "workstuff.tumblr.com": 0.8623257751778021,+ "www.lvh.io": 0.8623241846862385,+ "rs.io": 0.8623129582427793,+ "eslint.org": 0.862298463525373,+ "www.cucumbertown.com": 0.8622891062009898,+ "jquery.com": 0.8622837910960226,+ "z.cash": 0.8622770331746279,+ "www.ottawacitizen.com": 0.8622229843756593,+ "nanonets.com": 0.8621765227456938,+ "www.crunchgear.com": 0.8621559229318876,+ "mattruby.substack.com": 0.8621556881643411,+ "mbuffett.com": 0.8621556881643411,+ "cybergibbons.com": 0.8621443416178289,+ "www.lua.org": 0.8621439982277107,+ "monodraw.helftone.com": 0.8621272923907152,+ "vicki.substack.com": 0.8621255550009437,+ "www.davidgcohen.com": 0.8621062295375266,+ "jfrog.com": 0.8620661061833018,+ "joostdevblog.blogspot.com": 0.8620661061833018,+ "johnkary.net": 0.8620331171141896,+ "jakobgreenfeld.com": 0.8620331171141896,+ "bosker.wordpress.com": 0.8619844782836177,+ "encosia.com": 0.8619710029888306,+ "martinkool.com": 0.8619319552747774,+ "use-the-index-luke.com": 0.8619185059392996,+ "siteanalytics.compete.com": 0.8619154305011097,+ "madewithlove.be": 0.8619031112487354,+ "blog.digg.com": 0.8619019650013046,+ "wildlyinaccurate.com": 0.8618675915764481,+ "news.agu.org": 0.8618622951257247,+ "www.hnsearch.com": 0.8618535032580247,+ "bbc.com": 0.8618329035365963,+ "thevaluable.dev": 0.8618329035365963,+ "cve.mitre.org": 0.8618136566919955,+ "www.r2d3.us": 0.8618128425706462,+ "www.wsbtv.com": 0.8617435427240897,+ "wiki.php.net": 0.8617039994341269,+ "regex101.com": 0.861640779434757,+ "www.fastcodesign.com": 0.8616335728873852,+ "waitbutwhy.com": 0.8616048195074264,+ "wrapbootstrap.com": 0.8615996055432654,+ "www.oracle.com": 0.8615524291467332,+ "angrystaffofficer.com": 0.8615283953190601,+ "pythonprogramming.net": 0.8615283953190601,+ "codemirror.net": 0.8615237207033272,+ "erthalion.info": 0.8615038163918213,+ "news.wsu.edu": 0.8615031765236854,+ "jmtd.net": 0.8614607909822787,+ "www.businessinsider.com.au": 0.8614418925622166,+ "www.keithschwarz.com": 0.8614202461594263,+ "blog.testdouble.com": 0.8614030589984097,+ "rationalconspiracy.com": 0.8613615814645382,+ "michaelochurch.wordpress.com": 0.8613601391266915,+ "blogs.berkeley.edu": 0.8613538103460412,+ "blog.superfeedr.com": 0.8613308060874829,+ "hn-books.com": 0.8613200444920518,+ "techsolidarity.org": 0.8613153663268618,+ "annualletter.gatesfoundation.org": 0.8613153663268618,+ "www.ardanlabs.com": 0.8612708268948052,+ "www.mapbox.com": 0.8612700375538782,+ "protonvpn.com": 0.8612360355266332,+ "www.winrumors.com": 0.8611989655340138,+ "www.arcticstartup.com": 0.8611695091473522,+ "blogs.akamai.com": 0.8611395195993665,+ "penntoday.upenn.edu": 0.8611371780676135,+ "www.howitactuallyworks.com": 0.8611264027197298,+ "www.theage.com.au": 0.8611116664910229,+ "blogs.computerworld.com": 0.8610945749876592,+ "news.artnet.com": 0.8610390836029311,+ "yongfook.com": 0.8610327646216055,+ "ask.metafilter.com": 0.8610302755520367,+ "blog.zencoder.com": 0.8609595040886218,+ "labs.spotify.com": 0.8609558167629312,+ "ashfurrow.com": 0.8609452045938362,+ "rachelandrew.co.uk": 0.8609370906301821,+ "xem.github.io": 0.860936923468393,+ "natureofcode.com": 0.8609280600555332,+ "www.daedtech.com": 0.8609275762383382,+ "jnd.org": 0.8609065111890492,+ "cryptoslate.com": 0.8608950862608346,+ "googlepolicyeurope.blogspot.com": 0.8608494490780891,+ "electricliterature.com": 0.8608434109180494,+ "www.cs.toronto.edu": 0.8608065788416125,+ "mtgox.com": 0.8608060590690018,+ "laughingsquid.com": 0.8607982963555786,+ "nickdesaulniers.github.io": 0.8607761141579771,+ "news.netcraft.com": 0.8607587653123538,+ "www.mattfriz.com": 0.8607469264593318,+ "blogs.computerworlduk.com": 0.8606535841782695,+ "weblog.jamisbuck.org": 0.8606463446127512,+ "particletree.com": 0.860619018859172,+ "www.loopinsight.com": 0.8605648031182908,+ "blog.qualys.com": 0.86055640956746,+ "moz.com": 0.860495291636908,+ "vimeo.com": 0.8604892664547054,+ "www.chalmers.se": 0.860431654676259,+ "homepage.ntlworld.com": 0.8604242343472561,+ "www.jpost.com": 0.8604154523480785,+ "www.governor.ny.gov": 0.8603434360710707,+ "www.teenvogue.com": 0.8603394119266393,+ "blog.bufferapp.com": 0.8603306431458172,+ "pharr.org": 0.8603247905476864,+ "www.cforcoding.com": 0.8603197271193488,+ "mixpanel.com": 0.8603064014189074,+ "bits-please.blogspot.com": 0.860301774508791,+ "waypoint.vice.com": 0.8602882797773976,+ "elpais.com": 0.8602682993832683,+ "www.thevintagenews.com": 0.8602498276023521,+ "bdickason.com": 0.8602305475504323,+ "immunant.com": 0.8601526618229086,+ "madhadron.com": 0.8601439808675682,+ "www.dwheeler.com": 0.8600331290988332,+ "www.vim.org": 0.8600002647579501,+ "blog.prototypr.io": 0.859998158168068,+ "webpolicy.org": 0.8599719205993361,+ "coding.smashingmagazine.com": 0.8599291407790329,+ "wizardzines.com": 0.8598885971042833,+ "www.itnews.com.au": 0.8598724037037537,+ "www.codethinked.com": 0.8598208830600297,+ "www.technologynetworks.com": 0.8598085045605989,+ "abclocal.go.com": 0.8598024771455696,+ "blog.wolframalpha.com": 0.8597915197356469,+ "blog.juxt.pro": 0.8597891002608672,+ "newsroom.tiktok.com": 0.8597628867114886,+ "defector.com": 0.8597558066643536,+ "apps.apple.com": 0.8597457842210768,+ "www.jamesyu.org": 0.8597029099342595,+ "www.dailytech.com": 0.8596655487230563,+ "elie.net": 0.8596315170135848,+ "linear.app": 0.8596237337192475,+ "clerk.house.gov": 0.8596237337192475,+ "keepachangelog.com": 0.8596075354545292,+ "featherweightmusings.blogspot.com": 0.8596075354545292,+ "eng.localytics.com": 0.8595938002959398,+ "emacsrocks.com": 0.8595762853230908,+ "dassur.ma": 0.8595420928346226,+ "www.sightline.org": 0.859531902937568,+ "www.gamesradar.com": 0.8595242144121686,+ "sysdig.com": 0.8595202139290916,+ "www.cloudkick.com": 0.8595183443817482,+ "devopsu.com": 0.859493942235387,+ "believermag.com": 0.8594892467285535,+ "unhandledexpression.com": 0.8594741803219609,+ "efficiencyiseverything.com": 0.8594569005504344,+ "www.therobotreport.com": 0.859433089374823,+ "www.howtographql.com": 0.8594202898550725,+ "archive.today": 0.8593643247719568,+ "www.jacobinmag.com": 0.8593589358679861,+ "progrium.com": 0.8593468157466189,+ "redfin.engineering": 0.8593466877552149,+ "www.zillow.com": 0.8593304013971459,+ "www.nexedi.com": 0.8593210096380903,+ "madaidans-insecurities.github.io": 0.8593210096380903,+ "home.treasury.gov": 0.8592910289545252,+ "scholar.harvard.edu": 0.8592733512209211,+ "opensourceconnections.com": 0.8592682967350698,+ "bsdly.blogspot.com": 0.8592666292050983,+ "aras-p.info": 0.8592595047254012,+ "battlepenguin.com": 0.8592397001892514,+ "solokeys.com": 0.8592264014510466,+ "www.alexkyte.me": 0.8591861061610623,+ "www.digitalmusicnews.com": 0.8591609711742595,+ "kenneth.io": 0.8591232018072773,+ "mywiki.wooledge.org": 0.8590993783573315,+ "epic.org": 0.8590931061006938,+ "behavioralscientist.org": 0.8590931061006938,+ "jack.minardi.org": 0.8590798567329014,+ "zeptobars.ru": 0.8590350573746124,+ "bramcohen.com": 0.8590132997607233,+ "frankforce.com": 0.8590074352869126,+ "erniemiller.org": 0.8590074352869125,+ "robservatory.com": 0.8589982689488246,+ "engineering.squarespace.com": 0.8589982689488246,+ "www.artofmanliness.com": 0.8589698957080373,+ "blogs.bing.com": 0.8589523074975127,+ "www.world-nuclear-news.org": 0.8589473343420543,+ "openmirage.org": 0.8589305193408414,+ "prism-break.org": 0.8589248369205337,+ "slab.com": 0.8589172698916326,+ "monkeylearn.com": 0.8589030969473112,+ "journals.sagepub.com": 0.8588825279469426,+ "metaredux.com": 0.8588307379036221,+ "www.eurogamer.net": 0.8588239617965691,+ "www.wisdomination.com": 0.8587259126214145,+ "src.chromium.org": 0.8586820383432011,+ "www.tudelft.nl": 0.8586674667651873,+ "blog.project-insanity.org": 0.8586633233128,+ "hadihariri.com": 0.8586515656051527,+ "blog.xot.nl": 0.8586508044813143,+ "www.nypost.com": 0.8586366736605365,+ "andrewchenblog.com": 0.8586246895566572,+ "jordan-wright.com": 0.8586221711446911,+ "journal.burningman.org": 0.858622171144691,+ "blog.tensorflow.org": 0.8586139292072289,+ "googlewebmastercentral.blogspot.com": 0.8586030875489932,+ "standblog.org": 0.858600583090379,+ "ant.design": 0.858600583090379,+ "overapi.com": 0.8585933703160669,+ "www.suntimes.com": 0.8585797957168473,+ "thecable.foreignpolicy.com": 0.8585540700891416,+ "earther.gizmodo.com": 0.8584865305709988,+ "www.bestinclass.dk": 0.8584597924729722,+ "hn.algolia.com": 0.8584545053239319,+ "gwillem.gitlab.io": 0.8584541318402437,+ "elbenshira.com": 0.8584541318402437,+ "bankunderground.co.uk": 0.8583675490979057,+ "www.crowdsupply.com": 0.8583166788787248,+ "lists.linuxfoundation.org": 0.8583007929896874,+ "mikehadlow.blogspot.com": 0.8583007929896874,+ "prideout.net": 0.8582618132398327,+ "joinup.ec.europa.eu": 0.8582375872301639,+ "coderoom.wordpress.com": 0.8582225134679472,+ "andrewoneverything.com": 0.8582199611092117,+ "peterlyons.com": 0.8581871345029239,+ "facebook.com": 0.8581759828028657,+ "www.vogue.com": 0.8581630406503709,+ "future.a16z.com": 0.8581407364644609,+ "www.thedotpost.com": 0.8581294866962557,+ "davidwalsh.name": 0.85807314827283,+ "www.heise.de": 0.8580470280497292,+ "regex.info": 0.8580444001760147,+ "blog.jayway.com": 0.8580313921739998,+ "www.discovermagazine.com": 0.8580086852563162,+ "www.gamespot.com": 0.858006803378577,+ "www.mattmazur.com": 0.8579544667792158,+ "0x80.pl": 0.8579258153670124,+ "pine64.com": 0.8579245737044574,+ "www.gog.com": 0.8579147324679443,+ "jdh.hamkins.org": 0.857912419834021,+ "openjdk.java.net": 0.8578856841040577,+ "code.djangoproject.com": 0.8578761757124823,+ "airbnb.io": 0.8578759796916386,+ "www.neatorama.com": 0.8578511503394977,+ "intridea.com": 0.8578447662759853,+ "techchannel.att.com": 0.8578445745748121,+ "mule.substack.com": 0.8578445745748121,+ "www.fastmailstatus.com": 0.857833367118916,+ "nas.sr": 0.8578228253764041,+ "cloudblogs.microsoft.com": 0.8578026799771654,+ "danielbmarkham.com": 0.8577690138809076,+ "blogs.kde.org": 0.8577593606755378,+ "graphics.latimes.com": 0.8577592314786884,+ "www.wordfence.com": 0.8577572254294423,+ "quariety.com": 0.8577565674618444,+ "www.ritholtz.com": 0.8577423661740396,+ "jasonadriaan.com": 0.8576809443893051,+ "joelcalifa.com": 0.8576809443893051,+ "www.cs.umd.edu": 0.857661758811538,+ "zacharyvoase.com": 0.857656134895176,+ "www.decisionproblem.com": 0.8576347888613035,+ "steamdb.info": 0.8576311826444905,+ "blog.chef.io": 0.8576130873561624,+ "techraptor.net": 0.8575933683258181,+ "www.nanog.org": 0.857579845060974,+ "www.cs.yale.edu": 0.8575769410337604,+ "blogs.skype.com": 0.8575392141865088,+ "igoro.com": 0.8575204710620608,+ "www.math.toronto.edu": 0.8574671949652505,+ "tether.to": 0.8574671949652504,+ "adrianmejia.com": 0.8573817628591522,+ "adblockplus.org": 0.8573641935296868,+ "techxplore.com": 0.8573012751875033,+ "professional-troublemaker.com": 0.8572977062348858,+ "idealab.talkingpointsmemo.com": 0.8572223861972004,+ "nesslabs.com": 0.8572196165500797,+ "www.nationalgeographic.co.uk": 0.8571954684569548,+ "iridakos.com": 0.8571705482234938,+ "theagilepanda.com": 0.8571428571428571,+ "togototo.wordpress.com": 0.8571428571428571,+ "blog.skyliner.io": 0.8571354628699216,+ "eu.techcrunch.com": 0.8571248597503665,+ "datastudio.google.com": 0.8571140921734792,+ "www.django-cms.org": 0.8570965178487575,+ "blog.path.com": 0.8570914140075269,+ "www.villagevoice.com": 0.857081275011955,+ "somehowmanage.com": 0.857055205499945,+ "www.mnn.com": 0.8570470364584489,+ "abovethelaw.com": 0.8570463120033517,+ "blog.logrocket.com": 0.8570182021331599,+ "www.apress.com": 0.8569721488001009,+ "www.news.cornell.edu": 0.8569706892788174,+ "www.curbed.com": 0.8569597747993827,+ "digitizor.com": 0.856955455369854,+ "blog.lareviewofbooks.org": 0.8569521195010459,+ "www.cambus.net": 0.8569380680580816,+ "blog.rize.io": 0.8568954207368967,+ "2ality.com": 0.8568509445047684,+ "ux.stackexchange.com": 0.856847117644151,+ "www.thedomains.com": 0.8568397055760405,+ "timesofindia.indiatimes.com": 0.856835710146828,+ "examine.com": 0.8568332988347195,+ "speirs.org": 0.8568142257362032,+ "learnbyexample.github.io": 0.8568010189328112,+ "cs.brown.edu": 0.8567908824604603,+ "www.dnalounge.com": 0.8567908041212858,+ "www.nbcbayarea.com": 0.8567028634763063,+ "macperformanceguide.com": 0.8566332090796122,+ "social.techcrunch.com": 0.856633209079612,+ "gocardless.com": 0.8566325894779986,+ "www.driverless.id": 0.8565431663151448,+ "simonsarris.substack.com": 0.856507155170115,+ "blog.l0cal.com": 0.856499398164963,+ "poy.time.com": 0.8564993981649629,+ "labs.ripe.net": 0.8564206791243404,+ "britishlibrary.typepad.co.uk": 0.8564116348023619,+ "directorsblog.nih.gov": 0.8563781052354208,+ "www.trustedsec.com": 0.8563276469242246,+ "www.nicholaswindsorhoward.com": 0.8563276469242246,+ "wefunder.com": 0.8563033480182963,+ "grpc.io": 0.8563005637859097,+ "www.innovationaus.com": 0.8562719072138797,+ "algorithmwatch.org": 0.856263282232775,+ "kylewritescode.com": 0.8562626989641361,+ "www.iquilezles.org": 0.8562567878094767,+ "www.doc.ic.ac.uk": 0.8561753420541418,+ "www.r-bloggers.com": 0.8561384095933265,+ "www.kxan.com": 0.8561304599253935,+ "blog.shodan.io": 0.856101606479241,+ "blog.couchdb.org": 0.8560857595989197,+ "randsinrepose.com": 0.8560683987937139,+ "blogs.igalia.com": 0.8560658162982266,+ "erikhoel.substack.com": 0.8560552035555758,+ "www.columbia.edu": 0.8560502120089583,+ "adambard.com": 0.8560134134615376,+ "xania.org": 0.8559909491087049,+ "pinboard.in": 0.8559460897276601,+ "lobste.rs": 0.8559156074776079,+ "interestingengineering.com": 0.8558912746106632,+ "www.dilbert.com": 0.855888464977386,+ "jcdav.is": 0.8558808822556393,+ "zwbetz.com": 0.8558715995865966,+ "npr.org": 0.8558692421991085,+ "compellingsciencefiction.com": 0.8558593149932564,+ "headmelted.com": 0.8558229432148337,+ "blog.itlater.com": 0.8558223030664946,+ "www.joystiq.com": 0.8557770050631482,+ "blog.loadzero.com": 0.8557371228445998,+ "www.exploringbinary.com": 0.8557365123930964,+ "www.reason.com": 0.8557121404409874,+ "towardsdatascience.com": 0.8556587797122474,+ "ostif.org": 0.855654761904762,+ "articles.latimes.com": 0.855649649030829,+ "www.cs.ucr.edu": 0.8556340456009993,+ "www.transportenvironment.org": 0.8556309211595626,+ "www.sciencenews.org": 0.8556293530573256,+ "blog.portswigger.net": 0.855625404687521,+ "dshipper.posterous.com": 0.8556180321566556,+ "abopen.com": 0.8555419933626652,+ "www.twitch.tv": 0.8555327933571086,+ "www.realclearscience.com": 0.8555038318613801,+ "www.llnl.gov": 0.8554731239960308,+ "brandmark.io": 0.8554590545897166,+ "www.eflorenzano.com": 0.8554573688896427,+ "play.google.com": 0.8554165451041797,+ "www.ethanzuckerman.com": 0.8553991433840825,+ "lunduke.substack.com": 0.8553770673931268,+ "www.montulli.org": 0.8553552610577579,+ "www.dvorak.org": 0.8553507205142731,+ "pythoninsider.blogspot.com": 0.855339221122335,+ "octopart.com": 0.8553291642797942,+ "ithare.com": 0.8553097001258215,+ "isc.sans.edu": 0.8552998663643834,+ "doi.org": 0.8552929122812928,+ "www.starling-software.com": 0.855240263021274,+ "sedimental.org": 0.8552238805970149,+ "www.nitrokey.com": 0.8552078417856235,+ "www.fool.com": 0.8551227533080648,+ "ntietz.com": 0.8551173351343933,+ "thepointmag.com": 0.8551125688254727,+ "www.symantec.com": 0.8551104394486714,+ "highline.huffingtonpost.com": 0.8550936296505218,+ "blog.adacore.com": 0.8550882114429865,+ "techland.time.com": 0.8550381807790381,+ "www-03.ibm.com": 0.855023796089594,+ "blog.davidtate.org": 0.8550204472368161,+ "blog.gitlab.org": 0.8549666512282963,+ "blog.dijit.sh": 0.8549645308164788,+ "boredzo.org": 0.8549332224976328,+ "www.bishopfox.com": 0.8549242562625317,+ "blog.sproutcore.com": 0.8548755303332917,+ "www.nakedcapitalism.com": 0.8548518987841709,+ "hackaday.io": 0.8548507362141421,+ "www.nccgroup.com": 0.8548006867421194,+ "docs.microsoft.com": 0.8547764550021021,+ "www.kansas.com": 0.8547447271228046,+ "www.myfonts.com": 0.8547443058333942,+ "www.databreaches.net": 0.8547388607511365,+ "design.google": 0.8546474649453593,+ "www.bvp.com": 0.8546145212136451,+ "eriwen.com": 0.854565525324503,+ "flightaware.com": 0.854535067640072,+ "us2.campaign-archive1.com": 0.854531163801279,+ "engineering.skroutz.gr": 0.8544744689256348,+ "arcentry.com": 0.8544398366164802,+ "redmonk.com": 0.8544165230751263,+ "jesseduffield.com": 0.8543827046117866,+ "www.foodandwine.com": 0.8543802950724103,+ "www.jacobsschool.ucsd.edu": 0.8543584327039565,+ "www.vandenoever.info": 0.8543584327039565,+ "tubetime.us": 0.8543543543543544,+ "xenbits.xen.org": 0.85435049306025,+ "blogs.sun.com": 0.8543496525648886,+ "nofilmschool.com": 0.8543378482464506,+ "joi.ito.com": 0.8543316742759597,+ "blog.garlicsim.org": 0.8543018539678975,+ "akr.am": 0.854281488649609,+ "www.nypl.org": 0.8542056110271361,+ "blog.opendns.com": 0.8542056110271361,+ "shmuplations.com": 0.8542031037522354,+ "blog.statwing.com": 0.8542005291800069,+ "m.bbc.com": 0.8541979742875379,+ "boston.conman.org": 0.8541978420073731,+ "lilium.com": 0.8541923613616496,+ "phk.freebsd.dk": 0.8541720175820052,+ "underthehood.meltwater.com": 0.8541376157274252,+ "www.airliners.net": 0.8541362938673078,+ "www.gobankingrates.com": 0.8541353383458646,+ "maxplanck.nautil.us": 0.8540985777081339,+ "leandromoreira.com.br": 0.8540707480810653,+ "24ways.org": 0.8540629407199574,+ "www.federaltimes.com": 0.8540125885303304,+ "blog.trello.com": 0.8540087030052186,+ "www.slash7.com": 0.8539974953863674,+ "www.projectrho.com": 0.8539630717568143,+ "yahoopolicy.tumblr.com": 0.8539187064781893,+ "pencil.evolus.vn": 0.8539187064781893,+ "getkerf.wordpress.com": 0.8539187064781893,+ "newfoodeconomy.org": 0.853885872320658,+ "www.geekosystem.com": 0.8538783892446405,+ "bengtan.com": 0.8538510433274764,+ "dustin.github.com": 0.8538486125176874,+ "jasonlefkowitz.net": 0.8538441285736149,+ "teamtreehouse.com": 0.8538268240476388,+ "blog.quarkslab.com": 0.853807684540482,+ "www.artnome.com": 0.8537866111759274,+ "blog.chron.com": 0.8537460072706317,+ "blog.close.io": 0.8537188598181367,+ "lonesysadmin.net": 0.8536953242835595,+ "www.lemonde.fr": 0.8536772759118291,+ "www.guernicamag.com": 0.8536142442449905,+ "codeascraft.etsy.com": 0.8536109251434499,+ "blog.oleganza.com": 0.8535993869171182,+ "www.poetryfoundation.org": 0.8535856365759876,+ "archive.computerhistory.org": 0.8535651705660577,+ "www.philzimmermann.com": 0.8535638214444478,+ "www.sqlalchemy.org": 0.8535347372406801,+ "www.rubyinside.com": 0.8535314579904915,+ "rjbs.manxome.org": 0.8534743202416919,+ "www.itl.nist.gov": 0.8534657928784027,+ "lab.simurai.com": 0.8534578819472406,+ "www.shortcutfoo.com": 0.8534538731861366,+ "www.asymco.com": 0.8534535683227724,+ "johnmathews.eu": 0.8534414067561554,+ "www.msnbc.msn.com": 0.8534205452840495,+ "www.cs.brown.edu": 0.8534113013764459,+ "blog.vivekhaldar.com": 0.8534010422454464,+ "www.t3.org": 0.8533802695967294,+ "coloradosun.com": 0.8533683929679808,+ "www.gamesindustry.biz": 0.8533666288339562,+ "www.benjoffe.com": 0.8533494797004003,+ "windows.microsoft.com": 0.8533490913172264,+ "codeblog.jonskeet.uk": 0.8533155100042978,+ "mattdesl.svbtle.com": 0.8532886204160572,+ "wilshipley.com": 0.8532666831748213,+ "lunduke.com": 0.8532546707850448,+ "blog.documentcloud.org": 0.8532526475037822,+ "pratt.duke.edu": 0.8532505235078635,+ "forum.palemoon.org": 0.8532499119645807,+ "ring.cx": 0.8532409206090049,+ "www.healthimaging.com": 0.8532409206090049,+ "blog.lawnstarter.com": 0.853234538226177,+ "www.drorpoleg.com": 0.853234538226177,+ "www.abajournal.com": 0.8532334026694913,+ "volokh.com": 0.8532246743179585,+ "www.akitasoftware.com": 0.8532127948822967,+ "blog.algolia.com": 0.8531662937825892,+ "cgit.freedesktop.org": 0.8531345095634363,+ "creativegood.com": 0.8531247497260012,+ "www.bizjournals.com": 0.8530952665861291,+ "krypt.co": 0.85304727815771,+ "david-peter.de": 0.8530425774087601,+ "www.bitcoin.org": 0.853030303030303,+ "www.markdotto.com": 0.853030303030303,+ "donsbot.wordpress.com": 0.8530217272624623,+ "joyent.com": 0.8530153570981687,+ "googleplusplatform.blogspot.com": 0.8530153000371625,+ "modern-products.tumblr.com": 0.8530153000371625,+ "hashicorp.com": 0.8529426622739795,+ "www.tuaw.com": 0.8529238643536575,+ "jessepollak.me": 0.8528766579532876,+ "www.spectrum.ieee.org": 0.8528722385076482,+ "evasi0n.com": 0.8527890991537469,+ "continuum.io": 0.8527523717085685,+ "inconshreveable.com": 0.8527291982231584,+ "washingtonmonthly.com": 0.8527169016435672,+ "www.go350.com": 0.8526967137013025,+ "newhumanist.org.uk": 0.8526876181950579,+ "www.thelocal.se": 0.8526535411240088,+ "arielelkin.github.io": 0.8526249122103083,+ "ldpreload.com": 0.8526076535424412,+ "blog.valerieaurora.org": 0.8526007849075418,+ "tympanus.net": 0.8526005626429696,+ "www.pewresearch.org": 0.8525869052062965,+ "gcaptain.com": 0.8525025061695275,+ "www.fireeye.com": 0.8524966902771423,+ "mkdev.me": 0.8524961199751095,+ "thoughtmaybe.com": 0.8524858010387742,+ "blog.lukaszolejnik.com": 0.8524707074061916,+ "getmondo.co.uk": 0.852437604863552,+ "open.bufferapp.com": 0.852434953761713,+ "alumni.berkeley.edu": 0.8524296840496752,+ "forums.macrumors.com": 0.8523683544548385,+ "alastairreid.github.io": 0.8523644066969532,+ "www.scottberkun.com": 0.852359208523592,+ "ontwik.com": 0.8523482955385304,+ "git-scm.com": 0.8523082962459579,+ "news.un.org": 0.8522696996619386,+ "www.adafruit.com": 0.8522616020934412,+ "www.mathjax.org": 0.8522588583128298,+ "www.chinalawblog.com": 0.8522208346251817,+ "yahoolabs.tumblr.com": 0.8522048661669432,+ "www.dw.de": 0.8521590754317577,+ "www.udacity.com": 0.8521539143105872,+ "dianeravitch.net": 0.8521341463414633,+ "www.narrator.ai": 0.852097863618245,+ "broadbandnow.com": 0.852097863618245,+ "genius.com": 0.8520940394549934,+ "theaircurrent.com": 0.8519939439576096,+ "www.bytecellar.com": 0.8519428423381916,+ "www.rifters.com": 0.8519032600490041,+ "www.math.upenn.edu": 0.8518649443865483,+ "algorithmsoup.wordpress.com": 0.8518190799236097,+ "ellegriffin.substack.com": 0.8518190799236097,+ "mikewest.org": 0.8517824188629416,+ "loyaltylobby.com": 0.8516821355411338,+ "blog.hut8labs.com": 0.8516819571865444,+ "www.metmuseum.org": 0.8516671986689323,+ "www-users.cs.umn.edu": 0.8516590199664924,+ "undo.io": 0.8516245163512932,+ "news.discovery.com": 0.8515880430654605,+ "hulry.com": 0.8515702882943715,+ "www.greenheartgames.com": 0.8515702882943715,+ "ie.microsoft.com": 0.8515602032387396,+ "www.thehindu.com": 0.8515477928389076,+ "www.ably.io": 0.8515404721157463,+ "chrispenner.ca": 0.8514969974322367,+ "www.logodesignlove.com": 0.8514733938627516,+ "people.ischool.berkeley.edu": 0.8514695464544928,+ "timothycrosley.github.io": 0.8514548238897397,+ "www.neocomputer.org": 0.8514548238897397,+ "m.motherjones.com": 0.8514463981383273,+ "ro-che.info": 0.8514172067988788,+ "www.ofnumbers.com": 0.8514160013087799,+ "rdist.root.org": 0.8514077817668315,+ "blog.alexellis.io": 0.8514045841413571,+ "www.notebookcheck.net": 0.8513578715363501,+ "new.livestream.com": 0.8513411202985242,+ "www.sonyaellenmann.com": 0.8513367975602456,+ "builtin.com": 0.8513357366412594,+ "thecritic.co.uk": 0.8513136478418425,+ "bitworking.org": 0.851299858555184,+ "iwantmyname.com": 0.8512894868376749,+ "www.circleid.com": 0.851254578211912,+ "woboq.com": 0.8512444088430177,+ "www.instructables.com": 0.8512139095333348,+ "markmail.org": 0.8512136385469307,+ "da-data.blogspot.com": 0.8511940319698232,+ "www.inverse.com": 0.8511748736762975,+ "robinhood.com": 0.8511710385891778,+ "blog.tojicode.com": 0.8511685296886505,+ "usa.streetsblog.org": 0.8511588204743716,+ "sanderknape.com": 0.851058381116051,+ "technical.ly": 0.8510465053531708,+ "www.trek10.com": 0.8510328925328215,+ "theprint.in": 0.8510207129082807,+ "000fff.org": 0.8509998804696824,+ "journal.frontiersin.org": 0.8509377801906453,+ "on.wsj.com": 0.850920176035645,+ "sircmpwn.github.io": 0.8508709080357458,+ "www.networkworld.com": 0.8508447666287313,+ "apps.bostonglobe.com": 0.8508308486054927,+ "pfrazee.hashbase.io": 0.8508225328373329,+ "www.commentarymagazine.com": 0.8508100679431329,+ "petewarden.com": 0.8507826894953255,+ "www.exeter.ac.uk": 0.8507809015713883,+ "www.statwing.com": 0.8507692307692307,+ "www.unenvironment.org": 0.8507692307692307,+ "www.paintcodeapp.com": 0.8507513412377187,+ "andreschweighofer.com": 0.8507309300531638,+ "forums.unrealengine.com": 0.8507306931858649,+ "www.wikidata.org": 0.8507275608332324,+ "draculatheme.com": 0.8507209915723823,+ "hongkongfp.com": 0.8507154665320323,+ "psmag.com": 0.8507120935262249,+ "blog.flightcaster.com": 0.850683949182937,+ "vancouversun.com": 0.8506812814664019,+ "www.cc.gatech.edu": 0.8506656522986229,+ "news.wisc.edu": 0.8506650572190997,+ "madebymany.com": 0.850661432792337,+ "www.afp.com": 0.8506381100610537,+ "cudl.lib.cam.ac.uk": 0.8505963477950123,+ "www.huyng.com": 0.8505963477950121,+ "www.jeremymorgan.com": 0.8505908966433915,+ "thereformedbroker.com": 0.8505772906521217,+ "bigocheatsheet.com": 0.8505365958446607,+ "matduggan.com": 0.850500461698309,+ "deis.io": 0.8504704364122011,+ "poolp.org": 0.8504003265009309,+ "alteredqualia.com": 0.8503991615824809,+ "product.hubspot.com": 0.8503749082035804,+ "www.erlang-factory.com": 0.8503149886076794,+ "paranoia.dubfire.net": 0.850273893946855,+ "concurrencyfreaks.blogspot.com": 0.850256626665339,+ "www.cato-at-liberty.org": 0.8502310137099974,+ "databasearchitects.blogspot.com": 0.8502084998761508,+ "www.netflixprize.com": 0.8501934998584227,+ "www.utilitydive.com": 0.8501445783462607,+ "www.olioapps.com": 0.850137018218358,+ "rcrowley.org": 0.850105596274167,+ "kentonv.github.io": 0.8500969474584005,+ "hastebin.com": 0.8500573664501953,+ "www.dotnetfoundation.org": 0.850018648725371,+ "mattmazur.com": 0.8499955866420006,+ "breakoutstartups.substack.com": 0.8499877911408882,+ "www.platformonomics.com": 0.8499799037669202,+ "www.groovehq.com": 0.8499412891580047,+ "alf.nu": 0.8499136585227424,+ "antoinevastel.com": 0.8498761519134193,+ "liveweave.com": 0.8498511311102708,+ "causal.app": 0.8498452012383901,+ "liw.fi": 0.8498452012383901,+ "www.paypal.com": 0.8497695268466062,+ "redis.io": 0.8497424748729502,+ "collectiveidea.com": 0.8496851714802387,+ "felleisen.org": 0.8496847068727177,+ "blog.nobugware.com": 0.8496603118181073,+ "www.windowscentral.com": 0.8496154708571939,+ "www.xobni.com": 0.8495968094932279,+ "outage.report": 0.8495880396284066,+ "www.justsecurity.org": 0.849566454817199,+ "blog.rfox.eu": 0.8495467369935737,+ "way-cooler.org": 0.8495467369935737,+ "adamierymenko.com": 0.849538361390885,+ "mesosphere.com": 0.8495051866724436,+ "munchies.vice.com": 0.8495023041501157,+ "meatfighter.com": 0.8495018599221447,+ "dsc.discovery.com": 0.8494918847267826,+ "sciencenordic.com": 0.8494773842761618,+ "uxmovement.com": 0.8494740695810459,+ "death.andgravity.com": 0.8494550556394679,+ "www.idlewords.com": 0.8493746001892146,+ "staffeng.com": 0.8493531997757849,+ "godottutorials.pro": 0.8493096657310281,+ "earthobservatory.nasa.gov": 0.8493083274285178,+ "www.gladwell.com": 0.8492393112893071,+ "www.cgpgrey.com": 0.8492221809584313,+ "www.dailymail.co.uk": 0.8492114987186158,+ "source.opennews.org": 0.8492058945166755,+ "mikecanex.wordpress.com": 0.849180109488917,+ "unsharpen.com": 0.8491592756377232,+ "haseebq.com": 0.8491526060447331,+ "blog.diasporafoundation.org": 0.849136592576036,+ "www.pugetsystems.com": 0.8491094043930865,+ "roadmap.sh": 0.8491002596731841,+ "www.rawstory.com": 0.8490683893365794,+ "www.nylas.com": 0.8490240168878735,+ "www.tunetheweb.com": 0.8489936415581292,+ "www.subtraction.com": 0.8489862226571359,+ "adam.blog.heroku.com": 0.8489616756127251,+ "www.gdcvault.com": 0.8489549021982677,+ "n26.com": 0.8489269016604418,+ "www.passportindex.org": 0.8489269016604418,+ "www.sfweekly.com": 0.8489155778996055,+ "wiki.csswg.org": 0.8489096573208723,+ "remoteok.io": 0.8488395634361555,+ "www.guru3d.com": 0.8488079037622022,+ "www.wisdom.weizmann.ac.il": 0.8487791265218381,+ "fred.stlouisfed.org": 0.8487687545118902,+ "www.quantcast.com": 0.8487674782234258,+ "www.dspguide.com": 0.8487415764723351,+ "monitor.firefox.com": 0.8487326723774654,+ "www.manchester.ac.uk": 0.8487326070292353,+ "www.ecowatch.com": 0.8487187193792909,+ "exponents.co": 0.8487052872629832,+ "ubere.ng": 0.8486844576029813,+ "lists.cs.uiuc.edu": 0.8486469141688284,+ "www.erasmatazz.com": 0.8485843687777063,+ "prestonbyrne.com": 0.8485821394042593,+ "fakesteve.blogspot.com": 0.8485803980992321,+ "andyet.net": 0.8485375978388094,+ "blog.getfirebug.com": 0.8485293903506373,+ "joshtronic.com": 0.8485255486310321,+ "ngm.nationalgeographic.com": 0.8484980049334815,+ "www.cat-bus.com": 0.8484880761635857,+ "oos.moxiecode.com": 0.8484880761635857,+ "dlo.me": 0.8484699284663549,+ "www.openbazaar.org": 0.8484375,+ "tumbledry.org": 0.8484375,+ "boris-marinov.github.io": 0.8484375,+ "www.pcmag.com": 0.8484334813622002,+ "wsj.com": 0.8484234837206328,+ "ssd.eff.org": 0.8484099350915519,+ "www.worldometers.info": 0.8483883403559179,+ "www.denverpost.com": 0.8483741516385197,+ "blog.teambox.com": 0.8482961926347361,+ "www.staff.science.uu.nl": 0.8482766434239408,+ "chromium-review.googlesource.com": 0.8482702404753355,+ "www.antoarts.com": 0.8482404035930796,+ "www.redcode.nl": 0.8482404035930796,+ "www.morningstar.com": 0.8482320069770474,+ "www.nomachetejuggling.com": 0.8481585934322239,+ "influxdata.com": 0.8481227202820945,+ "ericholscher.com": 0.8481140078411193,+ "www.nfx.com": 0.8481012921108911,+ "wellpreparedmind.wordpress.com": 0.8480622757690688,+ "kara.allthingsd.com": 0.8480371905314535,+ "taptaptap.com": 0.8480285039672184,+ "rhizome.org": 0.8479990358005195,+ "c-faq.com": 0.8479835930179948,+ "lighttable.com": 0.8478713990382907,+ "www.ocweekly.com": 0.8478713990382906,+ "www.yesmagazine.org": 0.8478423076321161,+ "boston.cbslocal.com": 0.8477838372324837,+ "sethvargo.com": 0.847722900935325,+ "www.teamliquid.net": 0.8477100468877423,+ "kn100.me": 0.8476871371630261,+ "carsonified.com": 0.8476327362582359,+ "makecode.com": 0.8476290144215641,+ "www.leejamesrobinson.com": 0.8476290144215641,+ "journals.lww.com": 0.8476245506194432,+ "philfreo.com": 0.8476129588207604,+ "insights.som.yale.edu": 0.8475823333113045,+ "fredwu.me": 0.8475685309018646,+ "www.desertsun.com": 0.8475476379514038,+ "blog.noctua-software.com": 0.8475078292773957,+ "engineering.twitter.com": 0.8474442201925468,+ "elazzabi.com": 0.847403765595545,+ "camendesign.com": 0.8473892733773356,+ "spaceflightnow.com": 0.8473844409998718,+ "www.cs.uic.edu": 0.8473371233913075,+ "www.lpalmieri.com": 0.8473049751829188,+ "www.call-with-current-continuation.org": 0.8472612335852765,+ "pub.gajendra.net": 0.8472440944881889,+ "write.privacytools.io": 0.8471964059191153,+ "blog.gitprime.com": 0.8471964059191153,+ "deepstreamhub.com": 0.8471964059191153,+ "fosspost.org": 0.8471934496110164,+ "www.staradvertiser.com": 0.8471904258144719,+ "daggle.com": 0.8471392026866065,+ "text.sourcegraph.com": 0.8471325439607902,+ "graphics.reuters.com": 0.8470805937710428,+ "www.pulumi.com": 0.8470801636194034,+ "deepstream.io": 0.8470226785546888,+ "shop.oreilly.com": 0.8469734259689878,+ "www.rebol.com": 0.8469659113684828,+ "nylas.com": 0.8469368267588653,+ "sethgodin.typepad.com": 0.8469363395112108,+ "oilprice.com": 0.8469313973547731,+ "www.itsnicethat.com": 0.8469219290040122,+ "www.theblackvault.com": 0.8468972040529525,+ "jboriss.wordpress.com": 0.8468972040529525,+ "jeena.net": 0.8468879391324495,+ "www.mixnode.com": 0.8468817868572549,+ "danco.substack.com": 0.8468470187947723,+ "dyno-might.github.io": 0.8468364833167144,+ "extremelearning.com.au": 0.8467447980712459,+ "news.crunchbase.com": 0.8467020525586185,+ "the-japan-news.com": 0.8466850596011346,+ "www.tor.com": 0.8466742350710085,+ "graphics.pixar.com": 0.8466553705876948,+ "jackmott.github.io": 0.846643484398615,+ "www.deekit.com": 0.8466140516084654,+ "gkbrk.com": 0.8466121179458332,+ "washingtonexaminer.com": 0.846542334844308,+ "www.jayhuang.org": 0.8465047427289335,+ "lunchbag.ca": 0.8464844037983796,+ "www.vanschneider.com": 0.8464718715474767,+ "www.seobook.com": 0.8464370807656654,+ "lfe.io": 0.8464044390292613,+ "www.unitedstatescourts.org": 0.8463713866332889,+ "www.listennotes.com": 0.8463577921119287,+ "techblog.livingsocial.com": 0.8463371185563517,+ "blogoscoped.com": 0.8463347353586499,+ "www.density.io": 0.8463125089091263,+ "dbasch.posterous.com": 0.8462528298095997,+ "martiancraft.com": 0.8462422366153325,+ "blog.meldium.com": 0.8461981486766226,+ "blog.codeclimate.com": 0.8461758267783162,+ "www.abtinforouzandeh.com": 0.8461646472697207,+ "rachelnabors.com": 0.8461583302534714,+ "zencoder.com": 0.8461094533062384,+ "www.itworld.com": 0.8460477806046857,+ "blog.ipinfo.io": 0.846031746031746,+ "www.popsci.com": 0.8460295003872216,+ "www.autoweek.com": 0.8460205561195527,+ "bbot.org": 0.8459877925136515,+ "crookedtimber.org": 0.8459783522151496,+ "www.cultofmac.com": 0.8459180081962092,+ "en.wikisource.org": 0.8458715274833618,+ "www.kaggle.com": 0.8458458101824604,+ "www.europarl.europa.eu": 0.8458356717488885,+ "benchmarksgame.alioth.debian.org": 0.8458109252610799,+ "librepcb.org": 0.8457869634340223,+ "www.nationalobserver.com": 0.8457869634340223,+ "skitch.com": 0.8457433438672782,+ "www.bfilipek.com": 0.8457240963150365,+ "scottaaronson.com": 0.8456937780186609,+ "fee.org": 0.8456866411435615,+ "www.doomworld.com": 0.8456828105532045,+ "librivox.org": 0.845661772928326,+ "www.crispymtn.com": 0.845661772928326,+ "www.nydailynews.com": 0.8455856575577351,+ "brilliantmaps.com": 0.8455577844591506,+ "tiamat.tsotech.com": 0.8455576461751069,+ "npf.io": 0.8455538969123215,+ "jonathan.bergknoff.com": 0.8455538969123215,+ "belitsoft.com": 0.8455477879801876,+ "blog.thecybershadow.net": 0.8455414012738853,+ "blog.privacytools.io": 0.8455414012738853,+ "hardmath123.github.io": 0.8455383067302876,+ "elifesciences.org": 0.8455333018767387,+ "www.nemil.com": 0.845471203066534,+ "web.eecs.umich.edu": 0.8454557317112926,+ "500hats.typepad.com": 0.8454234389310107,+ "jsonfeed.org": 0.8454122902628886,+ "opendatastructures.org": 0.8454122902628886,+ "tex.stackexchange.com": 0.8454020068893373,+ "yifan.lu": 0.8453802898253059,+ "www.sierraclub.org": 0.8453802898253059,+ "axisofeval.blogspot.com": 0.845371248915586,+ "www.lbatalha.com": 0.8453195049851959,+ "theferrett.livejournal.com": 0.8452950558213717,+ "libmill.org": 0.8452950558213717,+ "www.epa.gov": 0.8452050900524932,+ "tricycle.org": 0.8451979459749552,+ "gking.harvard.edu": 0.8451920342583992,+ "seattletimes.nwsource.com": 0.8451827104383299,+ "killscreen.com": 0.8451534193795461,+ "www.pixeltrue.com": 0.8451429290751707,+ "www.randalolson.com": 0.8451252770638041,+ "en.blog.wordpress.com": 0.8450869071274824,+ "blog.thijsalkema.de": 0.845084400536627,+ "www.zephoria.org": 0.8450267106265884,+ "paulromer.net": 0.8450058741046084,+ "blog.gtk.org": 0.8449976566750211,+ "blog.flattr.net": 0.8449113529737324,+ "seedmagazine.com": 0.8448457357890011,+ "www.rt.com": 0.8448330221761664,+ "technosophos.com": 0.8448317661673463,+ "monzo.com": 0.844824966969565,+ "www.imperial.ac.uk": 0.8448169716366182,+ "andreyf.tumblr.com": 0.8448123793732512,+ "renesd.blogspot.com": 0.8448109932557698,+ "www.secretgeek.net": 0.8447678779550777,+ "alistapart.com": 0.8447602813793536,+ "facebook.design": 0.8446854923268982,+ "sachachua.com": 0.8446808335848872,+ "www.codesections.com": 0.8446790832287159,+ "davidwong.fr": 0.844658979317956,+ "blogs.office.com": 0.8446378605952511,+ "root.cern.ch": 0.8446152758619639,+ "indieweb.org": 0.8446147837430709,+ "blog.reemer.com": 0.8446120410396897,+ "www.change.org": 0.844610318859491,+ "www.mrspeaker.net": 0.8446002192794787,+ "www.sun-sentinel.com": 0.8445943201382862,+ "gorban.org": 0.844551282051282,+ "www.bgr.com": 0.8445425484905292,+ "coinmarketcap.com": 0.8445032134621009,+ "0day.work": 0.8444799253517218,+ "blog.realkinetic.com": 0.8444710920604793,+ "m.bbc.co.uk": 0.844465510423491,+ "earther.com": 0.8444605232524242,+ "link.springer.com": 0.8444514328769025,+ "positech.co.uk": 0.8444262068957591,+ "www.onthemedia.org": 0.8444066696037038,+ "www.cleveralgorithms.com": 0.8444062415001815,+ "mailchi.mp": 0.8443098020754962,+ "www.pixelapse.com": 0.8443017656500802,+ "berniesanders.com": 0.8443017656500802,+ "a.wholelottanothing.org": 0.8442952471954357,+ "events.ccc.de": 0.8442791650208269,+ "www.chicagobusiness.com": 0.8442628765380391,+ "nsl.com": 0.844239240640813,+ "srcco.de": 0.844239240640813,+ "testpilot.firefox.com": 0.8442195629693428,+ "noahveltman.com": 0.8441367921512271,+ "jamiequint.com": 0.8441367921512271,+ "reverb.com": 0.8441367921512271,+ "www.pri.org": 0.8441332017534635,+ "www.trustnodes.com": 0.8441221359793255,+ "blogs.mathworks.com": 0.8440748728050913,+ "www.cncf.io": 0.8440600291289487,+ "opensource.com": 0.844058190643116,+ "about.digg.com": 0.8440360228264422,+ "news.brown.edu": 0.8440209497481723,+ "www.forth.com": 0.8440090068688062,+ "blockstream.com": 0.843993466124853,+ "getaether.net": 0.8439867218274119,+ "www.hooktheory.com": 0.8439867218274119,+ "irssi.org": 0.8439867218274119,+ "jemma.dev": 0.8439780808263159,+ "gittup.org": 0.8439780808263159,+ "www.dropboxforum.com": 0.8439738032049753,+ "blog.neocities.org": 0.8439723488535399,+ "www.aopa.org": 0.8439413757138091,+ "sappingattention.blogspot.com": 0.8439256178572813,+ "math-blog.com": 0.843908007189824,+ "meetup.hackernewsdc.org": 0.8438982906033671,+ "www.ibtimes.sg": 0.8438980758876966,+ "curiousrat.com": 0.8438980758876966,+ "www.norvig.com": 0.8438704067825865,+ "michaelnielsen.org": 0.8438489596573671,+ "www.ponylang.io": 0.843843768902312,+ "www.noisebridge.net": 0.843843768902312,+ "apps.fcc.gov": 0.8438354432862912,+ "www.eenews.net": 0.8437777995615074,+ "www.dhs.gov": 0.8437568931274587,+ "www.uu.se": 0.8437434035754188,+ "blog.streak.com": 0.8437430051067691,+ "bluishcoder.co.nz": 0.8436990841434638,+ "trello.com": 0.8436685239481871,+ "prolost.com": 0.8436572621196788,+ "corgibytes.com": 0.843643069430385,+ "cbloomrants.blogspot.com": 0.8436415581159902,+ "grouptalent.com": 0.8436259181114076,+ "news.sky.com": 0.8435750425055707,+ "dankaminsky.com": 0.8435552716799266,+ "blog.kaspersky.com": 0.8435542294083143,+ "blog.rubbingalcoholic.com": 0.8435483870967742,+ "paul.copplest.one": 0.8435464913857005,+ "andrewhy.de": 0.8435425442649045,+ "www.techfounder.net": 0.8435095761996304,+ "spinroot.com": 0.8435095761996304,+ "garethrees.org": 0.8435072199487013,+ "blog.gedrap.me": 0.8434993251797909,+ "oceans.nautil.us": 0.8434897723987279,+ "www.autonlab.org": 0.8434766589029354,+ "kallithea-scm.org": 0.8433870123342087,+ "www.getrevue.co": 0.8433604592288586,+ "www.whatsonweibo.com": 0.8433258307054031,+ "tass.com": 0.843318150706682,+ "paste.lisp.org": 0.8433142035934179,+ "www.rollcall.com": 0.8433069334851104,+ "reference.wolfram.com": 0.8432544830728409,+ "opendotdotdot.blogspot.com": 0.8432276309668896,+ "kevin.burke.dev": 0.8432276309668896,+ "decisiondata.org": 0.8432276309668896,+ "xenaproject.wordpress.com": 0.8432246085096772,+ "benbernardblog.com": 0.8432189502435823,+ "usefathom.com": 0.8432043711877606,+ "www.pcpro.co.uk": 0.8431913701641113,+ "progit.org": 0.8431775195017707,+ "i.imgur.com": 0.8431547290624056,+ "developer.okta.com": 0.8431277870072948,+ "blog.medium.com": 0.8431066839897218,+ "ffii.org": 0.8430969026172707,+ "byrslf.co": 0.8430850673452497,+ "sigops.org": 0.8430850673452497,+ "blog.whoishiring.io": 0.8430812250163859,+ "www.mcvuk.com": 0.8430765289027752,+ "www.google.pl": 0.843042071197411,+ "blog.untogether.co.uk": 0.843042071197411,+ "www.uh.edu": 0.8430266348842538,+ "fable.io": 0.8429358535210576,+ "www.hizook.com": 0.8429332432339712,+ "www.ovh.com": 0.8429034962421136,+ "www.tedinski.com": 0.8428702241660795,+ "entrepreneur.venturebeat.com": 0.8428673093425917,+ "www.c-spanvideo.org": 0.8428673045430252,+ "www.historyextra.com": 0.8428269409461818,+ "www.defenseone.com": 0.8428181514777605,+ "volument.com": 0.8428160789962235,+ "www.imaging-resource.com": 0.8428098585582268,+ "popupcity.net": 0.8427876823338736,+ "www.skyandtelescope.com": 0.8427597458791004,+ "www.viva64.com": 0.8426735071847921,+ "existentialtype.wordpress.com": 0.8426727373097719,+ "www.ikea.com": 0.8426410914015982,+ "earnestcapital.com": 0.8426131831456453,+ "jmarbach.com": 0.8425928309863708,+ "chrisharrison.net": 0.8425324675324676,+ "www.jsonline.com": 0.8425180713598075,+ "parenting.nytimes.com": 0.8425153384707086,+ "tesla-cdn.thron.com": 0.8424060866368837,+ "www.neuroscientistnews.com": 0.8424034611022545,+ "www.wondersofphysics.com": 0.842390899623404,+ "www.lanl.gov": 0.8423253496487528,+ "skeptics.stackexchange.com": 0.8423210116773696,+ "www.rfc-editor.org": 0.8422724006868584,+ "www.industrialempathy.com": 0.8422674053917267,+ "performancejs.com": 0.8422235444245709,+ "infiniteundo.com": 0.8422087232147684,+ "www.heroku.com": 0.8422087232147684,+ "blog.lse.epita.fr": 0.8421833530068131,+ "lacker.io": 0.8421778420117882,+ "timeline.com": 0.8421633036155716,+ "wiki.blender.org": 0.8420940430794808,+ "www.freakonomics.com": 0.8420407955613295,+ "blog.fugue.co": 0.8420361620441877,+ "mediadecoder.blogs.nytimes.com": 0.8419912813368542,+ "www.irs.gov": 0.8419451366935324,+ "www.golem.de": 0.8419385789535662,+ "pingcap.com": 0.8419161509277815,+ "john.onolan.org": 0.8418365869145599,+ "blog.mocality.co.ke": 0.8417877979157168,+ "status.coinbase.com": 0.8417606283759475,+ "www.attejuvonen.fi": 0.8417377670517942,+ "serversforhackers.com": 0.8417376768816427,+ "work.caltech.edu": 0.8417166624718854,+ "fuelyourcoding.com": 0.841709160679554,+ "dyn.com": 0.8416196735446702,+ "blog.tabini.ca": 0.8415950088038011,+ "ieeexplore.ieee.org": 0.8415919239489138,+ "www.israel21c.org": 0.8415492534155486,+ "rubyonrails.org": 0.8415351989418526,+ "www.designweek.co.uk": 0.8415188999809792,+ "collison.ie": 0.8415120389596948,+ "www.evilmadscientist.com": 0.8414856456254624,+ "muddylemon.com": 0.8414700703965355,+ "agateau.com": 0.8414619982013722,+ "blog.miguelgrinberg.com": 0.8414611167027505,+ "nmap.org": 0.8414349271123382,+ "www.theblockcrypto.com": 0.8414118140696276,+ "www.haaretz.com": 0.8413639156892987,+ "codecapsule.com": 0.8413373129936518,+ "ethz.ch": 0.84131835872283,+ "blog.bloomca.me": 0.8413144015517519,+ "blog.snapeda.com": 0.8413079629673079,+ "evonomics.com": 0.8413074174618784,+ "www2.macleans.ca": 0.8412847684511929,+ "composition.al": 0.841279463048803,+ "js13kgames.com": 0.8412786874884526,+ "lists.basho.com": 0.8412692653556527,+ "people.redhat.com": 0.8412692653556527,+ "oceanservice.noaa.gov": 0.8412678170260773,+ "sciencenorway.no": 0.8412438625204582,+ "speaking.io": 0.8412438625204582,+ "www.exceptionnotfound.net": 0.8412220203835995,+ "www.2ality.com": 0.8411429398560686,+ "www.eecs.harvard.edu": 0.841119342861029,+ "bldgblog.blogspot.com": 0.8411187765808322,+ "thisismynext.com": 0.8410701280879982,+ "unchartedterritories.tomaspueyo.com": 0.8410441941601818,+ "www.skilled.io": 0.8410441941601818,+ "tenfourfox.blogspot.com": 0.8410170033847231,+ "thebreakfastpost.com": 0.8410159372229994,+ "blog.bumblebeelabs.com": 0.8409937866242836,+ "mmikowski.github.io": 0.8409753773821584,+ "paultyma.blogspot.com": 0.8409535720717558,+ "materialize.io": 0.8409502033267174,+ "www.ncdc.noaa.gov": 0.840942212674096,+ "www.haystacksoftware.com": 0.8409422126740959,+ "techiedelight.quora.com": 0.8409422126740959,+ "blog.modernmechanix.com": 0.8408958299079167,+ "www.math3ma.com": 0.8408873008599574,+ "retractionwatch.com": 0.8408852959781978,+ "www.appleoutsider.com": 0.8408203431401681,+ "www.dreamsongs.com": 0.8408038061650092,+ "blogs.cisco.com": 0.8407980680062469,+ "www.research.ibm.com": 0.8407779430526359,+ "gohugo.io": 0.8407728272201884,+ "nicolodavis.com": 0.8407632567723468,+ "www.andreykurenkov.com": 0.8407450453082168,+ "conversionxl.com": 0.8407343949968893,+ "www.qemu-advent-calendar.org": 0.8407224958949097,+ "enfranchisedmind.com": 0.8407038581523874,+ "ozkatz.github.com": 0.8406930647478468,+ "ahrefs.com": 0.8406628331444651,+ "paulmck.livejournal.com": 0.8406466273816793,+ "spf13.com": 0.8406252111185971,+ "adamschwartz.co": 0.8406212712895198,+ "scikit-learn.org": 0.8406196399338859,+ "www.lessonsoffailure.com": 0.8406081585039813,+ "mathenchant.wordpress.com": 0.8406026505628841,+ "charlesleifer.com": 0.8405838838741042,+ "zeux.io": 0.8405743435687888,+ "bicycledutch.wordpress.com": 0.8405743435687888,+ "ai.stanford.edu": 0.8405537358599618,+ "iximiuz.com": 0.8405441526018748,+ "kerkour.com": 0.8405182576561551,+ "www.racked.com": 0.8404972480623266,+ "news.morningstar.com": 0.8404838510660001,+ "hfet.org": 0.8404775873220368,+ "newcome.wordpress.com": 0.8404605263157895,+ "enterprisecraftsmanship.com": 0.8404567783699838,+ "dailynous.com": 0.8404286033548175,+ "usersknow.blogspot.com": 0.8402966516996352,+ "www.sandiegouniontribune.com": 0.8402646673456328,+ "www.playframework.com": 0.8402476003167625,+ "betterhumans.coach.me": 0.840238289918768,+ "thomaspark.co": 0.8402340830446674,+ "deadlockempire.github.io": 0.8401976935749588,+ "mimimatthews.com": 0.8401762295218094,+ "chris.beams.io": 0.8401743605753639,+ "www.whoismcafee.com": 0.8401743605753639,+ "www.stlouisfed.org": 0.8401103027746665,+ "fredrikdeboer.com": 0.8400945631732266,+ "blog.cerebralab.com": 0.8400945631732265,+ "weworkremotely.com": 0.8400536676760625,+ "english.yonhapnews.co.kr": 0.840015150270215,+ "blog.ffwll.ch": 0.8399806882990191,+ "www.kaspersky.com": 0.8399734036718564,+ "webmasters.stackexchange.com": 0.8399733386768045,+ "taoofmac.com": 0.8399516851214591,+ "webapps.stackexchange.com": 0.8399426081446101,+ "nenadseo.com": 0.839939548717707,+ "our.status.im": 0.8399358470807224,+ "justinhileman.info": 0.839916636052546,+ "www.financemagnates.com": 0.83991429571072,+ "srlabs.de": 0.8399137310646634,+ "emacsredux.com": 0.8399137310646634,+ "www.colorforth.com": 0.8398830755348644,+ "tech.trivago.com": 0.8398706352344396,+ "rusty.ozlabs.org": 0.8398201966467941,+ "www.unionleader.com": 0.8398201966467941,+ "tomverbeure.github.io": 0.8397956166500623,+ "www.huckmag.com": 0.8397408630849317,+ "www.3quarksdaily.com": 0.8397300193715044,+ "snarky.ca": 0.8397285354135811,+ "basicattentiontoken.org": 0.8396694214876033,+ "0patch.blogspot.com": 0.8396569050806407,+ "www.stackfield.com": 0.8396561419057332,+ "laktek.com": 0.8395904864603353,+ "www.elasticsearch.com": 0.8395809014444943,+ "thomashunter.name": 0.8395739505236344,+ "tapbots.com": 0.8395257149612579,+ "nlp.stanford.edu": 0.8395127943859381,+ "shime.sh": 0.8395015090274327,+ "news.consumerreports.org": 0.8395015090274327,+ "www.alwaysrightinstitute.com": 0.8395015090274327,+ "plover.net": 0.8394877101221607,+ "oleb.net": 0.8394619474972026,+ "www.mymoneyblog.com": 0.8394511334746066,+ "blog.clarifai.com": 0.8394429821320054,+ "technodrone.blogspot.com": 0.8394039735099338,+ "labs.earthpeople.se": 0.8394039735099338,+ "blog.greweb.fr": 0.8394025195959639,+ "truthout.org": 0.8393907588504428,+ "www.rowetel.com": 0.8393890552964602,+ "acha.ninja": 0.8393871342567361,+ "blog.carlmjohnson.net": 0.8393408901985292,+ "programming-journal.org": 0.8393406966484743,+ "www.news18.com": 0.8392713736775148,+ "sam.bluwiki.com": 0.8392570453444638,+ "www.monbiot.com": 0.8392563174673074,+ "rawgit.com": 0.8392483684869478,+ "www.foxla.com": 0.8392193702289447,+ "dvt.name": 0.839218830660763,+ "blog.facebook.com": 0.8392045415231093,+ "freddiedeboer.substack.com": 0.8391840290729782,+ "developer.marvel.com": 0.8391376451077944,+ "www.lowendtalk.com": 0.8391326019880078,+ "docs.celeryproject.org": 0.8391326019880078,+ "blog.trendmicro.com": 0.8391258095534089,+ "poststatus.com": 0.8390975027931495,+ "blockgeeks.com": 0.8390450127680085,+ "itnext.io": 0.8390022169180034,+ "blog.existentialize.com": 0.8389858546214449,+ "semiengineering.com": 0.8389442107551947,+ "www.digibarn.com": 0.8389114239700469,+ "mikemcquaid.com": 0.8388881112064441,+ "hackflow.com": 0.838888111206444,+ "lists.opensuse.org": 0.8388704318936877,+ "weblog.rogueamoeba.com": 0.8387522024258429,+ "pubs.acs.org": 0.838739869285439,+ "www.webdesignmuseum.org": 0.8387125258530213,+ "www.garron.blog": 0.8387125258530211,+ "instadeq.com": 0.8387105083280972,+ "www.brisbanetimes.com.au": 0.8386591269613846,+ "leanpub.com": 0.8386465804494829,+ "fendrich.se": 0.8386023294509152,+ "dl.dropboxusercontent.com": 0.8385995778248918,+ "lettier.github.io": 0.8385983762141392,+ "www.edn.com": 0.8384880352277337,+ "www.synthtopia.com": 0.8384691370060022,+ "www.michaelbromley.co.uk": 0.8384548675731243,+ "www.refsmmat.com": 0.8384548675731243,+ "jonbischke.com": 0.8384341839739798,+ "warzel.substack.com": 0.8384049720140551,+ "blogs.kqed.org": 0.8383873467097201,+ "www.xconomy.com": 0.8383614008234231,+ "www.timeslive.co.za": 0.838354450918394,+ "keithp.com": 0.8383347979461153,+ "psycnet.apa.org": 0.8383190430040641,+ "blog.online.net": 0.8383007137970923,+ "www.atariarchives.org": 0.8382952495330802,+ "www.fivethirtyeight.com": 0.8382374049516083,+ "www.wearethemighty.com": 0.8381915632322696,+ "www.arstechnica.com": 0.8381868933365827,+ "heatst.com": 0.8381531996802586,+ "blog.wekeroad.com": 0.838151852850549,+ "www.dnainfo.com": 0.8381308827626469,+ "wufoo.com": 0.8381135600270289,+ "www.zompist.com": 0.8381135600270289,+ "www.nbr.co.nz": 0.8380751982509018,+ "tomtunguz.com": 0.8380729812475479,+ "www.scripting.com": 0.8380697700961443,+ "blog.twilio.com": 0.8380613391841799,+ "www.nhplace.com": 0.8380324040052766,+ "pypi.python.org": 0.8380184467463045,+ "www.the-digital-reader.com": 0.837988405339907,+ "www.vpri.org": 0.8379878140892437,+ "www.bailis.org": 0.8379777722457122,+ "www.maximise.dk": 0.8379766121747607,+ "99designs.com": 0.8379580943514392,+ "www.crummy.com": 0.8379490514739079,+ "www.ibtimes.co.uk": 0.8379434303523721,+ "vmware.github.io": 0.8379411472179317,+ "www.6sqft.com": 0.8379298178385524,+ "symkat.com": 0.8379298178385524,+ "survata.com": 0.8378674714439094,+ "www.greenwichtime.com": 0.8378674714439094,+ "www.leandomainsearch.com": 0.8378176898959185,+ "penguindreams.org": 0.8377993988882627,+ "www.cs.ox.ac.uk": 0.8377993988882627,+ "windowsteamblog.com": 0.8377936722512888,+ "www.more-magic.net": 0.8377744800755205,+ "wundergraph.com": 0.8377666240744057,+ "www.stereogum.com": 0.8377124380298397,+ "media.defcon.org": 0.8376483665038642,+ "andrewsullivan.theatlantic.com": 0.8376455603446734,+ "physics.aps.org": 0.8376321975113651,+ "www.addedbytes.com": 0.8376314906854643,+ "theconversation.edu.au": 0.8376217948668653,+ "carwoo.com": 0.8376024057546919,+ "www.swiftbysundell.com": 0.8375837630694467,+ "bigthink.com": 0.837574858033942,+ "www.nltk.org": 0.8375336447475157,+ "pron.github.io": 0.8375247229364985,+ "atlas.cern": 0.8375209380234506,+ "www.nyse.com": 0.8375209380234506,+ "icyphox.sh": 0.8374905510071093,+ "hypothesis.works": 0.8374147657218561,+ "jacobobryant.com": 0.8373876028525398,+ "blogs.zappos.com": 0.8373722614685953,+ "blog.joellehman.com": 0.8373621224652893,+ "conal.net": 0.8373554518624804,+ "www.dadhacker.com": 0.837354651092982,+ "stars.chromeexperiments.com": 0.8372729777719929,+ "blog.thousandeyes.com": 0.8372663541460196,+ "www.drbunsen.org": 0.8372619145116595,+ "blog.tinfoilsecurity.com": 0.8372571688925584,+ "www.landley.net": 0.837248322147651,+ "jjbeshara.com": 0.837248322147651,+ "www.searchenginejournal.com": 0.8372190177985843,+ "samgentle.com": 0.8371607111617112,+ "www.mywinamp.com": 0.8371394760092047,+ "www.acm.uiuc.edu": 0.8371394760092047,+ "blogs.sfweekly.com": 0.8371151654308293,+ "theaviationist.com": 0.8370871690994972,+ "www.mailchimp.com": 0.8370705380744307,+ "incubator.apache.org": 0.8370705380744307,+ "www.mikealrogers.com": 0.8370700724965489,+ "www.unofficialgoogledatascience.com": 0.8370621601099476,+ "www.wildbunny.co.uk": 0.8370316066816674,+ "mega.co.nz": 0.8370228944247635,+ "vimcasts.org": 0.8370122300530931,+ "home.web.cern.ch": 0.8370107312738333,+ "news.virginia.edu": 0.8370064211588852,+ "blog.zawodny.com": 0.8370064211588852,+ "www.mdpi.com": 0.8370060708284108,+ "blog.mousereeve.com": 0.8370038016134355,+ "hackernewsbooks.com": 0.8369747899159664,+ "detexify.kirelabs.org": 0.8369455416288776,+ "blip.tv": 0.8369060266959822,+ "goodstuff.im": 0.8369038418527343,+ "opensignal.com": 0.8368950022832518,+ "the-pastry-box-project.net": 0.836851759413345,+ "parkerhiggins.net": 0.8368481032461187,+ "www.roughtype.com": 0.8367664523195075,+ "www.ecosia.org": 0.8367628729886891,+ "dougmccune.com": 0.8367628729886891,+ "lambdaisland.com": 0.8367557134451551,+ "blog.8thlight.com": 0.8367523868737199,+ "mml-book.github.io": 0.8367337334597409,+ "akaptur.com": 0.8367337334597409,+ "www.appneta.com": 0.8367099124724148,+ "eos.org": 0.836703598387807,+ "www.eia.gov": 0.836692477143149,+ "www.freesoftwaremagazine.com": 0.8366844191693678,+ "blog.isquaredsoftware.com": 0.8366450083184942,+ "www.effectgames.com": 0.8366086074590122,+ "danbirken.com": 0.8366082851176284,+ "www.georgesaines.com": 0.8365836924813026,+ "projects.nytimes.com": 0.8365185959005205,+ "cilium.io": 0.8364900150066928,+ "luisant.ca": 0.8364627688697146,+ "kde.org": 0.8364506685642159,+ "gengo.ai": 0.8364249578414841,+ "gigamonkeys.com": 0.836424957841484,+ "ask.slashdot.org": 0.8364157461593826,+ "www.inf.ethz.ch": 0.8364037125832664,+ "bufr.tumblr.com": 0.8363012314554059,+ "go.bloomberg.com": 0.8362895418561958,+ "www.mv-voice.com": 0.8362735866170948,+ "www.visual6502.org": 0.8362643122145562,+ "codeofrob.com": 0.8362576149569876,+ "www.ludumdare.com": 0.8362566416598247,+ "www.bradford-delong.com": 0.8361844302416513,+ "sidsite.com": 0.8361486486486487,+ "persquaremile.com": 0.8361417911687711,+ "boss.blogs.nytimes.com": 0.8361416007600664,+ "pathdependent.com": 0.8361365464427095,+ "www.marketplacepulse.com": 0.8361110009915448,+ "javascriptweblog.wordpress.com": 0.8361100871908929,+ "blog.steve.fi": 0.8360870917809617,+ "www.stefankendall.com": 0.8360443900320628,+ "machinelearningmastery.com": 0.8360325276933736,+ "tomcritchlow.com": 0.8360317433744368,+ "www.vqronline.org": 0.8359910491447964,+ "pmarchive.com": 0.8359774363850249,+ "blog.exodusintel.com": 0.8359613952379756,+ "icelandmonitor.mbl.is": 0.8359181324679799,+ "learnui.design": 0.8359165073352933,+ "www.wisdomandwonder.com": 0.8359037098645399,+ "gnupg.org": 0.8359037098645399,+ "www.otherhand.org": 0.8359037098645399,+ "andyswan.com": 0.8358651631273738,+ "vlaaad.github.io": 0.8358605537700576,+ "unikernel.org": 0.8358276165245331,+ "forums.adobe.com": 0.8358049697804542,+ "theappeal.org": 0.8358049697804542,+ "people.mozilla.org": 0.8357546553679396,+ "www.multivax.com": 0.8357546553679396,+ "www.hilarymason.com": 0.8357367418987102,+ "metro.co.uk": 0.8357169083210791,+ "www.fxguide.com": 0.8356995496213405,+ "mail.gnome.org": 0.8356930267331182,+ "www.ripe.net": 0.8356373665440678,+ "polygraph.cool": 0.8355932203389831,+ "newsinteractives.cbc.ca": 0.8355673184633825,+ "oonwoye.com": 0.8355085503926205,+ "www.centernetworks.com": 0.83550554167365,+ "blog.kenperlin.com": 0.8354986352909005,+ "codelabs.developers.google.com": 0.8354986352909004,+ "ericasadun.com": 0.8354812545307914,+ "voyager.jpl.nasa.gov": 0.8354770866238536,+ "glinden.blogspot.com": 0.8354248687848085,+ "www.jeffknupp.com": 0.8354226452254534,+ "0xax.blogspot.com": 0.8354177064677306,+ "home.comcast.net": 0.8354135974937222,+ "www.scrapingbee.com": 0.8353947775852016,+ "mail-index.netbsd.org": 0.8353813194329008,+ "mikeindustries.com": 0.835369856283584,+ "failory.com": 0.8353532155275709,+ "www.ritsumei.ac.jp": 0.835322744014879,+ "typing.io": 0.8353140916808149,+ "blog.britishmuseum.org": 0.835306193677583,+ "mailbox.org": 0.8352951727923229,+ "hackety.org": 0.8352886313345826,+ "invisible-island.net": 0.8352861612933625,+ "blog.minio.io": 0.8352821552252798,+ "www.etymonline.com": 0.8352819967060172,+ "founderdating.com": 0.8352750374933166,+ "users.softlab.ece.ntua.gr": 0.8352418158169491,+ "read.acloud.guru": 0.8352400454348197,+ "julian.digital": 0.8352206968521787,+ "ben.balter.com": 0.8352023832422831,+ "osc.universityofcalifornia.edu": 0.8351585723230902,+ "futureoflife.org": 0.8351040617937822,+ "www.multicians.org": 0.8350943317132041,+ "allthatsinteresting.com": 0.8350698686955953,+ "machinethink.net": 0.8350562425214455,+ "blogs.denverpost.com": 0.8350421047806545,+ "mishadoff.com": 0.8350340136054422,+ "getfirebug.com": 0.8350340136054422,+ "www.dxomark.com": 0.8349344540359945,+ "www.mycpu.org": 0.8349193509952372,+ "www.raphkoster.com": 0.8349185464697736,+ "bps-research-digest.blogspot.com": 0.8349065337932573,+ "cs229.stanford.edu": 0.834894479023565,+ "sadanand-singh.github.io": 0.834894479023565,+ "blog.doordash.com": 0.834891053529973,+ "paul.kinlan.me": 0.834891053529973,+ "sliderulemuseum.com": 0.8347529812606473,+ "www.hpcwire.com": 0.8347353682368306,+ "flaviocopes.com": 0.8347098860524725,+ "nocomplexity.com": 0.8346402286365225,+ "designbygravity.wordpress.com": 0.8346002566095803,+ "what-if.xkcd.com": 0.8345955143488654,+ "www.healthcareitnews.com": 0.834479735234925,+ "turingpi.com": 0.834470989761092,+ "pfrazee.github.io": 0.834470989761092,+ "csrc.nist.gov": 0.8344705397477052,+ "taskwarrior.org": 0.8344665233968941,+ "grepular.com": 0.8344590445370506,+ "nuclearsecrecy.com": 0.8344590445370506,+ "maxkle.in": 0.8344294161546372,+ "www.csd.uwo.ca": 0.8344273945101189,+ "alanhollis.com": 0.8343851939878961,+ "www.splitgraph.com": 0.8343399060316927,+ "www.instapainting.com": 0.834303164841344,+ "taxfoundation.org": 0.834303164841344,+ "delta.chat": 0.8342867534842825,+ "appft1.uspto.gov": 0.8342760797997661,+ "leerob.io": 0.8342207211363054,+ "chrisshort.net": 0.8342193690645514,+ "blog.venanti.us": 0.8342064686003738,+ "news.cnrs.fr": 0.8341880341880342,+ "500px.com": 0.8341880341880342,+ "krugman.blogs.nytimes.com": 0.8341820699049042,+ "fearlesssalarynegotiation.com": 0.8341803150678754,+ "www.frankmcsherry.org": 0.8341803150678754,+ "nts.strzibny.name": 0.8341550107611373,+ "developers.memsql.com": 0.8341526449980673,+ "sr.ht": 0.8341334625060556,+ "ebiquity.umbc.edu": 0.8340723188782359,+ "news.ubc.ca": 0.8340723188782359,+ "blog.wired.com": 0.8340173587394528,+ "www.dailyherald.com": 0.8340105184549977,+ "artur-martsinkovskyi.github.io": 0.8339880299244583,+ "felixge.de": 0.8339880299244583,+ "blog.ayoungprogrammer.com": 0.833983000311883,+ "inhabitat.com": 0.833970800806039,+ "www.lifehacker.com.au": 0.8339654277683552,+ "science.howstuffworks.com": 0.8338999781361612,+ "cancer.nautil.us": 0.8338999781361612,+ "appcubby.com": 0.8338994925108514,+ "www.math.utah.edu": 0.8338684428078268,+ "tooslowexception.com": 0.8338684428078268,+ "www.patrickcraig.co.uk": 0.8338645320250127,+ "perfectionkills.com": 0.8338619239102345,+ "ncona.com": 0.8337668211914532,+ "blogs.aws.amazon.com": 0.8337525489654461,+ "www.cnil.fr": 0.83372186106167,+ "guidovranken.wordpress.com": 0.8337106909166525,+ "littlebitofcode.com": 0.8337106909166525,+ "crpgaddict.blogspot.com": 0.8337028204720323,+ "zerodha.tech": 0.833697181290741,+ "medicine.wustl.edu": 0.8336839373019279,+ "blog.socialcops.com": 0.8336751280750007,+ "ifttt.com": 0.8336678541410665,+ "blog.scalyr.com": 0.8335986795297188,+ "www.psypost.org": 0.8335783172647908,+ "www.abcactionnews.com": 0.8335754162799685,+ "one.google.com": 0.8335754162799685,+ "blog.marksweep.com": 0.8335690009897165,+ "basecamp.com": 0.8334897559746374,+ "www.rdegges.com": 0.8334643316971914,+ "michael.otacoo.com": 0.8334458765497461,+ "toucharcade.com": 0.8334457758738516,+ "adage.com": 0.8334270713794664,+ "getbootstrap.com": 0.8334213784102003,+ "blog.klipse.tech": 0.8334108661664996,+ "classics.mit.edu": 0.8334006037794777,+ "dev.to": 0.8333163721881794,+ "deadline.com": 0.833286267441488,+ "www.expressvpn.com": 0.8332852925186139,+ "zverovich.net": 0.8332852925186139,+ "www.bnl.gov": 0.8332751437457405,+ "discuss.linuxcontainers.org": 0.8332368170670513,+ "codesketch.com": 0.8332263874970826,+ "www.leveragedsellout.com": 0.8331869545898674,+ "www.imore.com": 0.8331827749737907,+ "people.inf.ethz.ch": 0.8331734250731533,+ "blog.smartbear.com": 0.8331510235912066,+ "ryanflorence.com": 0.8331396140981399,+ "stephanpastis.wordpress.com": 0.8331396140981399,+ "static1.squarespace.com": 0.8331073989685633,+ "createdigitalmusic.com": 0.8330970061467121,+ "bbc.co.uk": 0.8330970061467121,+ "www.nhm.ac.uk": 0.8330761814047353,+ "press.princeton.edu": 0.8330608346746611,+ "blog.hartleybrody.com": 0.8329386129816697,+ "futureboy.us": 0.8329182087040424,+ "www.windowslatest.com": 0.8329104232060686,+ "ewontfix.com": 0.8328938411174854,+ "developer.yahoo.com": 0.8328635993581618,+ "jng.imagine27.com": 0.8328607138329057,+ "gogs.io": 0.8328565779067133,+ "geoffboeing.com": 0.8328409513000423,+ "www.phillymag.com": 0.8328409513000423,+ "graphthinking.blogspot.com": 0.8328383114974501,+ "blog.authy.com": 0.8327906592352505,+ "earthengine.google.com": 0.8327019997862043,+ "adamwathan.me": 0.8326583014571802,+ "distributionhacks.com": 0.8326485954892083,+ "www.thinkster.io": 0.832628817491836,+ "terminusdb.com": 0.832628817491836,+ "www.technoblogy.com": 0.8326183396075918,+ "30hourjobs.com": 0.8326092279817636,+ "infoproc.blogspot.com": 0.8326089175295298,+ "www.ftrain.com": 0.8326051014052062,+ "foundation.zurb.com": 0.8326012313075719,+ "blog.viraptor.info": 0.8325776839853443,+ "bennesvig.com": 0.8325751054908778,+ "www.expressionsofchange.org": 0.832572579886013,+ "mrl.nyu.edu": 0.8325111338579982,+ "blog.urfix.com": 0.8325111338579982,+ "lists.x.org": 0.8324697754749568,+ "huggingface.co": 0.8324625696123126,+ "zeuxcg.org": 0.8324088201420566,+ "vmssoftware.com": 0.8323629518973035,+ "jimplush.com": 0.8323629518973035,+ "thereboot.com": 0.8323490243960362,+ "antimatter15.com": 0.8323430057057912,+ "japhr.blogspot.com": 0.8323430057057912,+ "blog.davidecoppola.com": 0.8323083996896996,+ "www.kotaku.com.au": 0.832301047102968,+ "3perf.com": 0.8322876151248816,+ "ryxcommar.com": 0.8322876151248816,+ "www.mysql.com": 0.8322876151248816,+ "scitechdaily.com": 0.8322404618162444,+ "rjzaworski.com": 0.8321799307958477,+ "www.dolthub.com": 0.8321769413258827,+ "www.jperla.com": 0.8321740485435664,+ "roberto.open-lab.com": 0.8321725984943663,+ "www.tofugu.com": 0.8321694358359643,+ "www.preprints.org": 0.8321312662446396,+ "ntrs.nasa.gov": 0.8321306733695456,+ "blog.kaggle.com": 0.8321136466040312,+ "getcarina.com": 0.8321067502232443,+ "thebarentsobserver.com": 0.8320962403151547,+ "www.improbable.com": 0.8320824782702279,+ "type.method.ac": 0.8320667644923518,+ "www.aleph.se": 0.8320667644923518,+ "lists.macosforge.org": 0.8320667644923518,+ "zguide.zeromq.org": 0.8320667644923518,+ "onethingwell.org": 0.8320539832149718,+ "www.gittip.com": 0.8320539832149718,+ "electronics.stackexchange.com": 0.8320387304939284,+ "www.newsobserver.com": 0.8319542731572184,+ "voat.co": 0.8319340573214568,+ "www.viemu.com": 0.8319229644593481,+ "www.embeddeduse.com": 0.8318890814558059,+ "ideolalia.com": 0.831795173528973,+ "andrewdumont.me": 0.8317884623663075,+ "sriramk.com": 0.8317884108541596,+ "www.chenhuijing.com": 0.8317790598787409,+ "www.yegor256.com": 0.8317697748343722,+ "www.foxnews.com": 0.8317694725922751,+ "www.cs.nott.ac.uk": 0.8317650230135005,+ "www.trustedreviews.com": 0.8317627240153616,+ "greatergood.berkeley.edu": 0.8316786529650598,+ "mitxela.com": 0.8315323559022839,+ "mail.haskell.org": 0.8315275148175146,+ "cooking.stackexchange.com": 0.8315230832695719,+ "www.apnews.com": 0.831500326107284,+ "www.belfasttelegraph.co.uk": 0.8314967853302809,+ "zackshapiro.com": 0.8314155294643537,+ "www.collectspace.com": 0.831412959256786,+ "www.alchemistowl.org": 0.8313334448247652,+ "www.interfluidity.com": 0.8313143760797148,+ "mithril.js.org": 0.8313043478260869,+ "sunboxlabs.com": 0.8313043478260869,+ "css.csail.mit.edu": 0.83128825969837,+ "mattermark.com": 0.8312791828318407,+ "ballotpedia.org": 0.8312622600459071,+ "r2c.dev": 0.8312622600459071,+ "goaccess.io": 0.8312325519112657,+ "www.michaelwest.com.au": 0.8312257534350801,+ "redsymbol.net": 0.8312144135803776,+ "journals.aps.org": 0.8312128897645714,+ "blog.gingerlime.com": 0.8312016352070194,+ "bugsplat.info": 0.8311332389568854,+ "projects.aljazeera.com": 0.8310960202717635,+ "jlouisramblings.blogspot.com": 0.8310960202717635,+ "www.datastuff.tech": 0.8310748264197387,+ "www.cliffc.org": 0.8310367707450238,+ "www.merriam-webster.com": 0.8310169102957669,+ "washingtonpost.com": 0.831005263332199,+ "www.leapmotion.com": 0.8309966909104267,+ "dom.as": 0.8309526362308279,+ "dotnet.github.io": 0.8309273722835752,+ "rickcarlino.com": 0.8309273722835752,+ "p-cos.blogspot.com": 0.8309273722835752,+ "www.electronicsweekly.com": 0.8308205769875683,+ "blog.mythic-beasts.com": 0.8308141567715057,+ "blog.codiqa.com": 0.830787883150054,+ "www.urbandictionary.com": 0.8307663266042938,+ "blog.ericgoldman.org": 0.8307656053312806,+ "thatcomputerscientist.com": 0.8307366610510188,+ "shankman.com": 0.830715532286213,+ "blog.uservoice.com": 0.830715532286213,+ "shafyy.com": 0.8306912467549404,+ "www.23andme.com": 0.8306912467549404,+ "thebulletin.org": 0.8306726473588931,+ "sekao.net": 0.8306717904781525,+ "om.co": 0.8306518197608382,+ "reacttraining.com": 0.8306279342292152,+ "phili.pe": 0.8306279342292152,+ "kamerontanseli.ghost.io": 0.8306279342292152,+ "www.deccanherald.com": 0.830617757462069,+ "recipes.hypotheses.org": 0.8306075161940402,+ "www.scotthyoung.com": 0.8304676761207413,+ "countercomplex.blogspot.com": 0.8304599554901336,+ "blog.kraken.com": 0.8304102297264507,+ "ello.co": 0.8303950986221522,+ "asiatimes.com": 0.8303938402412525,+ "blog.tindie.com": 0.8303900100100166,+ "www.plagiarismtoday.com": 0.8303836069407874,+ "moviecode.tumblr.com": 0.8303351870840227,+ "www.the74million.org": 0.8303274336465168,+ "analytics.usa.gov": 0.8303274336465168,+ "blog.jpl-consulting.com": 0.8303274336465168,+ "archive.nytimes.com": 0.8303195537893885,+ "blogs.zdnet.com": 0.8302477853461946,+ "www.gosquared.com": 0.8302108084053713,+ "www.unite.ai": 0.8301971117330208,+ "pastie.org": 0.8301429578448887,+ "www.bluetooth.com": 0.830122591943958,+ "heartbeat.fritz.ai": 0.8300819862294267,+ "blog.liw.fi": 0.8300810779081896,+ "gilesbowkett.blogspot.com": 0.8300257142091928,+ "placesjournal.org": 0.8299577016504602,+ "interfacelab.com": 0.8299038213762403,+ "programming-motherfucker.com": 0.8298839065342439,+ "www.thespacereview.com": 0.8298526044878857,+ "www.zippia.com": 0.8298245614035087,+ "alumnit.ca": 0.8298220650253975,+ "pne.people.si.umich.edu": 0.8297232221938442,+ "blog.scienceexchange.com": 0.8296882900501562,+ "danielallendeutsch.com": 0.8296786018063418,+ "veyronb.wordpress.com": 0.8296786018063417,+ "www.learningclojure.com": 0.829663604832329,+ "android.stackexchange.com": 0.8296351227828602,+ "neo4j.com": 0.8296343605300973,+ "www.insentricity.com": 0.8295390134146845,+ "johnhawks.net": 0.8295371897944813,+ "gist.io": 0.829536713446392,+ "www.irishexaminer.com": 0.8295100407790094,+ "blog.runnable.com": 0.8294913771537328,+ "www.dmv.ca.gov": 0.8294663396318804,+ "technology.stitchfix.com": 0.8293836920149785,+ "anyasq.com": 0.8293440267345259,+ "eigenfoo.xyz": 0.8293416074439832,+ "sunlightfoundation.com": 0.8293282465454183,+ "developer.rackspace.com": 0.829323262325674,+ "rwmj.wordpress.com": 0.8293063560783351,+ "www.makegameswith.us": 0.8292959190283576,+ "wesleytansey.com": 0.8292534470294121,+ "startupljackson.com": 0.8292534470294121,+ "www.xprize.org": 0.8292534470294121,+ "www.mongodb.org": 0.829231224668068,+ "www.drive.com.au": 0.8292253521126759,+ "crate.io": 0.8291626909612831,+ "voice.kadira.io": 0.8291212925208525,+ "blog.drewolson.org": 0.8291191739941484,+ "slog.thestranger.com": 0.8291191739941484,+ "remarkedly.com": 0.8291054023142576,+ "artem.krylysov.com": 0.8290877591855479,+ "www.collabora.com": 0.8290844858349273,+ "www.handelsblatt.com": 0.8290824405762608,+ "www.dailycamera.com": 0.8289847063149993,+ "jozefg.bitbucket.org": 0.8289755956167212,+ "baus.net": 0.8289755956167212,+ "www.davisr.me": 0.8289669222429755,+ "ghc.haskell.org": 0.8289669222429755,+ "akkshaya.blog": 0.8289669222429755,+ "www.boristhebrave.com": 0.8289241622574957,+ "ootbcomp.com": 0.8289241622574957,+ "www.puzzlescript.net": 0.8289241622574957,+ "mattjibson.com": 0.828841660397834,+ "knowledge.wharton.upenn.edu": 0.8288260425307535,+ "simpleflying.com": 0.8288152386804701,+ "houseoflawandorder.com": 0.828808793096398,+ "abstractions.nautil.us": 0.8288076574107112,+ "www.scalescale.com": 0.8288034171529194,+ "www.ostechnix.com": 0.8288034171529194,+ "www.londonreconnections.com": 0.8288034171529193,+ "sandro.volpee.de": 0.8287903687677997,+ "www.reghardware.com": 0.8287468808977058,+ "www.shellcheck.net": 0.8287381145882802,+ "m.youtube.com": 0.8287237652975437,+ "www.freep.com": 0.828700914495102,+ "www.cs.hmc.edu": 0.8286794342224705,+ "www.post-gazette.com": 0.8286406919749909,+ "ecorner.stanford.edu": 0.8285837667040196,+ "olimex.wordpress.com": 0.8285738879971565,+ "englishrussia.com": 0.8285720380368012,+ "2ton.com.au": 0.8285684110847549,+ "debugandrelease.blogspot.com": 0.8285505526982948,+ "news.err.ee": 0.8285383023940318,+ "academictimes.com": 0.8285283281973459,+ "beyondgrep.com": 0.8285017969437128,+ "www.lesspenguiny.com": 0.8285017969437128,+ "tinyhack.com": 0.8285017969437128,+ "www.mailgun.com": 0.8284973918318256,+ "homepage.divms.uiowa.edu": 0.8284973918318256,+ "grimoire.ca": 0.82849280304493,+ "unicornfree.com": 0.8284746725029384,+ "www.afr.com": 0.8284438035910985,+ "www.routefifty.com": 0.8284215095159949,+ "www.nintendolife.com": 0.8284071908632212,+ "www.wolfram.com": 0.8283831975537147,+ "www.rfi.fr": 0.8282942879074087,+ "talkweb.eu": 0.8282723807936619,+ "drdobbs.com": 0.8282680775958894,+ "elidourado.com": 0.8282580762478886,+ "www.math.uwaterloo.ca": 0.8282472101761524,+ "www.industryweek.com": 0.8282421484626746,+ "tipjoys2cents.blogspot.com": 0.8282133103461377,+ "www.prerender.cloud": 0.8282133103461377,+ "www.thezdi.com": 0.8282032202463764,+ "www.ableton.com": 0.8281936977442643,+ "kylerush.net": 0.8281936977442643,+ "hbswk.hbs.edu": 0.8281850054053886,+ "www.hopesandfears.com": 0.8281077808394027,+ "mappingignorance.org": 0.8281015489846159,+ "code.jsoftware.com": 0.8281015489846159,+ "thefrailestthing.com": 0.828083220780241,+ "blog.gopheracademy.com": 0.8280826702520344,+ "threedots.tech": 0.8280375017895765,+ "robnapier.net": 0.8280259986658036,+ "blogs.nasa.gov": 0.8280019811960149,+ "blog.indutny.com": 0.8279912351566244,+ "cloudonaut.io": 0.8279487990535627,+ "boston.craigslist.org": 0.827928286178696,+ "www.techprowd.com": 0.827928286178696,+ "smalldata.tech": 0.8279186302269647,+ "notebook.drmaciver.com": 0.8278936902761703,+ "www.getpostman.com": 0.8278936902761703,+ "newcriterion.com": 0.82785314452846,+ "www.nationalaffairs.com": 0.8278141715678232,+ "roarmag.org": 0.8278111419376111,+ "blog.ezliu.com": 0.82779905891824,+ "engineering.heroku.com": 0.82779905891824,+ "sulami.github.io": 0.8277548514011955,+ "techdirt.com": 0.8277129992323573,+ "www.automobilemag.com": 0.827708703374778,+ "lihautan.com": 0.8275741663396405,+ "aschmann.net": 0.8275741663396404,+ "blog.apps.npr.org": 0.8275741663396404,+ "ionicframework.com": 0.8275048014166265,+ "google-engtools.blogspot.com": 0.8274021352313168,+ "blog.davidsingleton.org": 0.8274021352313168,+ "blog.jessitron.com": 0.8273903856669967,+ "www.orau.org": 0.8272273741633687,+ "digesttime.com": 0.8272248458013906,+ "kswizz.com": 0.8272248458013906,+ "wmpoweruser.com": 0.8272247723472747,+ "nimbleindustries.io": 0.8272187229694837,+ "jolieodell.wordpress.com": 0.8272099836896732,+ "www.bookforum.com": 0.8271999089538162,+ "genode.org": 0.8271844999040995,+ "decorrespondent.nl": 0.82714602901724,+ "theorangeduck.com": 0.8271367134087778,+ "blog.redfin.com": 0.8271155479281936,+ "threema.ch": 0.8270778121654588,+ "blog.gaborcselle.com": 0.8270675104289051,+ "www.eastbayexpress.com": 0.8270623051198946,+ "www.nilsjuenemann.de": 0.8270167583183482,+ "www.ausgamers.com": 0.8269871316556712,+ "www.nickkolenda.com": 0.8269846287715326,+ "freakonomics.blogs.nytimes.com": 0.8269510780284808,+ "datadome.co": 0.8269501507198006,+ "www.matusiak.eu": 0.8269501507198006,+ "marksaroufim.substack.com": 0.8269362550900476,+ "kernel.org": 0.8269362550900476,+ "foaas.com": 0.8268624447770888,+ "www.quiverquant.com": 0.8268223600931086,+ "www.pennmedicine.org": 0.8267857142857142,+ "panamapapers.icij.org": 0.8267786682199767,+ "rare-technologies.com": 0.8267627529251013,+ "www.wusa9.com": 0.826717577802525,+ "rustwasm.github.io": 0.8266828193880259,+ "exupero.org": 0.8266797238058068,+ "nodesource.com": 0.8266756851479922,+ "tryapl.org": 0.8266364460873337,+ "insights.sei.cmu.edu": 0.8266146306841327,+ "zeynep.substack.com": 0.8265133053820785,+ "adamcaudill.com": 0.8265133053820785,+ "flexport.engineering": 0.8265065205596224,+ "theneurosphere.com": 0.8264888595136076,+ "news.rice.edu": 0.8264758779932855,+ "blog.ninlabs.com": 0.8264758497316637,+ "blog.snapchat.com": 0.8264601098042835,+ "twistedoakstudios.com": 0.8264580233843033,+ "qvault.io": 0.8264470673072151,+ "zachaysan.tumblr.com": 0.826417360607573,+ "investigations.nbcnews.com": 0.8264128301922102,+ "ourmachinery.com": 0.8263907059824316,+ "www.fs.blog": 0.8263508838430117,+ "www.linuxatemyram.com": 0.8263506013533878,+ "blog.kraih.com": 0.8263506013533877,+ "blog.nyaruka.com": 0.8263442628141022,+ "jeffdechambeau.com": 0.8263442628141021,+ "opensoul.org": 0.8263238944048071,+ "www.cortes.design": 0.8263165312463789,+ "www.pawfal.org": 0.8263046319865304,+ "blog.typesafe.com": 0.8262863127448572,+ "blog.andrewcantino.com": 0.8262623863759415,+ "www.boeing.com": 0.8262623863759415,+ "www.ted.com": 0.8262579048280261,+ "blog.custora.com": 0.8262292214279253,+ "www.naftaliharris.com": 0.8261791079056829,+ "javascriptkicks.com": 0.8261791079056829,+ "the-toast.net": 0.8261672942679482,+ "www.cs.rit.edu": 0.8261648745519713,+ "jliszka.github.io": 0.8261587974287634,+ "m.economictimes.com": 0.8261099137432224,+ "vessenes.com": 0.8260709719869568,+ "www.ineteconomics.org": 0.826029111542426,+ "blogs.unity3d.com": 0.8260137334026378,+ "0fps.net": 0.8260060814624561,+ "justcramer.com": 0.8260060814624561,+ "www.nest.com": 0.8260056123053906,+ "qualiacomputing.com": 0.8259333270861942,+ "improbable.io": 0.8259007499406081,+ "blog.dilbert.com": 0.8258693250944908,+ "standpointmag.co.uk": 0.8258674439632754,+ "nelsonslog.wordpress.com": 0.8258527827648116,+ "www.pixelmator.com": 0.8258442158455263,+ "www.c2.com": 0.8258100134548192,+ "www.heraldscotland.com": 0.8257871536416392,+ "yuzu-emu.org": 0.8257634254559968,+ "www.physicscentral.com": 0.8257504892739511,+ "www.overclock3d.net": 0.8257367279398189,+ "www.crainsnewyork.com": 0.8257322176521333,+ "www.dtc.umn.edu": 0.8256884706720989,+ "livefreeordichotomize.com": 0.8256884706720989,+ "graphics.wsj.com": 0.8256623456620759,+ "lostechies.com": 0.8256485195117381,+ "weblog.bocoup.com": 0.8256267961489369,+ "www.rechargenews.com": 0.8256135856550654,+ "weatherspark.com": 0.8256135856550654,+ "wiki.apache.org": 0.8256007740509997,+ "ceylon-lang.org": 0.8256007740509996,+ "androidrookies.com": 0.8255966724327264,+ "nwn.blogs.com": 0.8255570752578437,+ "nathany.com": 0.8255395683453237,+ "www.deepdotweb.com": 0.8255335258309922,+ "jonathanischwartz.wordpress.com": 0.8254547893724321,+ "www.pitheringabout.com": 0.8254520771980598,+ "steve.yegge.googlepages.com": 0.8254511675613948,+ "www.useronboard.com": 0.8253939552044646,+ "www.kjmagnetics.com": 0.8253701708120639,+ "ftp.openbsd.org": 0.8253198620399047,+ "www.bogleheads.org": 0.8253198620399047,+ "v5.chriskrycho.com": 0.8253173977170429,+ "jordanmechner.com": 0.8253173977170429,+ "fare.tunes.org": 0.8253125699065577,+ "www.dangermouse.net": 0.8252739167424069,+ "pdf.yt": 0.82525521399989,+ "robbreport.com": 0.8252506344809964,+ "insideevs.com": 0.8252342355806511,+ "www.logolounge.com": 0.8252252252252251,+ "ftp.arl.army.mil": 0.8252200375406061,+ "www.linux.com": 0.8252089691299165,+ "www.csoonline.com": 0.8251776577952398,+ "catapult.co": 0.8251768351081376,+ "blog.webkid.io": 0.8251768351081376,+ "www.uwgb.edu": 0.8251526412738834,+ "girinstud.io": 0.8251526412738834,+ "adilmoujahid.com": 0.8251526412738834,+ "www.rntz.net": 0.8251450579360111,+ "www.smore.com": 0.8251318689390142,+ "news.rthk.hk": 0.8251206340702621,+ "www.c-span.org": 0.825114692730766,+ "m.stopa.io": 0.8250423585067803,+ "www.opb.org": 0.8250415548188467,+ "artofmanliness.com": 0.825002300673692,+ "opendylan.org": 0.825001402919603,+ "tailwindcss.com": 0.8249955991764671,+ "youtube-creators.googleblog.com": 0.8249955991764671,+ "www.plosone.org": 0.824994642431906,+ "la.curbed.com": 0.8249814854575737,+ "gfycat.com": 0.8249792022510777,+ "pressfreedomfoundation.org": 0.8249170036751421,+ "www.tested.com": 0.8249099846980117,+ "blog.gruntwork.io": 0.8248984430563132,+ "www.innoq.com": 0.8248966445207588,+ "granolashotgun.com": 0.8248948672452628,+ "articles.moneycentral.msn.com": 0.8248521762231498,+ "www.bu.edu": 0.8247468947911291,+ "mozillalabs.com": 0.8247395166074101,+ "thinkgrowth.org": 0.8247377515142629,+ "soledadpenades.com": 0.8247300709391115,+ "snips.ai": 0.8247300709391115,+ "insightdataengineering.com": 0.8247228989993084,+ "altdevblogaday.com": 0.8247202423028511,+ "bugs.webkit.org": 0.8246896410441447,+ "www.serpentine.com": 0.8246896410441446,+ "crimereads.com": 0.8246832358182735,+ "www.chadfowler.com": 0.8246492171080979,+ "flow.microsoft.com": 0.8245931283905966,+ "muut.com": 0.8245391490982966,+ "deque.blog": 0.8245391490982966,+ "www.nearform.com": 0.8245170505651781,+ "www.us-cert.gov": 0.82449055422799,+ "regexcrossword.com": 0.8244429611663081,+ "chris-lamb.co.uk": 0.824442961166308,+ "unretrofied.com": 0.8244331063039999,+ "www.rsaconference.com": 0.8244327243272294,+ "www.nhregister.com": 0.8244082580729432,+ "cryoshon.co": 0.8244082580729432,+ "www.dpmms.cam.ac.uk": 0.8243716227838693,+ "www.patreon.com": 0.8243522520227725,+ "salon.thefamily.co": 0.8243182801287786,+ "www.leakedsource.com": 0.8242753623188406,+ "www.clicktorelease.com": 0.824270209428051,+ "www.chipworks.com": 0.824267686738341,+ "www.hardkernel.com": 0.824267686738341,+ "news.slashdot.org": 0.8242672968534664,+ "developer.atlassian.com": 0.8242608280205023,+ "www.top500.org": 0.8242567265672777,+ "technicshistory.com": 0.8242340010424594,+ "segment.io": 0.8242217790014779,+ "mitsloan.mit.edu": 0.8242217790014779,+ "thebuild.com": 0.8242049759936478,+ "thistooshallgrow.com": 0.8241685536402895,+ "alliance.nautil.us": 0.8241562516385773,+ "www.slashgeek.net": 0.8241560987787111,+ "www.zeroequalsfalse.press": 0.8241315599752687,+ "poincare101.blogspot.com": 0.8240928369315027,+ "blog.zarfhome.com": 0.8240852612726679,+ "www.ryanpickren.com": 0.8240852612726679,+ "design.canonical.com": 0.8240707368381324,+ "www.theengineer.co.uk": 0.8240609124764349,+ "the-magazine.org": 0.8239745714397421,+ "dennybritz.com": 0.8239663447195944,+ "ben.casnocha.com": 0.8239663447195944,+ "www.alexirpan.com": 0.8239445529818444,+ "www.siliconvalleywatcher.com": 0.8239203009772859,+ "chill.com": 0.8239075286809479,+ "kanoki.org": 0.8238968676022924,+ "knowablemagazine.org": 0.8238968676022923,+ "www.fun-mooc.fr": 0.8238950612054784,+ "www.seas.upenn.edu": 0.823854197776325,+ "longbets.org": 0.8238504937824159,+ "papers.nips.cc": 0.8238165389819486,+ "dyske.com": 0.8237859610530819,+ "i.gizmodo.com": 0.8237724868134527,+ "www.kth.se": 0.8237610739927905,+ "www.tawheedkader.com": 0.8237541123759499,+ "www.gov.uk": 0.8237306525116473,+ "chsasank.github.io": 0.8237179544778666,+ "plasmasturm.org": 0.8236804766955088,+ "blog.backtype.com": 0.8236678521506691,+ "feaneron.com": 0.8236399179105635,+ "tech.backtype.com": 0.8236399179105635,+ "www.annashipman.co.uk": 0.8236363636363635,+ "sive.rs": 0.8235995425409803,+ "keen.io": 0.8235778391019275,+ "wensing.tumblr.com": 0.8234963171167604,+ "www1.informatik.uni-erlangen.de": 0.8234356896394766,+ "www.ewanleith.com": 0.8233853975881945,+ "tech.puredanger.com": 0.8233619779868596,+ "freshman.tech": 0.823356964994172,+ "weakdh.org": 0.8233151183970855,+ "boundary.com": 0.8233084145427331,+ "taitems.tumblr.com": 0.8232972027089648,+ "www2.eecs.berkeley.edu": 0.823294055901832,+ "arvindn.livejournal.com": 0.8232708263906926,+ "pcmonk.me": 0.8232633368501505,+ "www.wfla.com": 0.8232633368501505,+ "blog.adamnash.com": 0.8232369516538023,+ "english.martinvarsavsky.net": 0.8232086570141259,+ "www.usv.com": 0.8231252352764534,+ "federalnewsnetwork.com": 0.8231197830462293,+ "www.theatlas.com": 0.8231091015701053,+ "www.riknieu.com": 0.8231091015701053,+ "www.iterm2.com": 0.8231091015701053,+ "www.itwire.com": 0.8231086108233077,+ "www.standalone-sysadmin.com": 0.8231078625216017,+ "www.autoblog.com": 0.8230993668082046,+ "pv-magazine-usa.com": 0.8230980746736871,+ "blog.andrewvc.com": 0.8230893291673793,+ "www.mappingthejourney.com": 0.8230893291673792,+ "blogs.s-osg.org": 0.8230893291673792,+ "www.erica.biz": 0.8230405980636032,+ "digitallife.today.msnbc.msn.com": 0.822992700729927,+ "humantransit.org": 0.822978047436078,+ "anaxi.com": 0.8229457720254757,+ "bedrocklinux.org": 0.8229141632870333,+ "sina.is": 0.8229141632870332,+ "www.joachim-breitner.de": 0.8229052530564972,+ "o.canada.com": 0.8228104673631248,+ "michaelrbernste.in": 0.8228054773077562,+ "status.digitalocean.com": 0.8227743399626116,+ "www.computerworlduk.com": 0.8227710906704893,+ "tech.newstatesman.com": 0.8227353406126373,+ "abc7chicago.com": 0.8227318903356561,+ "bytes.yingw787.com": 0.8226344289193692,+ "www.mikekarnj.com": 0.822628692450413,+ "blogarithms.github.io": 0.8226216439253611,+ "designinstruct.com": 0.8226216439253611,+ "fsecurify.com": 0.8225830523375955,+ "deepsource.io": 0.8225746395058767,+ "n-gate.com": 0.8225612808865542,+ "www.25hoursaday.com": 0.8225607212300213,+ "global.handelsblatt.com": 0.8225208403926864,+ "pragli.com": 0.8225208403926864,+ "manuel.bernhardt.io": 0.8225011685480498,+ "blog.jooq.org": 0.8224250424416801,+ "www.portfolio.com": 0.8224243429262191,+ "web.nvd.nist.gov": 0.8224221426834478,+ "www.hn-books.com": 0.8223443223443224,+ "felixpalmer.github.io": 0.8223443223443223,+ "weev.livejournal.com": 0.8223443223443223,+ "www.harpersbazaar.com": 0.822307206378181,+ "snapframework.com": 0.822307206378181,+ "www.hussmanfunds.com": 0.822307206378181,+ "benlog.com": 0.8222776118268347,+ "www.fluentcpp.com": 0.8222033505775027,+ "www.toptal.com": 0.8221835594409774,+ "www.openssh.com": 0.8221823092541788,+ "blog.avast.com": 0.8221738853904804,+ "www.timesofisrael.com": 0.822154529331881,+ "www.tomdalling.com": 0.8221220478981254,+ "matloff.wordpress.com": 0.8221023761491743,+ "www.stuff.co.nz": 0.8220743929973073,+ "heather.cs.ucdavis.edu": 0.8220584263105031,+ "theincidentaleconomist.com": 0.8220326740109917,+ "radiapp.com": 0.8220183486238531,+ "blog.ramen.is": 0.8219861931408673,+ "geti2p.net": 0.8219861931408673,+ "bgr.com": 0.8218680813488343,+ "hey.com": 0.821866426984301,+ "33bits.org": 0.8218275173853028,+ "www.pressherald.com": 0.8218104832596405,+ "www.staycaffeinated.com": 0.8217905775466726,+ "www.igorkromin.net": 0.8217742285549731,+ "zackarymorris.tumblr.com": 0.8217716107859097,+ "seattletimes.com": 0.8217659740535453,+ "news.northwestern.edu": 0.8217641342619865,+ "canny.io": 0.8217480223498653,+ "www.modernhealthcare.com": 0.8217164092599815,+ "about.google": 0.8217164092599815,+ "www.ifweassume.com": 0.8216730309542566,+ "mobile.abc.net.au": 0.8216657761831399,+ "blog.readme.io": 0.8216640179430849,+ "www.bschaatsbergen.com": 0.8216640179430849,+ "bendyworks.com": 0.8216400745937131,+ "randywestergren.com": 0.8215693482108614,+ "www.crn.com.au": 0.8215655177172531,+ "yinwang0.wordpress.com": 0.8215439095876336,+ "substack.net": 0.8215269428046106,+ "www.toothycat.net": 0.8214683749692366,+ "vega.github.io": 0.8214578695182713,+ "pixelscommander.com": 0.8214578695182713,+ "www.thinq.co.uk": 0.8214038185099727,+ "www.huffingtonpost.com.au": 0.8213863864440609,+ "www.depesz.com": 0.8213734717251845,+ "www.w2lessons.com": 0.8213669213667488,+ "www.mostlymaths.net": 0.8213616628443653,+ "putanumonit.com": 0.8213391337481347,+ "www.executeprogram.com": 0.8213391337481347,+ "www.computing.co.uk": 0.8213046552380585,+ "eblong.com": 0.8212918616104282,+ "asylum.madhouse-project.org": 0.8212918616104282,+ "www.granolashotgun.com": 0.8212918616104282,+ "www.redditblog.com": 0.8212710455497659,+ "www.causal.app": 0.821219763084809,+ "valerieaurora.org": 0.8212075934194915,+ "www.owasp.org": 0.8211882087756882,+ "petition.parliament.uk": 0.8211640347006609,+ "engineering.columbia.edu": 0.821142906452761,+ "fastml.com": 0.8211297895691131,+ "nerderati.com": 0.8211239168679134,+ "www.nutsvolts.com": 0.8211239168679134,+ "vine.co": 0.821112630700933,+ "www.grobmeier.de": 0.8211091773217756,+ "html.adobe.com": 0.8211013298757308,+ "coursetro.com": 0.8210338632520735,+ "www.rleonardi.com": 0.8210332103321032,+ "make.wordpress.org": 0.8210118109358497,+ "iq.opengenus.org": 0.8209763562067588,+ "tech.slashdot.org": 0.820954477342236,+ "www.stanforddaily.com": 0.8209221578417347,+ "www.mapd.com": 0.8208973498293703,+ "campaignstops.blogs.nytimes.com": 0.8208298386512924,+ "www.theblaze.com": 0.8208160487478949,+ "codingkaiser.blog": 0.8208103239764403,+ "cushychicken.github.io": 0.8208101603109114,+ "techblog.badoo.com": 0.8207887125985329,+ "arcanesentiment.blogspot.com": 0.8207861411585611,+ "www.lawfareblog.com": 0.8207748819367903,+ "taskandpurpose.com": 0.8207638339548695,+ "programmingdesignsystems.com": 0.8207024029574862,+ "www.robots.ox.ac.uk": 0.8206399949682861,+ "us.pycon.org": 0.820630666964755,+ "market.android.com": 0.8206208283828789,+ "blog.kirigin.com": 0.8205932101360961,+ "www.pragprog.com": 0.820579551832526,+ "blog.perfectaudience.com": 0.8205466327449291,+ "mreinhold.org": 0.820546632744929,+ "www.thoughtworks.com": 0.8205459808249582,+ "blog.frankel.ch": 0.8205171785173707,+ "philipnilsson.github.io": 0.8205088545817216,+ "www.deathandtaxesmag.com": 0.8203995498364007,+ "7news.com.au": 0.8203793607701599,+ "keminglabs.com": 0.8203793607701598,+ "research.securitum.com": 0.8203793607701598,+ "blog.nanonets.com": 0.8203793607701598,+ "www.roguelynn.com": 0.8203703703703703,+ "zenhack.net": 0.8203703703703703,+ "corpgov.law.harvard.edu": 0.8203690845488739,+ "thelongandshort.org": 0.8203387274635711,+ "bramcohen.livejournal.com": 0.820330588464305,+ "www.nyu.edu": 0.82033015672031,+ "howtosplitanatom.com": 0.8203248926942794,+ "losangeles.cbslocal.com": 0.8202840702703775,+ "www.freebsdfoundation.org": 0.8202742664017543,+ "pqina.nl": 0.8202180020912505,+ "ramblemaps.com": 0.820186718359544,+ "blog.httpwatch.com": 0.8201707118020548,+ "aroberge.blogspot.com": 0.8201707118020547,+ "joanwestenberg.medium.com": 0.8201707118020546,+ "www.texastribune.org": 0.820127140783887,+ "vagosec.org": 0.8201145209512104,+ "blog.michielborkent.nl": 0.8200731194854671,+ "www.desmogblog.com": 0.8200731194854671,+ "informationarchitects.net": 0.8200687812178955,+ "www.mpg.de": 0.8200224577197879,+ "devcenter.heroku.com": 0.8200215315397114,+ "blog.hackerrank.com": 0.8199539191124647,+ "www.mailboxapp.com": 0.8199115962474446,+ "libregraphicsworld.org": 0.8198809873858414,+ "www.netmarketshare.com": 0.8198634237702345,+ "www.honeycomb.io": 0.8198446827660495,+ "wwwnc.cdc.gov": 0.8197995558341037,+ "davidhauser.com": 0.8197995558341036,+ "www.philanthropy.com": 0.8197755193144188,+ "www.armstrongeconomics.com": 0.8197658321751845,+ "www.techpowerup.com": 0.8197581725949339,+ "fourweekmba.com": 0.8197401870662347,+ "runwayml.com": 0.8197062084117198,+ "cs.uwaterloo.ca": 0.8197032651831856,+ "jeffarchibald.ca": 0.8196469792108286,+ "monograph.io": 0.8196424057593232,+ "jaredforsyth.com": 0.8196424057593232,+ "rework.withgoogle.com": 0.8196231659693813,+ "www.butwhatfor.com": 0.8195815182778377,+ "webglfundamentals.org": 0.8195389645544945,+ "skepchick.org": 0.8195389645544945,+ "t37.net": 0.8195018817046663,+ "blockstack.org": 0.8195018817046663,+ "www.ticalc.org": 0.8195018248001847,+ "konghq.com": 0.8194820842106035,+ "cseweb.ucsd.edu": 0.8194210882358477,+ "www.orlandosentinel.com": 0.8194144516611968,+ "ansuz.sooke.bc.ca": 0.8194061161585031,+ "platform.uno": 0.8193808271817485,+ "svs.gsfc.nasa.gov": 0.8193808271817484,+ "i-kh.net": 0.819366852886406,+ "www.t3x.org": 0.8193537789546194,+ "thinkingagriculture.io": 0.8193537789546194,+ "www.colorado.edu": 0.8192948515552042,+ "philipkiely.com": 0.8192790918819292,+ "theultralinx.com": 0.8192615833622455,+ "the-witness.net": 0.8192531710522758,+ "www.woot.com": 0.8192133344078468,+ "blog.crowdtilt.com": 0.8192133344078468,+ "docs.python-guide.org": 0.8192031083028783,+ "blog.indextank.com": 0.819186830384332,+ "bfi.uchicago.edu": 0.8191137736578458,+ "theirrelevantinvestor.com": 0.8190345082551709,+ "herman.asia": 0.8190298507462687,+ "quartzy.qz.com": 0.8189961869849737,+ "www.peterbe.com": 0.8189793251113663,+ "shakycode.com": 0.8189532189180286,+ "jeffreykegler.github.io": 0.8189309404959616,+ "programmer.97things.oreilly.com": 0.8189307834762161,+ "spectrum.chat": 0.8189034014413787,+ "alugy.com": 0.8189014793607566,+ "www.modern.ie": 0.8188594320310705,+ "ilovetypography.com": 0.8188542303813234,+ "parametric.press": 0.8188376402212918,+ "blog.staffjoy.com": 0.8187965320542665,+ "grahamcluley.com": 0.8187959495636524,+ "benediktmeurer.de": 0.8187751653221643,+ "www.loggly.com": 0.818750224724795,+ "typophile.com": 0.8187110947432705,+ "themorningnews.com": 0.8187110947432705,+ "blog.estimote.com": 0.8187092985024379,+ "www.nola.com": 0.8187092491447127,+ "blog.fanout.io": 0.8187064944602657,+ "theoreticalminimum.com": 0.8186915887850467,+ "snap.stanford.edu": 0.8186423752137325,+ "serhack.me": 0.8186312306298296,+ "digital.nhs.uk": 0.8186026992823302,+ "investor.yahoo.net": 0.8185979779164885,+ "www.gamingonlinux.com": 0.818574619301555,+ "interactions.acm.org": 0.8185585358755773,+ "htmlpreview.github.io": 0.8185193011459522,+ "codeberg.org": 0.8185193011459521,+ "www.arm.com": 0.8184839406823776,+ "www.fedscoop.com": 0.8184720810751717,+ "video.google.com": 0.8184638061300447,+ "eyeondesign.aiga.org": 0.8184578256865039,+ "vihart.com": 0.8184565521589099,+ "history.nasa.gov": 0.8184538842053415,+ "www.webmonkey.com": 0.8184424171038894,+ "www.pacifict.com": 0.8184091497388042,+ "www.anaconda.com": 0.8184074729195554,+ "jamesclear.com": 0.8183928113741262,+ "images.guide": 0.8183520599250935,+ "sandofsky.com": 0.818333099426741,+ "tessrinearson.com": 0.8182949341417263,+ "mysqlserverteam.com": 0.8182817858828835,+ "www.virology.ws": 0.8182801242351887,+ "transparencyreport.google.com": 0.8182801242351887,+ "b.lesseverything.com": 0.8182544712962565,+ "forklog.media": 0.8182524048349719,+ "mega.nz": 0.8182293546514141,+ "postcards.blogs.fortune.cnn.com": 0.8181713824074244,+ "kevinrose.com": 0.8181713824074243,+ "www.sankalpjonna.com": 0.8181713824074243,+ "nick.typepad.com": 0.8181333642385573,+ "mathbabe.org": 0.8180898841225986,+ "hacksoflife.blogspot.com": 0.8180862740536983,+ "nickmchardy.com": 0.8180611629393052,+ "www.spectrumnews.org": 0.8180586243621752,+ "www.fictiv.com": 0.8180206950485711,+ "www.riskbasedsecurity.com": 0.8179622866340774,+ "juliagalef.com": 0.8178989768184225,+ "wheels.blogs.nytimes.com": 0.8178917680568788,+ "medium.learningbyshipping.com": 0.8178382028024892,+ "travel.stackexchange.com": 0.8178270639852792,+ "developer.yahoo.net": 0.8177152660164336,+ "www.lostbookofsales.com": 0.8177143703947889,+ "blog.rainforestqa.com": 0.8176913530816636,+ "www.zeldman.com": 0.8176675642405572,+ "www.shogan.co.uk": 0.8176447363707375,+ "blog.friendfeed.com": 0.8176243006331482,+ "tech.instacart.com": 0.8176187126924949,+ "www.well-typed.com": 0.8176129455942598,+ "madebynathan.com": 0.8175850766542176,+ "www.dam.brown.edu": 0.8175673958400209,+ "www.electrive.com": 0.8175415128966471,+ "open.bekk.no": 0.8174977146526387,+ "launch.is": 0.8174957585928975,+ "mindyourdecisions.com": 0.8174283379047065,+ "avc.blogs.com": 0.8174249012744215,+ "underscore.io": 0.8174038787709598,+ "beakerbrowser.com": 0.8173898378490387,+ "tibleiz.net": 0.8173670622724108,+ "learnlayout.com": 0.8173662532870822,+ "statescoop.com": 0.8173342969093829,+ "pcottle.github.io": 0.8173258003766477,+ "mkaz.blog": 0.8173258003766477,+ "davmac.wordpress.com": 0.817306487680897,+ "khalilstemmler.com": 0.817286266376623,+ "mobile.slate.com": 0.817274377154757,+ "pepijndevos.nl": 0.8172690940621161,+ "en.arguman.org": 0.8172440600460996,+ "dougseven.com": 0.8172346051319319,+ "www.hsj.co.uk": 0.8172346051319318,+ "www.washingtoncitypaper.com": 0.8172227432631624,+ "www.stefanjudis.com": 0.8172227432631624,+ "remotehabits.com": 0.8170840840435508,+ "pawelurbanek.com": 0.8170758370632349,+ "www.linode.com": 0.8170518536342932,+ "mattrogish.com": 0.8170279345181233,+ "www.dataorienteddesign.com": 0.8170279345181233,+ "vtorosyan.github.io": 0.8170279345181232,+ "www.adapteva.com": 0.8170168040129737,+ "waldo.jaquith.org": 0.8170168040129737,+ "www.sentient.ai": 0.8169811320754717,+ "cheeaun.com": 0.8169811320754716,+ "adam.chlipala.net": 0.8169734054322659,+ "www.fastmail.com": 0.8169005980617469,+ "law.duke.edu": 0.8169005980617468,+ "uglyduck.ca": 0.8167837603451541,+ "labs.sentinelone.com": 0.8167337370996636,+ "whoisnnamdi.com": 0.8166875449806325,+ "eagain.net": 0.8166739998002963,+ "news.columbia.edu": 0.8166739998002963,+ "bibwild.wordpress.com": 0.8166660149109494,+ "docs.fedoraproject.org": 0.8166606678373952,+ "branchfree.org": 0.8166606678373952,+ "www.howmanypeopleareinspacerightnow.com": 0.8166606678373952,+ "devhints.io": 0.8166486478801723,+ "http.cat": 0.8166351606805292,+ "www.aspi.org.au": 0.8166351606805292,+ "lalo.li": 0.8166351606805292,+ "forum.linode.com": 0.8166342281215091,+ "www.cyanogenmod.com": 0.8166342281215091,+ "www.math.brown.edu": 0.8166342281215091,+ "embracethered.com": 0.8166342281215091,+ "insights.stackoverflow.com": 0.8166112476989804,+ "codemanship.co.uk": 0.816610401586209,+ "bombthrower.com": 0.8165653679484848,+ "open.dgraph.io": 0.8165613796925617,+ "dlewis.net": 0.8165199873064167,+ "www.infoworld.com": 0.8165194544092945,+ "simonsfoundation.org": 0.8164711754414633,+ "blog.evanweaver.com": 0.8164587254389714,+ "webdocs.cs.ualberta.ca": 0.8164587254389714,+ "runnable.com": 0.8164587254389714,+ "scienceblog.com": 0.816442930863004,+ "theinvisiblethings.blogspot.com": 0.8164118858386505,+ "caterina.net": 0.8163684777302764,+ "www.cockeyed.com": 0.8163493619189814,+ "garry.tv": 0.8163458866047911,+ "tech.mattmillman.com": 0.8163458866047911,+ "pavelfatin.com": 0.8163391912303228,+ "blog.testmunk.com": 0.8163326429746816,+ "righteousruminations.blogspot.com": 0.8163302805339576,+ "playnice.ly": 0.8163302805339576,+ "thedriven.io": 0.8162915628964128,+ "liorpachter.wordpress.com": 0.8162914041012925,+ "rr-project.org": 0.8162878787878787,+ "vis4.net": 0.8162878787878787,+ "purelyfunctional.tv": 0.8162705368577864,+ "cachestocaches.com": 0.816247477572823,+ "www.novelr.com": 0.8162474775728229,+ "www.myasmine.com": 0.8162289080621473,+ "www.hanshq.net": 0.8162230743510456,+ "urbankchoze.blogspot.com": 0.8160645118710772,+ "powazek.com": 0.8160565844295424,+ "www.mjt.me.uk": 0.8160235722456507,+ "www.loc.gov": 0.8160198829440672,+ "www.businessinsider.fr": 0.8160110236421557,+ "benmetcalfe.com": 0.816002952282756,+ "cloudant.com": 0.8159956502678918,+ "buddycloud.com": 0.8159660889619967,+ "devinhelton.com": 0.8159603862822139,+ "blog.onedrive.com": 0.8159603862822139,+ "hackoftheday.securitytube.net": 0.8159603862822139,+ "internetintel.oracle.com": 0.8159603862822139,+ "news.rpi.edu": 0.8159324993062729,+ "kyso.io": 0.8157856217718672,+ "blog.cryptoaustralia.org.au": 0.8157807107640108,+ "www.pixelmonkey.org": 0.8157110214426208,+ "gbracha.blogspot.com": 0.8156837220936173,+ "pandoralive.info": 0.8156756405717832,+ "www.lessmilk.com": 0.8156734149395104,+ "blog.everyblock.com": 0.815665920751951,+ "moyix.blogspot.com": 0.815665920751951,+ "blog.keep.network": 0.815665920751951,+ "www.dpreview.com": 0.8156119793953887,+ "blog.calbucci.com": 0.8156110920088071,+ "kowainik.github.io": 0.8156055311358869,+ "blogs.discovery.com": 0.8156055311358867,+ "flic.kr": 0.8155522718024564,+ "ledgersmbdev.blogspot.com": 0.8155503540881356,+ "www.bostonmagazine.com": 0.8155484449427585,+ "www.haydenplanetarium.org": 0.815539827164826,+ "lukesmith.xyz": 0.815539827164826,+ "blog.djy.io": 0.815539827164826,+ "www.treasury.gov": 0.8154535511908948,+ "abovethecrowd.com": 0.815439760100364,+ "yogthos.net": 0.8154331327745159,+ "www.adn.com": 0.8153813333062851,+ "www.eveonline.com": 0.8153685129167869,+ "blog.metamorphium.com": 0.8153509023913048,+ "dannyvankooten.com": 0.8152380952380952,+ "www.creativereview.co.uk": 0.8152232350076929,+ "blog.hasura.io": 0.8152023189276763,+ "artsbeat.blogs.nytimes.com": 0.8151914351876436,+ "dosync.posterous.com": 0.8151891543888505,+ "openreview.net": 0.8151568208322646,+ "blog.serverdensity.com": 0.8151341964704665,+ "manybutfinite.com": 0.8151251257545465,+ "mars.nasa.gov": 0.8150920162802555,+ "www.rbth.com": 0.8150730163714182,+ "www.fefe.de": 0.8150624797096998,+ "mcla.ug": 0.8150272592774294,+ "chem.tufts.edu": 0.8150272592774294,+ "code.jquery.com": 0.8149967373525299,+ "tantek.com": 0.8149534354896242,+ "www.grahamcluley.com": 0.8149532812632393,+ "query.nytimes.com": 0.814942807315685,+ "greig.cc": 0.8149045173924782,+ "www.xaprb.com": 0.8149019506115348,+ "badssl.com": 0.814885496183206,+ "bbirnbaum.com": 0.8148706344450946,+ "officialandroid.blogspot.com": 0.8147398134191081,+ "ruudvanasseldonk.com": 0.8147309156410547,+ "www.softwareatscale.dev": 0.8146603203498196,+ "heap.engineering": 0.8146603203498196,+ "videlalvaro.github.io": 0.8146574608017414,+ "norswap.com": 0.8146006935103474,+ "reactionwheel.net": 0.8145996500684587,+ "bitslog.wordpress.com": 0.8145723063065177,+ "rasterize.io": 0.8145327073791039,+ "adlervermillion.com": 0.8145327073791038,+ "timeless.judofyr.net": 0.81453154875717,+ "haskell.org": 0.8145294619307533,+ "githut.info": 0.8145279229959786,+ "www.gadgette.com": 0.8145279229959786,+ "frantic.im": 0.8145279229959786,+ "www.creativeapplications.net": 0.8145169207397831,+ "googlecommerce.blogspot.com": 0.8144066006321738,+ "mikeash.com": 0.8144025820168224,+ "wecode.wepay.com": 0.8143827137547628,+ "www.cs.ucdavis.edu": 0.8143765574898738,+ "data.heapanalytics.com": 0.8143226776111164,+ "blog.mixergy.com": 0.8143177092759811,+ "www.sudo.ws": 0.8142855202937566,+ "simplystatistics.org": 0.814273938401625,+ "pbpython.com": 0.81426991310501,+ "www.andrew.cmu.edu": 0.814256651162313,+ "www.techinasia.com": 0.8142474634536047,+ "www.scirra.com": 0.8142144867968754,+ "mrmacintosh.com": 0.8142070774434265,+ "omarabid.com": 0.8142001608362681,+ "db.cs.cmu.edu": 0.8141839403388533,+ "reactkungfu.com": 0.8141839403388533,+ "paulbridger.com": 0.8141723199425726,+ "openbazaar.org": 0.8141723199425726,+ "www.globalpost.com": 0.8141091737204239,+ "david-smith.org": 0.8140528179043202,+ "www.philipotoole.com": 0.8140231382115423,+ "20bits.com": 0.8139880326617631,+ "www.kickscondor.com": 0.813983802425237,+ "blog.zdsmith.com": 0.813920131333506,+ "www.iclarified.com": 0.8139139706368135,+ "www.x-plane.com": 0.8138853765840854,+ "mirage.io": 0.8138853765840854,+ "blog.expensify.com": 0.8138812059202145,+ "www.iso.org": 0.8138437732374225,+ "cryptoservices.github.io": 0.8138386793877036,+ "hedgehogreview.com": 0.8138372159261252,+ "www.simple.com": 0.8138043652042445,+ "ftp.cs.ucla.edu": 0.8137212614300459,+ "blog.laptopmag.com": 0.8137088732053118,+ "cliffmass.blogspot.com": 0.8136763300597577,+ "kubuszok.com": 0.8136519481838238,+ "whatblag.com": 0.813578419801053,+ "insight.kellogg.northwestern.edu": 0.8135739399890168,+ "skerritt.blog": 0.8135660771798076,+ "www.shacknews.com": 0.8135400926338027,+ "www.hcn.org": 0.8134966474909895,+ "www.desmoinesregister.com": 0.8134966474909894,+ "www.intelligentliving.co": 0.81349213300375,+ "www.huffpost.com": 0.8134799974360535,+ "nextbigfuture.com": 0.8134623168672729,+ "www.arneswinnen.net": 0.8134615384615383,+ "www.bellingcat.com": 0.8134267051774864,+ "www.zverovich.net": 0.8133918753159222,+ "blog.delicious.com": 0.8133799785061123,+ "hackthesystem.com": 0.8133799785061122,+ "blog.comae.io": 0.8133732585740193,+ "altdevblogaday.org": 0.8133173831188817,+ "w2.eff.org": 0.8132979261023464,+ "news.fastcompany.com": 0.8132816312546639,+ "blog.dominodatalab.com": 0.8132112268692228,+ "www.burlingtonfreepress.com": 0.8131021194605009,+ "betterwebtype.com": 0.8130859124005918,+ "tech.yipit.com": 0.8130859124005917,+ "www.capitolhillseattle.com": 0.8130859124005917,+ "flyosity.com": 0.8130320794442989,+ "juffalow.com": 0.8130140383642517,+ "www.alexstjohn.com": 0.8130023652493408,+ "awardwinningfjords.com": 0.8129597143337574,+ "humanbioscience.org": 0.8129551673058468,+ "programmersatwork.wordpress.com": 0.8129191502536695,+ "www.gao.gov": 0.8129073172130272,+ "www.sci-news.com": 0.8129054883394705,+ "www.teslarati.com": 0.8128722926988359,+ "www.creativedeletion.com": 0.8128708612164178,+ "hushkit.net": 0.8128656957185499,+ "www.sketch.com": 0.8127222898502466,+ "0xfe.blogspot.com": 0.8127222898502466,+ "www.governmentattic.org": 0.8127166548773207,+ "www.iflscience.com": 0.8127104066471765,+ "www.bmwblog.com": 0.8127009540598622,+ "thephd.github.io": 0.8126088980839451,+ "papercups.io": 0.8125137829701504,+ "deeplearning4j.org": 0.8125126453331284,+ "www.kaiostech.com": 0.8125018296149806,+ "www.kdab.com": 0.8124430720535263,+ "communities.intel.com": 0.8123620795777623,+ "www.knowablemagazine.org": 0.8123571766847992,+ "sci-hub.tw": 0.8122620383047426,+ "flink.apache.org": 0.8122287873617684,+ "www.mobilecrunch.com": 0.8120912628571249,+ "blog.liangzan.net": 0.8120625991864463,+ "paulcavallaro.com": 0.8120625991864463,+ "www.freebsdnews.net": 0.8120155038759689,+ "www.cs.dartmouth.edu": 0.8120120009628422,+ "slashdot.org": 0.8120103261102206,+ "beta.theglobeandmail.com": 0.8119800311478199,+ "www.franzoni.eu": 0.8119518701628052,+ "www.unz.com": 0.8119446655347015,+ "launch.gust.com": 0.8119242878514453,+ "mattlakeman.org": 0.8119242878514452,+ "us.battle.net": 0.8119135744478586,+ "vitess.io": 0.8119046881511972,+ "www.parsonsmatt.org": 0.8118880860804719,+ "www.sciam.com": 0.8118415645513156,+ "blog.inklingmarkets.com": 0.8118015537952591,+ "darknetdiaries.com": 0.8117495396709252,+ "blog.mixpanel.com": 0.8117419261805633,+ "blog.kippt.com": 0.8117273158415922,+ "danielrapp.github.io": 0.8117273158415922,+ "semantic-domain.blogspot.com": 0.8117250261807765,+ "blog.jayfields.com": 0.8117106131262737,+ "dronegarageblog.wordpress.com": 0.8116970637651796,+ "m.cnet.com": 0.8116871715030578,+ "bionic.fullstory.com": 0.8116504854368931,+ "blog.reybango.com": 0.8116192339099034,+ "baesystemsai.blogspot.com": 0.8116098514216743,+ "tasvideos.org": 0.811592254952584,+ "www.nathancunn.com": 0.8115765934419358,+ "theguardian.com": 0.8115466314959758,+ "www.bls.gov": 0.8114586352368218,+ "www.slashgear.com": 0.8114130074807454,+ "get21stnight.com": 0.8113811919256128,+ "wadefoster.net": 0.8113380585952003,+ "backstage.soundcloud.com": 0.8113341783615344,+ "blog.mongohq.com": 0.8113084373038951,+ "blog.samwhited.com": 0.8113018683008136,+ "lukethomas.com": 0.8112900298646131,+ "johnmurray.io": 0.811284046692607,+ "www.scpr.org": 0.8111931888951548,+ "serverless.com": 0.8111853186864186,+ "www.mintpressnews.com": 0.8111566169738434,+ "dev.hubspot.com": 0.8111491117505176,+ "uxdesign.smashingmagazine.com": 0.8110875951026609,+ "www.dcscience.net": 0.8110637872870197,+ "www.badlogicgames.com": 0.8110531473954321,+ "www.imdb.com": 0.8109585365005275,+ "www.kevinwilliampang.com": 0.810916179337232,+ "benjaminmayo.co.uk": 0.8109161793372319,+ "www.tonymacx86.com": 0.8109161793372319,+ "news.firedoglake.com": 0.8109161793372319,+ "gitorious.org": 0.810892038104006,+ "www.indystar.com": 0.810892038104006,+ "www.nextbigwhat.com": 0.810871453085769,+ "vectorized.io": 0.8108646859647678,+ "www.fakesteve.net": 0.8108603160330469,+ "theness.com": 0.8108582902726974,+ "blog.zenkit.com": 0.8108296140970532,+ "consequenceofsound.net": 0.8108001038237775,+ "dealbook.blogs.nytimes.com": 0.810782053555595,+ "raccoon.onyxbits.de": 0.8107413934848161,+ "www.linaro.org": 0.8107386685587079,+ "insights.hpe.com": 0.8107316248294252,+ "webflow.com": 0.8107174188583165,+ "insidesearch.blogspot.com": 0.8107157266121297,+ "www.hhs.gov": 0.8107124152561004,+ "blog.folyo.me": 0.8107124152561003,+ "blog.playcanvas.com": 0.8107042145283994,+ "oldblog.antirez.com": 0.8106726118909099,+ "computinged.wordpress.com": 0.8106676355919645,+ "www.beekeeperstudio.io": 0.8106491457950028,+ "www.meetup.com": 0.8106205529212165,+ "capiche.com": 0.8105820355692206,+ "www.cbp.gov": 0.810546875,+ "stackexchange.com": 0.8105386561469894,+ "kapeli.com": 0.8105257538297364,+ "blog.sigfpe.com": 0.8105035346007113,+ "nianticlabs.com": 0.8104898882641829,+ "econlog.econlib.org": 0.8104880381821702,+ "bitsofco.de": 0.8104598795589895,+ "blog.gyrosco.pe": 0.8104246070876013,+ "blog.n01se.net": 0.8104076949009623,+ "mozakai.blogspot.com": 0.8104076949009623,+ "iamwil.posterous.com": 0.8103954830327119,+ "www.minyanville.com": 0.8103877873546691,+ "jcjc-dev.com": 0.8103364623358701,+ "decryptmedia.com": 0.8103177363469551,+ "alexvermeer.com": 0.8103141664126979,+ "reddragdiva.dreamwidth.org": 0.8103141664126978,+ "anarchogeek.com": 0.8103141664126978,+ "truthonthemarket.com": 0.810302799232476,+ "numenta.com": 0.8102602204436639,+ "economictimes.indiatimes.com": 0.8102491144464327,+ "www.perceptualedge.com": 0.8102043958285661,+ "www.garykessler.net": 0.8101761252446185,+ "loopj.com": 0.8101761252446184,+ "newteevee.com": 0.8101251805166595,+ "morningconsult.com": 0.8101075221559366,+ "mrmrs.io": 0.810089337360651,+ "www.springboard.com": 0.8100885519221457,+ "www.computinghistory.org.uk": 0.8100885519221456,+ "ownlocal.com": 0.810072359778719,+ "wellcomecollection.org": 0.810072359778719,+ "owasp.org": 0.810072359778719,+ "linuxreviews.org": 0.8100650707969348,+ "monadical.com": 0.8100469597795646,+ "gamesfromwithin.com": 0.810042091205529,+ "www.geeksforgeeks.org": 0.8100336615814749,+ "www.universal-sci.com": 0.810032793083701,+ "esoteric.codes": 0.8099527252346412,+ "www.petermichaud.com": 0.8099337587386388,+ "www.freedom-to-tinker.com": 0.8099164380793796,+ "www.laptopmag.com": 0.809846769190124,+ "www.3news.co.nz": 0.8098352603019351,+ "sel4.systems": 0.8098186580704161,+ "www.accountingtoday.com": 0.8098186580704161,+ "host.madison.com": 0.8097889409717836,+ "www.nokia.com": 0.8097879208159734,+ "insanecoding.blogspot.com": 0.8097776259075163,+ "kdenlive.org": 0.8097696203436352,+ "www.bitstamp.net": 0.8097578373340156,+ "stonemaiergames.com": 0.809735836328836,+ "ceronman.com": 0.809735818851372,+ "airmail.news": 0.809735818851372,+ "nickwignall.com": 0.8097127006730107,+ "www.develop-online.net": 0.8096968183358267,+ "brunocalza.me": 0.809690221614063,+ "gadgetopia.com": 0.8096902216140629,+ "seekingalpha.com": 0.8096704138265437,+ "www.jofreeman.com": 0.8095931882025065,+ "muckandbrass.com": 0.8095931882025065,+ "nobelprize.org": 0.8095392106610299,+ "www.vg247.com": 0.8095156620373399,+ "chetansurpur.com": 0.8094671848396732,+ "haxiomic.github.io": 0.8094631273918713,+ "deislabs.io": 0.8094631273918712,+ "jmc.stanford.edu": 0.8094565130568238,+ "philpapers.org": 0.8094356549031609,+ "youtube-global.blogspot.com": 0.8094272365570714,+ "rubygems.org": 0.8093966911744022,+ "www.laquadrature.net": 0.8093644943397358,+ "dheera.net": 0.80935652930136,+ "use.perl.org": 0.8093278422294106,+ "dannorth.net": 0.8093207850460444,+ "orbitalindex.com": 0.8093095444635583,+ "historyofyesterday.com": 0.8092882173909588,+ "www.citymetric.com": 0.809287154904464,+ "www.codecommit.com": 0.8092420697380182,+ "danlec.com": 0.8092306399056751,+ "endlessparentheses.com": 0.8092053968063221,+ "www.vrfinal.com": 0.809205396806322,+ "www.the-american-interest.com": 0.8091888493515856,+ "cycle-gap.blogspot.com": 0.8091399291892002,+ "elinux.org": 0.8091062649443477,+ "agtb.wordpress.com": 0.8091062649443477,+ "addyosmani.github.com": 0.8090929406134104,+ "www.immadsnewworld.com": 0.8090591990898237,+ "interplayoflight.wordpress.com": 0.8090551181102362,+ "kylecordes.com": 0.8090551181102361,+ "www.ndss-symposium.org": 0.8090548860707052,+ "www.grantland.com": 0.808953355501278,+ "accu.org": 0.8089301368051649,+ "americans.org": 0.8089046493675752,+ "languagengine.co": 0.8088694832763874,+ "www.al-monitor.com": 0.8088667083388086,+ "www.marksimonson.com": 0.8088667083388086,+ "danilocampos.com": 0.8088667083388086,+ "money.yahoo.com": 0.8088648827181377,+ "www.mathpages.com": 0.8088432529287259,+ "nakamotoinstitute.org": 0.8088145420012367,+ "blog.ionelmc.ro": 0.8087651491972426,+ "www.strawpoll.me": 0.8087480632308696,+ "www.theweek.co.uk": 0.8087282943708363,+ "www.morningbrew.com": 0.8086875731228074,+ "ksj.mit.edu": 0.8086785009861932,+ "ozar.me": 0.808655053694382,+ "scheme2006.cs.uchicago.edu": 0.8085933953841323,+ "9fans.net": 0.8085933953841322,+ "pythonhosted.org": 0.8085610188346061,+ "lite.cnn.com": 0.8085535287872542,+ "www.fark.com": 0.808501385570164,+ "www.techendo.co": 0.808501106941333,+ "downlode.org": 0.8084934128064108,+ "upstart.me": 0.8084934128064108,+ "www.extendslogic.com": 0.8084934128064108,+ "theundefeated.com": 0.8084934128064108,+ "blog.appsignal.com": 0.8084690154513154,+ "www.informationarchitects.jp": 0.8084214332325731,+ "www.intrade.com": 0.8083885146980836,+ "lukasz.langa.pl": 0.8083885146980836,+ "www.energy.gov": 0.8083804906443909,+ "talkpython.fm": 0.8083525540804629,+ "yourbasic.org": 0.8083074906530393,+ "www.sachinagarwal.com": 0.8083003952569169,+ "openshift.redhat.com": 0.8082657786610448,+ "tech.adroll.com": 0.8082443572105479,+ "paleofuture.gizmodo.com": 0.8082365397238809,+ "tgceec.tumblr.com": 0.808209532747702,+ "bowery.io": 0.8081978171238972,+ "blog.sourcing.io": 0.8081838340004474,+ "www.9to5mac.com": 0.8081802724018405,+ "bostinno.com": 0.8081755973845505,+ "jatan.space": 0.8081563211953493,+ "www-inst.eecs.berkeley.edu": 0.8081204961840847,+ "evrone.com": 0.8080872755977898,+ "labs.domipheus.com": 0.8080351711070707,+ "www.thrillist.com": 0.8079971777160608,+ "www.gizmag.com": 0.8079969849430149,+ "landley.net": 0.807958662672192,+ "danweinreb.org": 0.8078992257179908,+ "tyk.io": 0.807867976487508,+ "mymicrocontroller.com": 0.8078630033918272,+ "www.samharris.org": 0.8078422754090425,+ "epetitions.direct.gov.uk": 0.8078284811604556,+ "blog.tildeloop.com": 0.8078284811604556,+ "discuss.emberjs.com": 0.8078241273575438,+ "blog.superuser.com": 0.8077665343961988,+ "oliveremberton.com": 0.807743372835657,+ "vpnpro.com": 0.8077402589309914,+ "www.lastwordonnothing.com": 0.8076974958304752,+ "stevenf.com": 0.8076913832175415,+ "www.timeanddate.com": 0.8076143611045982,+ "cornellsun.com": 0.8075830492659702,+ "letoverlambda.com": 0.8075830492659702,+ "snapcraft.io": 0.8075475712589127,+ "www.observer.com": 0.8074847637579465,+ "unit42.paloaltonetworks.com": 0.8074697991147913,+ "www.fastcocreate.com": 0.8074665167071722,+ "mixmax.com": 0.8074624950006974,+ "blog.goodaudience.com": 0.8074490566695363,+ "wikitravel.org": 0.8074371698943806,+ "elzr.com": 0.8074371698943806,+ "gerger.co": 0.8074101861989343,+ "developer.amazonwebservices.com": 0.8073933592944413,+ "blog.postmarkapp.com": 0.8073171397946091,+ "adereth.github.io": 0.8073171397946091,+ "www.gamepro.com": 0.8073017117935154,+ "www.sencha.com": 0.8072902381119292,+ "christianheilmann.com": 0.8072902381119292,+ "mg.co.za": 0.8072653978420726,+ "python-history.blogspot.com": 0.8072408370446265,+ "www.warhistoryonline.com": 0.807215873070334,+ "theatlantic.com": 0.8072127719392547,+ "sakurity.com": 0.8071790086703337,+ "blog.yalabot.com": 0.8071726013094649,+ "softwareengineering.stackexchange.com": 0.8071626259596408,+ "vnhacker.blogspot.com": 0.8071372108129627,+ "www.securityweek.com": 0.8070728244513765,+ "www.myce.com": 0.8070371527879234,+ "progressiveapp.store": 0.8070260217041482,+ "teamblog.supportbee.com": 0.8070260217041482,+ "arogozhnikov.github.io": 0.8070036402591226,+ "mechanical-sympathy.blogspot.com": 0.8069161416177959,+ "strangelyconsistent.org": 0.8069045196582395,+ "www.themarysue.com": 0.8068976247535697,+ "www.attendly.com": 0.8068580186201768,+ "www.snell-pym.org.uk": 0.8068326941714653,+ "tools.cisco.com": 0.8068196782315848,+ "influxdb.com": 0.8068173283593989,+ "worksinprogress.co": 0.8067896834805538,+ "www.scripps.edu": 0.8067572864866587,+ "dissenter.firedoglake.com": 0.806652591404526,+ "crackstation.net": 0.8066141922329747,+ "www.afar.com": 0.8065122291007617,+ "hasura.io": 0.8064237491553132,+ "www.cs.nyu.edu": 0.8063872255489021,+ "www.mtholyoke.edu": 0.8063872255489021,+ "blog.startifact.com": 0.8063862184736535,+ "www.colourlovers.com": 0.8063834547609555,+ "www.gettingmoreawesome.com": 0.8063764986530164,+ "www.themorningnews.org": 0.8063013263733552,+ "hardcoresoftware.learningbyshipping.com": 0.8062797912662047,+ "bramlambrecht.com": 0.8062668907413534,+ "www.spacesafetymagazine.com": 0.8062339137045579,+ "newscenter.berkeley.edu": 0.8062219609407214,+ "suckless.org": 0.8062207668887819,+ "www.staticgen.com": 0.8062025100696604,+ "www.eecs.umich.edu": 0.8062025100696603,+ "www.thefader.com": 0.8061953047624655,+ "www.independent.ie": 0.806109251382116,+ "pivotallabs.com": 0.8060806747033802,+ "www.opsian.com": 0.8060621471224426,+ "www.taigeair.com": 0.8060621471224426,+ "talkingpointsmemo.com": 0.8060591184167385,+ "williampross.com": 0.8060455947863886,+ "popmotion.io": 0.8060424719294661,+ "robb.weblaws.org": 0.806042471929466,+ "globaldev.co.uk": 0.806011328959103,+ "pencerw.com": 0.806,+ "michelenasti.com": 0.8059842345365967,+ "sourcesort.com": 0.8059044933660855,+ "www.monash.edu": 0.8059044933660855,+ "haydenjames.io": 0.8058768485324959,+ "stjepang.github.io": 0.805873630277718,+ "blog.webb.page": 0.805873630277718,+ "tcfast.com": 0.8058521366516195,+ "blog.figma.com": 0.8058473182210699,+ "blog.fikesfarm.com": 0.8058387400788318,+ "www.telecompaper.com": 0.8058387400788318,+ "mikegerwitz.com": 0.8058387400788318,+ "micheleincalifornia.blogspot.com": 0.8058373166902851,+ "cyber.harvard.edu": 0.8058229699826737,+ "content.time.com": 0.8057897338858303,+ "catless.ncl.ac.uk": 0.8057745286288139,+ "www.techflash.com": 0.8057273188584126,+ "omis.me": 0.805697502622664,+ "anildash.com": 0.8056599652743452,+ "www.authy.com": 0.8056112224448897,+ "bitterwinter.org": 0.8056112224448897,+ "news.illinois.edu": 0.8055525618659666,+ "trilema.com": 0.8055277480254573,+ "nshipster.com": 0.8055021399615895,+ "uppy.io": 0.8054909385894538,+ "probmods.org": 0.805478770001181,+ "labs.umbrella.com": 0.805478770001181,+ "www.lawyersgunsmoneyblog.com": 0.8054692502161738,+ "paulspontifications.blogspot.com": 0.8054688521990506,+ "wiki.python.org": 0.8054642788538674,+ "metaphysicaldeveloper.wordpress.com": 0.8054364779428177,+ "tech.okcupid.com": 0.805426569995148,+ "leejo.github.io": 0.8054207149429934,+ "scholar.google.com": 0.8053870344669396,+ "jseliger.com": 0.8053643358322942,+ "www.openrightsgroup.org": 0.8053456370357672,+ "zendev.com": 0.8053227485814811,+ "deepsense.ai": 0.8053174136210413,+ "googlepublicpolicy.blogspot.com": 0.8052966184857965,+ "reading.supply": 0.8052571979264512,+ "www.fullstackpython.com": 0.8052571979264512,+ "making.duolingo.com": 0.8052499518418211,+ "cloudrail.com": 0.8052345044507019,+ "crt.sh": 0.8052208835341366,+ "charity.wtf": 0.8052128017402501,+ "gettingreal.37signals.com": 0.8052119053726816,+ "products.office.com": 0.805195056791058,+ "blog.spiderlabs.com": 0.8051909890788225,+ "javascript.christmas": 0.8051778217183054,+ "www.ruby-forum.com": 0.8051778217183053,+ "webkul.github.io": 0.8051216600975734,+ "crypt.codemancers.com": 0.8050979962669772,+ "aomedia.org": 0.805084051401519,+ "www.physics-astronomy.org": 0.8050038695306542,+ "dothemath.ucsd.edu": 0.8049264940426683,+ "www.av8n.com": 0.8049169449005759,+ "signalsandthreads.com": 0.8049169449005759,+ "www.apptorium.com": 0.8048961063441398,+ "smarterware.org": 0.8048459862314374,+ "www.playstation.com": 0.8048413390083552,+ "toys.lerdorf.com": 0.8048289738430583,+ "blog.handcraft.com": 0.8047698525990352,+ "www.alex-ionescu.com": 0.8047611301824745,+ "team.adzerk.com": 0.8047527420132489,+ "www.trottercashion.com": 0.8047172511803042,+ "www.dylanpaulus.com": 0.8047172511803041,+ "shiroyasha.io": 0.8046842108009492,+ "sunlightlabs.com": 0.8046689741257063,+ "johnbender.us": 0.8046551925330852,+ "www.orionmagazine.org": 0.8046409236882213,+ "500ish.com": 0.8045980649418749,+ "mbostock.github.com": 0.8045837440928983,+ "www.eclogiselle.com": 0.8045630694400487,+ "j00ru.vexillium.org": 0.8045036919905235,+ "www.partow.net": 0.8045036919905235,+ "www.zpub.com": 0.8045036919905235,+ "www.hq.nasa.gov": 0.8043661879315178,+ "news.usni.org": 0.804355836696792,+ "charemza.name": 0.8043090754458252,+ "gondor.io": 0.8043052663473409,+ "gamelab.mit.edu": 0.8042844920724259,+ "starcraft.com": 0.8042844920724259,+ "sorentwo.com": 0.804261705269134,+ "thebrowser.com": 0.8042503564660758,+ "ericsink.com": 0.804229811533429,+ "blogs.loc.gov": 0.8041942737849643,+ "sslmate.com": 0.8041845438238772,+ "googlereader.blogspot.com": 0.8041760226307137,+ "pingcap.github.io": 0.8041760226307136,+ "blog.nearlyfreespeech.net": 0.8041648746111758,+ "blog.peer5.com": 0.8041599344447704,+ "understandlegacycode.com": 0.8041241866706869,+ "www3.imperial.ac.uk": 0.8040897254741548,+ "ovid.github.io": 0.8040060267347496,+ "perell.com": 0.8039678014067241,+ "marketingland.com": 0.8039660516562604,+ "maierfelix.github.io": 0.803954084248265,+ "www.plainsite.org": 0.803949941652702,+ "www.fabcapo.com": 0.8039204604178055,+ "www.osohq.com": 0.8039193847705445,+ "marc.merlins.org": 0.8038831339196976,+ "satyajit.ranjeev.in": 0.8038831339196976,+ "kirindave.tumblr.com": 0.8038831339196976,+ "transcend.io": 0.8038268126556799,+ "pozorvlak.livejournal.com": 0.8038248807746377,+ "elweb.co": 0.8038124805380112,+ "www.arqbackup.com": 0.8037337901625023,+ "www.turnkeylinux.org": 0.803725088601539,+ "www.box.com": 0.8037180799167778,+ "preciousplastic.com": 0.8036437246963563,+ "www.stonetemple.com": 0.8036218680934559,+ "oeis.org": 0.8036018584338912,+ "www.bath.ac.uk": 0.8036016394623412,+ "services.google.com": 0.803590777077633,+ "theodi.org": 0.8035882804728938,+ "saucelabs.com": 0.8035344497963749,+ "greydanus.github.io": 0.8035295033581054,+ "www.educative.io": 0.8035052925700027,+ "librarianshipwreck.wordpress.com": 0.8034837043428301,+ "sway.com": 0.803480126236027,+ "datacolada.org": 0.803472812615791,+ "convox.com": 0.8033973749313138,+ "www.tokbox.com": 0.8033848854367407,+ "review.firstround.com": 0.8033799488939632,+ "www.netmagazine.com": 0.8033798192718756,+ "www.stanfordalumni.org": 0.8033701218466093,+ "www.fox5ny.com": 0.8033480309814771,+ "phx.corporate-ir.net": 0.8033227542756777,+ "christophjanz.blogspot.com": 0.8033072428565003,+ "grugq.github.io": 0.8032867126150643,+ "gist.githubusercontent.com": 0.8032867126150643,+ "www.welivesecurity.com": 0.8032782303750765,+ "www.linfo.org": 0.8032479251672556,+ "www.vikingcodeschool.com": 0.8032454361054767,+ "talsraviv.com": 0.8032454361054766,+ "wyss.harvard.edu": 0.8032452848408873,+ "brainspl.at": 0.8032452848408873,+ "simblob.blogspot.com": 0.8032046395373721,+ "sefsar.com": 0.8031660842268623,+ "www.anmolsarma.in": 0.803136984148784,+ "www.semiaccurate.com": 0.8031240460985013,+ "www.mikekohn.net": 0.8031074147500573,+ "m.wsj.com": 0.8030996898658181,+ "doc-ok.org": 0.8030754588314308,+ "duncandavidson.com": 0.8030754588314308,+ "www.redbullstratos.com": 0.8030754588314308,+ "lodev.org": 0.8030754588314308,+ "www.sqreen.io": 0.8030754588314308,+ "weburbanist.com": 0.8030719460684711,+ "www.homelandsecuritynewswire.com": 0.8030383963381544,+ "programming.oreilly.com": 0.8029986620998482,+ "www.yuiblog.com": 0.8029870560713339,+ "mgalligan.com": 0.802938284099674,+ "startupsanonymous.com": 0.802896517417948,+ "genomebiology.biomedcentral.com": 0.8028333073906616,+ "www.commentary.org": 0.8028283532567575,+ "paidcontent.org": 0.8027935252684901,+ "www.entrepreneur.com": 0.8027573121471558,+ "opensource.google.com": 0.8027457335537694,+ "gus-massa.blogspot.com": 0.8027457335537693,+ "attractivechaos.wordpress.com": 0.8027428934082615,+ "timothyfitz.wordpress.com": 0.8027428934082615,+ "news.voyage.auto": 0.8027425585809037,+ "blog.safia.rocks": 0.802725147613407,+ "pagely.com": 0.802725147613407,+ "pedestrianobservations.wordpress.com": 0.8027251476134069,+ "www.huckmagazine.com": 0.8027158674008132,+ "amplitude.com": 0.8027111401704623,+ "academicvc.com": 0.8026691214318213,+ "zurb.com": 0.802669002882095,+ "www.fogcreek.com": 0.8026427258093481,+ "pubs.aeaweb.org": 0.8025918806108637,+ "ochronus.online": 0.8025797396414792,+ "mukulrathi.co.uk": 0.8025366605137177,+ "www.tinfoilsecurity.com": 0.8025110355010865,+ "blog.intercom.io": 0.802438081407932,+ "www.counterpointresearch.com": 0.802349177742619,+ "caines.ca": 0.8023472216796856,+ "bitmessage.org": 0.8023472216796855,+ "fare.livejournal.com": 0.802344964463485,+ "sameroom.io": 0.8023231254672252,+ "grapheneos.org": 0.8022611036781359,+ "ryanhoover.me": 0.8022215636421977,+ "ghost.org": 0.8021756677926977,+ "ably.com": 0.8021756677926974,+ "dgraph.io": 0.8021756677926974,+ "www.pcg-random.org": 0.8021607965447277,+ "www.electronicdesign.com": 0.802117143232502,+ "blog.benjamin-encz.de": 0.8021070483963555,+ "blog.apollographql.com": 0.8021070483963553,+ "interaktiv.morgenpost.de": 0.8021068976672623,+ "www.neogaf.com": 0.8020794141131092,+ "jasongullickson.posterous.com": 0.802070055097611,+ "www.lvguowei.me": 0.8020465972489538,+ "theapeiron.co.uk": 0.8020408163265306,+ "insideairbnb.com": 0.8020408163265306,+ "www.business-standard.com": 0.8020316557325065,+ "blog.chiariglione.org": 0.8020183701272139,+ "news.byu.edu": 0.8019177911188085,+ "ico.org.uk": 0.8019103627766435,+ "www.asktog.com": 0.8018750459311643,+ "www.thelightphone.com": 0.8018513951254583,+ "weblog.muledesign.com": 0.8018281461815683,+ "functional.christmas": 0.8017833933313524,+ "dendory.net": 0.8017566800074669,+ "css-irl.info": 0.8016782621735088,+ "m.spiegel.de": 0.8016676297112596,+ "blog.rubybestpractices.com": 0.8016652051765923,+ "buzzsumo.com": 0.8016526731161184,+ "edinburghhacklab.com": 0.8016359918200409,+ "www.uc.edu": 0.8016359918200409,+ "deprogrammaticaipsum.com": 0.8016284628618231,+ "blog.teamtreehouse.com": 0.8016183213966954,+ "www.poynter.org": 0.8015928533864275,+ "blog.thirdyearmba.com": 0.8015659486074044,+ "cmr.github.io": 0.8015566134648262,+ "www.hackingwithswift.com": 0.8015552594719607,+ "www.43folders.com": 0.801514257843383,+ "www.whispersystems.org": 0.801509946546816,+ "www.bravenewgeek.com": 0.8014600688018096,+ "blog.siliconpublishing.com": 0.8014553154641547,+ "psiloveyou.xyz": 0.8014114620819677,+ "social.msdn.microsoft.com": 0.8014114620819677,+ "thenewinquiry.com": 0.8014082764643022,+ "www.grc.com": 0.8014045918631542,+ "yle.fi": 0.8013958725561011,+ "www.buildzoom.com": 0.8013406095826441,+ "www.kmjn.org": 0.8013048425718535,+ "www.miaminewtimes.com": 0.8012606895008024,+ "pipedream.com": 0.8012606895008023,+ "xoofx.com": 0.8012606895008023,+ "9elements.com": 0.8012357442611142,+ "www.safetyresearch.net": 0.8012295081967213,+ "memcached.org": 0.8012295081967213,+ "letterstoanewdeveloper.com": 0.801226878568775,+ "viewfromthewing.boardingarea.com": 0.8012126203272316,+ "nanxiao.me": 0.8011961282535711,+ "ferrucc.io": 0.8011506248520655,+ "blog.mrmeyer.com": 0.8011434789943767,+ "www.dailycal.org": 0.8011077312639568,+ "tldp.org": 0.8011077312639568,+ "www.nbcphiladelphia.com": 0.8010985913944949,+ "ww2.kqed.org": 0.8010518269035773,+ "nitter.net": 0.8010512756640218,+ "www.mathieupassenaud.fr": 0.8010059231728947,+ "uploadvr.com": 0.8009858227111218,+ "lincolnloop.com": 0.800978810280933,+ "gaming.icrontic.com": 0.8009712545655284,+ "www.techvibes.com": 0.8009550432546098,+ "www.news.gatech.edu": 0.8009318982607027,+ "leastauthority.com": 0.8008308575748613,+ "find.xyz": 0.8008120556386431,+ "rhodesmill.org": 0.8007877241325981,+ "authzed.com": 0.8007485330706217,+ "www.agweb.com": 0.8007485330706217,+ "allegro.tech": 0.8007425067937589,+ "nex-3.com": 0.8007357953179356,+ "macaw.co": 0.8007357953179356,+ "nic.ferrier.me.uk": 0.8007109685993726,+ "www.planetscale.com": 0.8007054295824444,+ "arshadchowdhury.com": 0.8006990158865002,+ "www.adbusters.org": 0.8006901908979573,+ "techno-anthropology.blogspot.com": 0.8006290310742641,+ "noteflakes.com": 0.8006120189176902,+ "blog.ellenchisa.com": 0.8006120189176901,+ "ajaxian.com": 0.8005654734164377,+ "getchip.com": 0.8004935213803819,+ "www.dutchnews.nl": 0.8004918240352497,+ "www.efavdb.com": 0.8004115226337449,+ "mabez.dev": 0.8004115226337449,+ "www.azul.com": 0.80040857295631,+ "www.bonappetit.com": 0.8003933326639253,+ "microconf.com": 0.8003448118609695,+ "mxb.at": 0.8003448118609695,+ "rancher.com": 0.8003102115898207,+ "antoniocangiano.com": 0.8002939423538775,+ "www.modernretail.co": 0.8002715044965122,+ "gigamonkeys.wordpress.com": 0.8001954409713545,+ "blog.veitheller.de": 0.8001954409713545,+ "www.thehistoryblog.com": 0.8001885843714048,+ "www.javacodegeeks.com": 0.80016657689339,+ "blog.stevensanderson.com": 0.8001363248010883,+ "neilmitchell.blogspot.com": 0.8001226738254366,+ "www.foundingfuel.com": 0.800121664181143,+ "simpleprogrammer.com": 0.8001130650615119,+ "eclipse.org": 0.8000856208636091,+ "www.acs.org": 0.8000105725594235,+ "sigusr2.net": 0.8000105725594235,+}diff --git a/mwmbl/indexer/batch_cache.py b/mwmbl/indexer/batch_cache.pyindex 01d8cc9..3d460f4 100644--- a/mwmbl/indexer/batch_cache.py+++ b/mwmbl/indexer/batch_cache.py@@ -3,6 +3,7 @@ Store for local batches.We store them in a directory on the local machine."""+import gzipimport jsonimport os@@ -15,10 +16,9 @@ from pydantic import ValidationErrorfrom mwmbl.crawler.batch import HashedBatchfrom mwmbl.database import Database-from mwmbl.indexer.indexdb import IndexDatabase, BatchStatus+from mwmbl.indexer.indexdb import BatchStatus, IndexDatabasefrom mwmbl.retry import retry_requests-logger = getLogger(__name__)@@ -81,10 +81,10 @@ class BatchCache:path = self.get_path_from_url(url)logger.debug(f"Storing local batch at {path}")os.makedirs(path.parent, exist_ok=True)- with open(path, 'wb') as output_file:- data = gzip.compress(batch.json().encode('utf8'))+ with open(path, "wb") as output_file:+ data = gzip.compress(batch.json().encode("utf8"))output_file.write(data)def get_path_from_url(self, url) -> Path:url_path = urlparse(url).path- return Path(self.path) / url_path.lstrip('/')+ return Path(self.path) / url_path.lstrip("/")diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.pyindex 507c2bb..1d0f4f3 100644--- a/mwmbl/indexer/blacklist.py+++ b/mwmbl/indexer/blacklist.py@@ -9,16 +9,15 @@ from typing import Callable, Optional, Setfrom mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer.blacklist_providers import (- BuiltInRulesBlacklistProvider,- HaGeZiBlacklistProvider,AdultContentBlacklistProvider,+ BlacklistProvider,+ BuiltInRulesBlacklistProvider,CombinedBlacklistProvider,- BlacklistProvider+ HaGeZiBlacklistProvider,)-def get_default_blacklist_provider(- get_exempt_domains: Optional[Callable[[], Set[str]]] = None) -> BlacklistProvider:+def get_default_blacklist_provider(get_exempt_domains: Optional[Callable[[], Set[str]]] = None) -> BlacklistProvider:"""Get the default blacklist provider configuration.Domains a moderator has approved override the lists. Callers without a database - the@@ -28,7 +27,7 @@ def get_default_blacklist_provider(return CombinedBlacklistProvider([BuiltInRulesBlacklistProvider(),- HaGeZiBlacklistProvider('tif_medium'),+ HaGeZiBlacklistProvider("tif_medium"),AdultContentBlacklistProvider(),],get_exempt_domains=get_exempt_domains if get_exempt_domains is not None else get_curated_domains,diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.pyindex f1ccdf1..063a71c 100644--- a/mwmbl/indexer/blacklist_providers.py+++ b/mwmbl/indexer/blacklist_providers.py@@ -7,6 +7,7 @@ making the system more flexible and testable.from abc import ABC, abstractmethodfrom typing import Callable, Optional, Set+import requests@@ -26,10 +27,10 @@ def domain_and_parents(domain: str) -> list[str]:>>> domain_and_parents("nyou.booru.org")['nyou.booru.org', 'booru.org']"""- parts = domain.split('.')+ parts = domain.split(".")if len(parts) < 2:return [domain]- return ['.'.join(parts[i:]) for i in range(len(parts) - 1)]+ return [".".join(parts[i:]) for i in range(len(parts) - 1)]def domains_to_unblock(curated_domains: Set[str]) -> Set[str]:@@ -41,13 +42,13 @@ def domains_to_unblock(curated_domains: Set[str]) -> Set[str]:entry on the strength of an approved subdomain would unblock every one of its siblings."""unblocked = set(curated_domains)- unblocked.update(domain[len("www."):] for domain in curated_domains if domain.startswith("www."))+ unblocked.update(domain[len("www.") :] for domain in curated_domains if domain.startswith("www."))return unblockedclass BlacklistProvider(ABC):"""Abstract base class for blacklist providers."""-+@abstractmethoddef is_domain_blacklisted(self, domain: str) -> bool:"""Check if a domain should be blacklisted."""@@ -56,10 +57,10 @@ class BlacklistProvider(ABC):class StaticBlacklistProvider(BlacklistProvider):"""Provider that uses a static set of domains."""-+def __init__(self, domains: Set[str]):self.domains = domains.copy()-+def is_domain_blacklisted(self, domain: str) -> bool:"""Check if domain is in the static blacklist."""return domain in self.domains@@ -67,16 +68,16 @@ class StaticBlacklistProvider(BlacklistProvider):class BuiltInRulesBlacklistProvider(BlacklistProvider):"""Provider that implements the built-in spam detection and exclusion rules."""-+def __init__(self):# Import here to avoid circular imports- from mwmbl.settings import EXCLUDED_DOMAINS, DOMAIN_BLACKLIST_REGEXfrom mwmbl.hn_top_domains_filtered import DOMAINS-+ from mwmbl.settings import DOMAIN_BLACKLIST_REGEX, EXCLUDED_DOMAINS+self.excluded_domains = EXCLUDED_DOMAINSself.blacklist_regex = DOMAIN_BLACKLIST_REGEXself.trusted_domains = DOMAINS-+def is_domain_blacklisted(self, domain: str) -> bool:"""Check if domain should be blacklisted based on built-in rules."""# Check excluded domains. Matched as a suffix, so an apex entry covers its@@ -84,11 +85,11 @@ class BuiltInRulesBlacklistProvider(BlacklistProvider):# that host and below, which is what they were added for.if any(candidate in self.excluded_domains for candidate in domain_and_parents(domain)):return True-+# Check regex patterns (adult/spam content)if self.blacklist_regex.search(domain):return True-+# Trusted domains are never blacklistedif domain in self.trusted_domains:return False@@ -101,14 +102,14 @@ class BuiltInRulesBlacklistProvider(BlacklistProvider):# groups.google.com, images.example.com and every other three-label .com site with# a 6- or 8-character subdomain. Generated-hostname spam is better handled by the# threat feed than by guessing at label lengths.- domain_parts = domain.split('.')+ domain_parts = domain.split(".")# Domains with all-numeric generated-looking subdomains, e.g. 59648.etnomurcia.com.# Restricted to len > 2 so this doesn't catch a numeric *apex* domain like 350.org,# where the numeric part is the actual site name rather than a generated subdomain.if len(domain_parts) > 2 and set(domain_parts[0]) <= set("1234567890"):return True-+return False@@ -158,13 +159,13 @@ class RemoteListBlacklistProvider(BlacklistProvider):response.raise_for_status()domains = set()- for line in response.text.split('\n'):- line = line.split('#')[0].strip()+ for line in response.text.split("\n"):+ line = line.split("#")[0].strip()if not line:continue# Hosts format: "0.0.0.0 domain.com" or "127.0.0.1 domain.com"parts = line.split()- domains.add(parts[1] if len(parts) >= 2 and parts[0] in ('0.0.0.0', '127.0.0.1') else parts[0])+ domains.add(parts[1] if len(parts) >= 2 and parts[0] in ("0.0.0.0", "127.0.0.1") else parts[0])self._domains = domainsreturn domains@@ -198,16 +199,16 @@ class HaGeZiBlacklistProvider(RemoteListBlacklistProvider):"""HAGEZI_URLS = {- 'light': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/light-onlydomains.txt',- 'normal': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/multi-onlydomains.txt',- 'pro': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/pro-onlydomains.txt',- 'ultimate': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt',- 'tif': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif-onlydomains.txt',- 'tif_medium': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.medium-onlydomains.txt',- 'tif_mini': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.mini-onlydomains.txt',+ "light": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/light-onlydomains.txt",+ "normal": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/multi-onlydomains.txt",+ "pro": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/pro-onlydomains.txt",+ "ultimate": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt",+ "tif": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif-onlydomains.txt",+ "tif_medium": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.medium-onlydomains.txt",+ "tif_mini": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.mini-onlydomains.txt",}- def __init__(self, list_type: str = 'tif_medium'):+ def __init__(self, list_type: str = "tif_medium"):if list_type not in self.HAGEZI_URLS:raise ValueError(f"Invalid list_type. Must be one of: {list(self.HAGEZI_URLS.keys())}")@@ -222,7 +223,7 @@ class AdultContentBlacklistProvider(RemoteListBlacklistProvider):regex/domain rules (aimed at spam and a handful of known-bad domains) don't."""- URL = 'https://raw.githubusercontent.com/blocklistproject/Lists/master/porn.txt'+ URL = "https://raw.githubusercontent.com/blocklistproject/Lists/master/porn.txt"def __init__(self):super().__init__(self.URL)@@ -237,8 +238,7 @@ class CombinedBlacklistProvider(BlacklistProvider):crawler - and the result is resolved once and held, like the remote lists themselves."""- def __init__(self, providers: list[BlacklistProvider],- get_exempt_domains: Optional[Callable[[], Set[str]]] = None):+ def __init__(self, providers: list[BlacklistProvider], get_exempt_domains: Optional[Callable[[], Set[str]]] = None):self.providers = providersself.get_exempt_domains = get_exempt_domainsself._exempt_domains: Optional[Set[str]] = None@@ -248,7 +248,7 @@ class CombinedBlacklistProvider(BlacklistProvider):return Falseif self._exempt_domains is None:self._exempt_domains = domains_to_unblock(self.get_exempt_domains())- apex = domain[len("www."):] if domain.startswith("www.") else domain+ apex = domain[len("www.") :] if domain.startswith("www.") else domainreturn domain in self._exempt_domains or apex in self._exempt_domainsdef is_domain_blacklisted(self, domain: str) -> bool:@@ -263,5 +263,5 @@ class CombinedBlacklistProvider(BlacklistProvider):except Exception as e:print(f"Warning: Error from blacklist provider {provider.__class__.__name__}: {e}")continue-+return Falsediff --git a/mwmbl/indexer/blacklist_snapshot.py b/mwmbl/indexer/blacklist_snapshot.pyindex 6cf56d6..a5de0a3 100644--- a/mwmbl/indexer/blacklist_snapshot.py+++ b/mwmbl/indexer/blacklist_snapshot.py@@ -40,6 +40,7 @@ access, and every worker picks the exemption up with the next snapshot. The costapproval only takes effect on the next rebuild, which is why approving a submissionschedules one (mwmbl.signals)."""+import hashlibimport threadingimport time@@ -69,7 +70,7 @@ SNAPSHOT_KEY = "blacklist:domain-hashes:v1"SNAPSHOT_VERSION_KEY = "blacklist:domain-hashes:v1:version"# Hashes are stored little-endian so the blob is portable between machines.-HASH_DTYPE = np.dtype(np.uint64).newbyteorder('<')+HASH_DTYPE = np.dtype(np.uint64).newbyteorder("<")_redis: Optional[redis.Redis] = None@@ -95,6 +96,7 @@ def hash_domains(domains: Iterable[str]) -> np.ndarray:# Write side - runs in the background task only# ---------------------------------------------------------------------------+def collect_remote_domains(provider: BlacklistProvider) -> set[str]:"""Every domain from the remote lists reachable from this provider.@@ -128,9 +130,13 @@ def build_snapshot(provider: BlacklistProvider) -> bytes:num_unblocked = len(listed_domains) - len(domains)all_hashes = hash_domains(domains)hashes = np.unique(all_hashes) # np.unique sorts, which is what we need- logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB; "- "%d entries removed by curated domains",- len(domains), len(hashes), hashes.nbytes / 1e6, num_unblocked)+ logger.info(+ "Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB; %d entries removed by curated domains",+ len(domains),+ len(hashes),+ hashes.nbytes / 1e6,+ num_unblocked,+ )return hashes.astype(HASH_DTYPE).tobytes()@@ -149,8 +155,7 @@ def publish_snapshot(blob: bytes, redis_client: Optional[redis.Redis] = None) ->return version-def refresh_snapshot(provider: Optional[BlacklistProvider] = None,- redis_client: Optional[redis.Redis] = None) -> str:+def refresh_snapshot(provider: Optional[BlacklistProvider] = None, redis_client: Optional[redis.Redis] = None) -> str:if provider is None:provider = get_default_blacklist_provider()blob = build_snapshot(provider)@@ -161,6 +166,7 @@ def refresh_snapshot(provider: Optional[BlacklistProvider] = None,# Read side - runs in every search worker# ---------------------------------------------------------------------------+class SnapshotBlacklist:"""Membership queries against the published snapshot, plus the local built-in rules.@@ -170,8 +176,7 @@ class SnapshotBlacklist:against the previous array."""- def __init__(self, built_in_rules: Optional[BlacklistProvider] = None,- redis_client: Optional[redis.Redis] = None):+ def __init__(self, built_in_rules: Optional[BlacklistProvider] = None, redis_client: Optional[redis.Redis] = None):self._built_in_rules = built_in_rules if built_in_rules is not None else BuiltInRulesBlacklistProvider()self._redis = redis_clientself._array: Optional[np.ndarray] = None@@ -233,8 +238,12 @@ class SnapshotBlacklist:# np.frombuffer would raise, and this runs at import time via search_setup, so# a truncated blob would stop every web worker from starting. The published# snapshot is always a whole number of hashes, so this means a bad write.- logger.error("Blacklist snapshot version %s is %d bytes, not a multiple of %d; ignoring it",- version[:12], len(blob), HASH_DTYPE.itemsize)+ logger.error(+ "Blacklist snapshot version %s is %d bytes, not a multiple of %d; ignoring it",+ version[:12],+ len(blob),+ HASH_DTYPE.itemsize,+ )return Falsearray = np.frombuffer(blob, dtype=HASH_DTYPE)diff --git a/mwmbl/indexer/dedupe.py b/mwmbl/indexer/dedupe.pyindex 26caba7..7ae764c 100644--- a/mwmbl/indexer/dedupe.py+++ b/mwmbl/indexer/dedupe.py@@ -1,6 +1,7 @@"""Dedupe pages that have been crawled more than once and prepare them for indexing"""+import globimport gzipimport json@@ -17,8 +18,8 @@ def get_deduped_pages():seen_urls = set()for path in sorted(glob.glob(CRAWL_GLOB), reverse=True):data = json.load(gzip.open(path))- for item in data['items']:- url = item['url']+ for item in data["items"]:+ url = item["url"]if url in seen_urls:continue@@ -27,10 +28,10 @@ def get_deduped_pages():def queue_deduped_items(deduped_pages):- output_queue = FSQueue(TINYSEARCH_DATA_DIR, 'mwmbl-search-items', GzipJsonBlobSerializer())+ output_queue = FSQueue(TINYSEARCH_DATA_DIR, "mwmbl-search-items", GzipJsonBlobSerializer())for batch in grouper(BATCH_SIZE, deduped_pages):- data = {'items': batch}+ data = {"items": batch}output_queue.put(data)@@ -39,7 +40,7 @@ def run():queue_deduped_items(deduped_pages)-if __name__ == '__main__':+if __name__ == "__main__":run()@@ -48,4 +49,4 @@ def grouper(n: int, iterator: Iterator):chunk = tuple(islice(iterator, n))if not chunk:return- yield chunk\ No newline at end of file+ yield chunkdiff --git a/mwmbl/indexer/domains/domain_titles.py b/mwmbl/indexer/domains/domain_titles.pyindex 43cf520..6bc5993 100644--- a/mwmbl/indexer/domains/domain_titles.py+++ b/mwmbl/indexer/domains/domain_titles.py@@ -1,6 +1,7 @@"""Retrieve titles for each domain in the list of top domains"""+from multiprocessing import Processfrom time import sleepfrom urllib.parse import urlsplit, urlunsplit@@ -9,7 +10,7 @@ import bs4import requestsfrom mwmbl.indexer.fsqueue import FSQueue, ZstdJsonSerializer-from mwmbl.indexer.paths import TINYSEARCH_DATA_DIR, DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAME+from mwmbl.indexer.paths import DOMAINS_QUEUE_NAME, DOMAINS_TITLES_QUEUE_NAME, TINYSEARCH_DATA_DIRNUM_PROCESSES = 10@@ -23,10 +24,10 @@ def get_redirect_no_cookies(url, max_redirects=5):print("Unable to get with SSL", e)result = requests.get(url, allow_redirects=False, verify=False, timeout=10)if result.status_code // 100 == 3:- location = result.headers['Location']- if not location.startswith('http'):+ location = result.headers["Location"]+ if not location.startswith("http"):parsed_url = urlsplit(url)- location = urlunsplit(parsed_url[:2] + (location, '', ''))+ location = urlunsplit(parsed_url[:2] + (location, "", ""))return get_redirect_no_cookies(location, max_redirects=max_redirects - 1)return result@@ -64,8 +65,7 @@ def retrieve_title(domain):result = get_redirect_no_cookies(original_url)status = result.status_codeurl = result.url- except (RecursionError, requests.exceptions.ConnectionError,- requests.exceptions.ReadTimeout) as e:+ except (RecursionError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:print("Error retrieving URL", str(e))status = Noneurl = None@@ -74,7 +74,7 @@ def retrieve_title(domain):if status != 200:title = Noneelse:- title_tag = bs4.BeautifulSoup(result.content, features="lxml").find('title')+ title_tag = bs4.BeautifulSoup(result.content, features="lxml").find("title")title = str(title_tag.string) if title_tag is not None else domain# print("Title", domain, title)return status, title, url@@ -87,5 +87,5 @@ def run():sleep(3)-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/indexer/domains/queue_domains.py b/mwmbl/indexer/domains/queue_domains.pyindex 02df697..b85af37 100644--- a/mwmbl/indexer/domains/queue_domains.py+++ b/mwmbl/indexer/domains/queue_domains.py@@ -1,6 +1,7 @@"""Add domains to the queue to be retrieved"""+import csvimport gzip@@ -11,7 +12,7 @@ BATCH_SIZE = 250def get_domains():- reader = csv.reader(gzip.open(DOMAINS_PATH, 'rt'))+ reader = csv.reader(gzip.open(DOMAINS_PATH, "rt"))next(reader)for rank, domain, _ in reader:yield rank, domain@@ -30,5 +31,5 @@ def queue_domains():print("Queued:", queued)-if __name__ == '__main__':+if __name__ == "__main__":queue_domains()diff --git a/mwmbl/indexer/external_cache.py b/mwmbl/indexer/external_cache.pyindex 5f05d2e..7904eed 100644--- a/mwmbl/indexer/external_cache.py+++ b/mwmbl/indexer/external_cache.py@@ -29,6 +29,7 @@ not the fact that somebody searched in that area, since an entry still holds thethat came back.That is strictly less than the disk cache gave away."""+import hashlibimport hmacimport time@@ -132,8 +133,7 @@ def external_cache_term(query: str) -> str:in settings_prod."""normalised = " ".join(query.split()).casefold()- digest = hmac.new(settings.SECRET_KEY.encode("utf8"),- normalised.encode("utf8", errors="ignore"), hashlib.sha256)+ digest = hmac.new(settings.SECRET_KEY.encode("utf8"), normalised.encode("utf8", errors="ignore"), hashlib.sha256)return digest.hexdigest()[:16]@@ -144,13 +144,17 @@ def is_fresh(document: Document, now: int) -> bool:providers, so it cannot recover the source from the term - document.source is the onlything that identifies them, and it is where a per-source TTL would key off."""- ttl = (settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS if document.url == EXTERNAL_CACHE_EMPTY_URL- else settings.EXTERNAL_CACHE_TTL_SECONDS)+ ttl = (+ settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS+ if document.url == EXTERNAL_CACHE_EMPTY_URL+ else settings.EXTERNAL_CACHE_TTL_SECONDS+ )return document.last_crawled is not None and now - document.last_crawled < ttl-def get_cached_external_results(source: DocumentSource, query: str,- now: Optional[int] = None) -> Optional[list[Document]]:+def get_cached_external_results(+ source: DocumentSource, query: str, now: Optional[int] = None+) -> Optional[list[Document]]:"""A query's cached results from one provider.None means we have nothing for this (source, query) and should ask the provider. A list@@ -177,8 +181,11 @@ def get_cached_external_results(source: DocumentSource, query: str,# hashes to it, and it also lets through documents with no term at all. The term narrows# that to this query - across all providers, since the term does not name one - and the# source picks out the provider being asked about.- entries = [document for document in stored- if document.term == term and document.source == source and is_fresh(document, now)]+ entries = [+ document+ for document in stored+ if document.term == term and document.source == source and is_fresh(document, now)+ ]if not entries:return None@@ -190,13 +197,24 @@ def get_cached_external_results(source: DocumentSource, query: str,# is not a score: turning one into the other needs the caller's scale, which the cache# does not know and must not guess. Callers score by position - get_wiki_results does.ranked = sorted(entries, key=lambda document: document.score or 0.0)- return [Document(title=document.title, url=document.url, extract=document.extract,- score=None, term=query, state=document.state, source=document.source)- for document in ranked if document.url != EXTERNAL_CACHE_EMPTY_URL]---def store_external_results(source: DocumentSource, query: str, documents: list[Document],- now: Optional[int] = None) -> None:+ return [+ Document(+ title=document.title,+ url=document.url,+ extract=document.extract,+ score=None,+ term=query,+ state=document.state,+ source=document.source,+ )+ for document in ranked+ if document.url != EXTERNAL_CACHE_EMPTY_URL+ ]+++def store_external_results(+ source: DocumentSource, query: str, documents: list[Document], now: Optional[int] = None+) -> None:"""Store one provider's results for a query, replacing any existing entry for it.Never raises. A cache write that fails costs a re-fetch later; it must not cost a@@ -210,7 +228,7 @@ def store_external_results(source: DocumentSource, query: str, documents: list[Dentries = _entries_to_store(source, term, documents, now)try:- with TinyIndex(item_factory=Document, index_path=external_cache_path(), mode='w') as index:+ with TinyIndex(item_factory=Document, index_path=external_cache_path(), mode="w") as index:page_index = index.get_key_page_index(term)with index.page(page_index) as page:# Newest first. store() drops the tail that does not fit, so ordering by@@ -225,21 +243,26 @@ def store_external_results(source: DocumentSource, query: str, documents: list[D# provider's results would silently drop every other provider's results for# the same query.kept = sorted(- (document for document in page.documents- if not (document.term == term and document.source == source)- and is_fresh(document, now)),- key=lambda document: -(document.last_crawled or 0))+ (+ document+ for document in page.documents+ if not (document.term == term and document.source == source) and is_fresh(document, now)+ ),+ key=lambda document: -(document.last_crawled or 0),+ )stored = page.store(entries + kept)if stored < len(entries):logger.warning("External cache page %d held only %d of %d entries for one query",- page_index, stored, len(entries))+ page_index,+ stored,+ len(entries),+ )except Exception:logger.exception("Could not store %s results in the external cache index", source.name)-def _entries_to_store(source: DocumentSource, term: str, documents: list[Document],- now: int) -> list[Document]:+def _entries_to_store(source: DocumentSource, term: str, documents: list[Document], now: int) -> list[Document]:"""The documents to write for this query, or the empty-result sentinel if there are none.Every entry carries its source, the sentinel included - an entry has to say which@@ -257,10 +280,28 @@ def _entries_to_store(source: DocumentSource, term: str, documents: list[Documen"""usable = [document for document in documents if document.url and document.title]if not usable:- return [Document(title="", url=EXTERNAL_CACHE_EMPTY_URL, extract="", score=0.0, term=term,- last_crawled=now, source=source)]-- return [Document(title=document.title, url=document.url, extract=document.extract,- score=float(rank), term=term, state=document.state, last_crawled=now,- source=source)- for rank, document in enumerate(usable)]+ return [+ Document(+ title="",+ url=EXTERNAL_CACHE_EMPTY_URL,+ extract="",+ score=0.0,+ term=term,+ last_crawled=now,+ source=source,+ )+ ]++ return [+ Document(+ title=document.title,+ url=document.url,+ extract=document.extract,+ score=float(rank),+ term=term,+ state=document.state,+ last_crawled=now,+ source=source,+ )+ for rank, document in enumerate(usable)+ ]diff --git a/mwmbl/indexer/fsqueue.py b/mwmbl/indexer/fsqueue.pyindex 88787d9..13aee25 100644--- a/mwmbl/indexer/fsqueue.py+++ b/mwmbl/indexer/fsqueue.py@@ -2,14 +2,15 @@Filesystem-based queue that uses os.rename as an atomic operation to ensurethat items are handled correctly."""+import gzipimport jsonimport osfrom abc import ABCfrom enum import Enum-from typing import Union, Any-from uuid import uuid4from pathlib import Path+from typing import Any, Union+from uuid import uuid4from zstandard import ZstdCompressor, ZstdDecompressor@@ -21,11 +22,11 @@ class FSQueueError(Exception):class FSState(Enum):- CREATING = 'creating'- READY = 'ready'- LOCKED = 'locked'- DONE = 'done'- ERROR = 'error'+ CREATING = "creating"+ READY = "ready"+ LOCKED = "locked"+ DONE = "done"+ ERROR = "error"class Serializer(ABC):@@ -42,28 +43,28 @@ class ZstdJsonSerializer(Serializer):self.decompressor = ZstdDecompressor()def serialize(self, item) -> bytes:- return self.compressor.compress(json.dumps(item).encode('utf8'))+ return self.compressor.compress(json.dumps(item).encode("utf8"))def deserialize(self, serialized_item: bytes):- return json.loads(self.decompressor.decompress(serialized_item).decode('utf8'))+ return json.loads(self.decompressor.decompress(serialized_item).decode("utf8"))class GzipJsonRowSerializer(Serializer):def serialize(self, items: list[object]) -> bytes:json_items = [json.dumps(item) for item in items]- return gzip.compress('\n'.join(json_items).encode('utf8'))+ return gzip.compress("\n".join(json_items).encode("utf8"))def deserialize(self, serialized_items: bytes) -> list[object]:- lines = gzip.decompress(serialized_items).decode('utf8')- return [json.loads(line) for line in lines.strip().split('\n')]+ lines = gzip.decompress(serialized_items).decode("utf8")+ return [json.loads(line) for line in lines.strip().split("\n")]class GzipJsonBlobSerializer(Serializer):def serialize(self, items: Any) -> bytes:- return gzip.compress(json.dumps(items).encode('utf8'))+ return gzip.compress(json.dumps(items).encode("utf8"))def deserialize(self, serialized_items: bytes) -> Any:- data = gzip.decompress(serialized_items).decode('utf8')+ data = gzip.decompress(serialized_items).decode("utf8")return json.loads(data)@@ -76,7 +77,7 @@ class FSQueue:if not os.path.isdir(self.directory):raise ValueError("Given path is not a directory")- if '/' in name:+ if "/" in name:raise ValueError("Name should not contain '/'")os.makedirs(os.path.join(self.directory, self.name), exist_ok=True)@@ -97,7 +98,7 @@ class FSQueue:Push a new item into the ready state"""item_id = str(uuid4())- with open(self._get_path(FSState.CREATING, item_id), 'wb') as output_file:+ with open(self._get_path(FSState.CREATING, item_id), "wb") as output_file:output_file.write(self.serializer.serialize(item))self._move(item_id, FSState.CREATING, FSState.READY)@@ -121,12 +122,12 @@ class FSQueue:print("File not found", path.name)continue- with open(self._get_path(FSState.LOCKED, path.name), 'rb') as item_file:+ with open(self._get_path(FSState.LOCKED, path.name), "rb") as item_file:print("Opening file", path.name)try:return path.name, self.serializer.deserialize(item_file.read())except Exception as e:- raise FSQueueError(path.name, 'Error deserializing item') from e+ raise FSQueueError(path.name, "Error deserializing item") from edef done(self, item_id: str):"""@@ -146,4 +147,3 @@ class FSQueue:for path in paths:# Try and lock the fileself._move(path.name, FSState.LOCKED, FSState.READY)-diff --git a/mwmbl/indexer/historical.py b/mwmbl/indexer/historical.pyindex 9386680..5959294 100644--- a/mwmbl/indexer/historical.py+++ b/mwmbl/indexer/historical.py@@ -14,15 +14,15 @@ def run():index_db = IndexDatabase(db.connection)index_db.create_tables()batches = get_batches_for_date(date_str)- batch_urls = batches['batch_urls']+ batch_urls = batches["batch_urls"]print("Historical batches for date", date_str, len(batch_urls))infos = [BatchInfo(url, get_user_id_hash_from_url(url), BatchStatus.REMOTE) for url in batch_urls]index_db.record_batches(infos)def get_user_id_hash_from_url(url):- return url.split('/')[9]+ return url.split("/")[9]-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/indexer/index.py b/mwmbl/indexer/index.pyindex dc7a00c..d122b36 100644--- a/mwmbl/indexer/index.py+++ b/mwmbl/indexer/index.py@@ -1,47 +1,49 @@"""Create a search index"""-from typing import Iterable+from urllib.parse import unquotefrom mwmbl.tinysearchengine.indexer import TokenizedDocument-from mwmbl.tokenizer import tokenize, get_bigrams+from mwmbl.tokenizer import get_bigrams, tokenizeDEFAULT_SCORE = 0-HTTP_START = 'http://'-HTTPS_START = 'https://'+HTTP_START = "http://"+HTTPS_START = "https://"NUM_FIRST_TOKENS = 10NUM_BIGRAMS = 10-STOPWORDS = set("0,1,2,3,4,5,6,7,8,9,a,A,about,above,across,after,again,against,all,almost,alone,along,already,also," \- "although,always,am,among,an,and,another,any,anyone,anything,anywhere,are,aren't,around,as,at,b,B,back," \- "be,became,because,become,becomes,been,before,behind,being,below,between,both,but,by,c,C,can,cannot,can't," \- "could,couldn't,d,D,did,didn't,do,does,doesn't,doing,done,don't,down,during,e,E,each,either,enough,even," \- "ever,every,everyone,everything,everywhere,f,F,few,find,first,for,four,from,full,further,g,G,get,give,go," \- "h,H,had,hadn't,has,hasn't,have,haven't,having,he,he'd,he'll,her,here,here's,hers,herself,he's,him," \- "himself,his,how,however,how's,i,I,i'd,if,i'll,i'm,in,interest,into,is,isn't,it,it's,its,itself,i've," \- "j,J,k,K,keep,l,L,last,least,less,let's,m,M,made,many,may,me,might,more,most,mostly,much,must,mustn't," \- "my,myself,n,N,never,next,no,nobody,noone,nor,not,nothing,now,nowhere,o,O,of,off,often,on,once,one,only," \- "or,other,others,ought,our,ours,ourselves,out,over,own,p,P,part,per,perhaps,put,q,Q,r,R,rather,s,S,same," \- "see,seem,seemed,seeming,seems,several,shan't,she,she'd,she'll,she's,should,shouldn't,show,side,since,so," \- "some,someone,something,somewhere,still,such,t,T,take,than,that,that's,the,their,theirs,them,themselves," \- "then,there,therefore,there's,these,they,they'd,they'll,they're,they've,this,those,though,three,through," \- "thus,to,together,too,toward,two,u,U,under,until,up,upon,us,v,V,very,w,W,was,wasn't,we,we'd,we'll,well," \- "we're,were,weren't,we've,what,what's,when,when's,where,where's,whether,which,while,who,whole,whom,who's," \- "whose,why,why's,will,with,within,without,won't,would,wouldn't,x,X,y,Y,yet,you,you'd,you'll,your,you're," \- "yours,yourself,yourselves,you've,z,Z".split(','))+STOPWORDS = set(+ "0,1,2,3,4,5,6,7,8,9,a,A,about,above,across,after,again,against,all,almost,alone,along,already,also,"+ "although,always,am,among,an,and,another,any,anyone,anything,anywhere,are,aren't,around,as,at,b,B,back,"+ "be,became,because,become,becomes,been,before,behind,being,below,between,both,but,by,c,C,can,cannot,can't,"+ "could,couldn't,d,D,did,didn't,do,does,doesn't,doing,done,don't,down,during,e,E,each,either,enough,even,"+ "ever,every,everyone,everything,everywhere,f,F,few,find,first,for,four,from,full,further,g,G,get,give,go,"+ "h,H,had,hadn't,has,hasn't,have,haven't,having,he,he'd,he'll,her,here,here's,hers,herself,he's,him,"+ "himself,his,how,however,how's,i,I,i'd,if,i'll,i'm,in,interest,into,is,isn't,it,it's,its,itself,i've,"+ "j,J,k,K,keep,l,L,last,least,less,let's,m,M,made,many,may,me,might,more,most,mostly,much,must,mustn't,"+ "my,myself,n,N,never,next,no,nobody,noone,nor,not,nothing,now,nowhere,o,O,of,off,often,on,once,one,only,"+ "or,other,others,ought,our,ours,ourselves,out,over,own,p,P,part,per,perhaps,put,q,Q,r,R,rather,s,S,same,"+ "see,seem,seemed,seeming,seems,several,shan't,she,she'd,she'll,she's,should,shouldn't,show,side,since,so,"+ "some,someone,something,somewhere,still,such,t,T,take,than,that,that's,the,their,theirs,them,themselves,"+ "then,there,therefore,there's,these,they,they'd,they'll,they're,they've,this,those,though,three,through,"+ "thus,to,together,too,toward,two,u,U,under,until,up,upon,us,v,V,very,w,W,was,wasn't,we,we'd,we'll,well,"+ "we're,were,weren't,we've,what,what's,when,when's,where,where's,whether,which,while,who,whole,whom,who's,"+ "whose,why,why's,will,with,within,without,won't,would,wouldn't,x,X,y,Y,yet,you,you'd,you'll,your,you're,"+ "yours,yourself,yourselves,you've,z,Z".split(",")+)def prepare_url_for_tokenizing(url: str):if url.startswith(HTTP_START):- url = url[len(HTTP_START):]+ url = url[len(HTTP_START) :]elif url.startswith(HTTPS_START):- url = url[len(HTTPS_START):]- for c in '/._':+ url = url[len(HTTPS_START) :]+ for c in "/._":if c in url:- url = url.replace(c, ' ')+ url = url.replace(c, " ")return urldiff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex 2aaf142..87e26fc 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -1,10 +1,9 @@"""Index batches that are stored locally."""-import math-from collections import defaultdict, Counter++from collections import Counter, defaultdictfrom datetime import datetime-from functools import reducefrom logging import getLoggerfrom typing import Collection, Iterable, Optionalfrom urllib.parse import unquote@@ -14,24 +13,21 @@ from mwmbl.crawler.urls import URLStatusfrom mwmbl.indexer import process_batchfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist-from mwmbl.indexer.index import tokenize_document, prepare_url_for_tokenizing+from mwmbl.indexer.index import prepare_url_for_tokenizing, tokenize_documentfrom mwmbl.indexer.indexdb import BatchStatus-from mwmbl.tinysearchengine.indexer import (- CURATED_STATES, Document, DocumentState, PageError, TinyIndex)-from mwmbl.tinysearchengine.rank import score_result, DOCUMENT_FREQUENCIES, N_DOCUMENTS, HeuristicRanker-from mwmbl.tokenizer import tokenize, get_bigrams-from mwmbl.utils import add_term_infos, get_domain+from mwmbl.tinysearchengine.indexer import CURATED_STATES, Document, DocumentState, PageError, TinyIndex+from mwmbl.tinysearchengine.rank import HeuristicRanker+from mwmbl.tokenizer import get_bigrams, tokenize+from mwmbl.utils import get_domainlogger = getLogger(__name__)MAX_USER_IDS = 2-def _merge_user_ids(- existing: Optional[list[int]], incoming: Optional[list[int]]-) -> Optional[list[int]]:+def _merge_user_ids(existing: Optional[list[int]], incoming: Optional[list[int]]) -> Optional[list[int]]:combined = list(existing or [])- for uid in (incoming or []):+ for uid in incoming or []:if uid in combined:combined.remove(uid)combined.append(uid)@@ -43,7 +39,9 @@ def get_documents_from_batches(batches: Collection[HashedBatch]) -> Iterable[Docfor item in batch.items:if item.content is not None and not item.content.links_only:yield Document(- item.content.title, item.url, item.content.extract,+ item.content.title,+ item.url,+ item.content.extract,last_crawled=int(item.timestamp / 1000),)@@ -59,7 +57,7 @@ def run(batch_cache: BatchCache, index_path: str):def get_url_score(url):# TODO: compute a proper score for each document- return 1/len(url)+ return 1 / len(url)def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Counter:@@ -113,15 +111,17 @@ def filter_blacklisted_documents(documents: list[Document]) -> list[Document]:def index_pages(index_path: str, page_documents: dict[int, list[Document]], mark_synced: bool = False) -> Counter:term_new_doc_counts = Counter()- with TinyIndex(Document, index_path, 'w') as indexer:- ranker = HeuristicRanker(indexer, None, score_threshold=float('-inf'))+ with TinyIndex(Document, index_path, "w") as indexer:+ ranker = HeuristicRanker(indexer, None, score_threshold=float("-inf"))for page_index, documents in page_documents.items():try:with indexer.page(page_index) as page:combined_documents = combine_documents(page.documents, documents, mark_synced, ranker)num_stored = page.store(combined_documents)- logger.info(f"Storing {num_stored} of {len(combined_documents)} documents for "- f"page {page_index}, originally {len(page.documents)}")+ logger.info(+ f"Storing {num_stored} of {len(combined_documents)} documents for "+ f"page {page_index}, originally {len(page.documents)}"+ )except PageError:# One page we cannot safely write costs the documents bound for it, not the# rest of the batch. Letting it propagate would abort every remaining page,@@ -130,17 +130,18 @@ def index_pages(index_path: str, page_documents: dict[int, list[Document]], marklogger.exception("Skipping index page %d", page_index)continue- term_new_doc_counts.update(document.term for document in combined_documents[:num_stored]- if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX)+ term_new_doc_counts.update(+ document.term+ for document in combined_documents[:num_stored]+ if document.state != DocumentState.SYNCED_WITH_MAIN_INDEX+ )return term_new_doc_countsdef _document_token_set(doc: Document) -> set[str]:"""Unigram tokens of a document's title, URL and extract (no bigrams)."""prepared_url = prepare_url_for_tokenizing(unquote(doc.url))- return (set(tokenize(doc.title))- | set(tokenize(prepared_url))- | set(tokenize(doc.extract)))+ return set(tokenize(doc.title)) | set(tokenize(prepared_url)) | set(tokenize(doc.extract))def index_results_against_query(documents: list[Document], query: str, index_path: str) -> int:@@ -168,7 +169,7 @@ def index_results_against_query(documents: list[Document], query: str, index_pat# Read pass: build per-page candidates and track which (term, url) are new.page_documents: dict[int, list[Document]] = defaultdict(list)new_urls: set[str] = set()- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:existing_keys: dict[int, set[tuple]] = {}for doc in documents:if not (doc.url and doc.title):@@ -178,10 +179,15 @@ def index_results_against_query(documents: list[Document], query: str, index_patif not (words <= doc_tokens):continuepage = indexer.get_key_page_index(term)- page_documents[page].append(Document(- doc.title, doc.url, doc.extract,- term=term, last_crawled=doc.last_crawled,- ))+ page_documents[page].append(+ Document(+ doc.title,+ doc.url,+ doc.extract,+ term=term,+ last_crawled=doc.last_crawled,+ )+ )if page not in existing_keys:try:existing_keys[page] = {(d.term, d.url) for d in indexer.get_page(page)}@@ -255,7 +261,7 @@ def sort_documents(documents, all_existing_documents, ranker):def preprocess_documents(documents, index_path):page_documents = defaultdict(list)- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:for i, document in enumerate(documents):if i % 1000 == 0:logger.info(f"Preprocessing document {i} of {len(documents)}")@@ -264,7 +270,9 @@ def preprocess_documents(documents, index_path):for token in tokenized.tokens:page = indexer.get_key_page_index(token)term_document = Document(- document.title, document.url, document.extract,+ document.title,+ document.url,+ document.extract,term=token,user_ids=document.user_ids,last_crawled=document.last_crawled,@@ -278,8 +286,8 @@ def get_url_error_status(item: Item):if item.status == 404:return URLStatus.ERROR_404if item.error is not None:- if item.error.name == 'AbortError':+ if item.error.name == "AbortError":return URLStatus.ERROR_TIMEOUT- elif item.error.name == 'RobotsDenied':+ elif item.error.name == "RobotsDenied":return URLStatus.ERROR_ROBOTS_DENIEDreturn URLStatus.ERROR_OTHERdiff --git a/mwmbl/indexer/indexdb.py b/mwmbl/indexer/indexdb.pyindex 449f3bd..b616e04 100644--- a/mwmbl/indexer/indexdb.py+++ b/mwmbl/indexer/indexdb.py@@ -1,6 +1,7 @@"""Database interface for batches of crawled data."""+from dataclasses import dataclassfrom enum import Enum@@ -8,10 +9,10 @@ from psycopg2.extras import execute_valuesclass BatchStatus(Enum):- REMOTE = 0 # The batch only exists in long term storage- LOCAL = 10 # We have a copy of the batch locally in Postgresql- URLS_UPDATED = 20 # We've updated URLs from the batch- INDEXED = 30 # The batch has been indexed+ REMOTE = 0 # The batch only exists in long term storage+ LOCAL = 10 # We have a copy of the batch locally in Postgresql+ URLS_UPDATED = 20 # We've updated URLs from the batch+ INDEXED = 30 # The batch has been indexed@dataclass@@ -54,7 +55,7 @@ class IndexDatabase:"""with self.connection.cursor() as cursor:- cursor.execute(sql, {'status': status.value, 'num_batches': num_batches})+ cursor.execute(sql, {"status": status.value, "num_batches": num_batches})results = cursor.fetchall()return [BatchInfo(url, user_id_hash, status) for url, user_id_hash, status in results]@@ -68,4 +69,4 @@ class IndexDatabase:"""with self.connection.cursor() as cursor:- cursor.execute(sql, {'status': status.value, 'urls': tuple(batch_urls)})+ cursor.execute(sql, {"status": status.value, "urls": tuple(batch_urls)})diff --git a/mwmbl/indexer/links.py b/mwmbl/indexer/links.pyindex 437c9b0..81501f0 100644--- a/mwmbl/indexer/links.py+++ b/mwmbl/indexer/links.py@@ -1,6 +1,7 @@"""Analyse crawl data to find the most popular links"""+import globimport gzipimport json@@ -13,10 +14,10 @@ from mwmbl.indexer.paths import CRAWL_GLOB, LINK_COUNT_PATHdef get_urls():for path in glob.glob(CRAWL_GLOB):data = json.load(gzip.open(path))- for item in data['items']:- url = item['url']+ for item in data["items"]:+ url = item["url"]domain = urlparse(url).hostname- for link in item['links']:+ for link in item["links"]:yield domain, link@@ -31,9 +32,9 @@ def run():url_links = get_urls()collected = collect_links(url_links)link_counts = {url: len(links) for url, links in collected.items()}- with open(LINK_COUNT_PATH, 'w') as output_file:+ with open(LINK_COUNT_PATH, "w") as output_file:json.dump(link_counts, output_file, indent=2)-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/indexer/paths.py b/mwmbl/indexer/paths.pyindex 5b02c41..8f2472b 100644--- a/mwmbl/indexer/paths.py+++ b/mwmbl/indexer/paths.py@@ -1,28 +1,28 @@import osfrom pathlib import Path-HOME = os.getenv('HOME')+HOME = os.getenv("HOME")-DATA_DIR = Path(os.environ['HOME']) / 'data'-TINYSEARCH_DATA_DIR = DATA_DIR / 'tinysearch'-COMMON_CRAWL_TERMS_PATH = TINYSEARCH_DATA_DIR / 'common-craw-terms.csv'+DATA_DIR = Path(os.environ["HOME"]) / "data"+TINYSEARCH_DATA_DIR = DATA_DIR / "tinysearch"+COMMON_CRAWL_TERMS_PATH = TINYSEARCH_DATA_DIR / "common-craw-terms.csv"-TEST_INDEX_PATH = os.path.join(TINYSEARCH_DATA_DIR, 'index-test.tinysearch')-TEST_TERMS_PATH = os.path.join(TINYSEARCH_DATA_DIR, 'index-terms.csv')-WIKI_DATA_PATH = os.path.join(TINYSEARCH_DATA_DIR, 'enwiki-20210301-pages-articles1.xml-p1p41242.bz2')-WIKI_TITLES_PATH = os.path.join(TINYSEARCH_DATA_DIR, 'abstract-titles-sorted.txt.gz')+TEST_INDEX_PATH = os.path.join(TINYSEARCH_DATA_DIR, "index-test.tinysearch")+TEST_TERMS_PATH = os.path.join(TINYSEARCH_DATA_DIR, "index-terms.csv")+WIKI_DATA_PATH = os.path.join(TINYSEARCH_DATA_DIR, "enwiki-20210301-pages-articles1.xml-p1p41242.bz2")+WIKI_TITLES_PATH = os.path.join(TINYSEARCH_DATA_DIR, "abstract-titles-sorted.txt.gz")-URLS_PATH = TINYSEARCH_DATA_DIR / 'urls.sqlite3'-DOMAINS_QUEUE_NAME = 'domains-queue-fs'-DOMAINS_TITLES_QUEUE_NAME = 'domains-title-queue-fs'-DOMAINS_PATH = os.path.join(TINYSEARCH_DATA_DIR, 'top10milliondomains.csv.gz')+URLS_PATH = TINYSEARCH_DATA_DIR / "urls.sqlite3"+DOMAINS_QUEUE_NAME = "domains-queue-fs"+DOMAINS_TITLES_QUEUE_NAME = "domains-title-queue-fs"+DOMAINS_PATH = os.path.join(TINYSEARCH_DATA_DIR, "top10milliondomains.csv.gz")-LOCAL_DATA_DIR = Path(__file__).parent.parent.parent / 'data'-INDEX_PATH = LOCAL_DATA_DIR / 'index.tinysearch'-MWMBL_CRAWL_TERMS_PATH = LOCAL_DATA_DIR / 'mwmbl-crawl-terms.csv'+LOCAL_DATA_DIR = Path(__file__).parent.parent.parent / "data"+INDEX_PATH = LOCAL_DATA_DIR / "index.tinysearch"+MWMBL_CRAWL_TERMS_PATH = LOCAL_DATA_DIR / "mwmbl-crawl-terms.csv"-TOP_DOMAINS_JSON_PATH = TINYSEARCH_DATA_DIR / 'hn-top-domains.json'+TOP_DOMAINS_JSON_PATH = TINYSEARCH_DATA_DIR / "hn-top-domains.json"MWMBL_DATA_DIR = DATA_DIR / "mwmbl"CRAWL_GLOB = str(MWMBL_DATA_DIR / "b2") + "/*/*/*/*/*/*.json.gz"-LINK_COUNT_PATH = MWMBL_DATA_DIR / 'crawl-counts.json'+LINK_COUNT_PATH = MWMBL_DATA_DIR / "crawl-counts.json"diff --git a/mwmbl/indexer/process_batch.py b/mwmbl/indexer/process_batch.pyindex cd9df6b..c0f88de 100644--- a/mwmbl/indexer/process_batch.py+++ b/mwmbl/indexer/process_batch.py@@ -9,8 +9,14 @@ from mwmbl.indexer.indexdb import BatchStatus, IndexDatabaselogger = getLogger(__name__)-def run(batch_cache: BatchCache, start_status: BatchStatus, end_status: BatchStatus,- process: Callable[[Collection[HashedBatch], ...], None], num_batches, *args):+def run(+ batch_cache: BatchCache,+ start_status: BatchStatus,+ end_status: BatchStatus,+ process: Callable[[Collection[HashedBatch], ...], None],+ num_batches,+ *args,+):with Database() as db:index_db = IndexDatabase(db.connection)diff --git a/mwmbl/indexer/purge_blacklisted.py b/mwmbl/indexer/purge_blacklisted.pyindex ea3b6bf..c64300c 100644--- a/mwmbl/indexer/purge_blacklisted.py+++ b/mwmbl/indexer/purge_blacklisted.py@@ -19,6 +19,7 @@ for the queries that surface them, which is what makes the loop converge: everyremoves at least the copy that was queued, so no document can churn through the queueforever."""+from collections import defaultdictfrom logging import getLoggerfrom typing import Callable, Iterable, Optional@@ -43,8 +44,9 @@ def pages_for_document(index: TinyIndex, document: Document) -> set[int]:return pages-def purge_documents(index: TinyIndex, documents: Iterable[Document],- is_blacklisted: Optional[Callable[[str], bool]] = None) -> dict[str, int]:+def purge_documents(+ index: TinyIndex, documents: Iterable[Document], is_blacklisted: Optional[Callable[[str], bool]] = None+) -> dict[str, int]:"""Remove exactly these documents, by URL, from every page they are filed under.Returns a count of removed documents by domain.@@ -94,8 +96,9 @@ def purge_documents(index: TinyIndex, documents: Iterable[Document],removed_urls_by_domain[document.url].add(document.url)num_stored = page.store(kept)if num_stored < len(kept):- logger.warning("Page %d only held %d of %d documents kept after purging",- page_index, num_stored, len(kept))+ logger.warning(+ "Page %d only held %d of %d documents kept after purging", page_index, num_stored, len(kept)+ )except PageError:# The caller has already drained these documents out of the purge queue, so# letting one unreadable page propagate would lose the whole batch - including@@ -105,6 +108,10 @@ def purge_documents(index: TinyIndex, documents: Iterable[Document],removed_by_domain = {domain: len(urls) for domain, urls in removed_urls_by_domain.items()}if removed_by_domain:- logger.info("Purged %d documents across %d pages from %d URLs",- sum(removed_by_domain.values()), len(pages_to_urls), len(target_urls))+ logger.info(+ "Purged %d documents across %d pages from %d URLs",+ sum(removed_by_domain.values()),+ len(pages_to_urls),+ len(target_urls),+ )return removed_by_domaindiff --git a/mwmbl/indexer/purge_queue.py b/mwmbl/indexer/purge_queue.pyindex 9a5eb3c..b74e099 100644--- a/mwmbl/indexer/purge_queue.py+++ b/mwmbl/indexer/purge_queue.py@@ -25,6 +25,7 @@ serialised JSON is byte-identical between retrievals and SET dedupe works on it.The same URL under two different terms is two entries, which is correct: they are twocopies on two pages, and each needs its own removal."""+import jsonimport threadingimport time@@ -89,13 +90,16 @@ def get_redis() -> redis.Redis:def _payload(document: Document) -> str:- return json.dumps({- "url": document.url,- "title": document.title,- "extract": document.extract,- "score": document.score,- "term": document.term,- }, sort_keys=True)+ return json.dumps(+ {+ "url": document.url,+ "title": document.title,+ "extract": document.extract,+ "score": document.score,+ "term": document.term,+ },+ sort_keys=True,+ )def enqueue_for_purge(documents: Iterable[Document], redis_client: Optional[redis.Redis] = None) -> int:@@ -108,8 +112,7 @@ def enqueue_for_purge(documents: Iterable[Document], redis_client: Optional[reditry:client = redis_client if redis_client is not None else get_redis()if client.scard(PURGE_QUEUE_KEY) >= MAX_QUEUE_SIZE:- logger.warning("Blacklist purge queue is full (%d); dropping %d documents",- MAX_QUEUE_SIZE, len(payloads))+ logger.warning("Blacklist purge queue is full (%d); dropping %d documents", MAX_QUEUE_SIZE, len(payloads))return 0added = client.sadd(PURGE_QUEUE_KEY, *payloads)except Exception:@@ -130,9 +133,15 @@ def _documents_from_payloads(payloads: Iterable[str]) -> list[Document]:fields = json.loads(payload)# "term" is fetched with .get because entries queued before it joined the# payload are still in Redis; those purge by token pages alone, as they did.- documents.append(Document(title=fields["title"], url=fields["url"],- extract=fields["extract"], score=fields["score"],- term=fields.get("term")))+ documents.append(+ Document(+ title=fields["title"],+ url=fields["url"],+ extract=fields["extract"],+ score=fields["score"],+ term=fields.get("term"),+ )+ )except (ValueError, KeyError, TypeError):logger.warning("Discarding unreadable purge queue entry: %r", payload)return documentsdiff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex aa06815..e583a06 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -1,6 +1,6 @@import loggingfrom collections import defaultdict-from datetime import datetime, timezone, timedelta+from datetime import datetime, timedelta, timezonefrom logging import getLoggerfrom pathlib import Pathfrom time import sleep@@ -12,7 +12,7 @@ from redis import Redisfrom mwmbl.crawler.batch import HashedBatch, Linkfrom mwmbl.crawler.domains import DomainLinkDatabase-from mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURL+from mwmbl.crawler.urls import FoundURL, URLDatabase, URLStatusfrom mwmbl.indexer import process_batchfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist import get_default_blacklist_provider@@ -35,8 +35,9 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):def run(batch_cache: BatchCache, new_item_queue: RedisURLQueue, num_batches: int = 1000):- process_batch.run(batch_cache, BatchStatus.LOCAL, BatchStatus.URLS_UPDATED, record_urls_in_database, num_batches,- new_item_queue)+ process_batch.run(+ batch_cache, BatchStatus.LOCAL, BatchStatus.URLS_UPDATED, record_urls_in_database, num_batches, new_item_queue+ )def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: RedisURLQueue):@@ -45,8 +46,10 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Re# crawler is the live caller of this, and it fetches them over HTTP.blacklist_provider = get_default_blacklist_provider(new_item_queue.get_curated_domains_function)blacklist_retrieval_time = datetime.utcnow() - start- logger.info(f"Recording URLs in database for {len(batches)} batches, blacklist provider ready "- f"in {blacklist_retrieval_time.total_seconds()} seconds")+ logger.info(+ f"Recording URLs in database for {len(batches)} batches, blacklist provider ready "+ f"in {blacklist_retrieval_time.total_seconds()} seconds"+ )url_users = {}url_timestamps = {}@@ -67,11 +70,21 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Relogger.info(f"Couldn't parse URL {item.url}")continuefor link in item.content.all_links:- process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,- blacklist_provider, domain_links)-- found_urls = [FoundURL(url, url_users[url], url_statuses[url], url_timestamps[url])- for url in url_statuses.keys() | url_users.keys()]+ process_link(+ batch.user_id_hash,+ crawled_page_domain,+ link,+ timestamp,+ url_timestamps,+ url_users,+ blacklist_provider,+ domain_links,+ )++ found_urls = [+ FoundURL(url, url_users[url], url_statuses[url], url_timestamps[url])+ for url in url_statuses.keys() | url_users.keys()+ ]logger.info(f"Found URLs, {len(found_urls)}")@@ -93,24 +106,34 @@ def _add_found_urls_to_db_and_queue(found_urls, new_item_queue):def add_hn_links(new_item_queue: RedisURLQueue):- hn_count = new_item_queue.get_domain_count('news.ycombinator.com')- max_count = get_domain_max_urls('news.ycombinator.com')+ hn_count = new_item_queue.get_domain_count("news.ycombinator.com")+ max_count = get_domain_max_urls("news.ycombinator.com")- max_item = requests.get('https://hacker-news.firebaseio.com/v0/maxitem.json').json()+ max_item = requests.get("https://hacker-news.firebaseio.com/v0/maxitem.json").json()logger.info(f"Adding HN links, current count: {hn_count}, max count: {max_count}, max item ID: {max_item}")num_items_to_add = 100while hn_count < max_count:- urls = [f'https://news.ycombinator.com/item?id={item_id}' for item_id in range(max_item, max_item - num_items_to_add, -1)]- found_urls = [FoundURL(url, 'hn', URLStatus.NEW, datetime.now()) for url in urls]+ urls = [+ f"https://news.ycombinator.com/item?id={item_id}"+ for item_id in range(max_item, max_item - num_items_to_add, -1)+ ]+ found_urls = [FoundURL(url, "hn", URLStatus.NEW, datetime.now()) for url in urls]_add_found_urls_to_db_and_queue(found_urls, new_item_queue)- hn_count = new_item_queue.get_domain_count('news.ycombinator.com')+ hn_count = new_item_queue.get_domain_count("news.ycombinator.com")max_item -= num_items_to_addlogger.info(f"Added {len(urls)} HN links, {hn_count} total, max item ID: {max_item}")-def process_link(user_id_hash: str, crawled_page_domain: str, link: Link, timestamp: datetime,- url_timestamps: dict[str, datetime], url_users: dict[str, str], blacklist_provider,- domain_links: dict[str, set[str]]):+def process_link(+ user_id_hash: str,+ crawled_page_domain: str,+ link: Link,+ timestamp: datetime,+ url_timestamps: dict[str, datetime],+ url_users: dict[str, str],+ blacklist_provider,+ domain_links: dict[str, set[str]],+):try:parsed_link = parse_url(link.url)except ValueError:@@ -127,7 +150,7 @@ def process_link(user_id_hash: str, crawled_page_domain: str, link: Link, timesturl_users[link.url] = user_id_hashurl_timestamps[link.url] = timestamp- root_url = f'{parsed_link.scheme}://{parsed_link.netloc}/'+ root_url = f"{parsed_link.scheme}://{parsed_link.netloc}/"url_users[root_url] = user_id_hashurl_timestamps[root_url] = timestampdomain_links[crawled_page_domain].add(parsed_link.netloc)diff --git a/mwmbl/justext/core.py b/mwmbl/justext/core.pyindex 63a1eaa..da72576 100644--- a/mwmbl/justext/core.py+++ b/mwmbl/justext/core.py@@ -6,19 +6,17 @@ Copyright (c) 2011 Jan PomikalekThis software is licensed as described in the file LICENSE.rst."""-from __future__ import absolute_import-from __future__ import division, print_function, unicode_literals+from __future__ import absolute_import, division, print_function, unicode_literalsimport refrom contextlib import contextmanager+from functools import lru_cache+from xml.sax.handler import ContentHandlerimport lxml.htmlimport lxml.sax--from functools import lru_cache-from lxml.html.clean import Cleaner-from xml.sax.handler import ContentHandler+from .paragraph import Paragraphfrom .utils import is_blank@@ -33,16 +31,46 @@ NO_HEADINGS_DEFAULT = False# Short and near-good headings within MAX_HEADING_DISTANCE characters before# a good paragraph are classified as good unless --no-headings is specified.MAX_HEADING_DISTANCE_DEFAULT = 200-PARAGRAPH_TAGS = frozenset({- 'body', 'blockquote', 'caption', 'center', 'col', 'colgroup', 'dd',- 'div', 'dl', 'dt', 'fieldset', 'form', 'legend', 'optgroup', 'option',- 'p', 'pre', 'table', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr',- 'ul', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',-})-DEFAULT_ENCODING = 'utf8'-DEFAULT_ENC_ERRORS = 'replace'-CHARSET_META_TAG_PATTERN = re.compile(br"""<meta[^>]+charset=["']?([^'"/>\s]+)""", re.IGNORECASE)-GOOD_OR_BAD = {'good', 'bad'}+PARAGRAPH_TAGS = frozenset(+ {+ "body",+ "blockquote",+ "caption",+ "center",+ "col",+ "colgroup",+ "dd",+ "div",+ "dl",+ "dt",+ "fieldset",+ "form",+ "legend",+ "optgroup",+ "option",+ "p",+ "pre",+ "table",+ "td",+ "textarea",+ "tfoot",+ "th",+ "thead",+ "tr",+ "ul",+ "li",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ }+)+DEFAULT_ENCODING = "utf8"+DEFAULT_ENC_ERRORS = "replace"+CHARSET_META_TAG_PATTERN = re.compile(rb"""<meta[^>]+charset=["']?([^'"/>\s]+)""", re.IGNORECASE)+GOOD_OR_BAD = {"good", "bad"}@contextmanager@@ -52,6 +80,7 @@ def ignored(*exceptions):except tuple(exceptions):pass+class JustextError(Exception):"Base class for jusText exceptions."@@ -177,11 +206,11 @@ class ParagraphMaker(ContentHandler):else:self.br = bool(name == "br")if self.br:- self.paragraph.append_text(' ')- elif name == 'a':+ self.paragraph.append_text(" ")+ elif name == "a":self.link = Truetry:- self.paragraph.links.add(attrs.getValueByQName('href').strip())+ self.paragraph.links.add(attrs.getValueByQName("href").strip())except KeyError:passself.paragraph.tags_count += 1@@ -192,7 +221,7 @@ class ParagraphMaker(ContentHandler):if name in PARAGRAPH_TAGS:self._start_new_pragraph()- if name == 'a':+ if name == "a":self.link = Falsedef endDocument(self):@@ -250,10 +279,16 @@ def define_stoplist(stoplist):return stoplist-def classify_paragraphs(paragraphs, stoplist, length_low=LENGTH_LOW_DEFAULT,- length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,- stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,- no_headings=NO_HEADINGS_DEFAULT):+def classify_paragraphs(+ paragraphs,+ stoplist,+ length_low=LENGTH_LOW_DEFAULT,+ length_high=LENGTH_HIGH_DEFAULT,+ stopwords_low=STOPWORDS_LOW_DEFAULT,+ stopwords_high=STOPWORDS_HIGH_DEFAULT,+ max_link_density=MAX_LINK_DENSITY_DEFAULT,+ no_headings=NO_HEADINGS_DEFAULT,+):"Context-free paragraph classification."stoplist = define_stoplist(stoplist)@@ -264,25 +299,25 @@ def classify_paragraphs(paragraphs, stoplist, length_low=LENGTH_LOW_DEFAULT,paragraph.heading = bool(not no_headings and paragraph.is_heading)if link_density > max_link_density:- paragraph.cf_class = 'bad'- elif ('\xa9' in paragraph.text) or ('©' in paragraph.text):- paragraph.cf_class = 'bad'- elif 'select' in paragraph.dom_path:- paragraph.cf_class = 'bad'+ paragraph.cf_class = "bad"+ elif ("\xa9" in paragraph.text) or ("©" in paragraph.text):+ paragraph.cf_class = "bad"+ elif "select" in paragraph.dom_path:+ paragraph.cf_class = "bad"elif length < length_low:if paragraph.chars_count_in_links > 0:- paragraph.cf_class = 'bad'+ paragraph.cf_class = "bad"else:- paragraph.cf_class = 'short'+ paragraph.cf_class = "short"elif stopword_density >= stopwords_high:if length > length_high:- paragraph.cf_class = 'good'+ paragraph.cf_class = "good"else:- paragraph.cf_class = 'neargood'+ paragraph.cf_class = "neargood"elif stopword_density >= stopwords_low:- paragraph.cf_class = 'neargood'+ paragraph.cf_class = "neargood"else:- paragraph.cf_class = 'bad'+ paragraph.cf_class = "bad"def _get_neighbour(i, paragraphs, ignore_neargood, inc, boundary):@@ -291,9 +326,9 @@ def _get_neighbour(i, paragraphs, ignore_neargood, inc, boundary):c = paragraphs[i].class_typeif c in GOOD_OR_BAD:return c- if c == 'neargood' and not ignore_neargood:+ if c == "neargood" and not ignore_neargood:return c- return 'bad'+ return "bad"def get_prev_neighbour(i, paragraphs, ignore_neargood):@@ -324,13 +359,13 @@ def revise_paragraph_classification(paragraphs, max_heading_distance=MAX_HEADINGfor i, paragraph in enumerate(paragraphs):# copy classesparagraph.class_type = paragraph.cf_class- if not (paragraph.heading and paragraph.class_type == 'short'):+ if not (paragraph.heading and paragraph.class_type == "short"):continuej = i + 1distance = 0while j < len(paragraphs) and distance <= max_heading_distance:- if paragraphs[j].class_type == 'good':- paragraph.class_type = 'neargood'+ if paragraphs[j].class_type == "good":+ paragraph.class_type = "neargood"breakdistance += len(paragraphs[j].text)j += 1@@ -338,54 +373,62 @@ def revise_paragraph_classification(paragraphs, max_heading_distance=MAX_HEADING# classify shortnew_classes = {}for i, paragraph in enumerate(paragraphs):- if paragraph.class_type != 'short':+ if paragraph.class_type != "short":continueprev_neighbour = get_prev_neighbour(i, paragraphs, ignore_neargood=True)next_neighbour = get_next_neighbour(i, paragraphs, ignore_neargood=True)- if prev_neighbour == 'good' and next_neighbour == 'good':- new_classes[i] = 'good'- elif prev_neighbour == 'bad' and next_neighbour == 'bad':- new_classes[i] = 'bad'+ if prev_neighbour == "good" and next_neighbour == "good":+ new_classes[i] = "good"+ elif prev_neighbour == "bad" and next_neighbour == "bad":+ new_classes[i] = "bad"# it must be set(['good', 'bad'])- elif (prev_neighbour == 'bad' and get_prev_neighbour(i, paragraphs, ignore_neargood=False) == 'neargood') or \- (next_neighbour == 'bad' and get_next_neighbour(i, paragraphs, ignore_neargood=False) == 'neargood'):- new_classes[i] = 'good'+ elif (prev_neighbour == "bad" and get_prev_neighbour(i, paragraphs, ignore_neargood=False) == "neargood") or (+ next_neighbour == "bad" and get_next_neighbour(i, paragraphs, ignore_neargood=False) == "neargood"+ ):+ new_classes[i] = "good"else:- new_classes[i] = 'bad'+ new_classes[i] = "bad"for i, c in new_classes.items():paragraphs[i].class_type = c# revise neargoodfor i, paragraph in enumerate(paragraphs):- if paragraph.class_type != 'neargood':+ if paragraph.class_type != "neargood":continueprev_neighbour = get_prev_neighbour(i, paragraphs, ignore_neargood=True)next_neighbour = get_next_neighbour(i, paragraphs, ignore_neargood=True)- if (prev_neighbour, next_neighbour) == ('bad', 'bad'):- paragraph.class_type = 'bad'+ if (prev_neighbour, next_neighbour) == ("bad", "bad"):+ paragraph.class_type = "bad"else:- paragraph.class_type = 'good'+ paragraph.class_type = "good"# more good headingsfor i, paragraph in enumerate(paragraphs):- if not (paragraph.heading and paragraph.class_type == 'bad' and paragraph.cf_class != 'bad'):+ if not (paragraph.heading and paragraph.class_type == "bad" and paragraph.cf_class != "bad"):continuej = i + 1distance = 0while j < len(paragraphs) and distance <= max_heading_distance:- if paragraphs[j].class_type == 'good':- paragraph.class_type = 'good'+ if paragraphs[j].class_type == "good":+ paragraph.class_type = "good"breakdistance += len(paragraphs[j].text)j += 1-def justext_from_dom(dom, stoplist, length_low=LENGTH_LOW_DEFAULT,- length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,- stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,- max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT, no_headings=NO_HEADINGS_DEFAULT,- preprocessor=preprocessor):+def justext_from_dom(+ dom,+ stoplist,+ length_low=LENGTH_LOW_DEFAULT,+ length_high=LENGTH_HIGH_DEFAULT,+ stopwords_low=STOPWORDS_LOW_DEFAULT,+ stopwords_high=STOPWORDS_HIGH_DEFAULT,+ max_link_density=MAX_LINK_DENSITY_DEFAULT,+ max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT,+ no_headings=NO_HEADINGS_DEFAULT,+ preprocessor=preprocessor,+):"""Converts an HTML page into a list of classified paragraphs. Each paragraphis represented as instance of class ˙˙justext.paragraph.Paragraph˙˙.@@ -394,8 +437,9 @@ def justext_from_dom(dom, stoplist, length_low=LENGTH_LOW_DEFAULT,paragraphs = ParagraphMaker.make_paragraphs(dom_preprocessed)- classify_paragraphs(paragraphs, stoplist, length_low, length_high,- stopwords_low, stopwords_high, max_link_density, no_headings)+ classify_paragraphs(+ paragraphs, stoplist, length_low, length_high, stopwords_low, stopwords_high, max_link_density, no_headings+ )revise_paragraph_classification(paragraphs, max_heading_distance)return paragraphsdiff --git a/mwmbl/justext/paragraph.py b/mwmbl/justext/paragraph.pyindex 9d5ebde..19ace29 100644--- a/mwmbl/justext/paragraph.py+++ b/mwmbl/justext/paragraph.py@@ -1,18 +1,17 @@# -*- coding: utf-8 -*--from __future__ import absolute_import-from __future__ import division, print_function, unicode_literals+from __future__ import absolute_import, division, print_function, unicode_literalsimport refrom .utils import normalize_whitespace-HEADINGS_PATTERN = re.compile(r"\bh\d\b")class Paragraph(object):"""Object representing one block of text in HTML."""+def __init__(self, path):self.dom_path = path.domself.xpath = path.xpathdiff --git a/mwmbl/justext/utils.py b/mwmbl/justext/utils.pyindex 32ece57..955a876 100644--- a/mwmbl/justext/utils.py+++ b/mwmbl/justext/utils.py@@ -1,12 +1,10 @@# -*- coding: utf-8 -*--from __future__ import absolute_import-from __future__ import division, print_function, unicode_literals+from __future__ import absolute_import, division, print_function, unicode_literals-import reimport os-import sysimport pkgutil+import refrom pathlib import PathMULTIPLE_WHITESPACE_PATTERN = re.compile(r"\s+", re.UNICODE)diff --git a/mwmbl/main.py b/mwmbl/main.pyindex bb329f1..47cda9e 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -44,11 +44,12 @@ def run():django.setup()from django.conf import settings+from mwmbl import background- from mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.count_urls import count_urls_continuously- from mwmbl.indexer.update_urls import update_urls_continuouslyfrom mwmbl.curated_domains import get_curated_domains+ from mwmbl.indexer.update_urls import update_urls_continuously+ from mwmbl.redis_url_queue import RedisURLQueueif settings.STATIC_ROOT:call_command("collectstatic", "--clear", "--noinput")@@ -87,7 +88,8 @@ def run():# *scheduled*: apps.ready() writes a Task row and nothing executes it. Every# periodic task in production had been pending, unattempted, for months.process = multiprocessing.get_context("spawn").Process(- target=run_background_tasks, name="background-tasks", daemon=True)+ target=run_background_tasks, name="background-tasks", daemon=True+ )process.start()logger.info("Started the background task queue (pid %d)", process.pid)@@ -103,6 +105,7 @@ def run():def load(self):from mwmbl.asgi import application+return applicationGunicornApp().run()diff --git a/mwmbl/management/commands/backfill_domain_evidence.py b/mwmbl/management/commands/backfill_domain_evidence.pyindex 028c0a8..93501bd 100644--- a/mwmbl/management/commands/backfill_domain_evidence.py+++ b/mwmbl/management/commands/backfill_domain_evidence.py@@ -15,12 +15,12 @@ running it again picks up where it stopped.made up to two years ago, so the further back you reach the more the page text describes asite that has changed since a moderator judged it."""+import timefrom django.core.management.base import BaseCommand-from django.utils.dateparse import parse_date-from django.db import transaction+from django.utils.dateparse import parse_datefrom mwmbl.crawler.env_vars import CRAWL_DELAY_SECONDSfrom mwmbl.indexer.blacklist_snapshot import get_redis@@ -34,12 +34,12 @@ class Command(BaseCommand):help = "Crawl submitted domains that have no moderation evidence yet"def add_arguments(self, parser):- parser.add_argument("--status", action="append", default=None,- choices=list(DomainSubmission.DOMAIN_SUBMISSION_STATUS))+ parser.add_argument(+ "--status", action="append", default=None, choices=list(DomainSubmission.DOMAIN_SUBMISSION_STATUS)+ )parser.add_argument("--since", type=parse_date, default=None)parser.add_argument("--limit", type=int, default=None)- parser.add_argument("--delay", type=float, default=CRAWL_DELAY_SECONDS,- help="Seconds to wait between domains")+ parser.add_argument("--delay", type=float, default=CRAWL_DELAY_SECONDS, help="Seconds to wait between domains")def handle(self, *args, **options):domains = self._domains_to_crawl(options)@@ -54,8 +54,8 @@ class Command(BaseCommand):evidence = store_evidence(domain, crawl_domain(domain, redis))refresh_suggestion(evidence)self.stdout.write(- f"[{number}/{len(domains)}] {domain}: {evidence.suggested_action or '?'} "- f"({evidence.state})")+ f"[{number}/{len(domains)}] {domain}: {evidence.suggested_action or '?'} ({evidence.state})"+ )except Exception as exception:# One unfetchable domain must not end a backfill of thousands.store_failure(domain, type(exception).__name__)diff --git a/mwmbl/management/commands/export_judgments.py b/mwmbl/management/commands/export_judgments.pyindex 21dad81..3824ea4 100644--- a/mwmbl/management/commands/export_judgments.py+++ b/mwmbl/management/commands/export_judgments.py@@ -19,6 +19,7 @@ across the three files so per-user noise filtering still works downstream), andresult documents keep only url/title/extract/score/state — ``user_ids`` and``term`` are dropped."""+import gzipimport jsonfrom collections import Counter@@ -37,11 +38,7 @@ def sanitize_results(results) -> list[dict]:"""Keep only the document fields safe and useful for training."""if not isinstance(results, list):return []- return [- {field: doc.get(field) for field in DOCUMENT_FIELDS}- for doc in results- if isinstance(doc, dict)- ]+ return [{field: doc.get(field) for field in DOCUMENT_FIELDS} for doc in results if isinstance(doc, dict)]def query_from_results_url(url: str) -> str | None:@@ -59,14 +56,20 @@ class Command(BaseCommand):def add_arguments(self, parser):parser.add_argument(- "--output-dir", default="devdata/judgments",- help="Directory to write the export files to (created if missing)")+ "--output-dir",+ default="devdata/judgments",+ help="Directory to write the export files to (created if missing)",+ )parser.add_argument(- "--since", type=parse_date, default=None, metavar="YYYY-MM-DD",- help="Only export records with timestamp on or after this date")+ "--since",+ type=parse_date,+ default=None,+ metavar="YYYY-MM-DD",+ help="Only export records with timestamp on or after this date",+ )parser.add_argument(- "--stats-only", action="store_true",- help="Only compute and print the stats summary; write no data files")+ "--stats-only", action="store_true", help="Only compute and print the stats summary; write no data files"+ )def handle(self, *args, **options):output_dir = Path(options["output_dir"])@@ -107,8 +110,7 @@ class Command(BaseCommand):if since:flag_queryset = flag_queryset.filter(curation__timestamp__date__gte=since)for flag in flag_queryset:- flags_by_curation.setdefault(flag.curation_id, []).append(- {"flag": flag.flag, "status": flag.status})+ flags_by_curation.setdefault(flag.curation_id, []).append({"flag": flag.flag, "status": flag.status})count = 0no_ops = 0@@ -155,8 +157,9 @@ class Command(BaseCommand):"queries_curated_more_than_once": sum(1 for c in query_counts.values() if c > 1),"no_op_curations": no_ops,"num_changes_distribution": dict(sorted(num_changes.items())[:20]),- "mean_original_results_length": round(- sum(results_lengths) / len(results_lengths), 1) if results_lengths else None,+ "mean_original_results_length": round(sum(results_lengths) / len(results_lengths), 1)+ if results_lengths+ else None,"timestamp_range": timestamps.range(),"flag_status_counts": dict(flag_status_counts),"flag_type_counts": dict(flag_type_counts),@@ -239,7 +242,6 @@ class Command(BaseCommand):"timestamp_range": timestamps.range(),}-def export_domains(self, writer, anon, since):queryset = DomainSubmission.objects.order_by("id")if since:@@ -301,8 +303,7 @@ class AnonymousUserIds:return {"distinct_users": len(self._ids),"total_contributions": total,- "top_10_user_share": round(- sum(count for _, count in top) / total, 3) if total else None,+ "top_10_user_share": round(sum(count for _, count in top) / total, 3) if total else None,"top_10_user_contributions": [count for _, count in top],}diff --git a/mwmbl/management/commands/train_domain_moderation_model.py b/mwmbl/management/commands/train_domain_moderation_model.pyindex e772276..34f27f1 100644--- a/mwmbl/management/commands/train_domain_moderation_model.py+++ b/mwmbl/management/commands/train_domain_moderation_model.py@@ -16,6 +16,7 @@ Publishing writes a ModerationModelArtifact row rather than a file: the incumbenreads and the model the workers serve are then the same object, and neither is lost to adeploy. See mwmbl.moderation.model."""+import jsonfrom datetime import date@@ -45,29 +46,45 @@ class Command(BaseCommand):help = "Train the domain moderation approve/reject suggester"def add_arguments(self, parser):- parser.add_argument("--dry-run", action="store_true",- help="Train and report metrics without writing the artifact")- parser.add_argument("--force", action="store_true",- help="Publish even if the gate fails")parser.add_argument(- "--no-text", action="store_true", dest="no_text",- help="Fit on the domain name alone, ignoring the crawled page text vocabulary")+ "--dry-run", action="store_true", help="Train and report metrics without writing the artifact"+ )+ parser.add_argument("--force", action="store_true", help="Publish even if the gate fails")+ parser.add_argument(+ "--no-text",+ action="store_true",+ dest="no_text",+ help="Fit on the domain name alone, ignoring the crawled page text vocabulary",+ )parser.add_argument(- "--has-text", action="store_true", dest="has_text",- help=("Add the was-it-crawled indicator, which is off by default because it "- "measured worse on every metric: evidence is crawled newest-first, so it "- "doubles as a proxy for recency. Worth re-measuring with --ablate once "- "backfill_domain_evidence has evened out the coverage"))+ "--has-text",+ action="store_true",+ dest="has_text",+ help=(+ "Add the was-it-crawled indicator, which is off by default because it "+ "measured worse on every metric: evidence is crawled newest-first, so it "+ "doubles as a proxy for recency. Worth re-measuring with --ablate once "+ "backfill_domain_evidence has evened out the coverage"+ ),+ )parser.add_argument(- "--ablate", action="store_true",- help=("Also train with the has_text indicator and without the page-text "- "vocabulary, and report all three on the same split, so each feature is "- "measured rather than guessed"))+ "--ablate",+ action="store_true",+ help=(+ "Also train with the has_text indicator and without the page-text "+ "vocabulary, and report all three on the same split, so each feature is "+ "measured rather than guessed"+ ),+ )parser.add_argument(- "--derived", action="store_true",- help=("Add blocklist-derived rows for reason classes short of real data. Off by "- "default: measured at 100 rows this drops real SPAM F1 from 0.839 to 0.688, "- "and OFFENSIVE is already covered exactly by the blocklist check."))+ "--derived",+ action="store_true",+ help=(+ "Add blocklist-derived rows for reason classes short of real data. Off by "+ "default: measured at 100 rows this drops real SPAM F1 from 0.839 to 0.688, "+ "and OFFENSIVE is already covered exactly by the blocklist check."+ ),+ )def handle(self, *args, **options):rows = training_data.real_rows()@@ -83,16 +100,17 @@ class Command(BaseCommand):if needed:self.stdout.write(f"Reason classes with too little real data: {sorted(needed)}")rows += training_data.derived_rows(- needed, DERIVED_ROWS_PER_REASON,- exclude={row.domain for row in rows})+ needed, DERIVED_ROWS_PER_REASON, exclude={row.domain for row in rows}+ )version = f"domain-mod-{date.today():%Y-%m-%d}"# The model the workers are serving right now, so the gate can score it on the same# held-out rows as the candidate. None on a first run, and on an artifact this code can# no longer featurise - the gate falls back to the stored metrics and says so.incumbent = load_published_model()- model, metrics = train(rows, version, use_text=not options["no_text"],- use_has_text=options["has_text"], incumbent=incumbent)+ model, metrics = train(+ rows, version, use_text=not options["no_text"], use_has_text=options["has_text"], incumbent=incumbent+ )self.stdout.write(json.dumps(metrics, indent=2))if options["ablate"]:@@ -109,8 +127,7 @@ class Command(BaseCommand):returnpublish(model, metrics)- self.stdout.write(self.style.SUCCESS(- f"Published {model.version}; every worker picks it up within a minute."))+ self.stdout.write(self.style.SUCCESS(f"Published {model.version}; every worker picks it up within a minute."))rescore_pending_submissions(schedule=0)self.stdout.write("Scheduled a rescore of the pending queue with the new model.")@@ -130,11 +147,12 @@ class Command(BaseCommand):_, with_has_text = train(rows, f"{version}-has-text", use_has_text=True)self.stdout.write("\nAblation - are the page-text features earning their place?")- self.stdout.write(f" {'':>22} {'norm AP':>17} {'reject P':>16} "- f"{'reject R':>16} {'unsure':>7}")- for name, metrics in (("text (shipped)", full),- ("text + has_text", with_has_text),- ("domain name only", no_text)):+ self.stdout.write(f" {'':>22} {'norm AP':>17} {'reject P':>16} {'reject R':>16} {'unsure':>7}")+ for name, metrics in (+ ("text (shipped)", full),+ ("text + has_text", with_has_text),+ ("domain name only", no_text),+ ):cold = metrics.get("cold_start")if cold is None:self.stdout.write(f" {name:>22}: no cold-start slice")@@ -144,7 +162,8 @@ class Command(BaseCommand):f" {name:>22} {self._with_interval(cold['normalised_ap'], cold['normalised_ap_ci'])} "f"{self._with_interval(at['reject_precision'], at['reject_precision_ci'])} "f"{self._with_interval(at['reject_recall'], at['reject_recall_ci'])} "- f"{at['unsure_share']:>7.3f}")+ f"{at['unsure_share']:>7.3f}"+ )self.stdout.write("\n " + self._coverage_note(full) + "\n")@@ -163,14 +182,18 @@ class Command(BaseCommand):train_share = metrics.get("train_rows_with_text", 0) / train_rows if train_rows else 0test_share = (cold.get("rows_with_text", 0) / cold["rows"]) if cold.get("rows") else 0- coverage = (f"{metrics.get('train_rows_with_text', 0)} of {train_rows} training rows "- f"({train_share:.0%}) and {cold.get('rows_with_text', 0)} of "- f"{cold.get('rows', 0)} cold-start test rows ({test_share:.0%}) have text.")+ coverage = (+ f"{metrics.get('train_rows_with_text', 0)} of {train_rows} training rows "+ f"({train_share:.0%}) and {cold.get('rows_with_text', 0)} of "+ f"{cold.get('rows', 0)} cold-start test rows ({test_share:.0%}) have text."+ )if abs(train_share - test_share) > COVERAGE_GAP_TOLERANCE:- return (coverage + "\n That gap makes has_text a proxy for recency rather than "- "for evidence, and skews the text block's\n vocabulary towards the test "- "period. Run backfill_domain_evidence over the older submissions\n and "- "re-read this table before drawing conclusions from it.")+ return (+ coverage + "\n That gap makes has_text a proxy for recency rather than "+ "for evidence, and skews the text block's\n vocabulary towards the test "+ "period. Run backfill_domain_evidence over the older submissions\n and "+ "re-read this table before drawing conclusions from it."+ )return coverage + " Coverage is even, so these rows compare like for like."@staticmethod@@ -188,5 +211,8 @@ class Command(BaseCommand):for row in rows:if row.source == training_data.REAL and row.rejected and row.reason:counts[row.reason] = counts.get(row.reason, 0) + 1- return {reason for reason in DomainSubmission.DOMAIN_REJECTION_REASON- if counts.get(reason, 0) < MIN_REAL_ROWS_PER_REASON}+ return {+ reason+ for reason in DomainSubmission.DOMAIN_REJECTION_REASON+ if counts.get(reason, 0) < MIN_REAL_ROWS_PER_REASON+ }diff --git a/mwmbl/management/commands/train_super_search_xgb.py b/mwmbl/management/commands/train_super_search_xgb.pyindex 9d90843..c107030 100644--- a/mwmbl/management/commands/train_super_search_xgb.py+++ b/mwmbl/management/commands/train_super_search_xgb.py@@ -17,6 +17,7 @@ The artifact directory defaults to ``settings.SUPER_SEARCH_XGB_MODEL_DIR``;pass ``--out mwmbl/tinysearchengine/super_search_select/artifacts/xgb`` torefresh the bundled warm-start model."""+import jsonfrom pathlib import Path@@ -42,9 +43,7 @@ def profiles_from_fetch(path: Path, dim: int) -> dict:prof_cng.setdefault(name, np.zeros(dim))prof_bow[name] += vectors.project_bow(text, dim)prof_cng[name] += vectors.project_char_ngrams(text, dim)- return {name: (vectors._l2_normalise(prof_bow[name]),- vectors._l2_normalise(prof_cng[name]))- for name in prof_bow}+ return {name: (vectors._l2_normalise(prof_bow[name]), vectors._l2_normalise(prof_cng[name])) for name in prof_bow}class Command(BaseCommand):@@ -52,21 +51,23 @@ class Command(BaseCommand):def add_arguments(self, parser):source = parser.add_mutually_exclusive_group(required=True)- source.add_argument("--from-matrix", metavar="PATH",- help="RewardMatrix path (without extension)")- source.add_argument("--days", type=int,- help="train from SuperSearchImpression rows of the last N days")- parser.add_argument("--min-rows", type=int,- default=settings.SUPER_SEARCH_XGB_MIN_TRAIN_ROWS)- parser.add_argument("--out", default=settings.SUPER_SEARCH_XGB_MODEL_DIR,- help="artifact directory (default: SUPER_SEARCH_XGB_MODEL_DIR)")- parser.add_argument("--reward-kind", default="judge",- help="provenance label stored in meta.json for --from-matrix")- parser.add_argument("--profiles-from", metavar="JSONL",- help="fetch checkpoint to build profiles.npz from "- "(default: <matrix>.fetch.jsonl)")- parser.add_argument("--no-profiles", action="store_true",- help="skip writing profiles.npz for --from-matrix")+ source.add_argument("--from-matrix", metavar="PATH", help="RewardMatrix path (without extension)")+ source.add_argument("--days", type=int, help="train from SuperSearchImpression rows of the last N days")+ parser.add_argument("--min-rows", type=int, default=settings.SUPER_SEARCH_XGB_MIN_TRAIN_ROWS)+ parser.add_argument(+ "--out",+ default=settings.SUPER_SEARCH_XGB_MODEL_DIR,+ help="artifact directory (default: SUPER_SEARCH_XGB_MODEL_DIR)",+ )+ parser.add_argument(+ "--reward-kind", default="judge", help="provenance label stored in meta.json for --from-matrix"+ )+ parser.add_argument(+ "--profiles-from",+ metavar="JSONL",+ help="fetch checkpoint to build profiles.npz from (default: <matrix>.fetch.jsonl)",+ )+ parser.add_argument("--no-profiles", action="store_true", help="skip writing profiles.npz for --from-matrix")def handle(self, *args, **options):import numpy as np@@ -76,23 +77,24 @@ class Command(BaseCommand):X, y, vocab, source_means = xgb_model.build_training_data_from_matrix(matrix)if len(y) < options["min_rows"]:raise CommandError(- f"only {len(y)} training pairs in the matrix, "- f"need {options['min_rows']} (--min-rows)")+ f"only {len(y)} training pairs in the matrix, need {options['min_rows']} (--min-rows)"+ )model = xgb_model.train(X, y)metrics = {"train_rmse": float(np.sqrt(np.mean((model.predict(X) - y) ** 2)))}- xgb_model.save_artifact(model, vocab, options["out"],- reward_kind=options["reward_kind"],- n_rows=len(y), metrics=metrics,- source_reward_means=source_means)+ xgb_model.save_artifact(+ model,+ vocab,+ options["out"],+ reward_kind=options["reward_kind"],+ n_rows=len(y),+ metrics=metrics,+ source_reward_means=source_means,+ )if not options["no_profiles"]:- fetch = Path(options["profiles_from"]- or f"{options['from_matrix']}.fetch.jsonl")+ fetch = Path(options["profiles_from"] or f"{options['from_matrix']}.fetch.jsonl")if not fetch.exists():- raise CommandError(- f"fetch checkpoint {fetch} not found; pass --profiles-from "- f"or --no-profiles")- profiles = profiles_from_fetch(- fetch, settings.SUPER_SEARCH_PROJECTION_DIM)+ raise CommandError(f"fetch checkpoint {fetch} not found; pass --profiles-from or --no-profiles")+ profiles = profiles_from_fetch(fetch, settings.SUPER_SEARCH_PROJECTION_DIM)xgb_model.save_profiles(profiles, options["out"])self.stdout.write(f"wrote profiles.npz for {len(profiles)} sources")else:@@ -102,7 +104,5 @@ class Command(BaseCommand):out_dir=options["out"],)if metrics is None:- raise CommandError("not enough impression pairs to train "- "(see --min-rows)")- self.stdout.write(self.style.SUCCESS(- f"trained xgb source model -> {options['out']} ({metrics})"))+ raise CommandError("not enough impression pairs to train (see --min-rows)")+ self.stdout.write(self.style.SUCCESS(f"trained xgb source model -> {options['out']} ({metrics})"))diff --git a/mwmbl/migrations/0001_initial.py b/mwmbl/migrations/0001_initial.pyindex 1dc8d43..379dc17 100644--- a/mwmbl/migrations/0001_initial.py+++ b/mwmbl/migrations/0001_initial.py@@ -1,58 +1,108 @@# Generated by Django 4.2.6 on 2023-10-25 11:55-from django.conf import settingsimport django.contrib.auth.modelsimport django.contrib.auth.validators-from django.db import migrations, modelsimport django.db.models.deletionimport django.utils.timezone+from django.conf import settings+from django.db import migrations, modelsclass Migration(migrations.Migration):-initial = Truedependencies = [- ('auth', '0012_alter_user_first_name_max_length'),+ ("auth", "0012_alter_user_first_name_max_length"),]operations = [migrations.CreateModel(- name='MwmblUser',+ name="MwmblUser",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('password', models.CharField(max_length=128, verbose_name='password')),- ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),- ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),- ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')),- ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')),- ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')),- ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')),- ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')),- ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')),- ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')),- ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')),- ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("password", models.CharField(max_length=128, verbose_name="password")),+ ("last_login", models.DateTimeField(blank=True, null=True, verbose_name="last login")),+ (+ "is_superuser",+ models.BooleanField(+ default=False,+ help_text="Designates that this user has all permissions without explicitly assigning them.",+ verbose_name="superuser status",+ ),+ ),+ (+ "username",+ models.CharField(+ error_messages={"unique": "A user with that username already exists."},+ help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",+ max_length=150,+ unique=True,+ validators=[django.contrib.auth.validators.UnicodeUsernameValidator()],+ verbose_name="username",+ ),+ ),+ ("first_name", models.CharField(blank=True, max_length=150, verbose_name="first name")),+ ("last_name", models.CharField(blank=True, max_length=150, verbose_name="last name")),+ ("email", models.EmailField(blank=True, max_length=254, verbose_name="email address")),+ (+ "is_staff",+ models.BooleanField(+ default=False,+ help_text="Designates whether the user can log into this admin site.",+ verbose_name="staff status",+ ),+ ),+ (+ "is_active",+ models.BooleanField(+ default=True,+ help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",+ verbose_name="active",+ ),+ ),+ ("date_joined", models.DateTimeField(default=django.utils.timezone.now, verbose_name="date joined")),+ (+ "groups",+ models.ManyToManyField(+ blank=True,+ help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",+ related_name="user_set",+ related_query_name="user",+ to="auth.group",+ verbose_name="groups",+ ),+ ),+ (+ "user_permissions",+ models.ManyToManyField(+ blank=True,+ help_text="Specific permissions for this user.",+ related_name="user_set",+ related_query_name="user",+ to="auth.permission",+ verbose_name="user permissions",+ ),+ ),],options={- 'verbose_name': 'user',- 'verbose_name_plural': 'users',- 'abstract': False,+ "verbose_name": "user",+ "verbose_name_plural": "users",+ "abstract": False,},managers=[- ('objects', django.contrib.auth.models.UserManager()),+ ("objects", django.contrib.auth.models.UserManager()),],),migrations.CreateModel(- name='UserCuration',+ name="UserCuration",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('timestamp', models.DateTimeField()),- ('url', models.CharField(max_length=300)),- ('results', models.JSONField()),- ('curation_type', models.CharField(max_length=20)),- ('curation', models.JSONField()),- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("timestamp", models.DateTimeField()),+ ("url", models.CharField(max_length=300)),+ ("results", models.JSONField()),+ ("curation_type", models.CharField(max_length=20)),+ ("curation", models.JSONField()),+ ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),],),]diff --git a/mwmbl/migrations/0002_curation.py b/mwmbl/migrations/0002_curation.pyindex 17a81fb..066e8c5 100644--- a/mwmbl/migrations/0002_curation.py+++ b/mwmbl/migrations/0002_curation.py@@ -1,27 +1,31 @@# Generated by Django 4.2.6 on 2023-12-02 12:42+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletionclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0001_initial'),+ ("mwmbl", "0001_initial"),]operations = [migrations.CreateModel(- name='Curation',+ name="Curation",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('timestamp', models.DateTimeField()),- ('query', models.CharField(max_length=300)),- ('original_results', models.JSONField()),- ('new_results', models.JSONField()),- ('num_changes', models.IntegerField(default=0)),- ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("timestamp", models.DateTimeField()),+ ("query", models.CharField(max_length=300)),+ ("original_results", models.JSONField()),+ ("new_results", models.JSONField()),+ ("num_changes", models.IntegerField(default=0)),+ (+ "user",+ models.ForeignKey(+ null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL+ ),+ ),],),]diff --git a/mwmbl/migrations/0003_auto_20231203_0614.py b/mwmbl/migrations/0003_auto_20231203_0614.pyindex 7a4be6d..7a76399 100644--- a/mwmbl/migrations/0003_auto_20231203_0614.py+++ b/mwmbl/migrations/0003_auto_20231203_0614.py@@ -1,16 +1,16 @@# Generated by Django 4.2.6 on 2023-12-03 06:14from itertools import groupby-from urllib.parse import urlparse, parse_qs+from urllib.parse import parse_qs, urlparsefrom django.db import migrationsdef create_curations_from_user_curation(apps, schema_editor):- Curation = apps.get_model('mwmbl', 'Curation')- UserCuration = apps.get_model('mwmbl', 'UserCuration')+ Curation = apps.get_model("mwmbl", "Curation")+ UserCuration = apps.get_model("mwmbl", "UserCuration")# Order curations by timestamp- curations = UserCuration.objects.all().order_by('timestamp')+ curations = UserCuration.objects.all().order_by("timestamp")# Group by user and urlfor (user, url), curations in groupby(curations, lambda c: (c.user, c.url)):@@ -45,9 +45,8 @@ def create_curations_from_user_curation(apps, schema_editor):class Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0002_curation'),+ ("mwmbl", "0002_curation"),]operations = [diff --git a/mwmbl/migrations/0004_curation_original_index_results.py b/mwmbl/migrations/0004_curation_original_index_results.pyindex 494e1fe..2ab10b9 100644--- a/mwmbl/migrations/0004_curation_original_index_results.py+++ b/mwmbl/migrations/0004_curation_original_index_results.py@@ -4,15 +4,14 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0003_auto_20231203_0614'),+ ("mwmbl", "0003_auto_20231203_0614"),]operations = [migrations.AddField(- model_name='curation',- name='original_index_results',+ model_name="curation",+ name="original_index_results",field=models.JSONField(default=list),),]diff --git a/mwmbl/migrations/0005_oldindex.py b/mwmbl/migrations/0005_oldindex.pyindex 40c9c74..21bf8e9 100644--- a/mwmbl/migrations/0005_oldindex.py+++ b/mwmbl/migrations/0005_oldindex.py@@ -4,19 +4,18 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0004_curation_original_index_results'),+ ("mwmbl", "0004_curation_original_index_results"),]operations = [migrations.CreateModel(- name='OldIndex',+ name="OldIndex",fields=[- ('index_path', models.CharField(max_length=300, primary_key=True, serialize=False)),- ('index_total_pages', models.IntegerField()),- ('last_copied_time', models.DateTimeField()),- ('last_page_copied', models.IntegerField()),+ ("index_path", models.CharField(max_length=300, primary_key=True, serialize=False)),+ ("index_total_pages", models.IntegerField()),+ ("last_copied_time", models.DateTimeField()),+ ("last_page_copied", models.IntegerField()),],),]diff --git a/mwmbl/migrations/0005_oldindex_squashed.py b/mwmbl/migrations/0005_oldindex_squashed.pyindex f7d94d5..984ed59 100644--- a/mwmbl/migrations/0005_oldindex_squashed.py+++ b/mwmbl/migrations/0005_oldindex_squashed.py@@ -4,20 +4,23 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-- replaces = [('mwmbl', '0005_oldindex'), ('mwmbl', '0006_remove_oldindex_index_total_pages_and_more'), ('mwmbl', '0007_alter_oldindex_last_copied_time_and_more')]+ replaces = [+ ("mwmbl", "0005_oldindex"),+ ("mwmbl", "0006_remove_oldindex_index_total_pages_and_more"),+ ("mwmbl", "0007_alter_oldindex_last_copied_time_and_more"),+ ]dependencies = [- ('mwmbl', '0004_curation_original_index_results'),+ ("mwmbl", "0004_curation_original_index_results"),]operations = [migrations.CreateModel(- name='OldIndex',+ name="OldIndex",fields=[- ('index_path', models.CharField(max_length=300, primary_key=True, serialize=False)),- ('last_copied_time', models.DateTimeField(blank=True, null=True)),- ('last_page_copied', models.IntegerField(blank=True, null=True)),+ ("index_path", models.CharField(max_length=300, primary_key=True, serialize=False)),+ ("last_copied_time", models.DateTimeField(blank=True, null=True)),+ ("last_page_copied", models.IntegerField(blank=True, null=True)),],),]diff --git a/mwmbl/migrations/0006_flagcuration.py b/mwmbl/migrations/0006_flagcuration.pyindex b8b7929..2d4c8d5 100644--- a/mwmbl/migrations/0006_flagcuration.py+++ b/mwmbl/migrations/0006_flagcuration.py@@ -1,26 +1,37 @@# Generated by Django 4.2.11 on 2024-04-02 06:09+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletionclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0005_oldindex_squashed'),+ ("mwmbl", "0005_oldindex_squashed"),]operations = [migrations.CreateModel(- name='FlagCuration',+ name="FlagCuration",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('timestamp', models.DateTimeField()),- ('flag', models.CharField(choices=[('RELEVANCE', 'The curation is unlikely to be useful to a large number of users'), ('LANGUAGE', 'The curation is for a query in an unsupported language'), ('PROMOTION', 'The curation promotes a specific website or product'), ('OFFENSIVE', 'The curation contains offensive content'), ('OTHER', 'Other')], max_length=20)),- ('reason', models.CharField(max_length=300)),- ('curation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mwmbl.curation')),- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("timestamp", models.DateTimeField()),+ (+ "flag",+ models.CharField(+ choices=[+ ("RELEVANCE", "The curation is unlikely to be useful to a large number of users"),+ ("LANGUAGE", "The curation is for a query in an unsupported language"),+ ("PROMOTION", "The curation promotes a specific website or product"),+ ("OFFENSIVE", "The curation contains offensive content"),+ ("OTHER", "Other"),+ ],+ max_length=20,+ ),+ ),+ ("reason", models.CharField(max_length=300)),+ ("curation", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="mwmbl.curation")),+ ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),],),]diff --git a/mwmbl/migrations/0006_remove_oldindex_index_total_pages_and_more.py b/mwmbl/migrations/0006_remove_oldindex_index_total_pages_and_more.pyindex e656682..237d640 100644--- a/mwmbl/migrations/0006_remove_oldindex_index_total_pages_and_more.py+++ b/mwmbl/migrations/0006_remove_oldindex_index_total_pages_and_more.py@@ -4,24 +4,23 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0005_oldindex'),+ ("mwmbl", "0005_oldindex"),]operations = [migrations.RemoveField(- model_name='oldindex',- name='index_total_pages',+ model_name="oldindex",+ name="index_total_pages",),migrations.AlterField(- model_name='oldindex',- name='last_copied_time',+ model_name="oldindex",+ name="last_copied_time",field=models.DateTimeField(null=True),),migrations.AlterField(- model_name='oldindex',- name='last_page_copied',+ model_name="oldindex",+ name="last_page_copied",field=models.IntegerField(null=True),),]diff --git a/mwmbl/migrations/0007_alter_oldindex_last_copied_time_and_more.py b/mwmbl/migrations/0007_alter_oldindex_last_copied_time_and_more.pyindex c8e6025..aaa78ab 100644--- a/mwmbl/migrations/0007_alter_oldindex_last_copied_time_and_more.py+++ b/mwmbl/migrations/0007_alter_oldindex_last_copied_time_and_more.py@@ -4,20 +4,19 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0006_remove_oldindex_index_total_pages_and_more'),+ ("mwmbl", "0006_remove_oldindex_index_total_pages_and_more"),]operations = [migrations.AlterField(- model_name='oldindex',- name='last_copied_time',+ model_name="oldindex",+ name="last_copied_time",field=models.DateTimeField(blank=True, null=True),),migrations.AlterField(- model_name='oldindex',- name='last_page_copied',+ model_name="oldindex",+ name="last_page_copied",field=models.IntegerField(blank=True, null=True),),]diff --git a/mwmbl/migrations/0007_flagcuration_status_alter_flagcuration_reason.py b/mwmbl/migrations/0007_flagcuration_status_alter_flagcuration_reason.pyindex f2db076..2ba11e8 100644--- a/mwmbl/migrations/0007_flagcuration_status_alter_flagcuration_reason.py+++ b/mwmbl/migrations/0007_flagcuration_status_alter_flagcuration_reason.py@@ -4,20 +4,27 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0006_flagcuration'),+ ("mwmbl", "0006_flagcuration"),]operations = [migrations.AddField(- model_name='flagcuration',- name='status',- field=models.CharField(choices=[('PENDING', 'The flag has been submitted and is awaiting review'), ('REJECTED', 'The flag has been rejected'), ('ACCEPTED', 'The flag has been accepted and the curation has been reverted')], default='PENDING', max_length=20),+ model_name="flagcuration",+ name="status",+ field=models.CharField(+ choices=[+ ("PENDING", "The flag has been submitted and is awaiting review"),+ ("REJECTED", "The flag has been rejected"),+ ("ACCEPTED", "The flag has been accepted and the curation has been reverted"),+ ],+ default="PENDING",+ max_length=20,+ ),),migrations.AlterField(- model_name='flagcuration',- name='reason',+ model_name="flagcuration",+ name="reason",field=models.CharField(blank=True, max_length=300),),]diff --git a/mwmbl/migrations/0008_alter_flagcuration_options_and_more.py b/mwmbl/migrations/0008_alter_flagcuration_options_and_more.pyindex 308a8b6..eb4a44f 100644--- a/mwmbl/migrations/0008_alter_flagcuration_options_and_more.py+++ b/mwmbl/migrations/0008_alter_flagcuration_options_and_more.py@@ -1,38 +1,78 @@# Generated by Django 4.2.11 on 2024-06-14 18:59+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletionclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0007_flagcuration_status_alter_flagcuration_reason'),+ ("mwmbl", "0007_flagcuration_status_alter_flagcuration_reason"),]operations = [migrations.AlterModelOptions(- name='flagcuration',- options={'permissions': [('change_flag_status', 'Can change the flag status (approve or reject)')]},+ name="flagcuration",+ options={"permissions": [("change_flag_status", "Can change the flag status (approve or reject)")]},),migrations.AlterField(- model_name='flagcuration',- name='curation',- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='flag_curation_set', to='mwmbl.curation'),+ model_name="flagcuration",+ name="curation",+ field=models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE, related_name="flag_curation_set", to="mwmbl.curation"+ ),),migrations.CreateModel(- name='DomainSubmission',+ name="DomainSubmission",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('name', models.CharField(max_length=300)),- ('submitted_on', models.DateTimeField()),- ('status', models.CharField(choices=[('PENDING', 'The domain submission is awaiting review'), ('APPROVED', 'The domain submission has been approved'), ('REJECTED', 'The domain submission has been rejected')], default='PENDING', max_length=20)),- ('status_changed_on', models.DateTimeField(blank=True, null=True)),- ('rejection_reason', models.CharField(blank=True, choices=[('SPAM', 'The domain submission was rejected because it was spam'), ('OFFENSIVE', 'The domain submission was rejected because it was offensive'), ('LANGUAGE', 'The domain is in an unsupported language'), ('OTHER', 'The domain submission was rejected for another reason')], max_length=20)),- ('rejection_detail', models.CharField(blank=True, max_length=300)),- ('status_changed_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='domain_submissions_changed', to=settings.AUTH_USER_MODEL)),- ('submitted_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='domain_submissions', to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("name", models.CharField(max_length=300)),+ ("submitted_on", models.DateTimeField()),+ (+ "status",+ models.CharField(+ choices=[+ ("PENDING", "The domain submission is awaiting review"),+ ("APPROVED", "The domain submission has been approved"),+ ("REJECTED", "The domain submission has been rejected"),+ ],+ default="PENDING",+ max_length=20,+ ),+ ),+ ("status_changed_on", models.DateTimeField(blank=True, null=True)),+ (+ "rejection_reason",+ models.CharField(+ blank=True,+ choices=[+ ("SPAM", "The domain submission was rejected because it was spam"),+ ("OFFENSIVE", "The domain submission was rejected because it was offensive"),+ ("LANGUAGE", "The domain is in an unsupported language"),+ ("OTHER", "The domain submission was rejected for another reason"),+ ],+ max_length=20,+ ),+ ),+ ("rejection_detail", models.CharField(blank=True, max_length=300)),+ (+ "status_changed_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="domain_submissions_changed",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "submitted_by",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="domain_submissions",+ to=settings.AUTH_USER_MODEL,+ ),+ ),],),]diff --git a/mwmbl/migrations/0009_alter_domainsubmission_options.py b/mwmbl/migrations/0009_alter_domainsubmission_options.pyindex 323f0a2..efe55be 100644--- a/mwmbl/migrations/0009_alter_domainsubmission_options.py+++ b/mwmbl/migrations/0009_alter_domainsubmission_options.py@@ -4,14 +4,13 @@ from django.db import migrationsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0008_alter_flagcuration_options_and_more'),+ ("mwmbl", "0008_alter_flagcuration_options_and_more"),]operations = [migrations.AlterModelOptions(- name='domainsubmission',- options={'permissions': [('change_domain_submission_status', 'Can change the domain submission status')]},+ name="domainsubmission",+ options={"permissions": [("change_domain_submission_status", "Can change the domain submission status")]},),]diff --git a/mwmbl/migrations/0010_apikey.py b/mwmbl/migrations/0010_apikey.pyindex 505bb5d..9cc58f4 100644--- a/mwmbl/migrations/0010_apikey.py+++ b/mwmbl/migrations/0010_apikey.py@@ -1,25 +1,25 @@# Generated by Django 4.2.14 on 2024-08-09 18:21+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletion+import mwmbl.modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0009_alter_domainsubmission_options'),+ ("mwmbl", "0009_alter_domainsubmission_options"),]operations = [migrations.CreateModel(- name='ApiKey',+ name="ApiKey",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('key', models.CharField(default=mwmbl.models.random_api_key, max_length=300, unique=True)),- ('created_on', models.DateTimeField()),- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("key", models.CharField(default=mwmbl.models.random_api_key, max_length=300, unique=True)),+ ("created_on", models.DateTimeField()),+ ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),],),]diff --git a/mwmbl/migrations/0011_alter_domainsubmission_submitted_on_and_more.py b/mwmbl/migrations/0011_alter_domainsubmission_submitted_on_and_more.pyindex b477abe..a063f32 100644--- a/mwmbl/migrations/0011_alter_domainsubmission_submitted_on_and_more.py+++ b/mwmbl/migrations/0011_alter_domainsubmission_submitted_on_and_more.py@@ -1,23 +1,22 @@# Generated by Django 4.2.19 on 2025-06-08 20:33-from django.db import migrations, modelsimport django.utils.timezone+from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0010_apikey'),+ ("mwmbl", "0010_apikey"),]operations = [migrations.AlterField(- model_name='domainsubmission',- name='submitted_on',+ model_name="domainsubmission",+ name="submitted_on",field=models.DateTimeField(default=django.utils.timezone.now),),migrations.AddIndex(- model_name='domainsubmission',- index=models.Index(fields=['submitted_on'], name='mwmbl_domai_submitt_cb7865_idx'),+ model_name="domainsubmission",+ index=models.Index(fields=["submitted_on"], name="mwmbl_domai_submitt_cb7865_idx"),),]diff --git a/mwmbl/migrations/0012_wasmevaluationjob.py b/mwmbl/migrations/0012_wasmevaluationjob.pyindex 8c5ae0a..9384b97 100644--- a/mwmbl/migrations/0012_wasmevaluationjob.py+++ b/mwmbl/migrations/0012_wasmevaluationjob.py@@ -1,28 +1,40 @@# Generated by Django 4.2.23 on 2025-07-19 10:20+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletionclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0011_alter_domainsubmission_submitted_on_and_more'),+ ("mwmbl", "0011_alter_domainsubmission_submitted_on_and_more"),]operations = [migrations.CreateModel(- name='WasmEvaluationJob',+ name="WasmEvaluationJob",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('wasm_file', models.BinaryField()),- ('status', models.CharField(choices=[('PENDING', 'The evaluation job is pending'), ('VALIDATED', 'The WASM file has been validated'), ('RUNNING', 'The evaluation is currently running'), ('COMPLETED', 'The evaluation has completed successfully'), ('FAILED', 'The evaluation has failed')], default='PENDING', max_length=20)),- ('created_at', models.DateTimeField(auto_now_add=True)),- ('completed_at', models.DateTimeField(blank=True, null=True)),- ('results', models.JSONField(blank=True, null=True)),- ('error_message', models.TextField(blank=True, null=True)),- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("wasm_file", models.BinaryField()),+ (+ "status",+ models.CharField(+ choices=[+ ("PENDING", "The evaluation job is pending"),+ ("VALIDATED", "The WASM file has been validated"),+ ("RUNNING", "The evaluation is currently running"),+ ("COMPLETED", "The evaluation has completed successfully"),+ ("FAILED", "The evaluation has failed"),+ ],+ default="PENDING",+ max_length=20,+ ),+ ),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("completed_at", models.DateTimeField(blank=True, null=True)),+ ("results", models.JSONField(blank=True, null=True)),+ ("error_message", models.TextField(blank=True, null=True)),+ ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),],),]diff --git a/mwmbl/migrations/0013_searchresultvote.py b/mwmbl/migrations/0013_searchresultvote.pyindex 36eac65..9ccb233 100644--- a/mwmbl/migrations/0013_searchresultvote.py+++ b/mwmbl/migrations/0013_searchresultvote.py@@ -1,33 +1,43 @@# Generated by Django 4.2.23 on 2025-07-20 14:00+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletionclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0012_wasmevaluationjob'),+ ("mwmbl", "0012_wasmevaluationjob"),]operations = [migrations.CreateModel(- name='SearchResultVote',+ name="SearchResultVote",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('url', models.URLField(max_length=500)),- ('query', models.CharField(max_length=300)),- ('vote_type', models.CharField(choices=[('UPVOTE', 'User upvoted this result'), ('DOWNVOTE', 'User downvoted this result')], max_length=10)),- ('timestamp', models.DateTimeField(auto_now_add=True)),- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("url", models.URLField(max_length=500)),+ ("query", models.CharField(max_length=300)),+ (+ "vote_type",+ models.CharField(+ choices=[("UPVOTE", "User upvoted this result"), ("DOWNVOTE", "User downvoted this result")],+ max_length=10,+ ),+ ),+ ("timestamp", models.DateTimeField(auto_now_add=True)),+ ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),],options={- 'indexes': [models.Index(fields=['url', 'query'], name='mwmbl_search_url_b8e7b8_idx'), models.Index(fields=['timestamp'], name='mwmbl_search_timesta_b8e7b8_idx')],+ "indexes": [+ models.Index(fields=["url", "query"], name="mwmbl_search_url_b8e7b8_idx"),+ models.Index(fields=["timestamp"], name="mwmbl_search_timesta_b8e7b8_idx"),+ ],},),migrations.AddConstraint(- model_name='searchresultvote',- constraint=models.UniqueConstraint(fields=('user', 'url', 'query'), name='mwmbl_searchresultvote_user_url_query_unique'),+ model_name="searchresultvote",+ constraint=models.UniqueConstraint(+ fields=("user", "url", "query"), name="mwmbl_searchresultvote_user_url_query_unique"+ ),),]diff --git a/mwmbl/migrations/0014_remove_searchresultvote_mwmbl_searchresultvote_user_url_query_unique_and_more.py b/mwmbl/migrations/0014_remove_searchresultvote_mwmbl_searchresultvote_user_url_query_unique_and_more.pyindex 300f04b..4ff1520 100644--- a/mwmbl/migrations/0014_remove_searchresultvote_mwmbl_searchresultvote_user_url_query_unique_and_more.py+++ b/mwmbl/migrations/0014_remove_searchresultvote_mwmbl_searchresultvote_user_url_query_unique_and_more.py@@ -4,28 +4,27 @@ from django.db import migrationsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0013_searchresultvote'),+ ("mwmbl", "0013_searchresultvote"),]operations = [migrations.RemoveConstraint(- model_name='searchresultvote',- name='mwmbl_searchresultvote_user_url_query_unique',+ model_name="searchresultvote",+ name="mwmbl_searchresultvote_user_url_query_unique",),migrations.RenameIndex(- model_name='searchresultvote',- new_name='mwmbl_searc_url_03d737_idx',- old_name='mwmbl_search_url_b8e7b8_idx',+ model_name="searchresultvote",+ new_name="mwmbl_searc_url_03d737_idx",+ old_name="mwmbl_search_url_b8e7b8_idx",),migrations.RenameIndex(- model_name='searchresultvote',- new_name='mwmbl_searc_timesta_b8940f_idx',- old_name='mwmbl_search_timesta_b8e7b8_idx',+ model_name="searchresultvote",+ new_name="mwmbl_searc_timesta_b8940f_idx",+ old_name="mwmbl_search_timesta_b8e7b8_idx",),migrations.AlterUniqueTogether(- name='searchresultvote',- unique_together={('user', 'url', 'query')},+ name="searchresultvote",+ unique_together={("user", "url", "query")},),]diff --git a/mwmbl/migrations/0015_alter_searchresultvote_vote_type.py b/mwmbl/migrations/0015_alter_searchresultvote_vote_type.pyindex 4d33dee..8135b84 100644--- a/mwmbl/migrations/0015_alter_searchresultvote_vote_type.py+++ b/mwmbl/migrations/0015_alter_searchresultvote_vote_type.py@@ -4,15 +4,17 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0014_remove_searchresultvote_mwmbl_searchresultvote_user_url_query_unique_and_more'),+ ("mwmbl", "0014_remove_searchresultvote_mwmbl_searchresultvote_user_url_query_unique_and_more"),]operations = [migrations.AlterField(- model_name='searchresultvote',- name='vote_type',- field=models.CharField(choices=[('upvote', 'User upvoted this result'), ('downvote', 'User downvoted this result')], max_length=10),+ model_name="searchresultvote",+ name="vote_type",+ field=models.CharField(+ choices=[("upvote", "User upvoted this result"), ("downvote", "User downvoted this result")],+ max_length=10,+ ),),]diff --git a/mwmbl/migrations/0016_apikey_scopes_name_mwmbluser_tier.py b/mwmbl/migrations/0016_apikey_scopes_name_mwmbluser_tier.pyindex c06df4a..73a4d54 100644--- a/mwmbl/migrations/0016_apikey_scopes_name_mwmbluser_tier.py+++ b/mwmbl/migrations/0016_apikey_scopes_name_mwmbluser_tier.py@@ -1,27 +1,26 @@-from django.db import migrations, modelsimport django.contrib.postgres.fields+from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0015_alter_searchresultvote_vote_type'),+ ("mwmbl", "0015_alter_searchresultvote_vote_type"),]operations = [# ApiKey: add name fieldmigrations.AddField(- model_name='apikey',- name='name',- field=models.CharField(blank=True, default='', max_length=100),+ model_name="apikey",+ name="name",+ field=models.CharField(blank=True, default="", max_length=100),),# ApiKey: add scopes ArrayFieldmigrations.AddField(- model_name='apikey',- name='scopes',+ model_name="apikey",+ name="scopes",field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(- choices=[('crawl', 'Crawl'), ('search', 'Search')],+ choices=[("crawl", "Crawl"), ("search", "Search")],max_length=20,),default=list,@@ -31,25 +30,25 @@ class Migration(migrations.Migration):# ApiKey: change created_on to auto_now_add# We use AlterField; existing rows already have a value so this is safe.migrations.AlterField(- model_name='apikey',- name='created_on',+ model_name="apikey",+ name="created_on",field=models.DateTimeField(auto_now_add=True),preserve_default=False,),# MwmblUser: add tier fieldmigrations.AddField(- model_name='mwmbluser',- name='tier',+ model_name="mwmbluser",+ name="tier",field=models.CharField(- choices=[('free', 'Free'), ('starter', 'Starter'), ('pro', 'Pro')],- default='free',+ choices=[("free", "Free"), ("starter", "Starter"), ("pro", "Pro")],+ default="free",max_length=20,),),# MwmblUser: add monthly_search_count fieldmigrations.AddField(- model_name='mwmbluser',- name='monthly_search_count',+ model_name="mwmbluser",+ name="monthly_search_count",field=models.IntegerField(default=0),),]diff --git a/mwmbl/migrations/0017_backfill_apikey_scopes.py b/mwmbl/migrations/0017_backfill_apikey_scopes.pyindex 0404295..8b49864 100644--- a/mwmbl/migrations/0017_backfill_apikey_scopes.py+++ b/mwmbl/migrations/0017_backfill_apikey_scopes.py@@ -8,9 +8,8 @@ def set_crawl_scope(apps, schema_editor):class Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0016_apikey_scopes_name_mwmbluser_tier'),+ ("mwmbl", "0016_apikey_scopes_name_mwmbluser_tier"),]operations = [diff --git a/mwmbl/migrations/0018_usagebucket_remove_monthly_search_count.py b/mwmbl/migrations/0018_usagebucket_remove_monthly_search_count.pyindex 3a8b3c7..2d2d81b 100644--- a/mwmbl/migrations/0018_usagebucket_remove_monthly_search_count.py+++ b/mwmbl/migrations/0018_usagebucket_remove_monthly_search_count.py@@ -1,30 +1,29 @@+import django.db.models.deletionfrom django.conf import settingsfrom django.db import migrations, models-import django.db.models.deletionclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0017_backfill_apikey_scopes'),+ ("mwmbl", "0017_backfill_apikey_scopes"),]operations = [migrations.CreateModel(- name='UsageBucket',+ name="UsageBucket",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('year', models.IntegerField()),- ('month', models.IntegerField()),- ('count', models.IntegerField(default=0)),- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("year", models.IntegerField()),+ ("month", models.IntegerField()),+ ("count", models.IntegerField(default=0)),+ ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),],options={- 'unique_together': {('user', 'year', 'month')},+ "unique_together": {("user", "year", "month")},},),migrations.RemoveField(- model_name='mwmbluser',- name='monthly_search_count',+ model_name="mwmbluser",+ name="monthly_search_count",),]diff --git a/mwmbl/migrations/0019_add_userbilling.py b/mwmbl/migrations/0019_add_userbilling.pyindex 9f5b873..0a20ed8 100644--- a/mwmbl/migrations/0019_add_userbilling.py+++ b/mwmbl/migrations/0019_add_userbilling.py@@ -6,20 +6,24 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0018_usagebucket_remove_monthly_search_count'),+ ("mwmbl", "0018_usagebucket_remove_monthly_search_count"),]operations = [migrations.CreateModel(- name='UserBilling',+ name="UserBilling",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('polar_customer_id', models.CharField(blank=True, default='', max_length=100)),- ('polar_subscription_id', models.CharField(blank=True, default='', max_length=100)),- ('current_period_end', models.DateTimeField(blank=True, null=True)),- ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='billing', to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("polar_customer_id", models.CharField(blank=True, default="", max_length=100)),+ ("polar_subscription_id", models.CharField(blank=True, default="", max_length=100)),+ ("current_period_end", models.DateTimeField(blank=True, null=True)),+ (+ "user",+ models.OneToOneField(+ on_delete=django.db.models.deletion.CASCADE, related_name="billing", to=settings.AUTH_USER_MODEL+ ),+ ),],),]diff --git a/mwmbl/migrations/0020_usagebucket_year_month_index.py b/mwmbl/migrations/0020_usagebucket_year_month_index.pyindex b0f28fc..f434ffa 100644--- a/mwmbl/migrations/0020_usagebucket_year_month_index.py+++ b/mwmbl/migrations/0020_usagebucket_year_month_index.py@@ -2,14 +2,13 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0019_add_userbilling'),+ ("mwmbl", "0019_add_userbilling"),]operations = [migrations.AddIndex(- model_name='usagebucket',- index=models.Index(fields=['year', 'month'], name='mwmbl_usage_year_mo_idx'),+ model_name="usagebucket",+ index=models.Index(fields=["year", "month"], name="mwmbl_usage_year_mo_idx"),),]diff --git a/mwmbl/migrations/0021_hash_api_keys.py b/mwmbl/migrations/0021_hash_api_keys.pyindex 6d4a637..14a4d1c 100644--- a/mwmbl/migrations/0021_hash_api_keys.py+++ b/mwmbl/migrations/0021_hash_api_keys.py@@ -4,23 +4,22 @@ from django.db import migrations, modelsdef hash_existing_keys(apps, schema_editor):- ApiKey = apps.get_model('mwmbl', 'ApiKey')+ ApiKey = apps.get_model("mwmbl", "ApiKey")for api_key in ApiKey.objects.all():api_key.key = hashlib.sha256(api_key.key.encode()).hexdigest()- api_key.save(update_fields=['key'])+ api_key.save(update_fields=["key"])class Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0020_usagebucket_year_month_index'),+ ("mwmbl", "0020_usagebucket_year_month_index"),]operations = [migrations.RunPython(hash_existing_keys, migrations.RunPython.noop),migrations.AlterField(- model_name='apikey',- name='key',+ model_name="apikey",+ name="key",field=models.CharField(max_length=64, unique=True),),]diff --git a/mwmbl/migrations/0022_add_useragreement.py b/mwmbl/migrations/0022_add_useragreement.pyindex 961883d..1bdcbd4 100644--- a/mwmbl/migrations/0022_add_useragreement.py+++ b/mwmbl/migrations/0022_add_useragreement.py@@ -6,33 +6,48 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0021_hash_api_keys'),+ ("mwmbl", "0021_hash_api_keys"),]operations = [migrations.CreateModel(- name='UserAgreement',+ name="UserAgreement",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('agreement_type', models.CharField(choices=[('TERMS_OF_SERVICE_GUI', 'Terms of Service (GUI)'), ('TERMS_OF_SERVICE_API', 'Terms of Service (API)')], max_length=50)),- ('version_id', models.CharField(max_length=100)),- ('accepted_at', models.DateTimeField(auto_now_add=True)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ (+ "agreement_type",+ models.CharField(+ choices=[+ ("TERMS_OF_SERVICE_GUI", "Terms of Service (GUI)"),+ ("TERMS_OF_SERVICE_API", "Terms of Service (API)"),+ ],+ max_length=50,+ ),+ ),+ ("version_id", models.CharField(max_length=100)),+ ("accepted_at", models.DateTimeField(auto_now_add=True)),],),migrations.RenameIndex(- model_name='usagebucket',- new_name='mwmbl_usage_year_584313_idx',- old_name='mwmbl_usage_year_mo_idx',+ model_name="usagebucket",+ new_name="mwmbl_usage_year_584313_idx",+ old_name="mwmbl_usage_year_mo_idx",),migrations.AddField(- model_name='useragreement',- name='user',- field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='agreements', to=settings.AUTH_USER_MODEL),+ model_name="useragreement",+ name="user",+ field=models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="agreements",+ to=settings.AUTH_USER_MODEL,+ ),),migrations.AddIndex(- model_name='useragreement',- index=models.Index(fields=['user', 'agreement_type', '-accepted_at'], name='mwmbl_usera_user_id_e959ac_idx'),+ model_name="useragreement",+ index=models.Index(+ fields=["user", "agreement_type", "-accepted_at"], name="mwmbl_usera_user_id_e959ac_idx"+ ),),]diff --git a/mwmbl/migrations/0023_userbilling_cancel_at_period_end.py b/mwmbl/migrations/0023_userbilling_cancel_at_period_end.pyindex e7456b6..5a5a598 100644--- a/mwmbl/migrations/0023_userbilling_cancel_at_period_end.py+++ b/mwmbl/migrations/0023_userbilling_cancel_at_period_end.py@@ -4,15 +4,14 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0022_add_useragreement'),+ ("mwmbl", "0022_add_useragreement"),]operations = [migrations.AddField(- model_name='userbilling',- name='cancel_at_period_end',+ model_name="userbilling",+ name="cancel_at_period_end",field=models.BooleanField(default=False),),]diff --git a/mwmbl/migrations/0024_marketingconsent.py b/mwmbl/migrations/0024_marketingconsent.pyindex fcae3b4..ce798ff 100644--- a/mwmbl/migrations/0024_marketingconsent.py+++ b/mwmbl/migrations/0024_marketingconsent.py@@ -6,23 +6,38 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0023_userbilling_cancel_at_period_end'),+ ("mwmbl", "0023_userbilling_cancel_at_period_end"),]operations = [migrations.CreateModel(- name='MarketingConsent',+ name="MarketingConsent",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('source', models.CharField(choices=[('GUI', 'Consumer site (mwmbl.org)'), ('API', 'Developer site (developer.mwmbl.org)')], max_length=10)),- ('opted_in', models.BooleanField()),- ('timestamp', models.DateTimeField(auto_now_add=True)),- ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='marketing_consents', to=settings.AUTH_USER_MODEL)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ (+ "source",+ models.CharField(+ choices=[("GUI", "Consumer site (mwmbl.org)"), ("API", "Developer site (developer.mwmbl.org)")],+ max_length=10,+ ),+ ),+ ("opted_in", models.BooleanField()),+ ("timestamp", models.DateTimeField(auto_now_add=True)),+ (+ "user",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="marketing_consents",+ to=settings.AUTH_USER_MODEL,+ ),+ ),],options={- 'indexes': [models.Index(fields=['user', 'source', '-timestamp'], name='mwmbl_marke_user_id_c00b4e_idx')],+ "indexes": [+ models.Index(fields=["user", "source", "-timestamp"], name="mwmbl_marke_user_id_c00b4e_idx")+ ],},),]diff --git a/mwmbl/migrations/0025_supersearchimpression.py b/mwmbl/migrations/0025_supersearchimpression.pyindex f6abe7d..4814546 100644--- a/mwmbl/migrations/0025_supersearchimpression.py+++ b/mwmbl/migrations/0025_supersearchimpression.py@@ -4,25 +4,24 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0024_marketingconsent'),+ ("mwmbl", "0024_marketingconsent"),]operations = [migrations.CreateModel(- name='SuperSearchImpression',+ name="SuperSearchImpression",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('query', models.CharField(max_length=512)),- ('candidates', models.JSONField(default=list)),- ('selected', models.JSONField(default=list)),- ('features', models.JSONField(default=dict)),- ('rewards', models.JSONField(default=dict)),- ('timestamp', models.DateTimeField(auto_now_add=True)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("query", models.CharField(max_length=512)),+ ("candidates", models.JSONField(default=list)),+ ("selected", models.JSONField(default=list)),+ ("features", models.JSONField(default=dict)),+ ("rewards", models.JSONField(default=dict)),+ ("timestamp", models.DateTimeField(auto_now_add=True)),],options={- 'indexes': [models.Index(fields=['timestamp'], name='mwmbl_super_timesta_bbb2f8_idx')],+ "indexes": [models.Index(fields=["timestamp"], name="mwmbl_super_timesta_bbb2f8_idx")],},),]diff --git a/mwmbl/migrations/0026_supersearchbanditstate.py b/mwmbl/migrations/0026_supersearchbanditstate.pyindex 5a31996..d5ef5db 100644--- a/mwmbl/migrations/0026_supersearchbanditstate.py+++ b/mwmbl/migrations/0026_supersearchbanditstate.py@@ -4,21 +4,20 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0025_supersearchimpression'),+ ("mwmbl", "0025_supersearchimpression"),]operations = [migrations.CreateModel(- name='SuperSearchBanditState',+ name="SuperSearchBanditState",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('site', models.CharField(max_length=128, unique=True)),- ('dim', models.IntegerField()),- ('a', models.BinaryField()),- ('b', models.BinaryField()),- ('updated_at', models.DateTimeField(auto_now=True)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("site", models.CharField(max_length=128, unique=True)),+ ("dim", models.IntegerField()),+ ("a", models.BinaryField()),+ ("b", models.BinaryField()),+ ("updated_at", models.DateTimeField(auto_now=True)),],),]diff --git a/mwmbl/migrations/0027_sourceprovenance.py b/mwmbl/migrations/0027_sourceprovenance.pyindex bd58f25..87bb193 100644--- a/mwmbl/migrations/0027_sourceprovenance.py+++ b/mwmbl/migrations/0027_sourceprovenance.py@@ -4,25 +4,27 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0026_supersearchbanditstate'),+ ("mwmbl", "0026_supersearchbanditstate"),]operations = [migrations.CreateModel(- name='SourceProvenance',+ name="SourceProvenance",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('url', models.URLField(max_length=500, unique=True)),- ('source', models.CharField(max_length=128)),- ('query', models.CharField(blank=True, max_length=512, null=True)),- ('parent_url', models.URLField(blank=True, max_length=500, null=True)),- ('depth', models.IntegerField(default=0)),- ('timestamp', models.DateTimeField(auto_now_add=True)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("url", models.URLField(max_length=500, unique=True)),+ ("source", models.CharField(max_length=128)),+ ("query", models.CharField(blank=True, max_length=512, null=True)),+ ("parent_url", models.URLField(blank=True, max_length=500, null=True)),+ ("depth", models.IntegerField(default=0)),+ ("timestamp", models.DateTimeField(auto_now_add=True)),],options={- 'indexes': [models.Index(fields=['source'], name='mwmbl_sourc_source_e12d82_idx'), models.Index(fields=['timestamp'], name='mwmbl_sourc_timesta_bb15f3_idx')],+ "indexes": [+ models.Index(fields=["source"], name="mwmbl_sourc_source_e12d82_idx"),+ models.Index(fields=["timestamp"], name="mwmbl_sourc_timesta_bb15f3_idx"),+ ],},),]diff --git a/mwmbl/migrations/0028_remove_sourceprovenance_query_and_more.py b/mwmbl/migrations/0028_remove_sourceprovenance_query_and_more.pyindex a140873..e40d9d4 100644--- a/mwmbl/migrations/0028_remove_sourceprovenance_query_and_more.py+++ b/mwmbl/migrations/0028_remove_sourceprovenance_query_and_more.py@@ -4,18 +4,17 @@ from django.db import migrationsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0027_sourceprovenance'),+ ("mwmbl", "0027_sourceprovenance"),]operations = [migrations.RemoveField(- model_name='sourceprovenance',- name='query',+ model_name="sourceprovenance",+ name="query",),migrations.RemoveField(- model_name='supersearchimpression',- name='query',+ model_name="supersearchimpression",+ name="query",),]diff --git a/mwmbl/migrations/0029_delete_supersearchbanditstate.py b/mwmbl/migrations/0029_delete_supersearchbanditstate.pyindex 0b6e166..d6992db 100644--- a/mwmbl/migrations/0029_delete_supersearchbanditstate.py+++ b/mwmbl/migrations/0029_delete_supersearchbanditstate.py@@ -4,13 +4,12 @@ from django.db import migrationsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0028_remove_sourceprovenance_query_and_more'),+ ("mwmbl", "0028_remove_sourceprovenance_query_and_more"),]operations = [migrations.DeleteModel(- name='SuperSearchBanditState',+ name="SuperSearchBanditState",),]diff --git a/mwmbl/migrations/0030_remove_mwmbluser_tier_usagebucket_reported_overage_and_more.py b/mwmbl/migrations/0030_remove_mwmbluser_tier_usagebucket_reported_overage_and_more.pyindex d2c99cb..9d765b4 100644--- a/mwmbl/migrations/0030_remove_mwmbluser_tier_usagebucket_reported_overage_and_more.py+++ b/mwmbl/migrations/0030_remove_mwmbluser_tier_usagebucket_reported_overage_and_more.py@@ -4,24 +4,23 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0029_delete_supersearchbanditstate'),+ ("mwmbl", "0029_delete_supersearchbanditstate"),]operations = [migrations.RemoveField(- model_name='mwmbluser',- name='tier',+ model_name="mwmbluser",+ name="tier",),migrations.AddField(- model_name='usagebucket',- name='reported_overage',+ model_name="usagebucket",+ name="reported_overage",field=models.IntegerField(default=0),),migrations.AddField(- model_name='userbilling',- name='max_monthly_spend_cents',+ model_name="userbilling",+ name="max_monthly_spend_cents",field=models.IntegerField(default=0),),]diff --git a/mwmbl/migrations/0031_dedupe_background_task_schedules.py b/mwmbl/migrations/0031_dedupe_background_task_schedules.pyindex f304fc2..4be9022 100644--- a/mwmbl/migrations/0031_dedupe_background_task_schedules.py+++ b/mwmbl/migrations/0031_dedupe_background_task_schedules.py@@ -7,6 +7,7 @@ For each periodic task name, keeps a single unlocked row and deletes therest. Locked (in-flight) rows are left alone so we never interfere with atask that's currently running."""+from django.db import migrationsTASK_NAMES = [@@ -18,9 +19,7 @@ TASK_NAMES = [def dedupe_task_schedules(apps, schema_editor):Task = apps.get_model("background_task", "Task")for task_name in TASK_NAMES:- duplicates = list(- Task.objects.filter(task_name=task_name, locked_by__isnull=True).order_by("id")- )+ duplicates = list(Task.objects.filter(task_name=task_name, locked_by__isnull=True).order_by("id"))# Keep the oldest row, delete the rest.for task in duplicates[1:]:task.delete()@@ -31,7 +30,6 @@ def noop_reverse(apps, schema_editor):class Migration(migrations.Migration):-dependencies = [("mwmbl", "0030_remove_mwmbluser_tier_usagebucket_reported_overage_and_more"),("background_task", "0002_auto_20170927_1109"),diff --git a/mwmbl/migrations/0032_normalize_domain_submission_names.py b/mwmbl/migrations/0032_normalize_domain_submission_names.pyindex bb0a404..8169cd0 100644--- a/mwmbl/migrations/0032_normalize_domain_submission_names.py+++ b/mwmbl/migrations/0032_normalize_domain_submission_names.py@@ -29,7 +29,6 @@ def normalize_names(apps, schema_editor):class Migration(migrations.Migration):-dependencies = [("mwmbl", "0031_dedupe_background_task_schedules"),]diff --git a/mwmbl/migrations/0033_fix_domain_submission_names.py b/mwmbl/migrations/0033_fix_domain_submission_names.pyindex 445a826..752924d 100644--- a/mwmbl/migrations/0033_fix_domain_submission_names.py+++ b/mwmbl/migrations/0033_fix_domain_submission_names.py@@ -39,11 +39,10 @@ def fix_names(apps, schema_editor):DomainSubmission.objects.bulk_update(submissions_to_update, ["name"], batch_size=BATCH_SIZE)for start in range(0, len(ids_to_delete), BATCH_SIZE):- DomainSubmission.objects.filter(id__in=ids_to_delete[start:start + BATCH_SIZE]).delete()+ DomainSubmission.objects.filter(id__in=ids_to_delete[start : start + BATCH_SIZE]).delete()class Migration(migrations.Migration):-dependencies = [("mwmbl", "0032_normalize_domain_submission_names"),]diff --git a/mwmbl/migrations/0034_domainsubmission_suggested_reason_and_more.py b/mwmbl/migrations/0034_domainsubmission_suggested_reason_and_more.pyindex 873f50f..f7e84b8 100644--- a/mwmbl/migrations/0034_domainsubmission_suggested_reason_and_more.py+++ b/mwmbl/migrations/0034_domainsubmission_suggested_reason_and_more.py@@ -4,53 +4,68 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0033_fix_domain_submission_names'),+ ("mwmbl", "0033_fix_domain_submission_names"),]operations = [migrations.AddField(- model_name='domainsubmission',- name='suggested_reason',+ model_name="domainsubmission",+ name="suggested_reason",field=models.CharField(blank=True, max_length=20),),migrations.AddField(- model_name='domainsubmission',- name='suggested_status',+ model_name="domainsubmission",+ name="suggested_status",field=models.CharField(blank=True, max_length=20),),migrations.AddField(- model_name='domainsubmission',- name='suggestion_confidence',+ model_name="domainsubmission",+ name="suggestion_confidence",field=models.FloatField(blank=True, null=True),),migrations.AddField(- model_name='domainsubmission',- name='suggestion_model_version',+ model_name="domainsubmission",+ name="suggestion_model_version",field=models.CharField(blank=True, max_length=50),),migrations.CreateModel(- name='DomainEvidence',+ name="DomainEvidence",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('domain', models.CharField(max_length=300, unique=True)),- ('state', models.CharField(choices=[('PENDING', 'Queued for crawling'), ('READY', 'Crawled and scored'), ('FAILED', 'Crawling failed')], default='PENDING', max_length=20)),- ('fetched_at', models.DateTimeField(blank=True, null=True)),- ('http_status', models.IntegerField(blank=True, null=True)),- ('final_domain', models.CharField(blank=True, max_length=300)),- ('error', models.CharField(blank=True, max_length=100)),- ('pages', models.JSONField(default=list)),- ('signals', models.JSONField(default=dict)),- ('suggested_action', models.CharField(blank=True, max_length=10)),- ('suggested_reason', models.CharField(blank=True, max_length=20)),- ('confidence', models.FloatField(blank=True, null=True)),- ('review_priority', models.FloatField(blank=True, null=True)),- ('evidence', models.JSONField(default=list)),- ('model_version', models.CharField(blank=True, max_length=50)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("domain", models.CharField(max_length=300, unique=True)),+ (+ "state",+ models.CharField(+ choices=[+ ("PENDING", "Queued for crawling"),+ ("READY", "Crawled and scored"),+ ("FAILED", "Crawling failed"),+ ],+ default="PENDING",+ max_length=20,+ ),+ ),+ ("fetched_at", models.DateTimeField(blank=True, null=True)),+ ("http_status", models.IntegerField(blank=True, null=True)),+ ("final_domain", models.CharField(blank=True, max_length=300)),+ ("error", models.CharField(blank=True, max_length=100)),+ ("pages", models.JSONField(default=list)),+ ("signals", models.JSONField(default=dict)),+ ("suggested_action", models.CharField(blank=True, max_length=10)),+ ("suggested_reason", models.CharField(blank=True, max_length=20)),+ ("confidence", models.FloatField(blank=True, null=True)),+ ("review_priority", models.FloatField(blank=True, null=True)),+ ("evidence", models.JSONField(default=list)),+ ("model_version", models.CharField(blank=True, max_length=50)),],options={- 'indexes': [models.Index(fields=['state', '-review_priority'], name='mwmbl_domai_state_1c3406_idx'), models.Index(fields=['suggested_action', 'suggested_reason'], name='mwmbl_domai_suggest_68cee1_idx')],+ "indexes": [+ models.Index(fields=["state", "-review_priority"], name="mwmbl_domai_state_1c3406_idx"),+ models.Index(+ fields=["suggested_action", "suggested_reason"], name="mwmbl_domai_suggest_68cee1_idx"+ ),+ ],},),]diff --git a/mwmbl/migrations/0035_moderationmodelartifact_and_more.py b/mwmbl/migrations/0035_moderationmodelartifact_and_more.pyindex 837669b..1c31df7 100644--- a/mwmbl/migrations/0035_moderationmodelartifact_and_more.py+++ b/mwmbl/migrations/0035_moderationmodelartifact_and_more.py@@ -5,53 +5,52 @@ from django.db import migrations, modelsclass Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0034_domainsubmission_suggested_reason_and_more'),+ ("mwmbl", "0034_domainsubmission_suggested_reason_and_more"),]operations = [migrations.CreateModel(- name='ModerationModelArtifact',+ name="ModerationModelArtifact",fields=[- ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),- ('version', models.CharField(max_length=50, unique=True)),- ('model', models.BinaryField()),- ('metrics', models.JSONField(default=dict)),- ('created_on', models.DateTimeField(default=django.utils.timezone.now)),+ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),+ ("version", models.CharField(max_length=50, unique=True)),+ ("model", models.BinaryField()),+ ("metrics", models.JSONField(default=dict)),+ ("created_on", models.DateTimeField(default=django.utils.timezone.now)),],options={- 'get_latest_by': 'created_on',+ "get_latest_by": "created_on",},),migrations.RemoveIndex(- model_name='domainevidence',- name='mwmbl_domai_state_1c3406_idx',+ model_name="domainevidence",+ name="mwmbl_domai_state_1c3406_idx",),migrations.RemoveField(- model_name='domainevidence',- name='review_priority',+ model_name="domainevidence",+ name="review_priority",),migrations.AddField(- model_name='domainevidence',- name='reason_confidence',+ model_name="domainevidence",+ name="reason_confidence",field=models.FloatField(blank=True, null=True),),migrations.AddField(- model_name='domainevidence',- name='reason_source',+ model_name="domainevidence",+ name="reason_source",field=models.CharField(blank=True, max_length=10),),migrations.AddIndex(- model_name='domainsubmission',- index=models.Index(fields=['name', 'status'], name='mwmbl_domai_name_0a9d6d_idx'),+ model_name="domainsubmission",+ index=models.Index(fields=["name", "status"], name="mwmbl_domai_name_0a9d6d_idx"),),migrations.AddIndex(- model_name='domainsubmission',- index=models.Index(fields=['submitted_by', 'status'], name='mwmbl_domai_submitt_2d8fb1_idx'),+ model_name="domainsubmission",+ index=models.Index(fields=["submitted_by", "status"], name="mwmbl_domai_submitt_2d8fb1_idx"),),migrations.AddIndex(- model_name='moderationmodelartifact',- index=models.Index(fields=['-created_on'], name='mwmbl_moder_created_1c74be_idx'),+ model_name="moderationmodelartifact",+ index=models.Index(fields=["-created_on"], name="mwmbl_moder_created_1c74be_idx"),),]diff --git a/mwmbl/migrations/0036_search_result_vote_domain.py b/mwmbl/migrations/0036_search_result_vote_domain.pyindex bfc8983..c47b010 100644--- a/mwmbl/migrations/0036_search_result_vote_domain.py+++ b/mwmbl/migrations/0036_search_result_vote_domain.py@@ -38,20 +38,19 @@ def backfill_domains(apps, schema_editor):class Migration(migrations.Migration):-dependencies = [- ('mwmbl', '0035_moderationmodelartifact_and_more'),+ ("mwmbl", "0035_moderationmodelartifact_and_more"),]operations = [migrations.AddField(- model_name='searchresultvote',- name='domain',+ model_name="searchresultvote",+ name="domain",field=models.CharField(blank=True, max_length=300),),migrations.RunPython(backfill_domains, migrations.RunPython.noop),migrations.AddIndex(- model_name='searchresultvote',- index=models.Index(fields=['domain', 'vote_type'], name='mwmbl_searc_domain_7f4ebd_idx'),+ model_name="searchresultvote",+ index=models.Index(fields=["domain", "vote_type"], name="mwmbl_searc_domain_7f4ebd_idx"),),]diff --git a/mwmbl/migrations/0037_repair_domain_submission_statuses.py b/mwmbl/migrations/0037_repair_domain_submission_statuses.pyindex 0289dca..423a526 100644--- a/mwmbl/migrations/0037_repair_domain_submission_statuses.py+++ b/mwmbl/migrations/0037_repair_domain_submission_statuses.py@@ -18,6 +18,7 @@ Newly approved domains are subtracted from the remote blocklists when the snapshrebuilt (BLACKLIST_SNAPSHOT_REFRESH_SECONDS, six hours), and get_curated_domains caches forfive minutes, so both catch up on their own rather than being poked from a migration."""+from django.db import migrations, modelsREPAIRS = {"APPROVE": "APPROVED", "REJECT": "REJECTED"}@@ -35,7 +36,6 @@ def noop_reverse(apps, schema_editor):class Migration(migrations.Migration):-dependencies = [("mwmbl", "0036_search_result_vote_domain"),]@@ -53,8 +53,7 @@ class Migration(migrations.Migration):migrations.AddConstraint(model_name="domainsubmission",constraint=models.CheckConstraint(- condition=models.Q(- rejection_reason__in=["", "SPAM", "OFFENSIVE", "LANGUAGE", "OTHER"]),+ condition=models.Q(rejection_reason__in=["", "SPAM", "OFFENSIVE", "LANGUAGE", "OTHER"]),name="domain_submission_rejection_reason_valid",),),diff --git a/mwmbl/models.py b/mwmbl/models.pyindex 78e3eb6..9ad4423 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -1,14 +1,11 @@import hashlibimport secrets+from django.contrib.auth.models import AbstractUserfrom django.contrib.postgres.fields import ArrayFieldfrom django.db import models-from django.contrib.auth.models import AbstractUserfrom django.utils import timezone-from ninja import ModelSchema-from ninja.orm import create_schema-from mwmbl.usernames import generate_usernamefrom mwmbl.utils import bare_host@@ -20,6 +17,7 @@ class UserCuration(models.Model):"""Deprecated - use Curation instead"""+user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)timestamp = models.DateTimeField()url = models.CharField(max_length=300)@@ -99,12 +97,12 @@ class DomainSubmission(models.Model):("change_domain_submission_status", "Can change the domain submission status"),]indexes = [- models.Index(fields=['submitted_on']),+ models.Index(fields=["submitted_on"]),# The moderation queue asks "has this domain been decided before?" and "has this# submitter?" once per row it renders, as correlated subqueries. Both are lookups# into this table by a decided status. See mwmbl.moderation.suggest.- models.Index(fields=['name', 'status']),- models.Index(fields=['submitted_by', 'status']),+ models.Index(fields=["name", "status"]),+ models.Index(fields=["submitted_by", "status"]),]constraints = [# `choices` is documentation, not enforcement: it is checked by full_clean(),@@ -130,10 +128,16 @@ class DomainSubmission(models.Model):name = models.CharField(max_length=300)submitted_by = models.ForeignKey(MwmblUser, on_delete=models.CASCADE, related_name="domain_submissions")submitted_on = models.DateTimeField(default=timezone.now)- status = models.CharField(max_length=20, choices=[(k, v) for k, v in DOMAIN_SUBMISSION_STATUS.items()], default="PENDING")- status_changed_by = models.ForeignKey(MwmblUser, on_delete=models.CASCADE, null=True, blank=True, related_name="domain_submissions_changed")+ status = models.CharField(+ max_length=20, choices=[(k, v) for k, v in DOMAIN_SUBMISSION_STATUS.items()], default="PENDING"+ )+ status_changed_by = models.ForeignKey(+ MwmblUser, on_delete=models.CASCADE, null=True, blank=True, related_name="domain_submissions_changed"+ )status_changed_on = models.DateTimeField(null=True, blank=True)- rejection_reason = models.CharField(max_length=20, choices=[(k, v) for k, v in DOMAIN_REJECTION_REASON.items()], blank=True)+ rejection_reason = models.CharField(+ max_length=20, choices=[(k, v) for k, v in DOMAIN_REJECTION_REASON.items()], blank=True+ )rejection_detail = models.CharField(max_length=300, blank=True)# What the suggestion model was showing when the moderator made this decision. Deliberately@@ -162,14 +166,14 @@ def generate_api_key() -> tuple[str, str]:class ApiKey(models.Model):class Scope(models.TextChoices):- CRAWL = "crawl", "Crawl"+ CRAWL = "crawl", "Crawl"SEARCH = "search", "Search"- user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)- key = models.CharField(max_length=64, unique=True) # stores SHA-256 hash of the raw key+ user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)+ key = models.CharField(max_length=64, unique=True) # stores SHA-256 hash of the raw keycreated_on = models.DateTimeField(auto_now_add=True)- name = models.CharField(max_length=100, blank=True, default="")- scopes = ArrayField(+ name = models.CharField(max_length=100, blank=True, default="")+ scopes = ArrayField(models.CharField(max_length=20, choices=Scope.choices),default=list,)@@ -195,6 +199,7 @@ class WasmEvaluationJob(models.Model):class UsageBucket(models.Model):"""Records a user's API usage for a specific calendar month."""+user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)year = models.IntegerField()month = models.IntegerField()@@ -205,9 +210,9 @@ class UsageBucket(models.Model):reported_overage = models.IntegerField(default=0)class Meta:- unique_together = [('user', 'year', 'month')]+ unique_together = [("user", "year", "month")]indexes = [- models.Index(fields=['year', 'month']),+ models.Index(fields=["year", "month"]),]@@ -229,9 +234,7 @@ class AgreementType(models.TextChoices):class UserAgreement(models.Model):# SET_NULL rather than CASCADE so the audit record survives account deletion.- user = models.ForeignKey(- MwmblUser, on_delete=models.SET_NULL, null=True, related_name="agreements"- )+ user = models.ForeignKey(MwmblUser, on_delete=models.SET_NULL, null=True, related_name="agreements")agreement_type = models.CharField(max_length=50, choices=AgreementType.choices)version_id = models.CharField(max_length=100)accepted_at = models.DateTimeField(auto_now_add=True)@@ -254,11 +257,10 @@ class MarketingConsent(models.Model):Every opt-in and opt-out writes a new row; the current state for a source isthe `opted_in` of the latest row. Mirrors UserAgreement."""+# SET_NULL (not CASCADE) so the consent proof survives account deletion,# matching UserAgreement.- user = models.ForeignKey(- MwmblUser, on_delete=models.SET_NULL, null=True, related_name="marketing_consents"- )+ user = models.ForeignKey(MwmblUser, on_delete=models.SET_NULL, null=True, related_name="marketing_consents")source = models.CharField(max_length=10, choices=MarketingSource.choices)opted_in = models.BooleanField()timestamp = models.DateTimeField(auto_now_add=True)@@ -274,7 +276,7 @@ class SearchResultVote(models.Model):"upvote": "User upvoted this result","downvote": "User downvoted this result",}-+user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE)url = models.URLField(max_length=500) # The URL of the search resultquery = models.CharField(max_length=300) # The search query context@@ -290,13 +292,13 @@ class SearchResultVote(models.Model):super().save(*args, **kwargs)class Meta:- unique_together = ['user', 'url', 'query'] # One vote per user per result per query+ unique_together = ["user", "url", "query"] # One vote per user per result per queryindexes = [- models.Index(fields=['url', 'query']),- models.Index(fields=['timestamp']),+ models.Index(fields=["url", "query"]),+ models.Index(fields=["timestamp"]),# The per-domain rollup the moderation queue reads: count this domain's upvotes,# count its downvotes. See mwmbl.moderation.suggest.annotate_votes.- models.Index(fields=['domain', 'vote_type']),+ models.Index(fields=["domain", "vote_type"]),]@@ -308,15 +310,16 @@ class SuperSearchImpression(models.Model):durable training data for the contextual bandit. Deliberately does notstore the query text, to avoid persisting user search history."""- candidates = models.JSONField(default=list) # all selectable source names (action space)- selected = models.JSONField(default=list) # sources actually queried- features = models.JSONField(default=dict) # {source: [feature vector]} for selected sources- rewards = models.JSONField(default=dict) # {source: reward in [0, 1]}++ candidates = models.JSONField(default=list) # all selectable source names (action space)+ selected = models.JSONField(default=list) # sources actually queried+ features = models.JSONField(default=dict) # {source: [feature vector]} for selected sources+ rewards = models.JSONField(default=dict) # {source: reward in [0, 1]}timestamp = models.DateTimeField(auto_now_add=True)class Meta:indexes = [- models.Index(fields=['timestamp']),+ models.Index(fields=["timestamp"]),]@@ -328,16 +331,19 @@ class SourceProvenance(models.Model):offline including for descendant pages. Deliberately does not store thequery text, to avoid persisting user search history."""- url = models.URLField(max_length=500, unique=True) # first source wins, matches source_by_url semantics- source = models.CharField(max_length=128) # super-search source name (e.g. "gov.uk")- parent_url = models.URLField(max_length=500, null=True, blank=True) # page this URL was found on (null = direct result)- depth = models.IntegerField(default=0) # 0 = direct super-search result; +1 per crawl hop++ url = models.URLField(max_length=500, unique=True) # first source wins, matches source_by_url semantics+ source = models.CharField(max_length=128) # super-search source name (e.g. "gov.uk")+ parent_url = models.URLField(+ max_length=500, null=True, blank=True+ ) # page this URL was found on (null = direct result)+ depth = models.IntegerField(default=0) # 0 = direct super-search result; +1 per crawl hoptimestamp = models.DateTimeField(auto_now_add=True)class Meta:indexes = [- models.Index(fields=['source']),- models.Index(fields=['timestamp']),+ models.Index(fields=["source"]),+ models.Index(fields=["timestamp"]),]@@ -371,13 +377,13 @@ class DomainEvidence(models.Model):# Crawl results.http_status = models.IntegerField(null=True, blank=True)- final_domain = models.CharField(max_length=300, blank=True) # after redirects- error = models.CharField(max_length=100, blank=True) # RobotsDenied, AbortError, ...- pages = models.JSONField(default=list) # [{url, status, title, extract}], up to 3- signals = models.JSONField(default=dict) # derived: lang, has_links, ad_script_count, ...+ final_domain = models.CharField(max_length=300, blank=True) # after redirects+ error = models.CharField(max_length=100, blank=True) # RobotsDenied, AbortError, ...+ pages = models.JSONField(default=list) # [{url, status, title, extract}], up to 3+ signals = models.JSONField(default=dict) # derived: lang, has_links, ad_script_count, ...# Precomputed suggestion. The queue reads these columns; it never calls the model.- suggested_action = models.CharField(max_length=10, blank=True) # APPROVE | REJECT | UNSURE+ suggested_action = models.CharField(max_length=10, blank=True) # APPROVE | REJECT | UNSUREsuggested_reason = models.CharField(max_length=20, blank=True)confidence = models.FloatField(null=True, blank=True)# How sure we are of the *reason*, and where it came from - both separate from the numbers@@ -386,8 +392,8 @@ class DomainEvidence(models.Model):# and keeps `derived` (a reason class learned from blocklists rather than from moderator# decisions) visible to the moderator it is a caveat for.reason_confidence = models.FloatField(null=True, blank=True)- reason_source = models.CharField(max_length=10, blank=True) # rule | model | derived- evidence = models.JSONField(default=list) # cached rule + model evidence items+ reason_source = models.CharField(max_length=10, blank=True) # rule | model | derived+ evidence = models.JSONField(default=list) # cached rule + model evidence itemsmodel_version = models.CharField(max_length=50, blank=True)class Meta:@@ -417,8 +423,8 @@ class ModerationModelArtifact(models.Model):"""version = models.CharField(max_length=50, unique=True)- model = models.BinaryField() # joblib pickle of a ModerationModel, ~1 MB- metrics = models.JSONField(default=dict) # what train.evaluate measured for this model+ model = models.BinaryField() # joblib pickle of a ModerationModel, ~1 MB+ metrics = models.JSONField(default=dict) # what train.evaluate measured for this modelcreated_on = models.DateTimeField(default=timezone.now)class Meta:diff --git a/mwmbl/moderation/evidence.py b/mwmbl/moderation/evidence.pyindex da143f7..a2760d1 100644--- a/mwmbl/moderation/evidence.py+++ b/mwmbl/moderation/evidence.py@@ -15,6 +15,7 @@ caching, SSRF validation, a 3 second timeout, a 1 MB cap and justext extractionGraph fallbacks. Super Search already calls it from inside the server process, so nothinghere is a new capability - it is the existing crawler pointed at one domain."""+from __future__ import annotationsimport time@@ -33,16 +34,27 @@ from mwmbl.moderation import ruleslogger = getLogger(__name__)# Paths that tell you nothing about whether a site is worth crawling.-BORING_PATH_WORDS = ("privacy", "terms", "cookie", "legal", "login", "signin", "sign-in",- "register", "cart", "checkout", "contact", "rss", "feed")+BORING_PATH_WORDS = (+ "privacy",+ "terms",+ "cookie",+ "legal",+ "login",+ "signin",+ "sign-in",+ "register",+ "cart",+ "checkout",+ "contact",+ "rss",+ "feed",+)# Markers counted as signals for the moderator. Not model features - they are far too crude# for that - but "we found 4 ad-network scripts" is a useful line in an evidence list, and# moderators do reject for "obtrusive ads" and "paywall".-AD_MARKERS = ("googlesyndication", "doubleclick", "adsbygoogle", "adnxs", "taboola",- "outbrain", "criteo", "media.net")-PAYWALL_MARKERS = ("paywall", "subscribe to continue", "subscribers only",- "create a free account to")+AD_MARKERS = ("googlesyndication", "doubleclick", "adsbygoogle", "adnxs", "taboola", "outbrain", "criteo", "media.net")+PAYWALL_MARKERS = ("paywall", "subscribe to continue", "subscribers only", "create a free account to")def crawl_domain(domain: str, redis, max_pages: int = None) -> dict:@@ -132,7 +144,7 @@ def store_failure(domain: str, error: str) -> DomainEvidence:domain=domain,defaults={"state": DomainEvidence.State.FAILED,- "fetched_at": timezone.now(), # when we last tried, not when we last succeeded+ "fetched_at": timezone.now(), # when we last tried, not when we last succeeded"error": error,"http_status": None,"final_domain": "",@@ -223,5 +235,4 @@ def page_texts(evidence: DomainEvidence | None) -> list[str]:"""The text the model reads, in the same shape training built it from."""if evidence is None:return []- return [f"{page.get('title', '')} {page.get('extract', '')}".strip()- for page in (evidence.pages or [])]+ return [f"{page.get('title', '')} {page.get('extract', '')}".strip() for page in (evidence.pages or [])]diff --git a/mwmbl/moderation/features.py b/mwmbl/moderation/features.pyindex aef8d00..4fd5949 100644--- a/mwmbl/moderation/features.py+++ b/mwmbl/moderation/features.py@@ -10,6 +10,7 @@ were made up to two years ago, so a site that was live when it was approved in 2dead today would teach the model "dead -> approve". The model only judges what is stable abouta domain: whether its name and its text read as spam, promotion or a non-English site."""+from __future__ import annotationsfrom dataclasses import dataclass, field@@ -24,8 +25,25 @@ logger = getLogger(__name__)# Substrings that recur in the SEO/AI-slop domains moderators reject: aianimegenerator.cloud,# seobacklinkhub.org, stepupsipcalculator.net, bestaitoolsforthat.com, collegetools.io.SPAM_WORDS = [- "ai", "seo", "generator", "calculator", "tool", "free", "best", "top", "online", "shop",- "buy", "cheap", "review", "guide", "app", "hub", "pro", "download", "crypto",+ "ai",+ "seo",+ "generator",+ "calculator",+ "tool",+ "free",+ "best",+ "top",+ "online",+ "shop",+ "buy",+ "cheap",+ "review",+ "guide",+ "app",+ "hub",+ "pro",+ "download",+ "crypto",]# How much page text the model reads. Titles and extracts are already truncated by the@@ -42,13 +60,13 @@ INDICATOR_FEATURE_NAMES = ["www"]# Held out separately from the text block, because the two can fail independently: the block# can earn its place while the indicator is a trap. See Featuriser.OPTIONAL_INDICATOR_FEATURE_NAMES = ["has_text"]-NUM_SHAPE_FEATURES = (len(COUNT_FEATURE_NAMES) + len(INDICATOR_FEATURE_NAMES)- + len(OPTIONAL_INDICATOR_FEATURE_NAMES))+NUM_SHAPE_FEATURES = len(COUNT_FEATURE_NAMES) + len(INDICATOR_FEATURE_NAMES) + len(OPTIONAL_INDICATOR_FEATURE_NAMES)@dataclassclass ModerationExample:"""One thing to score: a domain, plus whatever page text we managed to crawl for it."""+domain: strpage_texts: list[str] = field(default_factory=list)@@ -130,15 +148,13 @@ class Featuriser:self.use_has_text = use_has_text# char_wb over the domain: catches the morphology of generated spam names, and the# word-boundary variant keeps n-grams from spanning label boundaries.- self.domain_chars = TfidfVectorizer(- analyzer="char_wb", ngram_range=(2, 5), min_df=2, sublinear_tf=True)+ self.domain_chars = TfidfVectorizer(analyzer="char_wb", ngram_range=(2, 5), min_df=2, sublinear_tf=True)# The TLD earns its own block rather than being left to the char n-grams: .ai was 30/33# rejected and .org 24/1079, and that is a whole-token effect, not a substring one.- self.tlds = TfidfVectorizer(- analyzer="word", tokenizer=_tld_tokens, token_pattern=None, min_df=1)+ self.tlds = TfidfVectorizer(analyzer="word", tokenizer=_tld_tokens, token_pattern=None, min_df=1)self.text = TfidfVectorizer(- analyzer="word", ngram_range=(1, 2), min_df=2, sublinear_tf=True,- strip_accents="unicode", lowercase=True)+ analyzer="word", ngram_range=(1, 2), min_df=2, sublinear_tf=True, strip_accents="unicode", lowercase=True+ )def fit_transform(self, examples: list[ModerationExample]) -> csr_matrix:domains = [example.domain for example in examples]@@ -157,8 +173,11 @@ class Featuriser:raise ValueError("text block disabled for this fit")blocks.append(self.text.fit_transform(texts))except ValueError:- logger.info("No text block over %d examples (disabled, or not enough text to fit "- "one); the model will judge on the domain name alone", len(examples))+ logger.info(+ "No text block over %d examples (disabled, or not enough text to fit "+ "one); the model will judge on the domain name alone",+ len(examples),+ )self.text = Noneblocks.append(self._shape_block(examples))@@ -177,8 +196,7 @@ class Featuriser:def _shape_block(self, examples: list[ModerationExample]) -> csr_matrix:counts = np.array([shape_features(example.domain) for example in examples], dtype=float)- indicators = np.array([indicator_features(example, self.use_has_text)- for example in examples], dtype=float)+ indicators = np.array([indicator_features(example, self.use_has_text) for example in examples], dtype=float)return csr_matrix(np.hstack([counts / SHAPE_SCALE, indicators]))def feature_names(self) -> np.ndarray:@@ -188,9 +206,7 @@ class Featuriser:np.array([f"tld={name}" for name in self.tlds.get_feature_names_out()]),]if self.text is not None:- names.append(np.array([f"text={name}"- for name in self.text.get_feature_names_out()]))- indicators = INDICATOR_FEATURE_NAMES + (- OPTIONAL_INDICATOR_FEATURE_NAMES if self.use_has_text else [])+ names.append(np.array([f"text={name}" for name in self.text.get_feature_names_out()]))+ indicators = INDICATOR_FEATURE_NAMES + (OPTIONAL_INDICATOR_FEATURE_NAMES if self.use_has_text else [])names.append(np.array(COUNT_FEATURE_NAMES + indicators))return np.concatenate(names)diff --git a/mwmbl/moderation/model.py b/mwmbl/moderation/model.pyindex c734cd8..e006808 100644--- a/mwmbl/moderation/model.py+++ b/mwmbl/moderation/model.py@@ -18,6 +18,7 @@ Loading mirrors mwmbl.tinysearchengine.super_search_select.judge: a lazy singletlock, memoized failure, and graceful degradation to rules-only when no artifact can be loadedso a deploy without one keeps working."""+from __future__ import annotationsimport io@@ -37,7 +38,7 @@ from django.utils import timezonefrom mwmbl.models import ModerationModelArtifactfrom mwmbl.moderation.features import Featuriser, ModerationExample-from mwmbl.moderation.rules import APPROVE, EvidenceItem, REJECT, decisive, implied_detail+from mwmbl.moderation.rules import APPROVE, REJECT, EvidenceItem, decisive, implied_detaillogger = getLogger(__name__)@@ -72,14 +73,14 @@ UNUSABLE_MODEL_REASONS = frozenset({"", "OTHER"})@dataclassclass Suggestion:- action: str # APPROVE | REJECT | UNSURE+ action: str # APPROVE | REJECT | UNSUREconfidence: floatreason: str = ""# What the submitter would be told. Non-empty whenever ``reason`` is OTHER and empty# otherwise: the other reasons are their own explanation.reason_detail: str = ""reason_confidence: float = 0.0- reason_source: str = "model" # model | derived | rule+ reason_source: str = "model" # model | derived | rulemodel_version: str = ""evidence: list[dict] = field(default_factory=list)@@ -106,8 +107,9 @@ class ModerationModel:# answers the question. Instances always set their own in __init__.train_domains: set = frozenset()- def __init__(self, featuriser: Featuriser, reject_head, reason_head, version: str,- train_domains: Optional[set] = None):+ def __init__(+ self, featuriser: Featuriser, reject_head, reason_head, version: str, train_domains: Optional[set] = None+ ):self.featuriser = featuriserself.reject_head = reject_headself.reason_head = reason_head@@ -132,14 +134,14 @@ class ModerationModel:confidences = reason_probabilities[np.arange(len(examples)), best]return [- (float(reject), str(reason), min(float(confidence),- REASON_CONFIDENCE_CAP.get(str(reason), 1.0)))+ (float(reject), str(reason), min(float(confidence), REASON_CONFIDENCE_CAP.get(str(reason), 1.0)))for reject, reason, confidence in zip(reject_probabilities, reasons, confidences)]-def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceItem],- model: Optional[ModerationModel] = None) -> Suggestion:+def suggest(+ domain: str, page_texts: list[str], evidence_items: list[EvidenceItem], model: Optional[ModerationModel] = None+) -> Suggestion:"""Compose a suggestion from the deterministic checks and, where they are silent, the model.A decisive check always wins. "Homepage returns 404" is not a matter of opinion, and@@ -164,12 +166,12 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIteif model is None:# Rules-only degradation: say we don't know rather than inventing a default.- return Suggestion(action="UNSURE", confidence=0.0, model_version="",- evidence=[item.to_dict() for item in evidence_items])+ return Suggestion(+ action="UNSURE", confidence=0.0, model_version="", evidence=[item.to_dict() for item in evidence_items]+ )try:- reject_probability, reason, reason_confidence = model.predict(- [ModerationExample(domain, page_texts)])[0]+ reject_probability, reason, reason_confidence = model.predict([ModerationExample(domain, page_texts)])[0]except Exception:# The same degradation, for a model that loaded but cannot score. is_compatible()# catches this at load time, but only for artifacts *older* than the running code -@@ -178,10 +180,10 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIte# August 2026 that failed all 2,409 rows of a queue rescore and left the task retrying# on the same exception. One unscorable domain is a suggestion we do not have; it is# not a reason to end the run.- logger.exception("Model %s could not score %s; falling back to the deterministic "- "checks", model.version, domain)- return Suggestion(action="UNSURE", confidence=0.0, model_version="",- evidence=[item.to_dict() for item in evidence_items])+ logger.exception("Model %s could not score %s; falling back to the deterministic checks", model.version, domain)+ return Suggestion(+ action="UNSURE", confidence=0.0, model_version="", evidence=[item.to_dict() for item in evidence_items]+ )if reject_probability >= settings.MODERATION_REJECT_THRESHOLD:action, confidence = "REJECT", reject_probability@@ -197,8 +199,12 @@ def suggest(domain: str, page_texts: list[str], evidence_items: list[EvidenceIte# can write the detail that goes with it, and a "Reject - other" button that fails# validation when pressed is worse than no suggestion. Back to the moderator, who is# the only one who can say what is wrong.- return Suggestion(action="UNSURE", confidence=0.0, model_version=model.version,- evidence=[item.to_dict() for item in evidence_items])+ return Suggestion(+ action="UNSURE",+ confidence=0.0,+ model_version=model.version,+ evidence=[item.to_dict() for item in evidence_items],+ )return Suggestion(action=action,@@ -248,7 +254,10 @@ def get_model() -> Optional[ModerationModel]:logger.warning("Published artifact %s is not usable by this code; continuing to serve %s. ""This is what a sibling deployment retraining against the same database "- "looks like.", published[0] if published else "(none)", _model.version)+ "looks like.",+ published[0] if published else "(none)",+ _model.version,+ )_loaded_stamp = published_checked_at = time.monotonic()return _model@@ -274,9 +283,8 @@ def publish(model: ModerationModel, metrics: dict) -> ModerationModelArtifact:# what tells the other workers this row has changed, since the version they compare# against has not.artifact, _ = ModerationModelArtifact.objects.update_or_create(- version=model.version,- defaults={"model": buffer.getvalue(), "metrics": metrics,- "created_on": timezone.now()})+ version=model.version, defaults={"model": buffer.getvalue(), "metrics": metrics, "created_on": timezone.now()}+ )reset_model_cache()return artifact@@ -297,8 +305,7 @@ def _published_stamp() -> Optional[tuple[str, datetime]]:same day replaces that row's bytes under the same version, and a worker comparing versionsalone would keep serving the superseded pickle until it restarted."""- row = (ModerationModelArtifact.objects.order_by("-created_on")- .values("version", "created_on").first())+ row = ModerationModelArtifact.objects.order_by("-created_on").values("version", "created_on").first()return (row["version"], row["created_on"]) if row else None@@ -327,9 +334,13 @@ def is_compatible(model: ModerationModel) -> bool:def _checked(model: Optional[ModerationModel], where: str) -> Optional[ModerationModel]:if model is None or is_compatible(model):return model- logger.error("Moderation model %s from %s was fitted on a different feature set than this "- "code builds; ignoring it and falling back to the deterministic checks. "- "Retrain to publish a compatible artifact.", model.version, where)+ logger.error(+ "Moderation model %s from %s was fitted on a different feature set than this "+ "code builds; ignoring it and falling back to the deterministic checks. "+ "Retrain to publish a compatible artifact.",+ model.version,+ where,+ )return None@@ -343,8 +354,7 @@ def _load(version: Optional[str]) -> Optional[ModerationModel]:# An artifact pickled by a different scikit-learn version is the realistic case, and# it must not take the enrichment task down with it: fall back to the model shipped# with this code, which was pickled by the dependencies this code is running.- logger.exception("Failed to load moderation model %s; falling back to the bundled "- "artifact", version)+ logger.exception("Failed to load moderation model %s; falling back to the bundled artifact", version)return _load_bundled()logger.info("Domain moderation model %s loaded from the database", model.version)return _checked(model, "the database")@@ -356,7 +366,9 @@ def _load_bundled() -> Optional[ModerationModel]:if not model_path.exists():logger.warning("Domain moderation model not found at %s and none published; suggestions will "- "use the deterministic checks only", model_path)+ "use the deterministic checks only",+ model_path,+ )return Nonetry:model = joblib.load(model_path)diff --git a/mwmbl/moderation/rules.py b/mwmbl/moderation/rules.pyindex fb60e14..157b429 100644--- a/mwmbl/moderation/rules.py+++ b/mwmbl/moderation/rules.py@@ -15,6 +15,7 @@ Two kinds of rule, split by whether the answer can change while the domain doesreads, and caching them would mean showing a moderator a stale "first submission from thisuser" after that user's tenth submission."""+from __future__ import annotationsfrom dataclasses import asdict, dataclass@@ -30,19 +31,53 @@ NEUTRAL = "neutral"# Domains we will not crawl whatever a submitter says, each of which has actually been# submitted and rejected with a hand-written explanation ("don't crawl ourselves", "same as# google.com no actual useful links apart from the search UI").-DO_NOT_CRAWL = frozenset({- "mwmbl.org", "www.mwmbl.org",- "google.com", "www.google.com", "scholar.google.com",-})+DO_NOT_CRAWL = frozenset(+ {+ "mwmbl.org",+ "www.mwmbl.org",+ "google.com",+ "www.google.com",+ "scholar.google.com",+ }+)# Country TLDs that mostly carry non-English sites. This only ever produces a *neutral* "worth# checking" line, never a rejection, because plenty of English sites live under .de and .nl -# the reject-or-not call belongs to the model, whose TLD block learned .pt, .tw and .cn as# reject-side features from actual decisions rather than from this list.-NON_ENGLISH_TLDS = frozenset({- "cn", "tw", "jp", "kr", "ru", "ua", "pl", "cz", "sk", "hu", "ro", "bg", "gr", "tr",- "pt", "es", "it", "fr", "de", "nl", "se", "no", "fi", "dk", "vn", "th", "id", "il", "ir",-})+NON_ENGLISH_TLDS = frozenset(+ {+ "cn",+ "tw",+ "jp",+ "kr",+ "ru",+ "ua",+ "pl",+ "cz",+ "sk",+ "hu",+ "ro",+ "bg",+ "gr",+ "tr",+ "pt",+ "es",+ "it",+ "fr",+ "de",+ "nl",+ "se",+ "no",+ "fi",+ "dk",+ "vn",+ "th",+ "id",+ "il",+ "ir",+ }+)# How much an earlier decision on the same domain is worth. Named because the queue's SQL# reproduces this check to filter and order on it (mwmbl.moderation.suggest), and the two@@ -54,9 +89,10 @@ PRIOR_DECISION_REASON = "OTHER"@dataclass(frozen=True)class EvidenceItem:"""One checkable fact, shown to the moderator as a line in the evidence list."""+kind: str- direction: str # REJECT | APPROVE | NEUTRAL- label: str # moderator-facing, e.g. "Homepage returns 404"+ direction: str # REJECT | APPROVE | NEUTRAL+ label: str # moderator-facing, e.g. "Homepage returns 404"# Set when the fact is decisive on its own. The suggester takes the strongest of these# in preference to the model, because a 404 is not a matter of opinion.implies_action: Optional[str] = None@@ -78,10 +114,16 @@ def crawl_evidence(domain: str, crawl: dict) -> list[EvidenceItem]:signals = crawl.get("signals") or {}if domain in DO_NOT_CRAWL:- items.append(EvidenceItem(- "do_not_crawl", REJECT, "On the do-not-crawl list (we don't crawl ourselves, "- "or it's another search engine)",- implies_action="REJECT", implies_reason="OTHER", implies_confidence=0.99))+ items.append(+ EvidenceItem(+ "do_not_crawl",+ REJECT,+ "On the do-not-crawl list (we don't crawl ourselves, or it's another search engine)",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.99,+ )+ )error = crawl.get("error") or ""status = crawl.get("http_status")@@ -91,23 +133,46 @@ def crawl_evidence(domain: str, crawl: dict) -> list[EvidenceItem]:# distinct rejection details moderators have written mentions robots.txt. The# crawler respects robots regardless of what is approved, so this is information a# moderator should have, not a decision to make for them.- items.append(EvidenceItem(- "robots", NEUTRAL,- "robots.txt forbids our crawler - approving will not make this site crawlable"))+ items.append(+ EvidenceItem(+ "robots", NEUTRAL, "robots.txt forbids our crawler - approving will not make this site crawlable"+ )+ )elif error:- items.append(EvidenceItem(- "unreachable", REJECT, f"Could not be fetched ({error})",- implies_action="REJECT", implies_reason="OTHER", implies_confidence=0.9))+ items.append(+ EvidenceItem(+ "unreachable",+ REJECT,+ f"Could not be fetched ({error})",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.9,+ )+ )elif status is not None and not 200 <= status < 300:- items.append(EvidenceItem(- "http_status", REJECT, f"Homepage returns HTTP {status}",- implies_action="REJECT", implies_reason="OTHER", implies_confidence=0.9))+ items.append(+ EvidenceItem(+ "http_status",+ REJECT,+ f"Homepage returns HTTP {status}",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.9,+ )+ )final_domain = crawl.get("final_domain") or ""if final_domain and registrable(final_domain) != registrable(domain):- items.append(EvidenceItem(- "redirect", REJECT, f"Redirects to {final_domain}",- implies_action="REJECT", implies_reason="OTHER", implies_confidence=0.85))+ items.append(+ EvidenceItem(+ "redirect",+ REJECT,+ f"Redirects to {final_domain}",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.85,+ )+ )# Only meaningful when we actually got a page. After a failed or forbidden fetch there is# trivially no title and no text, and reporting three further "problems" would pad the@@ -116,11 +181,9 @@ def crawl_evidence(domain: str, crawl: dict) -> list[EvidenceItem]:if fetched and not any(page.get("title") for page in pages):items.append(EvidenceItem("no_title", REJECT, "No page on the site has a title"))if fetched and not any(page.get("extract") for page in pages):- items.append(EvidenceItem(- "no_extract", REJECT, "No readable body text found on any page"))+ items.append(EvidenceItem("no_extract", REJECT, "No readable body text found on any page"))if fetched and not signals.get("has_links"):- items.append(EvidenceItem(- "no_links", REJECT, "No links found to crawl on from the homepage"))+ items.append(EvidenceItem("no_links", REJECT, "No links found to crawl on from the homepage"))# Language is left to the model rather than asserted here. crawl_url does not surface the# <html lang> attribute, and the one behavioural clue - justext finding no 'good'@@ -128,9 +191,9 @@ def crawl_evidence(domain: str, crawl: dict) -> list[EvidenceItem]:# on a JavaScript-rendered English site. So: a hint for the moderator, and the model's TLD# block (which learned .pt, .tw and .cn as reject-side features) does the calling.if _tld(domain) in NON_ENGLISH_TLDS:- items.append(EvidenceItem(- "tld_language", NEUTRAL,- f"Country domain (.{_tld(domain)}) - check the site is in English"))+ items.append(+ EvidenceItem("tld_language", NEUTRAL, f"Country domain (.{_tld(domain)}) - check the site is in English")+ )if signals.get("blacklisted"):# Strong evidence, but not decisive, and the reason is written down in@@ -139,10 +202,14 @@ def crawl_evidence(domain: str, crawl: dict) -> list[EvidenceItem]:# and character.ai are all in The Block List Project's porn.txt. Overriding those is# precisely what an approval is *for*, so a decisive rejection here would have the# tool arguing against the mechanism it is supposed to be serving.- items.append(EvidenceItem(- "blocklist", REJECT,- "On a public malware/adult blocklist - approving here also unblocks it. "- "These lists are built for ad-blocking and do have false positives."))+ items.append(+ EvidenceItem(+ "blocklist",+ REJECT,+ "On a public malware/adult blocklist - approving here also unblocks it. "+ "These lists are built for ad-blocking and do have false positives.",+ )+ )# Neutral for the same reason robots.txt is: a fact the moderator should have, not a# decision to make for them. Plenty of small personal sites - the ones this index exists@@ -151,14 +218,16 @@ def crawl_evidence(domain: str, crawl: dict) -> list[EvidenceItem]:# fetch "https: False" says the https attempt failed, which the unreachable line already# said better.if fetched and signals.get("https") is False:- items.append(EvidenceItem(- "no_tls", NEUTRAL, "Served over plain HTTP - no TLS certificate"))+ items.append(EvidenceItem("no_tls", NEUTRAL, "Served over plain HTTP - no TLS certificate"))if fetched:- items.append(EvidenceItem(- "reachable", APPROVE,- f"Crawled {sum(1 for page in pages if page.get('title'))} of {len(pages)} "- f"page(s) successfully"))+ items.append(+ EvidenceItem(+ "reachable",+ APPROVE,+ f"Crawled {sum(1 for page in pages if page.get('title'))} of {len(pages)} page(s) successfully",+ )+ )return items@@ -177,22 +246,37 @@ def live_evidence(submitter_record: dict, prior_decisions: dict) -> list[Evidencif total == 0:# Worth saying plainly: on the last year of decisions, submissions from someone with no# track record were rejected 54% of the time, against 1% for established submitters.- items.append(EvidenceItem(- "submitter", NEUTRAL, "First submission from this user - no track record yet"))+ items.append(EvidenceItem("submitter", NEUTRAL, "First submission from this user - no track record yet"))else:- items.append(EvidenceItem(- "submitter", APPROVE if rejected == 0 and approved >= 5 else NEUTRAL,- f"Submitter has {approved} approved and {rejected} rejected"))+ items.append(+ EvidenceItem(+ "submitter",+ APPROVE if rejected == 0 and approved >= 5 else NEUTRAL,+ f"Submitter has {approved} approved and {rejected} rejected",+ )+ )if prior_decisions.get("approved"):- items.append(EvidenceItem(- "prior_decision", APPROVE, "This domain has already been approved before",- implies_action="APPROVE", implies_confidence=PRIOR_DECISION_CONFIDENCE))+ items.append(+ EvidenceItem(+ "prior_decision",+ APPROVE,+ "This domain has already been approved before",+ implies_action="APPROVE",+ implies_confidence=PRIOR_DECISION_CONFIDENCE,+ )+ )elif prior_decisions.get("rejected"):- items.append(EvidenceItem(- "prior_decision", REJECT, "This domain has already been rejected before",- implies_action="REJECT", implies_reason=PRIOR_DECISION_REASON,- implies_confidence=PRIOR_DECISION_CONFIDENCE))+ items.append(+ EvidenceItem(+ "prior_decision",+ REJECT,+ "This domain has already been rejected before",+ implies_action="REJECT",+ implies_reason=PRIOR_DECISION_REASON,+ implies_confidence=PRIOR_DECISION_CONFIDENCE,+ )+ )return items@@ -251,8 +335,7 @@ def other_detail(items: list[EvidenceItem]) -> str:That is also the only form of the question SQL can ask of stored evidence - seeIMPLIES_OTHER - which is what lets the queue agree with what the moderator is shown."""- implying_other = [item for item in items- if item.implies_action == "REJECT" and item.implies_reason == "OTHER"]+ implying_other = [item for item in items if item.implies_action == "REJECT" and item.implies_reason == "OTHER"]if not implying_other:return ""return implied_detail(max(implying_other, key=lambda item: item.implies_confidence))diff --git a/mwmbl/moderation/suggest.py b/mwmbl/moderation/suggest.pyindex 8f01fd1..4866b4c 100644--- a/mwmbl/moderation/suggest.py+++ b/mwmbl/moderation/suggest.py@@ -18,20 +18,32 @@ decide which ones to return.queryset into the row-per-domain the moderator actually reviews. Both are SQL for the samereason."""+from __future__ import annotationsfrom logging import getLoggerfrom typing import Iterablefrom django.db.models import (- Case, CharField, Count, Exists, F, FloatField, IntegerField, OuterRef, Q, Subquery, Value,- When)+ Case,+ CharField,+ Count,+ Exists,+ F,+ FloatField,+ IntegerField,+ OuterRef,+ Q,+ Subquery,+ Value,+ When,+)from django.db.models.functions import Coalesce, Substrfrom mwmbl.models import DomainEvidence, DomainSubmission, SearchResultVotefrom mwmbl.moderation import rulesfrom mwmbl.moderation.evidence import page_texts-from mwmbl.moderation.model import Suggestion, UNUSABLE_MODEL_REASONS, suggest+from mwmbl.moderation.model import UNUSABLE_MODEL_REASONS, Suggestion, suggestlogger = getLogger(__name__)@@ -48,17 +60,26 @@ def refresh_suggestion(evidence: DomainEvidence) -> DomainEvidence:evidence.reason_source = suggestion.reason_sourceevidence.model_version = suggestion.model_versionevidence.evidence = suggestion.evidence- evidence.save(update_fields=[- "suggested_action", "suggested_reason", "confidence", "reason_confidence",- "reason_source", "model_version", "evidence",- ])+ evidence.save(+ update_fields=[+ "suggested_action",+ "suggested_reason",+ "confidence",+ "reason_confidence",+ "reason_source",+ "model_version",+ "evidence",+ ]+ )return evidence-def suggestion_for(submission: DomainSubmission,- evidence: DomainEvidence | None,- submitter: dict | None = None,- prior: dict | None = None) -> Suggestion | None:+def suggestion_for(+ submission: DomainSubmission,+ evidence: DomainEvidence | None,+ submitter: dict | None = None,+ prior: dict | None = None,+) -> Suggestion | None:"""The stored suggestion, plus live evidence. None while the domain is still being crawled.Returning None rather than a default is deliberate: a queue row that says "not assessed@@ -81,8 +102,8 @@ def suggestion_for(submission: DomainSubmission,cached = [rules.EvidenceItem(**item) for item in (evidence.evidence or [])]cached_decisive = rules.decisive(cached)if decisive_live is not None and (- cached_decisive is None- or decisive_live.implies_confidence > cached_decisive.implies_confidence):+ cached_decisive is None or decisive_live.implies_confidence > cached_decisive.implies_confidence+ ):return Suggestion(action=decisive_live.implies_action,confidence=decisive_live.implies_confidence,@@ -177,29 +198,25 @@ def annotate_queue(submissions):evidence_source=Subquery(evidence.values("reason_source")[:1]),# The live checks, as one boolean each. Only PENDING submissions are ever queued, so# a submission never counts itself in either of these.- prior_approved=Exists(DomainSubmission.objects.filter(- name=OuterRef("name"), status="APPROVED")),- prior_rejected=Exists(DomainSubmission.objects.filter(- name=OuterRef("name"), status="REJECTED")),- submitter_decided=Exists(DomainSubmission.objects.filter(- DECIDED, submitted_by_id=OuterRef("submitted_by_id"))),+ prior_approved=Exists(DomainSubmission.objects.filter(name=OuterRef("name"), status="APPROVED")),+ prior_rejected=Exists(DomainSubmission.objects.filter(name=OuterRef("name"), status="REJECTED")),+ submitter_decided=Exists(DomainSubmission.objects.filter(DECIDED, submitted_by_id=OuterRef("submitted_by_id"))),# rules.other_detail, as far as SQL can ask it: is there a cached check implying# OTHER, and so a sentence to send with an OTHER rejection. A containment test rather# than a column comparison because it is a question about the evidence list, and# Exists rather than a lookup on the annotation above so it reads as the join it is.- explained_by_a_check=Exists(DomainEvidence.objects.filter(- domain=OuterRef("name"), evidence__contains=rules.IMPLIES_OTHER)),+ explained_by_a_check=Exists(+ DomainEvidence.objects.filter(domain=OuterRef("name"), evidence__contains=rules.IMPLIES_OTHER)+ ),)scored = Q(evidence_state=DomainEvidence.State.READY)# A prior decision only overrides the stored suggestion when it is stronger than the check# that produced it. reason_source is how we know a check produced it at all: when it did,# the stored confidence *is* that check's implied confidence (see model.suggest).- stored_check_holds = (Q(evidence_source="rule")- & Q(evidence_confidence__gte=rules.PRIOR_DECISION_CONFIDENCE))+ stored_check_holds = Q(evidence_source="rule") & Q(evidence_confidence__gte=rules.PRIOR_DECISION_CONFIDENCE)prior_approval = scored & Q(prior_approved=True) & ~stored_check_holds- prior_rejection = (scored & Q(prior_rejected=True) & Q(prior_approved=False)- & ~stored_check_holds)+ prior_rejection = scored & Q(prior_rejected=True) & Q(prior_approved=False) & ~stored_check_holds# No cached check implies APPROVE - they are all reasons to reject - so a stored APPROVE# is always the model's, which is what the withheld-approval rule is about.withheld = scored & Q(evidence_action="APPROVE") & Q(submitter_decided=False)@@ -209,9 +226,11 @@ def annotate_queue(submissions):# produced the reason, and the evidence still has to contain the check that explains it -# a stale row can say "rule" and have nothing implying OTHER left in its evidence.explained = Q(evidence_source="rule") & Q(explained_by_a_check=True)- unexplained = scored & Q(evidence_action="REJECT") & (- (Q(evidence_reason="OTHER") & ~explained)- | Q(evidence_reason="") | Q(evidence_reason__isnull=True))+ unexplained = (+ scored+ & Q(evidence_action="REJECT")+ & ((Q(evidence_reason="OTHER") & ~explained) | Q(evidence_reason="") | Q(evidence_reason__isnull=True))+ )submissions = submissions.annotate(displayed_action=Case(@@ -224,8 +243,7 @@ def annotate_queue(submissions):output_field=CharField(),),displayed_confidence=Case(- When(prior_approval | prior_rejection,- then=Value(rules.PRIOR_DECISION_CONFIDENCE)),+ When(prior_approval | prior_rejection, then=Value(rules.PRIOR_DECISION_CONFIDENCE)),When(unexplained, then=Value(0.0)),When(withheld, then=Value(0.0)),When(scored, then=F("evidence_confidence")),@@ -234,8 +252,7 @@ def annotate_queue(submissions):),displayed_reason=Case(When(prior_rejection, then=Value(rules.PRIOR_DECISION_REASON)),- When(scored & Q(evidence_action="REJECT") & ~prior_approval & ~unexplained,- then=F("evidence_reason")),+ When(scored & Q(evidence_action="REJECT") & ~prior_approval & ~unexplained, then=F("evidence_reason")),When(scored, then=Value("")),default=Value(None),output_field=CharField(),@@ -291,17 +308,18 @@ def one_row_per_domain(submissions, status: str | None = "PENDING", pick: str =if pick == "first":beats = same_domain.filter(Q(submitted_on__lt=OuterRef("submitted_on"))- | Q(submitted_on=OuterRef("submitted_on"), pk__lt=OuterRef("pk")))+ | Q(submitted_on=OuterRef("submitted_on"), pk__lt=OuterRef("pk"))+ )else:beats = same_domain.annotate(touched=touched).filter(- Q(touched__gt=OuterRef("touched"))- | Q(touched=OuterRef("touched"), pk__gt=OuterRef("pk")))+ Q(touched__gt=OuterRef("touched")) | Q(touched=OuterRef("touched"), pk__gt=OuterRef("pk"))+ )return submissions.annotate(submission_count=Coalesce(- Subquery(same_domain.values("name").annotate(n=Count("pk")).values("n"),- output_field=IntegerField()),- Value(0)),+ Subquery(same_domain.values("name").annotate(n=Count("pk")).values("n"), output_field=IntegerField()),+ Value(0),+ ),).filter(~Exists(beats))@@ -316,17 +334,23 @@ def annotate_votes(submissions):Coalesced to zero rather than left NULL: a domain nobody has voted on has no votes, andordering on NULL would scatter those rows through the sort instead of ending it."""+def counted(vote_type):return Coalesce(- Subquery(SearchResultVote.objects- .filter(domain=OuterRef("bare_name"), vote_type=vote_type)- .values("domain").annotate(n=Count("pk")).values("n"),- output_field=IntegerField()),- Value(0))+ Subquery(+ SearchResultVote.objects.filter(domain=OuterRef("bare_name"), vote_type=vote_type)+ .values("domain")+ .annotate(n=Count("pk"))+ .values("n"),+ output_field=IntegerField(),+ ),+ Value(0),+ )return submissions.annotate(- bare_name=Case(When(name__startswith="www.", then=Substr("name", 5)),- default=F("name"), output_field=CharField()),+ bare_name=Case(+ When(name__startswith="www.", then=Substr("name", 5)), default=F("name"), output_field=CharField()+ ),).annotate(upvotes=counted("upvote"), downvotes=counted("downvote"))@@ -337,32 +361,30 @@ def submitter_record(user_id: int) -> dict:rejected 54% of the time against 1% for an established one, which is a bigger effect thananything the model reads off the domain name."""- counts = (DomainSubmission.objects- .filter(submitted_by_id=user_id)- .aggregate(approved=Count("pk", filter=Q(status="APPROVED")),- rejected=Count("pk", filter=Q(status="REJECTED"))))+ counts = DomainSubmission.objects.filter(submitted_by_id=user_id).aggregate(+ approved=Count("pk", filter=Q(status="APPROVED")), rejected=Count("pk", filter=Q(status="REJECTED"))+ )return {"approved": counts["approved"] or 0, "rejected": counts["rejected"] or 0}def prior_decisions(submission: DomainSubmission) -> dict:"""How other submissions of the same domain were decided."""- counts = (DomainSubmission.objects- .filter(name=submission.name)- .exclude(pk=submission.pk)- .aggregate(approved=Count("pk", filter=Q(status="APPROVED")),- rejected=Count("pk", filter=Q(status="REJECTED"))))+ counts = (+ DomainSubmission.objects.filter(name=submission.name)+ .exclude(pk=submission.pk)+ .aggregate(approved=Count("pk", filter=Q(status="APPROVED")), rejected=Count("pk", filter=Q(status="REJECTED")))+ )return {"approved": counts["approved"] or 0, "rejected": counts["rejected"] or 0}def submitter_records(user_ids: Iterable[int]) -> dict[int, dict]:"""Decision counts for several submitters in one query, for rendering a queue page."""- counts = (DomainSubmission.objects- .filter(submitted_by_id__in=list(user_ids))- .values("submitted_by_id")- .annotate(approved=Count("pk", filter=Q(status="APPROVED")),- rejected=Count("pk", filter=Q(status="REJECTED"))))- return {row["submitted_by_id"]: {"approved": row["approved"], "rejected": row["rejected"]}- for row in counts}+ counts = (+ DomainSubmission.objects.filter(submitted_by_id__in=list(user_ids))+ .values("submitted_by_id")+ .annotate(approved=Count("pk", filter=Q(status="APPROVED")), rejected=Count("pk", filter=Q(status="REJECTED")))+ )+ return {row["submitted_by_id"]: {"approved": row["approved"], "rejected": row["rejected"]} for row in counts}def prior_decision_counts(names: Iterable[str]) -> dict[str, dict]:@@ -372,10 +394,9 @@ def prior_decision_counts(names: Iterable[str]) -> dict[str, dict]:the row being rendered. That is harmless here because the queue only ever shows PENDINGsubmissions, which are by definition not counted."""- counts = (DomainSubmission.objects- .filter(name__in=list(names))- .values("name")- .annotate(approved=Count("pk", filter=Q(status="APPROVED")),- rejected=Count("pk", filter=Q(status="REJECTED"))))- return {row["name"]: {"approved": row["approved"], "rejected": row["rejected"]}- for row in counts}+ counts = (+ DomainSubmission.objects.filter(name__in=list(names))+ .values("name")+ .annotate(approved=Count("pk", filter=Q(status="APPROVED")), rejected=Count("pk", filter=Q(status="REJECTED")))+ )+ return {row["name"]: {"approved": row["approved"], "rejected": row["rejected"]} for row in counts}diff --git a/mwmbl/moderation/train.py b/mwmbl/moderation/train.pyindex 81cd0e8..002c16f 100644--- a/mwmbl/moderation/train.py+++ b/mwmbl/moderation/train.py@@ -24,6 +24,7 @@ a different population. It is :func:`compare_models`: score the incumbent and ththe *same* held-out rows and bootstrap the *difference*, which cancels the shared test-setvariance and is far more sensitive than comparing two independent intervals."""+from __future__ import annotationsfrom collections import Counter@@ -57,8 +58,15 @@ GATE_TOLERANCE = 0.05# Statistics bootstrapped alongside PR-AUC. Named here rather than inferred from the point# estimate, so a statistic that is undefined in *every* resample - reject precision on a slice# where nothing clears the threshold - still reports, as null, rather than vanishing.-OPERATING_POINT_KEYS = ["reject_share", "reject_precision", "reject_recall", "approve_share",- "approve_error_rate", "unsure_share", "recall_at_precision_75"]+OPERATING_POINT_KEYS = [+ "reject_share",+ "reject_precision",+ "reject_recall",+ "approve_share",+ "approve_error_rate",+ "unsure_share",+ "recall_at_precision_75",+]@dataclass@@ -85,8 +93,7 @@ class Evaluation:"pr_auc": round(self.pr_auc, 4),"pr_auc_ci": [round(self.pr_auc_ci[0], 4), round(self.pr_auc_ci[1], 4)],"normalised_ap": round(self.normalised_ap, 4),- "normalised_ap_ci": [round(self.normalised_ap_ci[0], 4),- round(self.normalised_ap_ci[1], 4)],+ "normalised_ap_ci": [round(self.normalised_ap_ci[0], 4), round(self.normalised_ap_ci[1], 4)],# What a moderator actually experiences. PR-AUC is a summary over every threshold,# including ones nothing runs at; the server suggests REJECT at# MODERATION_REJECT_THRESHOLD and APPROVE at MODERATION_APPROVE_THRESHOLD, and a@@ -112,16 +119,19 @@ def split_by_time(rows: list[TrainingRow]) -> tuple[list[TrainingRow], list[Traiphenomenon that does not appear in 2024 - and a random split would let the model learnfrom the future it is being tested on."""- real = sorted((row for row in rows if row.source == REAL),- key=lambda row: row.timestamp or "")+ real = sorted((row for row in rows if row.source == REAL), key=lambda row: row.timestamp or "")other = [row for row in rows if row.source != REAL]cut = int((1 - TEST_FRACTION) * len(real))return real[:cut] + other, real[cut:]-def train(rows: list[TrainingRow], version: str, use_text: bool = True,- use_has_text: bool = False,- incumbent: Optional[ModerationModel] = None) -> tuple[ModerationModel, dict]:+def train(+ rows: list[TrainingRow],+ version: str,+ use_text: bool = True,+ use_has_text: bool = False,+ incumbent: Optional[ModerationModel] = None,+) -> tuple[ModerationModel, dict]:"""Fit both heads and evaluate on held-out real rows only.``use_text`` and ``use_has_text`` are the two halves of the page-text ablation - the@@ -132,8 +142,12 @@ def train(rows: list[TrainingRow], version: str, use_text: bool = True,test rows, which is the only comparison that means anything."""train_rows, test_rows = split_by_time(rows)- logger.info("Training on %d rows (%d real), testing on %d real rows",- len(train_rows), sum(r.source == REAL for r in train_rows), len(test_rows))+ logger.info(+ "Training on %d rows (%d real), testing on %d real rows",+ len(train_rows),+ sum(r.source == REAL for r in train_rows),+ len(test_rows),+ )featuriser = Featuriser(use_text=use_text, use_has_text=use_has_text)features = featuriser.fit_transform([row.to_example() for row in train_rows])@@ -141,23 +155,25 @@ def train(rows: list[TrainingRow], version: str, use_text: bool = True,# The binary head sees real rows only: blanket augmentation was measured and did not help.real_mask = np.array([row.source == REAL for row in train_rows])reject_head = LogisticRegression(max_iter=4000, C=4.0, class_weight="balanced")- reject_head.fit(features[real_mask],- [row.rejected for row, keep in zip(train_rows, real_mask) if keep])+ reject_head.fit(features[real_mask], [row.rejected for row, keep in zip(train_rows, real_mask) if keep])# The reason head sees every source, because that is the whole point of the derived rows.reason_mask = np.array([row.rejected and bool(row.reason) for row in train_rows])reason_head = LogisticRegression(max_iter=4000, C=4.0, class_weight="balanced")- reason_head.fit(features[reason_mask],- [row.reason for row, keep in zip(train_rows, reason_mask) if keep])+ reason_head.fit(features[reason_mask], [row.reason for row, keep in zip(train_rows, reason_mask) if keep])- model = ModerationModel(featuriser, reject_head, reason_head, version,- train_domains={row.domain for row in train_rows})+ model = ModerationModel(+ featuriser, reject_head, reason_head, version, train_domains={row.domain for row in train_rows}+ )return model, evaluate(model, train_rows, test_rows, incumbent)-def evaluate(model: ModerationModel, train_rows: list[TrainingRow],- test_rows: list[TrainingRow],- incumbent: Optional[ModerationModel] = None) -> dict:+def evaluate(+ model: ModerationModel,+ train_rows: list[TrainingRow],+ test_rows: list[TrainingRow],+ incumbent: Optional[ModerationModel] = None,+) -> dict:if not test_rows:return {"error": "no held-out real rows to evaluate on"}@@ -184,20 +200,22 @@ def evaluate(model: ModerationModel, train_rows: list[TrainingRow],metrics["train_rows_by_source"] = _count_sources(train_rows)# Fit-time text coverage, alongside the per-slice test coverage above. A large gap between# the two is the train/test skew that makes the text block look worse than it is.- metrics["train_rows_with_text"] = sum(- 1 for row in train_rows if row.to_example().text.strip())+ metrics["train_rows_with_text"] = sum(1 for row in train_rows if row.to_example().text.strip())metrics["uses_text"] = model.featuriser.text is not Nonemetrics["uses_has_text"] = model.featuriser.use_has_textmetrics["suggestion_influence"] = _suggestion_influence(train_rows + test_rows)if incumbent is not None:- metrics["versus_incumbent"] = compare_models(- model, incumbent, test_rows, cold, reject_scores)+ metrics["versus_incumbent"] = compare_models(model, incumbent, test_rows, cold, reject_scores)return metrics-def compare_models(candidate: ModerationModel, incumbent: ModerationModel,- test_rows: list[TrainingRow], cold: np.ndarray,- candidate_scores: np.ndarray) -> dict:+def compare_models(+ candidate: ModerationModel,+ incumbent: ModerationModel,+ test_rows: list[TrainingRow],+ cold: np.ndarray,+ candidate_scores: np.ndarray,+) -> dict:"""Paired bootstrap of candidate minus incumbent on identical cold-start rows.Two models scored on the same rows share all the variance that comes from *which* rows@@ -211,16 +229,19 @@ def compare_models(candidate: ModerationModel, incumbent: ModerationModel,they trained on; the comparison still runs, and says so, because a candidate that wins undera bias in the incumbent's favour has won."""- incumbent_scores = np.array([prediction[0] for prediction in- incumbent.predict([row.to_example() for row in test_rows])])+ incumbent_scores = np.array(+ [prediction[0] for prediction in incumbent.predict([row.to_example() for row in test_rows])]+ )contamination_known = bool(incumbent.train_domains)seen = incumbent.train_domains or set()clean = np.array([row.domain not in seen for row in test_rows])mask = cold & cleanif not _has_both_classes(np.array([row.rejected for row in test_rows], dtype=int)[mask]):- return {"note": "not enough uncontaminated cold-start rows to compare on",- "incumbent_version": incumbent.version}+ return {+ "note": "not enough uncontaminated cold-start rows to compare on",+ "incumbent_version": incumbent.version,+ }truth = np.array([row.rejected for row in test_rows], dtype=int)[mask]candidate_slice = candidate_scores[mask]@@ -235,11 +256,13 @@ def compare_models(candidate: ModerationModel, incumbent: ModerationModel,base = truth[indices].mean()differences.append(normalise(average_precision_score(truth[indices], candidate_slice[indices]), base)- - normalise(average_precision_score(truth[indices], incumbent_slice[indices]), base))+ - normalise(average_precision_score(truth[indices], incumbent_slice[indices]), base)+ )base_rate = float(truth.mean())- difference = (normalise(average_precision_score(truth, candidate_slice), base_rate)- - normalise(average_precision_score(truth, incumbent_slice), base_rate))+ difference = normalise(average_precision_score(truth, candidate_slice), base_rate) - normalise(+ average_precision_score(truth, incumbent_slice), base_rate+ )return {"slice": "cold_start","incumbent_version": incumbent.version,@@ -249,13 +272,13 @@ def compare_models(candidate: ModerationModel, incumbent: ModerationModel,# False means the incumbent predates train-domain recording, so some of these rows may# be its own training data and the comparison leans in its favour."contamination_known": contamination_known,- "candidate_normalised_ap": round(- normalise(average_precision_score(truth, candidate_slice), base_rate), 4),- "incumbent_normalised_ap": round(- normalise(average_precision_score(truth, incumbent_slice), base_rate), 4),+ "candidate_normalised_ap": round(normalise(average_precision_score(truth, candidate_slice), base_rate), 4),+ "incumbent_normalised_ap": round(normalise(average_precision_score(truth, incumbent_slice), base_rate), 4),"difference": round(difference, 4),- "difference_ci": [round(float(np.percentile(differences, 2.5)), 4),- round(float(np.percentile(differences, 97.5)), 4)],+ "difference_ci": [+ round(float(np.percentile(differences, 2.5)), 4),+ round(float(np.percentile(differences, 97.5)), 4),+ ],# How often the candidate won a resample. Reads more directly than the interval when# the interval straddles zero, which on this much data it usually will."candidate_wins_fraction": round(float(np.mean(np.array(differences) > 0)), 4),@@ -292,8 +315,7 @@ def operating_point(truth: np.ndarray, scores: np.ndarray) -> dict:}-def _evaluate_slice(name: str, truth: np.ndarray, scores: np.ndarray,- has_text: np.ndarray) -> Evaluation:+def _evaluate_slice(name: str, truth: np.ndarray, scores: np.ndarray, has_text: np.ndarray) -> Evaluation:generator = np.random.default_rng(0)samples, normalised = [], []points: dict[str, list] = {key: [] for key in OPERATING_POINT_KEYS}@@ -324,11 +346,15 @@ def _evaluate_slice(name: str, truth: np.ndarray, scores: np.ndarray,positives=int(truth.sum()),base_rate=base_rate,pr_auc=float(average_precision_score(truth, scores)),- pr_auc_ci=(float(np.percentile(samples, 2.5)) if samples else 0.0,- float(np.percentile(samples, 97.5)) if samples else 0.0),+ pr_auc_ci=(+ float(np.percentile(samples, 2.5)) if samples else 0.0,+ float(np.percentile(samples, 97.5)) if samples else 0.0,+ ),normalised_ap=normalise(float(average_precision_score(truth, scores)), base_rate),- normalised_ap_ci=(float(np.percentile(normalised, 2.5)) if normalised else 0.0,- float(np.percentile(normalised, 97.5)) if normalised else 0.0),+ normalised_ap_ci=(+ float(np.percentile(normalised, 2.5)) if normalised else 0.0,+ float(np.percentile(normalised, 97.5)) if normalised else 0.0,+ ),operating_point=measured,rows_with_text=int(has_text.sum()),)@@ -338,8 +364,7 @@ def _interval(values: list) -> Optional[list]:"""A 95% interval, or None when no resample could measure the statistic at all."""if not values:return None- return [round(float(np.percentile(values, 2.5)), 4),- round(float(np.percentile(values, 97.5)), 4)]+ return [round(float(np.percentile(values, 2.5)), 4), round(float(np.percentile(values, 97.5)), 4)]def _evaluate_reason_head(predictions, test_rows: list[TrainingRow]) -> dict:@@ -388,8 +413,7 @@ def _suggestion_influence(rows: list[TrainingRow]) -> dict:# side, and a decision made against one agrees with nothing: it counts as shown, but is# not part of the population agreement is measured over.took_a_side = [row for row in shown if row.suggested_status in ("APPROVE", "REJECT")]- agreed = [row for row in took_a_side- if (row.suggested_status == "REJECT") == row.rejected]+ agreed = [row for row in took_a_side if (row.suggested_status == "REJECT") == row.rejected]return {"real_rows": len(real),"shown_a_suggestion": len(shown),@@ -420,8 +444,7 @@ def normalised_ap_of(slice_metrics: dict) -> Optional[tuple[float, float]]:if not slice_metrics.get("rows"):return Nonebase_rate = slice_metrics["positives"] / slice_metrics["rows"]- return (normalise(slice_metrics["pr_auc"], base_rate),- normalise(slice_metrics["pr_auc_ci"][0], base_rate))+ return (normalise(slice_metrics["pr_auc"], base_rate), normalise(slice_metrics["pr_auc_ci"][0], base_rate))def passes_gate(new: dict, current: dict) -> tuple[bool, str]:@@ -446,33 +469,41 @@ def passes_gate(new: dict, current: dict) -> tuple[bool, str]:comparison = new.get("versus_incumbent")if comparison is not None and "difference" in comparison:- caveat = "" if comparison["contamination_known"] else (- "; the incumbent has no record of its training domains, so some of these rows may "- "be its own and the comparison favours it")+ caveat = (+ ""+ if comparison["contamination_known"]+ else (+ "; the incumbent has no record of its training domains, so some of these rows may "+ "be its own and the comparison favours it"+ )+ )if comparison["difference_ci"][0] < -GATE_TOLERANCE:return False, (f"paired against {comparison['incumbent_version']} on {comparison['rows']} "f"cold-start rows, normalised AP is {comparison['difference']:+.3f} with "f"interval [{comparison['difference_ci'][0]:.3f}, "f"{comparison['difference_ci'][1]:.3f}], which admits a loss worse than the "- f"{GATE_TOLERANCE:.2f} tolerance{caveat}")+ f"{GATE_TOLERANCE:.2f} tolerance{caveat}"+ )return True, (f"paired against {comparison['incumbent_version']} on {comparison['rows']} "f"cold-start rows, normalised AP is {comparison['difference']:+.3f} "f"[{comparison['difference_ci'][0]:.3f}, {comparison['difference_ci'][1]:.3f}], "- f"winning {comparison['candidate_wins_fraction']:.0%} of resamples{caveat}")+ f"winning {comparison['candidate_wins_fraction']:.0%} of resamples{caveat}"+ )current_cold = current.get("cold_start")incumbent = normalised_ap_of(current_cold) if current_cold else Noneif incumbent is None:- return True, (f"no readable incumbent to compare against; cold-start normalised AP "- f"{candidate[0]:.3f}")+ return True, (f"no readable incumbent to compare against; cold-start normalised AP {candidate[0]:.3f}")point, floor = incumbentif candidate[0] < floor:return False, (f"cold-start normalised AP {candidate[0]:.3f} is below the incumbent's lower bound "f"{floor:.3f} (unpaired: the incumbent model could not be loaded, so these are "- f"different test sets and the comparison is weak)")- return True, (f"cold-start normalised AP {candidate[0]:.3f} "- f"(incumbent {point:.3f}, lower bound {floor:.3f}; unpaired)")+ f"different test sets and the comparison is weak)"+ )+ return True, (+ f"cold-start normalised AP {candidate[0]:.3f} (incumbent {point:.3f}, lower bound {floor:.3f}; unpaired)"+ )diff --git a/mwmbl/moderation/training_data.py b/mwmbl/moderation/training_data.pyindex 870a98d..192bb49 100644--- a/mwmbl/moderation/training_data.py+++ b/mwmbl/moderation/training_data.py@@ -44,6 +44,7 @@ Historic malformed submissions are excluded. 70 rows are ``null``, an IP addresscapitalised; migrations 0032/0033 fixed that at the API layer, so training on them would teachthe model to detect a problem that can no longer occur."""+from __future__ import annotationsimport json@@ -55,8 +56,7 @@ from typing import Iterable, Optionalimport requests-from mwmbl.indexer.blacklist_providers import (- AdultContentBlacklistProvider, HaGeZiBlacklistProvider)+from mwmbl.indexer.blacklist_providers import AdultContentBlacklistProvider, HaGeZiBlacklistProviderfrom mwmbl.models import DomainEvidence, DomainSubmissionfrom mwmbl.moderation.evidence import page_textsfrom mwmbl.moderation.features import ModerationExample@@ -80,8 +80,8 @@ MAX_DERIVED_DOMAIN_LENGTH = 40class TrainingRow:domain: strrejected: bool- reason: str # "" for approvals- source: str # REAL | DERIVED | SEED+ reason: str # "" for approvals+ source: str # REAL | DERIVED | SEEDpage_texts: list[str]timestamp: Optional[str] = None# Who submitted it. Not a feature - the model deliberately does not read the submitter,@@ -102,41 +102,39 @@ class TrainingRow:def is_trainable_domain(domain: str) -> bool:"""Reject the historic malformed names that the API no longer accepts."""- return (bool(domain)- and domain == domain.lower()- and VALID_DOMAIN_REGEX.fullmatch(domain) is not None)+ return bool(domain) and domain == domain.lower() and VALID_DOMAIN_REGEX.fullmatch(domain) is not Nonedef real_rows() -> list[TrainingRow]:"""Decided submissions, joined to whatever page text has been crawled for them."""evidence_by_domain = {- evidence.domain: evidence- for evidence in DomainEvidence.objects.filter(state=DomainEvidence.State.READY)+ evidence.domain: evidence for evidence in DomainEvidence.objects.filter(state=DomainEvidence.State.READY)}rows = []- submissions = (DomainSubmission.objects- .filter(status__in=("APPROVED", "REJECTED"))- .order_by("submitted_on"))+ submissions = DomainSubmission.objects.filter(status__in=("APPROVED", "REJECTED")).order_by("submitted_on")for submission in submissions.iterator():if not is_trainable_domain(submission.name):continuerejected = submission.status == "REJECTED"- rows.append(TrainingRow(- domain=submission.name,- rejected=rejected,- reason=submission.rejection_reason if rejected else "",- source=REAL,- page_texts=page_texts(evidence_by_domain.get(submission.name)),- timestamp=submission.submitted_on.isoformat() if submission.submitted_on else None,- submitter=str(submission.submitted_by_id),- suggested_status=submission.suggested_status,- ))+ rows.append(+ TrainingRow(+ domain=submission.name,+ rejected=rejected,+ reason=submission.rejection_reason if rejected else "",+ source=REAL,+ page_texts=page_texts(evidence_by_domain.get(submission.name)),+ timestamp=submission.submitted_on.isoformat() if submission.submitted_on else None,+ submitter=str(submission.submitted_by_id),+ suggested_status=submission.suggested_status,+ )+ )return rows-def derived_rows(reasons_needing_data: Iterable[str], per_reason: int,- exclude: set[str], seed: int = 0) -> list[TrainingRow]:+def derived_rows(+ reasons_needing_data: Iterable[str], per_reason: int, exclude: set[str], seed: int = 0+) -> list[TrainingRow]:"""Blocklist domains standing in for reason classes with too little real data.``exclude`` must contain every domain that appears in the real data, in either split, so a@@ -156,9 +154,9 @@ def derived_rows(reasons_needing_data: Iterable[str], per_reason: int,pool = [domain for domain in _fetch_apex_domains(url) if domain not in exclude]random.Random(seed).shuffle(pool)rows.extend(- TrainingRow(domain=domain, rejected=True, reason=reason, source=DERIVED,- page_texts=[])- for domain in pool[:per_reason])+ TrainingRow(domain=domain, rejected=True, reason=reason, source=DERIVED, page_texts=[])+ for domain in pool[:per_reason]+ )logger.info("Derived %d %s rows from %s", min(per_reason, len(pool)), reason, url)return rows@@ -173,13 +171,15 @@ def seed_rows(path: Path = SEED_LABELS_PATH) -> list[TrainingRow]:if not line or line.startswith("#"):continuerecord = json.loads(line)- rows.append(TrainingRow(- domain=record["domain"],- rejected=record["status"] == "REJECTED",- reason=record.get("reason", ""),- source=SEED,- page_texts=[record["text"]] if record.get("text") else [],- ))+ rows.append(+ TrainingRow(+ domain=record["domain"],+ rejected=record["status"] == "REJECTED",+ reason=record.get("reason", ""),+ source=SEED,+ page_texts=[record["text"]] if record.get("text") else [],+ )+ )return rows@@ -194,8 +194,7 @@ def _fetch_apex_domains(url: str) -> list[str]:if not line:continueparts = line.split()- domain = (parts[1] if len(parts) >= 2 and parts[0] in ("0.0.0.0", "127.0.0.1")- else parts[0])+ domain = parts[1] if len(parts) >= 2 and parts[0] in ("0.0.0.0", "127.0.0.1") else parts[0]labels = domain.split(".")if len(labels) != 2 or len(domain) > MAX_DERIVED_DOMAIN_LENGTH:continuediff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.pyindex e58b04d..e252d38 100644--- a/mwmbl/platform/api.py+++ b/mwmbl/platform/api.py@@ -2,7 +2,7 @@ import loggingfrom allauth.account.adapter import get_adapterfrom allauth.account.models import EmailConfirmationHMAC-from allauth.account.utils import setup_user_email, send_email_confirmation+from allauth.account.utils import send_email_confirmation, setup_user_emailfrom django.conf import settingsfrom django.core import signingfrom django.core.exceptions import ValidationError@@ -15,30 +15,70 @@ from ninja_jwt.authentication import JWTAuthfrom polar_sdk import Polarfrom polar_sdk import models as polar_modelsfrom polar_sdk.models import SubscriptionCancel-from polar_sdk.webhooks import validate_event, WebhookVerificationError+from polar_sdk.webhooks import WebhookVerificationError, validate_event-from mwmbl.exceptions import InvalidRequest-from mwmbl.search_auth import invalidate_api_key_cache, invalidate_user_api_key_cache-from mwmbl.utils import normalize_domain, validate_domainfrom mwmbl import pricingfrom mwmbl.background import enrich_domain_submission, stats_manager-from mwmbl.models import AgreementType, MwmblUser, DomainEvidence, DomainSubmission, SearchResultVote, ApiKey, UsageBucket, UserBilling, UserAgreement, MarketingConsent, MarketingSource, generate_username+from mwmbl.exceptions import InvalidRequest+from mwmbl.models import (+ AgreementType,+ ApiKey,+ DomainEvidence,+ DomainSubmission,+ MarketingConsent,+ MarketingSource,+ MwmblUser,+ SearchResultVote,+ UserAgreement,+ UserBilling,+)from mwmbl.moderation.suggest import (- annotate_queue, annotate_votes, one_row_per_domain, prior_decision_counts, prior_decisions,- submitter_record, submitter_records, suggestion_for,+ annotate_queue,+ annotate_votes,+ one_row_per_domain,+ prior_decision_counts,+ prior_decisions,+ submitter_record,+ submitter_records,+ suggestion_for,)-from mwmbl.signals import schedule_blacklist_rebuildfrom mwmbl.platform.schemas import (- Registration, ConfirmEmail, DomainSubmissionSchema, UpdateDomainSubmission,- VoteRequest, VoteRemoveRequest, VoteStatsRequest, VoteResponse, VoteStats, UserVoteHistory,- CreateApiKeyRequest, ApiKeyCreatedResponse, ApiKeyListItem,- UserProfileResponse, SubscriptionResponse, CheckoutRequest, CheckoutResponse, UpdateSpendLimitRequest,- ForgotPasswordRequest, ResetPasswordRequest,- AgreementAcceptRequest, AgreementResponse,- MarketingConsentRequest, MarketingConsentResponse, MarketingConsentListResponse,- BulkDecisionRequest, ModeratedDomainSchema, ModerationHistory, ModerationQueue,- QueueItemSchema, SubmissionDetailSchema,+ AgreementAcceptRequest,+ AgreementResponse,+ ApiKeyCreatedResponse,+ ApiKeyListItem,+ BulkDecisionRequest,+ CheckoutRequest,+ CheckoutResponse,+ ConfirmEmail,+ CreateApiKeyRequest,+ DomainSubmissionSchema,+ ForgotPasswordRequest,+ MarketingConsentListResponse,+ MarketingConsentRequest,+ MarketingConsentResponse,+ ModeratedDomainSchema,+ ModerationHistory,+ ModerationQueue,+ QueueItemSchema,+ Registration,+ ResetPasswordRequest,+ SubmissionDetailSchema,+ SubscriptionResponse,+ UpdateDomainSubmission,+ UpdateSpendLimitRequest,+ UserProfileResponse,+ UserVoteHistory,+ VoteRemoveRequest,+ VoteRequest,+ VoteResponse,+ VoteStats,+ VoteStatsRequest,)+from mwmbl.search_auth import invalidate_api_key_cache, invalidate_user_api_key_cache+from mwmbl.signals import schedule_blacklist_rebuild+from mwmbl.usernames import generate_username+from mwmbl.utils import normalize_domain, validate_domainlogger = logging.getLogger(__name__)@@ -52,7 +92,7 @@ def check_email_verified(request):@router.post(- '/register',+ "/register",summary="Register a new user",description=("Create a new Mwmbl user account. Only `email` and `password` are required. "@@ -87,7 +127,7 @@ def register(request, registration: Registration):return {"status": "ok","username": username,- "message": "User registered successfully. Check your email for confirmation."+ "message": "User registered successfully. Check your email for confirmation.",}@@ -115,7 +155,7 @@ def confirm_email(request, confirm: ConfirmEmail):return {"status": "ok","username": confirmation.email_address.user.username,- "message": "Email confirmed successfully."+ "message": "Email confirmed successfully.",}@@ -277,8 +317,7 @@ def apply_decision(submission: DomainSubmission, decision, user) -> None:def check_moderator(request):if not request.user.has_perm("mwmbl.change_domain_submission_status"):- raise InvalidRequest("You do not have permission to moderate domain submissions.",- status=403)+ raise InvalidRequest("You do not have permission to moderate domain submissions.", status=403)@router.get(@@ -303,14 +342,14 @@ def check_moderator(request):),)def get_moderation_queue(- request,- limit: int = 50,- offset: int = 0,- order_by: str = "submissions",- suggested_action: str = None,- suggested_reason: str = None,- submitted_by: int = None,- min_confidence: float = None,+ request,+ limit: int = 50,+ offset: int = 0,+ order_by: str = "submissions",+ suggested_action: str = None,+ suggested_reason: str = None,+ submitted_by: int = None,+ min_confidence: float = None,) -> dict:check_moderator(request)@@ -320,9 +359,9 @@ def get_moderation_queue(# list in Python would mean materialising every pending submission (there are ~4,000) on# every request, which is the whole reason suggestions are precomputed. one_row_per_domain# and annotate_votes are in SQL for the same reason: the ordering sorts on both.- submissions = annotate_votes(one_row_per_domain(- annotate_queue(DomainSubmission.objects.filter(status="PENDING"))- )).select_related("submitted_by")+ submissions = annotate_votes(+ one_row_per_domain(annotate_queue(DomainSubmission.objects.filter(status="PENDING")))+ ).select_related("submitted_by")if submitted_by is not None:submissions = submissions.filter(submitted_by_id=submitted_by)@@ -336,7 +375,7 @@ def get_moderation_queue(submissions = submissions.order_by(*_queue_ordering(order_by))count = submissions.count()- page = list(submissions[offset:offset + limit])+ page = list(submissions[offset : offset + limit])return {"items": _queue_items(page), "count": count}@@ -365,8 +404,10 @@ def _queue_items(submissions: list[DomainSubmission]) -> list[QueueItemSchema]:padlock come off the DomainEvidence rows already fetched here for the suggestion, soshowing them costs nothing."""- evidence_by_domain = {row.domain: row for row in DomainEvidence.objects.filter(- domain__in=[submission.name for submission in submissions])}+ evidence_by_domain = {+ row.domain: row+ for row in DomainEvidence.objects.filter(domain__in=[submission.name for submission in submissions])+ }submitters = submitter_records({submission.submitted_by_id for submission in submissions})priors = prior_decision_counts({submission.name for submission in submissions})@@ -375,23 +416,27 @@ def _queue_items(submissions: list[DomainSubmission]) -> list[QueueItemSchema]:for submission in submissions:evidence = evidence_by_domain.get(submission.name)suggestion = suggestion_for(- submission, evidence,+ submission,+ evidence,submitter=submitters.get(submission.submitted_by_id, empty),- prior=priors.get(submission.name, empty))+ prior=priors.get(submission.name, empty),+ )signals = evidence.signals if evidence else {}- items.append(QueueItemSchema(- name=submission.name,- submission_count=submission.submission_count,- first_submitted_on=submission.submitted_on,- first_submitted_by=submission.submitted_by_id,- first_submitted_by_username=submission.submitted_by.username,- upvotes=submission.upvotes,- downvotes=submission.downvotes,- https=signals.get("https"),- evidence_state=evidence.state if evidence else "PENDING",- pages=evidence.pages if evidence else [],- suggestion=suggestion.__dict__ if suggestion else None,- ))+ items.append(+ QueueItemSchema(+ name=submission.name,+ submission_count=submission.submission_count,+ first_submitted_on=submission.submitted_on,+ first_submitted_by=submission.submitted_by_id,+ first_submitted_by_username=submission.submitted_by.username,+ upvotes=submission.upvotes,+ downvotes=submission.downvotes,+ https=signals.get("https"),+ evidence_state=evidence.state if evidence else "PENDING",+ pages=evidence.pages if evidence else [],+ suggestion=suggestion.__dict__ if suggestion else None,+ )+ )return items@@ -409,8 +454,7 @@ def _queue_items(submissions: list[DomainSubmission]) -> list[QueueItemSchema]:def get_submission_detail(request, submission_id: int) -> SubmissionDetailSchema:check_moderator(request)- submission = (DomainSubmission.objects.filter(id=submission_id)- .select_related("submitted_by").first())+ submission = DomainSubmission.objects.filter(id=submission_id).select_related("submitted_by").first()if submission is None:raise InvalidRequest("Submission not found.", status=404)@@ -499,8 +543,12 @@ def undo_decision(request, domain: str):# suggestion - and re-posting a PENDING status through apply_decision, the nearest thing# to an undo before this endpoint, overwrote them from the request.undone = submissions.update(- status="PENDING", rejection_reason="", rejection_detail="",- status_changed_by=request.user, status_changed_on=timezone.now())+ status="PENDING",+ rejection_reason="",+ rejection_detail="",+ status_changed_by=request.user,+ status_changed_on=timezone.now(),+ )if not undone:raise InvalidRequest(f"No submissions found for {name}.", status=404)@@ -537,13 +585,13 @@ HISTORY_ORDERINGS = {),)def get_moderation_history(- request,- limit: int = 50,- offset: int = 0,- status: str = None,- moderator: int = None,- name: str = None,- order_by: str = "recent",+ request,+ limit: int = 50,+ offset: int = 0,+ status: str = None,+ moderator: int = None,+ name: str = None,+ order_by: str = "recent",) -> dict:check_moderator(request)@@ -558,43 +606,49 @@ def get_moderation_history(# "last" rather than "first": a history row shows the decision that currently stands, so# the domain is represented by its most recently touched submission, not its oldest.submissions = one_row_per_domain(submissions, status=status, pick="last")- submissions = (submissions.select_related("submitted_by", "status_changed_by")- .order_by(*HISTORY_ORDERINGS.get(order_by, HISTORY_ORDERINGS["recent"])))+ submissions = submissions.select_related("submitted_by", "status_changed_by").order_by(+ *HISTORY_ORDERINGS.get(order_by, HISTORY_ORDERINGS["recent"])+ )count = submissions.count()- return {"items": _moderated_items(list(submissions[offset:offset + limit])), "count": count}+ return {"items": _moderated_items(list(submissions[offset : offset + limit])), "count": count}def _moderated_items(submissions: list[DomainSubmission]) -> list[ModeratedDomainSchema]:"""Build the page's rows. The representative row here is the last-touched submission, so"who first asked for this domain" needs one more lookup - once for the page, not per row."""first_by_name = {}- for submission in (DomainSubmission.objects- .filter(name__in=[row.name for row in submissions])- .select_related("submitted_by").order_by("submitted_on", "pk")):+ for submission in (+ DomainSubmission.objects.filter(name__in=[row.name for row in submissions])+ .select_related("submitted_by")+ .order_by("submitted_on", "pk")+ ):first_by_name.setdefault(submission.name, submission)items = []for submission in submissions:first = first_by_name[submission.name]- items.append(ModeratedDomainSchema(- name=submission.name,- submission_count=submission.submission_count,- status=submission.status,- rejection_reason=submission.rejection_reason,- rejection_detail=submission.rejection_detail,- first_submitted_on=first.submitted_on,- first_submitted_by=first.submitted_by_id,- first_submitted_by_username=first.submitted_by.username,- status_changed_on=submission.status_changed_on,- status_changed_by=submission.status_changed_by_id,- status_changed_by_username=(submission.status_changed_by.username- if submission.status_changed_by else ""),- suggested_status=submission.suggested_status,- suggested_reason=submission.suggested_reason,- suggestion_confidence=submission.suggestion_confidence,- suggestion_model_version=submission.suggestion_model_version,- ))+ items.append(+ ModeratedDomainSchema(+ name=submission.name,+ submission_count=submission.submission_count,+ status=submission.status,+ rejection_reason=submission.rejection_reason,+ rejection_detail=submission.rejection_detail,+ first_submitted_on=first.submitted_on,+ first_submitted_by=first.submitted_by_id,+ first_submitted_by_username=first.submitted_by.username,+ status_changed_on=submission.status_changed_on,+ status_changed_by=submission.status_changed_by_id,+ status_changed_by_username=(+ submission.status_changed_by.username if submission.status_changed_by else ""+ ),+ suggested_status=submission.suggested_status,+ suggested_reason=submission.suggested_reason,+ suggestion_confidence=submission.suggestion_confidence,+ suggestion_model_version=submission.suggestion_model_version,+ )+ )return items@@ -653,7 +707,7 @@ def vote_on_search_result(request, vote_request: VoteRequest):user=request.user,url=vote_request.url,query=vote_request.query,- defaults={'vote_type': vote_request.vote_type}+ defaults={"vote_type": vote_request.vote_type},)action = "created" if created else "updated"@@ -682,8 +736,8 @@ def get_vote_counts(request, vote_stats_request: VoteStatsRequest):vote_data = {}for url in vote_stats_request.urls:votes = SearchResultVote.objects.filter(url=url, query=vote_stats_request.query)- upvotes = votes.filter(vote_type='upvote').count()- downvotes = votes.filter(vote_type='downvote').count()+ upvotes = votes.filter(vote_type="upvote").count()+ downvotes = votes.filter(vote_type="downvote").count()user_vote = Nonetry:@@ -692,11 +746,7 @@ def get_vote_counts(request, vote_stats_request: VoteStatsRequest):except SearchResultVote.DoesNotExist:pass- vote_data[url] = VoteStats(- upvotes=upvotes,- downvotes=downvotes,- user_vote=user_vote- )+ vote_data[url] = VoteStats(upvotes=upvotes, downvotes=downvotes, user_vote=user_vote)return VoteResponse(votes=vote_data)@@ -716,11 +766,7 @@ def remove_vote(request, vote_request: VoteRemoveRequest):check_email_verified(request)try:- vote = SearchResultVote.objects.get(- user=request.user,- url=vote_request.url,- query=vote_request.query- )+ vote = SearchResultVote.objects.get(user=request.user, url=vote_request.url, query=vote_request.query)vote.delete()return {"status": "ok", "message": "Vote removed successfully."}except SearchResultVote.DoesNotExist:@@ -742,13 +788,14 @@ def remove_vote(request, vote_request: VoteRemoveRequest):@paginatedef get_user_vote_history(request) -> list[SearchResultVote]:check_email_verified(request)- return SearchResultVote.objects.filter(user=request.user).order_by('-timestamp')+ return SearchResultVote.objects.filter(user=request.user).order_by("-timestamp")# ---------------------------------------------------------------------------# Agreements helpers# ---------------------------------------------------------------------------+def _record_agreements(user: MwmblUser, agreement_types: list) -> None:for agreement_type in agreement_types:version_id = settings.CURRENT_AGREEMENT_VERSIONS.get(agreement_type)@@ -768,11 +815,7 @@ def _record_marketing_consent(user: MwmblUser, source: MarketingSource, opted_inthe existing row instead of growing the audit trail. Returns the row representingthe current state."""- latest = (- MarketingConsent.objects.filter(user=user, source=source)- .order_by("-timestamp", "-id")- .first()- )+ latest = MarketingConsent.objects.filter(user=user, source=source).order_by("-timestamp", "-id").first()if latest is not None and latest.opted_in == opted_in:return latestreturn MarketingConsent.objects.create(user=user, source=source, opted_in=opted_in)@@ -807,6 +850,7 @@ def _require_current_agreement(user: MwmblUser, agreement_type: AgreementType) -# Agreement endpoints# ---------------------------------------------------------------------------+@router.post("/agreements/",auth=JWTAuth(),@@ -859,11 +903,13 @@ def get_agreements(request) -> list[AgreementResponse]:.first())if latest:- result.append(AgreementResponse(- agreement_type=latest.agreement_type,- version_id=latest.version_id,- accepted_at=latest.accepted_at,- ))+ result.append(+ AgreementResponse(+ agreement_type=latest.agreement_type,+ version_id=latest.version_id,+ accepted_at=latest.accepted_at,+ )+ )return result@@ -889,6 +935,7 @@ def get_agreement_history(request) -> list[AgreementResponse]:# Marketing consent# ---------------------------------------------------------------------------+@router.get("/marketing-consent",auth=JWTAuth(),@@ -906,17 +953,15 @@ def get_marketing_consent(request) -> MarketingConsentListResponse:check_email_verified(request)consent = []for source in MarketingSource:- latest = (- MarketingConsent.objects.filter(user=request.user, source=source)- .order_by("-timestamp", "-id")- .first()- )+ latest = MarketingConsent.objects.filter(user=request.user, source=source).order_by("-timestamp", "-id").first()if latest:- consent.append(MarketingConsentResponse(- source=latest.source,- opted_in=latest.opted_in,- timestamp=latest.timestamp,- ))+ consent.append(+ MarketingConsentResponse(+ source=latest.source,+ opted_in=latest.opted_in,+ timestamp=latest.timestamp,+ )+ )return MarketingConsentListResponse(consent=consent)@@ -974,6 +1019,7 @@ def unsubscribe_marketing(request, token: str):# API key management# ---------------------------------------------------------------------------+@router.post("/api-keys/",auth=JWTAuth(),@@ -994,6 +1040,7 @@ def create_api_key(request, body: CreateApiKeyRequest):else:_require_current_agreement(request.user, AgreementType.TERMS_OF_SERVICE_API)from mwmbl.models import generate_api_key+raw_key, key_hash = generate_api_key()api_key = ApiKey.objects.create(user=request.user,@@ -1062,6 +1109,7 @@ def delete_api_key(request, key_id: int):# User profile# ---------------------------------------------------------------------------+@router.get("/user",auth=JWTAuth(),@@ -1088,8 +1136,10 @@ def get_current_user(request):# Billing# ---------------------------------------------------------------------------+def _subscription_response(user, billing, status: str) -> SubscriptionResponse:from mwmbl.quota import get_monthly_count+spend_cents = billing.max_monthly_spend_cents if billing else 0usage = get_monthly_count(user.id)return SubscriptionResponse(@@ -1184,7 +1234,7 @@ def uncancel_subscription(request):response=SubscriptionResponse,summary="Cancel subscription at period end",description="Schedules the subscription to cancel at the end of the current billing period. "- "The spend limit remains active until then.",+ "The spend limit remains active until then.",tags=["Billing"],)def cancel_subscription(request):@@ -1271,7 +1321,13 @@ def polar_webhook(request):logger.warning("Polar webhook: no user found for user_id=%s", user_id)else:billing, created = UserBilling.objects.get_or_create(user=user)- logger.info("Polar webhook: UserBilling %s for user %s customer_id=%s subscription_id=%s", "created" if created else "updated", user.email, event.data.customer_id, event.data.id)+ logger.info(+ "Polar webhook: UserBilling %s for user %s customer_id=%s subscription_id=%s",+ "created" if created else "updated",+ user.email,+ event.data.customer_id,+ event.data.id,+ )billing.polar_customer_id = event.data.customer_id or billing.polar_customer_idbilling.polar_subscription_id = event.data.id or billing.polar_subscription_idbilling.current_period_end = event.data.current_period_end or billing.current_period_end@@ -1279,7 +1335,11 @@ def polar_webhook(request):billing.save()elif event_type == "subscription.canceled":user_id = event.data.metadata.get("user_id")- logger.info("Polar webhook: subscription.canceled user_id=%s cancel_at_period_end=%s", user_id, getattr(event.data, "cancel_at_period_end", None))+ logger.info(+ "Polar webhook: subscription.canceled user_id=%s cancel_at_period_end=%s",+ user_id,+ getattr(event.data, "cancel_at_period_end", None),+ )user = MwmblUser.objects.filter(id=user_id).first()if user is None:logger.warning("Polar webhook: no user found for user_id=%s", user_id)@@ -1289,9 +1349,15 @@ def polar_webhook(request):if billing:billing.cancel_at_period_end = Truebilling.save()- logger.info("Polar webhook: user %s (id=%s) subscription scheduled to cancel at period end", user.email, user_id)+ logger.info(+ "Polar webhook: user %s (id=%s) subscription scheduled to cancel at period end", user.email, user_id+ )else:- logger.info("Polar webhook: immediate cancellation for user %s (id=%s), resetting spend limit to $0", user.email, user_id)+ logger.info(+ "Polar webhook: immediate cancellation for user %s (id=%s), resetting spend limit to $0",+ user.email,+ user_id,+ )billing = getattr(user, "billing", None)if billing:billing.cancel_at_period_end = False@@ -1320,6 +1386,7 @@ def polar_webhook(request):# Password reset# ---------------------------------------------------------------------------+@router.post("/forgot-password",summary="Request password reset",@@ -1328,6 +1395,7 @@ def polar_webhook(request):)def forgot_password(request, body: ForgotPasswordRequest):from django.contrib.auth.forms import PasswordResetForm+form = PasswordResetForm({"email": body.email})if form.is_valid():form.save(request=request, use_https=request.is_secure())@@ -1342,6 +1410,7 @@ def forgot_password(request, body: ForgotPasswordRequest):)def reset_password(request, body: ResetPasswordRequest):from django.contrib.auth.tokens import default_token_generator+try:user = MwmblUser.objects.get(email=body.email)except MwmblUser.DoesNotExist:diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.pyindex 576ea5f..ac28fd1 100644--- a/mwmbl/platform/schemas.py+++ b/mwmbl/platform/schemas.py@@ -1,6 +1,7 @@from datetime import datetime-from typing import Optional, Literal-from ninja import Schema, ModelSchema, Field+from typing import Literal, Optional++from ninja import Field, ModelSchema, Schemafrom pydantic import model_validatorfrom mwmbl.models import AgreementType, DomainSubmission, MarketingSource@@ -50,8 +51,10 @@ class ResetPasswordRequest(Schema):# API key management schemas# ---------------------------------------------------------------------------+class CreateApiKeyRequest(Schema):"""Request body for creating a new API key."""+name: str = Field(default="",max_length=100,@@ -69,6 +72,7 @@ class ApiKeyCreatedResponse(Schema):Response returned when a new API key is created.The raw `key` value is only returned once — store it securely."""+id: int = Field(description="Unique ID of the API key.")key: str = Field(description="The raw API key token. Shown only on creation.")name: str = Field(description="Human-readable label for this key.")@@ -81,6 +85,7 @@ class ApiKeyListItem(Schema):A single API key entry returned by the list endpoint.The raw key value is intentionally omitted."""+id: int = Field(description="Unique ID of the API key.")name: str = Field(description="Human-readable label for this key.")created_on: datetime = Field(description="When the key was created.")@@ -91,6 +96,7 @@ class ApiKeyListItem(Schema):# Agreements# ---------------------------------------------------------------------------+class AgreementAcceptRequest(Schema):agreement_type: AgreementType = Field(description="The type of agreement being accepted.")@@ -105,6 +111,7 @@ class AgreementResponse(Schema):# Marketing consent# ---------------------------------------------------------------------------+class MarketingConsentRequest(Schema):source: MarketingSource = Field(description="The property the consent applies to (`GUI` or `API`).")opted_in: bool = Field(description="True to opt in to marketing emails, False to withdraw consent.")@@ -201,8 +208,8 @@ class RejectionFieldsMixin:raise ValueError("rejection_reason is only meaningful when status is REJECTED")if reason == "OTHER" and not detail.strip():raise ValueError(- "rejection_detail is required when rejection_reason is OTHER - it is what "- "the submitter is shown")+ "rejection_detail is required when rejection_reason is OTHER - it is what the submitter is shown"+ )return self@@ -225,20 +232,22 @@ class UpdateDomainSubmission(RejectionFieldsMixin, ModelSchema):# save(), so without them an over-long value is a 500 from Postgres where it should be a# 400 naming the field.suggested_status: Optional[str] = Field(- default=None, max_length=20,- description="The action the tool suggested, as shown to the moderator.")+ default=None, max_length=20, description="The action the tool suggested, as shown to the moderator."+ )suggested_reason: Optional[str] = Field(- default=None, max_length=20,- description="The rejection reason the tool suggested, if any.")+ default=None, max_length=20, description="The rejection reason the tool suggested, if any."+ )suggestion_confidence: Optional[float] = Field(- default=None, description="Confidence of the suggestion that was shown.")+ default=None, description="Confidence of the suggestion that was shown."+ )suggestion_model_version: Optional[str] = Field(- default=None, max_length=50,- description="Version of the model that produced the shown suggestion.")+ default=None, max_length=50, description="Version of the model that produced the shown suggestion."+ )class EvidenceItemSchema(Schema):"""One checkable fact behind a suggestion."""+kind: str = Field(description="Machine-readable check name, e.g. `http_status`.")direction: str = Field(description="`reject`, `approve` or `neutral`.")label: str = Field(description="Moderator-facing text, e.g. 'Homepage returns HTTP 404'.")@@ -288,33 +297,35 @@ class QueueItemSchema(Schema):Everything a card shows is here so that moving to the next one costs no request: thesample pages, the vote counts, the padlock and the suggestion all travel with the row."""+name: str- submission_count: int = Field(- description="How many submissions of this domain are still pending.")- first_submitted_on: datetime = Field(- description="When this domain was first submitted, by whoever asked first.")+ submission_count: int = Field(description="How many submissions of this domain are still pending.")+ first_submitted_on: datetime = Field(description="When this domain was first submitted, by whoever asked first.")first_submitted_by: intfirst_submitted_by_username: str- upvotes: int = Field(- description="Votes cast on results from this domain, ignoring a leading `www.`.")+ upvotes: int = Field(description="Votes cast on results from this domain, ignoring a leading `www.`.")downvotes: inthttps: Optional[bool] = Field(default=None,description=("Whether the homepage answered over TLS. `null` until the domain has been ""crawled - an uncrawled domain must not draw an open padlock."- ))+ ),+ )evidence_state: str = Field(- description="`PENDING` while the domain is still being crawled, then `READY` or `FAILED`.")+ description="`PENDING` while the domain is still being crawled, then `READY` or `FAILED`."+ )pages: list[CrawledPageSchema] = Field(- default=[], description="The pages crawled from the domain, as shown on the card.")+ default=[], description="The pages crawled from the domain, as shown on the card."+ )suggestion: Optional[SuggestionSchema] = Field(- default=None,- description="Absent until the domain has been crawled. Never a placeholder.")+ default=None, description="Absent until the domain has been crawled. Never a placeholder."+ )class SubmissionDetailSchema(Schema):"""Everything a moderator needs to decide one submission."""+id: intname: strsubmitted_by: int@@ -328,11 +339,10 @@ class SubmissionDetailSchema(Schema):pages: list[CrawledPageSchema] = []signals: dict = {}submitter_record: dict = Field(- default={}, description="Counts of this submitter's previously approved/rejected domains.")- prior_decisions: dict = Field(- default={}, description="Counts of earlier decisions on this same domain.")- index_stats: dict = Field(- default={}, description="What the crawler already knows about this domain.")+ default={}, description="Counts of this submitter's previously approved/rejected domains."+ )+ prior_decisions: dict = Field(default={}, description="Counts of earlier decisions on this same domain.")+ index_stats: dict = Field(default={}, description="What the crawler already knows about this domain.")class DomainDecision(RejectionFieldsMixin, Schema):@@ -342,6 +352,7 @@ class DomainDecision(RejectionFieldsMixin, Schema):decided about: a decision settles every submission of the name at once, so nine asks forthe same site cannot end up half approved and half rejected."""+domain: str = Field(max_length=300)status: str = Field(max_length=20)rejection_reason: str = Field(default="", max_length=20)@@ -358,6 +369,7 @@ class BulkDecisionRequest(Schema):Deliberately not "accept all suggestions": each entry carries its own status, so theclient can only send choices a moderator actually made."""+decisions: list[DomainDecision]@@ -368,6 +380,7 @@ class ModeratedDomainSchema(Schema):`suggested_*` audit columns so a moderator revisiting a call can see what was on screenwhen it was made, rather than what the model would say about the domain today."""+name: strsubmission_count: intstatus: str@@ -387,110 +400,72 @@ class ModeratedDomainSchema(Schema):class ModerationHistory(Schema):"""A page of past moderations. Same `items`/`count` shape as the queue."""+items: list[ModeratedDomainSchema]count: intclass VoteRequest(Schema):"""Request schema for voting on search results."""-- url: str = Field(- description="The URL of the search result being voted on",- example="https://example.com/article"- )- query: str = Field(- description="The search query that returned this result",- example="python tutorial"- )++ url: str = Field(description="The URL of the search result being voted on", example="https://example.com/article")+ query: str = Field(description="The search query that returned this result", example="python tutorial")vote_type: Literal["upvote", "downvote"] = Field(description="Type of vote - either 'upvote' for positive feedback or 'downvote' for negative feedback",- example="upvote"+ example="upvote",)class VoteRemoveRequest(Schema):"""Request schema for removing a vote on a search result."""-+url: str = Field(- description="The URL of the search result to remove the vote from",- example="https://example.com/article"- )- query: str = Field(- description="The search query that returned this result",- example="python tutorial"+ description="The URL of the search result to remove the vote from", example="https://example.com/article")+ query: str = Field(description="The search query that returned this result", example="python tutorial")class VoteStats(Schema):"""Statistics for votes on a specific search result."""-- upvotes: int = Field(- description="Total number of upvotes for this search result",- example=15- )- downvotes: int = Field(- description="Total number of downvotes for this search result",- example=3- )++ upvotes: int = Field(description="Total number of upvotes for this search result", example=15)+ downvotes: int = Field(description="Total number of downvotes for this search result", example=3)user_vote: Optional[Literal["upvote", "downvote"]] = Field(- default=None,- description="The current user's vote on this result, if any",- example="upvote"+ default=None, description="The current user's vote on this result, if any", example="upvote")class VoteStatsRequest(Schema):"""Request schema for getting vote statistics for multiple URLs."""-- query: str = Field(- description="The search query that returned these results",- example="python tutorial"- )++ query: str = Field(description="The search query that returned these results", example="python tutorial")urls: list[str] = Field(description="List of URLs to get vote statistics for",- example=["https://example.com/article", "https://another-site.com/page"]+ example=["https://example.com/article", "https://another-site.com/page"],)class VoteResponse(Schema):"""Response schema containing vote statistics for multiple URLs."""-+votes: dict[str, VoteStats] = Field(description="Dictionary mapping URLs to their vote statistics",example={- "https://example.com/article": {- "upvotes": 15,- "downvotes": 3,- "user_vote": "upvote"- },- "https://another-site.com/page": {- "upvotes": 8,- "downvotes": 1,- "user_vote": None- }- }+ "https://example.com/article": {"upvotes": 15, "downvotes": 3, "user_vote": "upvote"},+ "https://another-site.com/page": {"upvotes": 8, "downvotes": 1, "user_vote": None},+ },)class UserVoteHistory(Schema):"""Schema representing a user's voting history entry."""-+url: str = Field(- description="The URL of the search result that was voted on",- example="https://example.com/article"- )- query: str = Field(- description="The search query that returned this result",- example="python tutorial"- )- vote_type: Literal["upvote", "downvote"] = Field(- description="The type of vote cast by the user",- example="upvote"- )- timestamp: datetime = Field(- description="When the vote was cast",- example="2024-01-15T10:30:00Z"+ description="The URL of the search result that was voted on", example="https://example.com/article")+ query: str = Field(description="The search query that returned this result", example="python tutorial")+ vote_type: Literal["upvote", "downvote"] = Field(description="The type of vote cast by the user", example="upvote")+ timestamp: datetime = Field(description="When the vote was cast", example="2024-01-15T10:30:00Z")class ModerationQueue(Schema):@@ -500,5 +475,6 @@ class ModerationQueue(Schema):`count` is distinct pending *domains* - the "24 pending" in the header - not pendingsubmissions."""+items: list[QueueItemSchema]count: intdiff --git a/mwmbl/pricing.py b/mwmbl/pricing.pyindex da208a6..cb8298e 100644--- a/mwmbl/pricing.py+++ b/mwmbl/pricing.py@@ -5,8 +5,8 @@ billing endpoints, and the background usage-reporting job without risk ofimport cycles."""-FREE_KEYED_MONTHLY_LIMIT = 2_000 # free requests/month once an API key is presented-PRICE_PER_1000_QUERIES_CENTS = 500 # $5.00 per 1,000 queries+FREE_KEYED_MONTHLY_LIMIT = 2_000 # free requests/month once an API key is presented+PRICE_PER_1000_QUERIES_CENTS = 500 # $5.00 per 1,000 queries# Preset spend-limit options surfaced in the UI (mirrors Brave's Free/$10/$25/$100 presets).SPEND_LIMIT_PRESETS_CENTS = [0, 1_000, 2_500, 10_000]diff --git a/mwmbl/quota.py b/mwmbl/quota.pyindex a5fe140..ef66072 100644--- a/mwmbl/quota.py+++ b/mwmbl/quota.py@@ -6,18 +6,20 @@ backend can be swapped without changing this code. The only Redis-specificcall is get_all_monthly_keys(), which uses SCAN via django-redis and is onlycalled by the background flush/reset jobs."""+from datetime import datetime, timezonefrom django.core.cache import cache-RATE_LIMIT = 5 # maximum requests per second (all tiers)-MONTHLY_TTL = 60 * 60 * 24 * 35 # 35 days in seconds+RATE_LIMIT = 5 # maximum requests per second (all tiers)+MONTHLY_TTL = 60 * 60 * 24 * 35 # 35 days in seconds# ---------------------------------------------------------------------------# Key helpers# ---------------------------------------------------------------------------+def _monthly_key(user_id: int, year: int | None = None, month: int | None = None) -> str:now = datetime.now(timezone.utc)y = year if year is not None else now.year@@ -40,6 +42,7 @@ def _rate_key(user_id: int) -> str:# Rate limiting (fixed-window, 5 req/s)# ---------------------------------------------------------------------------+def check_rate_limit(user_id: int) -> bool:"""Fixed-window rate limit: at most RATE_LIMIT requests per second.@@ -53,6 +56,7 @@ def check_rate_limit(user_id: int) -> bool:# Key expired between add() and incr(); Redis created it without a TTL.# Set the TTL now to prevent the key from leaking indefinitely.from django_redis import get_redis_connection+get_redis_connection("default").expire(key, 1)return count <= RATE_LIMIT@@ -61,6 +65,7 @@ def check_rate_limit(user_id: int) -> bool:# Monthly quota# ---------------------------------------------------------------------------+def get_monthly_count(user_id: int) -> int:"""Return the current monthly request count for a user (0 if not set)."""return cache.get(_monthly_key(user_id), default=0)@@ -109,6 +114,7 @@ def decrement_monthly_super_search(user_id: int) -> None:# Key scanning (used by background jobs only — requires django-redis)# ---------------------------------------------------------------------------+def get_all_monthly_keys() -> list[str]:"""Return all active monthly counter keys for the current month.@@ -118,6 +124,7 @@ def get_all_monthly_keys() -> list[str]:now = datetime.now(timezone.utc)pattern = f"search:monthly:*:{now.year}:{now.month:02d}"from django_redis import get_redis_connection+conn = get_redis_connection("default")return [k.decode() if isinstance(k, bytes) else k for k in conn.scan_iter(pattern)]diff --git a/mwmbl/rankeval/dataset/extension_dataset.py b/mwmbl/rankeval/dataset/extension_dataset.pyindex 0acdabc..cc4974a 100644--- a/mwmbl/rankeval/dataset/extension_dataset.py+++ b/mwmbl/rankeval/dataset/extension_dataset.py@@ -38,7 +38,6 @@ from dotenv import load_dotenvfrom mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH, RANKINGS_DATASET_TRAIN_PATHfrom mwmbl.settings import BUCKET_NAME, ENDPOINT_URL, VERSION-REPO_ROOT = Path(__file__).resolve().parents[3]DOWNLOADS_DIR = REPO_ROOT / "scripts" / "downloads"DATE_PATTERN = re.compile(r"(\d{4}-\d{2}-\d{2})")@@ -85,8 +84,10 @@ def download_datasets(workers: int = 16):keys += [obj.key for obj in bucket.objects.filter(Prefix=f"{date_prefix}dataset/")]missing = [k for k in keys if not (DOWNLOADS_DIR / k).exists()]- print(f"{len(keys)} dataset files on Backblaze, {len(keys) - len(missing)} already "- f"present locally; downloading {len(missing)}...")+ print(+ f"{len(keys)} dataset files on Backblaze, {len(keys) - len(missing)} already "+ f"present locally; downloading {len(missing)}..."+ )def _fetch(key: str):dest = DOWNLOADS_DIR / key@@ -101,8 +102,7 @@ def download_datasets(workers: int = 16):if i % 500 == 0:print(f"Downloaded {i}/{len(missing)} new files...")- print(f"Download complete: {downloaded} new files, "- f"{len(keys) - len(missing)} already present.")+ print(f"Download complete: {downloaded} new files, {len(keys) - len(missing)} already present.")def create_dataset() -> pd.DataFrame:@@ -118,13 +118,15 @@ def create_dataset() -> pd.DataFrame:for item in data.get("searchResults", []):query = item["query"]for i, row in enumerate(item["results"]):- dataset.append({- "query": query,- "url": row["url"],- "snippet": row["extract"],- "rank": i + 1,- "date_retrieved": date_str,- })+ dataset.append(+ {+ "query": query,+ "url": row["url"],+ "snippet": row["extract"],+ "rank": i + 1,+ "date_retrieved": date_str,+ }+ )return pd.DataFrame(dataset)@@ -141,20 +143,22 @@ def save_dataset(dataset: pd.DataFrame):train_set = dataset[dataset["query"].isin(train_queries)]test_set = dataset[~dataset["query"].isin(train_queries)]- print(f"Saving dataset with {len(train_set)} train rows "- f"and {len(test_set)} test rows to {RANKINGS_DATASET_TRAIN_PATH.parent}")+ print(+ f"Saving dataset with {len(train_set)} train rows "+ f"and {len(test_set)} test rows to {RANKINGS_DATASET_TRAIN_PATH.parent}"+ )train_set.to_csv(RANKINGS_DATASET_TRAIN_PATH)test_set.to_csv(RANKINGS_DATASET_TEST_PATH)def main():- parser = argparse.ArgumentParser(- description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)parser.add_argument(- "--no-download", action="store_true",- help="Skip the Backblaze download and rebuild the CSVs from "- "files already in scripts/downloads/.")+ "--no-download",+ action="store_true",+ help="Skip the Backblaze download and rebuild the CSVs from files already in scripts/downloads/.",+ )args = parser.parse_args()if not args.no_download:diff --git a/mwmbl/rankeval/evaluation/compare_ltr_models.py b/mwmbl/rankeval/evaluation/compare_ltr_models.pyindex 87cb8f0..201e19f 100644--- a/mwmbl/rankeval/evaluation/compare_ltr_models.py+++ b/mwmbl/rankeval/evaluation/compare_ltr_models.py@@ -15,6 +15,7 @@ Usage::--model retrained=devdata/rankeval-2026-04/model.xgb \--fraction 0.1"""+import osfrom argparse import ArgumentParser@@ -24,8 +25,8 @@ import numpy as npos.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")django.setup()-import mwmbl.rankeval.evaluation.evaluate as evaluate_module # noqa: E402-from mwmbl.rankeval.evaluation.evaluate import evaluate # noqa: E402+import mwmbl.rankeval.evaluation.evaluate as evaluate_module+from mwmbl.rankeval.evaluation.evaluate import evaluatefrom mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModelfrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.tinysearchengine.ltr import RustXGBPipeline@@ -41,12 +42,15 @@ def model_from_path(path: str) -> MwmblRankingModel:def run():parser = ArgumentParser()- parser.add_argument("--model", action="append", required=True, metavar="LABEL=PATH",- help="A model to evaluate, e.g. current=path/to/model.xgb. Repeatable.")- parser.add_argument("--fraction", type=float, default=0.1,- help="Fraction of gold test queries to sample.")- parser.add_argument("--train", action="store_true",- help="Evaluate on the train split instead of test.")+ parser.add_argument(+ "--model",+ action="append",+ required=True,+ metavar="LABEL=PATH",+ help="A model to evaluate, e.g. current=path/to/model.xgb. Repeatable.",+ )+ parser.add_argument("--fraction", type=float, default=0.1, help="Fraction of gold test queries to sample.")+ parser.add_argument("--train", action="store_true", help="Evaluate on the train split instead of test.")args = parser.parse_args()labelled = [spec.split("=", 1) for spec in args.model]diff --git a/mwmbl/rankeval/evaluation/compare_search_modes.py b/mwmbl/rankeval/evaluation/compare_search_modes.pyindex 914c2e3..a2ed3bb 100644--- a/mwmbl/rankeval/evaluation/compare_search_modes.py+++ b/mwmbl/rankeval/evaluation/compare_search_modes.py@@ -17,6 +17,7 @@ Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \uv run python -m mwmbl.rankeval.evaluation.compare_search_modes --fraction 0.05"""+import osfrom argparse import ArgumentParser@@ -42,12 +43,11 @@ NEW_SOURCES = ["www_gov_uk", "imdb"]def run():parser = ArgumentParser()- parser.add_argument("--fraction", type=float, default=0.05,- help="Fraction of gold test queries to sample.")- parser.add_argument("--train", action="store_true",- help="Evaluate on the train split instead of test.")- parser.add_argument("--clear-cache", action="store_true",- help="Clear the Super Search doc-pool cache before evaluating.")+ parser.add_argument("--fraction", type=float, default=0.05, help="Fraction of gold test queries to sample.")+ parser.add_argument("--train", action="store_true", help="Evaluate on the train split instead of test.")+ parser.add_argument(+ "--clear-cache", action="store_true", help="Clear the Super Search doc-pool cache before evaluating."+ )args = parser.parse_args()if args.clear_cache:diff --git a/mwmbl/rankeval/evaluation/evaluate.py b/mwmbl/rankeval/evaluation/evaluate.pyindex b06acd2..1c10273 100644--- a/mwmbl/rankeval/evaluation/evaluate.py+++ b/mwmbl/rankeval/evaluation/evaluate.py@@ -1,15 +1,15 @@"""Perform an evaluation using NDCG against a gold standard set of results."""-from abc import ABC, abstractmethod+import time+from abc import ABC, abstractmethodimport numpy as npimport pandas as pdfrom scipy.stats import semfrom sklearn.metrics import ndcg_score-# Sourced from https://www.searchenginejournal.com/google-first-page-clicks/374516/from mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH, RANKINGS_DATASET_TRAIN_PATH@@ -51,7 +51,7 @@ def gold_scores_for(rankings) -> dict[str, float]:"""ranking = latest_ranking(rankings).drop_duplicates(subset="url", keep="first")top_ranked = ranking[["url"]].iloc[:NUM_RESULTS_FOR_EVAL].copy()- top_ranked["score"] = CLICK_PROPORTIONS[:len(top_ranked)]+ top_ranked["score"] = CLICK_PROPORTIONS[: len(top_ranked)]return top_ranked.set_index("url")["score"].to_dict()@@ -69,7 +69,7 @@ def evaluate(ranking_model: RankingModel, fraction: float = 1.0, use_test=False)ndcg_scores = []proportions = []- queries = dataset['query'].unique()+ queries = dataset["query"].unique()if fraction < 1.0:num_queries = int(fraction * len(queries))print("Num queries", num_queries)@@ -79,7 +79,7 @@ def evaluate(ranking_model: RankingModel, fraction: float = 1.0, use_test=False)wiki_counts = []durations = []- for query, rankings in dataset.groupby('query'):+ for query, rankings in dataset.groupby("query"):if query not in random_queries:continue@@ -121,6 +121,8 @@ def evaluate(ranking_model: RankingModel, fraction: float = 1.0, use_test=False)print("predict_time_sem:", sem(durations))print("Dataset\tFraction\tNDCG\tSEM\tProportion\tSEM\tWiki Count\tSEM\tPredict time\tSEM")- print(f"{'Test' if use_test else 'Train'}\t{fraction}\t{np.mean(ndcg_scores)}\t{sem(ndcg_scores)}\t"- f"{np.mean(proportions)}\t{sem(proportions)}\t{np.mean(wiki_counts)}\t{sem(wiki_counts)}"- f"\t{np.mean(durations)}\t{sem(durations)}")+ print(+ f"{'Test' if use_test else 'Train'}\t{fraction}\t{np.mean(ndcg_scores)}\t{sem(ndcg_scores)}\t"+ f"{np.mean(proportions)}\t{sem(proportions)}\t{np.mean(wiki_counts)}\t{sem(wiki_counts)}"+ f"\t{np.mean(durations)}\t{sem(durations)}"+ )diff --git a/mwmbl/rankeval/evaluation/evaluate_combined.py b/mwmbl/rankeval/evaluation/evaluate_combined.pyindex 9f6f669..749094d 100644--- a/mwmbl/rankeval/evaluation/evaluate_combined.py+++ b/mwmbl/rankeval/evaluation/evaluate_combined.py@@ -1,6 +1,7 @@"""Evaluate a ranking model that combines two other models"""+from mwmbl.rankeval.evaluation.evaluate import evaluatefrom mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModelfrom mwmbl.rankeval.evaluation.evaluate_wiki import WikiModel@@ -18,7 +19,7 @@ class CombinedModel:results1 = self.model1.predict(query)results2 = self.model2.predict(query)results1_set = set(results1)- return results1[:self.max_results1] + [x for x in results2 if x not in results1_set]+ return results1[: self.max_results1] + [x for x in results2 if x not in results1_set]def run():diff --git a/mwmbl/rankeval/evaluation/evaluate_fallback.py b/mwmbl/rankeval/evaluation/evaluate_fallback.pyindex 261cce2..03a332b 100644--- a/mwmbl/rankeval/evaluation/evaluate_fallback.py+++ b/mwmbl/rankeval/evaluation/evaluate_fallback.py@@ -39,6 +39,7 @@ Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \\uv run python -m mwmbl.rankeval.evaluation.evaluate_fallback --fraction 0.05"""+import osfrom argparse import ArgumentParser@@ -53,12 +54,18 @@ django.setup()import mwmbl.rankeval.evaluation.evaluate_super_search as ss_module # noqa: E402from mwmbl.rankeval.evaluation.evaluate import ( # noqa: E402- NUM_RESULTS_FOR_EVAL, RankingModel, gold_scores_for)+ NUM_RESULTS_FOR_EVAL,+ RankingModel,+ gold_scores_for,+)from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModel # noqa: E402from mwmbl.rankeval.evaluation.evaluate_super_search import SuperSearchRankingModel # noqa: E402from mwmbl.rankeval.evaluation.remote_index import RemoteIndex # noqa: E402from mwmbl.rankeval.paths import ( # noqa: E402- RANKINGS_DATASET_TEST_PATH, RANKINGS_DATASET_TRAIN_PATH, RUST_MODEL_PATH)+ RANKINGS_DATASET_TEST_PATH,+ RANKINGS_DATASET_TRAIN_PATH,+ RUST_MODEL_PATH,+)from mwmbl.tinysearchengine.ltr import RustXGBPipeline # noqa: E402from mwmbl.tinysearchengine.ltr_rank import LTRRanker # noqa: E402from mwmbl.tinysearchengine.mmr_rank import MMRRanker # noqa: E402@@ -72,6 +79,7 @@ def _standard_model(use_local: bool) -> MwmblRankingModel:"""if use_local:from mwmbl.search_setup import ranker # local index + LTR + MMR + wiki+return MwmblRankingModel(ranker)model = RustXGBPipeline.from_model_path(str(RUST_MODEL_PATH))ranker = MMRRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))@@ -117,21 +125,33 @@ def _mean_sem(values: list[float]) -> str:def run():parser = ArgumentParser()- parser.add_argument("--fraction", type=float, default=0.05,- help="Fraction of gold queries to sample (Super Search is slow).")- parser.add_argument("--thresholds", type=int, nargs="+", default=[1, 3, 5],- help="Fallback when standard returns <= threshold results.")- parser.add_argument("--train", action="store_true",- help="Evaluate on the train split instead of test.")- parser.add_argument("--local", action="store_true",- help="Use the local dev index for standard search instead of "- "the remote production index (fallback then fires on almost "- "every query — for debugging only).")- parser.add_argument("--fired-only", action="store_true",- help="Only run Super Search on queries where the fallback fires "- "(faster, but omits the always-Super-Search comparison arm).")- parser.add_argument("--clear-cache", action="store_true",- help="Clear the Super Search doc-pool cache before evaluating.")+ parser.add_argument(+ "--fraction", type=float, default=0.05, help="Fraction of gold queries to sample (Super Search is slow)."+ )+ parser.add_argument(+ "--thresholds",+ type=int,+ nargs="+",+ default=[1, 3, 5],+ help="Fallback when standard returns <= threshold results.",+ )+ parser.add_argument("--train", action="store_true", help="Evaluate on the train split instead of test.")+ parser.add_argument(+ "--local",+ action="store_true",+ help="Use the local dev index for standard search instead of "+ "the remote production index (fallback then fires on almost "+ "every query — for debugging only).",+ )+ parser.add_argument(+ "--fired-only",+ action="store_true",+ help="Only run Super Search on queries where the fallback fires "+ "(faster, but omits the always-Super-Search comparison arm).",+ )+ parser.add_argument(+ "--clear-cache", action="store_true", help="Clear the Super Search doc-pool cache before evaluating."+ )args = parser.parse_args()if args.clear_cache:@@ -182,24 +202,27 @@ def run():print("Super Search (always): NDCG =", _mean_sem(list(super_ndcg.values())))print(f"{'=' * 78}")- print(f"\n{'thresh':>6} {'fired':>12} {'fallback NDCG (all)':>22} "- f"{'fired-subset std':>17} {'fired-subset super':>18} {'Δ on fired':>11}")+ print(+ f"\n{'thresh':>6} {'fired':>12} {'fallback NDCG (all)':>22} "+ f"{'fired-subset std':>17} {'fired-subset super':>18} {'Δ on fired':>11}"+ )for threshold in sorted(args.thresholds):fired = [q for q, c in standard_count.items() if c <= threshold]- fallback_all = [- super_ndcg[q] if standard_count[q] <= threshold else standard_ndcg[q]- for q in standard_ndcg- ]+ fallback_all = [super_ndcg[q] if standard_count[q] <= threshold else standard_ndcg[q] for q in standard_ndcg]fired_std = [standard_ndcg[q] for q in fired]fired_super = [super_ndcg[q] for q in fired]delta = (np.mean(fired_super) - np.mean(fired_std)) if fired else float("nan")n_total = len(standard_ndcg)- print(f"{threshold:>6} {len(fired):>5}/{n_total:<6} {_mean_sem(fallback_all):>22} "- f"{_mean_sem(fired_std):>17} {_mean_sem(fired_super):>18} {delta:>+11.4f}")-- print("\nfired-subset Δ = mean(Super Search NDCG − standard NDCG) on queries where the "- "fallback fires;\npositive means falling back to Super Search helps where standard "- "is starved.")+ print(+ f"{threshold:>6} {len(fired):>5}/{n_total:<6} {_mean_sem(fallback_all):>22} "+ f"{_mean_sem(fired_std):>17} {_mean_sem(fired_super):>18} {delta:>+11.4f}"+ )++ print(+ "\nfired-subset Δ = mean(Super Search NDCG − standard NDCG) on queries where the "+ "fallback fires;\npositive means falling back to Super Search helps where standard "+ "is starved."+ )if __name__ == "__main__":diff --git a/mwmbl/rankeval/evaluation/evaluate_oracle.py b/mwmbl/rankeval/evaluation/evaluate_oracle.pyindex b100bfe..ca0a9db 100644--- a/mwmbl/rankeval/evaluation/evaluate_oracle.py+++ b/mwmbl/rankeval/evaluation/evaluate_oracle.py@@ -5,9 +5,9 @@ Evaluate the crawled URLs to see how well they cover our gold-standard search reimport pandas as pdfrom pandas import DataFrame-from mwmbl.rankeval.evaluation.evaluate import evaluate, RankingModel, NUM_RESULTS_FOR_EVAL+from mwmbl.rankeval.evaluation.evaluate import NUM_RESULTS_FOR_EVAL, RankingModel, evaluatefrom mwmbl.rankeval.evaluation.urldb import find_matching_urls-from mwmbl.rankeval.paths import URLS_PATH, RANKINGS_DATASET_TEST_PATH+from mwmbl.rankeval.paths import RANKINGS_DATASET_TEST_PATH, URLS_PATHclass OracleRankingModel(RankingModel):@@ -20,8 +20,8 @@ class OracleRankingModel(RankingModel):self.rankings = {}def prepare_rankings(self, gold_standard: DataFrame, url_database_path: str):- for query, rankings in gold_standard.groupby('query'):- gold_standard_urls = rankings['url'].to_list()[:NUM_RESULTS_FOR_EVAL]+ for query, rankings in gold_standard.groupby("query"):+ gold_standard_urls = rankings["url"].to_list()[:NUM_RESULTS_FOR_EVAL]matches = set(find_matching_urls(url_database_path, gold_standard_urls))oracle_ranking = [url for url in gold_standard_urls if url in matches]if oracle_ranking:@@ -39,5 +39,5 @@ def run():evaluate(model)-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/rankeval/evaluation/evaluate_ranker.py b/mwmbl/rankeval/evaluation/evaluate_ranker.pyindex 6b7ffa3..677b19d 100644--- a/mwmbl/rankeval/evaluation/evaluate_ranker.py+++ b/mwmbl/rankeval/evaluation/evaluate_ranker.py@@ -1,9 +1,8 @@from argparse import ArgumentParser-from mwmbl.tinysearchengine.indexer import TinyIndex, Document-from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker-from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate+from mwmbl.tinysearchengine.indexer import Document, TinyIndex+from mwmbl.tinysearchengine.rank import HeuristicRanker, Rankerclass MwmblRankingModel(RankingModel):@@ -22,8 +21,8 @@ class DummyCompleter:def run():arg_parser = ArgumentParser()- arg_parser.add_argument('--index', help='Path to the index', required=True)- arg_parser.add_argument('--note', required=True)+ arg_parser.add_argument("--index", help="Path to the index", required=True)+ arg_parser.add_argument("--note", required=True)args = arg_parser.parse_args()@@ -38,5 +37,5 @@ def run():evaluate(model)-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pyindex 21ce5f4..33690fc 100644--- a/mwmbl/rankeval/evaluation/evaluate_remote.py+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -1,23 +1,19 @@"""Evaluate against queries on a remote search engine."""+import pickle-from argparse import ArgumentParserimport requestsfrom joblib import Memory+from mwmbl.rankeval.evaluation.evaluate import evaluatefrom mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter, MwmblRankingModelfrom mwmbl.rankeval.evaluation.remote_index import RemoteIndexfrom mwmbl.rankeval.paths import MODEL_PATH, RUST_MODEL_PATH-from mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.ltr import RustXGBPipelinefrom mwmbl.tinysearchengine.ltr_rank import LTRRankerfrom mwmbl.tinysearchengine.mmr_rank import MMRRanker-from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, HeuristicAndWikiRanker--from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate-memory = Memory(location="devdata/cache")@@ -30,7 +26,7 @@ def fetch_results(url: str, query: str):def run():# ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())-+# model = pickle.load(open(MODEL_PATH, 'rb'))model = RustXGBPipeline.from_model_path(str(RUST_MODEL_PATH))ranker = MMRRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))@@ -42,7 +38,7 @@ def run():def single_query(query: str):- model = pickle.load(open(MODEL_PATH, 'rb'))+ model = pickle.load(open(MODEL_PATH, "rb"))ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3)# ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())results = ranker.search(query, [])@@ -50,7 +46,7 @@ def single_query(query: str):print(result)-if __name__ == '__main__':+if __name__ == "__main__":run()# import cProfile# cProfile.run("run()", sort="cumtime")diff --git a/mwmbl/rankeval/evaluation/evaluate_super_search.py b/mwmbl/rankeval/evaluation/evaluate_super_search.pyindex 71a7fd2..e6cbf11 100644--- a/mwmbl/rankeval/evaluation/evaluate_super_search.py+++ b/mwmbl/rankeval/evaluation/evaluate_super_search.py@@ -20,10 +20,10 @@ Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \uv run python -m mwmbl.rankeval.evaluation.evaluate_super_search --fraction 0.02"""+import asyncioimport osfrom argparse import ArgumentParser-from pathlib import Pathimport django@@ -40,7 +40,6 @@ from mwmbl.tinysearchengine.indexer import Document # noqa: E402from mwmbl.tinysearchengine.super_search import _emit_final_results, _run_pipeline # noqa: E402from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext # noqa: E402-memory = Memory(location=str(DATA_DIR / "super-search-eval-cache"), verbose=0)@@ -49,13 +48,20 @@ async def _noop_emit(event_type, data):def _doc_to_dict(doc: Document) -> dict:- return {"title": doc.title, "url": doc.url, "extract": doc.extract,- "score": doc.score, "term": doc.term, "state": doc.state}+ return {+ "title": doc.title,+ "url": doc.url,+ "extract": doc.extract,+ "score": doc.score,+ "term": doc.term,+ "state": doc.state,+ }def _dict_to_doc(d: dict) -> Document:- return Document(title=d["title"], url=d["url"], extract=d["extract"],- score=d["score"], term=d["term"], state=d["state"])+ return Document(+ title=d["title"], url=d["url"], extract=d["extract"], score=d["score"], term=d["term"], state=d["state"]+ )async def _collect(query: str) -> list[Document]:@@ -113,12 +119,11 @@ class SuperSearchRankingModel(RankingModel):def run():parser = ArgumentParser()- parser.add_argument("--fraction", type=float, default=0.02,- help="Fraction of gold queries to sample (collection is slow).")- parser.add_argument("--train", action="store_true",- help="Evaluate on the train split instead of test.")- parser.add_argument("--clear-cache", action="store_true",- help="Clear the cached document pools before evaluating.")+ parser.add_argument(+ "--fraction", type=float, default=0.02, help="Fraction of gold queries to sample (collection is slow)."+ )+ parser.add_argument("--train", action="store_true", help="Evaluate on the train split instead of test.")+ parser.add_argument("--clear-cache", action="store_true", help="Clear the cached document pools before evaluating.")args = parser.parse_args()if args.clear_cache:diff --git a/mwmbl/rankeval/evaluation/evaluate_wiki.py b/mwmbl/rankeval/evaluation/evaluate_wiki.pyindex 365ad44..3750446 100644--- a/mwmbl/rankeval/evaluation/evaluate_wiki.py+++ b/mwmbl/rankeval/evaluation/evaluate_wiki.py@@ -1,13 +1,13 @@"""Evaluate ranking where we only use Wikipedia as the index."""+import urllibimport requestsfrom joblib import Memoryfrom mwmbl.rankeval.evaluation.evaluate import evaluate-from mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.rank import WIKI_SEARCH_API_URL, get_wiki_urlmemory = Memory(location="devdata/cache")@@ -20,7 +20,7 @@ def fetch_results(query: str) -> list[str]:print("Getting url", url)results = requests.get(url).json()print("Results", results)- return [get_wiki_url(result['title']) for result in results['query']['search']]+ return [get_wiki_url(result["title"]) for result in results["query"]["search"]]class WikiModel:diff --git a/mwmbl/rankeval/evaluation/remote_index.py b/mwmbl/rankeval/evaluation/remote_index.pyindex e24da90..7aa05ed 100644--- a/mwmbl/rankeval/evaluation/remote_index.py+++ b/mwmbl/rankeval/evaluation/remote_index.py@@ -1,6 +1,7 @@"""Pretend to be an index but retrieve results from a remote index."""+import timefrom logging import getLogger@@ -9,7 +10,6 @@ import requestsfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.utils import request_cache-logger = getLogger(__name__)diff --git a/mwmbl/rankeval/evaluation/urldb.py b/mwmbl/rankeval/evaluation/urldb.pyindex 03acabc..5a4d44a 100644--- a/mwmbl/rankeval/evaluation/urldb.py+++ b/mwmbl/rankeval/evaluation/urldb.py@@ -6,9 +6,12 @@ def find_matching_urls(url_db_path: str, urls: list[str]) -> list[str]:return []connection = sqlite3.connect(url_db_path)- seq = ','.join(['?'] * len(urls))- result = connection.execute(f"""+ seq = ",".join(["?"] * len(urls))+ result = connection.execute(+ f"""SELECT url from urlsWHERE url IN ({seq})- """, urls)+ """,+ urls,+ )return [row[0] for row in result.fetchall()]diff --git a/mwmbl/rankeval/ltr/baseline.py b/mwmbl/rankeval/ltr/baseline.pyindex d9bb45b..a0ea809 100644--- a/mwmbl/rankeval/ltr/baseline.py+++ b/mwmbl/rankeval/ltr/baseline.py@@ -1,7 +1,6 @@from numpy.random import RandomStatefrom sklearn.base import BaseEstimator-random = RandomState(1)diff --git a/mwmbl/rankeval/ltr/dataset.py b/mwmbl/rankeval/ltr/dataset.pyindex 91ffdbf..b89d57c 100644--- a/mwmbl/rankeval/ltr/dataset.py+++ b/mwmbl/rankeval/ltr/dataset.py@@ -1,16 +1,13 @@"""Generate a learn-to-rank dataset."""-import sysimport pandas as pd-from mwmbl.rankeval.evaluation.remote_index import RemoteIndex-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState-from mwmbl.tinysearchengine.rank import HeuristicRanker, HeuristicAndWikiRanker-from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter-from mwmbl.rankeval.paths import RANKINGS_DATASET_TRAIN_PATH, LEARNING_TO_RANK_DATASET_PATH+from mwmbl.rankeval.evaluation.remote_index import RemoteIndex+from mwmbl.rankeval.paths import LEARNING_TO_RANK_DATASET_PATH, RANKINGS_DATASET_TRAIN_PATH+from mwmbl.tinysearchengine.rank import HeuristicAndWikiRankerdef run():@@ -26,15 +23,15 @@ def get_dataset(completer):index,completer,return_none_if_no_mwmbl_results=True,- score_threshold=float('-inf'),+ score_threshold=float("-inf"),)gold_standard = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)dataset = []- for query, rankings in gold_standard.groupby('query'):+ for query, rankings in gold_standard.groupby("query"):print("Query", query)- gold_standard = dict(zip(rankings['url'].tolist(), rankings.index.tolist()))+ gold_standard = dict(zip(rankings["url"].tolist(), rankings.index.tolist()))- predicted = ranker.search(query + ' ', [])+ predicted = ranker.search(query + " ", [])if len(predicted) == 0:continue@@ -45,15 +42,17 @@ def get_dataset(completer):found_gold = Falsefor item in predicted:in_gold_standard = item.url in gold_standard- new_items.append({- 'gold_standard_rank': gold_standard.get(item.url),- 'query': query,- 'url': item.url,- 'title': item.title,- 'extract': item.extract,- 'state': item.state,- 'score': item.score,- })+ new_items.append(+ {+ "gold_standard_rank": gold_standard.get(item.url),+ "query": query,+ "url": item.url,+ "title": item.title,+ "extract": item.extract,+ "state": item.state,+ "score": item.score,+ }+ )if in_gold_standard:found_gold = Trueif found_gold:@@ -62,5 +61,5 @@ def get_dataset(completer):return dataset-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/rankeval/ltr/evaluate.py b/mwmbl/rankeval/ltr/evaluate.pyindex dd8245d..29652f0 100644--- a/mwmbl/rankeval/ltr/evaluate.py+++ b/mwmbl/rankeval/ltr/evaluate.py@@ -1,12 +1,12 @@"""Evaluate a learning to rank dataset."""+import picklefrom argparse import ArgumentParserimport numpy as npimport pandas as pd-from mwmbl.tinysearchengine.ltr import ThresholdPredictor, FeatureExtractor, RankingPredictor, RustXGBPipelinefrom scipy.stats import semfrom sklearn.base import clonefrom sklearn.dummy import DummyRegressor@@ -14,41 +14,46 @@ from sklearn.metrics import ndcg_scorefrom sklearn.model_selection import GroupKFoldfrom sklearn.pipeline import make_pipelinefrom sklearn.tree import DecisionTreeClassifier-from xgboost import XGBClassifier, XGBRegressor, XGBRanker+from xgboost import XGBClassifier, XGBRanker, XGBRegressorfrom mwmbl.rankeval.evaluation.evaluate import CLICK_PROPORTIONSfrom mwmbl.rankeval.ltr.baseline import RandomRegressorfrom mwmbl.rankeval.paths import LEARNING_TO_RANK_DATASET_PATH, MODEL_PATH, RUST_MODEL_PATH+from mwmbl.tinysearchengine.ltr import FeatureExtractor, RankingPredictor, RustXGBPipeline, ThresholdPredictorPREDICTORS = {- 'random': RandomRegressor(),- 'constant': DummyRegressor(),- 'decision_tree': make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, DecisionTreeClassifier())),- 'xgb': make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(scale_pos_weight=0.1, reg_lambda=2))),- 'xgb_rust': RustXGBPipeline(threshold=0.0, scale_pos_weight=0.1, reg_lambda=2.0, num_rounds=100),- 'xgb_limit_terms': RankingPredictor(FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(scale_pos_weight=0.1, reg_lambda=2))),- 'xgb_regressor': make_pipeline(FeatureExtractor(), XGBRegressor(objective="reg:pseudohubererror")),- 'xgb_ranker': make_pipeline(FeatureExtractor(), XGBRanker(objective="rank:ndcg", reg_lambda=2)),+ "random": RandomRegressor(),+ "constant": DummyRegressor(),+ "decision_tree": make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, DecisionTreeClassifier())),+ "xgb": make_pipeline(+ FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(scale_pos_weight=0.1, reg_lambda=2))+ ),+ "xgb_rust": RustXGBPipeline(threshold=0.0, scale_pos_weight=0.1, reg_lambda=2.0, num_rounds=100),+ "xgb_limit_terms": RankingPredictor(+ FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(scale_pos_weight=0.1, reg_lambda=2))+ ),+ "xgb_regressor": make_pipeline(FeatureExtractor(), XGBRegressor(objective="reg:pseudohubererror")),+ "xgb_ranker": make_pipeline(FeatureExtractor(), XGBRanker(objective="rank:ndcg", reg_lambda=2)),}def print_feature_importances(predictor_name: str, model):"""Print top feature importances for XGBoost models."""xgb_model = None- if hasattr(model, 'steps'):+ if hasattr(model, "steps"):last_step = model.steps[-1][1]if isinstance(last_step, ThresholdPredictor):xgb_model = last_step.classifier- elif hasattr(last_step, 'feature_importances_'):+ elif hasattr(last_step, "feature_importances_"):xgb_model = last_stepelif isinstance(model, RankingPredictor):inner = model.modelif isinstance(inner, ThresholdPredictor):xgb_model = inner.classifier- elif hasattr(inner, 'feature_importances_'):+ elif hasattr(inner, "feature_importances_"):xgb_model = inner- if xgb_model is None or not hasattr(xgb_model, 'feature_importances_'):+ if xgb_model is None or not hasattr(xgb_model, "feature_importances_"):returnimportances = xgb_model.feature_importances_@@ -70,74 +75,86 @@ def get_discount(rank: float):def run():parser = ArgumentParser()- parser.add_argument('--predictor', required=True, choices=sorted(PREDICTORS))+ parser.add_argument("--predictor", required=True, choices=sorted(PREDICTORS))parser.add_argument("--binary-labels", required=False, action="store_true")- parser.add_argument('--note', required=True)- parser.add_argument('--max-depth', type=int, default=None, help='XGB max_depth parameter')- parser.add_argument('--scale-pos-weight', type=float, default=None, help='XGB scale_pos_weight parameter')- parser.add_argument('--n-estimators', type=int, default=None, help='XGB n_estimators parameter')- parser.add_argument('--min-child-weight', type=float, default=None, help='XGB min_child_weight parameter')- parser.add_argument('--gamma', type=float, default=None, help='XGB gamma (min_split_loss) parameter')- parser.add_argument('--reg-gamma', type=float, default=None, help='XGB reg_gamma parameter')- parser.add_argument('--reg-lambda', type=float, default=None, help='XGB reg_lambda parameter')- parser.add_argument('--subsample', type=float, default=None, help='XGB subsample parameter')+ parser.add_argument("--note", required=True)+ parser.add_argument("--max-depth", type=int, default=None, help="XGB max_depth parameter")+ parser.add_argument("--scale-pos-weight", type=float, default=None, help="XGB scale_pos_weight parameter")+ parser.add_argument("--n-estimators", type=int, default=None, help="XGB n_estimators parameter")+ parser.add_argument("--min-child-weight", type=float, default=None, help="XGB min_child_weight parameter")+ parser.add_argument("--gamma", type=float, default=None, help="XGB gamma (min_split_loss) parameter")+ parser.add_argument("--reg-gamma", type=float, default=None, help="XGB reg_gamma parameter")+ parser.add_argument("--reg-lambda", type=float, default=None, help="XGB reg_lambda parameter")+ parser.add_argument("--subsample", type=float, default=None, help="XGB subsample parameter")args = parser.parse_args()# Build a dict of only the XGB params that were explicitly provided,# so they override the hardcoded defaults in each predictor constructor.- xgb_params = {k: v for k, v in {- 'max_depth': args.max_depth,- 'scale_pos_weight': args.scale_pos_weight,- 'n_estimators': args.n_estimators,- 'min_child_weight': args.min_child_weight,- 'gamma': args.gamma,- 'reg_gamma': args.reg_gamma,- 'subsample': args.subsample,- }.items() if v is not None}-- rust_params = {k: v for k, v in {- 'scale_pos_weight': args.scale_pos_weight,- 'num_rounds': args.n_estimators,- 'max_depth': args.max_depth,- 'min_child_weight': args.min_child_weight,- 'gamma': args.gamma,- 'reg_lambda': args.reg_lambda,- 'subsample': args.subsample,- }.items() if v is not None}+ xgb_params = {+ k: v+ for k, v in {+ "max_depth": args.max_depth,+ "scale_pos_weight": args.scale_pos_weight,+ "n_estimators": args.n_estimators,+ "min_child_weight": args.min_child_weight,+ "gamma": args.gamma,+ "reg_gamma": args.reg_gamma,+ "subsample": args.subsample,+ }.items()+ if v is not None+ }++ rust_params = {+ k: v+ for k, v in {+ "scale_pos_weight": args.scale_pos_weight,+ "num_rounds": args.n_estimators,+ "max_depth": args.max_depth,+ "min_child_weight": args.min_child_weight,+ "gamma": args.gamma,+ "reg_lambda": args.reg_lambda,+ "subsample": args.subsample,+ }.items()+ if v is not None+ }# Merge hardcoded defaults with any CLI overrides (CLI wins).- xgb_classifier_params = {'scale_pos_weight': 0.1, 'reg_lambda': 2, **xgb_params}- xgb_ranker_params = {'reg_lambda': 2, **xgb_params}- rust_pipeline_params = {'scale_pos_weight': 0.1, 'reg_lambda': 2.0, 'num_rounds': 100, **rust_params}+ xgb_classifier_params = {"scale_pos_weight": 0.1, "reg_lambda": 2, **xgb_params}+ xgb_ranker_params = {"reg_lambda": 2, **xgb_params}+ rust_pipeline_params = {"scale_pos_weight": 0.1, "reg_lambda": 2.0, "num_rounds": 100, **rust_params}predictors = {- 'random': RandomRegressor(),- 'constant': DummyRegressor(),- 'decision_tree': make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, DecisionTreeClassifier())),- 'xgb': make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(**xgb_classifier_params))),- 'xgb_rust': RustXGBPipeline(threshold=0.0, **rust_pipeline_params),- 'xgb_limit_terms': RankingPredictor(FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(**xgb_classifier_params))),- 'xgb_regressor': make_pipeline(FeatureExtractor(), XGBRegressor(objective="reg:pseudohubererror", **xgb_params)),- 'xgb_ranker': make_pipeline(FeatureExtractor(), XGBRanker(objective="rank:ndcg", **xgb_ranker_params)),+ "random": RandomRegressor(),+ "constant": DummyRegressor(),+ "decision_tree": make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, DecisionTreeClassifier())),+ "xgb": make_pipeline(FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(**xgb_classifier_params))),+ "xgb_rust": RustXGBPipeline(threshold=0.0, **rust_pipeline_params),+ "xgb_limit_terms": RankingPredictor(+ FeatureExtractor(), ThresholdPredictor(0.0, XGBClassifier(**xgb_classifier_params))+ ),+ "xgb_regressor": make_pipeline(+ FeatureExtractor(), XGBRegressor(objective="reg:pseudohubererror", **xgb_params)+ ),+ "xgb_ranker": make_pipeline(FeatureExtractor(), XGBRanker(objective="rank:ndcg", **xgb_ranker_params)),}predictor = predictors[args.predictor]- dataset = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator='\n')+ dataset = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator="\n")if args.binary_labels:- dataset['gold_discount'] = dataset['gold_standard_rank'].apply(lambda x: 1 if x > 0 else 0)+ dataset["gold_discount"] = dataset["gold_standard_rank"].apply(lambda x: 1 if x > 0 else 0)else:- dataset['gold_discount'] = dataset['gold_standard_rank'].apply(get_discount)+ dataset["gold_discount"] = dataset["gold_standard_rank"].apply(get_discount)- print("Gold standard", dataset['gold_discount'])+ print("Gold standard", dataset["gold_discount"])- X = dataset[['query', 'url', 'title', 'extract', 'score']]- X['title'].fillna('', inplace=True)- X['extract'].fillna('', inplace=True)- y = dataset['gold_discount']- query_id, query_index = dataset['query'].factorize()- groups = dataset['query']+ X = dataset[["query", "url", "title", "extract", "score"]]+ X["title"].fillna("", inplace=True)+ X["extract"].fillna("", inplace=True)+ y = dataset["gold_discount"]+ query_id, query_index = dataset["query"].factorize()+ groups = dataset["query"]cross_validator = GroupKFold(n_splits=3)@@ -147,7 +164,7 @@ def run():for fold_idx, (train, test) in enumerate(splits):print(f"Fold {fold_idx + 1}: training on {len(train)} rows, testing on {len(test)} rows...")model = clone(predictor)- if args.predictor == 'xgb_ranker':+ if args.predictor == "xgb_ranker":model.fit(X.iloc[train], y.iloc[train], xgbranker__qid=query_id[train])else:model.fit(X.iloc[train], y.iloc[train])@@ -157,13 +174,13 @@ def run():print(f"Fold {fold_idx + 1}: predictions complete, computing NDCG...")test_dataset = dataset.iloc[test].copy()- test_dataset['prediction'] = predictions- for query, rankings in test_dataset.groupby('query'):+ test_dataset["prediction"] = predictions+ for query, rankings in test_dataset.groupby("query"):if len(rankings) == 1:continue- rankings_gold_discount = rankings['gold_discount'].tolist()- rankings_prediction = rankings['prediction'].tolist()+ rankings_gold_discount = rankings["gold_discount"].tolist()+ rankings_prediction = rankings["prediction"].tolist()score = ndcg_score([rankings_gold_discount], [rankings_prediction])scores.append(score)print(f"Fold {fold_idx + 1}: NDCG computed, mean so far = {np.mean(scores):.4f}")@@ -174,21 +191,21 @@ def run():final_model = clone(predictor)- if args.predictor == 'xgb_ranker':+ if args.predictor == "xgb_ranker":final_model.fit(X, y, xgbranker__qid=query_id)else:final_model.fit(X, y)- if args.predictor == 'xgb_rust':+ if args.predictor == "xgb_rust":# RustXGBPipeline cannot be pickled; use its native save_model() instead.print(f"Saving Rust XGBoost model to {RUST_MODEL_PATH}")final_model.save_model(str(RUST_MODEL_PATH))else:- with open(MODEL_PATH, 'wb') as output_file:+ with open(MODEL_PATH, "wb") as output_file:pickle.dump(final_model, output_file)print_feature_importances(args.predictor, final_model)-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/rankeval/ltr/llm_experiment.py b/mwmbl/rankeval/ltr/llm_experiment.pyindex 773fd56..ec54beb 100644--- a/mwmbl/rankeval/ltr/llm_experiment.py+++ b/mwmbl/rankeval/ltr/llm_experiment.py@@ -36,6 +36,7 @@ Usage:uv run python -m mwmbl.rankeval.ltr.llm_experiment --mode llm-only --overall-threshold 4 --scale-pos-weight 1.0 --note Buv run python -m mwmbl.rankeval.ltr.llm_experiment --mode mixed --ext-weight 0.25 --overall-threshold 4 --scale-pos-weight 1.0 --note C"""+import gzipimport jsonfrom argparse import ArgumentParser@@ -55,29 +56,29 @@ from mwmbl.rankeval.paths import ()from mwmbl.tinysearchengine.ltr import RustXGBPipeline-FEATURE_COLUMNS = ['query', 'url', 'title', 'extract', 'score']+FEATURE_COLUMNS = ["query", "url", "title", "extract", "score"]RELEVANT_OVERALL = 4-JUDGMENTS_EXPORT_DIR = ROOT_DIR / 'devdata' / 'judgments_export'+JUDGMENTS_EXPORT_DIR = ROOT_DIR / "devdata" / "judgments_export"CURATION_WEIGHT_CAP = 3.0 # bound on accumulated per-(query,url) evidence weightdef _normalise(dataset: pd.DataFrame) -> pd.DataFrame:dataset = dataset.copy()- dataset['qnorm'] = dataset['query'].str.lower().str.strip()- dataset['title'] = dataset['title'].fillna('')- dataset['extract'] = dataset['extract'].fillna('')- dataset['score'] = dataset['score'].fillna(0.0)+ dataset["qnorm"] = dataset["query"].str.lower().str.strip()+ dataset["title"] = dataset["title"].fillna("")+ dataset["extract"] = dataset["extract"].fillna("")+ dataset["score"] = dataset["score"].fillna(0.0)return datasetdef load_datasets() -> tuple[pd.DataFrame, pd.DataFrame]:- ext = _normalise(pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator='\n'))- llm = _normalise(pd.read_csv(LEARNING_TO_RANK_LLM_DATASET_PATH, lineterminator='\n'))+ ext = _normalise(pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator="\n"))+ llm = _normalise(pd.read_csv(LEARNING_TO_RANK_LLM_DATASET_PATH, lineterminator="\n"))return ext, llmdef split_llm_queries(llm: pd.DataFrame, test_size: float, split_seed: int) -> tuple[set[str], set[str]]:- queries = np.sort(llm['qnorm'].unique())+ queries = np.sort(llm["qnorm"].unique())rng = np.random.default_rng(split_seed)rng.shuffle(queries)num_test = int(round(len(queries) * test_size))@@ -99,74 +100,86 @@ def load_curation_frame(weak_neg_weight: float) -> pd.DataFrame:extract are dropped; `score` (the index score feature) is unknown forcuration rows and set to 0.0."""+def read_jsonl_gz(path: Path) -> list[dict]:- with gzip.open(path, 'rt') as f:+ with gzip.open(path, "rt") as f:return [json.loads(line) for line in f]evidence: dict[tuple[str, str], list[tuple[float, float]]] = defaultdict(list)texts: dict[tuple[str, str], tuple[str, str, str]] = {}def add(query: str, doc: dict, label: float, weight: float) -> None:- url = doc.get('url')- title = (doc.get('title') or '').strip()- extract = (doc.get('extract') or '').strip()+ url = doc.get("url")+ title = (doc.get("title") or "").strip()+ extract = (doc.get("extract") or "").strip()if not url or not (title or extract):returnqnorm = query.lower().strip()evidence[(qnorm, url)].append((label, weight))texts.setdefault((qnorm, url), (query, title, extract))- for pair in read_jsonl_gz(JUDGMENTS_EXPORT_DIR / 'pairs.jsonl.gz'):- add(pair['query'], pair['pos'], 1.0, 1.0)- neg_weight = 1.0 if pair['rule'] == 'delete' else weak_neg_weight- add(pair['query'], pair['neg'], 0.0, neg_weight)- for point in read_jsonl_gz(JUDGMENTS_EXPORT_DIR / 'pointwise.jsonl.gz'):- add(point['query'], point, 1.0 if point['label'] > 0 else 0.0, 1.0)+ for pair in read_jsonl_gz(JUDGMENTS_EXPORT_DIR / "pairs.jsonl.gz"):+ add(pair["query"], pair["pos"], 1.0, 1.0)+ neg_weight = 1.0 if pair["rule"] == "delete" else weak_neg_weight+ add(pair["query"], pair["neg"], 0.0, neg_weight)+ for point in read_jsonl_gz(JUDGMENTS_EXPORT_DIR / "pointwise.jsonl.gz"):+ add(point["query"], point, 1.0 if point["label"] > 0 else 0.0, 1.0)rows = []for (qnorm, url), points in evidence.items():total = sum(w for _, w in points)positive = sum(w for label, w in points if label > 0)query, title, extract = texts[(qnorm, url)]- rows.append({- 'query': query, 'qnorm': qnorm, 'url': url, 'title': title,- 'extract': extract, 'score': 0.0,- 'label': 1.0 if positive / total >= 0.5 else 0.0,- 'weight': min(total, CURATION_WEIGHT_CAP),- })+ rows.append(+ {+ "query": query,+ "qnorm": qnorm,+ "url": url,+ "title": title,+ "extract": extract,+ "score": 0.0,+ "label": 1.0 if positive / total >= 0.5 else 0.0,+ "weight": min(total, CURATION_WEIGHT_CAP),+ }+ )return pd.DataFrame(rows)def load_human_pairs() -> list[dict]:"""Human preference pairs with text on both sides, plus a qnorm key."""- with gzip.open(JUDGMENTS_EXPORT_DIR / 'pairs.jsonl.gz', 'rt') as f:+ with gzip.open(JUDGMENTS_EXPORT_DIR / "pairs.jsonl.gz", "rt") as f:pairs = [json.loads(line) for line in f]- pairs = [p for p in pairs if all(p[s]['title'] and p[s]['extract'] for s in ('pos', 'neg'))]+ pairs = [p for p in pairs if all(p[s]["title"] and p[s]["extract"] for s in ("pos", "neg"))]for pair in pairs:- pair['qnorm'] = pair['query'].lower().strip()+ pair["qnorm"] = pair["query"].lower().strip()return pairs-def split_human_pairs(pairs: list[dict], test_size: float, split_seed: int- ) -> tuple[list[dict], set[str]]:+def split_human_pairs(pairs: list[dict], test_size: float, split_seed: int) -> tuple[list[dict], set[str]]:"""Query-level test split of the human pairs: (test pairs, test queries)."""if test_size <= 0:return [], set()- queries = np.sort(pd.unique([p['qnorm'] for p in pairs]))+ queries = np.sort(pd.unique([p["qnorm"] for p in pairs]))rng = np.random.default_rng(split_seed)rng.shuffle(queries)- test_queries = set(queries[:int(round(len(queries) * test_size))])- return [p for p in pairs if p['qnorm'] in test_queries], test_queries+ test_queries = set(queries[: int(round(len(queries) * test_size))])+ return [p for p in pairs if p["qnorm"] in test_queries], test_queriesdef pair_accuracy(model, test_pairs: list[dict]) -> tuple[float, float]:"""Fraction of held-out human pairs ranked pos > neg (ties count 0.5)."""- def rec(pair, side):- return {'query': pair['query'], 'url': pair[side]['url'],- 'title': pair[side]['title'], 'extract': pair[side]['extract'], 'score': 0.0}- pos = np.array(model.predict([rec(p, 'pos') for p in test_pairs]))- neg = np.array(model.predict([rec(p, 'neg') for p in test_pairs]))+ def rec(pair, side):+ return {+ "query": pair["query"],+ "url": pair[side]["url"],+ "title": pair[side]["title"],+ "extract": pair[side]["extract"],+ "score": 0.0,+ }++ pos = np.array(model.predict([rec(p, "pos") for p in test_pairs]))+ neg = np.array(model.predict([rec(p, "neg") for p in test_pairs]))correct = np.where(pos > neg, 1.0, np.where(pos == neg, 0.5, 0.0))return float(correct.mean()), float((pos == neg).mean())@@ -184,46 +197,45 @@ def build_training_frame(curation: pd.DataFrame | None = None,curation_weight: float = 1.0,) -> pd.DataFrame:- llm_train = llm[llm['qnorm'].isin(train_queries)].copy()- llm_train['label'] = (llm_train['overall'] >= overall_threshold).astype(float)- llm_train['weight'] = 1.0+ llm_train = llm[llm["qnorm"].isin(train_queries)].copy()+ llm_train["label"] = (llm_train["overall"] >= overall_threshold).astype(float)+ llm_train["weight"] = 1.0- ext_train = ext[~ext['qnorm'].isin(test_queries)].copy()- ext_train['label'] = ext_train['gold_standard_rank'].notna().astype(float)- ext_train['weight'] = 1.0+ ext_train = ext[~ext["qnorm"].isin(test_queries)].copy()+ ext_train["label"] = ext_train["gold_standard_rank"].notna().astype(float)+ ext_train["weight"] = 1.0- if mode == 'llm-only':+ if mode == "llm-only":train = llm_train- elif mode == 'ext-only':+ elif mode == "ext-only":train = ext_train- elif mode == 'mixed':+ elif mode == "mixed":# Graded LLM labels supersede SERP presence for pairs both datasets cover.- llm_pairs = set(zip(llm_train['qnorm'], llm_train['url']))- pair_mask = [pair not in llm_pairs for pair in zip(ext_train['qnorm'], ext_train['url'])]+ llm_pairs = set(zip(llm_train["qnorm"], llm_train["url"]))+ pair_mask = [pair not in llm_pairs for pair in zip(ext_train["qnorm"], ext_train["url"])]ext_train = ext_train[pair_mask]if ext_downsample is not None:ext_train = ext_train.sample(frac=ext_downsample, random_state=seed)else:- ext_train['weight'] = ext_weight+ ext_train["weight"] = ext_weighttrain = pd.concat([llm_train, ext_train], ignore_index=True)else:raise ValueError(f"unknown training mode: {mode}")if curation is not None:- curation = curation[~curation['qnorm'].isin(test_queries)].copy()+ curation = curation[~curation["qnorm"].isin(test_queries)].copy()# On (query,url) collisions keep the existing row: the LLM's graded# judgment (or SERP gold) already covers that pair; curation adds the# pairs no other dataset has. Collisions are counted for visibility.- train_pairs = set(zip(train['qnorm'], train['url']))- collisions = [pair in train_pairs for pair in zip(curation['qnorm'], curation['url'])]+ train_pairs = set(zip(train["qnorm"], train["url"]))+ collisions = [pair in train_pairs for pair in zip(curation["qnorm"], curation["url"])]print(f"Curation: {len(curation)} rows, dropping {sum(collisions)} colliding with training frame")curation = curation[[not c for c in collisions]].copy()- curation['weight'] *= curation_weight- train = pd.concat([train, curation[FEATURE_COLUMNS + ['qnorm', 'label', 'weight']]],- ignore_index=True)+ curation["weight"] *= curation_weight+ train = pd.concat([train, curation[FEATURE_COLUMNS + ["qnorm", "label", "weight"]]], ignore_index=True)- assert not set(train['qnorm']) & test_queries, "test queries leaked into training frame"- return train[FEATURE_COLUMNS + ['label', 'weight']]+ assert not set(train["qnorm"]) & test_queries, "test queries leaked into training frame"+ return train[FEATURE_COLUMNS + ["label", "weight"]]def train(train_df: pd.DataFrame, scale_pos_weight: float, reg_lambda: float, num_rounds: int) -> RustXGBPipeline:@@ -234,96 +246,117 @@ def train(train_df: pd.DataFrame, scale_pos_weight: float, reg_lambda: float, nureg_lambda=reg_lambda,num_rounds=num_rounds,)- weights = train_df['weight'] if (train_df['weight'] != 1.0).any() else None- model.fit(train_df[FEATURE_COLUMNS], train_df['label'], sample_weight=weights)+ weights = train_df["weight"] if (train_df["weight"] != 1.0).any() else None+ model.fit(train_df[FEATURE_COLUMNS], train_df["label"], sample_weight=weights)return modeldef evaluate(model, test_df: pd.DataFrame) -> dict:test_df = test_df.copy()- test_df['prediction'] = model.predict(test_df[FEATURE_COLUMNS])+ test_df["prediction"] = model.predict(test_df[FEATURE_COLUMNS])ndcgs = []ndcgs_at_10 = []precisions = {5: [], 10: []}skipped = 0- for _, rankings in test_df.groupby('qnorm'):- gains = rankings['overall'].tolist()- predictions = rankings['prediction'].tolist()+ for _, rankings in test_df.groupby("qnorm"):+ gains = rankings["overall"].tolist()+ predictions = rankings["prediction"].tolist()if len(rankings) == 1 or sum(gains) == 0:skipped += 1continuendcgs.append(ndcg_score([gains], [predictions]))ndcgs_at_10.append(ndcg_score([gains], [predictions], k=10))- ranked = rankings.sort_values('prediction', ascending=False)- relevant = ranked['overall'] >= RELEVANT_OVERALL+ ranked = rankings.sort_values("prediction", ascending=False)+ relevant = ranked["overall"] >= RELEVANT_OVERALLfor k in precisions:top = relevant.head(k)precisions[k].append(top.sum() / len(top))return {- 'ndcg': np.mean(ndcgs),- 'ndcg_sem': sem(ndcgs),- 'ndcg@10': np.mean(ndcgs_at_10),- 'p@5': np.mean(precisions[5]),- 'p@10': np.mean(precisions[10]),- 'queries': len(ndcgs),- 'skipped': skipped,+ "ndcg": np.mean(ndcgs),+ "ndcg_sem": sem(ndcgs),+ "ndcg@10": np.mean(ndcgs_at_10),+ "p@5": np.mean(precisions[5]),+ "p@10": np.mean(precisions[10]),+ "queries": len(ndcgs),+ "skipped": skipped,}def run():parser = ArgumentParser(description=__doc__)- parser.add_argument('--mode', required=True, choices=['baseline', 'ext-only', 'llm-only', 'mixed'])- parser.add_argument('--note', required=True)- parser.add_argument('--model-path', default=str(CURRENT_MODEL_PATH),- help='model artifact to evaluate in baseline mode')- parser.add_argument('--overall-threshold', type=float, default=4.0,- help='LLM rows are positive when overall >= this')- parser.add_argument('--ext-weight', type=float, default=1.0,- help='per-row sample weight for extension rows in mixed mode')- parser.add_argument('--ext-downsample', type=float, default=None,- help='legacy behaviour: keep this fraction of extension rows instead of weighting')- parser.add_argument('--add-curation', action='store_true',- help='mix in the judgments_export curation/vote data as weighted pointwise rows')- parser.add_argument('--curation-weight', type=float, default=1.0,- help='multiplier on curation row weights')- parser.add_argument('--weak-neg-weight', type=float, default=0.25,- help='weight of out-ranked (add/move/approve) pair negatives')- parser.add_argument('--scale-pos-weight', type=float, default=0.1)- parser.add_argument('--reg-lambda', type=float, default=2.0)- parser.add_argument('--num-rounds', type=int, default=100)- parser.add_argument('--seed', type=int, default=0, help='downsampling seed (--ext-downsample only)')- parser.add_argument('--split-seed', type=int, default=42,- help='train/test query split seed; keep fixed across compared runs')- parser.add_argument('--test-size', type=float, default=0.3)- parser.add_argument('--save-model', default=None, help='optional path to save the trained model')+ parser.add_argument("--mode", required=True, choices=["baseline", "ext-only", "llm-only", "mixed"])+ parser.add_argument("--note", required=True)+ parser.add_argument(+ "--model-path", default=str(CURRENT_MODEL_PATH), help="model artifact to evaluate in baseline mode"+ )+ parser.add_argument(+ "--overall-threshold", type=float, default=4.0, help="LLM rows are positive when overall >= this"+ )+ parser.add_argument(+ "--ext-weight", type=float, default=1.0, help="per-row sample weight for extension rows in mixed mode"+ )+ parser.add_argument(+ "--ext-downsample",+ type=float,+ default=None,+ help="legacy behaviour: keep this fraction of extension rows instead of weighting",+ )+ parser.add_argument(+ "--add-curation",+ action="store_true",+ help="mix in the judgments_export curation/vote data as weighted pointwise rows",+ )+ parser.add_argument("--curation-weight", type=float, default=1.0, help="multiplier on curation row weights")+ parser.add_argument(+ "--weak-neg-weight", type=float, default=0.25, help="weight of out-ranked (add/move/approve) pair negatives"+ )+ parser.add_argument("--scale-pos-weight", type=float, default=0.1)+ parser.add_argument("--reg-lambda", type=float, default=2.0)+ parser.add_argument("--num-rounds", type=int, default=100)+ parser.add_argument("--seed", type=int, default=0, help="downsampling seed (--ext-downsample only)")+ parser.add_argument(+ "--split-seed", type=int, default=42, help="train/test query split seed; keep fixed across compared runs"+ )+ parser.add_argument("--test-size", type=float, default=0.3)+ parser.add_argument("--save-model", default=None, help="optional path to save the trained model")args = parser.parse_args()ext, llm = load_datasets()train_queries, test_queries = split_llm_queries(llm, args.test_size, args.split_seed)- test_df = llm[llm['qnorm'].isin(test_queries)]- human_test_pairs, human_test_queries = split_human_pairs(- load_human_pairs(), args.test_size, args.split_seed)- print(f"LLM split: {len(train_queries)} train / {len(test_queries)} test queries "- f"({len(test_df)} test rows); {len(human_test_pairs)} held-out human pairs "- f"({len(human_test_queries)} queries)")-- if args.mode == 'baseline':+ test_df = llm[llm["qnorm"].isin(test_queries)]+ human_test_pairs, human_test_queries = split_human_pairs(load_human_pairs(), args.test_size, args.split_seed)+ print(+ f"LLM split: {len(train_queries)} train / {len(test_queries)} test queries "+ f"({len(test_df)} test rows); {len(human_test_pairs)} held-out human pairs "+ f"({len(human_test_queries)} queries)"+ )++ if args.mode == "baseline":print(f"Loading model from {args.model_path}")model = RustXGBPipeline.from_model_path(args.model_path)else:curation = load_curation_frame(args.weak_neg_weight) if args.add_curation else Nonetrain_df = build_training_frame(- args.mode, ext, llm, train_queries - human_test_queries,+ args.mode,+ ext,+ llm,+ train_queries - human_test_queries,test_queries | human_test_queries,- args.overall_threshold, args.ext_weight, args.ext_downsample, args.seed,- curation=curation, curation_weight=args.curation_weight,+ args.overall_threshold,+ args.ext_weight,+ args.ext_downsample,+ args.seed,+ curation=curation,+ curation_weight=args.curation_weight,+ )+ print(+ f"Training frame: {len(train_df)} rows, "+ f"positive rate {train_df['label'].mean():.3f}, "+ f"weighted positive rate {np.average(train_df['label'], weights=train_df['weight']):.3f}")- print(f"Training frame: {len(train_df)} rows, "- f"positive rate {train_df['label'].mean():.3f}, "- f"weighted positive rate {np.average(train_df['label'], weights=train_df['weight']):.3f}")model = train(train_df, args.scale_pos_weight, args.reg_lambda, args.num_rounds)if args.save_model:print(f"Saving model to {args.save_model}")@@ -341,9 +374,8 @@ def run():print(f"ndcg@10: {metrics['ndcg@10']:.4f}")print(f"p@5: {metrics['p@5']:.4f}")print(f"p@10: {metrics['p@10']:.4f}")- print(f"pair-acc: {accuracy:.3f} on {len(human_test_pairs)} held-out human pairs "- f"(ties {ties:.1%})")+ print(f"pair-acc: {accuracy:.3f} on {len(human_test_pairs)} held-out human pairs (ties {ties:.1%})")-if __name__ == '__main__':+if __name__ == "__main__":run()diff --git a/mwmbl/rankeval/paths.py b/mwmbl/rankeval/paths.pyindex 8ac8873..5665f18 100644--- a/mwmbl/rankeval/paths.py+++ b/mwmbl/rankeval/paths.py@@ -2,17 +2,17 @@ import osfrom pathlib import PathROOT_DIR = Path(__file__).parent.parent.parent-DATA_DIR = ROOT_DIR / 'devdata' / 'rankeval-2026-04'+DATA_DIR = ROOT_DIR / "devdata" / "rankeval-2026-04"-REMOTE_DATA_DIR = DATA_DIR / 'remote-datasets'-RANKINGS_DATASET_TRAIN_PATH = REMOTE_DATA_DIR / 'rankings-train.csv'-RANKINGS_DATASET_TEST_PATH = REMOTE_DATA_DIR / 'rankings-test.csv'+REMOTE_DATA_DIR = DATA_DIR / "remote-datasets"+RANKINGS_DATASET_TRAIN_PATH = REMOTE_DATA_DIR / "rankings-train.csv"+RANKINGS_DATASET_TEST_PATH = REMOTE_DATA_DIR / "rankings-test.csv"-LEARNING_TO_RANK_DATASET_PATH = DATA_DIR / 'learning-to-rank.csv.gz'-LEARNING_TO_RANK_LLM_DATASET_PATH = DATA_DIR / 'learning-to-rank-llm.csv.gz'+LEARNING_TO_RANK_DATASET_PATH = DATA_DIR / "learning-to-rank.csv.gz"+LEARNING_TO_RANK_LLM_DATASET_PATH = DATA_DIR / "learning-to-rank-llm.csv.gz"# MODEL_PATH = ROOT_DIR / "mwmbl" / "resources" / 'model.pickle'-MODEL_PATH = DATA_DIR / 'model.pickle'+MODEL_PATH = DATA_DIR / "model.pickle"RUST_MODEL_PATH = DATA_DIR / "model.xgb"CURRENT_MODEL_PATH = DATA_DIR / "model-current.xgb"-URLS_PATH = Path(os.environ['HOME']) / 'data' / 'tinysearch' / 'urls.sqlite3'+URLS_PATH = Path(os.environ["HOME"]) / "data" / "tinysearch" / "urls.sqlite3"diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex db8b11d..a7736d7 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -8,7 +8,7 @@ from typing import Callablefrom redis import Redis-from mwmbl.crawler.domains import DomainLinkDatabase, TOP_DOMAINS+from mwmbl.crawler.domains import TOP_DOMAINS, DomainLinkDatabasefrom mwmbl.crawler.urls import FoundURLfrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.indexer.blacklist import get_default_blacklist_provider@@ -52,13 +52,14 @@ def get_domain_max_urls(domain: str, curated_domains: set[str]):class RedisURLQueue:- def __init__(self, redis: Redis, get_curated_domains_function: Callable[[], set[str]], blacklist_provider=None) -> None:+ def __init__(+ self, redis: Redis, get_curated_domains_function: Callable[[], set[str]], blacklist_provider=None+ ) -> None:self.redis = redisself.get_curated_domains_function = get_curated_domains_function# Curated domains override the blacklist, and the queue already knows where to get# them - over HTTP in the standalone crawler, which has no database.- self.blacklist_provider = blacklist_provider or get_default_blacklist_provider(- get_curated_domains_function)+ self.blacklist_provider = blacklist_provider or get_default_blacklist_provider(get_curated_domains_function)def queue_urls(self, found_urls: list[FoundURL]):curated_domains = self.get_curated_domains_function()@@ -67,20 +68,23 @@ class RedisURLQueue:domain_scores = {}with DomainLinkDatabase() as link_db:for url in found_urls:- time_since_crawled = (datetime.utcnow() - url.last_crawled- if url.last_crawled is not None else MAX_TIME_DELTA)+ time_since_crawled = (+ datetime.utcnow() - url.last_crawled if url.last_crawled is not None else MAX_TIME_DELTA+ )# Skip URLs crawled in the last monthif time_since_crawled < timedelta(days=30):continuedomain = parse_url(url.url).netloc- url_score = 1/len(url.url)+ url_score = 1 / len(url.url)# Discount URLs that were crawled recentlyscore_multiplier = 1 - math.exp(-time_since_crawled.total_seconds() / SCORE_TIME_CONSTANT)url_score *= score_multiplier- logger.info(f"URL score: {url_score}, score multiplier: {score_multiplier} for domain {domain} and age {time_since_crawled}")+ logger.info(+ f"URL score: {url_score}, score multiplier: {score_multiplier} for domain {domain} and age {time_since_crawled}"+ )url_scores[domain].append((url.url, url_score))domain_score = link_db.get_domain_score(domain) + url_score@@ -131,7 +135,8 @@ class RedisURLQueue:# Update the domain score if we removed a URLnew_domain_scores = self.redis.zrangebyscore(- DOMAIN_URLS_KEY.format(domain=domain), "-inf", "+inf", start=0, num=1, withscores=True)+ DOMAIN_URLS_KEY.format(domain=domain), "-inf", "+inf", start=0, num=1, withscores=True+ )if new_domain_scores:new_domain_score = new_domain_scores[0][1]self.redis.zadd(DOMAIN_SCORE_KEY, {domain: new_domain_score}, gt=True)diff --git a/mwmbl/retry.py b/mwmbl/retry.pyindex fd7cbb1..74e2b4c 100644--- a/mwmbl/retry.py+++ b/mwmbl/retry.py@@ -2,7 +2,6 @@ import requestsfrom requests.adapters import HTTPAdapterfrom urllib3.util.retry import Retry-MAX_RETRY = 2MAX_RETRY_FOR_SESSION = 2BACK_OFF_FACTOR = 0.3@@ -13,5 +12,5 @@ ERROR_CODES = (500, 502, 504)retry_requests = requests.Session()retry = Retry(total=5, backoff_factor=1)adapter = HTTPAdapter(max_retries=retry)-retry_requests.mount('http://', adapter)-retry_requests.mount('https://', adapter)+retry_requests.mount("http://", adapter)+retry_requests.mount("https://", adapter)diff --git a/mwmbl/search_auth.py b/mwmbl/search_auth.pyindex 2db3443..a1e06a6 100644--- a/mwmbl/search_auth.py+++ b/mwmbl/search_auth.py@@ -1,6 +1,7 @@"""Authentication for the search API using the X-API-Key header."""+import hashlibfrom asgiref.sync import sync_to_async@@ -76,7 +77,7 @@ async def authenticate_user(request) -> MwmblUser:from ninja_jwt.authentication import AsyncJWTAuthfrom ninja_jwt.exceptions import AuthenticationFailed, TokenError- token = auth_header[len("Bearer "):]+ token = auth_header[len("Bearer ") :]try:return await AsyncJWTAuth().async_jwt_authenticate(request, token)except (TokenError, AuthenticationFailed):diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex eebfd11..9b977dc 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -9,11 +9,10 @@ from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklistfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completer-from mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.tinysearchengine.indexer import Document, TinyIndexfrom mwmbl.tinysearchengine.ltr import RustXGBPipelinefrom mwmbl.tinysearchengine.ltr_rank import LTRRankerfrom mwmbl.tinysearchengine.mmr_rank import MMRRanker-from mwmbl.tinysearchengine.rank import HeuristicAndWikiRankerredis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)diff --git a/mwmbl/settings.py b/mwmbl/settings.pyindex fd777c8..76522ed 100644--- a/mwmbl/settings.py+++ b/mwmbl/settings.py@@ -2,26 +2,25 @@ import osimport refrom pathlib import Path--DATA_DIR = Path(os.environ['HOME']) / 'data' / 'tinysearch'-ALL_DOMAINS_PATH = DATA_DIR / 'hn-top-domains.csv'-TOP_DOMAINS_PATH = '../hn_top_domains_filtered.py'+DATA_DIR = Path(os.environ["HOME"]) / "data" / "tinysearch"+ALL_DOMAINS_PATH = DATA_DIR / "hn-top-domains.csv"+TOP_DOMAINS_PATH = "../hn_top_domains_filtered.py"MIN_COUNT = 10PROBABILITY_THRESHOLD = 0.8DATABASE_URL = os.environ.get("DATABASE_URL", "postgres://username:password@localhost/dbname")-APPLICATION_KEY = os.environ.get('MWMBL_APPLICATION_KEY', 'SECRETAPPLICATIONKEY')-KEY_ID = os.environ.get('MWMBL_KEY_ID', 'SECRETKEYID')-ENDPOINT_URL = os.environ.get('MWMBL_ENDPOINT_URL', 'https://s3.eu-central-003.backblazeb2.com')-BUCKET_NAME = os.environ.get('MWMBL_BUCKET_NAME', 'mwmbl-eu-crawl')+APPLICATION_KEY = os.environ.get("MWMBL_APPLICATION_KEY", "SECRETAPPLICATIONKEY")+KEY_ID = os.environ.get("MWMBL_KEY_ID", "SECRETKEYID")+ENDPOINT_URL = os.environ.get("MWMBL_ENDPOINT_URL", "https://s3.eu-central-003.backblazeb2.com")+BUCKET_NAME = os.environ.get("MWMBL_BUCKET_NAME", "mwmbl-eu-crawl")MAX_BATCH_SIZE = 100USER_ID_LENGTH = 36PUBLIC_USER_ID_LENGTH = 64-VERSION = 'v1'-DATE_REGEX = re.compile(r'\d{4}-\d{2}-\d{2}')-PUBLIC_URL_PREFIX = f'https://f003.backblazeb2.com/file/{BUCKET_NAME}/'-FILE_NAME_SUFFIX = '.json.gz'+VERSION = "v1"+DATE_REGEX = re.compile(r"\d{4}-\d{2}-\d{2}")+PUBLIC_URL_PREFIX = f"https://f003.backblazeb2.com/file/{BUCKET_NAME}/"+FILE_NAME_SUFFIX = ".json.gz"NUM_TITLE_CHARS = 65NUM_EXTRACT_CHARS = 155@@ -32,9 +31,16 @@ SCORE_FOR_SAME_DOMAIN = 0.01EXTRA_LINK_MULTIPLIER = 0.001UNKNOWN_DOMAIN_MULTIPLIER = 0.001EXCLUDED_DOMAINS = {- 'web.archive.org', 'forums.giantitp.com', 'www.crutchfield.com', 'plus.google.com', 'www.lukas-renggli.ch',+ "web.archive.org",+ "forums.giantitp.com",+ "www.crutchfield.com",+ "plus.google.com",+ "www.lukas-renggli.ch",# Adult content sites found ranking for suspicious Search Console queries (2026-08-14).- 'fineartteens.com', 'milforia.com', 'azgals.com', 'kusowanka.com',+ "fineartteens.com",+ "milforia.com",+ "azgals.com",+ "kusowanka.com",}# `booru.org` is a free hosting platform for imageboards; every subdomain we've found ranking# (nyou/hypno/inflate/futaba/captions.booru.org) hosts adult/fetish content, so block the whole domain.@@ -43,13 +49,13 @@ DOMAIN_BLACKLIST_REGEX = re.compile(r"jksu\.org|lwhyl\.org$|rgcd\.cn$|hzqwyou\.cn$|omgoat\.org$|pussyboy\.net$")CORE_DOMAINS = {- 'github.com',- 'en.wikipedia.org',- 'stackoverflow.com',- 'docs.google.com',- 'programmers.stackexchange.com',- 'developer.mozilla.org',- 'arxiv.org',- 'www.python.org',- 'news.ycombinator.com',+ "github.com",+ "en.wikipedia.org",+ "stackoverflow.com",+ "docs.google.com",+ "programmers.stackexchange.com",+ "developer.mozilla.org",+ "arxiv.org",+ "www.python.org",+ "news.ycombinator.com",}diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 04cbd5d..bb868a3 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -9,6 +9,7 @@ https://docs.djangoproject.com/en/4.2/topics/settings/For the full list of settings and their values, seehttps://docs.djangoproject.com/en/4.2/ref/settings/"""+import osfrom pathlib import Pathfrom urllib.parse import urlparse@@ -29,58 +30,57 @@ BASE_DIR = Path(__file__).resolve().parent.parent# Application definitionINSTALLED_APPS = [- 'django.contrib.admin',- 'django.contrib.auth',- 'django.contrib.contenttypes',- 'django.contrib.sessions',- 'django.contrib.messages',- 'django.contrib.staticfiles',- 'django.contrib.humanize',- 'django.contrib.postgres',- 'mwmbl',- 'django_htmx',- 'django_vite',- 'allauth',- 'allauth.account',- 'allauth.socialaccount',- 'ninja_extra',- 'debug_toolbar',- 'background_task',+ "django.contrib.admin",+ "django.contrib.auth",+ "django.contrib.contenttypes",+ "django.contrib.sessions",+ "django.contrib.messages",+ "django.contrib.staticfiles",+ "django.contrib.humanize",+ "django.contrib.postgres",+ "mwmbl",+ "django_htmx",+ "django_vite",+ "allauth",+ "allauth.account",+ "allauth.socialaccount",+ "ninja_extra",+ "debug_toolbar",+ "background_task",]MIDDLEWARE = ["debug_toolbar.middleware.DebugToolbarMiddleware",- 'django.middleware.security.SecurityMiddleware',- 'django.contrib.sessions.middleware.SessionMiddleware',- 'django.middleware.common.CommonMiddleware',- 'django.middleware.csrf.CsrfViewMiddleware',- 'django.contrib.auth.middleware.AuthenticationMiddleware',- 'django.contrib.messages.middleware.MessageMiddleware',- 'django.middleware.clickjacking.XFrameOptionsMiddleware',-+ "django.middleware.security.SecurityMiddleware",+ "django.contrib.sessions.middleware.SessionMiddleware",+ "django.middleware.common.CommonMiddleware",+ "django.middleware.csrf.CsrfViewMiddleware",+ "django.contrib.auth.middleware.AuthenticationMiddleware",+ "django.contrib.messages.middleware.MessageMiddleware",+ "django.middleware.clickjacking.XFrameOptionsMiddleware","django_htmx.middleware.HtmxMiddleware","allauth.account.middleware.AccountMiddleware",]-ROOT_URLCONF = 'mwmbl.urls'+ROOT_URLCONF = "mwmbl.urls"TEMPLATES = [{- 'BACKEND': 'django.template.backends.django.DjangoTemplates',- 'DIRS': [],- 'APP_DIRS': True,- 'OPTIONS': {- 'context_processors': [- 'django.template.context_processors.debug',- 'django.template.context_processors.request',- 'django.contrib.auth.context_processors.auth',- 'django.contrib.messages.context_processors.messages',+ "BACKEND": "django.template.backends.django.DjangoTemplates",+ "DIRS": [],+ "APP_DIRS": True,+ "OPTIONS": {+ "context_processors": [+ "django.template.context_processors.debug",+ "django.template.context_processors.request",+ "django.contrib.auth.context_processors.auth",+ "django.contrib.messages.context_processors.messages",],},},]-WSGI_APPLICATION = 'mwmbl.wsgi.application'+WSGI_APPLICATION = "mwmbl.wsgi.application"# Password validation@@ -88,16 +88,16 @@ WSGI_APPLICATION = 'mwmbl.wsgi.application'AUTH_PASSWORD_VALIDATORS = [{- 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',+ "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",},{- 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',+ "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",},{- 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',+ "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",},{- 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',+ "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",},]@@ -105,9 +105,9 @@ AUTH_PASSWORD_VALIDATORS = [# Internationalization# https://docs.djangoproject.com/en/4.2/topics/i18n/-LANGUAGE_CODE = 'en-us'+LANGUAGE_CODE = "en-us"-TIME_ZONE = 'UTC'+TIME_ZONE = "UTC"USE_I18N = True@@ -117,7 +117,7 @@ USE_TZ = True# Static files (CSS, JavaScript, Images)# https://docs.djangoproject.com/en/4.2/howto/static-files/-STATIC_URL = 'static/'+STATIC_URL = "static/"DJANGO_VITE_DEV_MODE = False@@ -125,17 +125,15 @@ DJANGO_VITE_DEV_MODE = False# Default primary key field type# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field-DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"AUTHENTICATION_BACKENDS = [# Accepts username or email in the username field — used by the JWT token endpoint- 'mwmbl.auth.UsernameOrEmailBackend',-+ "mwmbl.auth.UsernameOrEmailBackend",# Needed for Django's permission system (has_perm, has_module_perms) used by the admin- 'django.contrib.auth.backends.ModelBackend',-+ "django.contrib.auth.backends.ModelBackend",# allauth backend for allauth's own views (social auth, email confirmation, etc.)- 'allauth.account.auth_backends.AuthenticationBackend',+ "allauth.account.auth_backends.AuthenticationBackend",]@@ -187,11 +185,9 @@ FOOTER_LINKS = ["icon": "ph-discord-logo-bold","href": "https://discord.gg/2BGSUYFdkD",},--]-BATCH_DIR_NAME = 'batches'+BATCH_DIR_NAME = "batches"DATA_UPLOAD_MAX_NUMBER_FIELDS = None@@ -240,6 +236,7 @@ def strip_query_string(event):del event["request"]["query_string"]return event+# Django ninja-jwt settingsNINJA_JWT = {"USER_AUTHENTICATION_RULE": require_email_confirmation,@@ -249,9 +246,9 @@ NINJA_JWT = {# Apply PostgreSQL's default: if no database name is given in the URL, fall back to# the username (matching libpq's behaviour so that postgres://user@ works)._db = dj_database_url.config()-if _db and not _db.get('NAME'):- _db['NAME'] = _db.get('USER', '')-DATABASES = {'default': _db}+if _db and not _db.get("NAME"):+ _db["NAME"] = _db.get("USER", "")+DATABASES = {"default": _db}# Redis configurationREDIS_URL = os.environ.get("REDIS_URL", "redis://127.0.0.1:6379")@@ -274,7 +271,7 @@ INTERNAL_IPS = ["127.0.0.1",]-RUST_MODEL_PATH = Path(__file__).parent / 'resources' / "model.xgb"+RUST_MODEL_PATH = Path(__file__).parent / "resources" / "model.xgb"POLAR_ACCESS_TOKEN = os.environ.get("POLAR_ACCESS_TOKEN", "")POLAR_WEBHOOK_SECRET = os.environ.get("POLAR_WEBHOOK_SECRET", "")@@ -290,48 +287,48 @@ CURRENT_AGREEMENT_VERSIONS = {# Super SearchSUPER_SEARCH_MONTHLY_LIMIT = 100-SUPER_SEARCH_TOP_K = 10 # promote sources in top-K seen so far for crawling+SUPER_SEARCH_TOP_K = 10 # promote sources in top-K seen so far for crawlingSUPER_SEARCH_DEADLINE_SECONDS = 10.0SUPER_SEARCH_PER_SOURCE_TIMEOUT = 2.0SUPER_SEARCH_MAX_CONCURRENT_PER_HOST = 4SUPER_SEARCH_MAX_LINKS_PER_PAGE = 3SUPER_SEARCH_RESULTS_PER_SOURCE = 10SUPER_SEARCH_FINAL_RESULTS_LIMIT = 100-SUPER_SEARCH_CRAWL_WORKERS = 8 # threads in the dedicated crawl pool+SUPER_SEARCH_CRAWL_WORKERS = 8 # threads in the dedicated crawl pool# Fine-tuned relevance judge (dir with model.onnx + tokenizer.json; see# devdata/judge_train/RESULTS.md). If the artifact is missing, Super Search# falls back to LTR final ranking and top-K-survival bandit rewards.SUPER_SEARCH_JUDGE_MODEL_DIR = os.environ.get("SUPER_SEARCH_JUDGE_MODEL_DIR",- str(Path(__file__).parent.parent / "devdata" / "judge_train" / "models"- / "minilm-both-v1" / "onnx"))+ str(Path(__file__).parent.parent / "devdata" / "judge_train" / "models" / "minilm-both-v1" / "onnx"),+)# Super Search source selection (xgb contextual bandit over cosine-profile features)-SUPER_SEARCH_SOURCES_TO_QUERY = 10 # max sources queried per search-SUPER_SEARCH_PROJECTION_DIM = 64 # feature-hashing / random-projection dimension-SUPER_SEARCH_PROFILE_DECAY = 0.1 # decaying-mean weight for per-site content profiles-SUPER_SEARCH_QVEC_CACHE_TTL = 3600 # seconds to cache a query's projected vectors-SUPER_SEARCH_FORCE_INCLUDE = [] # source names always queried (high-value sources), like always_on+SUPER_SEARCH_SOURCES_TO_QUERY = 10 # max sources queried per search+SUPER_SEARCH_PROJECTION_DIM = 64 # feature-hashing / random-projection dimension+SUPER_SEARCH_PROFILE_DECAY = 0.1 # decaying-mean weight for per-site content profiles+SUPER_SEARCH_QVEC_CACHE_TTL = 3600 # seconds to cache a query's projected vectors+SUPER_SEARCH_FORCE_INCLUDE = [] # source names always queried (high-value sources), like always_on# XGBoost contextual-bandit source model. The runtime dir receives online# retrains (background task); before the first retrain, serving uses the# repo-bundled warm-start artifact in super_search_select/artifacts/xgb.SUPER_SEARCH_XGB_MODEL_DIR = os.environ.get(- "SUPER_SEARCH_XGB_MODEL_DIR",- str(Path(__file__).parent.parent / "devdata" / "super_search_xgb"))-SUPER_SEARCH_XGB_EPSILON = 0.1 # epsilon-greedy exploration rate-SUPER_SEARCH_XGB_MIN_TRAIN_ROWS = 2000 # (source, reward) pairs required before an online retrain-SUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS = 90 # impression window for online retrains-SOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pages+ "SUPER_SEARCH_XGB_MODEL_DIR", str(Path(__file__).parent.parent / "devdata" / "super_search_xgb")+)+SUPER_SEARCH_XGB_EPSILON = 0.1 # epsilon-greedy exploration rate+SUPER_SEARCH_XGB_MIN_TRAIN_ROWS = 2000 # (source, reward) pairs required before an online retrain+SUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS = 90 # impression window for online retrains+SOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pages# Blacklisted-domain filtering on the search path. Set BLACKLIST_FILTER_AT_RETRIEVAL to# False to turn retrieval-time filtering off without a rollback; the index-time filter in# index_documents() is unaffected.BLACKLIST_FILTER_AT_RETRIEVAL = os.environ.get("BLACKLIST_FILTER_AT_RETRIEVAL", "true").lower() != "false"-BLACKLIST_SNAPSHOT_CHECK_SECONDS = 300 # how often a worker checks Redis for a new snapshot-BLACKLIST_SNAPSHOT_REFRESH_SECONDS = 6 * 60 * 60 # how often the snapshot is rebuilt from the remote lists-BLACKLIST_PURGE_INTERVAL_SECONDS = 300 # how often the purge queue is drained-BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index per purge run+BLACKLIST_SNAPSHOT_CHECK_SECONDS = 300 # how often a worker checks Redis for a new snapshot+BLACKLIST_SNAPSHOT_REFRESH_SECONDS = 6 * 60 * 60 # how often the snapshot is rebuilt from the remote lists+BLACKLIST_PURGE_INTERVAL_SECONDS = 300 # how often the purge queue is drained+BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index per purge run# How far ahead approving a domain submission schedules a snapshot rebuild. Approvers work# through submissions in batches, so the delay collapses a batch into one rebuild rather# than one per approval - see mwmbl.signals.@@ -360,9 +357,9 @@ EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60# container filesystem is neither shared between workers nor kept across a deploy. This# directory holds the read-only warm start used until the first retrain publishes one.DOMAIN_MODERATION_MODEL_DIR = os.environ.get(- "DOMAIN_MODERATION_MODEL_DIR",- str(Path(__file__).parent / "moderation" / "artifacts"))-MODERATION_PAGES_PER_DOMAIN = 3 # homepage plus two, shown to the moderator and read by the model+ "DOMAIN_MODERATION_MODEL_DIR", str(Path(__file__).parent / "moderation" / "artifacts")+)+MODERATION_PAGES_PER_DOMAIN = 3 # homepage plus two, shown to the moderator and read by the model# Both measured on held-out decisions rather than picked as round numbers. At 0.75 a suggested# rejection is right 81% of the time overall and 88% for submitters with no track record; at# 0.25 a suggested approval is wrong 6% of the time overall. Between the two the suggestion is@@ -373,5 +370,5 @@ MODERATION_PAGES_PER_DOMAIN = 3 # homepage plus two, shown to the moderator# these two numbers, is where the asymmetry between approving and rejecting lives.MODERATION_REJECT_THRESHOLD = 0.75MODERATION_APPROVE_THRESHOLD = 0.25-MODERATION_EVIDENCE_MAX_AGE_DAYS = 30 # refetch rather than reuse evidence older than this-MODERATION_RETRAIN_INTERVAL_SECONDS = 30 * 24 * 60 * 60 # monthly; the gate guards each publish+MODERATION_EVIDENCE_MAX_AGE_DAYS = 30 # refetch rather than reuse evidence older than this+MODERATION_RETRAIN_INTERVAL_SECONDS = 30 * 24 * 60 * 60 # monthly; the gate guards each publishdiff --git a/mwmbl/settings_crawler.py b/mwmbl/settings_crawler.pyindex b9bd31c..df46595 100644--- a/mwmbl/settings_crawler.py+++ b/mwmbl/settings_crawler.py@@ -2,9 +2,8 @@ import osfrom mwmbl.settings_dev import *-DATA_PATH = f"{os.environ['HOME']}/.mwmbl"-INDEX_NAME = 'crawl-index.tinysearch'+INDEX_NAME = "crawl-index.tinysearch"# Index of around 400Mb = 4096b * 100_000NUM_PAGES = 100_000diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex a598abd..37af3a9 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -1,8 +1,7 @@from mwmbl.settings_common import *-# SECURITY WARNING: keep the secret key used in production secret!-SECRET_KEY = 'django-insecure-qqr#f(i3uf%m8%8u35vn=ov-uk(*8!a&1t-hxa%ev2^t1%j&sm'+SECRET_KEY = "django-insecure-qqr#f(i3uf%m8%8u35vn=ov-uk(*8!a&1t-hxa%ev2^t1%j&sm"STATIC_ROOT = ""@@ -15,14 +14,14 @@ STATICFILES_DIRS = [str(DJANGO_VITE_ASSETS_PATH)]DEBUG = TrueALLOWED_HOSTS = ["localhost", "127.0.0.1"]-EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'+EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"DATA_PATH = "./devdata"-INDEX_NAME = 'index-v2.tinysearch'+INDEX_NAME = "index-v2.tinysearch"NUM_PAGES = 2560-EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'+EXTERNAL_CACHE_INDEX_NAME = "external-cache.tinysearch"EXTERNAL_CACHE_NUM_PAGES = 2560URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex 1cb1f14..9fe9df0 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -2,7 +2,6 @@ import osfrom mwmbl.settings_common import *-SECRET_KEY = os.environ["DJANGO_SECRET_KEY"]@@ -18,20 +17,20 @@ CSRF_TRUSTED_ORIGINS = [f"https://{domain}" for domain in ALLOWED_HOSTS]# Sendgrid email settings-EMAIL_HOST = 'smtp.sendgrid.net'-EMAIL_HOST_USER = 'apikey'-EMAIL_HOST_PASSWORD = os.getenv('EMAIL_HOST_PASSWORD')+EMAIL_HOST = "smtp.sendgrid.net"+EMAIL_HOST_USER = "apikey"+EMAIL_HOST_PASSWORD = os.getenv("EMAIL_HOST_PASSWORD")EMAIL_PORT = 587EMAIL_USE_TLS = TrueDATA_PATH = "/app/storage"-INDEX_NAME = 'index-v2-400G.tinysearch'+INDEX_NAME = "index-v2-400G.tinysearch"# 400GB indexNUM_PAGES = 102400000-EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'+EXTERNAL_CACHE_INDEX_NAME = "external-cache.tinysearch"# ~15 GB. Sized by page occupancy rather than by naive capacity, because pages are assigned# by hash: the queries per page are Poisson-distributed, not evenly spread, so the pages# that run over start truncating while empty ones sit unused.diff --git a/mwmbl/signals.py b/mwmbl/signals.pyindex 0b58ce0..afa27ca 100644--- a/mwmbl/signals.py+++ b/mwmbl/signals.py@@ -11,6 +11,7 @@ Both of them are about a domain submission reaching work that happens elsewhere:queue the domain has already been fetched and scored. Doing that work on their requestinstead would put three page fetches in front of every row."""+from datetime import timedeltafrom logging import getLoggerdiff --git a/mwmbl/templatetags/humanbytes.py b/mwmbl/templatetags/humanbytes.pyindex 0044c9e..2529e4c 100644--- a/mwmbl/templatetags/humanbytes.py+++ b/mwmbl/templatetags/humanbytes.py@@ -1,14 +1,15 @@"""https://stackoverflow.com/a/63839503/660902"""+from django.template import Libraryclass HumanBytes:METRIC_LABELS = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]BINARY_LABELS = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"]- PRECISION_OFFSETS = [0.5, 0.05, 0.005, 0.0005] # PREDEFINED FOR SPEED.- PRECISION_FORMATS = ["{}{:.0f} {}", "{}{:.1f} {}", "{}{:.2f} {}", "{}{:.3f} {}"] # PREDEFINED FOR SPEED.+ PRECISION_OFFSETS = [0.5, 0.05, 0.005, 0.0005] # PREDEFINED FOR SPEED.+ PRECISION_FORMATS = ["{}{:.0f} {}", "{}{:.1f} {}", "{}{:.2f} {}", "{}{:.3f} {}"] # PREDEFINED FOR SPEED.@staticmethoddef format(num: int | float, metric: bool = False, precision: int = 1) -> str:@@ -27,7 +28,7 @@ class HumanBytes:unit_step_thresh = unit_step - HumanBytes.PRECISION_OFFSETS[precision]is_negative = num < 0- if is_negative: # Faster than ternary assignment or always running abs().+ if is_negative: # Faster than ternary assignment or always running abs().num = abs(num)for unit in unit_labels:diff --git a/mwmbl/templatetags/result_filters.py b/mwmbl/templatetags/result_filters.pyindex f26fc1b..2f46fd4 100644--- a/mwmbl/templatetags/result_filters.py+++ b/mwmbl/templatetags/result_filters.py@@ -4,7 +4,7 @@ from django.template import Libraryfrom django.utils.html import conditional_escapefrom django.utils.safestring import mark_safe-from mwmbl.format import get_query_regex, DOCUMENT_SOURCES, get_document_source+from mwmbl.format import get_document_source, get_query_regexfrom mwmbl.tinysearchengine.indexer import DocumentStatefrom mwmbl.tokenizer import tokenize@@ -22,8 +22,8 @@ def format_for_query(text: str | None, query: str, autoescape=True):formatted = []start = 0for match in matches:- formatted.append(escape(text[start:match.start()]))- formatted.append(f"<strong>{escape(text[match.start():match.end()])}</strong>")+ formatted.append(escape(text[start : match.start()]))+ formatted.append(f"<strong>{escape(text[match.start() : match.end()])}</strong>")start = match.end()formatted.append(escape(text[start:]))return mark_safe("".join(formatted))diff --git a/mwmbl/tinysearchengine/completer.py b/mwmbl/tinysearchengine/completer.pyindex d18c35a..8ef28d6 100644--- a/mwmbl/tinysearchengine/completer.py+++ b/mwmbl/tinysearchengine/completer.py@@ -3,8 +3,7 @@ from pathlib import Pathimport pandas as pd--TERMS_PATH = Path(__file__).parent.parent / 'resources' / 'mwmbl-crawl-terms.csv'+TERMS_PATH = Path(__file__).parent.parent / "resources" / "mwmbl-crawl-terms.csv"class Completer:@@ -12,12 +11,12 @@ class Completer:# Load term dataterms = self.get_terms()- terms_dict = terms.sort_values('term').set_index('term')['count'].to_dict()+ terms_dict = terms.sort_values("term").set_index("term")["count"].to_dict()self.terms = list(terms_dict.keys())self.counts = list(terms_dict.values())self.num_matches = num_matchesprint("Terms", self.terms[:100], self.counts[:100])-+def get_terms(self):return pd.read_csv(TERMS_PATH)@@ -27,7 +26,7 @@ class Completer:end_index = bisect_right(self.terms, term, key=lambda x: x[:term_length])matching_terms = zip(self.counts[start_index:end_index], self.terms[start_index:end_index])- top_terms = sorted(matching_terms, reverse=True)[:self.num_matches]+ top_terms = sorted(matching_terms, reverse=True)[: self.num_matches]if not top_terms:return []diff --git a/mwmbl/tinysearchengine/copy_index.py b/mwmbl/tinysearchengine/copy_index.pyindex 04f9ed3..040297c 100644--- a/mwmbl/tinysearchengine/copy_index.py+++ b/mwmbl/tinysearchengine/copy_index.py@@ -1,10 +1,11 @@"""Copy an old index into a new one"""+from collections import defaultdictfrom logging import getLogger-from mwmbl.indexer.index_batches import index_pages, get_url_score+from mwmbl.indexer.index_batches import get_url_score, index_pagesfrom mwmbl.tinysearchengine.indexer import Document, PageError, TinyIndexfrom mwmbl.utils import add_term_infos@@ -16,7 +17,7 @@ def copy_pages(old_index_path: str, new_index_path: str, start_page: int, num_pa# Get all old indexes:page_documents = defaultdict(list)- with (TinyIndex(item_factory=Document, index_path=new_index_path) as new_index):+ with TinyIndex(item_factory=Document, index_path=new_index_path) as new_index:with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:logger.info(f"Old index has {old_index.num_pages} pages")@@ -33,14 +34,17 @@ def copy_pages(old_index_path: str, new_index_path: str, start_page: int, num_palogger.warning("Skipping unreadable page %d of %s", page_index, old_index_path)continuedocuments_with_terms = add_term_infos(documents, old_index, page_index)- documents_with_scores = [Document(- document.title,- document.url,- document.extract,- get_url_score(document.url),- document.term,- state=document.state- ) for document in documents_with_terms]+ documents_with_scores = [+ Document(+ document.title,+ document.url,+ document.extract,+ get_url_score(document.url),+ document.term,+ state=document.state,+ )+ for document in documents_with_terms+ ]for document in documents_with_scores:new_page = new_index.get_key_page_index(document.term)page_documents[new_page].append(document)diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex a4b089c..1c25ea7 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -4,19 +4,19 @@ import jsonimport osimport structfrom contextlib import contextmanager-from dataclasses import dataclass, asdict, field+from dataclasses import asdict, dataclass, fieldfrom enum import IntEnumfrom io import UnsupportedOperationfrom json import JSONDecodeErrorfrom logging import getLogger-from mmap import mmap, PROT_READ, PROT_WRITE-from typing import TypeVar, Generic, Callable, List, Optional+from mmap import PROT_READ, PROT_WRITE, mmap+from typing import Callable, Generic, List, Optional, TypeVarimport mmh3-from zstandard import ZstdDecompressor, ZstdCompressor, ZstdError+from zstandard import ZstdCompressor, ZstdDecompressor, ZstdErrorVERSION = 1-METADATA_CONSTANT = b'mwmbl-tiny-search'+METADATA_CONSTANT = b"mwmbl-tiny-search"METADATA_SIZE = 4096PAGE_SIZE = 4096@@ -48,22 +48,21 @@ _FLOCK_STRUCT = "hhqqi4x"# one-off failure to take it: an old kernel rejects the F_OFD_SETLKW command with EINVAL,# and a filesystem with no record-lock support answers ENOLCK/ENOSYS/ENOTSUP. Only these# latch locking off for the process - see _locked_page.-UNSUPPORTED_LOCK_ERRNOS = frozenset({errno.EINVAL, errno.ENOLCK, errno.ENOSYS,- errno.ENOTSUP, errno.EOPNOTSUPP})+UNSUPPORTED_LOCK_ERRNOS = frozenset({errno.EINVAL, errno.ENOLCK, errno.ENOSYS, errno.ENOTSUP, errno.EOPNOTSUPP})# Set False the first time locking turns out to be unsupported here - see _locked_page._page_locking_supported = Truedef _set_page_lock(fileno: int, lock_type: int, start: int, length: int):- fcntl.fcntl(fileno, F_OFD_SETLKW,- struct.pack(_FLOCK_STRUCT, lock_type, os.SEEK_SET, start, length, 0))+ fcntl.fcntl(fileno, F_OFD_SETLKW, struct.pack(_FLOCK_STRUCT, lock_type, os.SEEK_SET, start, length, 0))class DocumentState(IntEnum):"""The state of the document in the index. A value of None indicates an organic search result."""+SYNCED_WITH_MAIN_INDEX = -2DELETED = -1FROM_USER = 2@@ -87,6 +86,7 @@ class DocumentSource(IntEnum):number for a different provider, or a stale cache entry becomes a lie about where itsresults came from."""+WIKIPEDIA = 1# Reserved, not yet wired up to a provider. The number is here rather than added later# because it is the numbering registry: with only one member the source field cannot be@@ -107,22 +107,22 @@ class Document:source: Optional[int] = Nonedef __init__(- self,- title: str,- url: str,- extract: str,- score: Optional[float] = None,- term: Optional[str] = None,- state: Optional[int | DocumentState] = None,- user_ids: Optional[List[int]] = None,- last_crawled: Optional[int] = None,- source: Optional[int | DocumentSource] = None,+ self,+ title: str,+ url: str,+ extract: str,+ score: Optional[float] = None,+ term: Optional[str] = None,+ state: Optional[int | DocumentState] = None,+ user_ids: Optional[List[int]] = None,+ last_crawled: Optional[int] = None,+ source: Optional[int | DocumentSource] = None,):# Sometimes the title or extract may be None, probably because of user generated content# It's not allowed to be None though, or things will break- self.title = title if title is not None else ''+ self.title = title if title is not None else ""self.url = url- self.extract = extract if extract is not None else ''+ self.extract = extract if extract is not None else ""self.score = scoreself.term = termself.state = None if state is None else DocumentState(state)@@ -155,7 +155,7 @@ class TokenizedDocument(Document):tokens: List[str] = field(default_factory=list)-T = TypeVar('T')+T = TypeVar("T")class PageError(Exception):@@ -170,7 +170,7 @@ class TinyIndexMetadata:item_factory: strdef to_bytes(self) -> bytes:- metadata_bytes = METADATA_CONSTANT + json.dumps(asdict(self)).encode('utf8')+ metadata_bytes = METADATA_CONSTANT + json.dumps(asdict(self)).encode("utf8")assert len(metadata_bytes) <= METADATA_SIZEreturn metadata_bytes@@ -181,7 +181,7 @@ class TinyIndexMetadata:if metadata_constant != METADATA_CONSTANT:raise ValueError("This doesn't seem to be an index file")- values = json.loads(data[constant_length:].decode('utf8'))+ values = json.loads(data[constant_length:].decode("utf8"))return TinyIndexMetadata(**values)@@ -189,22 +189,22 @@ class TinyIndexMetadata:# We do this by leveraging binary search to quickly find the index where:# - index+1 cannot fit onto a page# - <=index can fit on a page-def _binary_search_fitting_size(compressor: ZstdCompressor, page_size: int, items:list[T], lo:int, hi:int):+def _binary_search_fitting_size(compressor: ZstdCompressor, page_size: int, items: list[T], lo: int, hi: int):# Base case: our binary search has gone too farif lo > hi:return -1, None# Check the midpoint to see if it will fit onto a page- mid = (lo+hi)//2- compressed_data = compressor.compress(json.dumps(items[:mid]).encode('utf8'))+ mid = (lo + hi) // 2+ compressed_data = compressor.compress(json.dumps(items[:mid]).encode("utf8"))size = len(compressed_data)if size > page_size:# We cannot fit this much data into a page# Reduce the hi boundary, and try again- return _binary_search_fitting_size(compressor, page_size, items, lo, mid-1)+ return _binary_search_fitting_size(compressor, page_size, items, lo, mid - 1)else:# We can fit this data into a page, but maybe we can fit more data# Try to see if we have a better match- potential_target, potential_data = _binary_search_fitting_size(compressor, page_size, items, mid+1, hi)+ potential_target, potential_data = _binary_search_fitting_size(compressor, page_size, items, mid + 1, hi)if potential_target != -1:# We found a larger index that can still fit onto a page, so use thatreturn potential_target, potential_data@@ -213,7 +213,7 @@ def _binary_search_fitting_size(compressor: ZstdCompressor, page_size: int, itemreturn mid, compressed_data-def _trim_items_to_page(compressor: ZstdCompressor, page_size: int, items:list[T]):+def _trim_items_to_page(compressor: ZstdCompressor, page_size: int, items: list[T]):# Find max number of items that fit on a pagereturn _binary_search_fitting_size(compressor, page_size, items, 0, len(items))@@ -223,7 +223,7 @@ def _get_page_data(page_size: int, items: list[T]):compressor = ZstdCompressor()num_fitting, serialised_data = _trim_items_to_page(compressor, page_size, items)- compressed_data = compressor.compress(json.dumps(items[:num_fitting]).encode('utf8'))+ compressed_data = compressor.compress(json.dumps(items[:num_fitting]).encode("utf8"))assert len(compressed_data) <= page_size, "The data shouldn't get bigger"return _pad_to_page_size(compressed_data, page_size), num_fitting@@ -232,7 +232,7 @@ def _pad_to_page_size(data: bytes, page_size: int):page_length = len(data)if page_length > page_size:raise PageError(f"Data is too big ({page_length}) for page size ({page_size})")- padding = b'\x00' * (page_size - page_length)+ padding = b"\x00" * (page_size - page_length)page_data = data + paddingreturn page_data@@ -240,7 +240,7 @@ def _pad_to_page_size(data: bytes, page_size: int):class _OpenPage(Generic[T]):"""One page of the index, open for a read-modify-write. See TinyIndex.page."""- def __init__(self, index: 'TinyIndex[T]', page_index: int, locked: bool):+ def __init__(self, index: "TinyIndex[T]", page_index: int, locked: bool):self._index = indexself._page_index = page_indexself.reset = False@@ -273,18 +273,20 @@ class _OpenPage(Generic[T]):class TinyIndex(Generic[T]):- def __init__(self, item_factory: Callable[..., T], index_path, mode='r'):- if mode not in {'r', 'w'}:+ def __init__(self, item_factory: Callable[..., T], index_path, mode="r"):+ if mode not in {"r", "w"}:raise ValueError(f"Mode should be one of 'r' or 'w', got {mode}")- with open(index_path, 'rb') as index_file:+ with open(index_path, "rb") as index_file:metadata_page = index_file.read(METADATA_SIZE)- metadata_bytes = metadata_page.rstrip(b'\x00')+ metadata_bytes = metadata_page.rstrip(b"\x00")metadata = TinyIndexMetadata.from_bytes(metadata_bytes)if metadata.item_factory != item_factory.__name__:- raise ValueError(f"Metadata item factory '{metadata.item_factory}' in the index "- f"does not match the passed item factory: '{item_factory.__name__}'")+ raise ValueError(+ f"Metadata item factory '{metadata.item_factory}' in the index "+ f"does not match the passed item factory: '{item_factory.__name__}'"+ )self.item_factory = item_factoryself.index_path = index_path@@ -297,8 +299,8 @@ class TinyIndex(Generic[T]):self.mmap = Nonedef __enter__(self):- self.index_file = open(self.index_path, 'r+b')- prot = PROT_READ if self.mode == 'r' else PROT_READ | PROT_WRITE+ self.index_file = open(self.index_path, "r+b")+ prot = PROT_READ if self.mode == "r" else PROT_READ | PROT_WRITEself.mmap = mmap(self.index_file.fileno(), 0, prot=prot)return self@@ -349,7 +351,7 @@ class TinyIndex(Generic[T]):anything else is treated as a one-off and locking is tried again next time."""global _page_locking_supported- if self.mode != 'w':+ if self.mode != "w":raise UnsupportedOperation("The file is open in read mode, you cannot lock a page")start = i * self.page_size + METADATA_SIZE@@ -365,10 +367,11 @@ class TinyIndex(Generic[T]):logger.warning("Index page locking is not available here (%s); index writes will ""race as they did before it was added. Concurrent writers to one "- "page can lose documents.", e)+ "page can lose documents.",+ e,+ )else:- logger.warning("Could not lock index page %d (%s); writing it unlocked",- i, e)+ logger.warning("Could not lock index page %d (%s); writing it unlocked", i, e)try:yield lockedfinally:@@ -398,7 +401,7 @@ class TinyIndex(Generic[T]):if len(item) <= STATE_INDEX:logger.error(f"Could not recover item in index page {i}, skipping. Item: {item}")continue- fixed_item = list(item[:STATE_INDEX]) + [None] + list(item[STATE_INDEX + 1:])+ fixed_item = list(item[:STATE_INDEX]) + [None] + list(item[STATE_INDEX + 1 :])try:items.append(self.item_factory(*fixed_item))except Exception as e2:@@ -414,11 +417,11 @@ class TinyIndex(Generic[T]):over everything that was on the page. Readers that would rather have no resultsthan an error catch this - see retrieve."""- page_data = self.mmap[i * self.page_size + METADATA_SIZE:(i + 1) * self.page_size + METADATA_SIZE]+ page_data = self.mmap[i * self.page_size + METADATA_SIZE : (i + 1) * self.page_size + METADATA_SIZE]decompressor = ZstdDecompressor()try:decompressed_data = decompressor.decompress(page_data)- return json.loads(decompressed_data.decode('utf8'))+ return json.loads(decompressed_data.decode("utf8"))except (ZstdError, UnicodeDecodeError, JSONDecodeError) as e:# Damage that gets past zstd's checksum lands on the decode or the parse# instead, and it is the same kind of unreadable. Callers catch PageError and@@ -441,12 +444,12 @@ class TinyIndex(Generic[T]):If the data is too big, it will store the first items in the list and discard therest; the number actually stored is returned."""- if self.mode != 'w':+ if self.mode != "w":raise UnsupportedOperation("The file is open in read mode, you cannot write")page_data, num_stored = _get_page_data(self.page_size, data)logger.debug(f"Got page data of length {len(page_data)}")- self.mmap[i * self.page_size + METADATA_SIZE:(i+1) * self.page_size + METADATA_SIZE] = page_data+ self.mmap[i * self.page_size + METADATA_SIZE : (i + 1) * self.page_size + METADATA_SIZE] = page_datareturn num_stored@contextmanager@@ -489,10 +492,9 @@ class TinyIndex(Generic[T]):page_bytes, _ = _get_page_data(page_size, [])- with open(index_path, 'wb') as index_file:+ with open(index_path, "wb") as index_file:index_file.write(metadata_padded)for i in range(num_pages):index_file.write(page_bytes)return TinyIndex(item_factory, index_path=index_path)-diff --git a/mwmbl/tinysearchengine/ltr.py b/mwmbl/tinysearchengine/ltr.pyindex 428f187..8d8b792 100644--- a/mwmbl/tinysearchengine/ltr.py+++ b/mwmbl/tinysearchengine/ltr.py@@ -8,14 +8,14 @@ Contains:- RustXGBPipeline: thin Python shim over the Rust mwmbl_rank.RustXGBPipeline,providing a sklearn-compatible interface (fit / predict / save_model / load_model)."""-from pathlib import Path+from typing import Any-import mwmbl_rankimport numpy as npfrom pandas import DataFrame, Seriesfrom sklearn.base import BaseEstimator, RegressorMixin, TransformerMixin+import mwmbl_rankfrom mwmbl.tinysearchengine.rank import get_features@@ -37,8 +37,8 @@ class ThresholdPredictor(BaseEstimator, RegressorMixin):def get_features_as_series(item: Series):- terms = item['query'].lower().split()- features = get_features(terms, item['title'], item['url'], item['extract'], item['score'], True)+ terms = item["query"].lower().split()+ features = get_features(terms, item["title"], item["url"], item["extract"], item["score"], True)return Series(features)@@ -47,19 +47,16 @@ class FeatureExtractor(BaseEstimator, TransformerMixin):return selfdef transform(self, X: DataFrame, y=None):- records = X.to_dict('records')+ records = X.to_dict("records")all_features = []for item in records:- terms = item['query'].lower().split()+ terms = item["query"].lower().split()- features = get_features(- terms, item['title'], item['url'],- item['extract'], item['score'], True- )+ features = get_features(terms, item["title"], item["url"], item["extract"], item["score"], True)all_features.append(features)- return DataFrame(all_features).values.astype('float32')+ return DataFrame(all_features).values.astype("float32")class RankingPredictor(BaseEstimator, RegressorMixin):@@ -140,14 +137,14 @@ class RustXGBPipeline(BaseEstimator, RegressorMixin):@staticmethoddef _df_to_records(X: DataFrame) -> list:"""Convert a DataFrame to a list of dicts for the Rust boundary."""- cols = ['query', 'url', 'title', 'extract', 'score']+ cols = ["query", "url", "title", "extract", "score"]subset = X[cols].copy()- subset['title'] = subset['title'].fillna('')- subset['extract'] = subset['extract'].fillna('')- subset['score'] = subset['score'].fillna(0.0)- return subset.to_dict('records')+ subset["title"] = subset["title"].fillna("")+ subset["extract"] = subset["extract"].fillna("")+ subset["score"] = subset["score"].fillna(0.0)+ return subset.to_dict("records")- def fit(self, X: DataFrame, y, sample_weight=None) -> 'RustXGBPipeline':+ def fit(self, X: DataFrame, y, sample_weight=None) -> "RustXGBPipeline":"""Train the XGBoost model.@@ -158,6 +155,7 @@ class RustXGBPipeline(BaseEstimator, RegressorMixin):sample_weight : optional array-like of per-row weights"""import time+t0 = time.time()print(f"[RustXGBPipeline.fit] Converting {len(X)} rows to records...", flush=True)records = self._df_to_records(X)@@ -167,10 +165,15 @@ class RustXGBPipeline(BaseEstimator, RegressorMixin):weights = None if sample_weight is None else list(np.asarray(sample_weight, dtype=np.float32))print(f"[RustXGBPipeline.fit] Labels ready in {time.time() - t1:.2f}s. Getting inner pipeline...", flush=True)t2 = time.time()- print(f"[RustXGBPipeline.fit] Inner pipeline ready in {time.time() - t2:.2f}s. Calling Rust fit()...", flush=True)+ print(+ f"[RustXGBPipeline.fit] Inner pipeline ready in {time.time() - t2:.2f}s. Calling Rust fit()...", flush=True+ )t3 = time.time()self._inner.fit(records, labels, weights)- print(f"[RustXGBPipeline.fit] Rust fit() completed in {time.time() - t3:.2f}s (total: {time.time() - t0:.2f}s).", flush=True)+ print(+ f"[RustXGBPipeline.fit] Rust fit() completed in {time.time() - t3:.2f}s (total: {time.time() - t0:.2f}s).",+ flush=True,+ )return selfdef predict(self, X: DataFrame | list[dict[str, Any]]) -> np.ndarray:@@ -195,7 +198,7 @@ class RustXGBPipeline(BaseEstimator, RegressorMixin):"""Save the trained model to disk (XGBoost binary format)."""self._inner.save_model(path)- def load_model(self, path: str) -> 'RustXGBPipeline':+ def load_model(self, path: str) -> "RustXGBPipeline":"""Load a model from disk (XGBoost binary format)."""self._inner.load_model(path)return self@@ -212,7 +215,7 @@ class RustXGBPipeline(BaseEstimator, RegressorMixin):min_child_weight: float | None = None,gamma: float | None = None,subsample: float | None = None,- ) -> 'RustXGBPipeline':+ ) -> "RustXGBPipeline":"""Load a pre-trained model from disk and return a ready-to-predict pipeline."""pipeline = cls(threshold=threshold,diff --git a/mwmbl/tinysearchengine/ltr_rank.py b/mwmbl/tinysearchengine/ltr_rank.pyindex 49acbc3..a6f9cf8 100644--- a/mwmbl/tinysearchengine/ltr_rank.py+++ b/mwmbl/tinysearchengine/ltr_rank.py@@ -4,9 +4,8 @@ LTR (Learning-to-Rank) ranker that uses the Rust XGBoost pipeline for scoring.LTRRanker accepts any model with a sklearn-compatible predict(DataFrame) interface,including both the Python sklearn pipeline and the Rust RustXGBPipeline."""+import numpy as np-from pandas import DataFrame-from sklearn.base import BaseEstimatorfrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex@@ -59,15 +58,18 @@ class LTRRanker(Ranker):if len(results) == 0:return []- query = ' '.join(terms)+ query = " ".join(terms)- data = [{- 'query': query,- 'url': page.url,- 'title': page.title if page.title is not None else "",- 'extract': page.extract if page.extract is not None else "",- 'score': page.score if page.score is not None else 0.0,- } for page in results]+ data = [+ {+ "query": query,+ "url": page.url,+ "title": page.title if page.title is not None else "",+ "extract": page.extract if page.extract is not None else "",+ "score": page.score if page.score is not None else 0.0,+ }+ for page in results+ ]predictions = self.model.predict(data)mask = predictions > 0.0@@ -93,12 +95,15 @@ def score_documents(model, query: str, documents: list[Document]) -> list[float]"""if not documents:return []- data = [{- 'query': query,- 'url': page.url,- 'title': page.title if page.title is not None else "",- 'extract': page.extract if page.extract is not None else "",- 'score': page.score if page.score is not None else 0.0,- } for page in documents]+ data = [+ {+ "query": query,+ "url": page.url,+ "title": page.title if page.title is not None else "",+ "extract": page.extract if page.extract is not None else "",+ "score": page.score if page.score is not None else 0.0,+ }+ for page in documents+ ]predictions = model.predict(data)return [float(p) for p in predictions]diff --git a/mwmbl/tinysearchengine/mmr_rank.py b/mwmbl/tinysearchengine/mmr_rank.pyindex 6ad6b79..0ccdcf3 100644--- a/mwmbl/tinysearchengine/mmr_rank.py+++ b/mwmbl/tinysearchengine/mmr_rank.py@@ -7,6 +7,7 @@ search() output so near-duplicate / same-domain results are demoted rather thanIt can wrap any ranker (LTRRanker, HeuristicRanker, ...), which also makes it easy toevaluate a ranker with and without diversity."""+import mathfrom collections import Counterfrom urllib.parse import urlparse@@ -15,7 +16,6 @@ from mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.rank import Rankerfrom mwmbl.tokenizer import tokenize-# MMR tuning parameters.MMR_LAMBDA = 0.7 # weight on relevance vs. diversity (1.0 = pure relevance, 0.0 = max diversity)DOMAIN_SIMILARITY_WEIGHT = 0.8 # within the kernel: weight on same-domain vs. text overlap@@ -89,8 +89,7 @@ class MMRRanker:def __init__(self, ranker: Ranker):self.ranker = ranker- def search(self, s: str, additional_results: list[Document],- use_external_search: bool = True) -> list[Document]:+ def search(self, s: str, additional_results: list[Document], use_external_search: bool = True) -> list[Document]:return mmr_rerank(self.ranker.search(s, additional_results, use_external_search))def complete(self, q: str):diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex f6eedd1..734ae40 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -21,14 +21,13 @@ from requests.exceptions import RetryErrorfrom mwmbl.format import get_query_regexfrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist-from mwmbl.indexer.purge_queue import enqueue_for_purgefrom mwmbl.indexer.external_cache import get_cached_external_results, store_external_results+from mwmbl.indexer.purge_queue import enqueue_for_purgefrom mwmbl.tinysearchengine.completer import Completer-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentSource, DocumentState-from mwmbl.tokenizer import tokenize, get_bigrams+from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentState, TinyIndex+from mwmbl.tokenizer import get_bigrams, tokenizefrom mwmbl.utils import get_domain-logger = getLogger(__name__)@@ -38,7 +37,7 @@ SCORE_THRESHOLD = 0.0LENGTH_PENALTY = 0.04MATCH_EXPONENT = 2DOMAIN_SCORE_SMOOTHING = 0.1-HTTPS_STRING = 'https://'+HTTPS_STRING = "https://"WIKI_SCORES = json.load(open(Path(__file__).parent.parent / "resources" / "wiki_stats.json"))WIKI_MAX_SCORE = next(iter(WIKI_SCORES.values()))DOCUMENT_FREQUENCIES = json.load(open(Path(__file__).parent.parent / "resources" / "document_counts.json"))@@ -49,14 +48,19 @@ def score_result(terms: list[str], result: Document, is_complete: bool):features = get_features(terms, result.title, result.url, result.extract, result.score, is_complete)length_penalty = math.e ** (-LENGTH_PENALTY * len(result.url))- match_score = (4 * features['match_score_title'] + features['match_score_extract'] + 2 * features[- 'match_score_domain'] + 2 * features['match_score_domain_tokenized'] + features['match_score_path'])+ match_score = (+ 4 * features["match_score_title"]+ + features["match_score_extract"]+ + 2 * features["match_score_domain"]+ + 2 * features["match_score_domain_tokenized"]+ + features["match_score_path"]+ )- if features[f'match_terms'] <= len(terms) / 2 and result.state is None:+ if features["match_terms"] <= len(terms) / 2 and result.state is None:return 0.0if match_score > MATCH_SCORE_THRESHOLD:- return match_score * length_penalty * (features['domain_score'] + DOMAIN_SCORE_SMOOTHING) / 10+ return match_score * length_penalty * (features["domain_score"] + DOMAIN_SCORE_SMOOTHING) / 10return 0.0@@ -71,14 +75,14 @@ def score_result_whole(terms: list[str], result: Document, is_complete: bool) ->return 0.0features = get_features(terms,- result.title or '',+ result.title or "",result.url,- result.extract or '',+ result.extract or "",result.score or 0.0,is_complete,)length_penalty = math.e ** (-LENGTH_PENALTY * len(result.url))- return features['match_score_whole'] * length_penalty * (features['domain_score'] + DOMAIN_SCORE_SMOOTHING) / 10+ return features["match_score_whole"] * length_penalty * (features["domain_score"] + DOMAIN_SCORE_SMOOTHING) / 10def score_match(last_match_char, match_length, total_possible_match_length):@@ -140,35 +144,39 @@ def get_features(terms, title, url, extract, score, is_complete):domain = parsed_url.netlocpath = parsed_url.pathquery = parsed_url.query- whole = title + ' ' + extract + ' ' + domain + ' ' + path + ' ' + query- for part, name, is_url in [(title, 'title', False),- (extract, 'extract', False),- (domain, 'domain', True),- (domain, 'domain_tokenized', False),- (path, 'path', True),- (query, 'query', False),- (whole, 'whole', False)]:- last_match_char, match_length, total_possible_match_length, match_terms, match_counts = \- get_match_features(terms, part, is_complete, is_url)- features[f'last_match_char_{name}'] = last_match_char- features[f'match_length_{name}'] = match_length- features[f'total_possible_match_length_{name}'] = total_possible_match_length- features[f'match_score_{name}'] = score_match(last_match_char, match_length, total_possible_match_length)- features[f'match_terms_{name}'] = match_terms- features[f'match_term_proportion_{name}'] = match_terms / len(terms)+ whole = title + " " + extract + " " + domain + " " + path + " " + query+ for part, name, is_url in [+ (title, "title", False),+ (extract, "extract", False),+ (domain, "domain", True),+ (domain, "domain_tokenized", False),+ (path, "path", True),+ (query, "query", False),+ (whole, "whole", False),+ ]:+ last_match_char, match_length, total_possible_match_length, match_terms, match_counts = get_match_features(+ terms, part, is_complete, is_url+ )+ features[f"last_match_char_{name}"] = last_match_char+ features[f"match_length_{name}"] = match_length+ features[f"total_possible_match_length_{name}"] = total_possible_match_length+ features[f"match_score_{name}"] = score_match(last_match_char, match_length, total_possible_match_length)+ features[f"match_terms_{name}"] = match_terms+ features[f"match_term_proportion_{name}"] = match_terms / len(terms)# tf_idf_features = get_tf_idf_features(match_counts)# features.update({f"{name}_{k}": v for k, v in tf_idf_features.items()})- features['num_terms'] = len(terms)- features['num_chars'] = len(' '.join(terms))- features['domain_score'] = get_domain_score(url)- features['path_length'] = len(path)- features['domain_length'] = len(domain)- features['wiki_score'] = get_wiki_score(url)- features['item_score'] = score- features['match_terms'] = max(features[f'match_terms_{name}']- for name in ['title', 'extract', 'domain', 'domain_tokenized', 'path'])+ features["num_terms"] = len(terms)+ features["num_chars"] = len(" ".join(terms))+ features["domain_score"] = get_domain_score(url)+ features["path_length"] = len(path)+ features["domain_length"] = len(domain)+ features["wiki_score"] = get_wiki_score(url)+ features["item_score"] = score+ features["match_terms"] = max(+ features[f"match_terms_{name}"] for name in ["title", "extract", "domain", "domain_tokenized", "path"]+ )return features@@ -210,7 +218,7 @@ def get_match_features(terms, result_string, is_complete, is_url):def get_wiki_score(url):- title = url.split('/')[-1]+ title = url.split("/")[-1]return WIKI_SCORES.get(title, 0.0) / WIKI_MAX_SCORE@@ -278,8 +286,7 @@ class Ranker:def order_results(self, terms: list[str], pages: list[Document], is_complete: bool):pass- def search(self, s: str, additional_results: list[Document],- use_external_search: bool = True) -> list[Document]:+ def search(self, s: str, additional_results: list[Document], use_external_search: bool = True) -> list[Document]:# use_external_search=False is how the search-as-you-type path avoids a Wikipedia# call per keystroke; get_results has taken the flag all along, but only complete()# could reach it. See mwmbl.views.@@ -303,23 +310,26 @@ class Ranker:ordered_results, terms, completions = self.get_results(q, [], use_external_search=False)if len(ordered_results) == 0:# There are no results so suggest Google searches instead- completion_queries = [' '.join(terms[:-1] + [t]) for t in completions]+ completion_queries = [" ".join(terms[:-1] + [t]) for t in completions]adjusted_completions = completion_queries if q in completion_queries else [q] + completion_queriescompleted = ["search: google.com " + t for t in adjusted_completions]return [q, completed]else:adjusted_completions = [c for c in completions if c != terms[-1]]- urls = ["go: " + item.url[len(HTTPS_STRING):].rstrip('/') for item in ordered_results[:5]- if item.url.startswith(HTTPS_STRING) and all(term in item.url for term in terms)][:1]- completed = [' '.join(terms[:-1] + [t]) for t in adjusted_completions]+ urls = [+ "go: " + item.url[len(HTTPS_STRING) :].rstrip("/")+ for item in ordered_results[:5]+ if item.url.startswith(HTTPS_STRING) and all(term in item.url for term in terms)+ ][:1]+ completed = [" ".join(terms[:-1] + [t]) for t in adjusted_completions]return [q, urls + completed]def get_results(self, q: str, additional_results: list[Document], use_external_search: bool = True):logger.info(f"Get results with {len(additional_results)} additional results")terms = tokenize(q)- is_complete = q.endswith(' ')+ is_complete = q.endswith(" ")if len(terms) > 0 and not is_complete:completions = self.completer.complete(terms[-1])retrieval_terms = set(terms + completions)@@ -330,8 +340,7 @@ class Ranker:# Check for curationcuration_term = " ".join(terms)curation_items = self.tiny_index.retrieve(curation_term)- curated_items = [d for d in curation_items if d.state is not None- and d.term == curation_term]+ curated_items = [d for d in curation_items if d.state is not None and d.term == curation_term]bigrams = set(get_bigrams(len(terms), terms))@@ -343,13 +352,17 @@ class Ranker:else:items_wrong_state = self.tiny_index.retrieve(term)# If this is not a curation term, it is not curated for the current term- items = [Document(result.title,- result.url,- result.extract,- result.score,- result.term,- remove_curate_state(result.state))- for result in items_wrong_state]+ items = [+ Document(+ result.title,+ result.url,+ result.extract,+ result.score,+ result.term,+ remove_curate_state(result.state),+ )+ for result in items_wrong_state+ ]if items is not None:pages += items@@ -364,8 +377,9 @@ class Ranker:return state_fixed, terms, completions@staticmethod- def _remove_blacklisted(candidates: list[Document], curated_items: list[Document],- index_items: list[Document]) -> tuple[list[Document], list[Document]]:+ def _remove_blacklisted(+ candidates: list[Document], curated_items: list[Document], index_items: list[Document]+ ) -> tuple[list[Document], list[Document]]:"""Drop blacklisted documents from a result set and queue the indexed ones for purging.Curated items are filtered separately because they bypass order_results entirely -@@ -387,15 +401,17 @@ class Ranker:enqueue_for_purge([d for d in index_items if d.url in blacklisted_urls])- return ([d for d in candidates if d.url not in blacklisted_urls],- [d for d in curated_items if d.url not in blacklisted_urls])+ return (+ [d for d in candidates if d.url not in blacklisted_urls],+ [d for d in curated_items if d.url not in blacklisted_urls],+ )def external_search(self, q: str):return []def get_raw_results(self, query: str):tokens = tokenize(query)- term = ' '.join(tokens)+ term = " ".join(tokens)results = self.tiny_index.retrieve(term)# /raw bypasses get_results(), so it needs its own filter or it is a way to read# blacklisted documents straight out of the index.@@ -426,8 +442,9 @@ class HeuristicRanker(Ranker):return filtered_results-WIKI_SEARCH_API_URL = ("https://en.wikipedia.org/w/api.php?action=query&list=search"- "&srsearch={query}&srlimit={limit}&format=json")+WIKI_SEARCH_API_URL = (+ "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&srlimit={limit}&format=json"+)WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"# How many wiki results a caller gets unless it says otherwise, and the number the LTR@@ -458,8 +475,13 @@ WIKI_TOP_SCORE = 6.0# backoff, honouring any Retry-After, so a transient 429 doesn't silently drop a# query's wiki results. Live search never bursts, so this adds no latency in# normal operation. Successful responses are cached for months (see mwmbl.indexer.external_cache).-WIKI_RETRY = Retry(total=4, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504),- allowed_methods=frozenset({"GET"}), respect_retry_after_header=True)+WIKI_RETRY = Retry(+ total=4,+ backoff_factor=0.5,+ status_forcelist=(429, 502, 503, 504),+ allowed_methods=frozenset({"GET"}),+ respect_retry_after_header=True,+)# If Wikipedia has already rate-limited us hard enough that WIKI_RETRY exhausted its# retries, hammering it again on the very next query just makes things worse. Trip a@@ -498,11 +520,11 @@ def _is_wiki_rate_limited(e: RetryError) -> bool:return reason is not None and "429" in str(reason)-HTML_TAG_REGEX = re.compile(r'<[^>]+>')+HTML_TAG_REGEX = re.compile(r"<[^>]+>")def clean_html(s: str):- return html.unescape(HTML_TAG_REGEX.sub('', s))+ return html.unescape(HTML_TAG_REGEX.sub("", s))def wiki_score(rank: int) -> float:@@ -530,23 +552,19 @@ def get_wiki_results(s: str, max_wiki_results: int = NUM_WIKI_RESULTS) -> list[Dif _wiki_circuit_open():return []- escaped_query = urllib.parse.quote(query, safe='')- headers = {- 'User-Agent': 'Mwmbl/0.1.0 (https://mwmbl.org; [email protected])'- }+ escaped_query = urllib.parse.quote(query, safe="")+ headers = {"User-Agent": "Mwmbl/0.1.0 (https://mwmbl.org; [email protected])"}try:with requests.Session() as session:session.mount("https://en.wikipedia.org", HTTPAdapter(max_retries=WIKI_RETRY))response = session.get(- WIKI_SEARCH_API_URL.format(query=escaped_query, limit=WIKI_FETCH_LIMIT),- headers=headers, timeout=5)+ WIKI_SEARCH_API_URL.format(query=escaped_query, limit=WIKI_FETCH_LIMIT), headers=headers, timeout=5+ )wiki_response = response.json()except RetryError as e:if _is_wiki_rate_limited(e):_trip_wiki_circuit()- logger.warning(- "Wikipedia is rate-limiting us; pausing wiki lookups for %ds", WIKI_CIRCUIT_COOLDOWN_SECONDS- )+ logger.warning("Wikipedia is rate-limiting us; pausing wiki lookups for %ds", WIKI_CIRCUIT_COOLDOWN_SECONDS)else:# Don't log str(e)/exc_info here - RetryError embeds the request URL, which# embeds the (private) user query, and logger.exception would log both.@@ -556,17 +574,25 @@ def get_wiki_results(s: str, max_wiki_results: int = NUM_WIKI_RESULTS) -> list[Dlogger.warning("Failed to fetch Wikipedia results: %s", type(e).__name__)return []- if 'query' not in wiki_response or 'search' not in wiki_response['query']:- if 'error' in wiki_response:+ if "query" not in wiki_response or "search" not in wiki_response["query"]:+ if "error" in wiki_response:# Don't log the error body - it can echo back the (private) user query.logger.warning("Wikipedia API returned an error response")return []- wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),- wiki_score(rank), query, state=DocumentState.FROM_WIKI,- source=DocumentSource.WIKIPEDIA)- for rank, result in enumerate(wiki_response['query']['search'][:WIKI_FETCH_LIMIT])]+ wiki_results = [+ Document(+ result["title"],+ get_wiki_url(result["title"]),+ clean_html(result["snippet"]),+ wiki_score(rank),+ query,+ state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA,+ )+ for rank, result in enumerate(wiki_response["query"]["search"][:WIKI_FETCH_LIMIT])+ ]# Only a well-formed answer is stored, including one with no results in it. Every path# above returns [] without storing, so a transient failure is never remembered as@@ -581,19 +607,18 @@ def get_wiki_results(s: str, max_wiki_results: int = NUM_WIKI_RESULTS) -> list[Dclass HeuristicAndWikiRanker(HeuristicRanker):def __init__(- self,- tiny_index: TinyIndex,- completer: Completer,- return_none_if_no_mwmbl_results: bool = False,- score_threshold: float = 0.0,- max_wiki_results: int = NUM_WIKI_RESULTS+ self,+ tiny_index: TinyIndex,+ completer: Completer,+ return_none_if_no_mwmbl_results: bool = False,+ score_threshold: float = 0.0,+ max_wiki_results: int = NUM_WIKI_RESULTS,):super().__init__(tiny_index, completer, score_threshold)self.return_none_if_no_mwmbl_results = return_none_if_no_mwmbl_resultsself.max_wiki_results = max_wiki_results- def search(self, s: str, additional_results: list[Document],- use_external_search: bool = True) -> list[Document]:+ def search(self, s: str, additional_results: list[Document], use_external_search: bool = True) -> list[Document]:s_shortened = s[:MAX_QUERY_CHARS]wiki_results = get_wiki_results(s_shortened, self.max_wiki_results) if use_external_search else []diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.pyindex fdcc624..9fb4dfb 100644--- a/mwmbl/tinysearchengine/search.py+++ b/mwmbl/tinysearchengine/search.py@@ -7,10 +7,9 @@ from ninja.errors import HttpErrorfrom mwmbl import pricingfrom mwmbl.format import format_result, format_result_v2-from mwmbl.models import ApiKey, MwmblUser+from mwmbl.models import MwmblUserfrom mwmbl.quota import check_rate_limit, get_monthly_count, increment_monthlyfrom mwmbl.search_auth import SearchApiKeyAuth-from mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.rank import HeuristicRankerlogger = getLogger(__name__)@@ -19,16 +18,18 @@ logger = getLogger(__name__)SCORE_THRESHOLD = 0.25# Module-level routers-router = Router(tags=["Search"]) # v1-v2_router = Router(tags=["Search"]) # v2+router = Router(tags=["Search"]) # v1+v2_router = Router(tags=["Search"]) # v2# ---------------------------------------------------------------------------# Response schemas# ---------------------------------------------------------------------------+class TextSegment(Schema):"""A fragment of a title or extract, optionally highlighted."""+value: stris_bold: bool@@ -44,6 +45,7 @@ class TextSegment(Schema):class SearchResult(Schema):"""A single formatted search result returned by the main search endpoint."""+url: strtitle: list[TextSegment]extract: list[TextSegment]@@ -81,6 +83,7 @@ class SearchHit(Schema):class SearchResponse(Schema):"""Search response with optional Mwmbl usage metadata."""+query: strnumber_of_results: intresults: list[SearchHit]@@ -90,6 +93,7 @@ class SearchResponse(Schema):class RawDocument(Schema):"""A raw, unformatted document as stored in the index."""+title: strurl: strextract: str@@ -116,6 +120,7 @@ class RawDocument(Schema):class RawSearchResponse(Schema):"""Response from the raw search endpoint."""+query: strresults: list[RawDocument]@@ -155,10 +160,10 @@ _COMPLETE_RESPONSE_SCHEMA = {"type": "array","description": ("List of suggestion strings. Each entry is one of:\n"- "- A completed query term, e.g. `\"python tutorial\"`.\n"- "- A direct-navigation URL prefixed with `\"go: \"`, "- "e.g. `\"go: docs.python.org/3/tutorial\"`.\n"- "- A Google fallback prefixed with `\"search: google.com \"` "+ '- A completed query term, e.g. `"python tutorial"`.\n'+ '- A direct-navigation URL prefixed with `"go: "`, '+ 'e.g. `"go: docs.python.org/3/tutorial"`.\n'+ '- A Google fallback prefixed with `"search: google.com "` '"when no index results exist."),"items": {"type": "string"},@@ -195,22 +200,21 @@ _COMPLETE_RESPONSE_SCHEMA = {# Upgrade message helper# ---------------------------------------------------------------------------+def _upgrade_message(max_monthly_spend_cents: int) -> str:if max_monthly_spend_cents <= 0:return ("Increase your monthly spend limit at https://mwmbl.org/pricing to continue ""past the free 2,000 requests/month (billed at $5.00 per 1,000 additional queries).")- return (- "Increase your monthly spend limit at https://mwmbl.org/pricing to allow more "- "requests this month."- )+ return "Increase your monthly spend limit at https://mwmbl.org/pricing to allow more requests this month."# ---------------------------------------------------------------------------# Route registration helpers# ---------------------------------------------------------------------------+def _register_search_v1(r: Router | NinjaAPI, ranker: HeuristicRanker):"""Register the v1 search endpoint: returns a plain list of SearchResult."""@@ -346,10 +350,10 @@ def _register_common_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):"The response is a two-element list:\n""1. The original query string (echoed back).\n""2. A list of suggestion strings. Each suggestion is either:\n"- " - A completed query term (e.g. `\"python tutorial\"`).\n"- " - A direct URL prefixed with `\"go: \"` when the partial query "- "matches a URL in the index (e.g. `\"go: docs.python.org/3/tutorial\"`).\n"- " - A Google search suggestion prefixed with `\"search: google.com \"` "+ ' - A completed query term (e.g. `"python tutorial"`).\n'+ ' - A direct URL prefixed with `"go: "` when the partial query '+ 'matches a URL in the index (e.g. `"go: docs.python.org/3/tutorial"`).\n'+ ' - A Google search suggestion prefixed with `"search: google.com "` '"when no index results are found.\n\n""**Query parameter:** `q` — the partial query string (required)."),@@ -364,9 +368,7 @@ def _register_common_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):],"responses": {"200": {- "description": (- "A two-element list: the echoed query string and a list of suggestions."- ),+ "description": ("A two-element list: the echoed query string and a list of suggestions."),"content": {"application/json": {"schema": _COMPLETE_RESPONSE_SCHEMA,@@ -397,7 +399,7 @@ def _register_common_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):}},}- }+ },},)def complete(request, q: str):diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.pyindex 2a76c9e..5cd5e65 100644--- a/mwmbl/tinysearchengine/super_search.py+++ b/mwmbl/tinysearchengine/super_search.py@@ -11,14 +11,15 @@ and used to generate the OpenAPI documentation, so the two cannot drift.See plan: /api/v2/super-search/"""+import asyncioimport copyimport heapq-from concurrent.futures import ThreadPoolExecutorimport jsonimport loggingimport reimport time+from concurrent.futures import ThreadPoolExecutorfrom typing import Any, Literalfrom urllib.parse import unquote, urlparse@@ -95,6 +96,7 @@ async def _crawl(url: str):# the wrong shape.# ---------------------------------------------------------------------------+class ResultItem(Schema):"""A single ranked search result.@@ -102,14 +104,17 @@ class ResultItem(Schema):list. ``title`` and ``extract`` are nullable because upstream sources mayomit them."""- url: str = Field(description="Canonical URL of the result.",- examples=["https://docs.rs/tokio"])- title: str | None = Field(default=None, description="Page title, if known.",- examples=["Tokio — asynchronous Rust runtime"])- extract: str | None = Field(default=None, description="Short text snippet, if known.",- examples=["Tokio is an asynchronous runtime for Rust…"])- score: float = Field(description="Relevance score (higher is better), rounded to 4 dp.",- examples=[1.8423])++ url: str = Field(description="Canonical URL of the result.", examples=["https://docs.rs/tokio"])+ title: str | None = Field(+ default=None, description="Page title, if known.", examples=["Tokio — asynchronous Rust runtime"]+ )+ extract: str | None = Field(+ default=None,+ description="Short text snippet, if known.",+ examples=["Tokio is an asynchronous runtime for Rust…"],+ )+ score: float = Field(description="Relevance score (higher is better), rounded to 4 dp.", examples=[1.8423])source: str = Field(description=("Originating source for `result_promoted` (one of the Super Search "@@ -129,21 +134,23 @@ class ResultItem(Schema):class SourceStartedEvent(Schema):"""`source_started` — a source's query task has been launched (one per source)."""- source: str = Field(description="Name of the source whose query just started.",- examples=["hn"])++ source: str = Field(description="Name of the source whose query just started.", examples=["hn"])class SourceReturnedEvent(Schema):"""`source_returned` — a source finished successfully."""+source: str = Field(description="Name of the source that returned.", examples=["github"])count: int = Field(description="Number of raw documents the source returned.", examples=[10])class SourceFailedEvent(Schema):"""`source_failed` — a source errored or timed out and contributed nothing."""+source: str = Field(description="Name of the source that failed.", examples=["arxiv"])error: str = Field(- description="Failure reason: `\"timeout\"` or an exception message.",+ description='Failure reason: `"timeout"` or an exception message.',examples=["timeout"],)@@ -157,20 +164,20 @@ class ResultPromotedEvent(ResultItem):class PageFetchedEvent(Schema):"""`page_fetched` — a promoted page was crawled."""- url: str = Field(description="URL of the crawled page.",- examples=["https://docs.rs/tokio"])- links: int = Field(description="Number of outbound links discovered on the page.",- examples=[42])++ url: str = Field(description="URL of the crawled page.", examples=["https://docs.rs/tokio"])+ links: int = Field(description="Number of outbound links discovered on the page.", examples=[42])class LinkFollowedEvent(Schema):"""`link_followed` — an outbound link from a crawled page was fetched and addedto the candidate pool."""- url: str = Field(description="URL of the followed link.",- examples=["https://tokio.rs/tokio/tutorial"])- from_: str = Field(alias="from", description="URL of the parent page the link came from.",- examples=["https://docs.rs/tokio"])++ url: str = Field(description="URL of the followed link.", examples=["https://tokio.rs/tokio/tutorial"])+ from_: str = Field(+ alias="from", description="URL of the parent page the link came from.", examples=["https://docs.rs/tokio"]+ )class ResultsEvent(Schema):@@ -179,31 +186,35 @@ class ResultsEvent(Schema):Emitted progressively after each source returns, and once more as the finalranking. Clients should **replace** their displayed list on each event."""+results: list[ResultItem] = Field(description="Ranked results, best first.")count: int = Field(description="Number of results in this ranking.", examples=[100])class ErrorEvent(Schema):"""`error` — the pipeline crashed; the stream will end."""+message: str = Field(description="Error message.", examples=["internal error"])class DoneEvent(Schema):"""`done` — terminal event sent once when the stream finishes."""+reason: Literal["complete", "timed_out", "cancelled", "error"] = Field(- description="Why the stream ended.", examples=["complete"],+ description="Why the stream ended.",+ examples=["complete"],+ )+ elapsed_seconds: float = Field(description="Total wall-clock time for the request.", examples=[8.123])+ monthly_usage: int = Field(+ description="Caller's Super Search requests used this month (including this one).", examples=[3])- elapsed_seconds: float = Field(description="Total wall-clock time for the request.",- examples=[8.123])- monthly_usage: int = Field(description="Caller's Super Search requests used this month "- "(including this one).", examples=[3])monthly_limit: int = Field(description="Caller's monthly Super Search quota.", examples=[100])pages_indexed: int = Field(default=0,description="Number of distinct new pages (URLs) added to the Mwmbl index as a "- "result of this search. Can exceed the number of results returned, "- "since a page may match a unigram or bigram of the query without "- "matching the whole query.",+ "result of this search. Can exceed the number of results returned, "+ "since a page may match a unigram or bigram of the query without "+ "matching the whole query.",examples=[7],)@@ -259,6 +270,7 @@ def _event_oneof() -> list[dict]:# SSE helpers# ---------------------------------------------------------------------------+def _sse_frame(event_type: str, data: Any) -> bytes:if isinstance(data, BaseModel):data = data.model_dump(by_alias=True)@@ -287,10 +299,7 @@ def _doc_passes_term_filter(doc: Document, terms: list[str]) -> bool:if not terms:return Truetext = f"{doc.title or ''} {doc.extract or ''} {doc.url or ''}".lower()- matches = sum(- 1 for t in terms- if re.search(rf'\b{re.escape(t)}\b', text)- )+ matches = sum(1 for t in terms if re.search(rf"\b{re.escape(t)}\b", text))return matches > len(terms) / 2@@ -330,7 +339,7 @@ def _url_term_score(url: str, terms: list[str]) -> int:"kagi.com/discord" correctly matches both "kagi" and "discord"."""url_lower = url.lower()- return sum(1 for t in terms if re.search(rf'\b{re.escape(t)}\b', url_lower))+ return sum(1 for t in terms if re.search(rf"\b{re.escape(t)}\b", url_lower))def _title_from_url(url: str) -> str:@@ -347,11 +356,11 @@ def _title_from_url(url: str) -> str:return _URL_TOKEN_RE.sub(" ", last).strip() or parsed.netloc-# ---------------------------------------------------------------------------# Pipeline# ---------------------------------------------------------------------------+async def _call_source(name: str, fn, client: httpx.AsyncClient, query: str, limit: int):try:docs = await asyncio.wait_for(@@ -367,8 +376,12 @@ async def _call_source(name: str, fn, client: httpx.AsyncClient, query: str, limasync def _follow_links(- parent: Document, query: str, emit, all_docs: list[Document],- last_results_key: list, lock: asyncio.Lock,+ parent: Document,+ query: str,+ emit,+ all_docs: list[Document],+ last_results_key: list,+ lock: asyncio.Lock,) -> None:"""Crawl the parent URL, score its outbound links, and collect the best for final ranking."""max_links = settings.SUPER_SEARCH_MAX_LINKS_PER_PAGE@@ -398,8 +411,7 @@ async def _follow_links(terms = tokenize(query)# Filtered before the crawl below, so a blacklisted link is never even fetched.- proxy_docs = _drop_blacklisted(- [Document(title=_title_from_url(u), url=u, extract="") for u in raw_links])+ proxy_docs = _drop_blacklisted([Document(title=_title_from_url(u), url=u, extract="") for u in raw_links])proxy_scores = [_url_term_score(d.url, terms) for d in proxy_docs]ranked = sorted(zip(proxy_docs, proxy_scores), key=lambda x: -x[1])[:max_links]@@ -423,11 +435,13 @@ async def _follow_links(# pseudo-result, not worth ranking or indexing. The extract may be empty.crawled_title = (c.get("title") or "").strip()if proxy_doc.url and crawled_title:- all_docs.append(Document(- title=crawled_title,- url=proxy_doc.url,- extract=c.get("extract") or "",- ))+ all_docs.append(+ Document(+ title=crawled_title,+ url=proxy_doc.url,+ extract=c.get("extract") or "",+ )+ )await _emit_final_results(query, all_docs, emit, last_results_key, lock)@@ -467,10 +481,13 @@ async def _emit_final_results(if key == last_results_key[0]:returnlast_results_key[0] = key- await emit("results", ResultsEvent(- results=[_result_payload(doc, score, "", "final") for doc, score in ranked],- count=len(ranked),- ))+ await emit(+ "results",+ ResultsEvent(+ results=[_result_payload(doc, score, "", "final") for doc, score in ranked],+ count=len(ranked),+ ),+ )async def _index_results(query: str, docs: list[Document]) -> int:@@ -482,17 +499,13 @@ async def _index_results(query: str, docs: list[Document]) -> int:if not docs:return 0try:- return await asyncio.to_thread(- index_results_against_query, docs, query, str(index_path)- )+ return await asyncio.to_thread(index_results_against_query, docs, query, str(index_path))except Exception:logger.exception("super-search failed to index results")return 0-async def _run_pipeline(- query: str, emit, all_docs: list[Document], last_results_key: list, lock: asyncio.Lock-) -> None:+async def _run_pipeline(query: str, emit, all_docs: list[Document], last_results_key: list, lock: asyncio.Lock) -> None:per_source_limit = settings.SUPER_SEARCH_RESULTS_PER_SOURCEtop_k = getattr(settings, "SUPER_SEARCH_TOP_K", 10)limits = httpx.Limits(max_connections=20, max_keepalive_connections=10)@@ -528,9 +541,7 @@ async def _run_pipeline(source_tasks = []for name, fn in SOURCES.items():await emit("source_started", SourceStartedEvent(source=name))- source_tasks.append(- asyncio.create_task(_call_source(name, fn, client, query, per_source_limit))- )+ source_tasks.append(asyncio.create_task(_call_source(name, fn, client, query, per_source_limit)))secondary: list[asyncio.Task] = []@@ -567,6 +578,7 @@ async def _run_pipeline(# SSE generator# ---------------------------------------------------------------------------+async def _sse_stream(query: str, monthly_usage: int, monthly_limit: int):queue: asyncio.Queue = asyncio.Queue()started = time.monotonic()@@ -618,13 +630,16 @@ async def _sse_stream(query: str, monthly_usage: int, monthly_limit: int):event_type, data = itemyield _sse_frame(event_type, data)- yield _sse_frame("done", DoneEvent(- reason=reason,- elapsed_seconds=round(time.monotonic() - started, 3),- monthly_usage=monthly_usage,- monthly_limit=monthly_limit,- pages_indexed=pages_indexed,- ))+ yield _sse_frame(+ "done",+ DoneEvent(+ reason=reason,+ elapsed_seconds=round(time.monotonic() - started, 3),+ monthly_usage=monthly_usage,+ monthly_limit=monthly_limit,+ pages_indexed=pages_indexed,+ ),+ )finally:if not task.done():task.cancel()@@ -638,6 +653,7 @@ async def _sse_stream(query: str, monthly_usage: int, monthly_limit: int):# Route registration# ---------------------------------------------------------------------------+def init_router() -> None:@router.get("",@@ -662,23 +678,25 @@ def init_router() -> None:"|---|---|---|\n""| `source_started` | `{source}` | A source's query task was launched (one per source). |\n""| `source_returned` | `{source, count}` | A source finished; `count` is the number of raw results it returned. |\n"- "| `source_failed` | `{source, error}` | A source errored or timed out; `error` is `\"timeout\"` or an exception message. |\n"- "| `result_promoted` | result item (`origin=\"direct\"`) | A result entered the live top-K and will have its outbound links followed. |\n"+ '| `source_failed` | `{source, error}` | A source errored or timed out; `error` is `"timeout"` or an exception message. |\n'+ '| `result_promoted` | result item (`origin="direct"`) | A result entered the live top-K and will have its outbound links followed. |\n'"| `page_fetched` | `{url, links}` | A promoted page was crawled; `links` is the number of outbound links found. |\n""| `link_followed` | `{url, from}` | An outbound link from a crawled page was fetched and added to the candidate pool. |\n"- "| `results` | `{results[], count}` | The current authoritative ranking (items have `origin=\"final\"`). Emitted progressively after each source and once more at the end — **replace** your displayed list on each. |\n"+ '| `results` | `{results[], count}` | The current authoritative ranking (items have `origin="final"`). Emitted progressively after each source and once more at the end — **replace** your displayed list on each. |\n'"| `error` | `{message}` | The pipeline crashed; the stream ends. |\n""| `done` | `{reason, elapsed_seconds, monthly_usage, monthly_limit, pages_indexed}` | Terminal event. `reason` is `complete`, `timed_out`, `cancelled` or `error`; `pages_indexed` is the number of new pages added to the Mwmbl index by this search. |\n\n""The exact JSON shape of every payload is given by the `oneOf` schema ""of the 200 response below."),openapi_extra={- "parameters": [{- "name": "q",- "in": "query",- "required": True,- "schema": {"type": "string", "example": "rust async runtimes"},- }],+ "parameters": [+ {+ "name": "q",+ "in": "query",+ "required": True,+ "schema": {"type": "string", "example": "rust async runtimes"},+ }+ ],"responses": {"200": {"description": (diff --git a/mwmbl/tinysearchengine/super_search_select/domains.py b/mwmbl/tinysearchengine/super_search_select/domains.pyindex bebfe2c..52f767d 100644--- a/mwmbl/tinysearchengine/super_search_select/domains.py+++ b/mwmbl/tinysearchengine/super_search_select/domains.py@@ -7,6 +7,7 @@ last two labels — three for known multi-label suffixes like ``co.uk``); exact-string equality is what the NDCG harness scores on, but for "which source containsthis result" the registrable domain is the right granularity."""+from __future__ import annotationsfrom collections import defaultdict@@ -16,10 +17,31 @@ from urllib.parse import urlparse# e.g. bbc.co.uk -> co.uk. Not exhaustive (no full public-suffix list), but covers# the suffixes that actually appear in the gold set / source shortlist._MULTI_SUFFIXES = {- "co.uk", "org.uk", "ac.uk", "gov.uk", "me.uk", "ltd.uk", "plc.uk",- "com.au", "net.au", "org.au", "edu.au", "gov.au",- "co.nz", "co.za", "co.in", "co.jp", "co.kr", "com.br", "com.mx",- "com.cn", "com.tr", "com.sg", "com.hk", "or.jp", "ne.jp",+ "co.uk",+ "org.uk",+ "ac.uk",+ "gov.uk",+ "me.uk",+ "ltd.uk",+ "plc.uk",+ "com.au",+ "net.au",+ "org.au",+ "edu.au",+ "gov.au",+ "co.nz",+ "co.za",+ "co.in",+ "co.jp",+ "co.kr",+ "com.br",+ "com.mx",+ "com.cn",+ "com.tr",+ "com.sg",+ "com.hk",+ "or.jp",+ "ne.jp",}@@ -35,7 +57,7 @@ def host_of(url: str) -> str:def _strip_www(host: str) -> str:for prefix in ("www.", "m."):if host.startswith(prefix):- return host[len(prefix):]+ return host[len(prefix) :]return hostdiff --git a/mwmbl/tinysearchengine/super_search_select/evaluation.py b/mwmbl/tinysearchengine/super_search_select/evaluation.pyindex 7993475..e344f0f 100644--- a/mwmbl/tinysearchengine/super_search_select/evaluation.py+++ b/mwmbl/tinysearchengine/super_search_select/evaluation.py@@ -26,6 +26,7 @@ randomized pairs.This module is pure (numpy/sklearn/xgboost only); building the matrix from livesources lives in ``scripts/super_search_eval.py``."""+from __future__ import annotationsimport json@@ -34,26 +35,28 @@ from pathlib import Pathimport numpy as np-from mwmbl.tinysearchengine.super_search_select.features import FEATURE_NAMES-@dataclassclass RewardMatrix:queries: list[str]sources: list[str]feature_names: list[str]- X: np.ndarray # (Q, S, F) features- R: np.ndarray # (Q, S) reward in [0, 1]- mask: np.ndarray # (Q, S) bool: source returned results for the query+ X: np.ndarray # (Q, S, F) features+ R: np.ndarray # (Q, S) reward in [0, 1]+ mask: np.ndarray # (Q, S) bool: source returned results for the querydef save(self, path: str | Path) -> None:path = Path(path)np.savez_compressed(path.with_suffix(".npz"), X=self.X, R=self.R, mask=self.mask)- path.with_suffix(".json").write_text(json.dumps({- "queries": self.queries,- "sources": self.sources,- "feature_names": self.feature_names,- }))+ path.with_suffix(".json").write_text(+ json.dumps(+ {+ "queries": self.queries,+ "sources": self.sources,+ "feature_names": self.feature_names,+ }+ )+ )@classmethoddef load(cls, path: str | Path) -> "RewardMatrix":@@ -61,9 +64,12 @@ class RewardMatrix:arrs = np.load(path.with_suffix(".npz"))meta = json.loads(path.with_suffix(".json").read_text())return cls(- queries=meta["queries"], sources=meta["sources"],+ queries=meta["queries"],+ sources=meta["sources"],feature_names=meta["feature_names"],- X=arrs["X"], R=arrs["R"], mask=arrs["mask"],+ X=arrs["X"],+ R=arrs["R"],+ mask=arrs["mask"],)@@ -71,6 +77,7 @@ class RewardMatrix:# Metrics# ---------------------------------------------------------------------------+def coverage_at_k(scores: np.ndarray, R: np.ndarray, mask: np.ndarray, k: int) -> float:"""Mean (over queries) reward captured by the top-k scored sources, relativeto the oracle top-k. 1.0 = picked the best possible k every time."""@@ -94,6 +101,7 @@ def coverage_at_k(scores: np.ndarray, R: np.ndarray, mask: np.ndarray, k: int) -# Feature selection (XGBoost, grouped CV by query)# ---------------------------------------------------------------------------+def _flatten(matrix: RewardMatrix, feature_idx: list[int]):"""Flatten masked (query, source) cells into (X, y, groups) for sklearn."""Q, S, _ = matrix.X.shape@@ -109,14 +117,12 @@ def _flatten(matrix: RewardMatrix, feature_idx: list[int]):def _cell_index(matrix: RewardMatrix) -> list[tuple[int, int]]:"""The (query, source) cell behind each flattened masked row, in order."""- return [(q, s)- for q in range(matrix.X.shape[0])- for s in range(matrix.X.shape[1])- if matrix.mask[q, s]]+ return [(q, s) for q in range(matrix.X.shape[0]) for s in range(matrix.X.shape[1]) if matrix.mask[q, s]]-def evaluate_feature_set(matrix: RewardMatrix, feature_idx: list[int], k: int = 10,- n_splits: int = 5, seed: int = 0) -> dict:+def evaluate_feature_set(+ matrix: RewardMatrix, feature_idx: list[int], k: int = 10, n_splits: int = 5, seed: int = 0+) -> dict:"""Grouped CV (by query): train XGBoost on the chosen features, scatter theout-of-fold predictions to the (query, source) grid, and report held-outcoverage@k and RMSE."""@@ -129,8 +135,7 @@ def evaluate_feature_set(matrix: RewardMatrix, feature_idx: list[int], k: int =rmses = []gkf = GroupKFold(n_splits=min(n_splits, len(set(groups))))for train, test in gkf.split(X, y, groups):- model = XGBRegressor(n_estimators=200, max_depth=4, learning_rate=0.1,- subsample=0.8, random_state=seed)+ model = XGBRegressor(n_estimators=200, max_depth=4, learning_rate=0.1, subsample=0.8, random_state=seed)model.fit(X[train], y[train])p = model.predict(X[test])rmses.append(float(np.sqrt(np.mean((p - y[test]) ** 2))))@@ -162,8 +167,10 @@ def select_features(matrix: RewardMatrix, k: int = 10, seed: int = 0) -> dict:# Policy simulation# ----------------------------------------------------------------------------def simulate_ts(matrix: RewardMatrix, k: int, nu: float, sigma2: float = 0.25,- lam: float = 1.0, seed: int = 0) -> float:++def simulate_ts(+ matrix: RewardMatrix, k: int, nu: float, sigma2: float = 0.25, lam: float = 1.0, seed: int = 0+) -> float:"""Replay linear-Gaussian Thompson sampling; return mean per-query captured reward."""rng = np.random.default_rng(seed)Q, S, F = matrix.X.shape@@ -219,15 +226,22 @@ def simulate_baselines(matrix: RewardMatrix, k: int, seed: int = 0) -> dict:return out-def sweep_explore_scale(matrix: RewardMatrix, k: int, nus: list[float],- sigma2: float = 0.25, lam: float = 1.0, seed: int = 0) -> dict[float, float]:+def sweep_explore_scale(+ matrix: RewardMatrix, k: int, nus: list[float], sigma2: float = 0.25, lam: float = 1.0, seed: int = 0+) -> dict[float, float]:"""Mean captured reward for each candidate exploration scale ``nu``."""return {nu: simulate_ts(matrix, k, nu, sigma2, lam, seed) for nu in nus}-def simulate_xgb(matrix: RewardMatrix, k: int, epsilon: float,- refit_every: int = 200, min_rows: int = 300, seed: int = 0,- params: dict | None = None) -> float:+def simulate_xgb(+ matrix: RewardMatrix,+ k: int,+ epsilon: float,+ refit_every: int = 200,+ min_rows: int = 300,+ seed: int = 0,+ params: dict | None = None,+) -> float:"""Replay the epsilon-greedy XGBoost contextual bandit over the matrix.Mirrors ``simulate_ts``'s contract (mean per-query captured reward).@@ -274,8 +288,8 @@ def simulate_xgb(matrix: RewardMatrix, k: int, epsilon: float,enc = np.stack([encode(q, s) for s in avail])ranked = avail[np.argsort(model.predict(enc))[::-1]]n_explore = int(rng.binomial(k_q, epsilon))- chosen = list(ranked[:k_q - n_explore])- rest = ranked[k_q - n_explore:]+ chosen = list(ranked[: k_q - n_explore])+ rest = ranked[k_q - n_explore :]if n_explore and rest.size:chosen += list(rng.choice(rest, size=min(n_explore, rest.size), replace=False))chosen = np.asarray(chosen)@@ -294,20 +308,21 @@ def simulate_xgb(matrix: RewardMatrix, k: int, epsilon: float,return captured / Q-def sweep_epsilon(matrix: RewardMatrix, k: int, epsilons: list[float],- refit_every: int = 200, min_rows: int = 300,- seed: int = 0) -> dict[float, float]:+def sweep_epsilon(+ matrix: RewardMatrix, k: int, epsilons: list[float], refit_every: int = 200, min_rows: int = 300, seed: int = 0+) -> dict[float, float]:"""Mean captured reward for each candidate exploration rate ``epsilon``."""- return {eps: simulate_xgb(matrix, k, eps, refit_every, min_rows, seed)- for eps in epsilons}+ return {eps: simulate_xgb(matrix, k, eps, refit_every, min_rows, seed) for eps in epsilons}# ---------------------------------------------------------------------------# Holdout evaluation of the xgb source model# ----------------------------------------------------------------------------def _holdout_split(matrix: RewardMatrix, home: list[str | None], test_frac: float,- seed: int) -> tuple[np.ndarray, np.ndarray]:++def _holdout_split(+ matrix: RewardMatrix, home: list[str | None], test_frac: float, seed: int+) -> tuple[np.ndarray, np.ndarray]:"""Split query indices into (train, test), stratified by home source soevery source's home queries appear in both splits where possible."""rng = np.random.default_rng(seed)@@ -326,8 +341,9 @@ def _holdout_split(matrix: RewardMatrix, home: list[str | None], test_frac: floareturn train_idx, test_idx-def _home_recall_at_k(scores: np.ndarray, matrix: RewardMatrix,- home: list[str | None], rows: np.ndarray, k: int) -> float:+def _home_recall_at_k(+ scores: np.ndarray, matrix: RewardMatrix, home: list[str | None], rows: np.ndarray, k: int+) -> float:"""Fraction of queries whose home source lands in the top-k scored sources."""s_index = {name: i for i, name in enumerate(matrix.sources)}hits, n = 0, 0@@ -344,9 +360,15 @@ def _home_recall_at_k(scores: np.ndarray, matrix: RewardMatrix,return hits / n if n else 0.0-def _lints_posterior_means(matrix: RewardMatrix, train_idx: np.ndarray, k: int,- nu: float = 0.05, sigma2: float = 0.25,- lam: float = 1.0, seed: int = 0) -> np.ndarray:+def _lints_posterior_means(+ matrix: RewardMatrix,+ train_idx: np.ndarray,+ k: int,+ nu: float = 0.05,+ sigma2: float = 0.25,+ lam: float = 1.0,+ seed: int = 0,+) -> np.ndarray:"""Replay LinTS over the train queries; return per-arm posterior means forgreedy scoring of held-out queries."""rng = np.random.default_rng(seed)@@ -371,9 +393,14 @@ def _lints_posterior_means(matrix: RewardMatrix, train_idx: np.ndarray, k: int,return np.stack([np.linalg.inv(A[s]) @ b[s] for s in range(S)])-def evaluate_holdout(matrix: RewardMatrix, k: int = 10, test_frac: float = 0.2,- seed: int = 0, home_by_query: dict[str, str] | None = None,- params: dict | None = None) -> dict:+def evaluate_holdout(+ matrix: RewardMatrix,+ k: int = 10,+ test_frac: float = 0.2,+ seed: int = 0,+ home_by_query: dict[str, str] | None = None,+ params: dict | None = None,+) -> dict:"""Train the xgb source model on a query split, evaluate on the rest.``home_by_query`` maps a query to the source it was written for (the@@ -387,9 +414,12 @@ def evaluate_holdout(matrix: RewardMatrix, k: int = 10, test_frac: float = 0.2,if home_by_query is not None:home = [home_by_query.get(q) for q in matrix.queries]else:- home = [matrix.sources[int(np.argmax(np.where(matrix.mask[q], matrix.R[q], -np.inf)))]- if matrix.mask[q].any() else None- for q in range(len(matrix.queries))]+ home = [+ matrix.sources[int(np.argmax(np.where(matrix.mask[q], matrix.R[q], -np.inf)))]+ if matrix.mask[q].any()+ else None+ for q in range(len(matrix.queries))+ ]train_idx, test_idx = _holdout_split(matrix, home, test_frac, seed)vocab = xgb_model.build_vocab(matrix.sources)@@ -403,9 +433,9 @@ def evaluate_holdout(matrix: RewardMatrix, k: int = 10, test_frac: float = 0.2,train_mask[train_idx] = Trueseen = matrix.mask & train_mask[:, None]with np.errstate(invalid="ignore"):- source_mean = np.where(seen.sum(axis=0) > 0,- (matrix.R * seen).sum(axis=0) / np.maximum(seen.sum(axis=0), 1),- 0.0)+ source_mean = np.where(+ seen.sum(axis=0) > 0, (matrix.R * seen).sum(axis=0) / np.maximum(seen.sum(axis=0), 1), 0.0+ )def rows_for(idx: np.ndarray):xs, ys, cells = [], [], []diff --git a/mwmbl/tinysearchengine/super_search_select/features.py b/mwmbl/tinysearchengine/super_search_select/features.pyindex 877c1f2..9a7b100 100644--- a/mwmbl/tinysearchengine/super_search_select/features.py+++ b/mwmbl/tinysearchengine/super_search_select/features.py@@ -15,6 +15,7 @@ profiles are fine; raw query projections/embeddings are not and must never beadded here. ``test_persisted_features_are_scalar_and_reconstruction_safe``trips if a feature name suggests otherwise."""+from __future__ import annotationsimport re@@ -36,13 +37,13 @@ from mwmbl.tinysearchengine.super_search_select.registry import SiteMeta# to a vector; the dense (intent, source) co-occurrence is what makes this learnable# where raw per-term one-hot is ~93% singletons on the eval matrix.INTENT_NAMES = [- "code", # programming / software / dev tooling- "academic", # papers / research / formal study- "gaming", # video games, mods, speedruns- "music", # songs, lyrics, albums, instruments- "news", # current events / politics / recency- "howto", # questions / guides / instructional- "media", # books / film / art / reviews+ "code", # programming / software / dev tooling+ "academic", # papers / research / formal study+ "gaming", # video games, mods, speedruns+ "music", # songs, lyrics, albums, instruments+ "news", # current events / politics / recency+ "howto", # questions / guides / instructional+ "media", # books / film / art / reviews"reference", # definitions / encyclopedic / "what is"]@@ -51,39 +52,54 @@ _INTENT_PATTERNS = {r"\b(python|javascript|typescript|rust|golang|c\+\+|api|sdk|cli|npm|pip|"r"pypi|github|gitlab|git|docker|kubernetes|regex|compiler|runtime|library|"r"framework|package|module|function|async|bug|error|exception|traceback|"- r"syntax|code|coding|programming|developer|debug|install)\b", re.I),+ r"syntax|code|coding|programming|developer|debug|install)\b",+ re.I,+ ),"academic": re.compile(r"\b(paper|papers|study|studies|research|thesis|dissertation|theorem|proof|"r"lemma|journal|arxiv|preprint|citation|hypothesis|equation|dataset|"- r"benchmark|algorithm|peer.?review)\b", re.I),+ r"benchmark|algorithm|peer.?review)\b",+ re.I,+ ),"gaming": re.compile(r"\b(game|games|gaming|gamer|mod|mods|speedrun|playthrough|walkthrough|rpg|"r"fps|mmo|roguelike|pokemon|pokémon|minecraft|factorio|steam|itch|"- r"console|xbox|playstation|nintendo)\b", re.I),+ r"console|xbox|playstation|nintendo)\b",+ re.I,+ ),"music": re.compile(r"\b(song|songs|lyric|lyrics|album|albums|band|bands|music|musician|guitar|"- r"piano|synth|chord|chords|melody|track|tracks|remix|vinyl|discography)\b", re.I),+ r"piano|synth|chord|chords|melody|track|tracks|remix|vinyl|discography)\b",+ re.I,+ ),"news": re.compile(r"\b(news|election|elections|president|senate|congress|war|breaking|latest|"r"today|recent|court|verdict|policy|sanctions|protest|economy|inflation|"- r"20\d\d)\b", re.I),+ r"20\d\d)\b",+ re.I,+ ),"howto": re.compile(r"\b(how|what|why|who|when|where|guide|guides|tutorial|tutorials|tips|howto|"- r"learn|fix|setup|configure|versus|vs|difference)\b", re.I),+ r"learn|fix|setup|configure|versus|vs|difference)\b",+ re.I,+ ),"media": re.compile(r"\b(book|books|novel|novels|author|authors|film|films|movie|movies|cinema|"r"series|art|artist|painting|paintings|museum|gallery|poem|poetry|story|"- r"stories|review|reviews)\b", re.I),+ r"stories|review|reviews)\b",+ re.I,+ ),"reference": re.compile(r"\b(define|definition|meaning|means|history|origin|encyclopedia|wiki|"- r"wikipedia|biography|facts|overview|explained|explain)\b", re.I),+ r"wikipedia|biography|facts|overview|explained|explain)\b",+ re.I,+ ),}def classify_intent(query: str) -> list[float]:"""One-hot the query over ``INTENT_NAMES`` via lexicon rules (multi-label)."""- return [1.0 if _INTENT_PATTERNS[name].search(query or "") else 0.0- for name in INTENT_NAMES]+ return [1.0 if _INTENT_PATTERNS[name].search(query or "") else 0.0 for name in INTENT_NAMES]# Stable feature order. ``bias`` is the per-arm intercept (lets the bandit learn@@ -111,6 +127,7 @@ _CODE_RE = re.compile(r"[_/{}();]|::|->|[a-z][A-Z]|\.\w")@dataclassclass QueryContext:"""Per-query, site-independent context, computed once per search."""+bow: np.ndarraycng: np.ndarrayn_tokens: int@@ -132,9 +149,10 @@ class QueryContext:class SiteStats:"""Online per-site stats used as features (read from Redis via ``rstats``;defaults for cold sites)."""- contribution_ema: float = 0.0 # EMA of the source's per-request reward (judge score)- latency_ema: float = 0.0 # mean response time, seconds- failure_rate: float = 0.0 # fraction of recent queries that errored/timed out++ contribution_ema: float = 0.0 # EMA of the source's per-request reward (judge score)+ latency_ema: float = 0.0 # mean response time, seconds+ failure_rate: float = 0.0 # fraction of recent queries that errored/timed outdef feature_vector(@@ -147,19 +165,20 @@ def feature_vector(stats = stats or SiteStats()bow_profile, cng_profile = profiletimeout = settings.SUPER_SEARCH_PER_SOURCE_TIMEOUT or 1.0- x = np.array([- 1.0, # bias- vectors.cosine(qctx.bow, bow_profile), # cos_bow- vectors.cosine(qctx.cng, cng_profile), # cos_cng- meta.popularity, # popularity- meta.estimated_pages, # estimated_pages- stats.contribution_ema, # contribution_ema- min(stats.latency_ema / timeout, 1.0), # latency_penalty (0..1)- stats.failure_rate, # failure_rate- min(qctx.n_tokens, 10) / 10.0, # query_len (normalised)- 1.0 if qctx.has_code_token else 0.0, # has_code_token- *qctx.intent, # intent_* one-hot block- ], dtype=np.float64)+ x = np.array(+ [+ 1.0, # bias+ vectors.cosine(qctx.bow, bow_profile), # cos_bow+ vectors.cosine(qctx.cng, cng_profile), # cos_cng+ meta.popularity, # popularity+ meta.estimated_pages, # estimated_pages+ stats.contribution_ema, # contribution_ema+ min(stats.latency_ema / timeout, 1.0), # latency_penalty (0..1)+ stats.failure_rate, # failure_rate+ min(qctx.n_tokens, 10) / 10.0, # query_len (normalised)+ 1.0 if qctx.has_code_token else 0.0, # has_code_token+ *qctx.intent, # intent_* one-hot block+ ],+ dtype=np.float64,+ )return x--diff --git a/mwmbl/tinysearchengine/super_search_select/judge.py b/mwmbl/tinysearchengine/super_search_select/judge.pyindex ee9488e..d0d9692 100644--- a/mwmbl/tinysearchengine/super_search_select/judge.py+++ b/mwmbl/tinysearchengine/super_search_select/judge.py@@ -14,6 +14,7 @@ deploys without the model artifact keep working unchanged.Queries pass through this module in memory only; nothing is persisted."""+from __future__ import annotationsimport logging@@ -25,8 +26,8 @@ from django.conf import settingslogger = logging.getLogger(__name__)-DOC_CHARS = 1000 # matches judge training (scripts/judge_bakeoff.py DOC_CHARS)-MAX_TOKENS = 256 # matches training max_length+DOC_CHARS = 1000 # matches judge training (scripts/judge_bakeoff.py DOC_CHARS)+MAX_TOKENS = 256 # matches training max_lengthBATCH_SIZE = 64@@ -43,24 +44,21 @@ class Judge:self.tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))self.tokenizer.enable_truncation(max_length=MAX_TOKENS)self.tokenizer.enable_padding()- self.session = onnxruntime.InferenceSession(- str(model_dir / "model.onnx"), providers=["CPUExecutionProvider"])+ self.session = onnxruntime.InferenceSession(str(model_dir / "model.onnx"), providers=["CPUExecutionProvider"])self.input_names = {i.name for i in self.session.get_inputs()}def score(self, query: str, doc_texts: list[str]) -> list[float]:"""Relevance of each doc text to the query, each in [0, 1]."""scores: list[float] = []for start in range(0, len(doc_texts), BATCH_SIZE):- batch = doc_texts[start:start + BATCH_SIZE]+ batch = doc_texts[start : start + BATCH_SIZE]encodings = self.tokenizer.encode_batch([(query, text) for text in batch])feed = {"input_ids": np.array([e.ids for e in encodings], dtype=np.int64),- "attention_mask": np.array([e.attention_mask for e in encodings],- dtype=np.int64),+ "attention_mask": np.array([e.attention_mask for e in encodings], dtype=np.int64),}if "token_type_ids" in self.input_names:- feed["token_type_ids"] = np.array([e.type_ids for e in encodings],- dtype=np.int64)+ feed["token_type_ids"] = np.array([e.type_ids for e in encodings], dtype=np.int64)logits = self.session.run(None, feed)[0][:, 0].astype(np.float64)scores.extend(float(s) for s in 1 / (1 + np.exp(-logits)))return scores@@ -86,8 +84,8 @@ def get_judge() -> Judge | None:logger.info("relevance judge loaded from %s", model_dir)else:logger.warning(- "relevance judge model not found at %s; falling back to "- "LTR ranking and survival rewards", model_dir)+ "relevance judge model not found at %s; falling back to LTR ranking and survival rewards", model_dir+ )except Exception:logger.exception("failed to load relevance judge from %s", model_dir)_load_attempted = Truediff --git a/mwmbl/tinysearchengine/super_search_select/policy.py b/mwmbl/tinysearchengine/super_search_select/policy.pyindex 5403984..63201c0 100644--- a/mwmbl/tinysearchengine/super_search_select/policy.py+++ b/mwmbl/tinysearchengine/super_search_select/policy.py@@ -13,6 +13,7 @@ feature vectors used for the decision are stashed on the ``SelectionContext``so the reward attribution at request completion is consistent with the actiontaken."""+from __future__ import annotationsimport random@@ -76,7 +77,9 @@ def select_sources(elif name not in ctx.features:# pinned sources weren't scored; compute their features too.ctx.features[name] = feature_vector(- qctx, get_meta(name), profs.get(name, (None, None)),+ qctx,+ get_meta(name),+ profs.get(name, (None, None)),pinned_stats.get(name),).tolist()@@ -96,8 +99,8 @@ def _select_xgb(selectable, feats, budget) -> list[str]:ranked = sorted(selectable, key=lambda n: scores[n], reverse=True)epsilon = settings.SUPER_SEARCH_XGB_EPSILONn_explore = sum(random.random() < epsilon for _ in range(budget))- chosen = ranked[:budget - n_explore]- rest = ranked[budget - n_explore:]+ chosen = ranked[: budget - n_explore]+ rest = ranked[budget - n_explore :]chosen += random.sample(rest, min(n_explore, len(rest)))return chosendiff --git a/mwmbl/tinysearchengine/super_search_select/profiles.py b/mwmbl/tinysearchengine/super_search_select/profiles.pyindex 62963a4..dcbc8e7 100644--- a/mwmbl/tinysearchengine/super_search_select/profiles.py+++ b/mwmbl/tinysearchengine/super_search_select/profiles.py@@ -9,6 +9,7 @@ Vectors are stored as raw float32 bytes, so this module uses its ownbinary-safe Redis connection (the shared client in ``search_setup`` /``super_search`` uses ``decode_responses=True``, which would corrupt bytes)."""+from __future__ import annotationsimport numpy as np@@ -65,10 +66,7 @@ def get_profiles(sites: list[str]) -> dict[str, tuple[np.ndarray | None, np.ndarkeys = [_PROFILE_BOW.format(site=s) for s in sites] + [_PROFILE_CNG.format(site=s) for s in sites]raw = r.mget(keys)n = len(sites)- return {- site: (vectors.from_bytes(raw[i]), vectors.from_bytes(raw[n + i]))- for i, site in enumerate(sites)- }+ return {site: (vectors.from_bytes(raw[i]), vectors.from_bytes(raw[n + i])) for i, site in enumerate(sites)}def update_profile(site: str, docs: list[Document]) -> None:diff --git a/mwmbl/tinysearchengine/super_search_select/registry.py b/mwmbl/tinysearchengine/super_search_select/registry.pyindex 31983c2..508a061 100644--- a/mwmbl/tinysearchengine/super_search_select/registry.py+++ b/mwmbl/tinysearchengine/super_search_select/registry.py@@ -9,6 +9,7 @@ Recipe sources carry ``domain``/``field`` directly (parsed by ``recipe.py``);the handful of hand-written Python adapters have static metadata here. Theshortlist supplies popularity / page-count priors by domain."""+from __future__ import annotationsimport json@@ -31,12 +32,12 @@ _DEFAULT_SCORE = 0.6@dataclass(frozen=True)class SiteMeta:- name: str # source key in SOURCES+ name: str # source key in SOURCESdomain: strfield: str = "other"popularity: float = _DEFAULT_SCOREestimated_pages: float = _DEFAULT_SCORE- always_on: bool = False # global sources always included in selection+ always_on: bool = False # global sources always included in selection# Hand-written adapters: not in the shortlist (or special/global), so metadata is static.diff --git a/mwmbl/tinysearchengine/super_search_select/rewards.py b/mwmbl/tinysearchengine/super_search_select/rewards.pyindex ebe0110..ee396b1 100644--- a/mwmbl/tinysearchengine/super_search_select/rewards.py+++ b/mwmbl/tinysearchengine/super_search_select/rewards.py@@ -10,6 +10,7 @@ per-request state (which sources were selected, which URL came from whichsource, and the judge scores from the final re-rank) from the pipeline to thecompletion hook, where reward computation and ``log_impression`` run."""+from __future__ import annotationsimport logging@@ -23,8 +24,9 @@ logger = logging.getLogger(__name__)@dataclassclass SelectionContext:"""Per-request selection state, populated as the pipeline runs."""- candidates: list[str] = field(default_factory=list) # full action space- selected: list[str] = field(default_factory=list) # sources actually queried++ candidates: list[str] = field(default_factory=list) # full action space+ selected: list[str] = field(default_factory=list) # sources actually queriedsource_by_url: dict[str, str] = field(default_factory=dict) # url -> originating sourcefeatures: dict[str, list[float]] = field(default_factory=dict) # source -> feature vectorjudge_scores: dict[str, float] = field(default_factory=dict) # url -> judge score (final pool)@@ -71,8 +73,7 @@ def compute_judge_rewards(ctx: SelectionContext) -> dict[str, float] | None:if source in totals:totals[source][0] += scoretotals[source][1] += 1- return {name: total / count if count else 0.0- for name, (total, count) in totals.items()}+ return {name: total / count if count else 0.0 for name, (total, count) in totals.items()}def log_impression(ctx: SelectionContext, rewards: dict[str, float]) -> None:@@ -107,10 +108,7 @@ def record_source_provenance(ctx: SelectionContext) -> None:try:from mwmbl.models import SourceProvenance- rows = [- SourceProvenance(url=url, source=source, depth=0)- for url, source in ctx.source_by_url.items()- ]+ rows = [SourceProvenance(url=url, source=source, depth=0) for url, source in ctx.source_by_url.items()]SourceProvenance.objects.bulk_create(rows, ignore_conflicts=True)except Exception:logger.exception("failed to record super-search source provenance")diff --git a/mwmbl/tinysearchengine/super_search_select/rstats.py b/mwmbl/tinysearchengine/super_search_select/rstats.pyindex 8ef305a..15da927 100644--- a/mwmbl/tinysearchengine/super_search_select/rstats.py+++ b/mwmbl/tinysearchengine/super_search_select/rstats.py@@ -9,6 +9,7 @@ updates, and lets its trees interact it with the rest of the context.Aggregate per-source statistics only — nothing query-derived is stored here."""+from __future__ import annotationsimport redis@@ -39,10 +40,7 @@ def get_stats(sites: list[str]) -> dict[str, SiteStats]:for site in sites:pipe.get(_REWARD_EMA.format(site=site))values = pipe.execute()- return {- site: SiteStats(contribution_ema=float(v) if v is not None else 0.0)- for site, v in zip(sites, values)- }+ return {site: SiteStats(contribution_ema=float(v) if v is not None else 0.0) for site, v in zip(sites, values)}def update(rewards: dict[str, float]) -> None:diff --git a/mwmbl/tinysearchengine/super_search_select/vectors.py b/mwmbl/tinysearchengine/super_search_select/vectors.pyindex be4557e..095e942 100644--- a/mwmbl/tinysearchengine/super_search_select/vectors.py+++ b/mwmbl/tinysearchengine/super_search_select/vectors.py@@ -10,6 +10,7 @@ Vectors are dense ``numpy`` float32 arrays, L2-normalised so cosine similarityis a plain dot product. ``to_bytes`` / ``from_bytes`` give a compact Redisrepresentation."""+from __future__ import annotationsimport hashlib@@ -22,22 +23,135 @@ from mwmbl.tokenizer import tokenize# A compact English stop-word list. Removing these stops common function words# from dominating the projected bag-of-words. (No stop-word list existed in the# repo; this is intentionally small and self-contained.)-STOP_WORDS = frozenset({- "a", "about", "above", "after", "again", "all", "am", "an", "and", "any",- "are", "as", "at", "be", "because", "been", "before", "being", "below",- "between", "both", "but", "by", "can", "did", "do", "does", "doing", "down",- "during", "each", "few", "for", "from", "further", "had", "has", "have",- "having", "he", "her", "here", "hers", "herself", "him", "himself", "his",- "how", "i", "if", "in", "into", "is", "it", "its", "itself", "just", "me",- "more", "most", "my", "myself", "no", "nor", "not", "now", "of", "off",- "on", "once", "only", "or", "other", "our", "ours", "ourselves", "out",- "over", "own", "s", "same", "she", "should", "so", "some", "such", "t",- "than", "that", "the", "their", "theirs", "them", "themselves", "then",- "there", "these", "they", "this", "those", "through", "to", "too", "under",- "until", "up", "very", "was", "we", "were", "what", "when", "where",- "which", "while", "who", "whom", "why", "will", "with", "you", "your",- "yours", "yourself", "yourselves",-})+STOP_WORDS = frozenset(+ {+ "a",+ "about",+ "above",+ "after",+ "again",+ "all",+ "am",+ "an",+ "and",+ "any",+ "are",+ "as",+ "at",+ "be",+ "because",+ "been",+ "before",+ "being",+ "below",+ "between",+ "both",+ "but",+ "by",+ "can",+ "did",+ "do",+ "does",+ "doing",+ "down",+ "during",+ "each",+ "few",+ "for",+ "from",+ "further",+ "had",+ "has",+ "have",+ "having",+ "he",+ "her",+ "here",+ "hers",+ "herself",+ "him",+ "himself",+ "his",+ "how",+ "i",+ "if",+ "in",+ "into",+ "is",+ "it",+ "its",+ "itself",+ "just",+ "me",+ "more",+ "most",+ "my",+ "myself",+ "no",+ "nor",+ "not",+ "now",+ "of",+ "off",+ "on",+ "once",+ "only",+ "or",+ "other",+ "our",+ "ours",+ "ourselves",+ "out",+ "over",+ "own",+ "s",+ "same",+ "she",+ "should",+ "so",+ "some",+ "such",+ "t",+ "than",+ "that",+ "the",+ "their",+ "theirs",+ "them",+ "themselves",+ "then",+ "there",+ "these",+ "they",+ "this",+ "those",+ "through",+ "to",+ "too",+ "under",+ "until",+ "up",+ "very",+ "was",+ "we",+ "were",+ "what",+ "when",+ "where",+ "which",+ "while",+ "who",+ "whom",+ "why",+ "will",+ "with",+ "you",+ "your",+ "yours",+ "yourself",+ "yourselves",+ }+)# Seeds for the two independent hashes used by the hashing trick: one picks the# bucket, the other picks the sign. Independent seeds keep index and sign@@ -89,7 +203,7 @@ def project_char_ngrams(padded = f" {token} "for n in range(n_min, n_max + 1):for i in range(len(padded) - n + 1):- index, sign = _hash_token(padded[i:i + n], dim)+ index, sign = _hash_token(padded[i : i + n], dim)vec[index] += signreturn _l2_normalise(vec)diff --git a/mwmbl/tinysearchengine/super_search_select/xgb_model.py b/mwmbl/tinysearchengine/super_search_select/xgb_model.pyindex b5ccf45..5b431b2 100644--- a/mwmbl/tinysearchengine/super_search_select/xgb_model.py+++ b/mwmbl/tinysearchengine/super_search_select/xgb_model.py@@ -22,6 +22,7 @@ propagates out of source selection. The only quiet path is the designed one —no artifact in the runtime dir yet (normal before the first online retrain),in which case the repo-bundled warm-start artifact is used."""+from __future__ import annotationsimport json@@ -70,6 +71,7 @@ XGB_PARAMS = {# Encoding# ---------------------------------------------------------------------------+def build_vocab(names: Iterable[str]) -> list[str]:"""Frozen, sorted source vocabulary for the identity one-hot block."""return sorted(set(names))@@ -89,10 +91,9 @@ def encode(shared: Sequence[float], source: str, vocab_index: dict[str, int]) ->"""v = np.asarray(shared, dtype=np.float64)if v.ndim != 1 or v.size > NUM_FEATURES:- raise ValueError(- f"shared feature vector has {v.shape} shape; expected <= {NUM_FEATURES} values")+ raise ValueError(f"shared feature vector has {v.shape} shape; expected <= {NUM_FEATURES} values")x = np.zeros(NUM_FEATURES + len(vocab_index), dtype=np.float64)- x[:v.size] = v+ x[: v.size] = vidx = vocab_index.get(source)if idx is not None:x[NUM_FEATURES + idx] = 1.0@@ -103,6 +104,7 @@ def encode(shared: Sequence[float], source: str, vocab_index: dict[str, int]) -># Training data# ---------------------------------------------------------------------------+def build_training_data(rows: Iterable[tuple[dict[str, Sequence[float]], dict[str, float]]],vocab: Sequence[str],@@ -141,14 +143,16 @@ def build_training_data_from_matrix(if list(matrix.feature_names) != list(FEATURE_NAMES):raise ValueError(f"matrix feature names {matrix.feature_names} do not match the "- f"running code's FEATURE_NAMES {list(FEATURE_NAMES)}; rebuild the matrix")+ f"running code's FEATURE_NAMES {list(FEATURE_NAMES)}; rebuild the matrix"+ )vocab = build_vocab(matrix.sources)vocab_index = {name: i for i, name in enumerate(vocab)}ema_i = FEATURE_NAMES.index("contribution_ema")counts = matrix.mask.sum(axis=0)sums = (matrix.R * matrix.mask).sum(axis=0)- source_means = {matrix.sources[s]: float(sums[s] / counts[s]) if counts[s] else 0.0- for s in range(len(matrix.sources))}+ source_means = {+ matrix.sources[s]: float(sums[s] / counts[s]) if counts[s] else 0.0 for s in range(len(matrix.sources))+ }xs, ys = [], []for q in range(matrix.X.shape[0]):for s in range(matrix.X.shape[1]):@@ -169,7 +173,9 @@ def train(X: np.ndarray, y: np.ndarray, params: dict | None = None):def train_and_save_from_impressions(- window_days: int, min_rows: int, out_dir: str | Path,+ window_days: int,+ min_rows: int,+ out_dir: str | Path,) -> dict | None:"""Batch retrain from logged ``SuperSearchImpression`` rows.@@ -186,20 +192,17 @@ def train_and_save_from_impressions(from mwmbl.tinysearchengine.super_search_sources import SOURCEScutoff = datetime.now(timezone.utc) - timedelta(days=window_days)- rows = list(SuperSearchImpression.objects.filter(timestamp__gte=cutoff)- .values_list("features", "rewards"))+ rows = list(SuperSearchImpression.objects.filter(timestamp__gte=cutoff).values_list("features", "rewards"))n_pairs = sum(len(set(f) & set(r)) for f, r in rows)if n_pairs < min_rows:- logger.info("super-search xgb retrain skipped: %d pairs in %d days "- "(need %d)", n_pairs, window_days, min_rows)+ logger.info("super-search xgb retrain skipped: %d pairs in %d days (need %d)", n_pairs, window_days, min_rows)return Nonenames = {name for f, _ in rows for name in f} | set(SOURCES)vocab = build_vocab(names)X, y = build_training_data(rows, vocab)model = train(X, y)metrics = {"train_rmse": float(np.sqrt(np.mean((model.predict(X) - y) ** 2)))}- save_artifact(model, vocab, out_dir, reward_kind="judge",- n_rows=len(y), metrics=metrics)+ save_artifact(model, vocab, out_dir, reward_kind="judge", n_rows=len(y), metrics=metrics)return metrics@@ -207,6 +210,7 @@ def train_and_save_from_impressions(# Artifact save / load# ---------------------------------------------------------------------------+def save_artifact(model,vocab: Sequence[str],@@ -240,13 +244,12 @@ def save_artifact(tmp_meta = model_dir / (META_FILE + ".tmp")tmp_meta.write_text(json.dumps(meta, indent=2))os.replace(tmp_meta, model_dir / META_FILE)- logger.info("saved super-search xgb artifact to %s (%d rows, reward=%s)",- model_dir, n_rows, reward_kind)+ logger.info("saved super-search xgb artifact to %s (%d rows, reward=%s)", model_dir, n_rows, reward_kind)@dataclassclass XgbSourceModel:- model: object # fitted XGBRegressor+ model: object # fitted XGBRegressorvocab: list[str]vocab_index: dict[str, int]meta: dict@@ -269,14 +272,13 @@ def load_artifact(model_dir: str | Path) -> XgbSourceModel:model_dir = Path(model_dir)meta = json.loads((model_dir / META_FILE).read_text())if meta["format_version"] != FORMAT_VERSION:- raise ValueError(- f"artifact {model_dir} has format_version {meta['format_version']}, "- f"expected {FORMAT_VERSION}")+ raise ValueError(f"artifact {model_dir} has format_version {meta['format_version']}, expected {FORMAT_VERSION}")if meta["shared_feature_names"] != list(FEATURE_NAMES):raise ValueError(f"artifact {model_dir} was trained on features "f"{meta['shared_feature_names']} but the running code has "- f"{list(FEATURE_NAMES)}; retrain the artifact")+ f"{list(FEATURE_NAMES)}; retrain the artifact"+ )model = XGBRegressor()model.load_model(model_dir / MODEL_FILE)vocab = list(meta["source_vocab"])@@ -289,7 +291,8 @@ def load_artifact(model_dir: str | Path) -> XgbSourceModel:def save_profiles(- profiles: dict[str, tuple[np.ndarray, np.ndarray]], model_dir: str | Path,+ profiles: dict[str, tuple[np.ndarray, np.ndarray]],+ model_dir: str | Path,) -> None:"""Write the batch content profiles the model was trained against(``profiles.npz``: sources + bow/cng matrices) into the artifact dir."""@@ -309,8 +312,7 @@ def save_profiles(def load_profiles(model_dir: str | Path) -> dict[str, tuple[np.ndarray, np.ndarray]]:arrs = np.load(Path(model_dir) / PROFILES_FILE)- return {str(site): (arrs["bow"][i], arrs["cng"][i])- for i, site in enumerate(arrs["sources"])}+ return {str(site): (arrs["bow"][i], arrs["cng"][i]) for i, site in enumerate(arrs["sources"])}def seed_online_state() -> dict[str, int]:@@ -377,8 +379,12 @@ def get_model() -> XgbSourceModel:if key != _cached_key:_cached = load_artifact(model_dir)_cached_key = key- logger.info("super-search xgb model loaded from %s (trained %s, %d sources)",- model_dir, _cached.meta.get("trained_at"), len(_cached.vocab))+ logger.info(+ "super-search xgb model loaded from %s (trained %s, %d sources)",+ model_dir,+ _cached.meta.get("trained_at"),+ len(_cached.vocab),+ )_next_check = time.monotonic() + RELOAD_CHECK_SECONDSreturn _cacheddiff --git a/mwmbl/tinysearchengine/super_search_sources/__init__.py b/mwmbl/tinysearchengine/super_search_sources/__init__.pyindex 763019d..12a7f46 100644--- a/mwmbl/tinysearchengine/super_search_sources/__init__.py+++ b/mwmbl/tinysearchengine/super_search_sources/__init__.py@@ -5,6 +5,7 @@ and is responsible for one external API. Adapters never raise on HTTP/parseerrors — they log and return an empty list so one slow source can't sink theorchestrator."""+from mwmbl.tinysearchengine.super_search_sources.arxiv import search as search_arxivfrom mwmbl.tinysearchengine.super_search_sources.github import search as search_githubfrom mwmbl.tinysearchengine.super_search_sources.hn import search as search_hndiff --git a/mwmbl/tinysearchengine/super_search_sources/arxiv.py b/mwmbl/tinysearchengine/super_search_sources/arxiv.pyindex 5410a7f..3c86c45 100644--- a/mwmbl/tinysearchengine/super_search_sources/arxiv.py+++ b/mwmbl/tinysearchengine/super_search_sources/arxiv.py@@ -1,4 +1,5 @@"""ArXiv search via the public Atom API."""+import loggingfrom xml.etree.ElementTree import ParseErrordiff --git a/mwmbl/tinysearchengine/super_search_sources/github.py b/mwmbl/tinysearchengine/super_search_sources/github.pyindex b31d52d..baa9457 100644--- a/mwmbl/tinysearchengine/super_search_sources/github.py+++ b/mwmbl/tinysearchengine/super_search_sources/github.py@@ -4,6 +4,7 @@ Unauthenticated calls are limited to 10 requests/minute for the search endpoint,which is acceptable for the initial Super Search release. Add a token laterto lift the cap."""+import loggingimport httpxdiff --git a/mwmbl/tinysearchengine/super_search_sources/guardian.py b/mwmbl/tinysearchengine/super_search_sources/guardian.pyindex 9f3611e..a39ecc4 100644--- a/mwmbl/tinysearchengine/super_search_sources/guardian.py+++ b/mwmbl/tinysearchengine/super_search_sources/guardian.py@@ -5,6 +5,7 @@ Without a key the source is a no-op (returns []), so it degrades gracefully inenvironments where the key isn't configured. Covers the "news" intent that thetech/academic-skewed catalog otherwise misses."""+import loggingimport httpx@@ -23,13 +24,16 @@ async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Docuif not api_key:return []try:- response = await client.get(ENDPOINT, params={- "q": query,- "api-key": api_key,- "show-fields": "trailText",- "order-by": "relevance",- "page-size": max(1, min(limit, 10)),- })+ response = await client.get(+ ENDPOINT,+ params={+ "q": query,+ "api-key": api_key,+ "show-fields": "trailText",+ "order-by": "relevance",+ "page-size": max(1, min(limit, 10)),+ },+ )response.raise_for_status()results = response.json().get("response", {}).get("results", [])except (httpx.HTTPError, ValueError) as e:diff --git a/mwmbl/tinysearchengine/super_search_sources/hn.py b/mwmbl/tinysearchengine/super_search_sources/hn.pyindex 5f3cc40..40e3fa8 100644--- a/mwmbl/tinysearchengine/super_search_sources/hn.py+++ b/mwmbl/tinysearchengine/super_search_sources/hn.py@@ -1,4 +1,5 @@"""Hacker News via the Algolia search API (no auth required)."""+import loggingimport httpxdiff --git a/mwmbl/tinysearchengine/super_search_sources/homepage.py b/mwmbl/tinysearchengine/super_search_sources/homepage.pyindex cdb3ebe..dd52224 100644--- a/mwmbl/tinysearchengine/super_search_sources/homepage.py+++ b/mwmbl/tinysearchengine/super_search_sources/homepage.py@@ -7,6 +7,7 @@ title actually contains a query term — a cheap relevance guard against randomdomain squatters. It complements (does not replace) the general index: it justsurfaces an official homepage candidate the crawl may not have indexed yet."""+import loggingimport re@@ -34,8 +35,7 @@ async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Docubreakcandidate = f"https://{slug}{tld}/"try:- r = await client.get(candidate, follow_redirects=True,- headers={"User-Agent": _UA})+ r = await client.get(candidate, follow_redirects=True, headers={"User-Agent": _UA})except httpx.HTTPError:continueif r.status_code != 200 or "text/html" not in r.headers.get("content-type", ""):diff --git a/mwmbl/tinysearchengine/super_search_sources/imdb.py b/mwmbl/tinysearchengine/super_search_sources/imdb.pyindex 186aeaa..2c81107 100644--- a/mwmbl/tinysearchengine/super_search_sources/imdb.py+++ b/mwmbl/tinysearchengine/super_search_sources/imdb.py@@ -10,6 +10,7 @@ The query goes in the URL path (not a query-string param), so this can't be adeclarative recipe; the path is bucketed by the query's first alphanumeric char,as the endpoint expects."""+import loggingfrom urllib.parse import quotediff --git a/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py b/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.pyindex 2d8c54a..6c96506 100644--- a/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py+++ b/mwmbl/tinysearchengine/super_search_sources/mwmbl_index.py@@ -1,4 +1,5 @@"""Mwmbl index + Wikipedia via the standard search ranker."""+import asyncioimport loggingdiff --git a/mwmbl/tinysearchengine/super_search_sources/nhs.py b/mwmbl/tinysearchengine/super_search_sources/nhs.pyindex 506dc09..b2ae22f 100644--- a/mwmbl/tinysearchengine/super_search_sources/nhs.py+++ b/mwmbl/tinysearchengine/super_search_sources/nhs.py@@ -5,6 +5,7 @@ tech/academic-skewed source catalogue otherwise misses. The NHS search page isserver-rendered; each result is an ``a.app-search-results-item`` whose realtarget path is URL-encoded in the click-tracker's ``url`` query parameter."""+import loggingfrom urllib.parse import parse_qs, unquote, urljoin, urlsplitdiff --git a/mwmbl/tinysearchengine/super_search_sources/pypi.py b/mwmbl/tinysearchengine/super_search_sources/pypi.pyindex 8f7c1bf..7194df7 100644--- a/mwmbl/tinysearchengine/super_search_sources/pypi.py+++ b/mwmbl/tinysearchengine/super_search_sources/pypi.py@@ -3,6 +3,7 @@PyPI has no public full-text search endpoint, so we treat the query as apackage name and look it up directly. Returns at most one Document."""+import loggingimport rediff --git a/mwmbl/tinysearchengine/super_search_sources/recipe.py b/mwmbl/tinysearchengine/super_search_sources/recipe.pyindex 8b43cc3..521e811 100644--- a/mwmbl/tinysearchengine/super_search_sources/recipe.py+++ b/mwmbl/tinysearchengine/super_search_sources/recipe.py@@ -14,6 +14,7 @@ orchestrator.See ``recipes/*.yaml`` for examples covering JSON APIs (Wiktionary,archive.org) and HTML scraping (Project Gutenberg)."""+from __future__ import annotationsimport logging@@ -39,6 +40,7 @@ RECIPES_DIR = Path(__file__).parent / "recipes"@dataclassclass Recipe:"""A parsed search recipe. ``request`` and ``response`` are the raw YAML maps."""+name: strrequest: dictresponse: dict@@ -88,23 +90,21 @@ def load_recipes(directory: Path | str = RECIPES_DIR) -> dict[str, Recipe]:def make_recipe_source(recipe: Recipe) -> Callable[..., Coroutine[Any, Any, list[Document]]]:"""Wrap a recipe as a ``search(client, query, limit)`` adapter for ``SOURCES``."""+async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Document]:return await search_with_recipe(client, recipe, query, limit)+return search-async def search_with_recipe(- client: httpx.AsyncClient, recipe: Recipe, query: str, limit: int-) -> list[Document]:+async def search_with_recipe(client: httpx.AsyncClient, recipe: Recipe, query: str, limit: int) -> list[Document]:req = recipe.requestmethod = req.get("method", "GET").upper()params = _build_params(req.get("params"), query, limit)headers = _substitute(req.get("headers"), query, limit) if req.get("headers") else Nonejson_body = _substitute(req.get("json"), query, limit) if req.get("json") is not None else Nonetry:- response = await client.request(- method, req["url"], params=params, headers=headers, json=json_body- )+ response = await client.request(method, req["url"], params=params, headers=headers, json=json_body)response.raise_for_status()fmt = recipe.response_formatif fmt == "json":@@ -124,6 +124,7 @@ async def search_with_recipe(# Request building# ---------------------------------------------------------------------------+def _substitute(value, query: str, limit: int):if isinstance(value, str):return value.format(query=query, limit=limit)@@ -142,6 +143,7 @@ def _build_params(params: dict | None, query: str, limit: int) -> dict:# Shared helpers# ---------------------------------------------------------------------------+def _coerce_str(value, strip_html: bool = False) -> str:if value is None:return ""@@ -170,6 +172,7 @@ def _make_doc(values: dict, url: str) -> Document | None:# JSON# ---------------------------------------------------------------------------+def _walk(data, path: str):"""Walk a dotted path through nested dicts/lists; None if absent.@@ -231,6 +234,7 @@ def _resolve_url_json(url_spec, item: dict, values: dict, base_url: str = "") -># HTML# ---------------------------------------------------------------------------+def _parse_html(html_text: str, spec: dict) -> list[Document]:soup = BeautifulSoup(html_text, "html.parser")fields = spec["fields"]@@ -239,9 +243,7 @@ def _parse_html(html_text: str, spec: dict) -> list[Document]:docs: list[Document] = []for el in soup.select(spec["results"]):values = {- name: _select_html(el, field_spec, name in strip)- for name, field_spec in fields.items()- if name != "url"+ name: _select_html(el, field_spec, name in strip) for name, field_spec in fields.items() if name != "url"}doc = _make_doc(values, _resolve_url_html(fields.get("url"), el, base_url))if doc is not None:@@ -277,6 +279,7 @@ def _resolve_url_html(url_spec, el, base_url: str) -> str:# XML# ---------------------------------------------------------------------------+def _parse_xml(xml_text: str, spec: dict) -> list[Document]:root = fromstring(xml_text)results_path = spec.get("results", "")diff --git a/mwmbl/tinysearchengine/super_search_sources/smoke.py b/mwmbl/tinysearchengine/super_search_sources/smoke.pyindex d204e46..9c00f82 100644--- a/mwmbl/tinysearchengine/super_search_sources/smoke.py+++ b/mwmbl/tinysearchengine/super_search_sources/smoke.py@@ -9,6 +9,7 @@ a substantially different set of URLs. A recipe that's really scrapingnav/boilerplate returns the same links regardless of query, so a high overlapwith the control query's results is a failure."""+from __future__ import annotationsimport httpxdiff --git a/mwmbl/tinysearchengine/super_search_sources/stackexchange.py b/mwmbl/tinysearchengine/super_search_sources/stackexchange.pyindex 95704a5..3e550b1 100644--- a/mwmbl/tinysearchengine/super_search_sources/stackexchange.py+++ b/mwmbl/tinysearchengine/super_search_sources/stackexchange.py@@ -3,6 +3,7 @@Unauthenticated requests are subject to a 300/day quota per IP. The adaptersilently returns [] on quota or transport errors."""+import htmlimport loggingdiff --git a/mwmbl/tinysearchengine/super_search_sources/wikidata.py b/mwmbl/tinysearchengine/super_search_sources/wikidata.pyindex 5b265e7..830dfcd 100644--- a/mwmbl/tinysearchengine/super_search_sources/wikidata.py+++ b/mwmbl/tinysearchengine/super_search_sources/wikidata.py@@ -5,6 +5,7 @@ entity's official website (property ``P856``). This is the structured, Big-Tech-independent way to answer navigational/brand queries ("virgin media" ->https://www.virginmedia.com/) without a per-brand source."""+import loggingimport httpx@@ -22,10 +23,17 @@ _HEADERS = {"User-Agent": "mwmbl-supersearch/1.0 (https://mwmbl.org; hello@mwmblasync def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Document]:n = max(1, min(limit, 5))try:- r = await client.get(API, headers=_HEADERS, params={- "action": "wbsearchentities", "search": query,- "language": "en", "format": "json", "limit": n,- })+ r = await client.get(+ API,+ headers=_HEADERS,+ params={+ "action": "wbsearchentities",+ "search": query,+ "language": "en",+ "format": "json",+ "limit": n,+ },+ )r.raise_for_status()hits = r.json().get("search", [])except (httpx.HTTPError, ValueError) as e:@@ -37,10 +45,17 @@ async def search(client: httpx.AsyncClient, query: str, limit: int) -> list[Docureturn []try:- r2 = await client.get(API, headers=_HEADERS, params={- "action": "wbgetentities", "ids": "|".join(ids),- "props": "claims|labels|descriptions", "languages": "en", "format": "json",- })+ r2 = await client.get(+ API,+ headers=_HEADERS,+ params={+ "action": "wbgetentities",+ "ids": "|".join(ids),+ "props": "claims|labels|descriptions",+ "languages": "en",+ "format": "json",+ },+ )r2.raise_for_status()entities = r2.json().get("entities", {})except (httpx.HTTPError, ValueError) as e:diff --git a/mwmbl/tokenizer.py b/mwmbl/tokenizer.pyindex 0789de3..7e555fd 100644--- a/mwmbl/tokenizer.py+++ b/mwmbl/tokenizer.py@@ -1,7 +1,7 @@def tokenize(input_text):cleaned_text = clean_unicode(input_text)tokens = cleaned_text.lower().split()- if input_text.endswith('…'):+ if input_text.endswith("…"):# Discard the last two tokens since there will likely be a word cut in twotokens = tokens[:-2]return tokens@@ -9,9 +9,9 @@ def tokenize(input_text):def get_bigrams(num_bigrams, tokens):num_bigrams = min(num_bigrams, len(tokens) - 1)- bigrams = [f'{tokens[i]} {tokens[i + 1]}' for i in range(num_bigrams)]+ bigrams = [f"{tokens[i]} {tokens[i + 1]}" for i in range(num_bigrams)]return bigramsdef clean_unicode(s: str) -> str:- return s.encode('utf-8', errors='ignore').decode('utf-8')+ return s.encode("utf-8", errors="ignore").decode("utf-8")diff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex 6d8cb49..6aa043d 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -14,19 +14,35 @@ Including another URLconf1. Import the include() function: from django.urls import include, path2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))"""++from debug_toolbar.toolbar import debug_toolbar_urlsfrom django.contrib import admin-from django.urls import path, include+from django.urls import include, pathfrom django.views.generic import RedirectView-from debug_toolbar.toolbar import debug_toolbar_urlsimport mwmbl.crawler.app as crawlerfrom mwmbl.admin_views import blacklist_status_view-from mwmbl.api import api as v1_api, v2_api, register_routers-from mwmbl.search_setup import queued_batches, ranker, batch_cache+from mwmbl.api import api as v1_api+from mwmbl.api import register_routers, v2_api+from mwmbl.search_setup import batch_cache, queued_batches, rankerfrom mwmbl.tinysearchengine import search-from mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \- flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_view, CurationsView, submit_domain, \- DomainSubmissionListView, memory_view+from mwmbl.views import (+ CurationDetailView,+ CurationFlagListView,+ CurationsView,+ DomainSubmissionListView,+ add_url,+ approve,+ domain_view,+ domains_view,+ flag_curation,+ flag_curation_update,+ home_fragment,+ index,+ memory_view,+ revert_current_curation,+ submit_domain,+)# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.# This must be called before urlpatterns is evaluated.@@ -34,48 +50,41 @@ register_routers(ranker=ranker, batch_cache=batch_cache, queued_batches=queued_bdef trigger_error(request):- division_by_zero = 1 / 0+ 1 / 0urlpatterns = [# Before admin.site.urls so the admin catch-all does not shadow it. Access is gated by# staff_member_required on the view, matching the rest of the admin.- path('admin/blacklist-status/', blacklist_status_view, name="blacklist_status"),- path('admin/', admin.site.urls),- path('accounts/', include('allauth.urls')),-- path('', index, name="index"),- path('app/home/', home_fragment, name="home"),- path('app/add/', add_url, name="add_url"),- path('app/domain-submissions/', DomainSubmissionListView.as_view(), name="domain_submissions"),- path('app/domain-submissions/new', submit_domain, name="submit_domain"),- path('app/approve/', approve, name="approve"),- path('app/revert-current/', revert_current_curation, name="revert_current"),- path('app/curations/', CurationsView.as_view(), name="curations"),- path('app/curations/<int:pk>/', CurationDetailView.as_view(), name="curation"),- path('app/curations/<int:curation_id>/flag/', flag_curation, name="flag_curation"),- path('app/flags/curation', CurationFlagListView.as_view(), name="flag_curation_list"),- path('app/flags/curation/<int:flag_curation_id>/update', flag_curation_update, name="flag_curation_update"),- path('app/domains/', domains_view, name="domains"),- path('app/domains/<str:domain>/', domain_view, name="domain"),-+ path("admin/blacklist-status/", blacklist_status_view, name="blacklist_status"),+ path("admin/", admin.site.urls),+ path("accounts/", include("allauth.urls")),+ path("", index, name="index"),+ path("app/home/", home_fragment, name="home"),+ path("app/add/", add_url, name="add_url"),+ path("app/domain-submissions/", DomainSubmissionListView.as_view(), name="domain_submissions"),+ path("app/domain-submissions/new", submit_domain, name="submit_domain"),+ path("app/approve/", approve, name="approve"),+ path("app/revert-current/", revert_current_curation, name="revert_current"),+ path("app/curations/", CurationsView.as_view(), name="curations"),+ path("app/curations/<int:pk>/", CurationDetailView.as_view(), name="curation"),+ path("app/curations/<int:curation_id>/flag/", flag_curation, name="flag_curation"),+ path("app/flags/curation", CurationFlagListView.as_view(), name="flag_curation_list"),+ path("app/flags/curation/<int:flag_curation_id>/update", flag_curation_update, name="flag_curation_update"),+ path("app/domains/", domains_view, name="domains"),+ path("app/domains/<str:domain>/", domain_view, name="domain"),# TODO: these are the old APIs, deprecated and to be removed once all clients have moved overpath("search/", search.create_router(ranker, "0.1").urls),path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),-# Unified v1 API — search returns plain list; single docs page at /api/v1/docspath("api/v1/", v1_api.urls),-# v2 API — search returns object with quota metadata; docs at /api/v2/docspath("api/v2/", v2_api.urls),-# Redirects from old per-router docs URLs to the unified docs pagepath("api/v1/search/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),path("api/v1/crawler/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),path("api/v1/platform/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),path("api/v1/evaluate/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),-path("debug/memory", memory_view, name="memory"),-- path('sentry-debug/', trigger_error),+ path("sentry-debug/", trigger_error),] + debug_toolbar_urls()diff --git a/mwmbl/usernames.py b/mwmbl/usernames.pyindex 8421f35..1a796cf 100644--- a/mwmbl/usernames.py+++ b/mwmbl/usernames.py@@ -3,230 +3,1082 @@ import secrets_ADJECTIVES = [# A- "able", "abiding", "agile", "airy", "alabaster", "alert", "alpine", "amber",- "ample", "ancient", "ardent", "arctic", "arid", "ashen", "astral", "astute",- "atomic", "auburn", "august", "avid", "azure",+ "able",+ "abiding",+ "agile",+ "airy",+ "alabaster",+ "alert",+ "alpine",+ "amber",+ "ample",+ "ancient",+ "ardent",+ "arctic",+ "arid",+ "ashen",+ "astral",+ "astute",+ "atomic",+ "auburn",+ "august",+ "avid",+ "azure",# B- "balmy", "bare", "benevolent", "blazed", "blazing", "blithe", "blissful",- "blooming", "blushing", "bold", "boreal", "boundless", "brave", "breezy",- "bright", "brisk", "broad", "bronze", "buoyant", "burnished",+ "balmy",+ "bare",+ "benevolent",+ "blazed",+ "blazing",+ "blithe",+ "blissful",+ "blooming",+ "blushing",+ "bold",+ "boreal",+ "boundless",+ "brave",+ "breezy",+ "bright",+ "brisk",+ "broad",+ "bronze",+ "buoyant",+ "burnished",# C- "calm", "candid", "carefree", "caramel", "cedar", "celestial", "cerulean",- "chatty", "cheerful", "chestnut", "chill", "civic", "clear", "clever",- "clover", "cloudy", "coastal", "cobalt", "comet", "cool", "copper", "coral",- "cordial", "cosmic", "courageous", "cozy", "creative", "crisp", "crystal",- "curly", "cyan",+ "calm",+ "candid",+ "carefree",+ "caramel",+ "cedar",+ "celestial",+ "cerulean",+ "chatty",+ "cheerful",+ "chestnut",+ "chill",+ "civic",+ "clear",+ "clever",+ "clover",+ "cloudy",+ "coastal",+ "cobalt",+ "comet",+ "cool",+ "copper",+ "coral",+ "cordial",+ "cosmic",+ "courageous",+ "cozy",+ "creative",+ "crisp",+ "crystal",+ "curly",+ "cyan",# D- "dainty", "dapper", "daring", "dark", "dashing", "dauntless", "dawn",- "decisive", "deep", "devoted", "dewy", "diligent", "dim", "drifting",- "durable", "dusty", "dynamic",+ "dainty",+ "dapper",+ "daring",+ "dark",+ "dashing",+ "dauntless",+ "dawn",+ "decisive",+ "deep",+ "devoted",+ "dewy",+ "diligent",+ "dim",+ "drifting",+ "durable",+ "dusty",+ "dynamic",# E- "eager", "early", "earnest", "earthy", "easy", "ebony", "effervescent",- "electric", "elegant", "eloquent", "emerald", "empty", "enduring", "endless",- "energetic", "enlightened", "epic", "eternal", "everlasting", "even", "exact",+ "eager",+ "early",+ "earnest",+ "earthy",+ "easy",+ "ebony",+ "effervescent",+ "electric",+ "elegant",+ "eloquent",+ "emerald",+ "empty",+ "enduring",+ "endless",+ "energetic",+ "enlightened",+ "epic",+ "eternal",+ "everlasting",+ "even",+ "exact","exquisite",# F- "fabled", "fair", "faithful", "famous", "fancy", "far", "farsighted", "fast",- "felicitous", "fern", "fervent", "festive", "fierce", "fiery", "fine", "firm",- "first", "flexible", "flint", "floral", "flourishing", "flowing", "foggy",- "fond", "forest", "forward", "fragrant", "frank", "free", "fresh", "frosty",- "frozen", "full",+ "fabled",+ "fair",+ "faithful",+ "famous",+ "fancy",+ "far",+ "farsighted",+ "fast",+ "felicitous",+ "fern",+ "fervent",+ "festive",+ "fierce",+ "fiery",+ "fine",+ "firm",+ "first",+ "flexible",+ "flint",+ "floral",+ "flourishing",+ "flowing",+ "foggy",+ "fond",+ "forest",+ "forward",+ "fragrant",+ "frank",+ "free",+ "fresh",+ "frosty",+ "frozen",+ "full",# G- "gallant", "gentle", "genuine", "gifted", "gilded", "glacial", "glad",- "gleaming", "glorious", "glowing", "golden", "good", "graceful", "gracious",- "grand", "granite", "grateful", "great", "green", "grey", "grounded",+ "gallant",+ "gentle",+ "genuine",+ "gifted",+ "gilded",+ "glacial",+ "glad",+ "gleaming",+ "glorious",+ "glowing",+ "golden",+ "good",+ "graceful",+ "gracious",+ "grand",+ "granite",+ "grateful",+ "great",+ "green",+ "grey",+ "grounded",# H- "halcyon", "hardy", "harmonious", "hazy", "hearty", "heroic", "high",- "hollow", "honest", "hopeful", "humble",+ "halcyon",+ "hardy",+ "harmonious",+ "hazy",+ "hearty",+ "heroic",+ "high",+ "hollow",+ "honest",+ "hopeful",+ "humble",# I- "icy", "idle", "idyllic", "illumined", "imaginative", "immense", "impeccable",- "indigo", "infinite", "ingenious", "inland", "inner", "inspired", "intrepid",- "inventive", "invincible", "iron", "ivory",+ "icy",+ "idle",+ "idyllic",+ "illumined",+ "imaginative",+ "immense",+ "impeccable",+ "indigo",+ "infinite",+ "ingenious",+ "inland",+ "inner",+ "inspired",+ "intrepid",+ "inventive",+ "invincible",+ "iron",+ "ivory",# J- "jade", "jolly", "jovial", "joyful", "joyous", "jubilant", "judicious", "just",+ "jade",+ "jolly",+ "jovial",+ "joyful",+ "joyous",+ "jubilant",+ "judicious",+ "just",# K- "keen", "kind", "kindhearted", "kinetic",+ "keen",+ "kind",+ "kindhearted",+ "kinetic",# L- "lambent", "large", "lasting", "lavender", "leafy", "lean", "legendary",- "lemon", "level", "light", "lively", "lone", "long", "loyal", "lucid",- "lucky", "luminous", "lunar", "lush", "lyric",+ "lambent",+ "large",+ "lasting",+ "lavender",+ "leafy",+ "lean",+ "legendary",+ "lemon",+ "level",+ "light",+ "lively",+ "lone",+ "long",+ "loyal",+ "lucid",+ "lucky",+ "luminous",+ "lunar",+ "lush",+ "lyric",# M- "magic", "magnanimous", "majestic", "maple", "marble", "mature", "measured",- "meditative", "mellifluous", "mellow", "merciful", "methodical", "mighty",- "mild", "mindful", "misty", "modest", "moonlit", "mossy", "mountainous", "muted",+ "magic",+ "magnanimous",+ "majestic",+ "maple",+ "marble",+ "mature",+ "measured",+ "meditative",+ "mellifluous",+ "mellow",+ "merciful",+ "methodical",+ "mighty",+ "mild",+ "mindful",+ "misty",+ "modest",+ "moonlit",+ "mossy",+ "mountainous",+ "muted",# N- "narrow", "natural", "neat", "nimble", "noble", "nocturnal", "nordic",- "north", "nurturing",+ "narrow",+ "natural",+ "neat",+ "nimble",+ "noble",+ "nocturnal",+ "nordic",+ "north",+ "nurturing",# O- "oak", "observant", "obsidian", "oceanic", "olive", "open", "opulent",- "optimistic", "orange", "orderly", "outer",+ "oak",+ "observant",+ "obsidian",+ "oceanic",+ "olive",+ "open",+ "opulent",+ "optimistic",+ "orange",+ "orderly",+ "outer",# P- "pale", "passionate", "patient", "peaceful", "pearly", "perennial",- "persevering", "petal", "pine", "pioneering", "plain", "playful", "poised",- "polar", "polished", "pragmatic", "principled", "promising", "proud", "pure",- "purple", "purposeful",+ "pale",+ "passionate",+ "patient",+ "peaceful",+ "pearly",+ "perennial",+ "persevering",+ "petal",+ "pine",+ "pioneering",+ "plain",+ "playful",+ "poised",+ "polar",+ "polished",+ "pragmatic",+ "principled",+ "promising",+ "proud",+ "pure",+ "purple",+ "purposeful",# Q- "quiet", "quickened", "quintessential",+ "quiet",+ "quickened",+ "quintessential",# R- "radiant", "rapid", "rare", "ready", "receptive", "reflective", "refreshing",- "regal", "reliable", "resilient", "resolute", "resourceful", "respectful",- "responsive", "rich", "righteous", "roaming", "robust", "rooted", "rosy",- "rough", "round", "royal", "ruby", "rugged", "running", "rustic", "rustling",+ "radiant",+ "rapid",+ "rare",+ "ready",+ "receptive",+ "reflective",+ "refreshing",+ "regal",+ "reliable",+ "resilient",+ "resolute",+ "resourceful",+ "respectful",+ "responsive",+ "rich",+ "righteous",+ "roaming",+ "robust",+ "rooted",+ "rosy",+ "rough",+ "round",+ "royal",+ "ruby",+ "rugged",+ "running",+ "rustic",+ "rustling",# S- "sacred", "sage", "sandy", "sapphire", "scholarly", "seasoned", "serene",- "seraphic", "sharp", "shining", "silent", "silken", "silver", "simple",- "sincere", "skillful", "sleek", "slender", "slow", "smart", "smooth", "snowy",- "sociable", "solar", "solemn", "solid", "sonic", "soulful", "sovereign",- "spirited", "steadfast", "sterling", "still", "stoic", "stormy", "stout",- "strategic", "strong", "sturdy", "sublime", "subtle", "sunny", "supernal",- "supple", "supportive", "swift",+ "sacred",+ "sage",+ "sandy",+ "sapphire",+ "scholarly",+ "seasoned",+ "serene",+ "seraphic",+ "sharp",+ "shining",+ "silent",+ "silken",+ "silver",+ "simple",+ "sincere",+ "skillful",+ "sleek",+ "slender",+ "slow",+ "smart",+ "smooth",+ "snowy",+ "sociable",+ "solar",+ "solemn",+ "solid",+ "sonic",+ "soulful",+ "sovereign",+ "spirited",+ "steadfast",+ "sterling",+ "still",+ "stoic",+ "stormy",+ "stout",+ "strategic",+ "strong",+ "sturdy",+ "sublime",+ "subtle",+ "sunny",+ "supernal",+ "supple",+ "supportive",+ "swift",# T- "talented", "tall", "tawny", "tender", "tenacious", "thoughtful", "thriving",- "tidal", "tidy", "timeless", "timely", "tireless", "tranquil", "transparent",- "triumphant", "true", "trusty", "trusting", "twilight",+ "talented",+ "tall",+ "tawny",+ "tender",+ "tenacious",+ "thoughtful",+ "thriving",+ "tidal",+ "tidy",+ "timeless",+ "timely",+ "tireless",+ "tranquil",+ "transparent",+ "triumphant",+ "true",+ "trusty",+ "trusting",+ "twilight",# U- "ultra", "unassuming", "undaunted", "unwavering", "upbeat", "uplifting",- "unyielding", "upright",+ "ultra",+ "unassuming",+ "undaunted",+ "unwavering",+ "upbeat",+ "uplifting",+ "unyielding",+ "upright",# V- "valiant", "vast", "velvet", "venerable", "venturous", "verdant", "vernal",- "versatile", "vibrant", "vigilant", "visionary", "vital", "vivacious", "vivid",+ "valiant",+ "vast",+ "velvet",+ "venerable",+ "venturous",+ "verdant",+ "vernal",+ "versatile",+ "vibrant",+ "vigilant",+ "visionary",+ "vital",+ "vivacious",+ "vivid",# W- "warm", "watchful", "wavy", "whimsical", "wholesome", "willing", "windswept",- "wise", "witty", "wondrous", "worthy", "woven",+ "warm",+ "watchful",+ "wavy",+ "whimsical",+ "wholesome",+ "willing",+ "windswept",+ "wise",+ "witty",+ "wondrous",+ "worthy",+ "woven",# X Y Z- "xenial", "xeric", "yielding", "young", "youthful", "zealous", "zenith",- "zephyr", "zestful", "zippy",+ "xenial",+ "xeric",+ "yielding",+ "young",+ "youthful",+ "zealous",+ "zenith",+ "zephyr",+ "zestful",+ "zippy",# extras to reach 500- "affable", "aloft", "animated", "artful", "balanced", "bountiful", "bracing",- "candent", "capacious", "certain", "clean", "collected", "composed", "confident",- "conscientious", "considerate", "constant", "deft", "deliberate", "dependable",- "dignified", "discerning", "effulgent", "elevated", "eminent", "endeared",- "equable", "established", "expansive", "expressive", "felicific", "forthright",- "fulsome", "generative", "graced", "hallowed", "idiomatic", "imaginable",- "immovable", "incisive", "inquisitive", "insightful", "integral", "intuitive",- "jaunty", "knowing", "limber", "limpid", "lofty", "manifest",- "masterful", "meticulous", "musing", "nascent", "nuanced",- "openhearted", "ordinate", "pacific", "pellucid", "perceptive", "pious",- "plentiful", "precise", "pristine", "progressing",- "reasoned", "refined", "reposed", "revered", "rhapsodic", "sapient",- "sanguine", "settled", "sheer", "sideral", "spiraling",- "stately", "stellar", "temperate", "thorough", "tonal",- "torchlit", "tractable", "transcendent", "unbounded", "unfailing", "unfettered",+ "affable",+ "aloft",+ "animated",+ "artful",+ "balanced",+ "bountiful",+ "bracing",+ "candent",+ "capacious",+ "certain",+ "clean",+ "collected",+ "composed",+ "confident",+ "conscientious",+ "considerate",+ "constant",+ "deft",+ "deliberate",+ "dependable",+ "dignified",+ "discerning",+ "effulgent",+ "elevated",+ "eminent",+ "endeared",+ "equable",+ "established",+ "expansive",+ "expressive",+ "felicific",+ "forthright",+ "fulsome",+ "generative",+ "graced",+ "hallowed",+ "idiomatic",+ "imaginable",+ "immovable",+ "incisive",+ "inquisitive",+ "insightful",+ "integral",+ "intuitive",+ "jaunty",+ "knowing",+ "limber",+ "limpid",+ "lofty",+ "manifest",+ "masterful",+ "meticulous",+ "musing",+ "nascent",+ "nuanced",+ "openhearted",+ "ordinate",+ "pacific",+ "pellucid",+ "perceptive",+ "pious",+ "plentiful",+ "precise",+ "pristine",+ "progressing",+ "reasoned",+ "refined",+ "reposed",+ "revered",+ "rhapsodic",+ "sapient",+ "sanguine",+ "settled",+ "sheer",+ "sideral",+ "spiraling",+ "stately",+ "stellar",+ "temperate",+ "thorough",+ "tonal",+ "torchlit",+ "tractable",+ "transcendent",+ "unbounded",+ "unfailing",+ "unfettered",]_NOUNS = [# A- "adder", "alder", "algae", "aloe", "anchor", "antler", "anvil", "apex",- "apple", "arch", "arrow", "ash", "aspen", "atlas", "atoll", "aurora",- "autumn", "avalanche", "axe",+ "adder",+ "alder",+ "algae",+ "aloe",+ "anchor",+ "antler",+ "anvil",+ "apex",+ "apple",+ "arch",+ "arrow",+ "ash",+ "aspen",+ "atlas",+ "atoll",+ "aurora",+ "autumn",+ "avalanche",+ "axe",# B- "badger", "bark", "barn", "basalt", "basin", "bay", "beacon", "bear",- "beaver", "beech", "bell", "berry", "birch", "bison", "blade", "bloom",- "blossom", "bluebell", "bluff", "boar", "bog", "boulder", "bough", "bramble",- "brant", "bream", "brine", "brook", "buck", "bud", "burrow", "buttercup",+ "badger",+ "bark",+ "barn",+ "basalt",+ "basin",+ "bay",+ "beacon",+ "bear",+ "beaver",+ "beech",+ "bell",+ "berry",+ "birch",+ "bison",+ "blade",+ "bloom",+ "blossom",+ "bluebell",+ "bluff",+ "boar",+ "bog",+ "boulder",+ "bough",+ "bramble",+ "brant",+ "bream",+ "brine",+ "brook",+ "buck",+ "bud",+ "burrow",+ "buttercup","butte",# C- "canoe", "canopy", "canyon", "cape", "cardinal", "carp", "catfish", "cave",- "cedar", "channel", "charcoal", "chestnut", "chisel", "chough", "cicada",- "cinder", "cirque", "cirrus", "clam", "cliff", "clover", "cobble", "cobalt",- "cockle", "col", "comet", "compass", "condor", "coral", "cormorant", "corrie",- "cove", "coyote", "crane", "crater", "creek", "crest", "crocus", "crow",- "crystal", "curlew", "current", "cypress",+ "canoe",+ "canopy",+ "canyon",+ "cape",+ "cardinal",+ "carp",+ "catfish",+ "cave",+ "cedar",+ "channel",+ "charcoal",+ "chestnut",+ "chisel",+ "chough",+ "cicada",+ "cinder",+ "cirque",+ "cirrus",+ "clam",+ "cliff",+ "clover",+ "cobble",+ "cobalt",+ "cockle",+ "col",+ "comet",+ "compass",+ "condor",+ "coral",+ "cormorant",+ "corrie",+ "cove",+ "coyote",+ "crane",+ "crater",+ "creek",+ "crest",+ "crocus",+ "crow",+ "crystal",+ "curlew",+ "current",+ "cypress",# D- "dale", "darter", "dawn", "deer", "delta", "dew", "dipper", "dock",- "dolphin", "dormouse", "dotterel", "dove", "dragonfly", "drumlin", "dunlin",- "dune", "dyke",+ "dale",+ "darter",+ "dawn",+ "deer",+ "delta",+ "dew",+ "dipper",+ "dock",+ "dolphin",+ "dormouse",+ "dotterel",+ "dove",+ "dragonfly",+ "drumlin",+ "dunlin",+ "dune",+ "dyke",# E- "eagle", "echo", "eel", "egret", "elder", "elk", "elm", "ember",- "escarpment", "estuary",+ "eagle",+ "echo",+ "eel",+ "egret",+ "elder",+ "elk",+ "elm",+ "ember",+ "escarpment",+ "estuary",# F- "falcon", "fawn", "fern", "field", "fiord", "firecrest", "finch", "fir",- "fjord", "flame", "flint", "flood", "flora", "fluke", "flycatcher", "foam",- "fog", "forest", "forge", "fossil", "fox", "frost", "fulmar",+ "falcon",+ "fawn",+ "fern",+ "field",+ "fiord",+ "firecrest",+ "finch",+ "fir",+ "fjord",+ "flame",+ "flint",+ "flood",+ "flora",+ "fluke",+ "flycatcher",+ "foam",+ "fog",+ "forest",+ "forge",+ "fossil",+ "fox",+ "frost",+ "fulmar",# G- "gale", "gannet", "garganey", "garnet", "geyser", "glacier", "glade",- "glen", "godwit", "goldcrest", "goldfinch", "goosander", "gorge", "goshawk",- "granite", "gravel", "grove", "grouse", "guillemot", "gulch", "gull", "gust",+ "gale",+ "gannet",+ "garganey",+ "garnet",+ "geyser",+ "glacier",+ "glade",+ "glen",+ "godwit",+ "goldcrest",+ "goldfinch",+ "goosander",+ "gorge",+ "goshawk",+ "granite",+ "gravel",+ "grove",+ "grouse",+ "guillemot",+ "gulch",+ "gull",+ "gust",# H- "hamster", "hare", "harrier", "hartebeest", "hawk", "hazel", "heath",- "hedgehog", "heron", "hill", "hillside", "holly", "hoopoe", "horizon",+ "hamster",+ "hare",+ "harrier",+ "hartebeest",+ "hawk",+ "hazel",+ "heath",+ "hedgehog",+ "heron",+ "hill",+ "hillside",+ "holly",+ "hoopoe",+ "horizon","hornet",# I- "ibex", "ibis", "inlet", "iris", "island", "ivory",+ "ibex",+ "ibis",+ "inlet",+ "iris",+ "island",+ "ivory",# J- "jackal", "jackdaw", "jasper", "jay", "juniper",+ "jackal",+ "jackdaw",+ "jasper",+ "jay",+ "juniper",# K- "kelp", "kestrel", "kingfisher", "kite", "kiwi", "knoll", "knot",+ "kelp",+ "kestrel",+ "kingfisher",+ "kite",+ "kiwi",+ "knoll",+ "knot",# L- "lagoon", "lake", "lapwing", "larch", "lark", "lava", "leaf", "ledge",- "lemur", "leveret", "lichen", "limestone", "linden", "linnet", "lion",- "loam", "loess", "locust", "lodge", "loon", "lotus", "lynx",+ "lagoon",+ "lake",+ "lapwing",+ "larch",+ "lark",+ "lava",+ "leaf",+ "ledge",+ "lemur",+ "leveret",+ "lichen",+ "limestone",+ "linden",+ "linnet",+ "lion",+ "loam",+ "loess",+ "locust",+ "lodge",+ "loon",+ "lotus",+ "lynx",# M- "mackerel", "magpie", "mallard", "maple", "marl", "marsh", "martin",- "meadow", "merganser", "merlin", "mesa", "midstream", "milfoil", "mink",- "mist", "moorland", "moose", "moss", "moth", "mountain", "mudflat",- "mudstone", "muskrat",+ "mackerel",+ "magpie",+ "mallard",+ "maple",+ "marl",+ "marsh",+ "martin",+ "meadow",+ "merganser",+ "merlin",+ "mesa",+ "midstream",+ "milfoil",+ "mink",+ "mist",+ "moorland",+ "moose",+ "moss",+ "moth",+ "mountain",+ "mudflat",+ "mudstone",+ "muskrat",# N- "narwhal", "needle", "newt", "nightingale", "nightjar", "noctule", "north",- "nutmeg", "nuthatch",+ "narwhal",+ "needle",+ "newt",+ "nightingale",+ "nightjar",+ "noctule",+ "north",+ "nutmeg",+ "nuthatch",# O- "oak", "opal", "orca", "osprey", "otter", "outcrop", "owl", "oyster",+ "oak",+ "opal",+ "orca",+ "osprey",+ "otter",+ "outcrop",+ "owl",+ "oyster","oystercatcher",# P- "partridge", "pass", "pasture", "peak", "peat", "pebble", "peewit",- "pelican", "pennant", "perch", "peregrine", "petal", "pheasant", "pigeon",- "pine", "pintail", "pipit", "plover", "plume", "pochard", "polecat",- "pond", "poplar", "porcupine", "prairie", "ptarmigan", "puffin",+ "partridge",+ "pass",+ "pasture",+ "peak",+ "peat",+ "pebble",+ "peewit",+ "pelican",+ "pennant",+ "perch",+ "peregrine",+ "petal",+ "pheasant",+ "pigeon",+ "pine",+ "pintail",+ "pipit",+ "plover",+ "plume",+ "pochard",+ "polecat",+ "pond",+ "poplar",+ "porcupine",+ "prairie",+ "ptarmigan",+ "puffin",# Q- "quartz", "quill",+ "quartz",+ "quill",# R- "rabbit", "raven", "reed", "reedbed", "reef", "ridge", "rift", "rill",- "ringlet", "ripple", "robin", "rock", "rockpool", "roe", "roost",- "rosefinch", "root", "rowan", "runnel", "rush", "rushes",+ "rabbit",+ "raven",+ "reed",+ "reedbed",+ "reef",+ "ridge",+ "rift",+ "rill",+ "ringlet",+ "ripple",+ "robin",+ "rock",+ "rockpool",+ "roe",+ "roost",+ "rosefinch",+ "root",+ "rowan",+ "runnel",+ "rush",+ "rushes",# S- "sage", "salmon", "sandbar", "sandpiper", "sandstone", "sapphire", "scree",- "seabird", "seagrass", "sedge", "serin", "sett", "shale", "shearwater",- "shoal", "shore", "shoveler", "shrew", "shrike", "sierra", "siskin",- "skimmer", "skua", "skylark", "slate", "sluice", "smew", "snipe",- "snowdrop", "snowfield", "snout", "sora", "sorrel", "sparrow", "spire",- "spoonbill", "spruce", "squall", "squirrel", "stag", "starling", "stem",- "steppe", "stoat", "stone", "stonefly", "stork", "storm", "stream",- "stubble", "summit", "sundew", "sunfish", "swallow", "swamp", "swan",- "swift", "sycamore",+ "sage",+ "salmon",+ "sandbar",+ "sandpiper",+ "sandstone",+ "sapphire",+ "scree",+ "seabird",+ "seagrass",+ "sedge",+ "serin",+ "sett",+ "shale",+ "shearwater",+ "shoal",+ "shore",+ "shoveler",+ "shrew",+ "shrike",+ "sierra",+ "siskin",+ "skimmer",+ "skua",+ "skylark",+ "slate",+ "sluice",+ "smew",+ "snipe",+ "snowdrop",+ "snowfield",+ "snout",+ "sora",+ "sorrel",+ "sparrow",+ "spire",+ "spoonbill",+ "spruce",+ "squall",+ "squirrel",+ "stag",+ "starling",+ "stem",+ "steppe",+ "stoat",+ "stone",+ "stonefly",+ "stork",+ "storm",+ "stream",+ "stubble",+ "summit",+ "sundew",+ "sunfish",+ "swallow",+ "swamp",+ "swan",+ "swift",+ "sycamore",# T- "talon", "tanager", "tarpon", "teal", "tern", "thistle", "thorn",- "thornbill", "thicket", "thrush", "tide", "timber", "titlark", "toad",- "torchwood", "torrent", "trail", "treecreeper", "trogon", "trout", "tuatara",- "tumulus", "tundra", "turnstone", "turtle", "twite",+ "talon",+ "tanager",+ "tarpon",+ "teal",+ "tern",+ "thistle",+ "thorn",+ "thornbill",+ "thicket",+ "thrush",+ "tide",+ "timber",+ "titlark",+ "toad",+ "torchwood",+ "torrent",+ "trail",+ "treecreeper",+ "trogon",+ "trout",+ "tuatara",+ "tumulus",+ "tundra",+ "turnstone",+ "turtle",+ "twite",# U"upland",# V- "vale", "valley", "veery", "verdure", "vetch", "viburnum", "vireo",- "viper", "vixen", "vole",+ "vale",+ "valley",+ "veery",+ "verdure",+ "vetch",+ "viburnum",+ "vireo",+ "viper",+ "vixen",+ "vole",# W- "wader", "wagtail", "warbler", "waterfowl", "watershed", "wave", "waxwing",- "weasel", "weir", "wheatear", "whinchat", "whitethroat", "wigeon", "wildcat",- "willow", "wolf", "woodcock", "woodland", "woodpecker", "wormwood", "wren",+ "wader",+ "wagtail",+ "warbler",+ "waterfowl",+ "watershed",+ "wave",+ "waxwing",+ "weasel",+ "weir",+ "wheatear",+ "whinchat",+ "whitethroat",+ "wigeon",+ "wildcat",+ "willow",+ "wolf",+ "woodcock",+ "woodland",+ "woodpecker",+ "wormwood",+ "wren","wryneck",# Y Z- "yarrow", "yellowhammer", "yew", "zander", "zinnia", "zorilla",+ "yarrow",+ "yellowhammer",+ "yew",+ "zander",+ "zinnia",+ "zorilla",# extras to reach 500- "acorn", "agate", "albatross", "alcove", "almond", "alp", "amaranth",- "amethyst", "anemone", "antelope", "arroyo", "azalea", "barnacle", "bayou",- "blizzard", "bracken", "brushwood", "buffalo",- "bulrush", "bunting", "bushtit", "buzzard", "capercaillie", "caribou",- "catkin", "chaffinch", "chalcedony", "chamomile", "cheetah", "cinnabar",- "cistus", "clough", "coho", "columbine", "coppice", "crag", "crake",- "crossbill", "crowberry", "dace", "daisy", "damselfly", "dandelion",- "dewdrop", "dhole", "dingo", "dunock", "earthworm", "echidna", "ermine",- "esker", "fen", "flounder", "flyway", "forb", "gallinule", "garfish",- "gecko", "geode", "gerbil", "gibbon", "ginkgo", "gloaming", "glowworm",- "gneiss", "gnu",+ "acorn",+ "agate",+ "albatross",+ "alcove",+ "almond",+ "alp",+ "amaranth",+ "amethyst",+ "anemone",+ "antelope",+ "arroyo",+ "azalea",+ "barnacle",+ "bayou",+ "blizzard",+ "bracken",+ "brushwood",+ "buffalo",+ "bulrush",+ "bunting",+ "bushtit",+ "buzzard",+ "capercaillie",+ "caribou",+ "catkin",+ "chaffinch",+ "chalcedony",+ "chamomile",+ "cheetah",+ "cinnabar",+ "cistus",+ "clough",+ "coho",+ "columbine",+ "coppice",+ "crag",+ "crake",+ "crossbill",+ "crowberry",+ "dace",+ "daisy",+ "damselfly",+ "dandelion",+ "dewdrop",+ "dhole",+ "dingo",+ "dunock",+ "earthworm",+ "echidna",+ "ermine",+ "esker",+ "fen",+ "flounder",+ "flyway",+ "forb",+ "gallinule",+ "garfish",+ "gecko",+ "geode",+ "gerbil",+ "gibbon",+ "ginkgo",+ "gloaming",+ "glowworm",+ "gneiss",+ "gnu",]def generate_username():from mwmbl.models import MwmblUser+for _ in range(10):candidate = f"{random.choice(_ADJECTIVES)}_{random.choice(_NOUNS)}_{random.randint(100, 999)}"if not MwmblUser.objects.filter(username=candidate).exists():diff --git a/mwmbl/utils.py b/mwmbl/utils.pyindex 801b843..43f0ecb 100644--- a/mwmbl/utils.py+++ b/mwmbl/utils.py@@ -1,14 +1,11 @@-import osimport refrom dataclasses import dataclassfrom datetime import timedelta-from typing import Sequence, Optional+from typing import Optional, Sequencefrom django.conf import settingsfrom django.core.exceptions import ValidationError-from redis import Redis-from requests_cache import CachedSession, RedisCache-from requests.adapters import Retry, HTTPAdapter+from requests_cache import CachedSessionfrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex@@ -22,7 +19,7 @@ def batch(items: Sequence, batch_size):"""length = len(items)for ndx in range(0, length, batch_size):- yield items[ndx:min(ndx + batch_size, length)]+ yield items[ndx : min(ndx + batch_size, length)]def get_domain(url):diff --git a/mwmbl/views.py b/mwmbl/views.pyindex 8706101..48b78d2 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -5,33 +5,45 @@ from logging import getLoggerfrom typing import Optionalfrom urllib.parse import urlencode-from mwmbl import justextimport objgraphimport psutilimport requestsfrom django.conf import settingsfrom django.contrib.auth.decorators import login_required, permission_requiredfrom django.contrib.auth.mixins import LoginRequiredMixin-from django.forms import ModelForm, RadioSelect, CharField+from django.forms import CharField, ModelForm, RadioSelectfrom django.http import HttpResponseBadRequest-from django.shortcuts import render, redirect+from django.shortcuts import redirect, renderfrom django.views.decorators.http import require_http_methodsfrom django.views.generic import DetailView, ListView-from mwmbl.justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revise_paragraph_classification, \- LENGTH_LOW_DEFAULT, STOPWORDS_LOW_DEFAULT, MAX_LINK_DENSITY_DEFAULT, NO_HEADINGS_DEFAULT, LENGTH_HIGH_DEFAULT, \- STOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessorfrom requests.exceptions import RequestExceptionfrom mwmbl.crawler.app import stats_managerfrom mwmbl.crawler.ssrf import UnsafeURLError, validate_url+from mwmbl.justext.core import (+ DEFAULT_ENC_ERRORS,+ DEFAULT_ENCODING,+ LENGTH_HIGH_DEFAULT,+ LENGTH_LOW_DEFAULT,+ MAX_HEADING_DISTANCE_DEFAULT,+ MAX_LINK_DENSITY_DEFAULT,+ NO_HEADINGS_DEFAULT,+ STOPWORDS_HIGH_DEFAULT,+ STOPWORDS_LOW_DEFAULT,+ ParagraphMaker,+ classify_paragraphs,+ html_to_dom,+ preprocessor,+ revise_paragraph_classification,+)from mwmbl.justext.utils import get_stoplist-from mwmbl.models import Curation, FlagCuration, DomainSubmission-from mwmbl.search_setup import ranker, index_path+from mwmbl.models import Curation, DomainSubmission, FlagCuration+from mwmbl.search_setup import index_path, rankerfrom mwmbl.settings import NUM_EXTRACT_CHARSfrom mwmbl.tinysearchengine.indexer import Document, DocumentState, TinyIndexfrom mwmbl.tinysearchengine.rank import fix_document_statefrom mwmbl.tokenizer import tokenize-from mwmbl.utils import add_term_infos, normalize_domain, parse_url, validate_domain, float_or_none+from mwmbl.utils import add_term_infos, float_or_none, normalize_domain, parse_url, validate_domainMAX_CURATED_SCORE = 1_111_111.0@@ -39,12 +51,20 @@ MAX_CURATED_SCORE = 1_111_111.0logger = getLogger(__name__)-def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,- length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,- stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,- max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT, no_headings=NO_HEADINGS_DEFAULT,- encoding=None, default_encoding=DEFAULT_ENCODING,- enc_errors=DEFAULT_ENC_ERRORS):+def justext_with_dom(+ html_text,+ stoplist,+ length_low=LENGTH_LOW_DEFAULT,+ length_high=LENGTH_HIGH_DEFAULT,+ stopwords_low=STOPWORDS_LOW_DEFAULT,+ stopwords_high=STOPWORDS_HIGH_DEFAULT,+ max_link_density=MAX_LINK_DENSITY_DEFAULT,+ max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT,+ no_headings=NO_HEADINGS_DEFAULT,+ encoding=None,+ default_encoding=DEFAULT_ENCODING,+ enc_errors=DEFAULT_ENC_ERRORS,+):"""Converts an HTML page into a list of classified paragraphs. Each paragraphis represented as instance of class ˙˙justext.paragraph.Paragraph˙˙.@@ -58,8 +78,9 @@ def justext_with_dom(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,paragraphs = ParagraphMaker.make_paragraphs(dom)- classify_paragraphs(paragraphs, stoplist, length_low, length_high,- stopwords_low, stopwords_high, max_link_density, no_headings)+ classify_paragraphs(+ paragraphs, stoplist, length_low, length_high, stopwords_low, stopwords_high, max_link_density, no_headings+ )revise_paragraph_classification(paragraphs, max_heading_distance)return paragraphs, title@@ -79,13 +100,17 @@ def _prepare_results(results: Optional[list[Document]]) -> Optional[dict[str, lidef index(request):activity, query, results = _get_results_and_activity(request, use_external_search=True)- return render(request, "index.html", {- "results": _prepare_results(results),- "query": query,- "user": request.user,- "activity": activity,- "footer_links": settings.FOOTER_LINKS,- })+ return render(+ request,+ "index.html",+ {+ "results": _prepare_results(results),+ "query": query,+ "user": request.user,+ "activity": activity,+ "footer_links": settings.FOOTER_LINKS,+ },+ )def home_fragment(request):@@ -95,11 +120,15 @@ def home_fragment(request):# query cost us a dozen Wikipedia calls to answer one question - see mwmbl.templates.use_external_search = request.GET.get("external") == "1"activity, query, results = _get_results_and_activity(request, use_external_search)- response = render(request, "home.html", {- "results": _prepare_results(results),- "query": query,- "activity": activity,- })+ response = render(+ request,+ "home.html",+ {+ "results": _prepare_results(results),+ "query": query,+ "activity": activity,+ },+ )# Encode the new query stringif query:@@ -117,24 +146,30 @@ def _get_results_and_activity(request, use_external_search: bool):# There may be extra results in the request that we need to add in# format is ?enhanced=google&title=title1&url=url1&extract=extract1&title=title2&url=url2&extract=extract2# source = request.GET.get("enhanced", "unknown")- titles = request.GET.getlist(f"title")- urls = request.GET.getlist(f"url")- extracts = request.GET.getlist(f"extract")+ titles = request.GET.getlist("title")+ urls = request.GET.getlist("url")+ extracts = request.GET.getlist("extract")term = " ".join(tokenize(query))# For now, we only support the Google sourceadditional_results = [- Document(title=title, url=url, extract=extract, score=100.0 * 2 ** -i, term=term, state=DocumentState.FROM_GOOGLE)+ Document(+ title=title, url=url, extract=extract, score=100.0 * 2**-i, term=term, state=DocumentState.FROM_GOOGLE+ )for i, (title, url, extract) in enumerate(zip(titles, urls, extracts))]- results = ranker.search(query, additional_results=additional_results,- use_external_search=use_external_search)+ results = ranker.search(query, additional_results=additional_results, use_external_search=use_external_search)activity = Noneelse:results = None- activity = Curation.objects.filter(flag_curation_set__isnull=True).order_by("-timestamp")[:8].select_related("user").prefetch_related("flag_curation_set")+ activity = (+ Curation.objects.filter(flag_curation_set__isnull=True)+ .order_by("-timestamp")[:8]+ .select_related("user")+ .prefetch_related("flag_curation_set")+ )return activity, query, results@@ -156,14 +191,16 @@ def add_url(request):print("Content", response.encoding)paragraphs, title = justext_with_dom(response.text, get_stoplist("English"))- good_paragraphs = [p for p in paragraphs if p.class_type == 'good']+ good_paragraphs = [p for p in paragraphs if p.class_type == "good"]- extract = ' '.join([p.text for p in good_paragraphs])+ extract = " ".join([p.text for p in good_paragraphs])if len(extract) > NUM_EXTRACT_CHARS:- extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'+ extract = extract[: NUM_EXTRACT_CHARS - 1] + "…"term = " ".join(tokenize(query))- result = Document(title=title, url=new_url, extract=extract, score=0.0, term=term, state=DocumentState.FROM_USER_APPROVED)+ result = Document(+ title=title, url=new_url, extract=extract, score=0.0, term=term, state=DocumentState.FROM_USER_APPROVED+ )documents = _get_documents(request, term)reranked_documents = _insert_document(documents, result)@@ -171,12 +208,16 @@ def add_url(request):_save_to_index(query, reranked_documents)- return render(request, "home.html", {- "results": _prepare_results(reranked_documents),- "query": query,- "activity": None,- "curation": curation,- })+ return render(+ request,+ "home.html",+ {+ "results": _prepare_results(reranked_documents),+ "query": query,+ "activity": None,+ "curation": curation,+ },+ )class DomainSubmissionForm(ModelForm):@@ -270,12 +311,16 @@ def approve(request):_save_to_index(query, reranked_documents)- response = render(request, "home.html", {- "results": _prepare_results(reranked_documents),- "query": query,- "activity": None,- "curation": curation,- })+ response = render(+ request,+ "home.html",+ {+ "results": _prepare_results(reranked_documents),+ "query": query,+ "activity": None,+ "curation": curation,+ },+ )return response@@ -292,16 +337,20 @@ def revert_current_curation(request):original_documents_unfixed = [Document(**doc) for doc in curation.original_results]original_documents = [fix_document_state(doc) for doc in original_documents_unfixed]- return render(request, "home.html", {- "results": _prepare_results(original_documents),- "query": (curation.query),- "activity": None,- "curation": None,- })+ return render(+ request,+ "home.html",+ {+ "results": _prepare_results(original_documents),+ "query": (curation.query),+ "activity": None,+ "curation": None,+ },+ )def _revert_curation(curation):- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:term = " ".join(tokenize(curation.query))documents = [Document(**doc) for doc in curation.original_index_results]@@ -310,8 +359,7 @@ def _revert_curation(curation):# Replace all existing documents for the term with the original documentsall_documents = documents + [doc for doc in page.documents if doc.term != term]num_stored = page.store(all_documents)- logger.info(f"Reverted to {num_stored} of {len(all_documents)} documents "- f"at page {page_index}")+ logger.info(f"Reverted to {num_stored} of {len(all_documents)} documents at page {page_index}")def _get_curation(request, query, documents, reranked_documents):@@ -327,7 +375,7 @@ def _get_curation(request, query, documents, reranked_documents):if not user.is_authenticated:user = None- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:tokens = tokenize(query)term = " ".join(tokens)original_index_results = [doc for doc in indexer.retrieve(term) if doc.term == term]@@ -376,12 +424,13 @@ def _get_documents(request, term: str):except ValueError:state_enum = Nonedocuments[url] = Document(- title=title, url=url, extract=extract, score=float_or_none(score), term=term, state=state_enum)+ title=title, url=url, extract=extract, score=float_or_none(score), term=term, state=state_enum+ )return documentsdef _save_to_index(query: str, new_results: list[Document]):- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:term = " ".join(tokenize(query))documents = [Document(@@ -402,8 +451,10 @@ def _save_to_index(query: str, new_results: list[Document]):existing_documents = add_term_infos(page.documents, indexer, page_index)new_urls = {doc.url for doc in documents}other_documents = [doc for doc in existing_documents if doc.url not in new_urls]- logger.info(f"Found {len(other_documents)} other documents for term {term} at page {page_index} "- f"with terms { {doc.term for doc in other_documents} }")+ logger.info(+ f"Found {len(other_documents)} other documents for term {term} at page {page_index} "+ f"with terms { {doc.term for doc in other_documents} }"+ )# Update state for other documentsstates = {doc.url: doc.state for doc in new_results}@@ -439,7 +490,9 @@ class CurationsView(ListView):template_name = "mwmbl/curations.html"def get_queryset(self):- return Curation.objects.select_related("user").prefetch_related('flag_curation_set').all().order_by("-timestamp")+ return (+ Curation.objects.select_related("user").prefetch_related("flag_curation_set").all().order_by("-timestamp")+ )class CurationDetailView(DetailView):@@ -544,8 +597,12 @@ def memory_view(request):memory_info = process.memory_info()most_common_objects = objgraph.most_common_types(limit=100)growth = objgraph.growth(limit=100)- return render(request, "mwmbl/memory.html", {- "most_common_objects": most_common_objects,- "growth": growth,- "memory_info": memory_info,- })+ return render(+ request,+ "mwmbl/memory.html",+ {+ "most_common_objects": most_common_objects,+ "growth": growth,+ "memory_info": memory_info,+ },+ )diff --git a/mwmbl/wsgi.py b/mwmbl/wsgi.pyindex ebdf0ff..b136837 100644--- a/mwmbl/wsgi.py+++ b/mwmbl/wsgi.py@@ -11,6 +11,6 @@ import osfrom django.core.wsgi import get_wsgi_application-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mwmbl.settings_dev')+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")application = get_wsgi_application()diff --git a/scripts/_p3_make_wave.py b/scripts/_p3_make_wave.pyindex 2645f6e..6a9cb3c 100644--- a/scripts/_p3_make_wave.py+++ b/scripts/_p3_make_wave.py@@ -9,12 +9,18 @@ Usage::DJANGO_SETTINGS_MODULE=... uv run python scripts/_p3_make_wave.py \--tag w1 --n-queries 32 --chunk-queries 4 --outdir /path/to/scratch"""+import jsonimport osfrom argparse import ArgumentParserfrom llm_relabel_pass3_judge import (- JUDGE_PROMPT, EXTRACT_CHARS, MANIFEST, _pool, _intents, _done_pairs,+ EXTRACT_CHARS,+ JUDGE_PROMPT,+ MANIFEST,+ _done_pairs,+ _intents,+ _pool,)@@ -27,9 +33,8 @@ def main():a = p.parse_args()pool, intents, done = _pool(), _intents(), _done_pairs()- todo = [q for q, r in pool.items()- if any((q, c["url"]) not in done for c in r["candidates"])]- batch = todo[:a.n_queries]+ todo = [q for q, r in pool.items() if any((q, c["url"]) not in done for c in r["candidates"])]+ batch = todo[: a.n_queries]manifest, blocks, cid = [], [], 0for q in batch:@@ -52,7 +57,7 @@ def main():os.makedirs(a.outdir, exist_ok=True)n = 0for i in range(0, len(blocks), a.chunk_queries):- chunk = blocks[i:i + a.chunk_queries]+ chunk = blocks[i : i + a.chunk_queries]path = f"{a.outdir}/p3_{a.tag}_chunk{i // a.chunk_queries}.txt"with open(path, "w") as f:f.write(JUDGE_PROMPT + "\n".join(chunk))diff --git a/scripts/augment_matrix_intent.py b/scripts/augment_matrix_intent.pyindex afac2d5..a6282ee 100644--- a/scripts/augment_matrix_intent.py+++ b/scripts/augment_matrix_intent.py@@ -12,6 +12,7 @@ Usage:scripts/augment_matrix_intent.py --in devdata/ss_eval_matrix \--out devdata/ss_eval_matrix_intent"""+import argparseimport osimport sys@@ -20,12 +21,16 @@ from pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parent.parent))os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")import django # noqa: E402+django.setup()import numpy as np # noqa: E402+from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix # noqa: E402from mwmbl.tinysearchengine.super_search_select.features import ( # noqa: E402- FEATURE_NAMES, INTENT_NAMES, classify_intent,+ FEATURE_NAMES,+ INTENT_NAMES,+ classify_intent,)@@ -40,7 +45,8 @@ def main():intent_names = [f"intent_{n}" for n in INTENT_NAMES]assert m.feature_names == FEATURE_NAMES[:F], (f"matrix feature_names {m.feature_names} are not the prefix of the current "- f"FEATURE_NAMES; refusing to append (rebuild via build-matrix instead).")+ f"FEATURE_NAMES; refusing to append (rebuild via build-matrix instead)."+ )assert FEATURE_NAMES[F:] == intent_names, "intent block must be the appended tail"# (Q, n_intents) one-hot, then broadcast across the S source axis.@@ -48,9 +54,9 @@ def main():intent_block = np.broadcast_to(per_query[:, None, :], (Q, S, len(INTENT_NAMES)))X_aug = np.concatenate([m.X, intent_block], axis=2)- aug = RewardMatrix(queries=m.queries, sources=m.sources,- feature_names=list(FEATURE_NAMES),- X=X_aug, R=m.R, mask=m.mask)+ aug = RewardMatrix(+ queries=m.queries, sources=m.sources, feature_names=list(FEATURE_NAMES), X=X_aug, R=m.R, mask=m.mask+ )aug.save(args.out)fired = per_query.sum(axis=0)diff --git a/scripts/auto_recipe.py b/scripts/auto_recipe.pyindex 92f49ad..470a065 100644--- a/scripts/auto_recipe.py+++ b/scripts/auto_recipe.py@@ -26,6 +26,7 @@ Usage:recipe_chunks/input_000.json recipe_chunks/input_001.json ...DATABASE_URL="postgres://daoud@" uv run python scripts/auto_recipe.py --all"""+from __future__ import annotationsimport argparse@@ -39,13 +40,13 @@ REPO_ROOT = Path(__file__).resolve().parent.parentsys.path.insert(0, str(REPO_ROOT))os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")-import django # noqa: E402import logging # noqa: E402+import django # noqa: E402+django.setup()logging.getLogger("httpx").setLevel(logging.WARNING)-logging.getLogger("mwmbl.tinysearchengine.super_search_sources.recipe").setLevel(- logging.ERROR)+logging.getLogger("mwmbl.tinysearchengine.super_search_sources.recipe").setLevel(logging.ERROR)import httpx # noqa: E402import yaml # noqa: E402@@ -107,46 +108,65 @@ def _templates(domain: str) -> list[tuple[str, dict, dict]]:return [("wp-search",- {"url": f"{base}/wp-json/wp/v2/search",- "params": {"search": "{query}", "per_page": "{limit}"}},- {"format": "json", "results": "",- "fields": {"title": "title", "url": "url"}},+ {"url": f"{base}/wp-json/wp/v2/search", "params": {"search": "{query}", "per_page": "{limit}"}},+ {"format": "json", "results": "", "fields": {"title": "title", "url": "url"}},),("wp-posts",- {"url": f"{base}/wp-json/wp/v2/posts",- "params": {"search": "{query}", "per_page": "{limit}"}},- {"format": "json", "results": "",- "fields": {"title": "title.rendered", "extract": "excerpt.rendered",- "url": "link"},- "strip_html": ["title", "extract"]},+ {"url": f"{base}/wp-json/wp/v2/posts", "params": {"search": "{query}", "per_page": "{limit}"}},+ {+ "format": "json",+ "results": "",+ "fields": {"title": "title.rendered", "extract": "excerpt.rendered", "url": "link"},+ "strip_html": ["title", "extract"],+ },),("mediawiki",- {"url": f"{base}/w/api.php",- "params": {"action": "query", "list": "search", "format": "json",- "srsearch": "{query}", "srlimit": "{limit}"}},- {"format": "json", "results": "query.search",- "fields": {"title": "title", "extract": "snippet",- "url": {"template": f"{base}/wiki/{{title}}"}},- "strip_html": ["extract"]},+ {+ "url": f"{base}/w/api.php",+ "params": {+ "action": "query",+ "list": "search",+ "format": "json",+ "srsearch": "{query}",+ "srlimit": "{limit}",+ },+ },+ {+ "format": "json",+ "results": "query.search",+ "fields": {"title": "title", "extract": "snippet", "url": {"template": f"{base}/wiki/{{title}}"}},+ "strip_html": ["extract"],+ },),("mediawiki-root",- {"url": f"{base}/api.php",- "params": {"action": "query", "list": "search", "format": "json",- "srsearch": "{query}", "srlimit": "{limit}"}},- {"format": "json", "results": "query.search",- "fields": {"title": "title", "extract": "snippet",- "url": {"template": f"{base}/wiki/{{title}}"}},- "strip_html": ["extract"]},+ {+ "url": f"{base}/api.php",+ "params": {+ "action": "query",+ "list": "search",+ "format": "json",+ "srsearch": "{query}",+ "srlimit": "{limit}",+ },+ },+ {+ "format": "json",+ "results": "query.search",+ "fields": {"title": "title", "extract": "snippet", "url": {"template": f"{base}/wiki/{{title}}"}},+ "strip_html": ["extract"],+ },),("discourse",{"url": f"{base}/search.json", "params": {"q": "{query}"}},- {"format": "json", "results": "topics",- "fields": {"title": "title",- "url": {"template": f"{base}/t/{{slug}}/{{id}}"}}},+ {+ "format": "json",+ "results": "topics",+ "fields": {"title": "title", "url": {"template": f"{base}/t/{{slug}}/{{id}}"}},+ },),]@@ -160,8 +180,7 @@ async def _probe_domain(client: httpx.AsyncClient, entry: dict) -> dict:words = _probe_words(field)for label, request, response in _templates(domain):- recipe = Recipe(name=name, request=request, response=response,- domain=domain, field=field)+ recipe = Recipe(name=name, request=request, response=response, domain=domain, field=field)for word in words:try:docs = await search_with_recipe(client, recipe, word, 10)@@ -181,22 +200,23 @@ async def _probe_domain(client: httpx.AsyncClient, entry: dict) -> dict:"response": response,"smoke": {"query": word, "expect_title_contains": word},}- (RECIPES_DIR / f"{name}.yaml").write_text(- yaml.safe_dump(doc, sort_keys=False, allow_unicode=True))- return {"domain": domain, "status": "pass",- "label": label, "word": word, "n": len(docs)}+ (RECIPES_DIR / f"{name}.yaml").write_text(yaml.safe_dump(doc, sort_keys=False, allow_unicode=True))+ return {"domain": domain, "status": "pass", "label": label, "word": word, "n": len(docs)}return {"domain": domain, "status": "fail"}async def _run(entries: list[dict]) -> list[dict]:sem = asyncio.Semaphore(CONCURRENCY)async with httpx.AsyncClient(- follow_redirects=True, timeout=TIMEOUT,+ follow_redirects=True,+ timeout=TIMEOUT,headers={"User-Agent": "mwmbl-super-search-smoke/0.1 (+https://mwmbl.org)"},) as client:+async def guarded(entry):async with sem:return await _probe_domain(client, entry)+return await asyncio.gather(*[guarded(e) for e in entries])@@ -213,12 +233,10 @@ def _load_entries(paths: list[str]) -> list[dict]:def main() -> int:ap = argparse.ArgumentParser()ap.add_argument("batches", nargs="*", help="batch JSON files")- ap.add_argument("--all", action="store_true",- help="process every recipe_chunks/input_*.json")+ ap.add_argument("--all", action="store_true", help="process every recipe_chunks/input_*.json")args = ap.parse_args()- paths = sorted(str(p) for p in CHUNKS_DIR.glob("input_*.json")) if args.all \- else args.batches+ paths = sorted(str(p) for p in CHUNKS_DIR.glob("input_*.json")) if args.all else args.batchesif not paths:print("no batch files given (use --all or list files)", file=sys.stderr)return 2@@ -231,9 +249,11 @@ def main() -> int:exists = [r for r in results if r["status"] == "exists"]for r in passes:print(f"PASS {r['domain']} ({r['label']}, q={r['word']!r}, {r['n']} results)")- print(f"\n{len(passes)} passed, {len(exists)} already existed, "- f"{len(results) - len(passes) - len(exists)} failed "- f"(of {len(results)} domains).")+ print(+ f"\n{len(passes)} passed, {len(exists)} already existed, "+ f"{len(results) - len(passes) - len(exists)} failed "+ f"(of {len(results)} domains)."+ )return 0diff --git a/scripts/auto_recipe_html.py b/scripts/auto_recipe_html.pyindex 75afe51..8329129 100644--- a/scripts/auto_recipe_html.py+++ b/scripts/auto_recipe_html.py@@ -25,6 +25,7 @@ Input is the worklist JSON written by the caller: a list ofUsage:DATABASE_URL="postgres://daoud@" uv run python scripts/auto_recipe_html.py /tmp/htmlwork.json"""+from __future__ import annotationsimport asyncio@@ -38,25 +39,28 @@ REPO_ROOT = Path(__file__).resolve().parent.parentsys.path.insert(0, str(REPO_ROOT))os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")-import django # noqa: E402import logging # noqa: E402+import django # noqa: E402+django.setup()logging.getLogger("httpx").setLevel(logging.WARNING)-logging.getLogger("mwmbl.tinysearchengine.super_search_sources.recipe").setLevel(- logging.ERROR)+logging.getLogger("mwmbl.tinysearchengine.super_search_sources.recipe").setLevel(logging.ERROR)import httpx # noqa: E402import yaml # noqa: E402from bs4 import BeautifulSoup # noqa: E402from mwmbl.tinysearchengine.super_search_sources.recipe import ( # noqa: E402- Recipe, search_with_recipe,+ Recipe,+ search_with_recipe,)-from scripts.auto_recipe import FIELD_WORD, GENERIC_WORDS, _name # noqa: E402from mwmbl.tinysearchengine.super_search_sources.smoke import ( # noqa: E402- CONTROL_QUERY, MAX_CONTROL_OVERLAP, control_overlap,+ CONTROL_QUERY,+ MAX_CONTROL_OVERLAP,+ control_overlap,)+from scripts.auto_recipe import FIELD_WORD, GENERIC_WORDS, _name # noqa: E402RECIPES_DIR = REPO_ROOT / "mwmbl" / "tinysearchengine" / "super_search_sources" / "recipes"MIN_RESULTS = 3@@ -99,11 +103,13 @@ def _candidate_selectors(html: str) -> list[str]:# Keep only selectors that can isolate a result row: a heading tag, or any# selector carrying a class. Reject bare generic tags (li/div/p/td/...) and# obvious nav/menu chrome.- ranked = [(sel, n) for sel, n in groups.most_common()- if n >= MIN_RESULTS- and ("." in sel or sel in HEADING_TAGS)- and not any(bad in sel.lower()- for bad in ("menu", "nav", "footer", "header"))]+ ranked = [+ (sel, n)+ for sel, n in groups.most_common()+ if n >= MIN_RESULTS+ and ("." in sel or sel in HEADING_TAGS)+ and not any(bad in sel.lower() for bad in ("menu", "nav", "footer", "header"))+ ]return [sel for sel, _ in ranked[:3]]@@ -120,21 +126,20 @@ async def _author(client: httpx.AsyncClient, dom: str, field: str, url_tmpl: strbreak # network dead -> give up on this domainfor selector in _candidate_selectors(html):response = {- "format": "html", "results": selector, "base_url": base,+ "format": "html",+ "results": selector,+ "base_url": base,"fields": {"title": "a", "url": {"selector": "a", "attr": "href"}},}# Reconstruct request the same way the engine will (params from the# template's query string) by just pointing url at the templated form.- request = {"url": url_tmpl.split("?")[0],- "params": _params_from(url_tmpl)}- recipe = Recipe(name=name, request=request, response=response,- domain=dom, field=field)+ request = {"url": url_tmpl.split("?")[0], "params": _params_from(url_tmpl)}+ recipe = Recipe(name=name, request=request, response=response, domain=dom, field=field)try:docs = await search_with_recipe(client, recipe, word, 10)except Exception: # noqa: BLE001docs = []- if len(docs) < MIN_RESULTS or not any(- word in (d.title or "").lower() for d in docs):+ if len(docs) < MIN_RESULTS or not any(word in (d.title or "").lower() for d in docs):continue# Query-invariance guard (same as the hardened smoke gate): reject# selectors that return the same URLs for an unrelated control query.@@ -145,19 +150,23 @@ async def _author(client: httpx.AsyncClient, dom: str, field: str, url_tmpl: stroverlap = control_overlap(docs, control)if overlap is not None and overlap >= MAX_CONTROL_OVERLAP:continue- doc = {"name": name, "domain": dom, "field": field,- "request": request, "response": response,- "smoke": {"query": word, "expect_title_contains": word}}- (RECIPES_DIR / f"{name}.yaml").write_text(- yaml.safe_dump(doc, sort_keys=False, allow_unicode=True))- return {"domain": dom, "status": "pass",- "selector": selector, "word": word, "n": len(docs)}+ doc = {+ "name": name,+ "domain": dom,+ "field": field,+ "request": request,+ "response": response,+ "smoke": {"query": word, "expect_title_contains": word},+ }+ (RECIPES_DIR / f"{name}.yaml").write_text(yaml.safe_dump(doc, sort_keys=False, allow_unicode=True))+ return {"domain": dom, "status": "pass", "selector": selector, "word": word, "n": len(docs)}return {"domain": dom, "status": "fail"}def _params_from(url_tmpl: str) -> dict:"""Turn the query string of a templated URL into a recipe params map."""- from urllib.parse import urlsplit, parse_qsl+ from urllib.parse import parse_qsl, urlsplit+qs = urlsplit(url_tmpl).queryparams = {}for k, v in parse_qsl(qs, keep_blank_values=True):@@ -169,12 +178,15 @@ def _params_from(url_tmpl: str) -> dict:async def _run(work: list[list]) -> list[dict]:sem = asyncio.Semaphore(CONCURRENCY)async with httpx.AsyncClient(- follow_redirects=True, timeout=TIMEOUT,+ follow_redirects=True,+ timeout=TIMEOUT,headers={"User-Agent": "mwmbl-super-search-smoke/0.1 (+https://mwmbl.org)"},) as client:+async def guarded(w):async with sem:return await _author(client, w[0], w[1], w[3])+return await asyncio.gather(*[guarded(w) for w in work])@@ -190,9 +202,11 @@ def main() -> int:passes = [r for r in results if r["status"] == "pass"]for r in passes:print(f"PASS {r['domain']} (results={r['selector']!r}, q={r['word']!r}, {r['n']})")- print(f"\n{len(passes)} passed, "- f"{sum(1 for r in results if r['status']=='exists')} existed, "- f"{sum(1 for r in results if r['status']=='fail')} failed.")+ print(+ f"\n{len(passes)} passed, "+ f"{sum(1 for r in results if r['status'] == 'exists')} existed, "+ f"{sum(1 for r in results if r['status'] == 'fail')} failed."+ )return 0diff --git a/scripts/build_super_search_candidates.py b/scripts/build_super_search_candidates.pyindex 6e45f82..ca9d355 100644--- a/scripts/build_super_search_candidates.py+++ b/scripts/build_super_search_candidates.py@@ -20,6 +20,7 @@ Two-phase helper for the Super Search v2 candidate analysis:Run from the repo root (the directory containing curated-domains.json)."""+import argparseimport jsonimport sys@@ -46,28 +47,68 @@ RECIPE_BATCH_SIZE = 5# Domains already served by a hand-written adapter (recipe domains are detected# from the YAML files). The Stack Exchange adapter covers the whole SE network.ADAPTER_DOMAINS = {- "mwmbl.org", "news.ycombinator.com", "github.com", "pypi.org",- "arxiv.org", "info.arxiv.org",- "stackoverflow.com", "stackexchange.com", "superuser.com",- "mathoverflow.net", "serverfault.com", "askubuntu.com",+ "mwmbl.org",+ "news.ycombinator.com",+ "github.com",+ "pypi.org",+ "arxiv.org",+ "info.arxiv.org",+ "stackoverflow.com",+ "stackexchange.com",+ "superuser.com",+ "mathoverflow.net",+ "serverfault.com",+ "askubuntu.com",}# Fixed taxonomy / enums (keep in sync with the subagent prompt).FIELDS = {- "programming", "tech", "science", "academia", "books-literature",- "recipes-food", "history", "art-design", "gaming", "music", "film-tv",- "business-finance", "health-medicine", "news-politics", "law-government",- "education", "philosophy", "nature-environment", "sports", "lifestyle",+ "programming",+ "tech",+ "science",+ "academia",+ "books-literature",+ "recipes-food",+ "history",+ "art-design",+ "gaming",+ "music",+ "film-tv",+ "business-finance",+ "health-medicine",+ "news-politics",+ "law-government",+ "education",+ "philosophy",+ "nature-environment",+ "sports",+ "lifestyle","other",}LEVELS = {"low", "medium", "high"}SITE_TYPES = {- "blog", "docs", "wiki", "forum", "journal", "news", "store",- "personal", "reference", "tool", "org",+ "blog",+ "docs",+ "wiki",+ "forum",+ "journal",+ "news",+ "store",+ "personal",+ "reference",+ "tool",+ "org",}REQUIRED_KEYS = {- "name", "field", "popularity", "estimated_pages", "site_type",- "language", "has_search", "recommended", "reason",+ "name",+ "field",+ "popularity",+ "estimated_pages",+ "site_type",+ "language",+ "has_search",+ "recommended",+ "reason",}# Map classifier synonyms onto the fixed taxonomy (keeps the output reproducible@@ -106,6 +147,7 @@ def root_domain(d):def hn_prior_map(names):"""Map domain -> 'high' | 'medium' hint from HN top-domain scores, else absent."""from mwmbl.hn_top_domains_filtered import DOMAINS+hn = {k.lower(): v for k, v in DOMAINS.items()}prior = {}for name in names:@@ -127,16 +169,12 @@ def cmd_prepare():n_chunks = 0for i in range(0, len(names), CHUNK_SIZE):- chunk = names[i:i + CHUNK_SIZE]+ chunk = names[i : i + CHUNK_SIZE]idx = i // CHUNK_SIZE- payload = [- {"name": name, "hn_popularity_hint": prior.get(name)}- for name in chunk- ]+ payload = [{"name": name, "hn_popularity_hint": prior.get(name)} for name in chunk](CHUNK_DIR / f"input_{idx:03d}.json").write_text(json.dumps(payload, indent=2))n_chunks += 1- print(f"Prepared {len(names)} domains into {n_chunks} input chunks "- f"(size {CHUNK_SIZE}) in {CHUNK_DIR}")+ print(f"Prepared {len(names)} domains into {n_chunks} input chunks (size {CHUNK_SIZE}) in {CHUNK_DIR}")print(f"HN popularity prior available for {len(prior)} domains.")@@ -183,9 +221,11 @@ def cmd_merge():missing_names = [n for n in names if n not in by_name]extra_names = [n for n in by_name if n not in curated_set]- print(f"Classified {len(by_name)} / {len(names)} domains, "- f"{len(missing_names)} missing, {len(errors)} invalid, "- f"{len(extra_names)} unexpected.")+ print(+ f"Classified {len(by_name)} / {len(names)} domains, "+ f"{len(missing_names)} missing, {len(errors)} invalid, "+ f"{len(extra_names)} unexpected."+ )if missing_names:print(" Missing (first 20):", missing_names[:20])@@ -229,15 +269,9 @@ def cmd_shortlist():return 1domains = json.loads(OUTPUT.read_text())["domains"]pop_rank = {"high": 0, "medium": 1, "low": 2}- shortlist = [- e for e in domains- if e["recommended"] and e["has_search"]- and e["popularity"] in ("medium", "high")- ]+ shortlist = [e for e in domains if e["recommended"] and e["has_search"] and e["popularity"] in ("medium", "high")]shortlist.sort(key=lambda e: (pop_rank[e["popularity"]], e["field"], e["name"]))- SHORTLIST.write_text(- json.dumps({"domains": shortlist}, indent=2, ensure_ascii=False) + "\n"- )+ SHORTLIST.write_text(json.dumps({"domains": shortlist}, indent=2, ensure_ascii=False) + "\n")print(f"Wrote {SHORTLIST} with {len(shortlist)} / {len(domains)} domains.")print("Filter: recommended AND has_search AND popularity in {medium, high}")print("\n-- field --")@@ -255,6 +289,7 @@ def cmd_shortlist():def _covered_domains():"""Domains already served: hand-written adapters + existing recipe YAML files."""import yaml+covered = set(ADAPTER_DOMAINS)for path in RECIPES_DIR.glob("*.yaml"):try:@@ -304,8 +339,7 @@ def cmd_select_remaining():attempted = {e["name"] for e in _select_diverse(orig_uncovered, TARGET_COUNT)}covered_now = _covered_domains()- remaining = [e for e in entries- if e["name"] not in covered_now and e["name"] not in attempted]+ remaining = [e for e in entries if e["name"] not in covered_now and e["name"] not in attempted]remaining = _select_diverse(remaining, len(remaining)) # field-interleaved orderTARGETS.write_text(json.dumps({"domains": remaining}, indent=2, ensure_ascii=False) + "\n")@@ -314,13 +348,15 @@ def cmd_select_remaining():old.unlink()n_batches = 0for i in range(0, len(remaining), REMAINING_BATCH_SIZE):- batch = remaining[i:i + REMAINING_BATCH_SIZE]+ batch = remaining[i : i + REMAINING_BATCH_SIZE](RECIPE_CHUNK_DIR / f"input_{i // REMAINING_BATCH_SIZE:03d}.json").write_text(json.dumps(batch, indent=2, ensure_ascii=False))n_batches += 1- print(f"Remaining uncovered, not-yet-attempted: {len(remaining)} sites "- f"({len(covered_now)} covered, {len(attempted)} already attempted).")+ print(+ f"Remaining uncovered, not-yet-attempted: {len(remaining)} sites "+ f"({len(covered_now)} covered, {len(attempted)} already attempted)."+ )print(f"Wrote {n_batches} batches (size {REMAINING_BATCH_SIZE}) in {RECIPE_CHUNK_DIR}.")return 0@@ -343,16 +379,20 @@ def cmd_select_targets():old.unlink()n_batches = 0for i in range(0, len(targets), RECIPE_BATCH_SIZE):- batch = targets[i:i + RECIPE_BATCH_SIZE]+ batch = targets[i : i + RECIPE_BATCH_SIZE](RECIPE_CHUNK_DIR / f"input_{i // RECIPE_BATCH_SIZE:03d}.json").write_text(json.dumps(batch, indent=2, ensure_ascii=False))n_batches += 1- print(f"Selected {len(targets)} targets from {len(candidates)} uncovered "- f"shortlist sites ({len(covered)} already covered).")- print(f"Wrote {TARGETS} and {n_batches} batches (size {RECIPE_BATCH_SIZE}) "- f"in {RECIPE_CHUNK_DIR} for the recipe sub-agent fan-out.")+ print(+ f"Selected {len(targets)} targets from {len(candidates)} uncovered "+ f"shortlist sites ({len(covered)} already covered)."+ )+ print(+ f"Wrote {TARGETS} and {n_batches} batches (size {RECIPE_BATCH_SIZE}) "+ f"in {RECIPE_CHUNK_DIR} for the recipe sub-agent fan-out."+ )print("\n-- field --")for k, v in Counter(e["field"] for e in targets).most_common():print(f" {k:20} {v}")diff --git a/scripts/check_search_api_key.py b/scripts/check_search_api_key.pyindex 25ecfa6..459f024 100755--- a/scripts/check_search_api_key.py+++ b/scripts/check_search_api_key.py@@ -8,6 +8,7 @@ Check search API behaviour with an API key:1. Usage value increases with each successful request2. Rate limiting kicks in and returns 429 when exceeded"""+import osimport sysimport time@@ -47,9 +48,7 @@ def check_usage_increases() -> None:print(f" Request {i + 1}: monthly_usage={usage}, monthly_limit={limit}")assert usage is not None, "monthly_usage should not be null when using an API key"if prev_usage is not None:- assert usage > prev_usage, (- f"monthly_usage did not increase: was {prev_usage}, now {usage}"- )+ assert usage > prev_usage, f"monthly_usage did not increase: was {prev_usage}, now {usage}"prev_usage = usage# Small pause to avoid triggering rate limit between checkstime.sleep(0.3)@@ -109,9 +108,7 @@ def check_subscription_usage(expected_usage: int) -> None:print(f" plan={plan}, monthly_usage={usage}, monthly_limit={limit}")assert usage is not None, "monthly_usage missing from subscription response"assert limit is not None, "monthly_limit missing from subscription response"- assert usage >= expected_usage, (- f"subscription monthly_usage ({usage}) is less than search usage ({expected_usage})"- )+ assert usage >= expected_usage, f"subscription monthly_usage ({usage}) is less than search usage ({expected_usage})"print(f" PASS: subscription usage ({usage}) >= search usage ({expected_usage})\n")diff --git a/scripts/fetch_url.py b/scripts/fetch_url.pyindex 261aea1..76f5f08 100644--- a/scripts/fetch_url.py+++ b/scripts/fetch_url.py@@ -16,6 +16,7 @@ Usage:uv run python scripts/fetch_url.py "https://example.com/api?q=test" --max-chars 12000uv run python scripts/fetch_url.py "https://example.com/search?q=test" --out /tmp/page.html"""+from __future__ import annotationsimport argparse@@ -33,7 +34,9 @@ TIMEOUT = 15.0async def _fetch(url: str, max_chars: int, out: str | None) -> int:try:async with httpx.AsyncClient(- follow_redirects=True, timeout=TIMEOUT, headers=ENGINE_HEADERS,+ follow_redirects=True,+ timeout=TIMEOUT,+ headers=ENGINE_HEADERS,) as client:r = await client.get(url)except Exception as e: # noqa: BLE001diff --git a/scripts/judge_bakeoff.py b/scripts/judge_bakeoff.pyindex eecb09b..88ccf5f 100644--- a/scripts/judge_bakeoff.py+++ b/scripts/judge_bakeoff.py@@ -34,6 +34,7 @@ only ever be reported in this mode.Run: uv run --with fastembed python scripts/judge_bakeoff.py [--judges a,b,...]"""+from __future__ import annotationsimport argparse@@ -66,6 +67,7 @@ def load_dataset() -> list[dict]:# --- judges -----------------------------------------------------------------+def tokenize(text: str) -> list[str]:return re.findall(r"[a-z0-9]+", text.lower())@@ -86,16 +88,15 @@ def score_nomic_cosine(rows: list[dict]) -> np.ndarray:model = TextEmbedding("nomic-ai/nomic-embed-text-v1.5")queries = sorted({row["query"] for row in rows})- query_vectors = dict(zip(queries, model.embed(- [f"search_query: {q}" for q in queries], batch_size=64)))- doc_vectors = model.embed(- [f"search_document: {row['doc_text']}" for row in rows], batch_size=64)+ query_vectors = dict(zip(queries, model.embed([f"search_query: {q}" for q in queries], batch_size=64)))+ doc_vectors = model.embed([f"search_document: {row['doc_text']}" for row in rows], batch_size=64)scores = np.zeros(len(rows))for i, (row, doc_vector) in enumerate(zip(rows, doc_vectors)):query_vector = query_vectors[row["query"]]- scores[i] = float(np.dot(query_vector, doc_vector) /- (np.linalg.norm(query_vector) * np.linalg.norm(doc_vector) + 1e-9))+ scores[i] = float(+ np.dot(query_vector, doc_vector) / (np.linalg.norm(query_vector) * np.linalg.norm(doc_vector) + 1e-9)+ )if (i + 1) % 5000 == 0:print(f" nomic: {i + 1}/{len(rows)}", flush=True)return scores@@ -111,8 +112,7 @@ def _cross_encoder(rows: list[dict], model_name: str, label: str) -> np.ndarray:by_query[row["query"]].append(i)done = 0for query, indexes in by_query.items():- for index, score in zip(indexes, model.rerank(- query, [rows[i]["doc_text"] for i in indexes], batch_size=64)):+ for index, score in zip(indexes, model.rerank(query, [rows[i]["doc_text"] for i in indexes], batch_size=64)):scores[index] = float(score)done += len(indexes)if done // 5000 != (done - len(indexes)) // 5000:@@ -128,9 +128,9 @@ def score_jina_turbo_ce(rows: list[dict]) -> np.ndarray:return _cross_encoder(rows, "jinaai/jina-reranker-v1-turbo-en", "jina_turbo_ce")-def onnx_cross_encoder(rows: list[dict], onnx_dir: Path, model_file: str,- label: str, max_length: int = 256,- batch_size: int = 64) -> np.ndarray:+def onnx_cross_encoder(+ rows: list[dict], onnx_dir: Path, model_file: str, label: str, max_length: int = 256, batch_size: int = 64+) -> np.ndarray:"""Score (query, doc_text) rows with a local exported cross-encoder."""import onnxruntimefrom tokenizers import Tokenizer@@ -143,24 +143,27 @@ def onnx_cross_encoder(rows: list[dict], onnx_dir: Path, model_file: str,scores = np.zeros(len(rows))for start in range(0, len(rows), batch_size):- batch = rows[start:start + batch_size]- encodings = tokenizer.encode_batch(- [(row["query"], row["doc_text"]) for row in batch])- feed = {"input_ids": np.array([e.ids for e in encodings], dtype=np.int64),- "attention_mask": np.array([e.attention_mask for e in encodings],- dtype=np.int64)}+ batch = rows[start : start + batch_size]+ encodings = tokenizer.encode_batch([(row["query"], row["doc_text"]) for row in batch])+ feed = {+ "input_ids": np.array([e.ids for e in encodings], dtype=np.int64),+ "attention_mask": np.array([e.attention_mask for e in encodings], dtype=np.int64),+ }if "token_type_ids" in input_names:- feed["token_type_ids"] = np.array([e.type_ids for e in encodings],- dtype=np.int64)+ feed["token_type_ids"] = np.array([e.type_ids for e in encodings], dtype=np.int64)logits = session.run(None, feed)[0][:, 0]- scores[start:start + len(batch)] = 1 / (1 + np.exp(-logits.astype(np.float64)))+ scores[start : start + len(batch)] = 1 / (1 + np.exp(-logits.astype(np.float64)))if (start + batch_size) % 4992 < batch_size:print(f" {label}: {start + len(batch)}/{len(rows)}", flush=True)return scores-SCORERS = {"term_overlap": score_term_overlap, "nomic_cosine": score_nomic_cosine,- "minilm_ce": score_minilm_ce, "jina_turbo_ce": score_jina_turbo_ce}+SCORERS = {+ "term_overlap": score_term_overlap,+ "nomic_cosine": score_nomic_cosine,+ "minilm_ce": score_minilm_ce,+ "jina_turbo_ce": score_jina_turbo_ce,+}def register_finetuned(model_dir: Path) -> list[str]:@@ -171,8 +174,7 @@ def register_finetuned(model_dir: Path) -> list[str]:for suffix, model_file in (("", "model.onnx"), ("_int8", "model.int8.onnx")):if (onnx_dir / model_file).exists():name = f"ft_{run}{suffix}"- SCORERS[name] = lambda rows, f=model_file, n=f"ft_{run}{suffix}": \- onnx_cross_encoder(rows, onnx_dir, f, n)+ SCORERS[name] = lambda rows, f=model_file, n=f"ft_{run}{suffix}": onnx_cross_encoder(rows, onnx_dir, f, n)names.append(name)torch_scores = model_dir / "llm_scores.npy"if torch_scores.exists():@@ -200,6 +202,7 @@ def judge_scores(name: str, rows: list[dict]) -> np.ndarray:# --- metrics ----------------------------------------------------------------+def rankdata(values) -> np.ndarray:values = np.asarray(values, dtype=float)order = np.argsort(values)@@ -225,13 +228,13 @@ def roc_auc(labels: np.ndarray, scores: np.ndarray) -> float:if not len(positive) or not len(negative):return float("nan")ranks = rankdata(scores)- return float((ranks[labels].sum() - len(positive) * (len(positive) - 1) / 2)- / (len(positive) * len(negative)))+ return float((ranks[labels].sum() - len(positive) * (len(positive) - 1) / 2) / (len(positive) * len(negative)))def ndcg_at_k(gains_in_rank_order: list[float], ideal: list[float], k: int = 10) -> float:def dcg(gains):return sum(g / math.log2(i + 2) for i, g in enumerate(gains[:k]))+ideal_dcg = dcg(sorted(ideal, reverse=True))return dcg(gains_in_rank_order) / ideal_dcg if ideal_dcg > 0 else float("nan")@@ -266,9 +269,11 @@ def evaluate(name: str, rows: list[dict], scores: np.ndarray) -> dict:source_rows[row["query"]][row["ss_source"]].append(i)within_query_rho, best_agree, eligible = [], 0, 0for query, sources in source_rows.items():- aggregates = {s: (float(np.mean(scores[np.array(ix)])),- float(np.mean(overall[np.array(ix)])))- for s, ix in sources.items() if len(ix) >= 2}+ aggregates = {+ s: (float(np.mean(scores[np.array(ix)])), float(np.mean(overall[np.array(ix)])))+ for s, ix in sources.items()+ if len(ix) >= 2+ }if len(aggregates) < 2:continueeligible += 1@@ -288,8 +293,7 @@ def evaluate(name: str, rows: list[dict], scores: np.ndarray) -> dict:"auc_rel2": round(roc_auc(relevance >= 2, scores), 3),"ndcg@10": round(float(np.mean(ndcgs)), 3),"ndcg@10_random": round(float(np.mean(random_ndcgs)), 3),- "source_within_query_spearman": round(float(np.mean(within_query_rho)), 3)- if within_query_rho else None,+ "source_within_query_spearman": round(float(np.mean(within_query_rho)), 3) if within_query_rho else None,"source_best_agree@1": round(best_agree / eligible, 3) if eligible else None,"source_eligible_queries": eligible,}@@ -297,21 +301,28 @@ def evaluate(name: str, rows: list[dict], scores: np.ndarray) -> dict:def main():parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])- parser.add_argument("--judges", default=None,- help=f"comma-separated subset of {ALL_JUDGES} and any "- "registered ft_* judges (default: all)")- parser.add_argument("--model-dir", type=Path, action="append", default=[],- help="fine-tuned model artifact dir (repeatable); "- "registers ft_<run>[,_int8,_torch] judges")- parser.add_argument("--eval-manifest", type=Path, default=None,- help="eval_manifest.json: compute metrics on its "- "held-out rows only (results_heldout.json)")+ parser.add_argument(+ "--judges",+ default=None,+ help=f"comma-separated subset of {ALL_JUDGES} and any registered ft_* judges (default: all)",+ )+ parser.add_argument(+ "--model-dir",+ type=Path,+ action="append",+ default=[],+ help="fine-tuned model artifact dir (repeatable); registers ft_<run>[,_int8,_torch] judges",+ )+ parser.add_argument(+ "--eval-manifest",+ type=Path,+ default=None,+ help="eval_manifest.json: compute metrics on its held-out rows only (results_heldout.json)",+ )args = parser.parse_args()- finetuned = [name for model_dir in args.model_dir- for name in register_finetuned(model_dir)]- judges = (args.judges.split(",") if args.judges- else list(ALL_JUDGES) + finetuned)+ finetuned = [name for model_dir in args.model_dir for name in register_finetuned(model_dir)]+ judges = args.judges.split(",") if args.judges else list(ALL_JUDGES) + finetunedrows = load_dataset()print(f"dataset: {len(rows)} rows, {len({r['query'] for r in rows})} queries")@@ -323,12 +334,15 @@ def main():eval_slice = np.array(manifest["llm_eval_row_indexes"])eval_rows = [rows[i] for i in eval_slice]results_file = "results_heldout.json"- print(f"held-out eval: {len(eval_rows)} rows, "- f"{len({r['query'] for r in eval_rows})} queries "- f"({manifest['source_eligible_eval_queries']} source-eligible)")+ print(+ f"held-out eval: {len(eval_rows)} rows, "+ f"{len({r['query'] for r in eval_rows})} queries "+ f"({manifest['source_eligible_eval_queries']} source-eligible)"+ )elif finetuned:- parser.error("fine-tuned judges must be evaluated with --eval-manifest "- "(their training saw the non-held-out queries)")+ parser.error(+ "fine-tuned judges must be evaluated with --eval-manifest (their training saw the non-held-out queries)"+ )results = []for name in judges:@@ -342,8 +356,7 @@ def main():keys = list(results[0].keys())print(" | ".join(f"{k:>28s}" if i == 0 else k for i, k in enumerate(keys)))for result in results:- print(" | ".join(f"{str(result[k]):>28s}" if i == 0 else str(result[k])- for i, k in enumerate(keys)))+ print(" | ".join(f"{str(result[k]):>28s}" if i == 0 else str(result[k]) for i, k in enumerate(keys)))if __name__ == "__main__":diff --git a/scripts/judge_pairs_eval.py b/scripts/judge_pairs_eval.pyindex 179778b..27f7792 100644--- a/scripts/judge_pairs_eval.py+++ b/scripts/judge_pairs_eval.py@@ -16,6 +16,7 @@ arrays are cached in devdata/judge_bakeoff/pairscores_{judge}.npz.Run: uv run --with fastembed python scripts/judge_pairs_eval.py \[--judges a,b] [--model-dir devdata/judge_train/models/<run>]"""+import argparseimport gzipimport json@@ -37,8 +38,7 @@ def load_pairs() -> list[dict]:return [json.loads(line) for line in f]-def pair_scores(name: str, pairs: list[dict],- model_dirs: list[Path]) -> tuple[np.ndarray, np.ndarray]:+def pair_scores(name: str, pairs: list[dict], model_dirs: list[Path]) -> tuple[np.ndarray, np.ndarray]:cache = CACHE_DIR / f"pairscores_{name}.npz"if cache.exists():data = np.load(cache)@@ -55,10 +55,8 @@ def pair_scores(name: str, pairs: list[dict],else:print(f"scoring {name} ({len(pairs)} pairs)...", flush=True)docs = judge_bakeoff.DOC_CHARS- pos = SCORERS[name]([{"query": p["query"], "doc_text": p["pos"][:docs]}- for p in pairs])- neg = SCORERS[name]([{"query": p["query"], "doc_text": p["neg"][:docs]}- for p in pairs])+ pos = SCORERS[name]([{"query": p["query"], "doc_text": p["pos"][:docs]} for p in pairs])+ neg = SCORERS[name]([{"query": p["query"], "doc_text": p["neg"][:docs]} for p in pairs])CACHE_DIR.mkdir(parents=True, exist_ok=True)np.savez(cache, pos=pos, neg=neg)return pos, neg@@ -79,10 +77,8 @@ def main():parser.add_argument("--model-dir", type=Path, action="append", default=[])args = parser.parse_args()- finetuned = [name for model_dir in args.model_dir- for name in register_finetuned(model_dir)]- judges = (args.judges.split(",") if args.judges- else list(ALL_JUDGES) + finetuned)+ finetuned = [name for model_dir in args.model_dir for name in register_finetuned(model_dir)]+ judges = args.judges.split(",") if args.judges else list(ALL_JUDGES) + finetunedpairs = load_pairs()rules = sorted({p["rule"] for p in pairs})@@ -102,14 +98,12 @@ def main():results.append(row)print(json.dumps(row))- (CACHE_DIR / "results_pairs_eval.json").write_text(- json.dumps(results, indent=2))+ (CACHE_DIR / "results_pairs_eval.json").write_text(json.dumps(results, indent=2))print("\n=== pairs-eval accuracy ===")keys = list(results[0].keys())print(" | ".join(f"{k:>24s}" if i == 0 else k for i, k in enumerate(keys)))for row in results:- print(" | ".join(f"{str(row[k]):>24s}" if i == 0 else str(row[k])- for i, k in enumerate(keys)))+ print(" | ".join(f"{str(row[k]):>24s}" if i == 0 else str(row[k]) for i, k in enumerate(keys)))if __name__ == "__main__":diff --git a/scripts/judge_train_prep.py b/scripts/judge_train_prep.pyindex b1e3e58..cf8d318 100644--- a/scripts/judge_train_prep.py+++ b/scripts/judge_train_prep.py@@ -28,6 +28,7 @@ Outputs (devdata/judge_train/, refuses to overwrite without --force):Run: uv run python scripts/judge_train_prep.py [--force]"""+from __future__ import annotationsimport argparse@@ -70,8 +71,7 @@ def split_queries(queries: set[str], buckets: tuple[int, int, int]) -> dict[str,hundred queries. Deterministic for a fixed query set; the manifest freezesthe input files so the set is fixed."""- ordered = sorted(queries, key=lambda q: hashlib.md5(- normalize_query(q).encode()).hexdigest())+ ordered = sorted(queries, key=lambda q: hashlib.md5(normalize_query(q).encode()).hexdigest())n = len(ordered)train_end = round(n * buckets[0] / 100)val_end = train_end + round(n * buckets[1] / 100)@@ -87,8 +87,7 @@ def source_eligible_queries(rows: list[dict]) -> set[str]:for row in rows:if row["ss_source"] and "supersearch" in row["pools"]:sources[row["query"]][row["ss_source"]] += 1- return {query for query, counts in sources.items()- if sum(count >= 2 for count in counts.values()) >= 2}+ return {query for query, counts in sources.items() if sum(count >= 2 for count in counts.values()) >= 2}def load_pairs() -> list[dict]:@@ -99,8 +98,8 @@ def load_pairs() -> list[dict]:def doc_text(doc: dict, url_fallback: bool) -> str | None:title, extract = doc.get("title") or "", doc.get("extract") or ""if title and extract:- return f"{title}. {extract}"[:judge_bakeoff.DOC_CHARS]- return doc["url"][:judge_bakeoff.DOC_CHARS] if url_fallback else None+ return f"{title}. {extract}"[: judge_bakeoff.DOC_CHARS]+ return doc["url"][: judge_bakeoff.DOC_CHARS] if url_fallback else Nonedef sha256_file(path: Path) -> str:@@ -120,15 +119,17 @@ def write_jsonl_gz(path: Path, records: list[dict]) -> None:def main():parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])- parser.add_argument("--force", action="store_true",- help="overwrite an existing devdata/judge_train/")- parser.add_argument("--url-fallback", action="store_true",- help="use the URL as doc text when title/extract missing")+ parser.add_argument("--force", action="store_true", help="overwrite an existing devdata/judge_train/")+ parser.add_argument(+ "--url-fallback", action="store_true", help="use the URL as doc text when title/extract missing"+ )args = parser.parse_args()if OUT_DIR.exists() and any(OUT_DIR.iterdir()) and not args.force:- sys.exit(f"{OUT_DIR} already exists; pass --force to regenerate "- "(this invalidates any models trained on the old split)")+ sys.exit(+ f"{OUT_DIR} already exists; pass --force to regenerate "+ "(this invalidates any models trained on the old split)"+ )rng = random.Random(SEED)@@ -136,8 +137,7 @@ def main():rows = judge_bakeoff.load_dataset()queries = {row["query"] for row in rows}eligible = source_eligible_queries(rows)- print(f"LLM set: {len(rows)} rows, {len(queries)} queries, "- f"{len(eligible)} source-eligible")+ print(f"LLM set: {len(rows)} rows, {len(queries)} queries, {len(eligible)} source-eligible")llm_split = {}llm_split.update(split_queries(eligible, LLM_BUCKETS))@@ -150,24 +150,24 @@ def main():if part == "eval":eval_row_indexes.append(i)else:- pointwise[part].append({- "query": row["query"],- "doc_text": row["doc_text"],- "label": row["overall"] / LABEL_SCALE,- })+ pointwise[part].append(+ {+ "query": row["query"],+ "doc_text": row["doc_text"],+ "label": row["overall"] / LABEL_SCALE,+ }+ )split_queries_by_part = defaultdict(set)for query, part in llm_split.items():split_queries_by_part[part].add(query)for part in ("train", "val", "eval"):n_eligible = len(split_queries_by_part[part] & eligible)- print(f" llm {part}: {len(split_queries_by_part[part])} queries "- f"({n_eligible} source-eligible)")+ print(f" llm {part}: {len(split_queries_by_part[part])} queries ({n_eligible} source-eligible)")# --- preference pairs ------------------------------------------------------raw_pairs = load_pairs()- llm_heldout = {normalize_query(q)- for q in split_queries_by_part["val"] | split_queries_by_part["eval"]}+ llm_heldout = {normalize_query(q) for q in split_queries_by_part["val"] | split_queries_by_part["eval"]}pairs, seen = [], set()dropped = Counter()@@ -185,8 +185,9 @@ def main():if key[0] in llm_heldout:dropped["llm_heldout_leakage"] += 1continue- pairs.append({"query": pair["query"], "pos": pos_text, "neg": neg_text,- "rule": pair["rule"], "user": pair["user"]})+ pairs.append(+ {"query": pair["query"], "pos": pos_text, "neg": neg_text, "rule": pair["rule"], "user": pair["user"]}+ )print(f"pairs: {len(raw_pairs)} raw -> {len(pairs)} kept, dropped {dict(dropped)}")pair_split = split_queries({pair["query"] for pair in pairs}, PAIR_BUCKETS)@@ -204,8 +205,7 @@ def main():if len(query_pairs) > PAIR_CAP_PER_QUERY:query_pairs = rng.sample(query_pairs, PAIR_CAP_PER_QUERY)capped.extend(query_pairs)- print(f" pairs {part}: {len(pair_parts[part])} -> {len(capped)} "- f"after per-query cap")+ print(f" pairs {part}: {len(pair_parts[part])} -> {len(capped)} after per-query cap")pair_parts[part] = capped# per-user share cap on train only@@ -220,14 +220,18 @@ def main():top_pairs = [p for p in train if p["user"] == top_user]kept = set(map(id, rng.sample(top_pairs, keep)))train = [p for p in train if p["user"] != top_user or id(p) in kept]- print(f" pairs train: capped {top_user} {top_count} -> {keep} "- f"({max_share:.0%} -> {keep / len(train):.0%}); total {len(train)}")+ print(+ f" pairs train: capped {top_user} {top_count} -> {keep} "+ f"({max_share:.0%} -> {keep / len(train):.0%}); total {len(train)}"+ )pair_parts["train"] = trainfor part, part_pairs in pair_parts.items():rule_counts = Counter(pair["rule"] for pair in part_pairs)- print(f" pairs {part}: {len(part_pairs)} pairs, "- f"{len({p['query'] for p in part_pairs})} queries, rules {dict(rule_counts)}")+ print(+ f" pairs {part}: {len(part_pairs)} pairs, "+ f"{len({p['query'] for p in part_pairs})} queries, rules {dict(rule_counts)}"+ )# --- write outputs ---------------------------------------------------------OUT_DIR.mkdir(parents=True, exist_ok=True)@@ -255,9 +259,11 @@ def main():},}(OUT_DIR / "eval_manifest.json").write_text(json.dumps(manifest, indent=2))- print(f" wrote {OUT_DIR / 'eval_manifest.json'} "- f"({len(eval_row_indexes)} eval rows, "- f"{manifest['source_eligible_eval_queries']} source-eligible eval queries)")+ print(+ f" wrote {OUT_DIR / 'eval_manifest.json'} "+ f"({len(eval_row_indexes)} eval rows, "+ f"{manifest['source_eligible_eval_queries']} source-eligible eval queries)"+ )if __name__ == "__main__":diff --git a/scripts/judge_verify_onnx.py b/scripts/judge_verify_onnx.pyindex 2a6f25f..1aa1179 100644--- a/scripts/judge_verify_onnx.py+++ b/scripts/judge_verify_onnx.py@@ -11,6 +11,7 @@ If int8 fails, serve the fp32 model.Run: uv run --with onnxruntime --with tokenizers python \scripts/judge_verify_onnx.py --model-dir devdata/judge_train/models/<run>"""+import argparseimport jsonimport sys@@ -40,18 +41,22 @@ def main():fp32 = onnx_cross_encoder(sample, onnx_dir, "model.onnx", "fp32")fp32_diff = float(np.max(np.abs(fp32 - torch_scores)))fp32_ok = fp32_diff < args.fp32_tol- print(f"fp32 vs torch: max|diff| {fp32_diff:.2e} "- f"(spearman {spearman(fp32, torch_scores):.6f}) "- f"-> {'PASS' if fp32_ok else 'FAIL'}")+ print(+ f"fp32 vs torch: max|diff| {fp32_diff:.2e} "+ f"(spearman {spearman(fp32, torch_scores):.6f}) "+ f"-> {'PASS' if fp32_ok else 'FAIL'}"+ )int8_ok = Trueif (onnx_dir / "model.int8.onnx").exists():int8 = onnx_cross_encoder(sample, onnx_dir, "model.int8.onnx", "int8")rho = spearman(int8, fp32)int8_ok = rho > args.int8_min_spearman- print(f"int8 vs fp32: spearman {rho:.6f}, "- f"max|diff| {float(np.max(np.abs(int8 - fp32))):.2e} "- f"-> {'PASS' if int8_ok else 'FAIL (serve fp32)'}")+ print(+ f"int8 vs fp32: spearman {rho:.6f}, "+ f"max|diff| {float(np.max(np.abs(int8 - fp32))):.2e} "+ f"-> {'PASS' if int8_ok else 'FAIL (serve fp32)'}"+ )else:print("int8 model not found, skipping")diff --git a/scripts/judgments_to_pairs.py b/scripts/judgments_to_pairs.pyindex 11de491..2d438ae 100644--- a/scripts/judgments_to_pairs.py+++ b/scripts/judgments_to_pairs.py@@ -33,6 +33,7 @@ first filled from a free local lookup (every documented URL in the export plusthe Pass-2 candidate pool); ``--backfill`` then crawls the remainder over HTTP(checkpointed to backfill_cache.jsonl.gz, resumable)."""+from __future__ import annotationsimport argparse@@ -43,10 +44,10 @@ from collections import Counter, defaultdictfrom pathlib import Pathfrom urllib.parse import parse_qs, urlparse-PAIR_CAP = 5 # max losers per positive event-MIN_ADDS_FOR_SPAM = 20 # dominance filter threshold+PAIR_CAP = 5 # max losers per positive event+MIN_ADDS_FOR_SPAM = 20 # dominance filter thresholdSPAM_DOMAIN_SHARE = 0.5-APPROVED_STATE = 7 # DocumentState values >= this are curated/approved+APPROVED_STATE = 7 # DocumentState values >= this are curated/approvedPASS2_POOL = Path("devdata/llm_relabel/pass2_pool.jsonl")@@ -72,7 +73,8 @@ NSFW_RE = re.compile(r"brazzers|onlyfans|camgirls?|camsex|livejasmin|chaturbate|stripchat|"r"erotic\w*|escorts?|sex|sexcam\w*|blowjob\w*|anal|cumshot\w*|gangbang\w*|"r"nudes?|naked|fetish\w*|bdsm|dildos?|masturbat\w*)\b",- re.IGNORECASE)+ re.IGNORECASE,+)def is_nsfw(*texts) -> bool:@@ -124,8 +126,7 @@ def find_spammers(curations: list[dict], user_curations: list[dict]) -> dict[strdef flagged_users(curations: list[dict]) -> set[str]:- return {c["user"] for c in curations- if c["user"] and any(f["status"] == "ACCEPTED" for f in c["flags"])}+ return {c["user"] for c in curations if c["user"] and any(f["status"] == "ACCEPTED" for f in c["flags"])}class TextLookup:@@ -149,9 +150,11 @@ class TextLookup:def slim(doc: dict) -> dict:- return {"url": doc["url"],- "title": (doc.get("title") or "").strip() or None,- "extract": (doc.get("extract") or "").strip() or None}+ return {+ "url": doc["url"],+ "title": (doc.get("title") or "").strip() or None,+ "extract": (doc.get("extract") or "").strip() or None,+ }class PairEmitter:@@ -175,12 +178,16 @@ class PairEmitter:if key in self.seen:continueself.seen.add(key)- self.pairs.append({- "query": query,- "pos": slim(self.lookup.fill(pos)),- "neg": slim(self.lookup.fill(neg)),- "rule": rule, "table": table, "user": user,- })+ self.pairs.append(+ {+ "query": query,+ "pos": slim(self.lookup.fill(pos)),+ "neg": slim(self.lookup.fill(neg)),+ "rule": rule,+ "table": table,+ "user": user,+ }+ )self.rule_counts[rule] += 1@@ -190,18 +197,15 @@ def pairs_from_curation(curation: dict, emitter: PairEmitter):original, new = curation["original_results"], curation["new_results"]original_urls = {d["url"] for d in original}original_rank = {d["url"]: i for i, d in enumerate(original)}- curated_now = {d["url"] for d in new- if d["url"] not in original_urls or (d.get("state") or 0) >= APPROVED_STATE}+ curated_now = {d["url"] for d in new if d["url"] not in original_urls or (d.get("state") or 0) >= APPROVED_STATE}for position, doc in enumerate(new):- below = [d for d in new[position + 1:]- if d["url"] in original_urls and d["url"] not in curated_now]+ below = [d for d in new[position + 1 :] if d["url"] in original_urls and d["url"] not in curated_now]if doc["url"] not in original_urls:emitter.emit(query, doc, below, "add", "curations", user)else:# order inversions: originals this doc now outranks but didn't before- passed = [d for d in below- if original_rank[d["url"]] < original_rank[doc["url"]]]+ passed = [d for d in below if original_rank[d["url"]] < original_rank[doc["url"]]]if passed:rule = "approve" if (doc.get("state") or 0) >= APPROVED_STATE else "move"emitter.emit(query, doc, passed, rule, "curations", user)@@ -210,7 +214,7 @@ def pairs_from_curation(curation: dict, emitter: PairEmitter):def pairs_from_user_curations(events: list[dict], emitter: PairEmitter, pointwise: list[dict]):"""Old interface: per-action events; results list is post-action."""last_results: dict[tuple, list[dict]] = {}- for event in sorted(events, key=lambda e: (e["timestamp"] or "")):+ for event in sorted(events, key=lambda e: e["timestamp"] or ""):query, user = event["query"], event["user"]if not query:continue@@ -221,20 +225,18 @@ def pairs_from_user_curations(events: list[dict], emitter: PairEmitter, pointwisif kind == "curate_move":old, new = action.get("old_index"), action.get("new_index")if old is not None and new is not None and 0 <= new < old <= len(results):- emitter.emit(query, results[new], results[new + 1:old + 1],- "move", "user_curations", user)+ emitter.emit(query, results[new], results[new + 1 : old + 1], "move", "user_curations", user)elif kind == "curate_add":index = action.get("insert_index")if index is not None and 0 <= index < len(results):- emitter.emit(query, results[index], results[index + 1:],- "add", "user_curations", user)+ emitter.emit(query, results[index], results[index + 1 :], "add", "user_curations", user)elif kind == "curate_delete":index = action.get("delete_index")previous = last_results.get(session)if previous and index is not None and 0 <= index < len(previous):deleted = previous[index]kept = {d["url"] for d in results}- winners = [d for d in previous[index + 1:] if d["url"] in kept]+ winners = [d for d in previous[index + 1 :] if d["url"] in kept]for winner in winners[:PAIR_CAP]:emitter.emit(query, winner, [deleted], "delete", "user_curations", user)elif kind == "curate_validate":@@ -243,8 +245,9 @@ def pairs_from_user_curations(events: list[dict], emitter: PairEmitter, pointwisdoc = slim(emitter.lookup.fill(results[index]))if is_nsfw(query, doc["url"], doc["title"], doc["extract"]):continue- pointwise.append({"query": query, **doc, "label": 1,- "rule": "validate", "table": "user_curations", "user": user})+ pointwise.append(+ {"query": query, **doc, "label": 1, "rule": "validate", "table": "user_curations", "user": user}+ )if results:last_results[session] = results@@ -252,8 +255,8 @@ def pairs_from_user_curations(events: list[dict], emitter: PairEmitter, pointwisTITLE_RE = re.compile(rb"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)META_DESC_RE = re.compile(- rb'<meta[^>]+(?:name|property)=["\'](?:og:)?description["\'][^>]+content=["\']([^"\']+)',- re.IGNORECASE)+ rb'<meta[^>]+(?:name|property)=["\'](?:og:)?description["\'][^>]+content=["\']([^"\']+)', re.IGNORECASE+)PARAGRAPH_RE = re.compile(rb"<p[^>]*>(.*?)</p>", re.IGNORECASE | re.DOTALL)TAG_RE = re.compile(rb"<[^>]+>")@@ -261,6 +264,7 @@ TAG_RE = re.compile(rb"<[^>]+>")def extract_text(body: bytes) -> tuple[str | None, str | None]:def clean(raw: bytes) -> str:import html+return html.unescape(TAG_RE.sub(b" ", raw).decode("utf-8", "replace")).strip()title_match = TITLE_RE.search(body)@@ -275,8 +279,9 @@ def extract_text(body: bytes) -> tuple[str | None, str | None]:return title, None-def crawl_backfill(urls: list[str], cache_path: Path, limit: int | None,- workers: int = 16) -> dict[str, tuple[str, str]]:+def crawl_backfill(+ urls: list[str], cache_path: Path, limit: int | None, workers: int = 16+) -> dict[str, tuple[str, str]]:"""Fetch missing texts over HTTP in parallel; checkpointed and resumable."""from concurrent.futures import ThreadPoolExecutor, as_completed@@ -302,9 +307,11 @@ def crawl_backfill(urls: list[str], cache_path: Path, limit: int | None,row["status"] = f"error: {type(exc).__name__}"return row- with gzip.open(cache_path, "at") as out, \- httpx.Client(follow_redirects=True, timeout=10.0, headers=headers) as client, \- ThreadPoolExecutor(max_workers=workers) as pool:+ with (+ gzip.open(cache_path, "at") as out,+ httpx.Client(follow_redirects=True, timeout=10.0, headers=headers) as client,+ ThreadPoolExecutor(max_workers=workers) as pool,+ ):futures = [pool.submit(fetch, url, client) for url in todo]for n, future in enumerate(as_completed(futures), 1):row = future.result()@@ -314,15 +321,15 @@ def crawl_backfill(urls: list[str], cache_path: Path, limit: int | None,out.flush()print(f" fetched {n}/{len(todo)}", flush=True)- return {u: (r["title"], r["extract"]) for u, r in cache.items()- if r.get("title") and r.get("extract")}+ return {u: (r["title"], r["extract"]) for u, r in cache.items() if r.get("title") and r.get("extract")}def main():parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])parser.add_argument("--export-dir", default="devdata/judgments_export", type=Path)- parser.add_argument("--backfill", action="store_true",- help="crawl URLs whose title/extract can't be filled locally")+ parser.add_argument(+ "--backfill", action="store_true", help="crawl URLs whose title/extract can't be filled locally"+ )parser.add_argument("--backfill-limit", type=int, default=None)args = parser.parse_args()@@ -333,9 +340,11 @@ def main():spammers = find_spammers(curations, user_curations)flagged = flagged_users(curations)dropped_users = set(spammers) | flagged- print(f"dropping {len(dropped_users)} users: "- f"{len(spammers)} domain spammers {dict(list(spammers.items())[:5])}, "- f"{len(flagged)} with accepted flags")+ print(+ f"dropping {len(dropped_users)} users: "+ f"{len(spammers)} domain spammers {dict(list(spammers.items())[:5])}, "+ f"{len(flagged)} with accepted flags"+ )lookup = TextLookup()for curation in curations:@@ -354,22 +363,19 @@ def main():pointwise: list[dict] = []report = {"dropped_users": len(dropped_users), "spammer_domains": spammers}- kept = [c for c in curations- if c["user"] not in dropped_users and not is_urlish_query(c["query"])]+ kept = [c for c in curations if c["user"] not in dropped_users and not is_urlish_query(c["query"])]urlish = sum(1 for c in curations if is_urlish_query(c["query"]))no_ops = 0for curation in kept:- if [d["url"] for d in curation["original_results"]] == \- [d["url"] for d in curation["new_results"]]:+ if [d["url"] for d in curation["original_results"]] == [d["url"] for d in curation["new_results"]]:no_ops += 1continuepairs_from_curation(curation, emitter)- report["curations"] = {"total": len(curations), "kept": len(kept),- "urlish_queries": urlish, "no_ops": no_ops}+ report["curations"] = {"total": len(curations), "kept": len(kept), "urlish_queries": urlish, "no_ops": no_ops}- kept_events = [e for e in user_curations- if e["user"] not in dropped_users- and not (e["query"] and is_urlish_query(e["query"]))]+ kept_events = [+ e for e in user_curations if e["user"] not in dropped_users and not (e["query"] and is_urlish_query(e["query"]))+ ]pairs_from_user_curations(kept_events, emitter, pointwise)report["user_curations"] = {"total": len(user_curations), "kept": len(kept_events)}@@ -379,9 +385,16 @@ def main():filled = lookup.fill({"url": vote["url"], "title": None, "extract": None})if is_nsfw(vote["query"], vote["url"], filled.get("title"), filled.get("extract")):continue- pointwise.append({"query": vote["query"], **slim(filled),- "label": 1 if vote["vote_type"] == "upvote" else -1,- "rule": "vote", "table": "votes", "user": vote["user"]})+ pointwise.append(+ {+ "query": vote["query"],+ **slim(filled),+ "label": 1 if vote["vote_type"] == "upvote" else -1,+ "rule": "vote",+ "table": "votes",+ "user": vote["user"],+ }+ )def missing_urls() -> list[str]:urls = set()@@ -393,8 +406,7 @@ def main():return sorted(urls)if args.backfill:- crawled = crawl_backfill(missing_urls(), args.export_dir / "backfill_cache.jsonl.gz",- args.backfill_limit)+ crawled = crawl_backfill(missing_urls(), args.export_dir / "backfill_cache.jsonl.gz", args.backfill_limit)lookup.texts.update(crawled)for pair in emitter.pairs:for side in ("pos", "neg"):@@ -410,14 +422,15 @@ def main():for point in pointwise:f.write(json.dumps(point) + "\n")- complete = sum(1 for p in emitter.pairs- if all(p[s]["title"] and p[s]["extract"] for s in ("pos", "neg")))- report["pairs"] = {"total": len(emitter.pairs), "by_rule": dict(emitter.rule_counts),- "distinct_queries": len({p["query"] for p in emitter.pairs}),- "both_sides_have_text": complete,- "urls_still_missing_text": len(missing_urls())}- report["pointwise"] = {"total": len(pointwise),- "by_rule": dict(Counter(p["rule"] for p in pointwise))}+ complete = sum(1 for p in emitter.pairs if all(p[s]["title"] and p[s]["extract"] for s in ("pos", "neg")))+ report["pairs"] = {+ "total": len(emitter.pairs),+ "by_rule": dict(emitter.rule_counts),+ "distinct_queries": len({p["query"] for p in emitter.pairs}),+ "both_sides_have_text": complete,+ "urls_still_missing_text": len(missing_urls()),+ }+ report["pointwise"] = {"total": len(pointwise), "by_rule": dict(Counter(p["rule"] for p in pointwise))}print(json.dumps(report, indent=2))(args.export_dir / "transform_report.json").write_text(json.dumps(report, indent=2))diff --git a/scripts/llm_relabel_build_dataset.py b/scripts/llm_relabel_build_dataset.pyindex c2bc42b..ef88352 100644--- a/scripts/llm_relabel_build_dataset.py+++ b/scripts/llm_relabel_build_dataset.py@@ -13,6 +13,7 @@ Pass-3 judgment are dropped.Usage::DATABASE_URL="postgres://daoud@" uv run python scripts/llm_relabel_build_dataset.py"""+import jsonimport pandas as pd@@ -43,20 +44,22 @@ def main():j = judged.get((q, c["url"]))if not j:continue- rows.append({- "gold_standard_rank": c["gold_rank"],- "query": q,- "url": c["url"],- "title": c["title"],- "extract": c["extract"],- "state": c["state"],- "score": c["score"],- "relevance": j["relevance"],- "ethos": j["ethos"],- "overall": j["overall"],- "pools": "|".join(c["pools"]),- "ss_source": c["ss_source"] or "",- })+ rows.append(+ {+ "gold_standard_rank": c["gold_rank"],+ "query": q,+ "url": c["url"],+ "title": c["title"],+ "extract": c["extract"],+ "state": c["state"],+ "score": c["score"],+ "relevance": j["relevance"],+ "ethos": j["ethos"],+ "overall": j["overall"],+ "pools": "|".join(c["pools"]),+ "ss_source": c["ss_source"] or "",+ }+ )df = pd.DataFrame(rows)df.to_csv(OUT, errors="replace")@@ -65,15 +68,19 @@ def main():for tag in ("standard", "supersearch", "google"):sub = df[df["pools"].str.contains(tag)]if len(sub):- print(f" {tag:12s} n={len(sub):6d} relevance={sub.relevance.mean():.2f} "- f"ethos={sub.ethos.mean():.2f} overall={sub.overall.mean():.2f}")+ print(+ f" {tag:12s} n={len(sub):6d} relevance={sub.relevance.mean():.2f} "+ f"ethos={sub.ethos.mean():.2f} overall={sub.overall.mean():.2f}"+ )# how much graded signal lives outside the old binary (in-gold) labelin_gold = df["gold_standard_rank"].notna()rel = df["relevance"] >= 2print(f"\nrelevant (relevance>=2): {rel.sum()}")print(f" of which in Google gold : {(rel & in_gold).sum()}")- print(f" of which NOT in gold : {(rel & ~in_gold).sum()} "- f"({100*(rel & ~in_gold).sum()/rel.sum():.0f}% recovered by the relabel)")+ print(+ f" of which NOT in gold : {(rel & ~in_gold).sum()} "+ f"({100 * (rel & ~in_gold).sum() / rel.sum():.0f}% recovered by the relabel)"+ )if __name__ == "__main__":diff --git a/scripts/llm_relabel_pass1.py b/scripts/llm_relabel_pass1.pyindex db53c3c..5b015d4 100644--- a/scripts/llm_relabel_pass1.py+++ b/scripts/llm_relabel_pass1.py@@ -10,6 +10,7 @@ Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev DATABASE_URL="postgres://daoud@" \uv run python scripts/llm_relabel_pass1.py --num 100 --seed 42"""+import jsonimport osfrom argparse import ArgumentParser@@ -36,10 +37,22 @@ ADAPTER_DESC = {"arxiv": ("academia", "arXiv — academic preprints in physics, CS, maths, biology, etc."),"pypi": ("code", "PyPI — the Python package index."),"imdb": ("entertainment", "IMDb — films, TV shows, actors, and entertainment."),- "nhs": ("health", "NHS — authoritative UK consumer health: conditions, symptoms, treatments, medicines and dosages."),- "openstreetmap_org": ("maps-places", "OpenStreetMap — places, towns, addresses, and local points of interest (maps/local)."),- "wikidata_official": ("navigational", "Official website of a named brand, company, organisation or person (resolved via Wikidata)."),- "homepage": ("navigational", "Best-guess official homepage for a brand or site name — for navigational brand queries."),+ "nhs": (+ "health",+ "NHS — authoritative UK consumer health: conditions, symptoms, treatments, medicines and dosages.",+ ),+ "openstreetmap_org": (+ "maps-places",+ "OpenStreetMap — places, towns, addresses, and local points of interest (maps/local).",+ ),+ "wikidata_official": (+ "navigational",+ "Official website of a named brand, company, organisation or person (resolved via Wikidata).",+ ),+ "homepage": (+ "navigational",+ "Best-guess official homepage for a brand or site name — for navigational brand queries.",+ ),"guardian": ("news", "The Guardian — UK/world news, politics, sport and current events."),}@@ -113,7 +126,7 @@ def main():# Catalog text block (for the Haiku prompt)print("\n===CATALOG===")for name, meta in sorted(catalog.items()):- print(f"{name} [{meta.get('field','')}]: {meta['description']}")+ print(f"{name} [{meta.get('field', '')}]: {meta['description']}")print("\n===QUERIES===")for q in sample:print(q)diff --git a/scripts/llm_relabel_pass1_run.py b/scripts/llm_relabel_pass1_run.pyindex 1fff4d1..1864d04 100644--- a/scripts/llm_relabel_pass1_run.py+++ b/scripts/llm_relabel_pass1_run.py@@ -21,6 +21,7 @@ Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev DATABASE_URL="postgres://daoud@" \uv run python scripts/llm_relabel_pass1_run.py --dump-batch 200"""+import jsonimport osfrom argparse import ArgumentParser@@ -34,12 +35,11 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")django.setup()from mwmbl.rankeval.paths import RANKINGS_DATASET_TRAIN_PATH # noqa: E402-from scripts.llm_relabel_pass1 import INTENT_SOURCES, augment_sources # noqa: E402+from scripts.llm_relabel_pass1 import augment_sources # noqa: E402CHECKPOINT = "devdata/llm_relabel/pass1_intents.jsonl"SEED = 42-INTENTS = {"navigational", "entity", "informational", "transactional",- "local", "news", "tool", "ambiguous"}+INTENTS = {"navigational", "entity", "informational", "transactional", "local", "news", "tool", "ambiguous"}# Curated catalog the judge chooses from (matches the validated Pass-1 prompt).CATALOG_BLOCK = """\@@ -171,9 +171,7 @@ def cmd_merge(path: str):skipped += 1continuepicks = [] if src.lower() == "none" else [s.strip() for s in src.split(",") if s.strip()]- rec = {"query": query, "intent": intent,- "haiku_sources": picks,- "sources": augment_sources(intent, picks)}+ rec = {"query": query, "intent": intent, "haiku_sources": picks, "sources": augment_sources(intent, picks)}out.write(json.dumps(rec) + "\n")done[query] = recadded += 1diff --git a/scripts/llm_relabel_pass2_collect.py b/scripts/llm_relabel_pass2_collect.pyindex a13cc0a..83ca464 100644--- a/scripts/llm_relabel_pass2_collect.py+++ b/scripts/llm_relabel_pass2_collect.py@@ -22,6 +22,7 @@ Usage::... uv run python scripts/llm_relabel_pass2_collect.py # full run... uv run python scripts/llm_relabel_pass2_collect.py --status"""+import asyncioimport jsonimport os@@ -54,8 +55,10 @@ ALWAYS_ON = {name for name in SOURCES if get_meta(name).always_on}# that dataset.py uses to build the existing LTR data, so the relabel is a# drop-in replacement (score_threshold=-inf keeps all candidates for pooling)._std_ranker = HeuristicAndWikiRanker(- RemoteIndex(), DummyCompleter(),- return_none_if_no_mwmbl_results=True, score_threshold=float("-inf"),+ RemoteIndex(),+ DummyCompleter(),+ return_none_if_no_mwmbl_results=True,+ score_threshold=float("-inf"),)@@ -80,15 +83,34 @@ async def _collect_ss(query: str, sources: list[str]) -> tuple[list[Document], dreturn all_docs, dict(ctx.source_by_url)-def _add(pool: dict, url: str, title: str, extract: str, state, score, *,- pool_tag: str, ss_source: str | None = None, gold_rank: int | None = None):+def _add(+ pool: dict,+ url: str,+ title: str,+ extract: str,+ state,+ score,+ *,+ pool_tag: str,+ ss_source: str | None = None,+ gold_rank: int | None = None,+):"""Merge one result into the per-url pool, keeping the richest title/extract."""if not url:return- item = pool.setdefault(url, {- "url": url, "title": "", "extract": "", "state": None, "score": None,- "pools": [], "ss_source": None, "gold_rank": None,- })+ item = pool.setdefault(+ url,+ {+ "url": url,+ "title": "",+ "extract": "",+ "state": None,+ "score": None,+ "pools": [],+ "ss_source": None,+ "gold_rank": None,+ },+ )if title and len(title) > len(item["title"]):item["title"] = titleif extract and len(extract) > len(item["extract"]):@@ -113,8 +135,11 @@ def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k:# The heuristic ranker returns the whole recall set; TREC-style we pool only# the top-K by score (what could realistically rank), but always keep any# gold URL as a calibration anchor.- std = sorted((_std_ranker.search(query + " ", []) or []),- key=lambda d: d.score if d.score is not None else float("-inf"), reverse=True)+ std = sorted(+ (_std_ranker.search(query + " ", []) or []),+ key=lambda d: d.score if d.score is not None else float("-inf"),+ reverse=True,+ )kept = std[:std_top_k] + [d for d in std[std_top_k:] if d.url in gold_urls]for d in kept:_add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")@@ -129,13 +154,11 @@ def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k:src = source_by_url.get(d.url)if src in stray:continue- _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score,- pool_tag="supersearch", ss_source=src)+ _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="supersearch", ss_source=src)# google gold (snippet only)for _, row in gold.iterrows():- _add(pool, row["url"], "", str(row.get("snippet") or ""), None, None,- pool_tag="google", gold_rank=row["rank"])+ _add(pool, row["url"], "", str(row.get("snippet") or ""), None, None, pool_tag="google", gold_rank=row["rank"])return {"query": query, "sources": sources, "candidates": list(pool.values())}@@ -153,18 +176,19 @@ def _load_pass1() -> dict:def _done() -> set:if not os.path.exists(CHECKPOINT):return set()- return {json.loads(l)["query"] for l in open(CHECKPOINT) if l.strip()}+ return {json.loads(line)["query"] for line in open(CHECKPOINT) if line.strip()}def cmd_status():p1, done = _load_pass1(), _done()print(f"collected {len(done)} / {len(p1)} pass-1 queries ({CHECKPOINT})")if done:- recs = [json.loads(l) for l in open(CHECKPOINT) if l.strip()]+ recs = [json.loads(line) for line in open(CHECKPOINT) if line.strip()]n_cand = sum(len(r["candidates"]) for r in recs)from collections import Counter+pools = Counter(t for r in recs for c in r["candidates"] for t in c["pools"])- print(f" {n_cand} candidates total, {n_cand / max(len(recs),1):.1f} avg/query")+ print(f" {n_cand} candidates total, {n_cand / max(len(recs), 1):.1f} avg/query")print(f" pool membership: {dict(pools)}")@@ -179,8 +203,7 @@ def run(limit: int | None, std_top_k: int):print(f"collecting {len(todo)} queries (skipping {len(done)} done)")with open(CHECKPOINT, "a") as out:for i, query in enumerate(todo, 1):- rec = collect_query(query, p1[query]["sources"],- by_query.get(query, gold_all.iloc[0:0]), std_top_k)+ rec = collect_query(query, p1[query]["sources"], by_query.get(query, gold_all.iloc[0:0]), std_top_k)out.write(json.dumps(rec) + "\n")out.flush()print(f"[{i}/{len(todo)}] {query!r}: {len(rec['candidates'])} candidates", flush=True)@@ -189,8 +212,9 @@ def run(limit: int | None, std_top_k: int):def main():parser = ArgumentParser()parser.add_argument("--limit", type=int, default=None)- parser.add_argument("--std-top-k", type=int, default=30,- help="Pool only the top-K standard candidates by score (+ any gold URL).")+ parser.add_argument(+ "--std-top-k", type=int, default=30, help="Pool only the top-K standard candidates by score (+ any gold URL)."+ )parser.add_argument("--status", action="store_true")args = parser.parse_args()if args.status:diff --git a/scripts/llm_relabel_pass3_judge.py b/scripts/llm_relabel_pass3_judge.pyindex a922e74..e864b21 100644--- a/scripts/llm_relabel_pass3_judge.py+++ b/scripts/llm_relabel_pass3_judge.py@@ -26,6 +26,7 @@ Usage::... --merge judge_out.txt --tag cal1... --calibration"""+import jsonimport osfrom argparse import ArgumentParser@@ -131,7 +132,7 @@ CANDIDATES TO JUDGE:def _load_jsonl(path):if not os.path.exists(path):return []- return [json.loads(l) for l in open(path) if l.strip()]+ return [json.loads(line) for line in open(path) if line.strip()]def _pool():@@ -148,8 +149,7 @@ def _done_pairs():def cmd_dump_batch(n_queries: int, tag: str):pool, intents, done = _pool(), _intents(), _done_pairs()- todo = [q for q, r in pool.items()- if any((q, c["url"]) not in done for c in r["candidates"])]+ todo = [q for q, r in pool.items() if any((q, c["url"]) not in done for c in r["candidates"])]batch = todo[:n_queries]manifest, lines, cid = [], [], 0@@ -187,8 +187,7 @@ def cmd_merge(path: str, tag: str):bad += 1continuetry:- cid, rel, eth, ov = (int(parts[0]), int(parts[1]),- int(parts[2]), int(parts[3]))+ cid, rel, eth, ov = (int(parts[0]), int(parts[1]), int(parts[2]), int(parts[3]))except ValueError:bad += 1continue@@ -200,8 +199,9 @@ def cmd_merge(path: str, tag: str):if key in done:skipped += 1continue- out.write(json.dumps({"query": m["query"], "url": m["url"],- "relevance": rel, "ethos": eth, "overall": ov}) + "\n")+ out.write(+ json.dumps({"query": m["query"], "url": m["url"], "relevance": rel, "ethos": eth, "overall": ov}) + "\n"+ )done.add(key)added += 1print(f"merged {path} (tag={tag}): +{added} added, {skipped} dup, {bad} malformed")@@ -212,8 +212,7 @@ def cmd_status():judged = _load_jsonl(CHECKPOINT)total_cand = sum(len(r["candidates"]) for r in pool.values())qs_done = {j["query"] for j in judged}- print(f"judged {len(judged)} / {total_cand} candidates; "- f"queries touched {len(qs_done)} / {len(pool)}")+ print(f"judged {len(judged)} / {total_cand} candidates; queries touched {len(qs_done)} / {len(pool)}")if judged:for axis in ("relevance", "ethos", "overall"):dist = Counter(j[axis] for j in judged)@@ -225,7 +224,7 @@ def cmd_calibration():pool = _pool()judged = {(j["query"], j["url"]): j for j in _load_jsonl(CHECKPOINT)}# url provenance from the pool- buckets = defaultdict(list) # pool tag -> list of judgments+ buckets = defaultdict(list) # pool tag -> list of judgmentsfor q, r in pool.items():for c in r["candidates"]:j = judged.get((q, c["url"]))@@ -256,7 +255,7 @@ def cmd_calibration():for s, js in sorted(src.items(), key=lambda kv: -len(kv[1])):n = len(js)rel = sum(j["relevance"] for j in js) / n- print(f" {s:18s} n={n:4d} rel={rel:.2f} overall={sum(j['overall'] for j in js)/n:.2f}")+ print(f" {s:18s} n={n:4d} rel={rel:.2f} overall={sum(j['overall'] for j in js) / n:.2f}")def main():diff --git a/scripts/modal_judge_train.py b/scripts/modal_judge_train.pyindex c36bf68..7a6c37d 100644--- a/scripts/modal_judge_train.py+++ b/scripts/modal_judge_train.py@@ -23,6 +23,7 @@ One-time auth: uv run --with modal modal token newRun: uv run --with modal modal run scripts/modal_judge_train.py \--base minilm --tasks both --run-name minilm-both-v1"""+import ioimport jsonimport subprocess@@ -42,10 +43,10 @@ app = modal.App("judge-train")image = (modal.Image.debian_slim(python_version="3.11")- .pip_install("torch", "sentence-transformers>=4.1", "transformers",- "accelerate", "datasets", "optimum[onnxruntime]>=1.24")- .add_local_dir("devdata/judge_train", "/repo/devdata/judge_train",- ignore=["models/**"])+ .pip_install(+ "torch", "sentence-transformers>=4.1", "transformers", "accelerate", "datasets", "optimum[onnxruntime]>=1.24"+ )+ .add_local_dir("devdata/judge_train", "/repo/devdata/judge_train", ignore=["models/**"]).add_local_file(LLM_DATASET, f"/repo/{LLM_DATASET}").add_local_file("scripts/judge_bakeoff.py", "/repo/scripts/judge_bakeoff.py"))@@ -55,13 +56,23 @@ volume = modal.Volume.from_name("judge-train", create_if_missing=True)def load_jsonl_gz(path):import gzip+with gzip.open(path, "rt") as f:return [json.loads(line) for line in f]@app.function(image=image, gpu="T4", timeout=3600, volumes={"/ckpt": volume})-def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,- batch_size: int, eval_steps: int, seed: int, git_sha: str) -> bytes:+def train(+ base: str,+ tasks: str,+ run_name: str,+ epochs: float,+ lr: float,+ batch_size: int,+ eval_steps: int,+ seed: int,+ git_sha: str,+) -> bytes:import osimport sysimport time@@ -69,10 +80,8 @@ def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,import numpy as npimport torchfrom datasets import Dataset- from sentence_transformers.cross_encoder import (- CrossEncoder, CrossEncoderTrainer, CrossEncoderTrainingArguments)- from sentence_transformers.cross_encoder.losses import (- BinaryCrossEntropyLoss, RankNetLoss)+ from sentence_transformers.cross_encoder import CrossEncoder, CrossEncoderTrainer, CrossEncoderTrainingArguments+ from sentence_transformers.cross_encoder.losses import BinaryCrossEntropyLoss, RankNetLossfrom sentence_transformers.evaluation import SentenceEvaluatorfrom sentence_transformers.training_args import MultiDatasetBatchSamplersfrom transformers import set_seed@@ -97,13 +106,13 @@ def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,model.max_length = MAX_LENGTHif tasks in ("both", "pointwise"):train_datasets["pointwise"] = Dataset.from_list(- [{"query": r["query"], "doc": r["doc_text"], "label": float(r["label"])}- for r in pointwise_train])+ [{"query": r["query"], "doc": r["doc_text"], "label": float(r["label"])} for r in pointwise_train]+ )train_losses["pointwise"] = BinaryCrossEntropyLoss(model)if tasks in ("both", "pairs"):train_datasets["pairs"] = Dataset.from_list(- [{"query": r["query"], "docs": [r["pos"], r["neg"]],- "labels": [1.0, 0.0]} for r in pairs_train])+ [{"query": r["query"], "docs": [r["pos"], r["neg"]], "labels": [1.0, 0.0]} for r in pairs_train]+ )train_losses["pairs"] = RankNetLoss(model)class JudgeEvaluator(SentenceEvaluator):@@ -112,18 +121,15 @@ def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,def __init__(self):super().__init__()self.primary_metric = "pairs_accuracy"- self.pair_inputs = [(p["query"], p["pos"]) for p in pairs_val] + \- [(p["query"], p["neg"]) for p in pairs_val]+ self.pair_inputs = [(p["query"], p["pos"]) for p in pairs_val] + [(p["query"], p["neg"]) for p in pairs_val]self.point_inputs = [(r["query"], r["doc_text"]) for r in pointwise_val]self.point_labels = np.array([r["label"] for r in pointwise_val])def __call__(self, model, output_path=None, epoch=-1, steps=-1):- scores = model.predict(self.pair_inputs, batch_size=256,- show_progress_bar=False)+ scores = model.predict(self.pair_inputs, batch_size=256, show_progress_bar=False)half = len(pairs_val)accuracy = float(np.mean(scores[:half] > scores[half:]))- point = model.predict(self.point_inputs, batch_size=256,- show_progress_bar=False)+ point = model.predict(self.point_inputs, batch_size=256, show_progress_bar=False)rho = judge_bakeoff.spearman(point, self.point_labels)metrics = {"pairs_accuracy": accuracy, "pointwise_spearman": rho}print(f"eval @ step {steps}: {metrics}", flush=True)@@ -153,9 +159,9 @@ def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,multi_dataset_batch_sampler=MultiDatasetBatchSamplers.PROPORTIONAL,report_to="none",)- trainer = CrossEncoderTrainer(model=model, args=args,- train_dataset=train_datasets,- loss=train_losses, evaluator=evaluator)+ trainer = CrossEncoderTrainer(+ model=model, args=args, train_dataset=train_datasets, loss=train_losses, evaluator=evaluator+ )trainer.train()final = evaluator(model, steps=-1)@@ -166,56 +172,58 @@ def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,artifact_dir = Path(f"/tmp/artifact/{run_name}")artifact_dir.mkdir(parents=True)rows = judge_bakeoff.load_dataset()- llm_scores = model.predict([(r["query"], r["doc_text"]) for r in rows],- batch_size=256, show_progress_bar=False)+ llm_scores = model.predict([(r["query"], r["doc_text"]) for r in rows], batch_size=256, show_progress_bar=False)np.save(artifact_dir / "llm_scores.npy", np.asarray(llm_scores, dtype=np.float64))- pos = model.predict([(p["query"], p["pos"]) for p in pairs_eval],- batch_size=256, show_progress_bar=False)- neg = model.predict([(p["query"], p["neg"]) for p in pairs_eval],- batch_size=256, show_progress_bar=False)+ pos = model.predict([(p["query"], p["pos"]) for p in pairs_eval], batch_size=256, show_progress_bar=False)+ neg = model.predict([(p["query"], p["neg"]) for p in pairs_eval], batch_size=256, show_progress_bar=False)np.savez(artifact_dir / "pairs_eval_scores.npz", pos=pos, neg=neg)parity_indexes = np.random.default_rng(0).choice(len(rows), 512, replace=False)- parity = [{"query": rows[i]["query"], "doc_text": rows[i]["doc_text"],- "torch_score": float(llm_scores[i])} for i in parity_indexes]+ parity = [+ {"query": rows[i]["query"], "doc_text": rows[i]["doc_text"], "torch_score": float(llm_scores[i])}+ for i in parity_indexes+ ](artifact_dir / "parity_sample.json").write_text(json.dumps(parity))# --- ONNX export: fp32 (O2-optimized) + dynamic int8 ----------------------onnx_error = Nonetry:- from optimum.onnxruntime import (ORTModelForSequenceClassification,- ORTOptimizer, ORTQuantizer)- from optimum.onnxruntime.configuration import (AutoQuantizationConfig,- OptimizationConfig)+ from optimum.onnxruntime import ORTModelForSequenceClassification, ORTOptimizer, ORTQuantizer+ from optimum.onnxruntime.configuration import AutoQuantizationConfig, OptimizationConfigonnx_dir = artifact_dir / "onnx"- ort_model = ORTModelForSequenceClassification.from_pretrained(- final_dir, export=True)+ ort_model = ORTModelForSequenceClassification.from_pretrained(final_dir, export=True)ort_model.save_pretrained(onnx_dir) # raw model.onnx# quantize from the raw export — the O2-optimized fused graph breaks# onnx shape inference inside the quantizerquantizer = ORTQuantizer.from_pretrained(onnx_dir, file_name="model.onnx")- quantizer.quantize(save_dir=onnx_dir,- quantization_config=AutoQuantizationConfig.avx2(- is_static=False, per_channel=False))+ quantizer.quantize(+ save_dir=onnx_dir, quantization_config=AutoQuantizationConfig.avx2(is_static=False, per_channel=False)+ )(onnx_dir / "model_quantized.onnx").rename(onnx_dir / "model.int8.onnx")optimizer = ORTOptimizer.from_pretrained(ort_model)- optimizer.optimize(save_dir=onnx_dir, optimization_config=OptimizationConfig(- optimization_level=2))+ optimizer.optimize(save_dir=onnx_dir, optimization_config=OptimizationConfig(optimization_level=2))(onnx_dir / "model_optimized.onnx").rename(onnx_dir / "model.onnx")model.tokenizer.save_pretrained(str(onnx_dir))except Exception as exc: # jina custom code may not export; keep torch armonnx_error = f"{type(exc).__name__}: {exc}"- print(f"ONNX export FAILED (torch scores still usable): {onnx_error}",- flush=True)+ print(f"ONNX export FAILED (torch scores still usable): {onnx_error}", flush=True)meta = {- "run_name": run_name, "base": base, "tasks": tasks,- "epochs": epochs, "lr": lr, "batch_size": batch_size,- "eval_steps": eval_steps, "seed": seed, "max_length": MAX_LENGTH,- "git_sha": git_sha, "train_pairs": len(pairs_train),+ "run_name": run_name,+ "base": base,+ "tasks": tasks,+ "epochs": epochs,+ "lr": lr,+ "batch_size": batch_size,+ "eval_steps": eval_steps,+ "seed": seed,+ "max_length": MAX_LENGTH,+ "git_sha": git_sha,+ "train_pairs": len(pairs_train),"train_pointwise": len(pointwise_train),- "zero_shot_val": baseline, "final_val": final,+ "zero_shot_val": baseline,+ "final_val": final,"best_checkpoint": trainer.state.best_model_checkpoint,"best_metric": trainer.state.best_metric,"onnx_error": onnx_error,@@ -233,17 +241,22 @@ def train(base: str, tasks: str, run_name: str, epochs: float, lr: float,@app.local_entrypoint()-def main(base: str = "minilm", tasks: str = "both", run_name: str = None,- epochs: float = 2.0, lr: float = 2e-5, batch_size: int = 64,- eval_steps: int = 200, seed: int = 42):+def main(+ base: str = "minilm",+ tasks: str = "both",+ run_name: str = None,+ epochs: float = 2.0,+ lr: float = 2e-5,+ batch_size: int = 64,+ eval_steps: int = 200,+ seed: int = 42,+):assert base in BASES, f"--base must be one of {sorted(BASES)}"assert tasks in ("both", "pairs", "pointwise")run_name = run_name or f"{base}-{tasks}-v1"- git_sha = subprocess.run(["git", "rev-parse", "--short", "HEAD"],- capture_output=True, text=True).stdout.strip()+ git_sha = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()- artifact = train.remote(BASES[base], tasks, run_name, epochs, lr,- batch_size, eval_steps, seed, git_sha)+ artifact = train.remote(BASES[base], tasks, run_name, epochs, lr, batch_size, eval_steps, seed, git_sha)out_dir = Path("devdata/judge_train/models") / run_nameout_dir.mkdir(parents=True, exist_ok=True)diff --git a/scripts/moderation_eval.py b/scripts/moderation_eval.pyindex eff45e4..c8bf41b 100644--- a/scripts/moderation_eval.py+++ b/scripts/moderation_eval.py@@ -12,6 +12,7 @@ Reports every metric with a bootstrap interval. That is not decoration: the coldhas ~80 positives, and the augmentation experiment that motivated the derived-data designproduced a 0.04 PR-AUC "gain" whose interval spanned 0.17."""+import argparseimport gzipimport json@@ -28,10 +29,15 @@ import djangodjango.setup()-from mwmbl.moderation.train import train # noqa: E402from mwmbl.moderation.model import METRICS_FILENAME, MODEL_FILENAME # noqa: E402-from mwmbl.moderation.training_data import ( # noqa: E402- REAL, TrainingRow, is_trainable_domain, derived_rows, seed_rows)+from mwmbl.moderation.train import train # noqa: E402+from mwmbl.moderation.training_data import ( # noqa: E402+ REAL,+ TrainingRow,+ derived_rows,+ is_trainable_domain,+ seed_rows,+)DEFAULT_EXPORT = Path("devdata/judgments_export/domains.jsonl.gz")ARTIFACT_DIR = Path("mwmbl/moderation/artifacts")@@ -46,33 +52,40 @@ def rows_from_export(path: Path) -> list[TrainingRow]:if not is_trainable_domain(record["domain"]):continuerejected = record["status"] == "REJECTED"- rows.append(TrainingRow(- domain=record["domain"],- rejected=rejected,- reason=record.get("rejection_reason") or "" if rejected else "",- source=REAL,- # The export carries no page text: it predates the evidence crawl. So this- # measures the domain-name-only floor, which is what the model falls back to for- # any submission whose crawl failed.- page_texts=[],- timestamp=record["timestamp"],- submitter=record.get("user"),- ))+ rows.append(+ TrainingRow(+ domain=record["domain"],+ rejected=rejected,+ reason=record.get("rejection_reason") or "" if rejected else "",+ source=REAL,+ # The export carries no page text: it predates the evidence crawl. So this+ # measures the domain-name-only floor, which is what the model falls back to for+ # any submission whose crawl failed.+ page_texts=[],+ timestamp=record["timestamp"],+ submitter=record.get("user"),+ )+ )return rowsdef main() -> int:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--export", type=Path, default=DEFAULT_EXPORT)- parser.add_argument("--derived", type=int, default=0,- help="Blocklist rows per under-represented reason (0 = skip the fetch)")+ parser.add_argument(+ "--derived", type=int, default=0, help="Blocklist rows per under-represented reason (0 = skip the fetch)"+ )parser.add_argument("--no-seed", action="store_true")parser.add_argument(- "--write-artifact", action="store_true",- help=(f"Overwrite the committed warm start in {ARTIFACT_DIR}. Needed after a change to "- "the feature set: a featuriser is pickled whole, so the shipped artifact then "- "describes a matrix the code no longer builds, fails the load-time probe, and "- "every deploy degrades to the deterministic checks until the first retrain."))+ "--write-artifact",+ action="store_true",+ help=(+ f"Overwrite the committed warm start in {ARTIFACT_DIR}. Needed after a change to "+ "the feature set: a featuriser is pickled whole, so the shipped artifact then "+ "describes a matrix the code no longer builds, fails the load-time probe, and "+ "every deploy degrades to the deterministic checks until the first retrain."+ ),+ )options = parser.parse_args()if not options.export.exists():@@ -80,30 +93,29 @@ def main() -> int:return 1rows = rows_from_export(options.export)- print(f"real rows: {len(rows)} "- f"({sum(row.rejected for row in rows)} rejected)")+ print(f"real rows: {len(rows)} ({sum(row.rejected for row in rows)} rejected)")if not options.no_seed:rows += seed_rows()if options.derived:- rows += derived_rows(["OFFENSIVE"], options.derived,- exclude={row.domain for row in rows})+ rows += derived_rows(["OFFENSIVE"], options.derived, exclude={row.domain for row in rows})# Dated like a retrain's, because this artifact is served in production until the first# retrain publishes and "offline-eval" in a worker log says nothing about how old it is -# but suffixed, because it is *not* one. DomainEvidence.model_version records which model# scored a row, and a warm start sharing a retrain's version string makes "has the rescore# run yet?" unanswerable from the data.- version = (f"domain-mod-{date.today():%Y-%m-%d}-warmstart" if options.write_artifact- else "offline-eval")+ version = f"domain-mod-{date.today():%Y-%m-%d}-warmstart" if options.write_artifact else "offline-eval"model, metrics = train(rows, version)print(json.dumps(metrics, indent=2))if options.write_artifact:joblib.dump(model, ARTIFACT_DIR / MODEL_FILENAME)(ARTIFACT_DIR / METRICS_FILENAME).write_text(json.dumps(metrics, indent=2) + "\n")- print(f"\nWrote the warm start to {ARTIFACT_DIR}. This is the day-one model only - a "- f"retrain against the production database still replaces it.")+ print(+ f"\nWrote the warm start to {ARTIFACT_DIR}. This is the day-one model only - a "+ f"retrain against the production database still replaces it."+ )return 0diff --git a/scripts/moderation_llm_bakeoff.py b/scripts/moderation_llm_bakeoff.pyindex c515153..a43a47a 100644--- a/scripts/moderation_llm_bakeoff.py+++ b/scripts/moderation_llm_bakeoff.py@@ -20,8 +20,8 @@ Three steps, because the middle one runs outside this process:The export deliberately carries no labels and no submitter, so the file handed to a judge -which may be a third party - cannot leak the answer or say who submitted what."""+import argparse-import gzipimport jsonimport osimport sys@@ -33,15 +33,19 @@ import djangodjango.setup()-import numpy as np # noqa: E402-from sklearn.metrics import average_precision_score # noqa: E402+import numpy as np # noqa: E402+from sklearn.metrics import average_precision_score # noqa: E402-from mwmbl.moderation.train import ( # noqa: E402- BOOTSTRAP_SAMPLES, normalise, split_by_time, train)-from mwmbl.moderation.training_data import seed_rows # noqa: E402+from mwmbl.moderation.train import ( # noqa: E402+ BOOTSTRAP_SAMPLES,+ normalise,+ split_by_time,+ train,+)+from mwmbl.moderation.training_data import seed_rows # noqa: E402sys.path.insert(0, str(Path(__file__).resolve().parent))-from moderation_eval import DEFAULT_EXPORT, rows_from_export # noqa: E402+from moderation_eval import DEFAULT_EXPORT, rows_from_export # noqa: E402DOMAINS_FILE = "domains.txt"TRUTH_FILE = "truth.jsonl"@@ -73,14 +77,13 @@ def few_shot_examples(train_rows, wanted: int) -> list:# Round-robin across reasons so a rare class - OFFENSIVE has one real example in the whole# dataset - is not crowded out by SPAM, which has hundreds.rejections, index = [], 0- while len(rejections) < wanted // 2 and any(- len(rows) > index for rows in by_reason.values()):+ while len(rejections) < wanted // 2 and any(len(rows) > index for rows in by_reason.values()):for reason in sorted(by_reason):if len(by_reason[reason]) > index and len(rejections) < wanted // 2:rejections.append(by_reason[reason][index])index += 1- chosen = approvals[:wanted - len(rejections)] + rejections+ chosen = approvals[: wanted - len(rejections)] + rejectionsreturn sorted(chosen, key=lambda row: row.domain)@@ -96,8 +99,7 @@ def cold_start_test_rows(export: Path):known = {row.submitter for row in train_rows if row.submitter}cold = [row for row in test_rows if row.submitter not in known]- scores = [prediction[0] for prediction in- model.predict([row.to_example() for row in cold])]+ scores = [prediction[0] for prediction in model.predict([row.to_example() for row in cold])]return cold, np.array(scores)@@ -109,29 +111,38 @@ def export(options) -> int:train_rows, _ = split_by_time(all_rows)examples = few_shot_examples(train_rows, options.examples)held_out = {row.domain for row in rows}- assert not {row.domain for row in examples} & held_out, (- "a few-shot example leaked in from the held-out rows")+ assert not {row.domain for row in examples} & held_out, "a few-shot example leaked in from the held-out rows"(options.out / EXAMPLES_FILE).write_text(- "\n".join(f"- `{row.domain}` -> "- + (f"REJECT ({row.reason or 'OTHER'})" if row.rejected else "APPROVE")- for row in examples) + "\n")-- (options.out / DOMAINS_FILE).write_text(- "".join(f"{row.domain}\n" for row in rows))+ "\n".join(+ f"- `{row.domain}` -> " + (f"REJECT ({row.reason or 'OTHER'})" if row.rejected else "APPROVE")+ for row in examples+ )+ + "\n"+ )++ (options.out / DOMAINS_FILE).write_text("".join(f"{row.domain}\n" for row in rows))# The labels and the model's own scores stay on this side, keyed by domain, so scoring can# join them back without the judge ever having seen them.with (options.out / TRUTH_FILE).open("w") as truth:for row, score in zip(rows, scores):- truth.write(json.dumps({"domain": row.domain, "rejected": row.rejected,- "reason": row.reason, "model_score": float(score)}) + "\n")-- print(f"{len(examples)} training-split examples "- f"({sum(row.rejected for row in examples)} rejections) -> "- f"{options.out / EXAMPLES_FILE}")+ truth.write(+ json.dumps(+ {"domain": row.domain, "rejected": row.rejected, "reason": row.reason, "model_score": float(score)}+ )+ + "\n"+ )++ print(+ f"{len(examples)} training-split examples "+ f"({sum(row.rejected for row in examples)} rejections) -> "+ f"{options.out / EXAMPLES_FILE}"+ )print(f"{len(rows)} cold-start held-out domains -> {options.out / DOMAINS_FILE}")- print(f"{sum(row.rejected for row in rows)} of them were rejected "- f"(base rate {sum(row.rejected for row in rows) / len(rows):.3f})")+ print(+ f"{sum(row.rejected for row in rows)} of them were rejected "+ f"(base rate {sum(row.rejected for row in rows) / len(rows):.3f})"+ )return 0@@ -173,8 +184,10 @@ def paired_bootstrap(truth: np.ndarray, left: np.ndarray, right: np.ndarray) ->if sample.sum() < 5 or (1 - sample).sum() < 5:continuebase = sample.mean()- differences.append(normalise(average_precision_score(sample, left[indices]), base)- - normalise(average_precision_score(sample, right[indices]), base))+ differences.append(+ normalise(average_precision_score(sample, left[indices]), base)+ - normalise(average_precision_score(sample, right[indices]), base)+ )base_rate = float(truth.mean())return {@@ -182,16 +195,19 @@ def paired_bootstrap(truth: np.ndarray, left: np.ndarray, right: np.ndarray) ->"right": round(normalise(average_precision_score(truth, right), base_rate), 4),"difference": round(normalise(average_precision_score(truth, left), base_rate)- - normalise(average_precision_score(truth, right), base_rate), 4),- "difference_ci": [round(float(np.percentile(differences, 2.5)), 4),- round(float(np.percentile(differences, 97.5)), 4)],+ - normalise(average_precision_score(truth, right), base_rate),+ 4,+ ),+ "difference_ci": [+ round(float(np.percentile(differences, 2.5)), 4),+ round(float(np.percentile(differences, 97.5)), 4),+ ],"left_wins_fraction": round(float(np.mean(np.array(differences) > 0)), 4),}def score(options) -> int:- truth_rows = [json.loads(line) for line in- (options.dir / TRUTH_FILE).read_text().splitlines() if line.strip()]+ truth_rows = [json.loads(line) for line in (options.dir / TRUTH_FILE).read_text().splitlines() if line.strip()]# One entry per distinct domain, not per submission. A domain submitted nine times is one# thing a moderator decides, and counting it nine times would weight the score by how# popular a submission was rather than by how hard it was to judge. Both rankers are@@ -215,28 +231,34 @@ def score(options) -> int:result = paired_bootstrap(truth, llm, model)print(f"\n{path.name}")- print(f" answered {len(answered)}/{len(by_domain)} domains, "- f"{int(truth.sum())} rejected (base rate {truth.mean():.3f})")- print(f" normalised AP: LLM {result['left']:.4f} model {result['right']:.4f} "- f"difference {result['difference']:+.4f} "- f"[{result['difference_ci'][0]:+.4f}, {result['difference_ci'][1]:+.4f}]")+ print(+ f" answered {len(answered)}/{len(by_domain)} domains, "+ f"{int(truth.sum())} rejected (base rate {truth.mean():.3f})"+ )+ print(+ f" normalised AP: LLM {result['left']:.4f} model {result['right']:.4f} "+ f"difference {result['difference']:+.4f} "+ f"[{result['difference_ci'][0]:+.4f}, {result['difference_ci'][1]:+.4f}]"+ )print(f" the LLM ranks better in {result['left_wins_fraction']:.0%} of resamples")- print(f" raw PR-AUC: LLM "- f"{average_precision_score(truth, llm):.4f} model "- f"{average_precision_score(truth, model):.4f}")+ print(+ f" raw PR-AUC: LLM "+ f"{average_precision_score(truth, llm):.4f} model "+ f"{average_precision_score(truth, model):.4f}"+ )return 0def main() -> int:- parser = argparse.ArgumentParser(description=__doc__,- formatter_class=argparse.RawDescriptionHelpFormatter)+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)subparsers = parser.add_subparsers(dest="command", required=True)exporter = subparsers.add_parser("export")exporter.add_argument("--export", type=Path, default=DEFAULT_EXPORT)exporter.add_argument("--out", type=Path, default=Path("devdata/moderation_bakeoff"))- exporter.add_argument("--examples", type=int, default=60,- help="Labelled training-split decisions to write for few-shot prompts")+ exporter.add_argument(+ "--examples", type=int, default=60, help="Labelled training-split decisions to write for few-shot prompts"+ )exporter.set_defaults(function=export)scorer = subparsers.add_parser("score")diff --git a/scripts/prescreen_sites.py b/scripts/prescreen_sites.pyindex a55de5a..237b8d9 100644--- a/scripts/prescreen_sites.py+++ b/scripts/prescreen_sites.py@@ -21,6 +21,7 @@ Skips domains that already have a recipe. Runs entirely in the main thread.Usage:uv run python scripts/prescreen_sites.py --all --out prescreen.json"""+from __future__ import annotationsimport argparse@@ -43,10 +44,18 @@ CONCURRENCY = 16# Substrings that mark a bot-challenge / block page (lowercased body match).BLOCK_MARKERS = (- "just a moment", "checking your browser", "attention required",- "cf-chl", "cf-challenge", "enable javascript and cookies",- "ddos protection by", "access denied", "request unsuccessful",- "are you a robot", "captcha-delivery", "px-captcha",+ "just a moment",+ "checking your browser",+ "attention required",+ "cf-chl",+ "cf-challenge",+ "enable javascript and cookies",+ "ddos protection by",+ "access denied",+ "request unsuccessful",+ "are you a robot",+ "captcha-delivery",+ "px-captcha",)BLOCK_STATUSES = {401, 403, 405, 406, 429, 503}SEARCH_INPUT_NAMES = {"q", "s", "query", "search", "keyword", "keywords", "term", "k"}@@ -87,8 +96,7 @@ def _harvest(base: str, html: str) -> dict:text_names = [(i.get("name") or "").strip()for i in form.find_all("input")- if (i.get("name") or "").strip()- and (i.get("type") or "text").lower() in ("", "text", "search")+ if (i.get("name") or "").strip() and (i.get("type") or "text").lower() in ("", "text", "search")]hit = [n for n in text_names if n.lower() in SEARCH_INPUT_NAMES]cls = (form.get("class") or [""])[0]@@ -120,8 +128,7 @@ async def _screen(client: httpx.AsyncClient, entry: dict) -> dict:except Exception as e: # noqa: BLE001return {"domain": domain, "status": "error", "detail": repr(e)[:120]}cls = _classify(r.status_code, r.text)- out = {"domain": domain, "field": entry.get("field", ""),- "status": cls, "http": r.status_code}+ out = {"domain": domain, "field": entry.get("field", ""), "status": cls, "http": r.status_code}if cls == "ok":out.update(_harvest(str(r.url), r.text))return out@@ -129,11 +136,12 @@ async def _screen(client: httpx.AsyncClient, entry: dict) -> dict:async def _run(entries: list[dict]) -> list[dict]:sem = asyncio.Semaphore(CONCURRENCY)- async with httpx.AsyncClient(follow_redirects=True, timeout=TIMEOUT,- headers=ENGINE_HEADERS) as client:+ async with httpx.AsyncClient(follow_redirects=True, timeout=TIMEOUT, headers=ENGINE_HEADERS) as client:+async def guarded(e):async with sem:return await _screen(client, e)+return await asyncio.gather(*[guarded(e) for e in entries])@@ -154,8 +162,7 @@ def main() -> int:ap.add_argument("--out", default="prescreen.json")args = ap.parse_args()- paths = sorted(str(p) for p in CHUNKS_DIR.glob("input_*.json")) if args.all \- else args.batches+ paths = sorted(str(p) for p in CHUNKS_DIR.glob("input_*.json")) if args.all else args.batchesif not paths:print("no batch files (use --all or list files)", file=sys.stderr)return 2@@ -171,13 +178,14 @@ def main() -> int:with_search = [r for r in reachable if r.get("opensearch") or r.get("forms")]Path(args.out).write_text(json.dumps(results, indent=2))- print(f"\n ok/reachable : {len(reachable)} "- f"(of which {len(with_search)} expose an OpenSearch/form search URL)")+ print(f"\n ok/reachable : {len(reachable)} (of which {len(with_search)} expose an OpenSearch/form search URL)")print(f" blocked : {len(by.get('blocked', []))}")print(f" error : {len(by.get('error', []))}")print(f" already exist: {len(by.get('exists', []))}")- print(f"\nWrote {args.out}. Reachable sites with a discoverable search URL "- f"are the worthwhile targets for the generation stage.")+ print(+ f"\nWrote {args.out}. Reachable sites with a discoverable search URL "+ f"are the worthwhile targets for the generation stage."+ )return 0diff --git a/scripts/smoke_recipe.py b/scripts/smoke_recipe.pyindex 034b063..77a297b 100644--- a/scripts/smoke_recipe.py+++ b/scripts/smoke_recipe.py@@ -14,6 +14,7 @@ Usage:Prints "PASS <name>" or "FAIL <name>: <reason>" and exits 0/1 accordingly."""+import asyncioimport osimport sys@@ -36,7 +37,8 @@ from mwmbl.tinysearchengine.super_search_sources.smoke import check_recipe # noasync def _run(path: str) -> int:recipe = load_recipe(path)async with httpx.AsyncClient(- follow_redirects=True, timeout=15.0,+ follow_redirects=True,+ timeout=15.0,headers={"User-Agent": "mwmbl-super-search-smoke/0.1 (+https://mwmbl.org)"},) as client:ok, reason = await check_recipe(client, recipe)diff --git a/scripts/super_search_coverage.py b/scripts/super_search_coverage.pyindex 14e72ff..7adfce5 100644--- a/scripts/super_search_coverage.py+++ b/scripts/super_search_coverage.py@@ -20,6 +20,7 @@ Run:DATABASE_URL="postgres://daoud@" DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \uv run python scripts/super_search_coverage.py"""+import argparseimport osimport sys@@ -50,8 +51,7 @@ from mwmbl.tinysearchengine.super_search_select.registry import get_registry #def load_gold() -> pd.DataFrame:frames = []- for split, path in (("train", RANKINGS_DATASET_TRAIN_PATH),- ("test", RANKINGS_DATASET_TEST_PATH)):+ for split, path in (("train", RANKINGS_DATASET_TRAIN_PATH), ("test", RANKINGS_DATASET_TEST_PATH)):df = pd.read_csv(path)df["split"] = splitframes.append(df)@@ -89,7 +89,7 @@ def cmd_report():q_reg: dict[str, set[str]] = defaultdict(set)exact_rows = reg_rows = 0exact_mass = total_mass = 0.0- rank_hist_exact: Counter = Counter() # rank -> hit count (exact)+ rank_hist_exact: Counter = Counter() # rank -> hit count (exact)source_queries_exact: dict[str, set[str]] = defaultdict(set)source_queries_reg: dict[str, set[str]] = defaultdict(set)@@ -239,6 +239,7 @@ def cmd_recall(source: str, per_source_limit: int, max_queries: int):print(f"source {source!r} (registrable domain {dom}): {len(qs)} gold queries touch it")if max_queries and len(qs) > max_queries:import random as _random+_random.Random(0).shuffle(qs)qs = qs[:max_queries]print(f"sampling {len(qs)} of them")@@ -247,8 +248,9 @@ def cmd_recall(source: str, per_source_limit: int, max_queries: int):timeout = max(float(settings.SUPER_SEARCH_PER_SOURCE_TIMEOUT), 10.0)async def run() -> dict[str, list[str]]:- async with httpx.AsyncClient(follow_redirects=True, timeout=timeout,- headers={"User-Agent": "mwmbl-super-search-eval/0.1"}) as client:+ async with httpx.AsyncClient(+ follow_redirects=True, timeout=timeout, headers={"User-Agent": "mwmbl-super-search-eval/0.1"}+ ) as client:out: dict[str, list[str]] = {}for q in qs:try:@@ -274,8 +276,8 @@ def cmd_recall(source: str, per_source_limit: int, max_queries: int):if not ex and len(misses) < 10:misses.append((q, sorted(tgt)[0], ret[:2]))n = len(qs) or 1- print(f"\nexact-URL recall: {exact}/{len(qs)} ({100*exact/n:.1f}%)")- print(f"near-match recall (host+path): {near}/{len(qs)} ({100*near/n:.1f}%)")+ print(f"\nexact-URL recall: {exact}/{len(qs)} ({100 * exact / n:.1f}%)")+ print(f"near-match recall (host+path): {near}/{len(qs)} ({100 * near / n:.1f}%)")print(f"queries where source returned nothing: {empty}/{len(qs)}")print("\nmisses (gold target vs our top returns):")for q, t, ret in misses:@@ -312,10 +314,10 @@ def cmd_missing(top: int):miss.sort(key=lambda x: -x[1])ours_mass = sum(m for d, m in mass.items() if d in ours)print(f"total gold mass {total:.0f} | distinct domains {len(mass)} | missing {len(miss)}")- print(f"domains already a source (registrable match) = {100*ours_mass/total:.2f}% of gold mass\n")+ print(f"domains already a source (registrable match) = {100 * ours_mass / total:.2f}% of gold mass\n")print(f"{'#':>3} {'domain':32} {'mass':>7} {'%mass':>6} {'queries':>8} {'rows':>6} {'rank1-3':>7}")for i, (d, m, nq, nr, t3) in enumerate(miss[:top]):- print(f"{i+1:>3} {d:32} {m:7.1f} {100*m/total:5.1f}% {nq:8} {nr:6} {t3:7}")+ print(f"{i + 1:>3} {d:32} {m:7.1f} {100 * m / total:5.1f}% {nq:8} {nr:6} {t3:7}")print()for n in (10, 20, 40):share = 100 * sum(x[1] for x in miss[:n]) / total@@ -323,8 +325,7 @@ def cmd_missing(top: int):def main():- parser = argparse.ArgumentParser(description=__doc__,- formatter_class=argparse.RawDescriptionHelpFormatter)+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)sub = parser.add_subparsers(dest="command")sub.add_parser("report", help="gold-coverage feasibility report (default)")sub.add_parser("missing", help="rank gold domains we lack by gold mass")diff --git a/scripts/super_search_eval.py b/scripts/super_search_eval.pyindex db08ee6..af6c107 100644--- a/scripts/super_search_eval.py+++ b/scripts/super_search_eval.py@@ -40,6 +40,7 @@ Usage:uv run python scripts/super_search_eval.py select --matrix eval_matrixuv run python scripts/super_search_eval.py simulate --matrix eval_matrix"""+import argparseimport asyncioimport json@@ -59,7 +60,8 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")# host_of/registrable are pure stdlib helpers (no Django) — safe to import at module# load, before django.setup(); the registry-backed source_domain_map is loaded lazily.from mwmbl.tinysearchengine.super_search_select.domains import ( # noqa: E402- host_of, registrable,+ host_of,+ registrable,)@@ -71,20 +73,25 @@ def _bootstrap_django():# build-matrix (network + Django)# ---------------------------------------------------------------------------+async def _query_all_sources(query: str, limit: int):import httpxfrom django.conf import settings+from mwmbl.tinysearchengine.super_search_sources import SOURCEStimeout = settings.SUPER_SEARCH_PER_SOURCE_TIMEOUT- async with httpx.AsyncClient(follow_redirects=True, timeout=timeout,- headers={"User-Agent": "mwmbl-super-search-eval/0.1"}) as client:+ async with httpx.AsyncClient(+ follow_redirects=True, timeout=timeout, headers={"User-Agent": "mwmbl-super-search-eval/0.1"}+ ) as client:+async def one(name, fn):try:docs = await asyncio.wait_for(fn(client, query, limit), timeout=timeout)return name, docsexcept Exception:return name, []+results = await asyncio.gather(*[one(n, f) for n, f in SOURCES.items()])return dict(results)@@ -96,7 +103,6 @@ def _fetch_all_queries(queries: list[str], checkpoint: Path, limit: int) -> dictin the checkpoint are not re-fetched, so a multi-hour network run can beinterrupted and resumed."""- from mwmbl.tinysearchengine.super_search_sources import SOURCESfetched: dict[str, dict] = {}if checkpoint.exists():@@ -108,18 +114,15 @@ def _fetch_all_queries(queries: list[str], checkpoint: Path, limit: int) -> dictwith checkpoint.open("a") as fh:for qi, query in enumerate(todo):docs_by_source = asyncio.run(_query_all_sources(query, limit))- docs = {name: [[d.url, d.title, d.extract] for d in ds]- for name, ds in docs_by_source.items() if ds}+ docs = {name: [[d.url, d.title, d.extract] for d in ds] for name, ds in docs_by_source.items() if ds}fetched[query] = docsfh.write(json.dumps({"query": query, "docs": docs}) + "\n")fh.flush()- print(f" [{qi + 1}/{len(todo)}] {query!r}: "- f"{sum(len(d) for d in docs.values())} docs")+ print(f" [{qi + 1}/{len(todo)}] {query!r}: {sum(len(d) for d in docs.values())} docs")return fetched-def _survival_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int,- limit: int, top_k: int) -> None:+def _survival_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int, limit: int, top_k: int) -> None:"""Reward = fraction of a source's results surviving the LTR model's top-K."""from mwmbl.search_setup import ltr_modelfrom mwmbl.tinysearchengine.indexer import Document@@ -152,24 +155,26 @@ def _judge_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int)judge = get_judge()if judge is None:raise RuntimeError(- "--reward judge requires the relevance judge artifact "- "(settings.SUPER_SEARCH_JUDGE_MODEL_DIR)")+ "--reward judge requires the relevance judge artifact (settings.SUPER_SEARCH_JUDGE_MODEL_DIR)"+ )for name, docs in docs_by_source.items():texts = [doc_text(title, extract) for _, title, extract in docs]if texts:R[qi, s_index[name]] = float(np.mean(judge.score(query, texts)))-def build_matrix(queries: list[str], out: str, reward: str = "survival",- checkpoint: str | None = None):+def build_matrix(queries: list[str], out: str, reward: str = "survival", checkpoint: str | None = None):from django.conf import settings- from mwmbl.tinysearchengine.super_search_sources import SOURCES+from mwmbl.tinysearchengine.super_search_select import vectors+ from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrixfrom mwmbl.tinysearchengine.super_search_select.features import (- FEATURE_NAMES, QueryContext, feature_vector,+ FEATURE_NAMES,+ QueryContext,+ feature_vector,)from mwmbl.tinysearchengine.super_search_select.registry import get_meta- from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix+ from mwmbl.tinysearchengine.super_search_sources import SOURCESsources = list(SOURCES.keys())s_index = {name: i for i, name in enumerate(sources)}@@ -189,8 +194,7 @@ def build_matrix(queries: list[str], out: str, reward: str = "survival",text = " ".join(f"{title or ''} {extract or ''}" for _, title, extract in docs)prof_bow[name] += vectors.project_bow(text, dim)prof_cng[name] += vectors.project_char_ngrams(text, dim)- profile = {n: (vectors._l2_normalise(prof_bow[n]), vectors._l2_normalise(prof_cng[n]))- for n in sources}+ profile = {n: (vectors._l2_normalise(prof_bow[n]), vectors._l2_normalise(prof_cng[n])) for n in sources}# Pass 2: features + rewards.Q, S = len(queries), len(sources)@@ -213,17 +217,18 @@ def build_matrix(queries: list[str], out: str, reward: str = "survival",if (qi + 1) % 50 == 0:print(f" scored [{qi + 1}/{Q}]")- matrix = RewardMatrix(queries=queries, sources=sources,- feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)+ matrix = RewardMatrix(queries=queries, sources=sources, feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)matrix.save(out)- print(f"Wrote matrix {out}.npz/.json ({reward} rewards): {Q} queries x {S} sources, "- f"{int(mask.sum())} filled cells.")+ print(+ f"Wrote matrix {out}.npz/.json ({reward} rewards): {Q} queries x {S} sources, {int(mask.sum())} filled cells."+ )# ---------------------------------------------------------------------------# build-gold-matrix (offline: LTR dataset gold labels, no network)# ---------------------------------------------------------------------------+def _is_gold(rank) -> bool:"""True if ``rank`` is a real gold rank (not pandas NaN / None / blank)."""if rank is None:@@ -244,7 +249,7 @@ def attribute_rows(rows, reg_map):source has a gold-relevant row for the query (else False = available-but-not-gold),and ``prof_text[source]`` accumulates that source's title/extract text."""- per_query: dict[str, dict[str, bool]] = {} # query -> {source -> has_gold}+ per_query: dict[str, dict[str, bool]] = {} # query -> {source -> has_gold}prof_text: dict[str, list[str]] = defaultdict(list)for query, url, title, extract, gold in rows:names = reg_map.get(registrable(host_of(str(url))))@@ -276,14 +281,17 @@ def build_gold_matrix(out: str):"""import pandas as pdfrom django.conf import settings+from mwmbl.rankeval.paths import LEARNING_TO_RANK_DATASET_PATHfrom mwmbl.tinysearchengine.super_search_select import vectorsfrom mwmbl.tinysearchengine.super_search_select.domains import source_domain_map+ from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrixfrom mwmbl.tinysearchengine.super_search_select.features import (- FEATURE_NAMES, QueryContext, feature_vector,+ FEATURE_NAMES,+ QueryContext,+ feature_vector,)from mwmbl.tinysearchengine.super_search_select.registry import get_meta, get_registry- from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrixdim = settings.SUPER_SEARCH_PROJECTION_DIMF = len(FEATURE_NAMES)@@ -328,8 +336,9 @@ def build_gold_matrix(out: str):if has_gold:R[qi, si] = 1.0- matrix = RewardMatrix(queries=in_cov_queries, sources=sources,- feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)+ matrix = RewardMatrix(+ queries=in_cov_queries, sources=sources, feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask+ )matrix.save(out)# Diagnostics: does selection even have room to matter? If most queries have@@ -337,23 +346,28 @@ def build_gold_matrix(out: str):avail_per_q = mask.sum(axis=1)gold_per_q = (R > 0).sum(axis=1)n_gold_q = int((gold_per_q > 0).sum())- print(f"\nWrote matrix {out}.npz/.json: {Q} in-coverage queries x {S} sources, "- f"{int(mask.sum())} filled cells.")- print(f" queries with >=1 gold source: {n_gold_q} ({100*n_gold_q/max(Q,1):.1f}%)")- print(f" mean available sources / query: {avail_per_q.mean():.2f} "- f"(median {int(np.median(avail_per_q))}, max {int(avail_per_q.max())})")+ print(f"\nWrote matrix {out}.npz/.json: {Q} in-coverage queries x {S} sources, {int(mask.sum())} filled cells.")+ print(f" queries with >=1 gold source: {n_gold_q} ({100 * n_gold_q / max(Q, 1):.1f}%)")+ print(+ f" mean available sources / query: {avail_per_q.mean():.2f} "+ f"(median {int(np.median(avail_per_q))}, max {int(avail_per_q.max())})"+ )for k in (1, 2, 3, 5, 10):n = int((avail_per_q > k).sum())- print(f" queries with > {k:>2} available sources: {n:>4} "- f"({100*n/max(Q,1):.1f}%) <- room for top-{k} selection to matter")+ print(+ f" queries with > {k:>2} available sources: {n:>4} "+ f"({100 * n / max(Q, 1):.1f}%) <- room for top-{k} selection to matter"+ )# ---------------------------------------------------------------------------# select / simulate (pure, no network)# ---------------------------------------------------------------------------+def cmd_select(matrix_path: str, k: int):from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix, select_features+m = RewardMatrix.load(matrix_path)result = select_features(m, k=k)print(f"baseline coverage@{k}: {result['baseline_coverage']:.4f}\n")@@ -364,8 +378,11 @@ def cmd_select(matrix_path: str, k: int):def cmd_simulate(matrix_path: str, k: int):from mwmbl.tinysearchengine.super_search_select.evaluation import (- RewardMatrix, simulate_baselines, sweep_explore_scale,+ RewardMatrix,+ simulate_baselines,+ sweep_explore_scale,)+m = RewardMatrix.load(matrix_path)base = simulate_baselines(m, k=k)print("baselines (mean captured reward per query):")@@ -379,11 +396,14 @@ def cmd_simulate(matrix_path: str, k: int):print(f" nu={nu:<4} {val:.4f}{marker}")-def cmd_simulate_xgb(matrix_path: str, k: int, epsilons: list[float],- refit_every: int, min_rows: int):+def cmd_simulate_xgb(matrix_path: str, k: int, epsilons: list[float], refit_every: int, min_rows: int):from mwmbl.tinysearchengine.super_search_select.evaluation import (- RewardMatrix, simulate_baselines, sweep_epsilon, sweep_explore_scale,+ RewardMatrix,+ simulate_baselines,+ sweep_epsilon,+ sweep_explore_scale,)+m = RewardMatrix.load(matrix_path)base = simulate_baselines(m, k=k)print("baselines (mean captured reward per query):")@@ -393,28 +413,26 @@ def cmd_simulate_xgb(matrix_path: str, k: int, epsilons: list[float],best_nu = max(ts, key=ts.get)print(f"\nbest LinTS: nu={best_nu} {ts[best_nu]:.4f}")print("\nXGB contextual bandit by epsilon:")- sweep = sweep_epsilon(m, k=k, epsilons=epsilons,- refit_every=refit_every, min_rows=min_rows)+ sweep = sweep_epsilon(m, k=k, epsilons=epsilons, refit_every=refit_every, min_rows=min_rows)best = max(sweep, key=sweep.get)for eps, val in sweep.items():marker = " <- best" if eps == best else ""print(f" eps={eps:<5} {val:.4f}{marker}")-def cmd_holdout(matrix_path: str, k: int, test_frac: float,- queries_by_source: str | None, seed: int):+def cmd_holdout(matrix_path: str, k: int, test_frac: float, queries_by_source: str | None, seed: int):from mwmbl.tinysearchengine.super_search_select.evaluation import (- RewardMatrix, evaluate_holdout,+ RewardMatrix,+ evaluate_holdout,)+m = RewardMatrix.load(matrix_path)home_by_query = Noneif queries_by_source:by_source = json.loads(Path(queries_by_source).read_text())home_by_query = {q: source for source, qs in by_source.items() for q in qs}- result = evaluate_holdout(m, k=k, test_frac=test_frac, seed=seed,- home_by_query=home_by_query)- print(f"train queries: {result['n_train_queries']}, "- f"test queries: {result['n_test_queries']}")+ result = evaluate_holdout(m, k=k, test_frac=test_frac, seed=seed, home_by_query=home_by_query)+ print(f"train queries: {result['n_train_queries']}, test queries: {result['n_test_queries']}")print(f"xgb test RMSE: {result['rmse']:.4f}\n")for metric in ("coverage_at_k", "home_recall_at_k"):print(f"{metric} (k={k}):")@@ -424,16 +442,14 @@ def cmd_holdout(matrix_path: str, k: int, test_frac: float,def main():- parser = argparse.ArgumentParser(description=__doc__,- formatter_class=argparse.RawDescriptionHelpFormatter)+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)sub = parser.add_subparsers(dest="command", required=True)p_build = sub.add_parser("build-matrix")p_build.add_argument("--queries", required=True, help="file with one query per line")p_build.add_argument("--out", default="eval_matrix")p_build.add_argument("--reward", choices=["survival", "judge"], default="survival")- p_build.add_argument("--checkpoint", default=None,- help="fetch checkpoint JSONL (default: <out>.fetch.jsonl)")+ p_build.add_argument("--checkpoint", default=None, help="fetch checkpoint JSONL (default: <out>.fetch.jsonl)")p_gold = sub.add_parser("build-gold-matrix")p_gold.add_argument("--out", default="devdata/ss_gold_matrix")@@ -449,8 +465,7 @@ def main():p_sxgb = sub.add_parser("simulate-xgb")p_sxgb.add_argument("--matrix", default="eval_matrix")p_sxgb.add_argument("--k", type=int, default=10)- p_sxgb.add_argument("--epsilons", type=float, nargs="+",- default=[0.0, 0.05, 0.1, 0.2])+ p_sxgb.add_argument("--epsilons", type=float, nargs="+", default=[0.0, 0.05, 0.1, 0.2])p_sxgb.add_argument("--refit-every", type=int, default=200)p_sxgb.add_argument("--min-rows", type=int, default=300)@@ -458,8 +473,9 @@ def main():p_hold.add_argument("--matrix", default="eval_matrix")p_hold.add_argument("--k", type=int, default=10)p_hold.add_argument("--test-frac", type=float, default=0.2)- p_hold.add_argument("--queries-by-source", default=None,- help="JSON {source: [queries]} home map (ss_source_queries.json)")+ p_hold.add_argument(+ "--queries-by-source", default=None, help="JSON {source: [queries]} home map (ss_source_queries.json)"+ )p_hold.add_argument("--seed", type=int, default=0)args = parser.parse_args()@@ -475,11 +491,9 @@ def main():elif args.command == "simulate":cmd_simulate(args.matrix, args.k)elif args.command == "simulate-xgb":- cmd_simulate_xgb(args.matrix, args.k, args.epsilons,- args.refit_every, args.min_rows)+ cmd_simulate_xgb(args.matrix, args.k, args.epsilons, args.refit_every, args.min_rows)elif args.command == "holdout":- cmd_holdout(args.matrix, args.k, args.test_frac,- args.queries_by_source, args.seed)+ cmd_holdout(args.matrix, args.k, args.test_frac, args.queries_by_source, args.seed)if __name__ == "__main__":diff --git a/scripts/super_search_manual_poc.py b/scripts/super_search_manual_poc.pyindex 1b48db8..4881117 100644--- a/scripts/super_search_manual_poc.py+++ b/scripts/super_search_manual_poc.py@@ -16,6 +16,7 @@ Usage::Needs the local index + LTR model (via search_setup), network access for the firstSuper Search run (joblib-cached afterwards), and Redis for crawl/robots caching."""+import asyncioimport osfrom argparse import ArgumentParser@@ -55,10 +56,15 @@ def collect_super_search(query: str) -> list[dict]:if event_type == "results":captured.clear()for item in data.results:- captured.append({- "url": item.url, "title": item.title, "extract": item.extract,- "score": item.score, "source": item.source,- })+ captured.append(+ {+ "url": item.url,+ "title": item.title,+ "extract": item.extract,+ "score": item.score,+ "source": item.source,+ }+ )async def run():await _emit_final_results(query, docs, emit, [None], asyncio.Lock())@@ -94,8 +100,7 @@ def main():parser = ArgumentParser()parser.add_argument("--num", type=int, default=10, help="Number of queries to sample.")parser.add_argument("--seed", type=int, default=42, help="RNG seed for sampling.")- parser.add_argument("--out", default=os.path.join(- os.environ.get("CLAUDE_SCRATCHPAD", "."), "ss_manual_poc.md"))+ parser.add_argument("--out", default=os.path.join(os.environ.get("CLAUDE_SCRATCHPAD", "."), "ss_manual_poc.md"))args = parser.parse_args()dataset = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH)@@ -112,16 +117,16 @@ def main():for n, query in enumerate(sample, 1):print(f"[{n}/{len(sample)}] {query!r}")- gold = (dataset[dataset["query"] == query]- .sort_values("rank")- .head(NUM_RESULTS_FOR_EVAL))- gold_rows = [{"url": r.url, "title": None, "extract": r.snippet,- "score": None, "source": ""} for r in gold.itertuples()]+ gold = dataset[dataset["query"] == query].sort_values("rank").head(NUM_RESULTS_FOR_EVAL)+ gold_rows = [+ {"url": r.url, "title": None, "extract": r.snippet, "score": None, "source": ""} for r in gold.itertuples()+ ]gold_domains = {domain(r["url"]) for r in gold_rows}std_docs = ranker.search(query, [])- std_rows = [{"url": d.url, "title": d.title, "extract": d.extract,- "score": d.score, "source": ""} for d in std_docs]+ std_rows = [+ {"url": d.url, "title": d.title, "extract": d.extract, "score": d.score, "source": ""} for d in std_docs+ ]try:ss_rows = collect_super_search(query)diff --git a/scripts/super_search_new_sources_eval.py b/scripts/super_search_new_sources_eval.pyindex 089f8a2..1e8f3c5 100644--- a/scripts/super_search_new_sources_eval.py+++ b/scripts/super_search_new_sources_eval.py@@ -26,6 +26,7 @@ Run:DATABASE_URL="postgres://daoud@" DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \uv run python scripts/super_search_new_sources_eval.py --max-queries 80"""+import argparseimport osimport sys@@ -66,6 +67,7 @@ def _standard_ranker_and_model(use_local: bool):"""if use_local:from mwmbl.search_setup import ltr_model, ranker # local index + LTR + MMR + wiki+return ranker, ltr_modelltr_model = RustXGBPipeline.from_model_path(str(RUST_MODEL_PATH))return MMRRanker(LTRRanker(RemoteIndex(), DummyCompleter(), ltr_model, True, 3)), ltr_model@@ -74,7 +76,7 @@ def _standard_ranker_and_model(use_local: bool):def _registrable(host: str) -> str:for prefix in ("www.", "m."):if host.startswith(prefix):- host = host[len(prefix):]+ host = host[len(prefix) :]labels = host.split(".")return ".".join(labels[-2:]) if len(labels) > 2 else host@@ -96,41 +98,53 @@ def _ndcg(predicted_urls: list[str], gold_scores: dict[str, float]) -> float:def main():parser = argparse.ArgumentParser()parser.add_argument("--max-queries", type=int, default=80)- parser.add_argument("--standard-index", choices=["remote", "local"], default="remote",- help="Index for the standard-search arm and the fallback gate "- "(default remote = api.mwmbl.org production index).")- parser.add_argument("--fallback-thresholds", type=int, nargs="+", default=[1, 3, 5],- help="Fall back to Super Search when standard returns <= threshold "- "results. Evaluated post-hoc for each value.")- parser.add_argument("--score-thresholds", type=float, nargs="+", default=None,- help="Relevance-score trigger: fall back when standard's top-result "- "LTR score < threshold. Default: deciles of the observed score "- "distribution (data-driven sweep).")+ parser.add_argument(+ "--standard-index",+ choices=["remote", "local"],+ default="remote",+ help="Index for the standard-search arm and the fallback gate "+ "(default remote = api.mwmbl.org production index).",+ )+ parser.add_argument(+ "--fallback-thresholds",+ type=int,+ nargs="+",+ default=[1, 3, 5],+ help="Fall back to Super Search when standard returns <= threshold results. Evaluated post-hoc for each value.",+ )+ parser.add_argument(+ "--score-thresholds",+ type=float,+ nargs="+",+ default=None,+ help="Relevance-score trigger: fall back when standard's top-result "+ "LTR score < threshold. Default: deciles of the observed score "+ "distribution (data-driven sweep).",+ )args = parser.parse_args()df = pd.read_csv(RANKINGS_DATASET_TEST_PATH)new_domains = {_registrable(get_meta(n).domain.lower()) for n in NEW_SOURCES}# In-coverage queries: a gold URL of theirs sits on one of the new domains.- covered = {q for q, u in zip(df["query"], df["url"])- if _registrable(_host(u)) in new_domains}+ covered = {q for q, u in zip(df["query"], df["url"]) if _registrable(_host(u)) in new_domains}queries = sorted(covered)rng = np.random.default_rng(0)rng.shuffle(queries)- queries = queries[:args.max_queries]+ queries = queries[: args.max_queries]print(f"{len(covered)} in-coverage test queries; evaluating {len(queries)}")# Gold score dicts per query (same construction as evaluate.py).gold = {}for q in queries:rows = df[df["query"] == q][["url"]].iloc[:NUM_RESULTS_FOR_EVAL].copy()- rows["score"] = CLICK_PROPORTIONS[:len(rows)]+ rows["score"] = CLICK_PROPORTIONS[: len(rows)]gold[q] = rows.set_index("url")["score"].to_dict()print(f"standard-search index: {args.standard_index}")per_query = {"standard": {}, "ss-baseline": {}, "ss+new": {}}- standard_count = {} # number of results standard returned (count-based gate)- standard_top_score = {} # LTR relevance of standard's top result (score-based gate)+ standard_count = {} # number of results standard returned (count-based gate)+ standard_top_score = {} # LTR relevance of standard's top result (score-based gate)# Standard arm: handled directly (not via the generic loop) so we can read the# ranker's Documents and re-score the top one with the LTR model.@@ -163,8 +177,7 @@ def main():for n in sorted(args.fallback_thresholds):for ss_name in ("ss-baseline", "ss+new"):per_query[f"fallback@{n}/{ss_name}"] = {- q: (per_query[ss_name][q] if standard_count[q] <= n else per_query["standard"][q])- for q in queries+ q: (per_query[ss_name][q] if standard_count[q] <= n else per_query["standard"][q]) for q in queries}print("\n" + "=" * 60)@@ -182,8 +195,10 @@ def main():wins = sum(new[q] > base[q] + 1e-9 for q in queries)losses = sum(new[q] < base[q] - 1e-9 for q in queries)delta = np.mean([new[q] - base[q] for q in queries])- print(f"\nss+new vs ss-baseline: mean Δ {delta:+.4f} | better {wins}, worse {losses}, "- f"same {len(queries) - wins - losses}")+ print(+ f"\nss+new vs ss-baseline: mean Δ {delta:+.4f} | better {wins}, worse {losses}, "+ f"same {len(queries) - wins - losses}"+ )# The headline fallback question: does gating Super Search behind standard-search# failure beat serving standard search alone, on this in-coverage subset?@@ -194,15 +209,14 @@ def main():fb = per_query[f"fallback@{n}/{ss_name}"]d = np.mean([fb[q] - std[q] for q in queries])w = sum(fb[q] > std[q] + 1e-9 for q in queries)- l = sum(fb[q] < std[q] - 1e-9 for q in queries)- print(f" fallback@{n}/{ss_name:11} mean Δ {d:+.4f} | better {w}, worse {l}")+ worse = sum(fb[q] < std[q] - 1e-9 for q in queries)+ print(f" fallback@{n}/{ss_name:11} mean Δ {d:+.4f} | better {w}, worse {worse}")# Why the fallback nets ~0: show, for the queries it fires on, whether Super# Search actually rescues them. If standard and Super Search both score ~0 on the# starved queries, the result-count gate is selecting queries SS cannot help.nmax = max(args.fallback_thresholds)- fired = sorted([q for q in queries if standard_count[q] <= nmax],- key=lambda q: standard_count[q])+ fired = sorted([q for q in queries if standard_count[q] <= nmax], key=lambda q: standard_count[q])print(f"\nfired queries at n={nmax} (standard count, NDCG: standard / ss-baseline / ss+new):")for q in fired:print(f" cnt={standard_count[q]:2d} {std[q]:.3f} / {base[q]:.3f} / {new[q]:.3f} {q!r}")@@ -213,8 +227,7 @@ def main():# where standard is *confidently wrong* rather than merely sparse.finite = sorted(s for s in standard_top_score.values() if np.isfinite(s))n_empty = len(queries) - len(finite)- print("\nstandard top-result LTR score distribution"- f" ({n_empty} queries with no results → -inf, always fire):")+ print(f"\nstandard top-result LTR score distribution ({n_empty} queries with no results → -inf, always fire):")if finite:pcts = [0, 10, 25, 50, 75, 90, 100]print(" " + " ".join(f"p{p}={v:+.3f}" for p, v in zip(pcts, np.percentile(finite, pcts))))@@ -222,13 +235,11 @@ def main():if args.score_thresholds is not None:score_thresholds = args.score_thresholdselif finite:- score_thresholds = sorted(set(round(float(t), 3)- for t in np.percentile(finite, [10, 20, 30, 40, 50])))+ score_thresholds = sorted(set(round(float(t), 3) for t in np.percentile(finite, [10, 20, 30, 40, 50])))else:score_thresholds = []- print("\nrelevance-score fallback (fire when standard top score < threshold),"- " paired vs standard-always:")+ print("\nrelevance-score fallback (fire when standard top score < threshold), paired vs standard-always:")print(f" {'arm':26} {'fired':>11} {'mean NDCG':>10} {'Δ vs std':>9} {'better/worse':>13}")for t in score_thresholds:n_fired = sum(standard_top_score[q] < t for q in queries)@@ -236,10 +247,12 @@ def main():fb = {q: (per_query[ss_name][q] if standard_top_score[q] < t else std[q]) for q in queries}d = np.mean([fb[q] - std[q] for q in queries])w = sum(fb[q] > std[q] + 1e-9 for q in queries)- l = sum(fb[q] < std[q] - 1e-9 for q in queries)+ worse = sum(fb[q] < std[q] - 1e-9 for q in queries)label = f"score<{t:+.3f}/{ss_name}"- print(f" {label:26} {n_fired:3d} ({n_fired / len(queries):3.0%}) "- f"{np.mean(list(fb.values())):>10.4f} {d:>+9.4f} {w:>5}/{l:<5}")+ print(+ f" {label:26} {n_fired:3d} ({n_fired / len(queries):3.0%}) "+ f"{np.mean(list(fb.values())):>10.4f} {d:>+9.4f} {w:>5}/{worse:<5}"+ )if __name__ == "__main__":diff --git a/scripts/super_search_queryset.py b/scripts/super_search_queryset.pyindex 129ee6e..f318afc 100644--- a/scripts/super_search_queryset.py+++ b/scripts/super_search_queryset.py@@ -23,6 +23,7 @@ Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev uv run python \scripts/super_search_queryset.py --dump-batch 30"""+import argparseimport asyncioimport json@@ -45,8 +46,8 @@ CHECKPOINT = REPO_ROOT / "devdata" / "ss_source_queries.json"CATALOG = REPO_ROOT / "devdata" / "llm_relabel" / "source_catalog.json"MIN_QUERIES, MAX_QUERIES = 5, 10-PROBE_SEEDS = 3 # description-derived probe queries per source-PROBE_LIMIT = 5 # results per probe+PROBE_SEEDS = 3 # description-derived probe queries per source+PROBE_LIMIT = 5 # results per probeGROUNDING_TERMS = 15_WORD_RE = re.compile(r"[a-z][a-z0-9'-]{3,}")@@ -56,7 +57,8 @@ _STOPWORDS = frozenset("more most some such only very also than then there here each every and ""the for are was were has had can could should would may might must not ""its his her our you all any but out off own same too who whom does did "- "site sites page pages website websites online free best".split())+ "site sites page pages website websites online free best".split()+)GENERATION_BRIEF = """\For each source below, write {min_q}-{max_q} realistic user search queries that this@@ -88,14 +90,17 @@ def save_checkpoint(data: dict[str, list[str]]) -> None:def all_sources() -> list[str]:from mwmbl.tinysearchengine.super_search_sources import SOURCES+return list(SOURCES.keys())def source_info(name: str) -> tuple[str, str, str]:"""(domain, field, description) from the registry + LLM source catalog + shortlist."""from mwmbl.tinysearchengine.super_search_select.registry import (- _load_shortlist, get_meta,+ _load_shortlist,+ get_meta,)+meta = get_meta(name)catalog = json.loads(CATALOG.read_text()) if CATALOG.exists() else {}desc = catalog.get(name, {}).get("description", "")@@ -112,13 +117,15 @@ def _content_words(text: str) -> list[str]:async def _probe_source(name: str, seeds: list[str]) -> list[str]:"""Query the source with description-derived seeds; return its top result terms."""import httpx+from mwmbl.tinysearchengine.super_search_sources import SOURCESfn = SOURCES[name]timeout = settings.SUPER_SEARCH_PER_SOURCE_TIMEOUTcounts: Counter = Counter()- async with httpx.AsyncClient(follow_redirects=True, timeout=timeout,- headers={"User-Agent": "mwmbl-super-search-eval/0.1"}) as client:+ async with httpx.AsyncClient(+ follow_redirects=True, timeout=timeout, headers={"User-Agent": "mwmbl-super-search-eval/0.1"}+ ) as client:for seed in seeds:try:docs = await asyncio.wait_for(fn(client, seed, PROBE_LIMIT), timeout=timeout)@@ -144,11 +151,12 @@ def dump_batch(n: int, probe: bool) -> None:if probe:seeds = list(dict.fromkeys(_content_words(desc)))[:PROBE_SEEDS] or [field]terms = asyncio.run(_probe_source(name, seeds))- print(f"grounding (live results for seeds {seeds}): "- f"{', '.join(terms) if terms else '(no results)'}")+ print(f"grounding (live results for seeds {seeds}): {', '.join(terms) if terms else '(no results)'}")print()- print(f"[{len(todo)} sources dumped; {len(done)} done, "- f"{len(all_sources()) - len(done) - len(todo)} remaining after these]")+ print(+ f"[{len(todo)} sources dumped; {len(done)} done, "+ f"{len(all_sources()) - len(done) - len(todo)} remaining after these]"+ )_URLISH_RE = re.compile(r"https?://|www\.", re.I)@@ -167,15 +175,13 @@ def merge(path: str) -> None:raise ValueError(f"source {name!r} already has queries; refusing to overwrite")deduped = list(dict.fromkeys(q.strip() for q in queries if q and q.strip()))if not MIN_QUERIES <= len(deduped) <= MAX_QUERIES:- raise ValueError(f"{name!r}: {len(deduped)} queries after dedup, "- f"need {MIN_QUERIES}-{MAX_QUERIES}")+ raise ValueError(f"{name!r}: {len(deduped)} queries after dedup, need {MIN_QUERIES}-{MAX_QUERIES}")for q in deduped:if _URLISH_RE.search(q) or len(q) > 80:raise ValueError(f"{name!r}: bad query {q!r}")done[name] = dedupedsave_checkpoint(done)- print(f"merged {len(generated)} sources; checkpoint now covers "- f"{len(done)}/{len(known)} sources.")+ print(f"merged {len(generated)} sources; checkpoint now covers {len(done)}/{len(known)} sources.")def status() -> None:@@ -194,8 +200,7 @@ def main():group.add_argument("--status", action="store_true")group.add_argument("--dump-batch", type=int, metavar="N")group.add_argument("--merge", metavar="FILE")- parser.add_argument("--no-probe", action="store_true",- help="skip the live grounding probe when dumping")+ parser.add_argument("--no-probe", action="store_true", help="skip the live grounding probe when dumping")args = parser.parse_args()if args.status:status()diff --git a/scripts/verify_user_ids_beta.py b/scripts/verify_user_ids_beta.pyindex 4c61066..9d67a42 100644--- a/scripts/verify_user_ids_beta.py+++ b/scripts/verify_user_ids_beta.py@@ -12,10 +12,10 @@ Usage:"""import argparse-import sys+import jsonimport os+import sysimport time-import jsonimport redisimport requests@@ -23,7 +23,7 @@ import requests# Must be set before importing retrieve (used in the User-Agent string)os.environ.setdefault("MWMBL_CONTACT_INFO", "[email protected]")-from mwmbl.crawler.retrieve import crawl_url, CRAWLER_VERSION+from mwmbl.crawler.retrieve import CRAWLER_VERSION, crawl_urlfrom mwmbl.settings import REDIS_URL_redis = None@@ -36,6 +36,7 @@ def _get_redis() -> redis.Redis:_redis = redis.from_url(REDIS_URL, decode_responses=True)return _redis+BETA_BASE = "http://localhost:8000"MAX_URLS = 5@@ -76,12 +77,14 @@ def main():content = raw.get("content")if content and not raw.get("error") and content.get("title"):last_crawled = int(raw["timestamp"] / 1000) # ms → seconds- results.append({- "url": raw["url"],- "title": content["title"],- "extract": content.get("extract", ""),- "last_crawled": last_crawled,- })+ results.append(+ {+ "url": raw["url"],+ "title": content["title"],+ "extract": content.get("extract", ""),+ "last_crawled": last_crawled,+ }+ )print(f" OK — {content['title'][:70]}")else:err = (raw.get("error") or {}).get("name", "unknown error")diff --git a/test/conftest.py b/test/conftest.pyindex b0da0ed..3fab119 100644--- a/test/conftest.py+++ b/test/conftest.py@@ -10,8 +10,6 @@ callback that runs longer than 100 ms triggers a warning. Combined with`-W error::RuntimeWarning` (set per test if desired), this catches mostaccidental sync calls leaking into async paths."""-import asyncio-import sysimport pytestdiff --git a/test/test_admin_blacklist_status.py b/test/test_admin_blacklist_status.pyindex 79a0583..9e62e7f 100644--- a/test/test_admin_blacklist_status.py+++ b/test/test_admin_blacklist_status.py@@ -1,4 +1,5 @@"""Tests for the blacklist status admin page (mwmbl.admin_views)."""+import fakeredisimport numpy as npimport pytest@@ -8,7 +9,10 @@ from redis import ConnectionError as RedisConnectionErrorfrom mwmbl import admin_viewsfrom mwmbl.indexer import blacklist_snapshot, purge_queuefrom mwmbl.indexer.blacklist_snapshot import (- HASH_DTYPE, SnapshotBlacklist, hash_domains, publish_snapshot,+ HASH_DTYPE,+ SnapshotBlacklist,+ hash_domains,+ publish_snapshot,)from mwmbl.indexer.purge_queue import drain_purge_queue, enqueue_for_purge, peek_purge_queue, queue_sizefrom mwmbl.tinysearchengine.indexer import Document@@ -57,8 +61,7 @@ def wired_redis(monkeypatch, redis_server, redis_client):binary_client = fakeredis.FakeRedis(server=redis_server)monkeypatch.setattr(blacklist_snapshot, "_redis", binary_client)monkeypatch.setattr(purge_queue, "_redis", redis_client)- monkeypatch.setattr(blacklist_snapshot, "_snapshot_blacklist",- SnapshotBlacklist(redis_client=binary_client))+ monkeypatch.setattr(blacklist_snapshot, "_snapshot_blacklist", SnapshotBlacklist(redis_client=binary_client))return redis_client@@ -73,6 +76,7 @@ def staff_client(client, db):# peek_purge_queue# ---------------------------------------------------------------------------+def test_peek_does_not_consume_the_queue(redis_client, documents):enqueue_for_purge(documents, redis_client)@@ -101,6 +105,7 @@ def test_peek_on_an_empty_queue(redis_client):# Status gathering# ---------------------------------------------------------------------------+def test_snapshot_status_reports_a_published_blob(wired_redis):blob = np.array([1, 2, 3], dtype=HASH_DTYPE).tobytes()version = publish_snapshot(blob, wired_redis)@@ -169,6 +174,7 @@ def test_removed_counts_covers_the_requested_window(wired_redis):# The view# ---------------------------------------------------------------------------+def test_view_requires_staff(client, db):user = User.objects.create_user(username="ordinary_user_2", password="correctpassword")client.force_login(user)@@ -216,8 +222,7 @@ def test_view_reports_the_background_tasks(staff_client, wired_redis):assert task_names == [admin_views.SNAPSHOT_TASK_NAME, admin_views.PURGE_TASK_NAME]# Nothing scheduled in the test DB, which is what a missing process_tasks worker looks# like - the page has to render that rather than assume the rows exist.- assert all(task["task"] is None and task["last_completed"] is None- for task in response.context["tasks"])+ assert all(task["task"] is None and task["last_completed"] is None for task in response.context["tasks"])def test_admin_index_links_to_the_status_page(staff_client):@@ -230,6 +235,7 @@ def test_admin_index_links_to_the_status_page(staff_client):# Curated domains# ---------------------------------------------------------------------------+def test_curated_status_reports_nothing_still_blacklisted(wired_redis, monkeypatch):monkeypatch.setattr(admin_views, "get_curated_domains", lambda: {"pudding.cool"})publish_snapshot(np.array(hash_domains(["badsite.test"]), dtype=HASH_DTYPE).tobytes(), wired_redis)diff --git a/test/test_apps_background_scheduling.py b/test/test_apps_background_scheduling.pyindex 55e9d95..7b38e87 100644--- a/test/test_apps_background_scheduling.py+++ b/test/test_apps_background_scheduling.py@@ -92,6 +92,7 @@ def test_a_one_off_snapshot_rebuild_does_not_suppress_the_periodic_one():six-hourly task being registered - that would silently leave the snapshot to whateverapprovals happened to trigger."""from mwmbl.background import refresh_blacklist_snapshot+refresh_blacklist_snapshot(schedule=600)MwmblConfig._schedule_background_tasks()diff --git a/test/test_auth.py b/test/test_auth.pyindex a675132..f2723ce 100644--- a/test/test_auth.py+++ b/test/test_auth.py@@ -1,6 +1,6 @@import pytest-from django.contrib.auth import get_user_modelfrom allauth.account.models import EmailAddress+from django.contrib.auth import get_user_modelUser = get_user_model()@@ -14,9 +14,7 @@ def verified_user(db):email="[email protected]",password="correctpassword",)- EmailAddress.objects.create(- user=user, email="[email protected]", verified=True, primary=True- )+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@@ -27,9 +25,7 @@ def unverified_user(db):email="[email protected]",password="correctpassword",)- EmailAddress.objects.create(- user=user, email="[email protected]", verified=False, primary=True- )+ EmailAddress.objects.create(user=user, email="[email protected]", verified=False, primary=True)return user@@ -102,7 +98,8 @@ def test_login_email_and_username_return_same_user(client, verified_user):content_type="application/json",).json()# Both tokens are for the same user — decode the subject claim to verify- import base64, json as _json+ import base64+ import json as _jsondef subject(token):payload = token.split(".")[1]diff --git a/test/test_background_polar_reporting.py b/test/test_background_polar_reporting.pyindex 764548b..c3d8196 100644--- a/test/test_background_polar_reporting.py+++ b/test/test_background_polar_reporting.py@@ -22,7 +22,9 @@ User = get_user_model()@pytest.fixturedef billed_user(db):user = User.objects.create_user(- username="billeduser", email="[email protected]", password="testpass123",+ username="billeduser",+ email="[email protected]",+ password="testpass123",)EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)UserBilling.objects.create(@@ -38,8 +40,11 @@ def billed_user(db):def test_report_usage_to_polar_ingests_delta_only(billed_user):now = datetime.now(timezone.utc)bucket = UsageBucket.objects.create(- user=billed_user, year=now.year, month=now.month,- count=pricing.FREE_KEYED_MONTHLY_LIMIT + 500, reported_overage=0,+ user=billed_user,+ year=now.year,+ month=now.month,+ count=pricing.FREE_KEYED_MONTHLY_LIMIT + 500,+ reported_overage=0,)with patch("polar_sdk.Polar") as MockPolar:@@ -60,13 +65,18 @@ def test_report_usage_to_polar_ingests_delta_only(billed_user):@pytest.mark.django_dbdef test_report_usage_to_polar_skips_users_without_polar_customer(db):user = User.objects.create_user(- username="nocustomer", email="[email protected]", password="testpass123",+ username="nocustomer",+ email="[email protected]",+ password="testpass123",)EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)now = datetime.now(timezone.utc)UsageBucket.objects.create(- user=user, year=now.year, month=now.month,- count=pricing.FREE_KEYED_MONTHLY_LIMIT + 500, reported_overage=0,+ user=user,+ year=now.year,+ month=now.month,+ count=pricing.FREE_KEYED_MONTHLY_LIMIT + 500,+ reported_overage=0,)with patch("polar_sdk.Polar") as MockPolar:@@ -80,8 +90,11 @@ def test_report_usage_to_polar_skips_users_without_polar_customer(db):def test_report_usage_to_polar_no_op_below_free_allowance(billed_user):now = datetime.now(timezone.utc)bucket = UsageBucket.objects.create(- user=billed_user, year=now.year, month=now.month,- count=pricing.FREE_KEYED_MONTHLY_LIMIT - 1, reported_overage=0,+ user=billed_user,+ year=now.year,+ month=now.month,+ count=pricing.FREE_KEYED_MONTHLY_LIMIT - 1,+ reported_overage=0,)with patch("polar_sdk.Polar") as MockPolar:@@ -97,8 +110,11 @@ def test_report_usage_to_polar_no_op_below_free_allowance(billed_user):def test_report_usage_to_polar_incremental_second_run(billed_user):now = datetime.now(timezone.utc)bucket = UsageBucket.objects.create(- user=billed_user, year=now.year, month=now.month,- count=pricing.FREE_KEYED_MONTHLY_LIMIT + 500, reported_overage=500,+ user=billed_user,+ year=now.year,+ month=now.month,+ count=pricing.FREE_KEYED_MONTHLY_LIMIT + 500,+ reported_overage=500,)# More usage has accumulated since the last report.diff --git a/test/test_blacklist_approval_signal.py b/test/test_blacklist_approval_signal.pyindex ca87d91..160b173 100644--- a/test/test_blacklist_approval_signal.py+++ b/test/test_blacklist_approval_signal.py@@ -4,6 +4,7 @@ Approved domains are subtracted from the remote blocklists when the snapshot iswithout this an approval would sit inert until the next periodic rebuild - six hours bydefault. The scheduling is debounced because moderators work through the queue in batches."""+from datetime import timedeltaimport pytest@@ -61,6 +62,7 @@ def test_a_rebuild_already_due_inside_the_window_suppresses_a_new_one(user):"""The periodic task's own row counts, so an approval shortly before the six-hourlyrebuild adds nothing at all."""from mwmbl.background import refresh_blacklist_snapshot+refresh_blacklist_snapshot(repeat=6 * 60 * 60, repeat_until=None)assert snapshot_tasks().count() == 1@@ -73,6 +75,7 @@ def test_a_rebuild_already_due_inside_the_window_suppresses_a_new_one(user):@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)def test_a_rebuild_due_after_the_window_does_not_suppress_a_new_one(user):from mwmbl.background import refresh_blacklist_snapshot+refresh_blacklist_snapshot(schedule=APPROVAL_DELAY * 10)assert snapshot_tasks().count() == 1diff --git a/test/test_blacklist_providers.py b/test/test_blacklist_providers.pyindex e44ec25..22612d0 100644--- a/test/test_blacklist_providers.py+++ b/test/test_blacklist_providers.py@@ -3,113 +3,113 @@ Tests for blacklist providers and the abstraction system."""import re+from unittest.mock import MagicMock, patchimport pytestimport requests-from unittest.mock import patch, MagicMock+from mwmbl.indexer.blacklist import get_default_blacklist_providerfrom mwmbl.indexer.blacklist_providers import (- BuiltInRulesBlacklistProvider,- StaticBlacklistProvider,- HaGeZiBlacklistProvider,AdultContentBlacklistProvider,+ BuiltInRulesBlacklistProvider,CombinedBlacklistProvider,+ HaGeZiBlacklistProvider,+ StaticBlacklistProvider,domain_and_parents,domains_to_unblock,)-from mwmbl.indexer.blacklist import get_default_blacklist_providerdef test_static_blacklist_provider():"""Test StaticBlacklistProvider with is_domain_blacklisted method."""- test_domains = {'spam.com', 'malware.example'}+ test_domains = {"spam.com", "malware.example"}provider = StaticBlacklistProvider(test_domains)-+# Test domains that should be blacklisted- assert provider.is_domain_blacklisted('spam.com') == True- assert provider.is_domain_blacklisted('malware.example') == True-+ assert provider.is_domain_blacklisted("spam.com")+ assert provider.is_domain_blacklisted("malware.example")+# Test domain that should not be blacklisted- assert provider.is_domain_blacklisted('github.com') == False+ assert not provider.is_domain_blacklisted("github.com")def test_hagezi_blacklist_provider_success():"""Test HaGeZiBlacklistProvider with successful HTTP response."""- with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:mock_response = MagicMock()- mock_response.text = '''# HaGeZi DNS Blocklist+ mock_response.text = """# HaGeZi DNS Blocklist# Comments should be ignoredspam.commalware.example# Another commentbadsite.net-'''+"""mock_get.return_value = mock_response- provider = HaGeZiBlacklistProvider('light')+ provider = HaGeZiBlacklistProvider("light")# Test domains that should be blacklisted- assert provider.is_domain_blacklisted('spam.com') == True- assert provider.is_domain_blacklisted('malware.example') == True- assert provider.is_domain_blacklisted('badsite.net') == True-+ assert provider.is_domain_blacklisted("spam.com")+ assert provider.is_domain_blacklisted("malware.example")+ assert provider.is_domain_blacklisted("badsite.net")+# Test domain that should not be blacklisted- assert provider.is_domain_blacklisted('github.com') == False+ assert not provider.is_domain_blacklisted("github.com")def test_adult_content_blacklist_provider_success():"""Test AdultContentBlacklistProvider parses hosts-format lines correctly."""- with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:mock_response = MagicMock()- mock_response.text = '''# Title: Porn Block List+ mock_response.text = """# Title: Porn Block List# Homepage: https://github.com/blocklistproject/Lists0.0.0.0 fineartteens.com0.0.0.0 milforia.com-'''+"""mock_get.return_value = mock_responseprovider = AdultContentBlacklistProvider()- assert provider.is_domain_blacklisted('fineartteens.com') == True- assert provider.is_domain_blacklisted('milforia.com') == True- assert provider.is_domain_blacklisted('github.com') == False+ assert provider.is_domain_blacklisted("fineartteens.com")+ assert provider.is_domain_blacklisted("milforia.com")+ assert not provider.is_domain_blacklisted("github.com")def test_hagezi_blacklist_provider_invalid_type():"""Test HaGeZiBlacklistProvider rejects invalid list types."""with pytest.raises(ValueError):- HaGeZiBlacklistProvider('invalid_type')+ HaGeZiBlacklistProvider("invalid_type")def test_combined_blacklist_provider():"""Test CombinedBlacklistProvider with is_domain_blacklisted method."""- provider1 = StaticBlacklistProvider({'spam.com', 'malware.example'})- provider2 = StaticBlacklistProvider({'badsite.net', 'phishing.site'})-+ provider1 = StaticBlacklistProvider({"spam.com", "malware.example"})+ provider2 = StaticBlacklistProvider({"badsite.net", "phishing.site"})+combined = CombinedBlacklistProvider([provider1, provider2])-+# Test domains from both providers- assert combined.is_domain_blacklisted('spam.com') == True- assert combined.is_domain_blacklisted('badsite.net') == True- assert combined.is_domain_blacklisted('phishing.site') == True-+ assert combined.is_domain_blacklisted("spam.com")+ assert combined.is_domain_blacklisted("badsite.net")+ assert combined.is_domain_blacklisted("phishing.site")+# Test domain that should not be blacklisted- assert combined.is_domain_blacklisted('github.com') == False+ assert not combined.is_domain_blacklisted("github.com")def test_combined_blacklist_provider_handles_failures():"""Test CombinedBlacklistProvider continues even if one provider fails."""- good_provider = StaticBlacklistProvider({'spam.com'})+ good_provider = StaticBlacklistProvider({"spam.com"})bad_provider = MagicMock()bad_provider.is_domain_blacklisted.side_effect = Exception("Provider failure")-+combined = CombinedBlacklistProvider([good_provider, bad_provider])-+# Should still work with the good provider despite the bad one failing- assert combined.is_domain_blacklisted('spam.com') == True- assert combined.is_domain_blacklisted('github.com') == False+ assert combined.is_domain_blacklisted("spam.com")+ assert not combined.is_domain_blacklisted("github.com")def test_integration_with_blacklist_module():@@ -117,13 +117,13 @@ def test_integration_with_blacklist_module():# Test the default provider factorydefault_provider = get_default_blacklist_provider()assert default_provider is not None-+# Test that it can check domains# Test with a domain that should be blacklisted by built-in rules- assert default_provider.is_domain_blacklisted('59648.etnomurcia.com') == True-+ assert default_provider.is_domain_blacklisted("59648.etnomurcia.com")+# Test with a domain that should not be blacklisted- assert default_provider.is_domain_blacklisted('github.com') == False+ assert not default_provider.is_domain_blacklisted("github.com")def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():@@ -133,7 +133,7 @@ def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():including get_wiki_results() on the live search path (which then retries a live fetch4x per uncached query until the worker dies). Regression test: fetch must not gothrough request_cache."""- with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:mock_response = MagicMock()mock_response.text = "spam.com\n"mock_get.return_value = mock_response@@ -141,11 +141,11 @@ def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():# If the module still imported request_cache, patching it would succeed; assert the# name is gone from the module so this can't silently regress.import mwmbl.indexer.blacklist_providers as bp- assert not hasattr(bp, "request_cache"), \- "blacklist_providers must not use the shared filesystem request_cache"- provider = HaGeZiBlacklistProvider('light')- assert provider.is_domain_blacklisted('spam.com') is True+ assert not hasattr(bp, "request_cache"), "blacklist_providers must not use the shared filesystem request_cache"++ provider = HaGeZiBlacklistProvider("light")+ assert provider.is_domain_blacklisted("spam.com") is Trueassert mock_get.called@@ -155,11 +155,10 @@ def test_fetch_failure_propagates():build_snapshot() publishes the union of these lists to every search worker, so aprovider that swallowed the error and returned an empty set would replace a goodsnapshot with one missing its entire contribution."""- provider = HaGeZiBlacklistProvider('light')- with patch('mwmbl.indexer.blacklist_providers.requests.get',- side_effect=requests.RequestException("boom")):+ provider = HaGeZiBlacklistProvider("light")+ with patch("mwmbl.indexer.blacklist_providers.requests.get", side_effect=requests.RequestException("boom")):with pytest.raises(requests.RequestException):- provider.is_domain_blacklisted('spam.com')+ provider.is_domain_blacklisted("spam.com")def test_the_list_is_fetched_once_per_instance():@@ -167,29 +166,29 @@ def test_the_list_is_fetched_once_per_instance():There is no cache behind the module - that made get_default_blacklist_provider() lookcheap enough to call from a request handler. Reuse comes from holding the instance."""- with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:mock_response = MagicMock()mock_response.text = "spam.com\n"mock_get.return_value = mock_response- provider = HaGeZiBlacklistProvider('light')- assert provider.is_domain_blacklisted('spam.com') is True- assert provider.is_domain_blacklisted('other.com') is False+ provider = HaGeZiBlacklistProvider("light")+ assert provider.is_domain_blacklisted("spam.com") is True+ assert provider.is_domain_blacklisted("other.com") is Falseassert mock_get.call_count == 1# A second instance is a second download - nothing is shared between them.- assert HaGeZiBlacklistProvider('light').is_domain_blacklisted('spam.com') is True+ assert HaGeZiBlacklistProvider("light").is_domain_blacklisted("spam.com") is Trueassert mock_get.call_count == 2def test_hosts_lines_with_trailing_comments_keep_the_domain():- with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:mock_response = MagicMock()mock_response.text = "0.0.0.0 spam.com # added 2026-01-01\n# a comment line\nplain.com\n"mock_get.return_value = mock_responseprovider = AdultContentBlacklistProvider()- assert provider._get_blacklisted_domains() == {'spam.com', 'plain.com'}+ assert provider._get_blacklisted_domains() == {"spam.com", "plain.com"}def test_domain_and_parents():@@ -202,30 +201,30 @@ def test_domain_and_parents():def test_a_remote_list_entry_covers_subdomains():"""The lists are apex rules, so www.spam.com must be caught by an entry for spam.com."""- with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:mock_response = MagicMock()mock_response.text = "spam.com\n"mock_get.return_value = mock_response- provider = HaGeZiBlacklistProvider('light')- assert provider.is_domain_blacklisted('spam.com') is True- assert provider.is_domain_blacklisted('www.spam.com') is True- assert provider.is_domain_blacklisted('cdn.images.spam.com') is True- assert provider.is_domain_blacklisted('notspam.com') is False- assert provider.is_domain_blacklisted('spam.com.example.org') is False+ provider = HaGeZiBlacklistProvider("light")+ assert provider.is_domain_blacklisted("spam.com") is True+ assert provider.is_domain_blacklisted("www.spam.com") is True+ assert provider.is_domain_blacklisted("cdn.images.spam.com") is True+ assert provider.is_domain_blacklisted("notspam.com") is False+ assert provider.is_domain_blacklisted("spam.com.example.org") is Falsedef test_excluded_domains_cover_subdomains():"""EXCLUDED_DOMAINS entries are written apex-first; the www. form must still match."""- with patch.object(BuiltInRulesBlacklistProvider, '__init__', lambda self: None):+ with patch.object(BuiltInRulesBlacklistProvider, "__init__", lambda self: None):provider = BuiltInRulesBlacklistProvider()- provider.excluded_domains = {'fineartteens.com'}+ provider.excluded_domains = {"fineartteens.com"}provider.blacklist_regex = re.compile(r"^$")provider.trusted_domains = set()- assert provider.is_domain_blacklisted('fineartteens.com') is True- assert provider.is_domain_blacklisted('www.fineartteens.com') is True- assert provider.is_domain_blacklisted('otherfineartteens.com') is False+ assert provider.is_domain_blacklisted("fineartteens.com") is True+ assert provider.is_domain_blacklisted("www.fineartteens.com") is True+ assert provider.is_domain_blacklisted("otherfineartteens.com") is Falsedef test_domains_to_unblock_adds_the_apex_for_www_submissions():@@ -243,8 +242,8 @@ def test_curated_domains_override_the_providers():get_exempt_domains=lambda: {"pudding.cool"},)- assert provider.is_domain_blacklisted('pudding.cool') is False- assert provider.is_domain_blacklisted('reallybad.example') is True+ assert provider.is_domain_blacklisted("pudding.cool") is False+ assert provider.is_domain_blacklisted("reallybad.example") is Truedef test_curated_domains_override_the_built_in_rules():@@ -255,8 +254,8 @@ def test_curated_domains_override_the_built_in_rules():get_exempt_domains=lambda: {"adultswim.example"},)- assert provider.is_domain_blacklisted('adultswim.example') is False- assert provider.is_domain_blacklisted('otheradult.example') is True+ assert provider.is_domain_blacklisted("adultswim.example") is False+ assert provider.is_domain_blacklisted("otheradult.example") is Truedef test_curated_domains_match_either_www_form():@@ -265,8 +264,8 @@ def test_curated_domains_match_either_www_form():get_exempt_domains=lambda: {"www.contactmusic.com", "queer.example"},)- assert provider.is_domain_blacklisted('contactmusic.com') is False- assert provider.is_domain_blacklisted('www.queer.example') is False+ assert provider.is_domain_blacklisted("contactmusic.com") is False+ assert provider.is_domain_blacklisted("www.queer.example") is Falsedef test_exempt_domains_are_resolved_once():@@ -279,10 +278,11 @@ def test_exempt_domains_are_resolved_once():return {"pudding.cool"}provider = CombinedBlacklistProvider(- [StaticBlacklistProvider({"pudding.cool"})], get_exempt_domains=get_exempt_domains)+ [StaticBlacklistProvider({"pudding.cool"})], get_exempt_domains=get_exempt_domains+ )for _ in range(5):- provider.is_domain_blacklisted('pudding.cool')+ provider.is_domain_blacklisted("pudding.cool")assert len(calls) == 1diff --git a/test/test_blacklist_snapshot.py b/test/test_blacklist_snapshot.pyindex 41a6552..337ffce 100644--- a/test/test_blacklist_snapshot.py+++ b/test/test_blacklist_snapshot.py@@ -3,6 +3,11 @@ import numpy as npimport pytestfrom mwmbl.indexer import blacklist_snapshot+from mwmbl.indexer.blacklist_providers import (+ CombinedBlacklistProvider,+ RemoteListBlacklistProvider,+ StaticBlacklistProvider,+)from mwmbl.indexer.blacklist_snapshot import (SNAPSHOT_KEY,SNAPSHOT_VERSION_KEY,@@ -12,11 +17,6 @@ from mwmbl.indexer.blacklist_snapshot import (publish_snapshot,refresh_snapshot,)-from mwmbl.indexer.blacklist_providers import (- CombinedBlacklistProvider,- RemoteListBlacklistProvider,- StaticBlacklistProvider,-)class FakeRemoteProvider(RemoteListBlacklistProvider):@@ -37,11 +37,13 @@ def redis_client():@pytest.fixturedef provider():- return CombinedBlacklistProvider([- StaticBlacklistProvider({"not-remote.test"}),- FakeRemoteProvider({"badsite.test", "evil.test"}),- FakeRemoteProvider({"evil.test", "spam.test"}),- ])+ return CombinedBlacklistProvider(+ [+ StaticBlacklistProvider({"not-remote.test"}),+ FakeRemoteProvider({"badsite.test", "evil.test"}),+ FakeRemoteProvider({"evil.test", "spam.test"}),+ ]+ )def test_collect_remote_domains_walks_combined_providers_and_skips_local_ones(provider):@@ -59,12 +61,13 @@ def test_build_snapshot_is_sorted_and_deduplicated(provider):def test_publish_and_load_round_trip(provider, redis_client):version = refresh_snapshot(provider, redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)assert blacklist.load_now() is True- assert blacklist.filter_blacklisted(- ["badsite.test", "evil.test", "spam.test", "good.test"]- ) == {"badsite.test", "evil.test", "spam.test"}+ assert blacklist.filter_blacklisted(["badsite.test", "evil.test", "spam.test", "good.test"]) == {+ "badsite.test",+ "evil.test",+ "spam.test",+ }assert redis_client.get(SNAPSHOT_VERSION_KEY).decode() == version@@ -76,8 +79,7 @@ def test_load_is_a_no_op_when_the_version_is_unchanged(provider, redis_client):def test_load_picks_up_a_republished_snapshot(redis_client):- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"first.test"})]), redis_client)blacklist.load_now()@@ -91,8 +93,7 @@ def test_load_picks_up_a_republished_snapshot(redis_client):def test_built_in_rules_apply_without_a_snapshot(redis_client):"""The local rules must work even before the first snapshot lands, so a domain addedto mwmbl/settings.py takes effect on deploy rather than at the next refresh."""- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({"local.test"}),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({"local.test"}), redis_client=redis_client)assert blacklist.loaded is Falseassert blacklist.filter_blacklisted(["local.test", "good.test"]) == {"local.test"}@@ -102,8 +103,7 @@ def test_eviction_keeps_the_loaded_snapshot(provider, redis_client):"""Production Redis runs allkeys-lru, so the snapshot can vanish. Losing it must notsilently disable filtering."""refresh_snapshot(provider, redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)blacklist.load_now()redis_client.delete(SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY)@@ -118,8 +118,7 @@ def test_redis_failure_keeps_the_loaded_snapshot(provider):raise ConnectionError("redis is down")refresh_snapshot(provider, (working := fakeredis.FakeRedis()))- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=working)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=working)blacklist.load_now()blacklist._redis = BrokenRedis()@@ -143,8 +142,7 @@ def test_filter_blacklisted_handles_an_empty_input(provider, redis_client):def test_is_domain_blacklisted(provider, redis_client):refresh_snapshot(provider, redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)blacklist.load_now()assert blacklist.is_domain_blacklisted("badsite.test") is True@@ -154,8 +152,7 @@ def test_is_domain_blacklisted(provider, redis_client):def test_hashes_above_every_entry_do_not_index_out_of_bounds(redis_client):"""np.searchsorted returns len(array) for a hash larger than everything in it."""refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"only.test"})]), redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)blacklist.load_now()# Try enough domains that at least one hashes above and one below the single entry.@@ -171,30 +168,36 @@ def test_hashes_above_every_entry_do_not_index_out_of_bounds(redis_client):# subdomains)" - so an entry has to cover the subdomains of that domain too, starting# with the www. form most sites actually serve.+@pytest.fixturedef apex_blacklist(redis_client):refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"badsite.test"})]), redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)blacklist.load_now()return blacklist[email protected]("domain", [- "badsite.test",- "www.badsite.test",- "images.badsite.test",- "a.deeply.nested.badsite.test",-])+ "domain",+ [+ "badsite.test",+ "www.badsite.test",+ "images.badsite.test",+ "a.deeply.nested.badsite.test",+ ],+)def test_an_apex_entry_covers_its_subdomains(apex_blacklist, domain):assert apex_blacklist.is_domain_blacklisted(domain) is True[email protected]("domain", [- "badsite.test.example.com", # the entry is a prefix, not a suffix- "notbadsite.test", # suffix of the string, but not a parent domain- "example.test", # shares only the TLD-])+ "domain",+ [+ "badsite.test.example.com", # the entry is a prefix, not a suffix+ "notbadsite.test", # suffix of the string, but not a parent domain+ "example.test", # shares only the TLD+ ],+)def test_an_apex_entry_does_not_cover_unrelated_domains(apex_blacklist, domain):assert apex_blacklist.is_domain_blacklisted(domain) is False@@ -202,8 +205,7 @@ def test_an_apex_entry_does_not_cover_unrelated_domains(apex_blacklist, domain):def test_a_tld_is_never_a_candidate(redis_client):"""Matching down to a bare TLD would take out every domain under it."""refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"test"})]), redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)blacklist.load_now()assert blacklist.is_domain_blacklisted("innocent.test") is False@@ -213,8 +215,7 @@ def test_a_truncated_blob_is_ignored_rather_than_raising(provider, redis_client)"""load_now() runs at import time via search_setup, so raising here would stop everyweb worker from starting."""refresh_snapshot(provider, redis_client)- blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),- redis_client=redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()), redis_client=redis_client)blacklist.load_now()assert blacklist.is_domain_blacklisted("badsite.test") is Truediff --git a/test/test_comma_urls.py b/test/test_comma_urls.pyindex ce56a15..44a7f7a 100644--- a/test/test_comma_urls.py+++ b/test/test_comma_urls.py@@ -1,30 +1,20 @@-import pytestimport json-from urllib.parse import quote-from django.contrib.auth import get_user_model++import pytestfrom allauth.account.models import EmailAddress+from django.contrib.auth import get_user_modelfrom ninja_jwt.tokens import RefreshTokenfrom mwmbl.models import SearchResultVote-User = get_user_model()@pytest.fixturedef verified_user():"""Create a user with verified email"""- user = User.objects.create_user(- username='testuser',- email='[email protected]',- password='testpass123'- )- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=True,- primary=True- )+ user = User.objects.create_user(username="testuser", email="[email protected]", password="testpass123")+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@@ -41,38 +31,30 @@ def test_urls_with_commas_single_url(client, verified_user, access_token):# URL with comma in the path (like the original issue)url_with_comma = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'query = "hello world"-+# Create a vote for this URL- SearchResultVote.objects.create(- user=verified_user,- url=url_with_comma,- query=query,- vote_type='upvote'- )-+ SearchResultVote.objects.create(user=verified_user, url=url_with_comma, query=query, vote_type="upvote")+# Test getting vote counts using POST- vote_stats_request = {- "query": query,- "urls": [url_with_comma]- }-+ vote_stats_request = {"query": query, "urls": [url_with_comma]}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert 'votes' in response_data- assert url_with_comma in response_data['votes']-- vote_stats = response_data['votes'][url_with_comma]- assert vote_stats['upvotes'] == 1- assert vote_stats['downvotes'] == 0- assert vote_stats['user_vote'] == 'upvote'+ assert "votes" in response_data+ assert url_with_comma in response_data["votes"]++ vote_stats = response_data["votes"][url_with_comma]+ assert vote_stats["upvotes"] == 1+ assert vote_stats["downvotes"] == 0+ assert vote_stats["user_vote"] == "upvote"@pytest.mark.django_db@@ -80,98 +62,77 @@ def test_urls_with_commas_multiple_urls(client, verified_user, access_token):"""Test handling multiple URLs with commas using POST"""# Multiple URLs with commas (from the original issue)url1 = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'- url2 = 'https://en.wikipedia.org/wiki/Hello_Twelve,_Hello_Thirteen,_Hello_Love'+ url2 = "https://en.wikipedia.org/wiki/Hello_Twelve,_Hello_Thirteen,_Hello_Love"query = "hello"-+# Create votes for these URLs- SearchResultVote.objects.create(- user=verified_user,- url=url1,- query=query,- vote_type='upvote'- )- SearchResultVote.objects.create(- user=verified_user,- url=url2,- query=query,- vote_type='downvote'- )-+ SearchResultVote.objects.create(user=verified_user, url=url1, query=query, vote_type="upvote")+ SearchResultVote.objects.create(user=verified_user, url=url2, query=query, vote_type="downvote")+# Test getting vote counts using POST- vote_stats_request = {- "query": query,- "urls": [url1, url2]- }-+ vote_stats_request = {"query": query, "urls": [url1, url2]}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert len(response_data['votes']) == 2- assert url1 in response_data['votes']- assert url2 in response_data['votes']-+ assert len(response_data["votes"]) == 2+ assert url1 in response_data["votes"]+ assert url2 in response_data["votes"]+# Verify vote counts are correct- assert response_data['votes'][url1]['upvotes'] == 1- assert response_data['votes'][url1]['downvotes'] == 0- assert response_data['votes'][url1]['user_vote'] == 'upvote'-- assert response_data['votes'][url2]['upvotes'] == 0- assert response_data['votes'][url2]['downvotes'] == 1- assert response_data['votes'][url2]['user_vote'] == 'downvote'+ assert response_data["votes"][url1]["upvotes"] == 1+ assert response_data["votes"][url1]["downvotes"] == 0+ assert response_data["votes"][url1]["user_vote"] == "upvote"++ assert response_data["votes"][url2]["upvotes"] == 0+ assert response_data["votes"][url2]["downvotes"] == 1+ assert response_data["votes"][url2]["user_vote"] == "downvote"@pytest.mark.django_dbdef test_urls_with_various_special_characters(client, verified_user, access_token):"""Test URLs with various special characters that could cause parsing issues"""special_urls = [- 'https://example.com/path?param=value,with,commas',- 'https://example.com/path?param=value%26other=a,b,c', # URL-encoded ampersand- 'https://example.com/path/with,comma/in/path',- 'https://example.com/path?query=hello%2Cworld', # URL-encoded comma+ "https://example.com/path?param=value,with,commas",+ "https://example.com/path?param=value%26other=a,b,c", # URL-encoded ampersand+ "https://example.com/path/with,comma/in/path",+ "https://example.com/path?query=hello%2Cworld", # URL-encoded comma'https://example.com/path?data={"key":"value,with,comma"}',]query = "special characters test"-+# Create votes for all URLsfor i, url in enumerate(special_urls):- vote_type = 'upvote' if i % 2 == 0 else 'downvote'- SearchResultVote.objects.create(- user=verified_user,- url=url,- query=query,- vote_type=vote_type- )-+ vote_type = "upvote" if i % 2 == 0 else "downvote"+ SearchResultVote.objects.create(user=verified_user, url=url, query=query, vote_type=vote_type)+# Test getting vote counts using POST- vote_stats_request = {- "query": query,- "urls": special_urls- }-+ vote_stats_request = {"query": query, "urls": special_urls}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert len(response_data['votes']) == len(special_urls)-+ assert len(response_data["votes"]) == len(special_urls)+# Verify all URLs are present and have correct vote countsfor i, url in enumerate(special_urls):- assert url in response_data['votes']- expected_vote_type = 'upvote' if i % 2 == 0 else 'downvote'- assert response_data['votes'][url]['user_vote'] == expected_vote_type+ assert url in response_data["votes"]+ expected_vote_type = "upvote" if i % 2 == 0 else "downvote"+ assert response_data["votes"][url]["user_vote"] == expected_vote_type@pytest.mark.django_db@@ -179,49 +140,38 @@ def test_voting_on_url_with_commas(client, verified_user, access_token):"""Test voting on URLs with commas works correctly"""url_with_comma = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'query = "hello world"-- vote_data = {- "url": url_with_comma,- "query": query,- "vote_type": "upvote"- }-++ vote_data = {"url": url_with_comma, "query": query, "vote_type": "upvote"}+# Test votingresponse = client.post(- '/api/v1/platform/search-results/vote',+ "/api/v1/platform/search-results/vote",data=json.dumps(vote_data),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert response_data['status'] == 'ok'- assert 'Vote created successfully' in response_data['message']-+ assert response_data["status"] == "ok"+ assert "Vote created successfully" in response_data["message"]+# Verify vote was created in database with exact URL- vote = SearchResultVote.objects.get(- user=verified_user,- url=url_with_comma,- query=query- )- assert vote.vote_type == 'upvote'-+ vote = SearchResultVote.objects.get(user=verified_user, url=url_with_comma, query=query)+ assert vote.vote_type == "upvote"+# Test retrieving the vote using POST- vote_stats_request = {- "query": query,- "urls": [url_with_comma]- }-+ vote_stats_request = {"query": query, "urls": [url_with_comma]}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200response_data = response.json()- assert url_with_comma in response_data['votes']- assert response_data['votes'][url_with_comma]['user_vote'] == 'upvote'+ assert url_with_comma in response_data["votes"]+ assert response_data["votes"][url_with_comma]["user_vote"] == "upvote"diff --git a/test/test_completer.py b/test/test_completer.pyindex e9612a3..8f4362f 100644--- a/test/test_completer.py+++ b/test/test_completer.py@@ -1,84 +1,62 @@import pandas as pd-from mwmbl import tinysearchengine+from mwmbl.tinysearchengine.completer import Completerdef mockCompleterData(mocker, data):- testDataFrame = pd.DataFrame(data, columns=['','term','count'])- mocker.patch('mwmbl.tinysearchengine.completer.Completer.get_terms',- return_value = testDataFrame)+ testDataFrame = pd.DataFrame(data, columns=["", "term", "count"])+ mocker.patch("mwmbl.tinysearchengine.completer.Completer.get_terms", return_value=testDataFrame)def test_correctCompletions(mocker):# Mock completer with custom data- testdata = [- [0, 'build', 4],- [1, 'builder', 3],- [2, 'announce', 2],- [3, 'buildings', 1]]+ testdata = [[0, "build", 4], [1, "builder", 3], [2, "announce", 2], [3, "buildings", 1]]mockCompleterData(mocker, testdata)-+completer = Completer()- completion = completer.complete('build')- assert ['build', 'builder', 'buildings'] == completion+ completion = completer.complete("build")+ assert ["build", "builder", "buildings"] == completiondef test_correctSortOrder(mocker):# Mock completer with custom data- testdata = [- [0, 'build', 4],- [1, 'builder', 1],- [2, 'announce', 2],- [3, 'buildings', 3]]+ testdata = [[0, "build", 4], [1, "builder", 1], [2, "announce", 2], [3, "buildings", 3]]mockCompleterData(mocker, testdata)-+completer = Completer()- completion = completer.complete('build')- assert ['build', 'buildings', 'builder'] == completion+ completion = completer.complete("build")+ assert ["build", "buildings", "builder"] == completiondef test_noCompletions(mocker):# Mock completer with custom data- testdata = [- [0, 'build', 4],- [1, 'builder', 3],- [2, 'announce', 2],- [3, 'buildings', 1]]+ testdata = [[0, "build", 4], [1, "builder", 3], [2, "announce", 2], [3, "buildings", 1]]mockCompleterData(mocker, testdata)-+completer = Completer()- completion = completer.complete('test')+ completion = completer.complete("test")assert [] == completiondef test_singleCompletions(mocker):# Mock completer with custom data- testdata = [- [0, 'build', 4],- [1, 'builder', 3],- [2, 'announce', 2],- [3, 'buildings', 1]]+ testdata = [[0, "build", 4], [1, "builder", 3], [2, "announce", 2], [3, "buildings", 1]]mockCompleterData(mocker, testdata)-+completer = Completer()- completion = completer.complete('announce')- assert ['announce'] == completion+ completion = completer.complete("announce")+ assert ["announce"] == completiondef test_idempotencyWithSameScoreCompletions(mocker):# Mock completer with custom data- testdata = [- [0, 'build', 1],- [1, 'builder', 1],- [2, 'announce', 1],- [3, 'buildings', 1]]+ testdata = [[0, "build", 1], [1, "builder", 1], [2, "announce", 1], [3, "buildings", 1]]mockCompleterData(mocker, testdata)-+completer = Completer()for i in range(3):print(f"iteration: {i}")- completion = completer.complete('build')+ completion = completer.complete("build")# Results expected in reverse order- expected = ['buildings','builder','build']+ expected = ["buildings", "builder", "build"]assert expected == completion-\ No newline at end of filediff --git a/test/test_copy_index.py b/test/test_copy_index.pyindex 25f7e31..69a22a0 100644--- a/test/test_copy_index.py+++ b/test/test_copy_index.py@@ -1,11 +1,11 @@from pathlib import Pathfrom tempfile import TemporaryDirectory-from mwmbl.tinysearchengine.copy_index import copy_pages-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE-from django.conf import settings+from mwmbl.tinysearchengine.copy_index import copy_pages+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex+NUM_PAGES = 10000diff --git a/test/test_crawl_functional.py b/test/test_crawl_functional.pyindex b16a8cb..b5f91e9 100644--- a/test/test_crawl_functional.py+++ b/test/test_crawl_functional.py@@ -5,17 +5,17 @@ import timefrom collections import Counterfrom datetime import datetime, timedeltafrom pathlib import Path-from unittest.mock import patch, MagicMock, mock_open+from unittest.mock import MagicMock, patchimport fakeredisimport pytestfrom django.test import override_settings-from mwmbl.crawl import Crawler, crawl_url, process_batch, run_indexing-from mwmbl.crawler.batch import HashedBatch, Result, Item, ItemContent+from mwmbl.crawl import Crawler, crawl_url+from mwmbl.crawler.batch import HashedBatch, Item, ItemContentfrom mwmbl.crawler.urls import FoundURL, URLStatus-from mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.indexer.blacklist_providers import StaticBlacklistProvider+from mwmbl.redis_url_queue import RedisURLQueue@pytest.fixture@@ -38,7 +38,7 @@ def mock_settings(temp_data_path):DATA_PATH=str(temp_data_path),REDIS_URL="redis://localhost:6379",HAS_DATABASE=False,- NUM_PAGES=10 # Small number for testing+ NUM_PAGES=10, # Small number for testing):yield@@ -47,10 +47,10 @@ def mock_settings(temp_data_path):def mock_environment():"""Mock environment variables"""env_vars = {- 'MWMBL_API_KEY': 'test-api-key',- 'MWMBL_CONTACT_INFO': '[email protected]',- 'CRAWLER_WORKERS': '1',- 'CRAWL_DELAY_SECONDS': '0.0'+ "MWMBL_API_KEY": "test-api-key",+ "MWMBL_CONTACT_INFO": "[email protected]",+ "CRAWLER_WORKERS": "1",+ "CRAWL_DELAY_SECONDS": "0.0",}with patch.dict(os.environ, env_vars):yield@@ -60,16 +60,16 @@ def mock_environment():def mock_crawl_response():"""Mock successful crawl response"""return {- 'url': 'https://example.com',- 'status': 200,- 'timestamp': int(time.time() * 1000),- 'content': {- 'title': 'Example Page',- 'extract': 'This is an example page with some content.',- 'links': ['https://example.com/page1', 'https://example.com/page2'],- 'extra_links': ['https://example.com/extra1']+ "url": "https://example.com",+ "status": 200,+ "timestamp": int(time.time() * 1000),+ "content": {+ "title": "Example Page",+ "extract": "This is an example page with some content.",+ "links": ["https://example.com/page1", "https://example.com/page2"],+ "extra_links": ["https://example.com/extra1"],},- 'error': None+ "error": None,}@@ -78,37 +78,33 @@ def sample_found_urls():"""Create sample FoundURL objects for testing"""return [FoundURL(- url='https://example.com/page1',- user_id_hash='test_user_hash',+ url="https://example.com/page1",+ user_id_hash="test_user_hash",status=URLStatus.NEW,timestamp=datetime.utcnow(),- last_crawled=datetime.utcnow() - timedelta(days=60)+ last_crawled=datetime.utcnow() - timedelta(days=60),),FoundURL(- url='https://example.com/page2',- user_id_hash='test_user_hash',+ url="https://example.com/page2",+ user_id_hash="test_user_hash",status=URLStatus.NEW,timestamp=datetime.utcnow(),- last_crawled=datetime.utcnow() - timedelta(days=90)+ last_crawled=datetime.utcnow() - timedelta(days=90),),FoundURL(- url='https://test.com/article',- user_id_hash='test_user_hash',+ url="https://test.com/article",+ user_id_hash="test_user_hash",status=URLStatus.NEW,timestamp=datetime.utcnow(),- last_crawled=None- )+ last_crawled=None,+ ),]@pytest.fixturedef test_blacklist_provider():"""Create a static blacklist provider for tests to avoid network calls."""- test_blacklist_domains = {- 'spam.com',- 'malware.example',- 'badsite.net'- }+ test_blacklist_domains = {"spam.com", "malware.example", "badsite.net"}return StaticBlacklistProvider(test_blacklist_domains)@@ -123,11 +119,11 @@ class TestCrawlFunctional:crawler._redis = fake_redis # Set the private attribute directly# Should not raise an exceptioncrawler.check_redis()-+# Test with broken Redisbroken_redis = MagicMock()broken_redis.ping.side_effect = ConnectionError("Redis unavailable")-+crawler._redis = broken_redis # Set the private attribute directlywith pytest.raises(SystemExit):crawler.check_redis()@@ -135,43 +131,43 @@ class TestCrawlFunctional:def test_url_queue_operations(self, fake_redis, sample_found_urls, test_blacklist_provider):"""Test RedisURLQueue operations with fake Redis"""url_queue = RedisURLQueue(fake_redis, lambda: set(), test_blacklist_provider)-+# Test queuing URLsurl_queue.queue_urls(sample_found_urls)-+# Verify URLs were queuedassert fake_redis.zcard("domain-scores") > 0-+# Test getting a batchbatch = url_queue.get_batch("test_user")assert len(batch) > 0- assert all(url.startswith('http') for url in batch)+ assert all(url.startswith("http") for url in batch)def test_process_batch_workflow(self, fake_redis, mock_settings, mock_environment, mock_crawl_response):"""Test the complete batch processing workflow"""crawler = Crawler()crawler._redis = fake_redis # Set the private attribute directly-+# Mock the URL queuemock_url_queue = MagicMock()- mock_url_queue.get_batch.return_value = ['https://example.com']+ mock_url_queue.get_batch.return_value = ["https://example.com"]crawler._url_queue = mock_url_queue-- with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):- with patch('mwmbl.crawl.record_urls_in_database'):++ with patch("mwmbl.crawl.crawl_url", return_value=mock_crawl_response):+ with patch("mwmbl.crawl.record_urls_in_database"):# Should complete without errorscrawler.process_batch()-+# Verify batch was pushed to Redisbatch_data = fake_redis.lpop("batch-queue")assert batch_data is not None-+# Verify batch structurebatch = json.loads(batch_data)- assert 'user_id_hash' in batch- assert 'timestamp' in batch- assert 'items' in batch- assert len(batch['items']) == 1+ assert "user_id_hash" in batch+ assert "timestamp" in batch+ assert "items" in batch+ assert len(batch["items"]) == 1def test_indexing_workflow(self, fake_redis, mock_settings, mock_environment, temp_data_path):"""Test the indexing workflow with fake Redis"""@@ -181,49 +177,42 @@ class TestCrawlFunctional:url="https://example.com",status=200,timestamp=time.time(),- content=ItemContent(- title="Example",- extract="Test content"- )- )-- sample_batch = HashedBatch(- user_id_hash="test_user",- timestamp=int(time.time() * 1000),- items=[sample_item]+ content=ItemContent(title="Example", extract="Test content"),)++ sample_batch = HashedBatch(user_id_hash="test_user", timestamp=int(time.time() * 1000), items=[sample_item])fake_redis.rpush("batch-queue", sample_batch.json())-+crawler = Crawler()crawler._redis = fake_redis # Set the private attribute directly-+# Mock Counter object for index_batches return value- mock_counter = Counter({'example': 1})- with patch('mwmbl.crawl.index_batches', return_value=mock_counter):- with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:- with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:- with patch('mwmbl.crawl.index_pages') as mock_index_pages:- with patch('requests.post') as mock_post:- # Setup mocks- mock_remote_index_instance = MagicMock()- mock_remote_index.return_value = mock_remote_index_instance- mock_remote_index_instance.retrieve.return_value = []-- mock_tiny_index_instance = MagicMock()- mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance- mock_tiny_index_instance.retrieve.return_value = []- mock_tiny_index_instance.get_key_page_index.return_value = 0- mock_tiny_index_instance.get_page.return_value = "test page content"-- mock_post.return_value.status_code = 200- mock_post.return_value.text = "OK"-- # Should complete without errors- crawler.run_indexing()-- # Verify batch was consumed from Redis- remaining_batches = fake_redis.llen("batch-queue")- assert remaining_batches == 0+ mock_counter = Counter({"example": 1})+ with patch("mwmbl.crawl.index_batches", return_value=mock_counter):+ with patch("mwmbl.crawl.RemoteIndex") as mock_remote_index:+ with patch("mwmbl.crawl.TinyIndex") as mock_tiny_index:+ with patch("mwmbl.crawl.index_pages"):+ with patch("requests.post") as mock_post:+ # Setup mocks+ mock_remote_index_instance = MagicMock()+ mock_remote_index.return_value = mock_remote_index_instance+ mock_remote_index_instance.retrieve.return_value = []++ mock_tiny_index_instance = MagicMock()+ mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance+ mock_tiny_index_instance.retrieve.return_value = []+ mock_tiny_index_instance.get_key_page_index.return_value = 0+ mock_tiny_index_instance.get_page.return_value = "test page content"++ mock_post.return_value.status_code = 200+ mock_post.return_value.text = "OK"++ # Should complete without errors+ crawler.run_indexing()++ # Verify batch was consumed from Redis+ remaining_batches = fake_redis.llen("batch-queue")+ assert remaining_batches == 0def test_crawl_url_functionality(self, mock_environment):"""Test individual URL crawling functionality"""@@ -239,149 +228,153 @@ class TestCrawlFunctional:</body></html>"""-- with patch('mwmbl.crawler.retrieve.fetch') as mock_fetch:- with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=True):- mock_fetch.return_value = (200, mock_html_content, 'https://example.com')-- result = crawl_url('https://example.com', redis=None)-- assert result['url'] == 'https://example.com'- assert result['resolved_url'] == 'https://example.com'- assert result['status'] == 200- assert result['content'] is not None- assert result['content']['title'] == 'Test Page'++ with patch("mwmbl.crawler.retrieve.fetch") as mock_fetch:+ with patch("mwmbl.crawler.retrieve.robots_allowed", return_value=True):+ mock_fetch.return_value = (200, mock_html_content, "https://example.com")++ result = crawl_url("https://example.com", redis=None)++ assert result["url"] == "https://example.com"+ assert result["resolved_url"] == "https://example.com"+ assert result["status"] == 200+ assert result["content"] is not None+ assert result["content"]["title"] == "Test Page"# The extract might be empty due to justext filtering, so let's just check it exists- assert 'extract' in result['content']- assert result['error'] is None+ assert "extract" in result["content"]+ assert result["error"] is Nonedef test_crawl_url_error_handling(self, mock_environment):"""Test URL crawling error handling"""- with patch('mwmbl.crawler.retrieve.fetch') as mock_fetch:- with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=True):+ with patch("mwmbl.crawler.retrieve.fetch") as mock_fetch:+ with patch("mwmbl.crawler.retrieve.robots_allowed", return_value=True):# Test connection errormock_fetch.side_effect = ConnectionError("Connection failed")-- result = crawl_url('https://example.com', redis=None)-- assert result['url'] == 'https://example.com'- assert result['status'] is None- assert result['content'] is None- assert result['error'] is not None- assert result['error']['name'] == 'AbortError'++ result = crawl_url("https://example.com", redis=None)++ assert result["url"] == "https://example.com"+ assert result["status"] is None+ assert result["content"] is None+ assert result["error"] is not None+ assert result["error"]["name"] == "AbortError"def test_robots_denied_handling(self, mock_environment):"""Test robots.txt denial handling"""- with patch('mwmbl.crawler.retrieve.robots_allowed', return_value=False):- result = crawl_url('https://example.com', redis=None)-- assert result['url'] == 'https://example.com'- assert result['status'] is None- assert result['content'] is None- assert result['error'] is not None- assert result['error']['name'] == 'RobotsDenied'+ with patch("mwmbl.crawler.retrieve.robots_allowed", return_value=False):+ result = crawl_url("https://example.com", redis=None)++ assert result["url"] == "https://example.com"+ assert result["status"] is None+ assert result["content"] is None+ assert result["error"] is not None+ assert result["error"]["name"] == "RobotsDenied"def test_batch_queue_operations(self, fake_redis):"""Test batch queue operations in Redis"""# Test pushing and popping batches- sample_batch = {- 'user_id_hash': 'test_user',- 'timestamp': int(time.time() * 1000),- 'items': []- }-+ sample_batch = {"user_id_hash": "test_user", "timestamp": int(time.time() * 1000), "items": []}+# Push batchfake_redis.rpush("batch-queue", json.dumps(sample_batch))-+# Pop batchbatch_data = fake_redis.lpop("batch-queue")assert batch_data is not None-+retrieved_batch = json.loads(batch_data)- assert retrieved_batch['user_id_hash'] == 'test_user'+ assert retrieved_batch["user_id_hash"] == "test_user"def test_domain_scoring_and_url_management(self, fake_redis, sample_found_urls, test_blacklist_provider):"""Test domain scoring and URL management in Redis"""url_queue = RedisURLQueue(fake_redis, lambda: set(), test_blacklist_provider)-+# Queue URLsurl_queue.queue_urls(sample_found_urls)-+# Check domain scores were createddomain_count = fake_redis.zcard("domain-scores")assert domain_count > 0-+# Check individual domain URL countsexample_count = url_queue.get_domain_count("example.com")test_count = url_queue.get_domain_count("test.com")-+assert example_count >= 0assert test_count >= 0def test_user_url_assignment(self, fake_redis, sample_found_urls, test_blacklist_provider):"""Test user URL assignment and tracking"""url_queue = RedisURLQueue(fake_redis, lambda: set(), test_blacklist_provider)-+# Queue URLs and get batchurl_queue.queue_urls(sample_found_urls)batch = url_queue.get_batch("test_user")-+# Check user assignmentuser_urls = fake_redis.get("user-urls-test_user")assert user_urls is not None-+assigned_urls = json.loads(user_urls)assert len(assigned_urls) == len(batch)-+# Test URL checkinguncrawled = url_queue.check_user_crawled_urls("test_user", batch)assert len(uncrawled) == 0 # All URLs should be assigned to this user- def test_integration_workflow(self, fake_redis, mock_settings, mock_environment,- sample_found_urls, mock_crawl_response, temp_data_path, test_blacklist_provider):+ def test_integration_workflow(+ self,+ fake_redis,+ mock_settings,+ mock_environment,+ sample_found_urls,+ mock_crawl_response,+ temp_data_path,+ test_blacklist_provider,+ ):"""Test the complete integration workflow"""# Setup URL queueurl_queue = RedisURLQueue(fake_redis, lambda: set(), test_blacklist_provider)url_queue.queue_urls(sample_found_urls)-+# Create crawler instancecrawler = Crawler()crawler._redis = fake_rediscrawler._url_queue = url_queue-- with patch('mwmbl.crawl.crawl_url', return_value=mock_crawl_response):- with patch('mwmbl.crawl.record_urls_in_database'):++ with patch("mwmbl.crawl.crawl_url", return_value=mock_crawl_response):+ with patch("mwmbl.crawl.record_urls_in_database"):# Mock Counter object for index_batches return value- mock_counter = Counter({'example': 1})- with patch('mwmbl.crawl.index_batches', return_value=mock_counter):- with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:- with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:- with patch('mwmbl.crawl.index_pages') as mock_index_pages:- with patch('requests.post') as mock_post:+ mock_counter = Counter({"example": 1})+ with patch("mwmbl.crawl.index_batches", return_value=mock_counter):+ with patch("mwmbl.crawl.RemoteIndex") as mock_remote_index:+ with patch("mwmbl.crawl.TinyIndex") as mock_tiny_index:+ with patch("mwmbl.crawl.index_pages"):+ with patch("requests.post") as mock_post:# Setup mocks for indexingmock_remote_index_instance = MagicMock()mock_remote_index.return_value = mock_remote_index_instancemock_remote_index_instance.retrieve.return_value = []-+mock_tiny_index_instance = MagicMock()mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instancemock_tiny_index_instance.retrieve.return_value = []mock_tiny_index_instance.get_key_page_index.return_value = 0mock_tiny_index_instance.get_page.return_value = "test content"-+mock_post.return_value.status_code = 200mock_post.return_value.text = "OK"-+# Run complete workflowcrawler.process_batch() # Should create batch in Redis-+# Verify batch was createdbatch_count = fake_redis.llen("batch-queue")assert batch_count == 1-+# Run indexingcrawler.run_indexing() # Should process the batch-+# Verify batch was consumedremaining_batches = fake_redis.llen("batch-queue")assert remaining_batches == 0@@ -391,32 +384,32 @@ class TestCrawlFunctional:# Create crawler instancecrawler = Crawler()crawler._redis = fake_redis-+# Mock the URL queue for empty batch testmock_url_queue = MagicMock()mock_url_queue.get_batch.return_value = []crawler._url_queue = mock_url_queue-- with patch('mwmbl.crawl.record_urls_in_database'):++ with patch("mwmbl.crawl.record_urls_in_database"):# Should handle empty batch gracefullycrawler.process_batch()-+# Test indexing with no batches - mock sleep to avoid waiting- with patch('time.sleep'):- with patch('mwmbl.crawl.index_batches') as mock_index_batches:- with patch('mwmbl.crawl.TinyIndex') as mock_tiny_index:- with patch('mwmbl.crawl.RemoteIndex') as mock_remote_index:+ with patch("time.sleep"):+ with patch("mwmbl.crawl.index_batches") as mock_index_batches:+ with patch("mwmbl.crawl.TinyIndex") as mock_tiny_index:+ with patch("mwmbl.crawl.RemoteIndex") as mock_remote_index:# Mock the index_batches function to avoid file system accessmock_index_batches.return_value = Counter()-+# Setup TinyIndex mockmock_tiny_index_instance = MagicMock()mock_tiny_index.return_value.__enter__.return_value = mock_tiny_index_instance-+# Setup RemoteIndex mockmock_remote_index_instance = MagicMock()mock_remote_index.return_value = mock_remote_index_instancemock_remote_index_instance.retrieve.return_value = []-+# Should handle no batches gracefully (will sleep and return)crawler.run_indexing()diff --git a/test/test_crawl_import_order.py b/test/test_crawl_import_order.pyindex 85b102a..414b757 100644--- a/test/test_crawl_import_order.py+++ b/test/test_crawl_import_order.py@@ -4,6 +4,7 @@ mwmbl.crawl sets the settings module and calls django.setup() itself, so every min it must come after that call - the mwmbl package reaches Django models (viacurated_domains) and raises ImproperlyConfigured if the app registry is not ready."""+import astfrom pathlib import Path@@ -12,9 +13,13 @@ CRAWL_PATH = Path(__file__).parent.parent / "mwmbl" / "crawl.py"def _django_setup_line(module: ast.Module) -> int:for node in ast.walk(module):- if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)- and node.func.attr == "setup" and isinstance(node.func.value, ast.Name)- and node.func.value.id == "django"):+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "setup"+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id == "django"+ ):return node.linenoraise AssertionError("mwmbl/crawl.py does not call django.setup()")@@ -24,8 +29,11 @@ def test_mwmbl_imports_come_after_django_setup():setup_line = _django_setup_line(module)early_imports = [- node.module for node in module.body- if isinstance(node, ast.ImportFrom) and node.lineno < setup_line- and node.module is not None and node.module.startswith("mwmbl")+ node.module+ for node in module.body+ if isinstance(node, ast.ImportFrom)+ and node.lineno < setup_line+ and node.module is not None+ and node.module.startswith("mwmbl")]assert early_imports == []diff --git a/test/test_curated_domains.py b/test/test_curated_domains.pyindex 0c43e6a..13f130f 100644--- a/test/test_curated_domains.py+++ b/test/test_curated_domains.py@@ -1,4 +1,5 @@"""Tests for the approved-domain lookup that overrides the blacklists."""+from unittest.mock import patchimport pytestdiff --git a/test/test_domain_moderation.py b/test/test_domain_moderation.pyindex 87f994b..951a979 100644--- a/test/test_domain_moderation.py+++ b/test/test_domain_moderation.py@@ -5,6 +5,7 @@ the training gate - but that a moderator is never shown something misleading: aeither backed by evidence or absent, a deterministic check always beats a probability, and thequeue never runs a model on the request path."""+import ioimport jsonimport sys@@ -21,14 +22,12 @@ from django.contrib.auth.models import Permissionfrom django.utils import timezonefrom ninja_jwt.tokens import RefreshToken-from mwmbl.models import (- DomainEvidence, DomainSubmission, ModerationModelArtifact, MwmblUser, SearchResultVote)+from mwmbl.models import DomainEvidence, DomainSubmission, ModerationModelArtifact, MwmblUser, SearchResultVotefrom mwmbl.moderation import model as model_modulefrom mwmbl.moderation import rulesfrom mwmbl.moderation.evidence import crawl_domainfrom mwmbl.moderation.features import Featuriser, ModerationExample-from mwmbl.moderation.model import (- Suggestion, get_model, load_metrics, publish, reset_model_cache, suggest)+from mwmbl.moderation.model import Suggestion, get_model, load_metrics, publish, reset_model_cache, suggestfrom mwmbl.moderation.suggest import annotate_queue, refresh_suggestion, suggestion_forfrom mwmbl.moderation.train import _suggestion_influence, operating_point, passes_gatefrom mwmbl.moderation.training_data import TrainingRow, is_trainable_domain@@ -38,8 +37,7 @@ QUEUE_URL = "/api/v1/platform/domain-submissions/queue"@pytest.fixturedef submitter(db):- return MwmblUser.objects.create_user(- username="submitter", email="[email protected]", password="password")+ return MwmblUser.objects.create_user(username="submitter", email="[email protected]", password="password")@pytest.fixture(autouse=True)@@ -53,21 +51,17 @@ def clear_the_model_cache():@pytest.fixturedef established(db):"""A submitter with a track record, whose approvals are not withheld."""- user = MwmblUser.objects.create_user(- username="established", email="[email protected]", password="password")+ user = MwmblUser.objects.create_user(username="established", email="[email protected]", password="password")for index in range(3):- DomainSubmission.objects.create(name=f"past{index}.example", submitted_by=user,- status="APPROVED")+ DomainSubmission.objects.create(name=f"past{index}.example", submitted_by=user, status="APPROVED")return user@pytest.fixturedef moderator(db):- user = MwmblUser.objects.create_user(- username="moderator", email="[email protected]", password="password")+ user = MwmblUser.objects.create_user(username="moderator", email="[email protected]", password="password")EmailAddress.objects.create(user=user, email=user.email, verified=True, primary=True)- user.user_permissions.add(- Permission.objects.get(codename="change_domain_submission_status"))+ user.user_permissions.add(Permission.objects.get(codename="change_domain_submission_status"))return user@@ -80,8 +74,16 @@ def ready_evidence(domain, **overrides):"state": DomainEvidence.State.READY,"fetched_at": timezone.now(),"http_status": 200,- "pages": [{"url": f"https://{domain}/", "status": 200, "title": "A title",- "extract": "Some body text", "num_links": 12, "error": ""}],+ "pages": [+ {+ "url": f"https://{domain}/",+ "status": 200,+ "title": "A title",+ "extract": "Some body text",+ "num_links": 12,+ "error": "",+ }+ ],"signals": {"has_links": True, "num_pages_fetched": 1, "blacklisted": False},"suggested_action": "APPROVE","confidence": 0.8,@@ -100,16 +102,24 @@ def ready_evidence(domain, **overrides):# --------------------------------------------------------------------- rules[email protected]("crawl,expected_kind", [- ({"http_status": 404, "pages": [{"title": "", "extract": "", "num_links": 0}],- "signals": {}}, "http_status"),- ({"http_status": None, "error": "RobotsDenied", "pages": [], "signals": {}}, "robots"),- ({"http_status": None, "error": "AbortError", "pages": [], "signals": {}}, "unreachable"),- ({"http_status": 200, "final_domain": "somewhere-else.com", "pages": [],- "signals": {}}, "redirect"),- ({"http_status": 200, "pages": [{"title": "t", "extract": "e", "num_links": 1}],- "signals": {"has_links": True, "blacklisted": True}}, "blocklist"),-])++ "crawl,expected_kind",+ [+ ({"http_status": 404, "pages": [{"title": "", "extract": "", "num_links": 0}], "signals": {}}, "http_status"),+ ({"http_status": None, "error": "RobotsDenied", "pages": [], "signals": {}}, "robots"),+ ({"http_status": None, "error": "AbortError", "pages": [], "signals": {}}, "unreachable"),+ ({"http_status": 200, "final_domain": "somewhere-else.com", "pages": [], "signals": {}}, "redirect"),+ (+ {+ "http_status": 200,+ "pages": [{"title": "t", "extract": "e", "num_links": 1}],+ "signals": {"has_links": True, "blacklisted": True},+ },+ "blocklist",+ ),+ ],+)def test_each_deterministic_check_fires(crawl, expected_kind):kinds = {item.kind for item in rules.crawl_evidence("example.com", crawl)}assert expected_kind in kinds@@ -124,8 +134,8 @@ def test_do_not_crawl_list_is_decisive():def test_subdomain_redirect_is_not_treated_as_off_domain():items = rules.crawl_evidence(- "example.com",- {"http_status": 200, "final_domain": "www.example.com", "pages": [], "signals": {}})+ "example.com", {"http_status": 200, "final_domain": "www.example.com", "pages": [], "signals": {}}+ )assert "redirect" not in {item.kind for item in items}@@ -141,14 +151,14 @@ def test_previous_approval_of_the_same_domain_is_decisive():# --------------------------------------------------------------------- suggestions+def test_a_deterministic_check_beats_the_model():"""A 404 is not a matter of opinion, so no probability may override it."""model = mock.Mock()model.version = "test"- model.predict.return_value = [(0.01, "SPAM", 0.9)] # the model says "approve"+ model.predict.return_value = [(0.01, "SPAM", 0.9)] # the model says "approve"- items = rules.crawl_evidence(- "example.com", {"http_status": 404, "pages": [], "signals": {}})+ items = rules.crawl_evidence("example.com", {"http_status": 404, "pages": [], "signals": {}})suggestion = suggest("example.com", [], items, model=model)assert suggestion.action == "REJECT"@@ -159,8 +169,7 @@ def test_a_deterministic_check_beats_the_model():def test_a_check_that_implies_other_carries_the_detail_the_submitter_is_shown():"""OTHER is the one reason that explains nothing on its own, so a suggestion carrying ithas to carry the sentence too - the API refuses a decision that does not."""- items = rules.crawl_evidence(- "example.com", {"http_status": 404, "pages": [], "signals": {}})+ items = rules.crawl_evidence("example.com", {"http_status": 404, "pages": [], "signals": {}})suggestion = suggest("example.com", [], items, model=mock.Mock(version="test"))assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")@@ -170,22 +179,19 @@ def test_a_check_that_implies_other_carries_the_detail_the_submitter_is_shown():def test_a_check_written_for_the_moderator_tells_the_submitter_something_readable():"""A label is read by whoever is deciding the case and a detail by the person whose sitewas rejected, so a label naming the exception we caught is not the sentence to send."""- items = rules.crawl_evidence(- "example.com", {"error": "AbortError", "pages": [], "signals": {}})+ items = rules.crawl_evidence("example.com", {"error": "AbortError", "pages": [], "signals": {}})suggestion = suggest("example.com", [], items, model=mock.Mock(version="test"))assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")assert suggestion.reason_detail == "We could not fetch this site when we tried to crawl it."# The moderator still gets the exception, in the evidence list where it belongs.- assert "Could not be fetched (AbortError)" in [- item["label"] for item in suggestion.evidence]+ assert "Could not be fetched (AbortError)" in [item["label"] for item in suggestion.evidence]def test_the_do_not_crawl_list_explains_itself_from_the_submitters_side():"""Its label is written from ours - "we don't crawl ourselves" - and is about our policyrather than about the site the submitter sent."""- items = rules.crawl_evidence(- "google.com", {"http_status": 200, "pages": [], "signals": {}})+ items = rules.crawl_evidence("google.com", {"http_status": 200, "pages": [], "signals": {}})suggestion = suggest("google.com", [], items, model=mock.Mock(version="test"))assert (suggestion.action, suggestion.reason) == ("REJECT", "OTHER")@@ -224,6 +230,7 @@ def test_missing_model_degrades_to_unsure_not_to_a_default():def test_offensive_confidence_is_capped_because_it_has_no_real_labels():from mwmbl.moderation.model import REASON_CONFIDENCE_CAP+assert REASON_CONFIDENCE_CAP["OFFENSIVE"] < 1.0@@ -236,13 +243,16 @@ def test_review_priority_puts_confident_rejects_first_and_approvals_last():# --------------------------------------------------------------------- features+def test_featuriser_produces_the_same_width_for_unseen_input():featuriser = Featuriser()- fitted = featuriser.fit_transform([- ModerationExample("aianimegenerator.cloud", ["free ai anime generator"]),- ModerationExample("docs.python.org", ["python language reference"]),- ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),- ])+ fitted = featuriser.fit_transform(+ [+ ModerationExample("aianimegenerator.cloud", ["free ai anime generator"]),+ ModerationExample("docs.python.org", ["python language reference"]),+ ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),+ ]+ )transformed = featuriser.transform([ModerationExample("unseen.example", ["nothing alike"])])assert transformed.shape[1] == fitted.shape[1]assert len(featuriser.feature_names()) == fitted.shape[1]@@ -254,11 +264,13 @@ def test_has_text_separates_an_uncrawled_domain_from_one_whose_words_are_unknownrows do; without this indicator the model cannot tell the two cases apart and thedifference is absorbed into an intercept fitted on the training mix."""featuriser = Featuriser(use_has_text=True)- featuriser.fit_transform([- ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),- ModerationExample("docs.python.org", ["python language reference tool"]),- ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),- ])+ featuriser.fit_transform(+ [+ ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),+ ModerationExample("docs.python.org", ["python language reference tool"]),+ ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),+ ]+ )assert featuriser.text is not Nonecolumn = list(featuriser.feature_names()).index("has_text")@@ -271,9 +283,11 @@ def test_has_text_separates_an_uncrawled_domain_from_one_whose_words_are_unknowndef test_the_ablation_fits_the_same_features_minus_the_text_block():"""--no-text has to run through the real featuriser, not a copy of it, or the ablationmeasures the copy."""- examples = [ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),- ModerationExample("docs.python.org", ["python language reference tool"]),- ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"])]+ examples = [+ ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),+ ModerationExample("docs.python.org", ["python language reference tool"]),+ ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),+ ]with_text = Featuriser().fit_transform(examples).shape[1]without = Featuriser(use_text=False)assert without.fit_transform(examples).shape[1] < with_text@@ -291,7 +305,8 @@ def test_a_judges_most_confident_approval_is_not_read_as_a_certain_rejection(tmppath = tmp_path / "judge.judgements.jsonl"path.write_text('{"domain": "obviously-fine.org", "reject_probability": 1, "reason": ""}\n'- '{"domain": "aislopgenerator.xyz", "reject_probability": 95, "reason": "SPAM"}\n')+ '{"domain": "aislopgenerator.xyz", "reject_probability": 95, "reason": "SPAM"}\n'+ )judgements = read_judgements(path)assert judgements["obviously-fine.org"] == 0.01@@ -305,7 +320,8 @@ def test_judgements_already_on_a_zero_to_one_scale_are_left_alone(tmp_path):path = tmp_path / "judge.judgements.jsonl"path.write_text('{"domain": "obviously-fine.org", "reject_probability": 0.01, "reason": ""}\n'- '{"domain": "aislopgenerator.xyz", "reject_probability": 0.95, "reason": "SPAM"}\n')+ '{"domain": "aislopgenerator.xyz", "reject_probability": 0.95, "reason": "SPAM"}\n'+ )assert read_judgements(path) == {"obviously-fine.org": 0.01, "aislopgenerator.xyz": 0.95}@@ -345,9 +361,11 @@ def test_has_text_can_be_held_out_independently_of_the_text_block():"""The two halves of the page-text features can fail independently - the vocabulary canearn its place while the indicator is a proxy for recency - so the ablation holds out eachon its own rather than both together."""- examples = [ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),- ModerationExample("docs.python.org", ["python language reference tool"]),- ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"])]+ examples = [+ ModerationExample("aianimegenerator.cloud", ["free ai anime generator tool"]),+ ModerationExample("docs.python.org", ["python language reference tool"]),+ ModerationExample("seobacklinkhub.org", ["best seo backlinks tool"]),+ ]both = Featuriser(use_has_text=True)shipped = Featuriser()@@ -357,7 +375,7 @@ def test_has_text_can_be_held_out_independently_of_the_text_block():assert "has_text" in list(both.feature_names())# Off by default: measured against production data it made every metric worse.assert "has_text" not in list(shipped.feature_names())- assert shipped.text is not None # the vocabulary block is still there+ assert shipped.text is not None # the vocabulary block is still thereassert shipped.transform(examples).shape[1] == both.transform(examples).shape[1] - 1@@ -373,10 +391,11 @@ def test_malformed_historic_names_are_excluded_from_training():def test_derived_rows_never_reach_the_test_split():from mwmbl.moderation.train import split_by_time- real = [TrainingRow(f"real{index}.com", index % 2 == 0, "", "real", [], f"2025-01-{index:02d}")- for index in range(1, 21)]- derived = [TrainingRow(f"derived{index}.com", True, "OFFENSIVE", "derived", [])- for index in range(5)]+ real = [+ TrainingRow(f"real{index}.com", index % 2 == 0, "", "real", [], f"2025-01-{index:02d}")+ for index in range(1, 21)+ ]+ derived = [TrainingRow(f"derived{index}.com", True, "OFFENSIVE", "derived", []) for index in range(5)]seed = [TrainingRow("seed.com", True, "OTHER", "seed", [])]train_rows, test_rows = split_by_time(real + derived + seed)@@ -388,23 +407,26 @@ def test_agreement_is_measured_against_the_action_that_was_shown():"""suggested_status records what the tool displayed - APPROVE, REJECT or UNSURE - socomparing it against a submission status ("REJECTED") matched nothing and inverted thecount: every rejection the moderator agreed with was scored as a disagreement."""+def decided(rejected, shown):return TrainingRow("example.com", rejected, "", "real", [], suggested_status=shown)- influence = _suggestion_influence([- decided(rejected=True, shown="REJECT"), # agreed- decided(rejected=False, shown="REJECT"), # disagreed- decided(rejected=False, shown="APPROVE"), # agreed- decided(rejected=True, shown="UNSURE"), # shown, but took no side- decided(rejected=True, shown=""), # nothing was shown- ])+ influence = _suggestion_influence(+ [+ decided(rejected=True, shown="REJECT"), # agreed+ decided(rejected=False, shown="REJECT"), # disagreed+ decided(rejected=False, shown="APPROVE"), # agreed+ decided(rejected=True, shown="UNSURE"), # shown, but took no side+ decided(rejected=True, shown=""), # nothing was shown+ ]+ )- assert influence == {"real_rows": 5, "shown_a_suggestion": 4,- "suggested_a_side": 3, "agreed_with_it": 2}+ assert influence == {"real_rows": 5, "shown_a_suggestion": 4, "suggested_a_side": 3, "agreed_with_it": 2}# --------------------------------------------------------------------- API+@pytest.mark.django_dbdef test_queue_requires_the_moderator_permission(client, submitter):response = client.get(QUEUE_URL, headers={"Authorization": token(submitter)})@@ -437,16 +459,12 @@ def test_uncrawled_submission_reports_that_rather_than_guessing(client, moderato@pytest.mark.django_dbdef test_queue_orders_rows_that_need_a_human_first(client, moderator, established):- for name, action in [("approve-me.com", "APPROVE"),- ("reject-me.com", "REJECT"),- ("unsure.com", "UNSURE")]:+ for name, action in [("approve-me.com", "APPROVE"), ("reject-me.com", "REJECT"), ("unsure.com", "UNSURE")]:DomainSubmission.objects.create(name=name, submitted_by=established)ready_evidence(name, suggested_action=action, confidence=0.8)- response = client.get(f"{QUEUE_URL}?order_by=needs_review",- headers={"Authorization": token(moderator)})- assert [item["name"] for item in response.json()["items"]] == [- "reject-me.com", "unsure.com", "approve-me.com"]+ response = client.get(f"{QUEUE_URL}?order_by=needs_review", headers={"Authorization": token(moderator)})+ assert [item["name"] for item in response.json()["items"]] == ["reject-me.com", "unsure.com", "approve-me.com"]@pytest.mark.django_db@@ -455,8 +473,7 @@ def test_queue_filters_on_the_suggestion(client, moderator, submitter):DomainSubmission.objects.create(name=name, submitted_by=submitter)ready_evidence(name, suggested_action=action)- response = client.get(f"{QUEUE_URL}?suggested_action=REJECT",- headers={"Authorization": token(moderator)})+ response = client.get(f"{QUEUE_URL}?suggested_action=REJECT", headers={"Authorization": token(moderator)})assert [item["name"] for item in response.json()["items"]] == ["b.com"]@@ -466,10 +483,18 @@ def test_decision_records_the_suggestion_that_was_shown(client, moderator, submiresponse = client.post(f"/api/v1/platform/domain-submissions/ids/{submission.id}",- data={"status": "REJECTED", "rejection_reason": "SPAM", "rejection_detail": "",- "suggested_status": "APPROVE", "suggested_reason": "",- "suggestion_confidence": 0.62, "suggestion_model_version": "test-model"},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={+ "status": "REJECTED",+ "rejection_reason": "SPAM",+ "rejection_detail": "",+ "suggested_status": "APPROVE",+ "suggested_reason": "",+ "suggestion_confidence": 0.62,+ "suggestion_model_version": "test-model",+ },+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 200submission.refresh_from_db()@@ -488,11 +513,15 @@ def test_bulk_decisions_apply_each_choice_separately(client, moderator, submitteresponse = client.post("/api/v1/platform/domain-submissions/decisions",- data={"decisions": [- {"domain": "a.com", "status": "APPROVED"},- {"domain": "b.com", "status": "REJECTED", "rejection_reason": "SPAM"},- ]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={+ "decisions": [+ {"domain": "a.com", "status": "APPROVED"},+ {"domain": "b.com", "status": "REJECTED", "rejection_reason": "SPAM"},+ ]+ },+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.json()["updated"] == 2first.refresh_from_db()@@ -508,7 +537,8 @@ def test_refetch_clears_the_evidence_so_the_crawl_actually_reruns(client, moderawith mock.patch("mwmbl.platform.api.enrich_domain_submission") as enrich:response = client.post(f"/api/v1/platform/domain-submissions/ids/{submission.id}/refetch",- headers={"Authorization": token(moderator)})+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 200assert not DomainEvidence.objects.filter(domain="was-down.com").exists()@@ -523,14 +553,14 @@ HISTORY_URL = "/api/v1/platform/domain-submissions/moderated"def upvote(url, username):SearchResultVote.objects.create(- user=MwmblUser.objects.create_user(username=username), url=url, query="q",- vote_type="upvote")+ user=MwmblUser.objects.create_user(username=username), url=url, query="q", vote_type="upvote"+ )def downvote(url, username):SearchResultVote.objects.create(- user=MwmblUser.objects.create_user(username=username), url=url, query="q",- vote_type="downvote")+ user=MwmblUser.objects.create_user(username=username), url=url, query="q", vote_type="downvote"+ )@pytest.mark.django_db@@ -556,10 +586,8 @@ def test_the_row_is_the_first_submission_of_the_domain(client, moderator):has to be the row those two fields come from."""first = MwmblUser.objects.create_user(username="anon_4417")later = MwmblUser.objects.create_user(username="someone_else")- DomainSubmission.objects.create(name="a.com", submitted_by=first,- submitted_on=timezone.now() - timedelta(days=6))- DomainSubmission.objects.create(name="a.com", submitted_by=later,- submitted_on=timezone.now() - timedelta(days=1))+ DomainSubmission.objects.create(name="a.com", submitted_by=first, submitted_on=timezone.now() - timedelta(days=6))+ DomainSubmission.objects.create(name="a.com", submitted_by=later, submitted_on=timezone.now() - timedelta(days=1))ready_evidence("a.com")item = client.get(QUEUE_URL, headers={"Authorization": token(moderator)}).json()["items"][0]@@ -582,9 +610,10 @@ def test_votes_are_rolled_up_to_the_domain(client, moderator, established):DomainSubmission.objects.create(name="silent.example", submitted_by=established)ready_evidence("silent.example")- items = {item["name"]: item- for item in client.get(QUEUE_URL,- headers={"Authorization": token(moderator)}).json()["items"]}+ items = {+ item["name"]: item+ for item in client.get(QUEUE_URL, headers={"Authorization": token(moderator)}).json()["items"]+ }assert (items["solarpunk.zone"]["upvotes"], items["solarpunk.zone"]["downvotes"]) == (2, 1)# Zero, not null: a domain nobody has voted on has to sort with the rest.@@ -603,8 +632,12 @@ def test_a_www_submission_still_finds_its_votes(client, moderator, established):@pytest.mark.django_dbdef test_the_default_order_is_most_submitted_then_most_upvoted(client, moderator, established):- for name, submissions, upvotes in [("popular.com", 3, 1), ("tied-low.com", 2, 1),- ("tied-high.com", 2, 40), ("lonely.com", 1, 99)]:+ for name, submissions, upvotes in [+ ("popular.com", 3, 1),+ ("tied-low.com", 2, 1),+ ("tied-high.com", 2, 40),+ ("lonely.com", 1, 99),+ ]:for _ in range(submissions):DomainSubmission.objects.create(name=name, submitted_by=established)for index in range(upvotes):@@ -612,8 +645,7 @@ def test_the_default_order_is_most_submitted_then_most_upvoted(client, moderatorready_evidence(name)body = client.get(QUEUE_URL, headers={"Authorization": token(moderator)}).json()- assert [item["name"] for item in body["items"]] == [- "popular.com", "tied-high.com", "tied-low.com", "lonely.com"]+ assert [item["name"] for item in body["items"]] == ["popular.com", "tied-high.com", "tied-low.com", "lonely.com"]@pytest.mark.django_db@@ -624,12 +656,12 @@ def test_the_card_carries_its_sample_pages_and_padlock(client, moderator, establready_evidence("crawled.com", signals={"has_links": True, "https": False})DomainSubmission.objects.create(name="uncrawled.com", submitted_by=established)DomainSubmission.objects.create(name="unreachable.com", submitted_by=established)- ready_evidence("unreachable.com", signals={"has_links": False, "https": None},- error="AbortError", http_status=None)+ ready_evidence("unreachable.com", signals={"has_links": False, "https": None}, error="AbortError", http_status=None)- items = {item["name"]: item- for item in client.get(QUEUE_URL,- headers={"Authorization": token(moderator)}).json()["items"]}+ items = {+ item["name"]: item+ for item in client.get(QUEUE_URL, headers={"Authorization": token(moderator)}).json()["items"]+ }assert items["crawled.com"]["pages"][0]["title"] == "A title"assert items["crawled.com"]["https"] is False@@ -647,30 +679,30 @@ def test_one_decision_settles_every_submission_of_the_domain(client, moderator,response = client.post(DECISIONS_URL,- data={"decisions": [{"domain": "cheap-rolex.biz", "status": "REJECTED",- "rejection_reason": "SPAM"}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={"decisions": [{"domain": "cheap-rolex.biz", "status": "REJECTED", "rejection_reason": "SPAM"}]},+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.json() == {"status": "ok", "domains": 1, "updated": 9, "not_found": []}- assert not DomainSubmission.objects.filter(name="cheap-rolex.biz",- status="PENDING").exists()+ assert not DomainSubmission.objects.filter(name="cheap-rolex.biz", status="PENDING").exists()@pytest.mark.django_dbdef test_a_decision_can_be_changed_after_the_fact(client, moderator, submitter):"""Re-deciding is the same request as deciding: a domain already rejected can be approvedwithout a second endpoint, and does not end up half one and half the other."""- DomainSubmission.objects.create(name="a.com", submitted_by=submitter, status="REJECTED",- rejection_reason="SPAM")+ DomainSubmission.objects.create(name="a.com", submitted_by=submitter, status="REJECTED", rejection_reason="SPAM")DomainSubmission.objects.create(name="a.com", submitted_by=submitter)- client.post(DECISIONS_URL,- data={"decisions": [{"domain": "a.com", "status": "APPROVED"}]},- content_type="application/json",- headers={"Authorization": token(moderator)})+ client.post(+ DECISIONS_URL,+ data={"decisions": [{"domain": "a.com", "status": "APPROVED"}]},+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )- statuses = set(DomainSubmission.objects.filter(name="a.com")- .values_list("status", "rejection_reason"))+ statuses = set(DomainSubmission.objects.filter(name="a.com").values_list("status", "rejection_reason"))assert statuses == {("APPROVED", "")}@@ -679,7 +711,9 @@ def test_a_decision_on_an_unknown_domain_is_reported_not_invented(client, moderaresponse = client.post(DECISIONS_URL,data={"decisions": [{"domain": "never-submitted.com", "status": "APPROVED"}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.json()["not_found"] == ["never-submitted.com"]@@ -690,24 +724,32 @@ UNDO_URL = "/api/v1/platform/domain-submissions/domains/{}/undo"@pytest.mark.django_db-def test_undo_returns_a_domain_to_the_queue_without_rewriting_the_audit(client, moderator,- submitter):+def test_undo_returns_a_domain_to_the_queue_without_rewriting_the_audit(client, moderator, submitter):"""The suggested_* columns record what was on screen when the decision was made. Postinga PENDING status back through the decision endpoint - the nearest thing to an undo beforethis - overwrote them from the request, destroying the one thing they exist for."""submission = DomainSubmission.objects.create(name="a.com", submitted_by=submitter)ready_evidence("a.com")- client.post(DECISIONS_URL,- data={"decisions": [{"domain": "a.com", "status": "REJECTED",- "rejection_reason": "SPAM",- "suggested_status": "REJECT",- "suggested_reason": "SPAM",- "suggestion_confidence": 0.91,- "suggestion_model_version": "2026-08-01"}]},- content_type="application/json", headers={"Authorization": token(moderator)})-- response = client.post(UNDO_URL.format("a.com"),- headers={"Authorization": token(moderator)})+ client.post(+ DECISIONS_URL,+ data={+ "decisions": [+ {+ "domain": "a.com",+ "status": "REJECTED",+ "rejection_reason": "SPAM",+ "suggested_status": "REJECT",+ "suggested_reason": "SPAM",+ "suggestion_confidence": 0.91,+ "suggestion_model_version": "2026-08-01",+ }+ ]+ },+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )++ response = client.post(UNDO_URL.format("a.com"), headers={"Authorization": token(moderator)})assert response.status_code == 200submission.refresh_from_db()@@ -740,8 +782,7 @@ def test_undoing_an_approval_rebuilds_the_blacklist_snapshot(client, moderator,def test_undoing_a_rejection_does_not_rebuild_the_snapshot(client, moderator, submitter):"""Only an approval ever changed the snapshot, so only undoing one has to change it back -a rebuild downloads and parses tens of megabytes."""- DomainSubmission.objects.create(name="a.com", submitted_by=submitter, status="REJECTED",- rejection_reason="SPAM")+ DomainSubmission.objects.create(name="a.com", submitted_by=submitter, status="REJECTED", rejection_reason="SPAM")Task.objects.all().delete()with mock.patch("mwmbl.background.refresh_blacklist_snapshot") as refresh:@@ -752,22 +793,26 @@ def test_undoing_a_rejection_does_not_rebuild_the_snapshot(client, moderator, su@pytest.mark.django_dbdef test_undoing_a_domain_nobody_submitted_is_a_404(client, moderator):- response = client.post(UNDO_URL.format("never-submitted.com"),- headers={"Authorization": token(moderator)})+ response = client.post(UNDO_URL.format("never-submitted.com"), headers={"Authorization": token(moderator)})assert response.status_code == 404@pytest.mark.django_dbdef test_history_lists_past_decisions_by_status_newest_first(client, moderator, submitter):- for name, status, when in [("old-reject.com", "REJECTED", 5),- ("new-reject.com", "REJECTED", 1),- ("approved.com", "APPROVED", 2)]:+ for name, status, when in [+ ("old-reject.com", "REJECTED", 5),+ ("new-reject.com", "REJECTED", 1),+ ("approved.com", "APPROVED", 2),+ ]:DomainSubmission.objects.create(- name=name, submitted_by=submitter, status=status,- status_changed_by=moderator, status_changed_on=timezone.now() - timedelta(days=when))+ name=name,+ submitted_by=submitter,+ status=status,+ status_changed_by=moderator,+ status_changed_on=timezone.now() - timedelta(days=when),+ )- body = client.get(f"{HISTORY_URL}?status=REJECTED",- headers={"Authorization": token(moderator)}).json()+ body = client.get(f"{HISTORY_URL}?status=REJECTED", headers={"Authorization": token(moderator)}).json()assert [item["name"] for item in body["items"]] == ["new-reject.com", "old-reject.com"]assert body["count"] == 2@@ -778,9 +823,13 @@ def test_history_lists_past_decisions_by_status_newest_first(client, moderator,def test_history_shows_one_row_per_domain(client, moderator, submitter):for index in range(4):DomainSubmission.objects.create(- name="a.com", submitted_by=submitter, status="REJECTED", rejection_reason="SPAM",+ name="a.com",+ submitted_by=submitter,+ status="REJECTED",+ rejection_reason="SPAM",status_changed_by=moderator,- status_changed_on=timezone.now() - timedelta(days=index))+ status_changed_on=timezone.now() - timedelta(days=index),+ )body = client.get(HISTORY_URL, headers={"Authorization": token(moderator)}).json()@@ -791,13 +840,19 @@ def test_history_shows_one_row_per_domain(client, moderator, submitter):@pytest.mark.django_dbdef test_history_carries_what_the_moderator_was_shown(client, moderator, submitter):DomainSubmission.objects.create(- name="a.com", submitted_by=submitter, status="REJECTED", rejection_reason="SPAM",- status_changed_by=moderator, status_changed_on=timezone.now(),- suggested_status="REJECT", suggested_reason="SPAM", suggestion_confidence=0.77,- suggestion_model_version="2026-07-01")-- item = client.get(HISTORY_URL,- headers={"Authorization": token(moderator)}).json()["items"][0]+ name="a.com",+ submitted_by=submitter,+ status="REJECTED",+ rejection_reason="SPAM",+ status_changed_by=moderator,+ status_changed_on=timezone.now(),+ suggested_status="REJECT",+ suggested_reason="SPAM",+ suggestion_confidence=0.77,+ suggestion_model_version="2026-07-01",+ )++ item = client.get(HISTORY_URL, headers={"Authorization": token(moderator)}).json()["items"][0]assert item["suggested_status"] == "REJECT"assert item["suggestion_confidence"] == 0.77@@ -812,14 +867,19 @@ def test_history_requires_the_moderator_permission(client, submitter):# --------------------------------------------------------------------- rescoring+@pytest.mark.django_dbdef test_rescore_updates_pending_rows_only(submitter):from mwmbl.background import rescore_pending_submissionsDomainSubmission.objects.create(name="pending.com", submitted_by=submitter, status="PENDING")decided = DomainSubmission.objects.create(- name="decided.com", submitted_by=submitter, status="APPROVED",- suggested_status="REJECT", suggestion_confidence=0.9)+ name="decided.com",+ submitted_by=submitter,+ status="APPROVED",+ suggested_status="REJECT",+ suggestion_confidence=0.9,+ )ready_evidence("pending.com", model_version="old")ready_evidence("decided.com", model_version="old")@@ -849,15 +909,13 @@ def test_enrichment_recrawls_stale_evidence(submitter):ready_evidence("stale.com", fetched_at=timezone.now() - timedelta(days=365))with mock.patch("mwmbl.background.crawl_domain") as crawl:- crawl.return_value = {"http_status": 200, "final_domain": "", "error": "",- "pages": [], "signals": {}}+ crawl.return_value = {"http_status": 200, "final_domain": "", "error": "", "pages": [], "signals": {}}enrich_domain_submission.now("stale.com")crawl.assert_called_once()@pytest.mark.django_db-def test_queue_cost_does_not_grow_with_the_backlog(client, moderator, established,- django_assert_max_num_queries):+def test_queue_cost_does_not_grow_with_the_backlog(client, moderator, established, django_assert_max_num_queries):"""Ordering, filtering and slicing must happen in SQL.The backlog is ~4,000 pending submissions. An implementation that builds the list in@@ -874,13 +932,15 @@ def test_queue_cost_does_not_grow_with_the_backlog(client, moderator, establishefor voter in range(index % 4):SearchResultVote.objects.create(user=MwmblUser.objects.create_user(username=f"voter{index}-{voter}"),- url=f"https://{name}/page{voter}", query="q", vote_type="upvote")+ url=f"https://{name}/page{voter}",+ query="q",+ vote_type="upvote",+ )ready_evidence(name, suggested_action="REJECT", confidence=index / 30)# Auth, permissions, count, page, evidence, submitter counts, prior-decision counts.with django_assert_max_num_queries(12):- response = client.get(f"{QUEUE_URL}?limit=5&order_by=needs_review",- headers={"Authorization": token(moderator)})+ response = client.get(f"{QUEUE_URL}?limit=5&order_by=needs_review", headers={"Authorization": token(moderator)})body = response.json()# Distinct pending domains, not the 90 pending submissions behind them.@@ -900,12 +960,12 @@ def test_queue_paginates_over_the_whole_ordering(client, moderator, established)ready_evidence(name, suggested_action="REJECT", confidence=index / 10)def page(offset):- response = client.get(f"{QUEUE_URL}?limit=4&offset={offset}&order_by=needs_review",- headers={"Authorization": token(moderator)})+ response = client.get(+ f"{QUEUE_URL}?limit=4&offset={offset}&order_by=needs_review", headers={"Authorization": token(moderator)}+ )return [item["name"] for item in response.json()["items"]]- assert page(0) + page(4) + page(8) == [f"site{index}.example"- for index in range(9, -1, -1)]+ assert page(0) + page(4) + page(8) == [f"site{index}.example" for index in range(9, -1, -1)]def test_robots_denied_is_evidence_not_a_verdict():@@ -913,8 +973,8 @@ def test_robots_denied_is_evidence_not_a_verdict():distinct rejection details mentions robots.txt. Suggesting a confident rejection herewould be confidently wrong on a domain moderators actually wanted."""items = rules.crawl_evidence(- "lobste.rs", {"http_status": None, "error": "RobotsDenied", "pages": [{}],- "signals": {}})+ "lobste.rs", {"http_status": None, "error": "RobotsDenied", "pages": [{}], "signals": {}}+ )robots = next(item for item in items if item.kind == "robots")assert robots.implies_action is Noneassert rules.decisive(items) is None@@ -926,8 +986,12 @@ def test_blocklist_membership_is_evidence_not_a_verdict():must not argue against the mechanism it serves."""items = rules.crawl_evidence("pudding.cool",- {"http_status": 200, "pages": [{"title": "t", "extract": "e", "num_links": 4}],- "signals": {"has_links": True, "blacklisted": True}})+ {+ "http_status": 200,+ "pages": [{"title": "t", "extract": "e", "num_links": 4}],+ "signals": {"has_links": True, "blacklisted": True},+ },+ )blocklist = next(item for item in items if item.kind == "blocklist")assert blocklist.direction == rules.REJECTassert blocklist.implies_action is None@@ -936,9 +1000,13 @@ def test_blocklist_membership_is_evidence_not_a_verdict():def test_a_failed_fetch_reports_one_problem_not_four():items = rules.crawl_evidence("gone.example",- {"http_status": None, "error": "AbortError", "pages": [{"title": "", "extract": "",- "num_links": 0}],- "signals": {}})+ {+ "http_status": None,+ "error": "AbortError",+ "pages": [{"title": "", "extract": "", "num_links": 0}],+ "signals": {},+ },+ )kinds = {item.kind for item in items}assert kinds == {"unreachable"}@@ -946,7 +1014,7 @@ def test_a_failed_fetch_reports_one_problem_not_four():def test_an_approval_is_not_labelled_with_the_reason_heads_argmax():model = mock.Mock()model.version = "test"- model.predict.return_value = [(0.02, "OFFENSIVE", 0.5)] # clearly an approval+ model.predict.return_value = [(0.02, "OFFENSIVE", 0.5)] # clearly an approvalsuggestion = suggest("example.com", ["wholesome text"], [], model=model)@@ -963,29 +1031,35 @@ def test_redirect_check_errs_towards_reporting_nothing():assert rules.registrable("example.co.uk") == rules.registrable("somewhere-else.co.uk")items = rules.crawl_evidence(- "example.com",- {"http_status": 200, "final_domain": "elsewhere.net", "pages": [], "signals": {}})+ "example.com", {"http_status": 200, "final_domain": "elsewhere.net", "pages": [], "signals": {}}+ )assert rules.decisive(items).kind == "redirect"# --------------------------------------------------------------------- retrain gate+def metrics(pr_auc, low, high, rows=150, positives=81):"""Cold-start metrics in the shape a stored artifact carries.rows and positives are what make the PR-AUC readable: the gate corrects for prevalence, soa slice cannot be described by its score alone."""- return {"cold_start": {"pr_auc": pr_auc, "pr_auc_ci": [low, high],- "rows": rows, "positives": positives}}+ return {"cold_start": {"pr_auc": pr_auc, "pr_auc_ci": [low, high], "rows": rows, "positives": positives}}def paired(difference, low, high, contamination_known=True):- return {"cold_start": metrics(0.7, 0.6, 0.8)["cold_start"],- "versus_incumbent": {- "incumbent_version": "domain-mod-2026-08-01", "rows": 150,- "contamination_known": contamination_known, "difference": difference,- "difference_ci": [low, high], "candidate_wins_fraction": 0.5}}+ return {+ "cold_start": metrics(0.7, 0.6, 0.8)["cold_start"],+ "versus_incumbent": {+ "incumbent_version": "domain-mod-2026-08-01",+ "rows": 150,+ "contamination_known": contamination_known,+ "difference": difference,+ "difference_ci": [low, high],+ "candidate_wins_fraction": 0.5,+ },+ }def test_gate_publishes_when_there_is_no_incumbent():@@ -994,8 +1068,7 @@ def test_gate_publishes_when_there_is_no_incumbent():def test_gate_blocks_a_model_below_the_incumbents_lower_bound():- allowed, explanation = passes_gate(metrics(0.60, 0.51, 0.69),- metrics(0.78, 0.69, 0.86))+ allowed, explanation = passes_gate(metrics(0.60, 0.51, 0.69), metrics(0.78, 0.69, 0.86))assert not allowedassert "unpaired" in explanation@@ -1018,7 +1091,8 @@ def test_gate_is_not_fooled_by_a_change_in_prevalence():ships."""allowed, explanation = passes_gate(metrics(0.6716, 0.5927, 0.7472, rows=635, positives=189),- metrics(0.7788, 0.6885, 0.8588, rows=150, positives=81))+ metrics(0.7788, 0.6885, 0.8588, rows=150, positives=81),+ )assert allowed, explanation@@ -1026,8 +1100,7 @@ def test_the_paired_comparison_is_preferred_to_the_stored_metrics():"""Two numbers from two different test sets can only be compared weakly. When the incumbentcould be loaded and scored on the same rows, that is the answer - even when the storedmetrics, describing a different population, would have said otherwise."""- allowed, explanation = passes_gate(paired(0.02, -0.03, 0.07),- metrics(0.95, 0.93, 0.97))+ allowed, explanation = passes_gate(paired(0.02, -0.03, 0.07), metrics(0.95, 0.93, 0.97))assert allowedassert "paired against domain-mod-2026-08-01" in explanation@@ -1049,8 +1122,7 @@ def test_an_incumbent_with_no_training_record_is_flagged_not_trusted_silently():"""Artifacts pickled before train_domains was recorded may be scored on rows they werefitted on, which flatters them. The candidate still has to clear the bar - the point isthat the explanation says the bar was tilted."""- allowed, explanation = passes_gate(paired(0.01, -0.02, 0.05, contamination_known=False),- metrics(0.5, 0.4, 0.6))+ allowed, explanation = passes_gate(paired(0.01, -0.02, 0.05, contamination_known=False), metrics(0.5, 0.4, 0.6))assert allowedassert "no record of its training domains" in explanation@@ -1064,6 +1136,7 @@ def test_gate_refuses_when_there_is_no_cold_start_slice():# --------------------------------------------------------------------- prior shift+@pytest.mark.django_dbdef test_no_confident_approval_for_a_submitter_with_no_track_record(submitter):"""The model is trained on a population that rejects 11% and asked about one that rejects@@ -1080,8 +1153,7 @@ def test_no_confident_approval_for_a_submitter_with_no_track_record(submitter):@pytest.mark.django_dbdef test_approval_stands_for_a_submitter_with_a_record(submitter):for index in range(3):- DomainSubmission.objects.create(name=f"past{index}.example", submitted_by=submitter,- status="APPROVED")+ DomainSubmission.objects.create(name=f"past{index}.example", submitted_by=submitter, status="APPROVED")submission = DomainSubmission.objects.create(name="known.example", submitted_by=submitter)evidence = ready_evidence("known.example", suggested_action="APPROVE", confidence=0.9)@@ -1095,8 +1167,7 @@ def test_approval_stands_for_a_submitter_with_a_record(submitter):def test_rejections_are_not_withheld_from_first_time_submitters(submitter):"""Rejection precision is 0.88 on exactly this slice, so only the approve side is held."""submission = DomainSubmission.objects.create(name="spammy.example", submitted_by=submitter)- evidence = ready_evidence("spammy.example", suggested_action="REJECT",- suggested_reason="SPAM", confidence=0.9)+ evidence = ready_evidence("spammy.example", suggested_action="REJECT", suggested_reason="SPAM", confidence=0.9)suggestion = suggestion_for(submission, evidence)@@ -1106,6 +1177,7 @@ def test_rejections_are_not_withheld_from_first_time_submitters(submitter):# --------------------------------------------------------------------- crawl evidence+class _FakeResponse:"""Enough of a requests Response for retrieve.fetch."""@@ -1123,9 +1195,11 @@ class _FakeResponse:pass-A_PAGE = (b"<html><head><title>Buy this domain</title></head><body>"- b"<p>This premium domain name is for sale. Make an offer today and we will be in "- b"touch with you shortly to complete the transfer.</p></body></html>")+A_PAGE = (+ b"<html><head><title>Buy this domain</title></head><body>"+ b"<p>This premium domain name is for sale. Make an offer today and we will be in "+ b"touch with you shortly to complete the transfer.</p></body></html>"+)def crawl_with(responses: dict, domain: str) -> dict:@@ -1134,6 +1208,7 @@ def crawl_with(responses: dict, domain: str) -> dict:Driven from requests.get rather than by stubbing crawl_url, because everything thiscovers lives in the plumbing between the fetch and the evidence."""+def get(url, **kwargs):response = responses[url]if isinstance(response, Exception):@@ -1141,11 +1216,12 @@ def crawl_with(responses: dict, domain: str) -> dict:return responseblacklist = mock.Mock(is_domain_blacklisted=mock.Mock(return_value=False))- with mock.patch("mwmbl.crawler.retrieve.requests.get", get), \- mock.patch("mwmbl.crawler.retrieve.validate_url"), \- mock.patch("mwmbl.crawler.retrieve.robots_allowed", return_value=True), \- mock.patch("mwmbl.moderation.evidence.get_snapshot_blacklist",- return_value=blacklist):+ with (+ mock.patch("mwmbl.crawler.retrieve.requests.get", get),+ mock.patch("mwmbl.crawler.retrieve.validate_url"),+ mock.patch("mwmbl.crawler.retrieve.robots_allowed", return_value=True),+ mock.patch("mwmbl.moderation.evidence.get_snapshot_blacklist", return_value=blacklist),+ ):return crawl_domain(domain, redis=None)@@ -1154,11 +1230,13 @@ def test_a_domain_that_redirects_elsewhere_is_reported_as_a_redirect():domains, and "redirects to somewhere else" is the fetch that proves it. The check is onlyas good as the URL the crawler reports: reporting the requested one compares the domainagainst itself and the rule can never fire."""- crawl = crawl_with({- "https://squatted.example/": _FakeResponse(- 301, headers={"Location": "https://parked.test/lander"}),- "https://parked.test/lander": _FakeResponse(200, body=A_PAGE),- }, "squatted.example")+ crawl = crawl_with(+ {+ "https://squatted.example/": _FakeResponse(301, headers={"Location": "https://parked.test/lander"}),+ "https://parked.test/lander": _FakeResponse(200, body=A_PAGE),+ },+ "squatted.example",+ )assert crawl["final_domain"] == "parked.test"decisive = rules.decisive(rules.crawl_evidence("squatted.example", crawl))@@ -1169,8 +1247,7 @@ def test_a_domain_that_redirects_elsewhere_is_reported_as_a_redirect():def test_a_domain_that_serves_its_own_pages_reports_no_redirect():"""The other direction, because the check is decisive: a site that answers for itself mustnever be rejected for redirecting to itself."""- crawl = crawl_with(- {"https://honest.example/": _FakeResponse(200, body=A_PAGE)}, "honest.example")+ crawl = crawl_with({"https://honest.example/": _FakeResponse(200, body=A_PAGE)}, "honest.example")assert crawl["final_domain"] == ""kinds = {item.kind for item in rules.crawl_evidence("honest.example", crawl)}@@ -1181,10 +1258,13 @@ def test_a_site_without_a_certificate_is_a_site_not_a_dead_domain():"""Only https used to be tried, so "no TLS" and "not there" were the same result - andthat result is a decisive REJECT for being unreachable. A site served over plain http isa site; whether it has a certificate is a line for the moderator, not a verdict."""- crawl = crawl_with({- "https://no-cert.example/": ConnectionError("certificate verify failed"),- "http://no-cert.example/": _FakeResponse(200, body=A_PAGE),- }, "no-cert.example")+ crawl = crawl_with(+ {+ "https://no-cert.example/": ConnectionError("certificate verify failed"),+ "http://no-cert.example/": _FakeResponse(200, body=A_PAGE),+ },+ "no-cert.example",+ )assert crawl["signals"]["https"] is Falseassert crawl["error"] == ""@@ -1196,10 +1276,13 @@ def test_a_site_without_a_certificate_is_a_site_not_a_dead_domain():def test_a_domain_that_is_simply_down_is_still_reported_as_unreachable():"""The other direction: falling back to http must not turn a dead domain into a live one,and must not label it "no TLS" - the https attempt is the one that describes production."""- crawl = crawl_with({- "https://gone.example/": ConnectionError("no route to host"),- "http://gone.example/": ConnectionError("no route to host"),- }, "gone.example")+ crawl = crawl_with(+ {+ "https://gone.example/": ConnectionError("no route to host"),+ "http://gone.example/": ConnectionError("no route to host"),+ },+ "gone.example",+ )# Unknown, not absent: the rules layer already gates "no TLS" behind having reached the# site, but the queue card reads this signal straight out of the row, and False there@@ -1212,16 +1295,15 @@ def test_a_domain_that_is_simply_down_is_still_reported_as_unreachable():def test_a_normal_https_site_is_not_asked_for_over_http():- crawl = crawl_with(- {"https://secure.example/": _FakeResponse(200, body=A_PAGE)}, "secure.example")+ crawl = crawl_with({"https://secure.example/": _FakeResponse(200, body=A_PAGE)}, "secure.example")assert crawl["signals"]["https"] is True- assert "no_tls" not in {item.kind- for item in rules.crawl_evidence("secure.example", crawl)}+ assert "no_tls" not in {item.kind for item in rules.crawl_evidence("secure.example", crawl)}# --------------------------------------------------------------------- enrichment+@pytest.mark.django_dbdef test_enrichment_never_commits_a_scored_row_without_the_score():"""READY with a fresh timestamp is what makes the task skip a domain for a month. A row@@ -1230,9 +1312,10 @@ def test_enrichment_never_commits_a_scored_row_without_the_score():from mwmbl.background import enrich_domain_submissioncrawl = {"http_status": 200, "final_domain": "", "error": "", "pages": [], "signals": {}}- with mock.patch("mwmbl.background.crawl_domain", return_value=crawl), \- mock.patch("mwmbl.background.refresh_suggestion",- side_effect=RuntimeError("scoring blew up")):+ with (+ mock.patch("mwmbl.background.crawl_domain", return_value=crawl),+ mock.patch("mwmbl.background.refresh_suggestion", side_effect=RuntimeError("scoring blew up")),+ ):with pytest.raises(RuntimeError):enrich_domain_submission.now("half-written.example")@@ -1246,8 +1329,7 @@ def test_a_failed_recrawl_does_not_leave_the_old_crawl_on_screen():from mwmbl.background import enrich_domain_submissionready_evidence("gone.example", fetched_at=timezone.now() - timedelta(days=365))- with mock.patch("mwmbl.background.crawl_domain",- side_effect=ConnectionError("no route to host")):+ with mock.patch("mwmbl.background.crawl_domain", side_effect=ConnectionError("no route to host")):enrich_domain_submission.now("gone.example")evidence = DomainEvidence.objects.get(domain="gone.example")@@ -1284,9 +1366,9 @@ def test_a_stored_unexplainable_rejection_is_not_shown_as_a_suggestion(submitter"""Rows scored before the model stopped proposing a bare OTHER are still in the table, anda moderator pressing the button on one gets a 422 rather than a decision."""submission = DomainSubmission.objects.create(name="stale.example", submitted_by=submitter)- evidence = ready_evidence("stale.example", suggested_action="REJECT",- suggested_reason="OTHER", confidence=0.93,- reason_source="model")+ evidence = ready_evidence(+ "stale.example", suggested_action="REJECT", suggested_reason="OTHER", confidence=0.93, reason_source="model"+ )suggestion = suggestion_for(submission, evidence)@@ -1296,12 +1378,22 @@ def test_a_stored_unexplainable_rejection_is_not_shown_as_a_suggestion(submitter@pytest.mark.django_dbdef test_a_rule_scored_rejection_keeps_its_reason_and_carries_the_detail(submitter):a_404 = rules.EvidenceItem(- "http_status", rules.REJECT, "Homepage returns HTTP 404", implies_action="REJECT",- implies_reason="OTHER", implies_confidence=0.9).to_dict()+ "http_status",+ rules.REJECT,+ "Homepage returns HTTP 404",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.9,+ ).to_dict()submission = DomainSubmission.objects.create(name="dead.example", submitted_by=submitter)- evidence = ready_evidence("dead.example", suggested_action="REJECT",- suggested_reason="OTHER", confidence=0.9,- reason_source="rule", evidence=[a_404])+ evidence = ready_evidence(+ "dead.example",+ suggested_action="REJECT",+ suggested_reason="OTHER",+ confidence=0.9,+ reason_source="rule",+ evidence=[a_404],+ )suggestion = suggestion_for(submission, evidence)@@ -1312,8 +1404,7 @@ def test_a_rule_scored_rejection_keeps_its_reason_and_carries_the_detail(submitt@pytest.mark.django_dbdef test_a_previous_rejection_of_the_same_domain_says_so_as_the_detail(submitter, established):"""The prior-decision check implies OTHER too, and its label is the sentence."""- DomainSubmission.objects.create(name="again.example", submitted_by=established,- status="REJECTED")+ DomainSubmission.objects.create(name="again.example", submitted_by=established, status="REJECTED")submission = DomainSubmission.objects.create(name="again.example", submitted_by=submitter)evidence = ready_evidence("again.example")@@ -1330,12 +1421,16 @@ def test_a_rule_scored_other_with_no_check_left_behind_it_is_not_suggested(submistored rows claiming "rule" with nothing left to explain them, and a rejection whosedetail cannot be written is one the API would refuse. So it goes back as UNSURE, and thedetail is read off the evidence rather than assumed from the column."""- demoted = rules.EvidenceItem(- "http_status", rules.REJECT, "Homepage returns HTTP 404").to_dict()+ demoted = rules.EvidenceItem("http_status", rules.REJECT, "Homepage returns HTTP 404").to_dict()submission = DomainSubmission.objects.create(name="stale.example", submitted_by=submitter)- evidence = ready_evidence("stale.example", suggested_action="REJECT",- suggested_reason="OTHER", confidence=0.9,- reason_source="rule", evidence=[demoted])+ evidence = ready_evidence(+ "stale.example",+ suggested_action="REJECT",+ suggested_reason="OTHER",+ confidence=0.9,+ reason_source="rule",+ evidence=[demoted],+ )suggestion = suggestion_for(submission, evidence)@@ -1344,46 +1439,66 @@ def test_a_rule_scored_other_with_no_check_left_behind_it_is_not_suggested(submi# --------------------------------------------------------------------- queue parity+@pytest.mark.django_dbdef test_queue_display_matches_suggestion_for(submitter, established):"""The queue filters and orders in SQL; the rows are rendered by suggestion_for. Where thetwo disagree, a filter hides rows that are on screen and the ordering sorts on a numbernobody is shown - so every branch of the adjustment is checked against the other."""- DomainSubmission.objects.create(name="approved-before.example",- submitted_by=established, status="APPROVED")- DomainSubmission.objects.create(name="rejected-before.example",- submitted_by=established, status="REJECTED")- DomainSubmission.objects.create(name="dead-and-approved-before.example",- submitted_by=established, status="APPROVED")+ DomainSubmission.objects.create(name="approved-before.example", submitted_by=established, status="APPROVED")+ DomainSubmission.objects.create(name="rejected-before.example", submitted_by=established, status="REJECTED")+ DomainSubmission.objects.create(+ name="dead-and-approved-before.example", submitted_by=established, status="APPROVED"+ )a_404 = rules.EvidenceItem(- "http_status", rules.REJECT, "Homepage returns HTTP 404", implies_action="REJECT",- implies_reason="OTHER", implies_confidence=0.9).to_dict()+ "http_status",+ rules.REJECT,+ "Homepage returns HTTP 404",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.9,+ ).to_dict()cases = [# (domain, submitter, evidence overrides) - one per branch of the adjustment.("first-timer-approve.example", submitter, {}),("known-approve.example", established, {}),- ("reject.example", submitter, {"suggested_action": "REJECT",- "suggested_reason": "SPAM", "confidence": 0.91}),+ ("reject.example", submitter, {"suggested_action": "REJECT", "suggested_reason": "SPAM", "confidence": 0.91}),("unsure.example", submitter, {"suggested_action": "UNSURE", "confidence": 0.1}),# A reason the model cannot explain, so not shown as a suggestion at all.- ("unexplained.example", submitter, {"suggested_action": "REJECT",- "suggested_reason": "OTHER", "confidence": 0.93}),+ (+ "unexplained.example",+ submitter,+ {"suggested_action": "REJECT", "suggested_reason": "OTHER", "confidence": 0.93},+ ),# The same, from the other side: a row that says a check decided the reason, with# nothing implying OTHER left in its evidence to write the detail from.- ("stale-rule.example", submitter,- {"suggested_action": "REJECT", "suggested_reason": "OTHER", "confidence": 0.93,- "reason_source": "rule",- "evidence": [rules.EvidenceItem(- "http_status", rules.REJECT, "Homepage returns HTTP 404").to_dict()]}),- ("approved-before.example", submitter, {"suggested_action": "REJECT",- "suggested_reason": "SPAM"}),+ (+ "stale-rule.example",+ submitter,+ {+ "suggested_action": "REJECT",+ "suggested_reason": "OTHER",+ "confidence": 0.93,+ "reason_source": "rule",+ "evidence": [rules.EvidenceItem("http_status", rules.REJECT, "Homepage returns HTTP 404").to_dict()],+ },+ ),+ ("approved-before.example", submitter, {"suggested_action": "REJECT", "suggested_reason": "SPAM"}),("rejected-before.example", submitter, {}),# A check that already decided it, at least as strongly as the prior decision does.- ("dead-and-approved-before.example", submitter,- {"suggested_action": "REJECT", "suggested_reason": "OTHER", "confidence": 0.9,- "reason_source": "rule", "evidence": [a_404]}),+ (+ "dead-and-approved-before.example",+ submitter,+ {+ "suggested_action": "REJECT",+ "suggested_reason": "OTHER",+ "confidence": 0.9,+ "reason_source": "rule",+ "evidence": [a_404],+ },+ ),("still-crawling.example", submitter, None),]for domain, user, overrides in cases:@@ -1415,8 +1530,7 @@ def test_queue_filters_find_a_withheld_approval_where_it_is_shown(client, moderaDomainSubmission.objects.create(name="withheld.example", submitted_by=submitter)ready_evidence("withheld.example", suggested_action="APPROVE", confidence=0.9)- response = client.get(f"{QUEUE_URL}?suggested_action=UNSURE",- headers={"Authorization": token(moderator)})+ response = client.get(f"{QUEUE_URL}?suggested_action=UNSURE", headers={"Authorization": token(moderator)})items = response.json()["items"]assert [item["name"] for item in items] == ["withheld.example"]@@ -1424,8 +1538,7 @@ def test_queue_filters_find_a_withheld_approval_where_it_is_shown(client, modera@pytest.mark.django_db-def test_a_withheld_approval_outranks_a_confident_one(client, moderator, submitter,- established):+def test_a_withheld_approval_outranks_a_confident_one(client, moderator, submitter, established):"""It is the 44%-wrong slice: it needs a human more than an approval we trust does."""DomainSubmission.objects.create(name="withheld.example", submitted_by=submitter)ready_evidence("withheld.example", suggested_action="APPROVE", confidence=0.95)@@ -1433,12 +1546,12 @@ def test_a_withheld_approval_outranks_a_confident_one(client, moderator, submittready_evidence("trusted.example", suggested_action="APPROVE", confidence=0.95)response = client.get(QUEUE_URL, headers={"Authorization": token(moderator)})- assert [item["name"] for item in response.json()["items"]] == [- "withheld.example", "trusted.example"]+ assert [item["name"] for item in response.json()["items"]] == ["withheld.example", "trusted.example"]# --------------------------------------------------------------------- the artifact+class _StubModel:"""A stand-in for ModerationModel. joblib only needs it to pickle and carry a version."""@@ -1452,6 +1565,7 @@ class _StubModel:class _StaleFeatureSetModel:"""An artifact whose pickled featuriser no longer matches the code that loaded it."""+version = "stale"def predict(self, examples):@@ -1470,7 +1584,7 @@ def test_a_published_model_outlives_the_process_that_trained_it():worker replica kept its own copy."""publish(_StubModel("domain-mod-2026-09-01"), {"cold_start": {"pr_auc": 0.81}})- reset_model_cache() # a different worker, loading for the first time+ reset_model_cache() # a different worker, loading for the first timeassert get_model().version == "domain-mod-2026-09-01"assert load_metrics()["cold_start"]["pr_auc"] == 0.81@@ -1483,8 +1597,8 @@ def test_a_worker_picks_up_a_retrain_it_did_not_run(monkeypatch):monkeypatch.setattr(model_module, "MODEL_REFRESH_SECONDS", 0)ModerationModelArtifact.objects.create(- version="domain-mod-2026-10-01", metrics={},- model=artifact_bytes(_StubModel("domain-mod-2026-10-01")))+ version="domain-mod-2026-10-01", metrics={}, model=artifact_bytes(_StubModel("domain-mod-2026-10-01"))+ )assert get_model().version == "domain-mod-2026-10-01"@@ -1527,8 +1641,7 @@ def test_an_unloadable_artifact_falls_back_rather_than_failing(tmp_path, settingnot go down with it."""joblib.dump(_StubModel("bundled"), tmp_path / "model.joblib")settings.DOMAIN_MODERATION_MODEL_DIR = str(tmp_path)- ModerationModelArtifact.objects.create(- version="corrupt", model=b"not a pickle", metrics={})+ ModerationModelArtifact.objects.create(version="corrupt", model=b"not a pickle", metrics={})assert get_model().version == "bundled"@@ -1569,9 +1682,16 @@ def test_a_model_newer_than_the_code_degrades_instead_of_ending_the_run(tmp_pathdef test_a_decisive_check_still_wins_over_a_model_that_cannot_score():"""The deterministic checks do not need the model, so an unscorable one must not downgradea decisive answer to UNSURE."""- items = [rules.EvidenceItem(kind="http_error", direction="REJECT",- label="Homepage returns 404", implies_action="REJECT",- implies_reason="OTHER", implies_confidence=0.95)]+ items = [+ rules.EvidenceItem(+ kind="http_error",+ direction="REJECT",+ label="Homepage returns 404",+ implies_action="REJECT",+ implies_reason="OTHER",+ implies_confidence=0.95,+ )+ ]suggestion = suggest("a.com", [], items, model=_StaleFeatureSetModel())@@ -1590,8 +1710,8 @@ def test_a_sibling_deployments_artifact_does_not_evict_a_working_model(monkeypatmonkeypatch.setattr(model_module, "MODEL_REFRESH_SECONDS", 0)ModerationModelArtifact.objects.create(- version="from-the-other-app", metrics={},- model=artifact_bytes(_StaleFeatureSetModel()))+ version="from-the-other-app", metrics={}, model=artifact_bytes(_StaleFeatureSetModel())+ )assert get_model().version == "domain-mod-2026-09-01"@@ -1605,13 +1725,12 @@ def test_an_unusable_artifact_is_not_unpickled_on_every_refresh(monkeypatch):monkeypatch.setattr(model_module, "MODEL_REFRESH_SECONDS", 0)ModerationModelArtifact.objects.create(- version="from-the-other-app", metrics={},- model=artifact_bytes(_StaleFeatureSetModel()))+ version="from-the-other-app", metrics={}, model=artifact_bytes(_StaleFeatureSetModel())+ )loads = []original = model_module._load- monkeypatch.setattr(model_module, "_load",- lambda version: loads.append(version) or original(version))+ monkeypatch.setattr(model_module, "_load", lambda version: loads.append(version) or original(version))for _ in range(3):assert get_model().version == "domain-mod-2026-09-01"@@ -1620,18 +1739,24 @@ def test_an_unusable_artifact_is_not_unpickled_on_every_refresh(monkeypatch):# --------------------------------------------------------------------- request validation+@pytest.mark.django_db-def test_an_over_long_audit_field_is_a_validation_error_not_a_database_one(- client, moderator, submitter):+def test_an_over_long_audit_field_is_a_validation_error_not_a_database_one(client, moderator, submitter):"""Django does not enforce max_length on save(), so without the schema constraint this isa Postgres DataError and a 500 rather than a 400 naming the field."""submission = DomainSubmission.objects.create(name="example.com", submitted_by=submitter)response = client.post(f"/api/v1/platform/domain-submissions/ids/{submission.id}",- data={"status": "APPROVED", "rejection_reason": "", "rejection_detail": "",- "suggestion_model_version": "v" * 100},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={+ "status": "APPROVED",+ "rejection_reason": "",+ "rejection_detail": "",+ "suggestion_model_version": "v" * 100,+ },+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 422submission.refresh_from_db()@@ -1646,9 +1771,14 @@ def test_other_without_a_detail_is_refused(client, moderator, submitter):response = client.post(DECISIONS_URL,- data={"decisions": [{"domain": "example.com", "status": "REJECTED",- "rejection_reason": "OTHER", "rejection_detail": " "}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={+ "decisions": [+ {"domain": "example.com", "status": "REJECTED", "rejection_reason": "OTHER", "rejection_detail": " "}+ ]+ },+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 422assert DomainSubmission.objects.get(name="example.com").status == "PENDING"@@ -1662,26 +1792,27 @@ def test_a_rejection_reason_outside_the_four_choices_is_refused(client, moderatoresponse = client.post(DECISIONS_URL,- data={"decisions": [{"domain": "example.com", "status": "REJECTED",- "rejection_reason": "BECAUSE"}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={"decisions": [{"domain": "example.com", "status": "REJECTED", "rejection_reason": "BECAUSE"}]},+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 422@pytest.mark.django_db-def test_a_decision_carrying_the_suggestion_action_as_its_status_is_refused(- client, moderator, submitter):- """"APPROVE" is what the tool suggests; "APPROVED" is what a submission becomes. Django+def test_a_decision_carrying_the_suggestion_action_as_its_status_is_refused(client, moderator, submitter):+ """ "APPROVE" is what the tool suggests; "APPROVED" is what a submission becomes. Djangodoes not check choices on save(), so the near-miss used to be written to every submissionof the domain, which then matched neither the pending queue nor the approved set."""DomainSubmission.objects.create(name="example.com", submitted_by=submitter)response = client.post(DECISIONS_URL,- data={"decisions": [{"domain": "example.com", "status": "APPROVE",- "suggested_status": "APPROVE"}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={"decisions": [{"domain": "example.com", "status": "APPROVE", "suggested_status": "APPROVE"}]},+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 422assert DomainSubmission.objects.get(name="example.com").status == "PENDING"@@ -1693,9 +1824,10 @@ def test_an_approval_cannot_carry_a_rejection_reason(client, moderator, submitteresponse = client.post(DECISIONS_URL,- data={"decisions": [{"domain": "example.com", "status": "APPROVED",- "rejection_reason": "SPAM"}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={"decisions": [{"domain": "example.com", "status": "APPROVED", "rejection_reason": "SPAM"}]},+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 422@@ -1706,10 +1838,19 @@ def test_other_with_a_detail_is_accepted(client, moderator, submitter):response = client.post(DECISIONS_URL,- data={"decisions": [{"domain": "example.com", "status": "REJECTED",- "rejection_reason": "OTHER",- "rejection_detail": "Mirrors content we already index."}]},- content_type="application/json", headers={"Authorization": token(moderator)})+ data={+ "decisions": [+ {+ "domain": "example.com",+ "status": "REJECTED",+ "rejection_reason": "OTHER",+ "rejection_detail": "Mirrors content we already index.",+ }+ ]+ },+ content_type="application/json",+ headers={"Authorization": token(moderator)},+ )assert response.status_code == 200assert DomainSubmission.objects.get(name="example.com").status == "REJECTED"diff --git a/test/test_domain_submission_status_repair.py b/test/test_domain_submission_status_repair.pyindex 8194bd8..f9560e2 100644--- a/test/test_domain_submission_status_repair.py+++ b/test/test_domain_submission_status_repair.py@@ -7,6 +7,7 @@ to every query that looks for a decided submission - the queue, the history, thedomains, the training data. These cover both halves of the fix: the rows are repaired, andthe column now refuses the bad value at the database, whatever writes it."""+import importlibimport pytest@@ -18,8 +19,7 @@ from django.test import override_settingsfrom mwmbl.curated_domains import CURATED_DOMAINS_CACHE_KEY, get_curated_domainsfrom mwmbl.models import DomainSubmission, MwmblUser-migration_module = importlib.import_module(- "mwmbl.migrations.0037_repair_domain_submission_statuses")+migration_module = importlib.import_module("mwmbl.migrations.0037_repair_domain_submission_statuses")repair_statuses = migration_module.repair_statusesSTATUS_CONSTRAINT = "domain_submission_status_valid"@@ -39,8 +39,7 @@ def broken_column(db):in a transaction, so the constraint comes back on rollback."""with connection.cursor() as cursor:- cursor.execute(- f"ALTER TABLE {DomainSubmission._meta.db_table} DROP CONSTRAINT {STATUS_CONSTRAINT}")+ cursor.execute(f"ALTER TABLE {DomainSubmission._meta.db_table} DROP CONSTRAINT {STATUS_CONSTRAINT}")yield@@ -52,8 +51,7 @@ def clear_curated_cache():def make_submission(user, name, status, **kwargs):- return DomainSubmission.objects.create(- name=name, submitted_by=user, status=status, **kwargs)+ return DomainSubmission.objects.create(name=name, submitted_by=user, status=status, **kwargs)@pytest.mark.django_dbdiff --git a/test/test_domain_submissions.py b/test/test_domain_submissions.pyindex 9823f37..e8eadb7 100644--- a/test/test_domain_submissions.py+++ b/test/test_domain_submissions.py@@ -15,8 +15,7 @@ from ninja_jwt.tokens import RefreshTokenfrom mwmbl.models import DomainSubmission, MwmblUser-normalize_names = importlib.import_module(- "mwmbl.migrations.0032_normalize_domain_submission_names").normalize_names+normalize_names = importlib.import_module("mwmbl.migrations.0032_normalize_domain_submission_names").normalize_namesfix_names = importlib.import_module("mwmbl.migrations.0033_fix_domain_submission_names").fix_names@@ -164,7 +163,7 @@ def test_api_update_submission_status_grants_moderator_permission(client, verifi@pytest.mark.django_dbdef test_api_update_status_rejects_a_moderation_action_as_a_status(client, verified_user, access_token):- """"APPROVE" is the word a client shows its moderator; "APPROVED" is the status.+ """ "APPROVE" is the word a client shows its moderator; "APPROVED" is the status.Posting the former used to be accepted and written straight to the column - `choices` ischecked by full_clean(), which save() never calls - and the submission then matched@@ -172,8 +171,9 @@ def test_api_update_status_rejects_a_moderation_action_as_a_status(client, verifnaming the field now."""submission = DomainSubmission.objects.create(name="example.com", submitted_by=verified_user)- verified_user.user_permissions.add(Permission.objects.get(- codename="change_domain_submission_status", content_type__app_label="mwmbl"))+ verified_user.user_permissions.add(+ Permission.objects.get(codename="change_domain_submission_status", content_type__app_label="mwmbl")+ )response = client.post(f"/api/v1/platform/domain-submissions/ids/{submission.id}",@@ -193,8 +193,9 @@ def test_api_update_status_accepts_an_approval_without_rejection_fields(client,"""Both rejection columns are NOT NULL, and ninja types them Optional, so omitting themon an approval sent None into the column and came back as a 500."""submission = DomainSubmission.objects.create(name="example.com", submitted_by=verified_user)- verified_user.user_permissions.add(Permission.objects.get(- codename="change_domain_submission_status", content_type__app_label="mwmbl"))+ verified_user.user_permissions.add(+ Permission.objects.get(codename="change_domain_submission_status", content_type__app_label="mwmbl")+ )response = client.post(f"/api/v1/platform/domain-submissions/ids/{submission.id}",diff --git a/test/test_evaluation_api.py b/test/test_evaluation_api.pyindex 0dd5f12..114ef48 100644--- a/test/test_evaluation_api.py+++ b/test/test_evaluation_api.py@@ -1,51 +1,33 @@-import pytestimport io-from django.contrib.auth import get_user_model++import pytestfrom allauth.account.models import EmailAddress+from django.contrib.auth import get_user_modelfrom ninja_jwt.tokens import RefreshTokenfrom mwmbl.models import WasmEvaluationJob-User = get_user_model()@pytest.fixturedef user():"""Create a test user"""- return User.objects.create_user(- username='testuser',- email='[email protected]',- password='testpass123'- )+ return User.objects.create_user(username="testuser", email="[email protected]", password="testpass123")@pytest.fixturedef verified_user(user):"""Create a user with verified email"""- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=True,- primary=True- )+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@pytest.fixturedef unverified_user():"""Create a user with unverified email"""- user = User.objects.create_user(- username='unverified',- email='[email protected]',- password='testpass123'- )- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=False,- primary=True- )+ user = User.objects.create_user(username="unverified", email="[email protected]", password="testpass123")+ EmailAddress.objects.create(user=user, email="[email protected]", verified=False, primary=True)return user@@ -71,18 +53,41 @@ def valid_wasm_bytes():# (module# (memory (export "memory") 1)# )- return bytes([- 0x00, 0x61, 0x73, 0x6d, # WASM magic number- 0x01, 0x00, 0x00, 0x00, # WASM version- 0x05, 0x03, 0x01, 0x00, 0x01, # Memory section: 1 page minimum- 0x07, 0x0a, 0x01, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x02, 0x00 # Export section: export "memory"- ])+ return bytes(+ [+ 0x00,+ 0x61,+ 0x73,+ 0x6D, # WASM magic number+ 0x01,+ 0x00,+ 0x00,+ 0x00, # WASM version+ 0x05,+ 0x03,+ 0x01,+ 0x00,+ 0x01, # Memory section: 1 page minimum+ 0x07,+ 0x0A,+ 0x01,+ 0x06,+ 0x6D,+ 0x65,+ 0x6D,+ 0x6F,+ 0x72,+ 0x79,+ 0x02,+ 0x00, # Export section: export "memory"+ ]+ )@pytest.fixturedef invalid_wasm_bytes():"""Create invalid WASM file bytes"""- return b'invalid wasm content'+ return b"invalid wasm content"@pytest.mark.skip("Evaluation API is disabled for now")@@ -90,27 +95,21 @@ def invalid_wasm_bytes():def test_submit_valid_wasm_file(client, verified_user, access_token, valid_wasm_bytes):"""Test submitting a valid WASM file"""wasm_file = io.BytesIO(valid_wasm_bytes)- wasm_file.name = 'test_ranker.wasm'-- response = client.post(- '/api/v1/evaluate/submit',- {- 'file': wasm_file- },- HTTP_AUTHORIZATION=f'Bearer {access_token}'- )-+ wasm_file.name = "test_ranker.wasm"++ response = client.post("/api/v1/evaluate/submit", {"file": wasm_file}, HTTP_AUTHORIZATION=f"Bearer {access_token}")+assert response.status_code == 200-+response_data = response.json()- assert 'job_id' in response_data- assert response_data['status'] == 'VALIDATED'- assert 'message' in response_data-+ assert "job_id" in response_data+ assert response_data["status"] == "VALIDATED"+ assert "message" in response_data+# Verify job was created in database- job = WasmEvaluationJob.objects.get(id=response_data['job_id'])+ job = WasmEvaluationJob.objects.get(id=response_data["job_id"])assert job.user == verified_user- assert job.status == 'VALIDATED'+ assert job.status == "VALIDATED"assert job.wasm_file == valid_wasm_bytes@@ -119,43 +118,31 @@ def test_submit_valid_wasm_file(client, verified_user, access_token, valid_wasm_def test_submit_invalid_wasm_file(client, access_token, invalid_wasm_bytes):"""Test submitting an invalid WASM file"""wasm_file = io.BytesIO(invalid_wasm_bytes)- wasm_file.name = 'invalid.wasm'-- response = client.post(- '/api/v1/evaluate/submit',- {- 'file': wasm_file- },- HTTP_AUTHORIZATION=f'Bearer {access_token}'- )-+ wasm_file.name = "invalid.wasm"++ response = client.post("/api/v1/evaluate/submit", {"file": wasm_file}, HTTP_AUTHORIZATION=f"Bearer {access_token}")+assert response.status_code == 400-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'WASM validation failed' in response_data['message']+ assert response_data["status"] == "error"+ assert "WASM validation failed" in response_data["message"]@pytest.mark.skip("Evaluation API is disabled for now")@pytest.mark.django_dbdef test_submit_non_wasm_file(client, access_token):"""Test submitting a file without .wasm extension"""- text_file = io.BytesIO(b'some text content')- text_file.name = 'test.txt'-- response = client.post(- '/api/v1/evaluate/submit',- {- 'file': text_file- },- HTTP_AUTHORIZATION=f'Bearer {access_token}'- )-+ text_file = io.BytesIO(b"some text content")+ text_file.name = "test.txt"++ response = client.post("/api/v1/evaluate/submit", {"file": text_file}, HTTP_AUTHORIZATION=f"Bearer {access_token}")+assert response.status_code == 400-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'File must have .wasm extension' in response_data['message']+ assert response_data["status"] == "error"+ assert "File must have .wasm extension" in response_data["message"]@pytest.mark.skip("Evaluation API is disabled for now")@@ -163,23 +150,17 @@ def test_submit_non_wasm_file(client, access_token):def test_submit_large_file(client, access_token):"""Test submitting a file that exceeds size limit"""# Create a file larger than 10MB- large_content = b'x' * (11 * 1024 * 1024) # 11MB+ large_content = b"x" * (11 * 1024 * 1024) # 11MBlarge_file = io.BytesIO(large_content)- large_file.name = 'large.wasm'-- response = client.post(- '/api/v1/evaluate/submit',- {- 'file': large_file- },- HTTP_AUTHORIZATION=f'Bearer {access_token}'- )-+ large_file.name = "large.wasm"++ response = client.post("/api/v1/evaluate/submit", {"file": large_file}, HTTP_AUTHORIZATION=f"Bearer {access_token}")+assert response.status_code == 400-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'File too large' in response_data['message']+ assert response_data["status"] == "error"+ assert "File too large" in response_data["message"]@pytest.mark.skip("Evaluation API is disabled for now")@@ -187,15 +168,10 @@ def test_submit_large_file(client, access_token):def test_submit_without_authentication(client, valid_wasm_bytes):"""Test submitting without JWT token"""wasm_file = io.BytesIO(valid_wasm_bytes)- wasm_file.name = 'test.wasm'-- response = client.post(- '/api/v1/evaluate/submit',- {- 'file': wasm_file- }- )-+ wasm_file.name = "test.wasm"++ response = client.post("/api/v1/evaluate/submit", {"file": wasm_file})+assert response.status_code == 401@@ -204,33 +180,25 @@ def test_submit_without_authentication(client, valid_wasm_bytes):def test_submit_with_unverified_email(client, unverified_access_token, valid_wasm_bytes):"""Test submitting with unverified email address"""wasm_file = io.BytesIO(valid_wasm_bytes)- wasm_file.name = 'test.wasm'-+ wasm_file.name = "test.wasm"+response = client.post(- '/api/v1/evaluate/submit',- {- 'file': wasm_file- },- HTTP_AUTHORIZATION=f'Bearer {unverified_access_token}'+ "/api/v1/evaluate/submit", {"file": wasm_file}, HTTP_AUTHORIZATION=f"Bearer {unverified_access_token}")-+assert response.status_code == 403-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'Email address is not verified' in response_data['message']+ assert response_data["status"] == "error"+ assert "Email address is not verified" in response_data["message"]@pytest.mark.skip("Evaluation API is disabled for now")@pytest.mark.django_dbdef test_submit_without_file(client, access_token):"""Test submitting without a file"""- response = client.post(- '/api/v1/evaluate/submit',- {},- HTTP_AUTHORIZATION=f'Bearer {access_token}'- )-+ response = client.post("/api/v1/evaluate/submit", {}, HTTP_AUTHORIZATION=f"Bearer {access_token}")+assert response.status_code == 422 # Validation error@@ -240,37 +208,33 @@ def test_multiple_submissions_same_user(client, verified_user, access_token, val"""Test that a user can submit multiple WASM files"""# Submit first filewasm_file1 = io.BytesIO(valid_wasm_bytes)- wasm_file1.name = 'ranker1.wasm'-+ wasm_file1.name = "ranker1.wasm"+response1 = client.post(- '/api/v1/evaluate/submit',- {'file': wasm_file1},- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ "/api/v1/evaluate/submit", {"file": wasm_file1}, HTTP_AUTHORIZATION=f"Bearer {access_token}")-+assert response1.status_code == 200- job_id1 = response1.json()['job_id']-+ job_id1 = response1.json()["job_id"]+# Submit second filewasm_file2 = io.BytesIO(valid_wasm_bytes)- wasm_file2.name = 'ranker2.wasm'-+ wasm_file2.name = "ranker2.wasm"+response2 = client.post(- '/api/v1/evaluate/submit',- {'file': wasm_file2},- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ "/api/v1/evaluate/submit", {"file": wasm_file2}, HTTP_AUTHORIZATION=f"Bearer {access_token}")-+assert response2.status_code == 200- job_id2 = response2.json()['job_id']-+ job_id2 = response2.json()["job_id"]+# Verify both jobs exist and belong to the same userassert job_id1 != job_id2-+job1 = WasmEvaluationJob.objects.get(id=job_id1)job2 = WasmEvaluationJob.objects.get(id=job_id2)-+assert job1.user == verified_userassert job2.user == verified_user- assert job1.status == 'VALIDATED'- assert job2.status == 'VALIDATED'+ assert job1.status == "VALIDATED"+ assert job2.status == "VALIDATED"diff --git a/test/test_export_judgments.py b/test/test_export_judgments.pyindex 10ab132..d7fc3fe 100644--- a/test/test_export_judgments.py+++ b/test/test_export_judgments.py@@ -12,8 +12,15 @@ User = get_user_model()def doc(url, title="title", extract="extract", score=1.0, state=None, user_ids=None):- return {"url": url, "title": title, "extract": extract, "score": score,- "state": state, "term": "secret term", "user_ids": user_ids or [42]}+ return {+ "url": url,+ "title": title,+ "extract": extract,+ "score": score,+ "state": state,+ "term": "secret term",+ "user_ids": user_ids or [42],+ }@pytest.fixture@@ -42,20 +49,30 @@ def curation(user):@pytest.mark.django_dbdef test_export_writes_sanitized_records(tmp_path, user, flagger, curation):FlagCuration.objects.create(- user=flagger, timestamp=datetime(2025, 3, 2, tzinfo=timezone.utc),- curation=curation, flag="PROMOTION", status="ACCEPTED", reason="self promo")+ user=flagger,+ timestamp=datetime(2025, 3, 2, tzinfo=timezone.utc),+ curation=curation,+ flag="PROMOTION",+ status="ACCEPTED",+ reason="self promo",+ )UserCuration.objects.create(- user=user, timestamp=datetime(2024, 1, 1, tzinfo=timezone.utc),+ user=user,+ timestamp=datetime(2024, 1, 1, tzinfo=timezone.utc),url="https://mwmbl.org/?q=old+query",results=[doc("https://c.com")],curation_type="curate_delete",curation={"delete_index": 0},)- SearchResultVote.objects.create(- user=user, url="https://a.com", query="test query", vote_type="upvote")+ SearchResultVote.objects.create(user=user, url="https://a.com", query="test query", vote_type="upvote")DomainSubmission.objects.create(- name="spammy.example.com", submitted_by=user, status="REJECTED",- status_changed_by=flagger, rejection_reason="SPAM", rejection_detail="link farm")+ name="spammy.example.com",+ submitted_by=user,+ status="REJECTED",+ status_changed_by=flagger,+ rejection_reason="SPAM",+ rejection_detail="link farm",+ )call_command("export_judgments", output_dir=str(tmp_path))@@ -112,9 +129,14 @@ def test_stats_only_writes_no_files(tmp_path, curation):@pytest.mark.django_dbdef test_since_filters_old_records(tmp_path, user, curation):Curation.objects.create(- user=user, timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc),- query="recent query", original_index_results=[], original_results=[],- new_results=[], num_changes=1)+ user=user,+ timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc),+ query="recent query",+ original_index_results=[],+ original_results=[],+ new_results=[],+ num_changes=1,+ )call_command("export_judgments", output_dir=str(tmp_path), since="2026-01-01")@@ -126,9 +148,14 @@ def test_since_filters_old_records(tmp_path, user, curation):@pytest.mark.django_dbdef test_anonymous_and_malformed_data(tmp_path):Curation.objects.create(- user=None, timestamp=datetime(2025, 5, 1, tzinfo=timezone.utc),- query="anon query", original_index_results=[], original_results="garbage",- new_results=[doc("https://a.com")], num_changes=1)+ user=None,+ timestamp=datetime(2025, 5, 1, tzinfo=timezone.utc),+ query="anon query",+ original_index_results=[],+ original_results="garbage",+ new_results=[doc("https://a.com")],+ num_changes=1,+ )call_command("export_judgments", output_dir=str(tmp_path))diff --git a/test/test_external_cache.py b/test/test_external_cache.pyindex 34859be..eaca04a 100644--- a/test/test_external_cache.py+++ b/test/test_external_cache.py@@ -6,6 +6,7 @@ gone is everything that existed only because cache entries shared a file with recontent: the anonymisation gate, the Redis queue, the general-indexing path, and the guardsstopping a cache term reaching curation or /raw."""+import jsonimport randomimport string@@ -26,37 +27,49 @@ from mwmbl.indexer.external_cache import (store_external_results,)from mwmbl.tinysearchengine import rank-from mwmbl.tinysearchengine.indexer import (PAGE_SIZE, Document, DocumentSource, DocumentState,- TinyIndex)-from mwmbl.tinysearchengine.rank import (WIKI_FETCH_LIMIT, Ranker, get_wiki_results,- wiki_score)+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentSource, DocumentState, TinyIndex+from mwmbl.tinysearchengine.rank import WIKI_FETCH_LIMIT, Ranker, get_wiki_results, wiki_scoreNUM_PAGES = 8# state and source are set because get_wiki_results sets them, and the cache preserves what# the provider produced rather than stamping its own. The scores are not preserved - what# an entry stores is the rank - and the tests below rely on that.-PYTHON = Document("Python (programming language)", "https://en.wikipedia.org/wiki/Python",- "A high-level programming language.", 3.0, state=DocumentState.FROM_WIKI,- source=DocumentSource.WIKIPEDIA)-MONTY = Document("Monty Python", "https://en.wikipedia.org/wiki/Monty_Python",- "A British comedy troupe.", 2.0, state=DocumentState.FROM_WIKI,- source=DocumentSource.WIKIPEDIA)+PYTHON = Document(+ "Python (programming language)",+ "https://en.wikipedia.org/wiki/Python",+ "A high-level programming language.",+ 3.0,+ state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA,+)+MONTY = Document(+ "Monty Python",+ "https://en.wikipedia.org/wiki/Monty_Python",+ "A British comedy troupe.",+ 2.0,+ state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA,+)@pytest.fixturedef cache_index(tmp_path):"""A real, empty external results cache index that the module-level helpers pick up."""- with override_settings(DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="external-cache.tinysearch",- EXTERNAL_CACHE_NUM_PAGES=NUM_PAGES, EXTERNAL_CACHE_ENABLED=True):- TinyIndex.create(item_factory=Document, index_path=str(external_cache_path()),- num_pages=NUM_PAGES, page_size=PAGE_SIZE)+ with override_settings(+ DATA_PATH=str(tmp_path),+ EXTERNAL_CACHE_INDEX_NAME="external-cache.tinysearch",+ EXTERNAL_CACHE_NUM_PAGES=NUM_PAGES,+ EXTERNAL_CACHE_ENABLED=True,+ ):+ TinyIndex.create(+ item_factory=Document, index_path=str(external_cache_path()), num_pages=NUM_PAGES, page_size=PAGE_SIZE+ )yield str(external_cache_path())def _wiki_api_response(*titles):- return {"query": {"search": [{"title": title, "snippet": f"<b>{title}</b> snippet"}- for title in titles]}}+ return {"query": {"search": [{"title": title, "snippet": f"<b>{title}</b> snippet"} for title in titles]}}def _patched_wikipedia(response):@@ -76,6 +89,7 @@ def _patched_wikipedia(response):# The cache term# ---------------------------------------------------------------------------+def test_cache_term_is_stable_and_normalised():"""Whitespace and case must not split one query across several entries. The disk cachekeyed on the raw request URL, so " python " and "Python" missed each other."""@@ -120,6 +134,7 @@ def test_cache_term_is_keyed_so_it_cannot_be_recomputed_without_the_secret():# Round trip# ---------------------------------------------------------------------------+def test_results_come_back_from_the_index(cache_index):store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON, MONTY])@@ -175,7 +190,7 @@ def test_a_query_we_have_never_asked_about_is_a_miss(cache_index):def _all_stored_documents(index_path):"""Every document in the file, decompressed. Grepping the raw bytes proves little: thepages are zstd-compressed, so plaintext would not show up there either way."""- with TinyIndex(Document, index_path, 'r') as index:+ with TinyIndex(Document, index_path, "r") as index:return [document for page in range(index.num_pages) for document in index.get_page(page)]@@ -198,8 +213,13 @@ def test_the_file_holds_no_query_text(cache_index):# alone, so the tests below exercise it even though nothing fetches from it yet.OTHER_SOURCE = DocumentSource.STAAN-STAAN_RESULT = Document("Python tutorial", "https://example.com/python",- "Somewhere that is not Wikipedia.", 9.0, source=DocumentSource.STAAN)+STAAN_RESULT = Document(+ "Python tutorial",+ "https://example.com/python",+ "Somewhere that is not Wikipedia.",+ 9.0,+ source=DocumentSource.STAAN,+)def test_two_providers_do_not_share_an_entry_for_the_same_query(cache_index):@@ -245,7 +265,7 @@ def test_both_providers_results_for_a_query_live_on_one_page(cache_index):store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])- with TinyIndex(Document, cache_index, 'r') as index:+ with TinyIndex(Document, cache_index, "r") as index:page = index.get_page(index.get_key_page_index(external_cache_term("python")))assert {document.source for document in page} == {DocumentSource.WIKIPEDIA, DocumentSource.STAAN}@@ -276,6 +296,7 @@ def test_a_providers_state_and_source_survive_the_round_trip(cache_index):# Freshness# ---------------------------------------------------------------------------+def test_a_stale_entry_is_not_a_cache_hit(cache_index):now = int(time.time())store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)@@ -303,7 +324,7 @@ def test_stale_entries_are_dropped_when_the_page_is_rewritten(cache_index):much_later = now + 27 * 7 * 24 * 60 * 60store_external_results(DocumentSource.WIKIPEDIA, new_query, [MONTY], now=much_later)- with TinyIndex(Document, cache_index, 'r') as index:+ with TinyIndex(Document, cache_index, "r") as index:page = index.get_page(index.get_key_page_index(external_cache_term(new_query)))assert [document.url for document in page] == [MONTY.url]@@ -313,6 +334,7 @@ def test_stale_entries_are_dropped_when_the_page_is_rewritten(cache_index):# Negative caching# ---------------------------------------------------------------------------+def test_a_query_a_provider_has_nothing_for_is_remembered(cache_index):"""Without this the query is re-fetched forever - #357 listed it as unfixable, becausea sentinel in the search index would surface in /raw and in every candidate list."""@@ -361,6 +383,7 @@ def test_documents_without_a_url_or_title_are_not_stored(cache_index):# The fetch path# ---------------------------------------------------------------------------+def test_a_cache_miss_calls_wikipedia_and_stores_the_result(cache_index):mock, session = _patched_wikipedia(_wiki_api_response("Python (programming language)"))try:@@ -448,8 +471,9 @@ def test_a_cache_hit_scores_the_same_as_the_live_fetch_that_filled_it(cache_indecached = get_wiki_results("python", 3)- assert [(document.url, document.score) for document in cached] == \- [(document.url, document.score) for document in live]+ assert [(document.url, document.score) for document in cached] == [+ (document.url, document.score) for document in live+ ]def test_a_hit_is_served_while_the_circuit_breaker_is_open(cache_index):@@ -518,6 +542,7 @@ def test_the_query_is_truncated_before_it_reaches_wikipedia(cache_index):# The kill switch# ---------------------------------------------------------------------------+def test_the_kill_switch_stops_reads_and_writes(cache_index):with override_settings(EXTERNAL_CACHE_ENABLED=False):store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])@@ -527,8 +552,9 @@ def test_the_kill_switch_stops_reads_and_writes(cache_index):def test_a_missing_cache_file_is_a_miss_rather_than_an_error(tmp_path):- with override_settings(DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="absent.tinysearch",- EXTERNAL_CACHE_ENABLED=True):+ with override_settings(+ DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="absent.tinysearch", EXTERNAL_CACHE_ENABLED=True+ ):assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None@@ -536,9 +562,10 @@ def test_a_missing_cache_file_is_a_miss_rather_than_an_error(tmp_path):# Page competition# ---------------------------------------------------------------------------+def _colliding_queries(index_path):"""Two queries whose cache terms land on the same page."""- with TinyIndex(Document, index_path, 'r') as index:+ with TinyIndex(Document, index_path, "r") as index:first = "collide-0"first_page = index.get_key_page_index(external_cache_term(first))for i in range(1, 10000):@@ -559,8 +586,7 @@ def test_the_newest_entry_survives_a_full_page_and_the_oldest_is_evicted(cache_iordering, which would evict by relevance instead of by age."""old_query, new_query = _colliding_queries(cache_index)now = int(time.time())- bulky = [Document(f"Title {i}", f"https://en.wikipedia.org/wiki/{i}", _filler(2000), 3.0)- for i in range(4)]+ bulky = [Document(f"Title {i}", f"https://en.wikipedia.org/wiki/{i}", _filler(2000), 3.0) for i in range(4)]store_external_results(DocumentSource.WIKIPEDIA, old_query, bulky, now=now)assert get_cached_external_results(DocumentSource.WIKIPEDIA, old_query, now=now) is not None@@ -582,6 +608,7 @@ def test_an_entry_replaces_its_own_previous_results(cache_index):# Keeping Wikipedia off the keystroke path# ---------------------------------------------------------------------------+class _RecordingRanker(Ranker):"""The smallest thing that exercises Ranker.search's external_search plumbing, which iswhat LTRRanker inherits and what the production ranker therefore uses."""@@ -602,7 +629,7 @@ def test_search_does_not_call_external_search_when_it_is_turned_off(cache_index)"""The search-as-you-type trigger passes use_external_search=False. Typing a15-character query otherwise costs a Wikipedia call per keystroke, and 92% of thoseprefixes are ones nobody ever searches for, so no cache can absorb them."""- with TinyIndex(Document, cache_index, 'r') as index:+ with TinyIndex(Document, cache_index, "r") as index:ranker = _RecordingRanker(index)ranker.search("python", [], use_external_search=False)@@ -610,7 +637,7 @@ def test_search_does_not_call_external_search_when_it_is_turned_off(cache_index)def test_search_calls_external_search_by_default(cache_index):- with TinyIndex(Document, cache_index, 'r') as index:+ with TinyIndex(Document, cache_index, "r") as index:ranker = _RecordingRanker(index)ranker.search("python", [])@@ -621,6 +648,7 @@ def test_search_calls_external_search_by_default(cache_index):# Bootstrap# ---------------------------------------------------------------------------+def test_a_resized_cache_index_is_rebuilt_rather_than_crashing_startup(tmp_path):"""Resizing a cache should be a config change, not a startup crash - the cost ofrebuilding is re-fetching."""@@ -630,7 +658,7 @@ def test_a_resized_cache_index_is_rebuilt_rather_than_crashing_startup(tmp_path)with override_settings(DATA_PATH=str(tmp_path)):create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)- with TinyIndex(Document, str(path), 'r') as index:+ with TinyIndex(Document, str(path), "r") as index:assert index.num_pages == 16@@ -654,7 +682,7 @@ def test_an_unreadable_cache_index_is_rebuilt_rather_than_crashing_startup(tmp_pwith override_settings(DATA_PATH=str(tmp_path)):create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)- with TinyIndex(Document, str(path), 'r') as index:+ with TinyIndex(Document, str(path), "r") as index:assert index.num_pages == 16@@ -681,8 +709,7 @@ def test_a_rebuild_leaves_no_window_with_the_index_missing(tmp_path):seen.append((path.exists(), kwargs.get("index_path", args[1] if len(args) > 1 else None)))return real_create(*args, **kwargs)- with override_settings(DATA_PATH=str(tmp_path)), \- patch.object(TinyIndex, "create", side_effect=watched_create):+ with override_settings(DATA_PATH=str(tmp_path)), patch.object(TinyIndex, "create", side_effect=watched_create):create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)assert seen, "nothing was rebuilt, so this would pass for the wrong reason"diff --git a/test/test_fallback_model.py b/test/test_fallback_model.pyindex 131a01f..6f7bd46 100644--- a/test/test_fallback_model.py+++ b/test/test_fallback_model.py@@ -1,4 +1,5 @@"""Trigger logic for the Super-Search-as-fallback ranking model (no network)."""+from mwmbl.rankeval.evaluation.evaluate import RankingModel@@ -17,6 +18,7 @@ def _fallback(primary_urls, fallback_urls, threshold, fb_calls=None):# Imported lazily so the module's django.setup()/search_setup import cost is only# paid when these tests actually run.from mwmbl.rankeval.evaluation.evaluate_fallback import FallbackRankingModel+return FallbackRankingModel(_FixedModel(primary_urls),_FixedModel(fallback_urls, calls=fb_calls),diff --git a/test/test_format.py b/test/test_format.pyindex c04e15d..aed7498 100644--- a/test/test_format.py+++ b/test/test_format.py@@ -1,6 +1,3 @@---from mwmbl.format import format_resultfrom mwmbl.tinysearchengine.indexer import Documentdiff --git a/test/test_index_batches.py b/test/test_index_batches.pyindex ec4859f..79dce14 100644--- a/test/test_index_batches.py+++ b/test/test_index_batches.py@@ -8,10 +8,13 @@ import pytestfrom mwmbl.indexer.blacklist_providers import StaticBlacklistProviderfrom mwmbl.indexer.blacklist_snapshot import SnapshotBlacklistfrom mwmbl.indexer.index_batches import (- sort_documents, combine_documents, _merge_user_ids, MAX_USER_IDS,- index_results_against_query, index_documents,+ _merge_user_ids,+ combine_documents,+ index_documents,+ index_results_against_query,+ sort_documents,)-from mwmbl.tinysearchengine.indexer import Document, DocumentState, PAGE_SIZE, TinyIndex+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentState, TinyIndexclass UrlRanker:@@ -31,7 +34,6 @@ def test_sort_documents():documents = [Document(title="title5", url="2", extract="extract5", term="term1"),Document(title="title6", url="3", extract="extract6", term="term2"),-]# Sort the documents@@ -59,7 +61,6 @@ def test_sort_documents_curated_items_first():documents = [Document(title="title5", url="2", extract="extract5", term="term1"),Document(title="title6", url="3", extract="extract6", term="term2"),-]# Sort the documents@@ -99,6 +100,7 @@ def test_sort_documents_duplicates_keep_synced_state():# index_results_against_query# ---------------------------------------------------------------------------+def test_index_results_against_query():# "rust", "async" and the bigram "rust async" land on distinct pages here,# so cross-term URL dedup within a page does not interfere with the asserts.@@ -109,7 +111,7 @@ def test_index_results_against_query():docs = [a, b, c]with TemporaryDirectory() as temp_dir:- index_path = str(Path(temp_dir) / 'temp-index.tinysearch')+ index_path = str(Path(temp_dir) / "temp-index.tinysearch")with TinyIndex.create(Document, index_path, num_pages=num_pages, page_size=4096):pass@@ -118,7 +120,7 @@ def test_index_results_against_query():# All three pages are newly added (each matches at least one term).assert new_count == 3- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:rust_urls = {d.url for d in indexer.retrieve("rust")}async_urls = {d.url for d in indexer.retrieve("async")}bigram_urls = {d.url for d in indexer.retrieve("rust async")}@@ -142,14 +144,14 @@ def test_index_results_against_query_keeps_title_only_documents():docs = [doc]with TemporaryDirectory() as temp_dir:- index_path = str(Path(temp_dir) / 'temp-index.tinysearch')+ index_path = str(Path(temp_dir) / "temp-index.tinysearch")with TinyIndex.create(Document, index_path, num_pages=num_pages, page_size=4096):passnew_count = index_results_against_query(docs, "kitsas", index_path)assert new_count == 1- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:kitsas_urls = {d.url for d in indexer.retrieve("kitsas")}assert kitsas_urls == {doc.url}@@ -158,6 +160,7 @@ def test_index_results_against_query_keeps_title_only_documents():# _merge_user_ids# ---------------------------------------------------------------------------+def test_merge_user_ids_empty_existing():assert _merge_user_ids(None, [1]) == [1]@@ -183,6 +186,7 @@ def test_merge_user_ids_both_none():# combine_documents: user_ids and last_crawled merging# ---------------------------------------------------------------------------+def test_combine_documents_merges_user_ids_for_same_url():existing = [Document(title="t", url="http://a.com", extract="e", term="q", user_ids=[1])]new_docs = [Document(title="t", url="http://a.com", extract="e", term="q", user_ids=[2])]@@ -229,8 +233,7 @@ def snapshot_blacklist(domains: set[str]) -> SnapshotBlacklist:as its built-in rules keeps these tests off the snapshot machinery, whichtest_blacklist_snapshot.py covers."""- return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(domains),- redis_client=fakeredis.FakeRedis())+ return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(domains), redis_client=fakeredis.FakeRedis())@pytest.fixturediff --git a/test/test_index_page_writes.py b/test/test_index_page_writes.pyindex 4f375d0..44581ef 100644--- a/test/test_index_page_writes.py+++ b/test/test_index_page_writes.py@@ -4,6 +4,7 @@ _write_page copies ~4 KB into the mmap, and a reader catching it half-written gethat will not decompress. Every read-modify-write goes through TinyIndex.page(), whichholds the page's lock for the whole block, so no caller has to know locking exists."""+import errnoimport fcntlimport multiprocessing@@ -19,9 +20,7 @@ import pytestfrom mwmbl.indexer.index_batches import index_pages, index_results_against_queryfrom mwmbl.indexer.purge_blacklisted import purge_documents-from mwmbl.tinysearchengine.indexer import (- METADATA_SIZE, Document, DocumentState, PageError, TinyIndex)-+from mwmbl.tinysearchengine.indexer import METADATA_SIZE, Document, DocumentState, PageError, TinyIndexNUM_PAGES = 64@@ -43,12 +42,14 @@ def _try_lock_in_child(index_path: str, page: int, queue):"""Child process: report whether the page lock is free."""from mwmbl.tinysearchengine.indexer import _FLOCK_STRUCT- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:start = page * indexer.page_size + METADATA_SIZEtry:- fcntl.fcntl(indexer.index_file.fileno(), F_OFD_SETLK,- struct.pack(_FLOCK_STRUCT, fcntl.F_WRLCK, os.SEEK_SET,- start, indexer.page_size, 0))+ fcntl.fcntl(+ indexer.index_file.fileno(),+ F_OFD_SETLK,+ struct.pack(_FLOCK_STRUCT, fcntl.F_WRLCK, os.SEEK_SET, start, indexer.page_size, 0),+ )queue.put(True)except OSError:queue.put(False)@@ -62,7 +63,7 @@ def test_page_holds_the_lock_across_the_read_and_the_write(index_path):nothing."""context = multiprocessing.get_context("fork")- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:target = indexer.get_key_page_index("zebra")def child_can_lock(page_index=target):@@ -72,7 +73,7 @@ def test_page_holds_the_lock_across_the_read_and_the_write(index_path):child.join(timeout=30)return queue.get(timeout=5)- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:with indexer.page(target) as page:assert not child_can_lock(), "another process took a lock we were holding"@@ -80,10 +81,9 @@ def test_page_holds_the_lock_across_the_read_and_the_write(index_path):# moment any fd on the file is closed - and index_results_against_query opens# and closes a read handle on every store, so in a threaded worker that would# silently release the lock mid-write. OFD locks survive it.- with TinyIndex(Document, index_path, 'r'):+ with TinyIndex(Document, index_path, "r"):pass- assert not child_can_lock(), \- "closing an unrelated fd released the lock - this needs an OFD lock"+ assert not child_can_lock(), "closing an unrelated fd released the lock - this needs an OFD lock"assert child_can_lock((target + 1) % NUM_PAGES), "a different page was blocked"assert page.documents == []@@ -92,16 +92,15 @@ def test_page_holds_the_lock_across_the_read_and_the_write(index_path):def test_page_writes_nothing_unless_store_is_called(index_path):- seeded = [Document("Zebra", "https://en.wikipedia.org/wiki/Zebra", "", 3.0, "zebra",- state=DocumentState.FROM_WIKI)]- with TinyIndex(Document, index_path, 'w') as indexer:+ seeded = [Document("Zebra", "https://en.wikipedia.org/wiki/Zebra", "", 3.0, "zebra", state=DocumentState.FROM_WIKI)]+ with TinyIndex(Document, index_path, "w") as indexer:target = indexer.get_key_page_index("zebra")indexer.store_in_page(target, seeded)with indexer.page(target) as page:assert [d.url for d in page.documents] == [seeded[0].url] # read, then do nothing- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:assert [d.url for d in indexer.get_page(target)] == [seeded[0].url]@@ -111,22 +110,29 @@ def test_store_reports_how_many_documents_actually_fit(index_path):# Distinct, incompressible extracts - zstd would squeeze 500 identical ones onto a# single page and there would be nothing to truncate.filler = random.Random(0)- many = [Document(f"Zebra {i}", f"https://en.wikipedia.org/wiki/Zebra_{i}",- "".join(filler.choices(string.ascii_letters, k=400)),- 3.0, "zebra", state=DocumentState.FROM_WIKI)- for i in range(500)]- with TinyIndex(Document, index_path, 'w') as indexer:+ many = [+ Document(+ f"Zebra {i}",+ f"https://en.wikipedia.org/wiki/Zebra_{i}",+ "".join(filler.choices(string.ascii_letters, k=400)),+ 3.0,+ "zebra",+ state=DocumentState.FROM_WIKI,+ )+ for i in range(500)+ ]+ with TinyIndex(Document, index_path, "w") as indexer:with indexer.page(indexer.get_key_page_index("zebra")) as page:num_stored = page.store(many)assert 0 < num_stored < len(many), f"expected truncation, stored {num_stored}"- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:assert len(indexer.get_page(indexer.get_key_page_index("zebra"))) == num_storeddef test_page_needs_write_mode(index_path):- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:with pytest.raises(UnsupportedOperation):with indexer.page(0):pass@@ -136,8 +142,7 @@ def _fail_to_lock(monkeypatch, err: OSError):from mwmbl.tinysearchengine import indexer as indexer_modulemonkeypatch.setattr(indexer_module, "_page_locking_supported", True)- monkeypatch.setattr(indexer_module, "_set_page_lock",- lambda *args: (_ for _ in ()).throw(err))+ monkeypatch.setattr(indexer_module, "_set_page_lock", lambda *args: (_ for _ in ()).throw(err))return indexer_module@@ -145,7 +150,7 @@ def _store_one(index_path: str) -> Document:document = Document("Zebra", "https://en.wikipedia.org/wiki/Zebra", "", 3.0, None)assert index_results_against_query([document], "zebra", index_path) == 1- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:assert {d.url for d in indexer.retrieve("zebra")} == {document.url}return document@@ -181,24 +186,23 @@ def _store_repeatedly(index_path: str, term: str, url_prefix: str, count: int):for i in range(count):# The title has to contain the term, or the containment rule stores nothing.index_results_against_query(- [Document(f"Zebra {url_prefix} {i}", f"https://example.com/{url_prefix}/{i}", "",- 3.0, None)],- term, index_path)+ [Document(f"Zebra {url_prefix} {i}", f"https://example.com/{url_prefix}/{i}", "", 3.0, None)],+ term,+ index_path,+ )def test_concurrent_writers_leave_the_page_readable(index_path):"""Smoke test: four processes hammering one page leave it decodable, not empty."""context = multiprocessing.get_context("fork")- workers = [context.Process(target=_store_repeatedly,- args=(index_path, "zebra", f"w{n}", 30))- for n in range(4)]+ workers = [context.Process(target=_store_repeatedly, args=(index_path, "zebra", f"w{n}", 30)) for n in range(4)]for worker in workers:worker.start()for worker in workers:worker.join(timeout=60)assert worker.exitcode == 0- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:page = indexer.get_page(indexer.get_key_page_index("zebra"))assert page, "the page was left empty"@@ -209,6 +213,7 @@ def test_concurrent_writers_leave_the_page_readable(index_path):# A page that will not decode# ---------------------------------------------------------------------------+def _doc(i: int, term: str) -> Document:return Document(f"Title {i}", f"https://example.com/{i}", "extract", 1.0, term)@@ -219,14 +224,14 @@ def _corrupt(index_path: str, page_index: int):It has to be written first: a freshly created page is a few bytes of compressed `[]`followed by zero padding, so overwriting the padding with zeros changes nothing."""- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:indexer.store_in_page(page_index, [_doc(i, "seeded") for i in range(5)])with open(index_path, "r+b") as index_file:index_file.seek(METADATA_SIZE + page_index * 4096 + 16)index_file.write(b"\0" * 64)- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:with pytest.raises(PageError):indexer.get_page(page_index)@@ -241,7 +246,7 @@ def test_one_unreadable_page_does_not_abandon_the_rest_of_the_batch(index_path):counts = index_pages(index_path, {3: [_doc(1, "a")], 4: [_doc(2, "b")], 5: [_doc(3, "c")]})- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:assert len(indexer.get_page(4)) == 1, "a later page was skipped"assert len(indexer.get_page(5)) == 1, "a later page was skipped"assert counts["b"] == 1 and counts["c"] == 1@@ -255,7 +260,7 @@ def test_a_corrupt_page_is_reset_by_a_writer_holding_its_lock(index_path):index_pages(index_path, {3: [_doc(1, "a")]})- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:assert [d.url for d in indexer.get_page(3)] == ["https://example.com/1"]@@ -268,7 +273,7 @@ def test_an_unlocked_writer_refuses_a_page_it_could_not_read(index_path, monkeypindex_pages(index_path, {3: [_doc(1, "a")], 4: [_doc(2, "b")]})- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:assert len(indexer.get_page(4)) == 1, "a later page was skipped"with pytest.raises(PageError):indexer.get_page(3) # left alone, not reset and not merged onto@@ -278,17 +283,16 @@ def test_purging_skips_a_page_it_cannot_read(index_path):"""purge_documents' caller has already drained these documents out of the purge queue,so one propagating page would lose the whole batch on every run."""blacklisted = Document("Spam", "https://spam.example.com/1", "extract", 1.0, None)- with TinyIndex(Document, index_path, 'w') as indexer:- pages = {indexer.get_key_page_index(term)- for term in ("spam", "extract", "spam example com")}+ with TinyIndex(Document, index_path, "w") as indexer:+ pages = {indexer.get_key_page_index(term) for term in ("spam", "extract", "spam example com")}assert len(pages) > 1, "expected the document to be filed under several terms"- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:for page_index in pages:indexer.store_in_page(page_index, [blacklisted])_corrupt(index_path, sorted(pages)[0])- with TinyIndex(Document, index_path, 'w') as indexer:+ with TinyIndex(Document, index_path, "w") as indexer:removed = purge_documents(indexer, [blacklisted], lambda domain: True)assert removed == {"spam.example.com": 1}, "the readable pages were not purged"@@ -301,7 +305,7 @@ def test_damage_that_gets_past_zstd_still_degrades_to_no_results(index_path):from mwmbl.tinysearchengine.indexer import _pad_to_page_size- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:page_index = indexer.get_key_page_index("zebra")payload = ZstdCompressor().compress(b"\xff\xfe definitely not json")@@ -309,7 +313,7 @@ def test_damage_that_gets_past_zstd_still_degrades_to_no_results(index_path):index_file.seek(METADATA_SIZE + page_index * 4096)index_file.write(_pad_to_page_size(payload, 4096))- with TinyIndex(Document, index_path, 'r') as indexer:+ with TinyIndex(Document, index_path, "r") as indexer:with pytest.raises(PageError):indexer.get_page(page_index)assert indexer.retrieve("zebra") == []diff --git a/test/test_indexdb.py b/test/test_indexdb.pyindex 14b1a3c..2dff0ef 100644--- a/test/test_indexdb.py+++ b/test/test_indexdb.py@@ -2,5 +2,5 @@ from mwmbl.tokenizer import clean_unicodedef test_clean_unicode():- result = clean_unicode('Bad extract text \ud83c')- assert result == 'Bad extract text '+ result = clean_unicode("Bad extract text \ud83c")+ assert result == "Bad extract text "diff --git a/test/test_indexer.py b/test/test_indexer.pyindex 97fcb3e..1a092cd 100644--- a/test/test_indexer.py+++ b/test/test_indexer.py@@ -3,8 +3,17 @@ from tempfile import TemporaryDirectoryfrom zstandard import ZstdCompressor-from mwmbl.tinysearchengine.indexer import STATE_INDEX, TinyIndex, Document, DocumentSource, \- DocumentState, _binary_search_fitting_size, _trim_items_to_page, _pad_to_page_size, _get_page_data+from mwmbl.tinysearchengine.indexer import (+ STATE_INDEX,+ Document,+ DocumentSource,+ DocumentState,+ TinyIndex,+ _binary_search_fitting_size,+ _get_page_data,+ _pad_to_page_size,+ _trim_items_to_page,+)def test_create_index():@@ -12,61 +21,64 @@ def test_create_index():page_size = 4096with TemporaryDirectory() as temp_dir:- index_path = Path(temp_dir) / 'temp-index.tinysearch'+ index_path = Path(temp_dir) / "temp-index.tinysearch"with TinyIndex.create(Document, str(index_path), num_pages=num_pages, page_size=page_size) as indexer:for i in range(num_pages):page = indexer.get_page(i)assert page == []+def test_binary_search_fitting_size_all_fit():- items = [1,2,3,4,5,6,7,8,9]+ items = [1, 2, 3, 4, 5, 6, 7, 8, 9]compressor = ZstdCompressor()page_size = 4096- count_fit, data = _binary_search_fitting_size(compressor,page_size,items,0,len(items))-+ count_fit, data = _binary_search_fitting_size(compressor, page_size, items, 0, len(items))+# We should fit everythingassert count_fit == len(items)-++def test_binary_search_fitting_size_subset_fit():- items = [1,2,3,4,5,6,7,8,9]+ items = [1, 2, 3, 4, 5, 6, 7, 8, 9]compressor = ZstdCompressor()page_size = 15- count_fit, data = _binary_search_fitting_size(compressor,page_size,items,0,len(items))-+ count_fit, data = _binary_search_fitting_size(compressor, page_size, items, 0, len(items))+# We should not fit everythingassert count_fit < len(items)-++def test_binary_search_fitting_size_none_fit():- items = [1,2,3,4,5,6,7,8,9]+ items = [1, 2, 3, 4, 5, 6, 7, 8, 9]compressor = ZstdCompressor()page_size = 5- count_fit, data = _binary_search_fitting_size(compressor,page_size,items,0,len(items))-+ count_fit, data = _binary_search_fitting_size(compressor, page_size, items, 0, len(items))+# We should not fit anythingassert count_fit == -1assert data is Nonedef test_get_page_data_single_doc():- document1 = Document(title='title1',url='url1',extract='extract1',score=1.0)+ document1 = Document(title="title1", url="url1", extract="extract1", score=1.0)items = [document1.as_tuple()]compressor = ZstdCompressor()page_size = 4096-+# Trim data- num_fitting,trimmed_data = _trim_items_to_page(compressor,4096,items)-+ num_fitting, trimmed_data = _trim_items_to_page(compressor, 4096, items)+# We should be able to fit the 1 item into a pageassert num_fitting == 1-+# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmmed data, then it should be equal to the data we persistpadded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)serialized_data, num_stored = _get_page_data(page_size, items)assert serialized_data == padded_trimmed_dataassert num_stored == num_fitting-+def test_get_page_data_many_docs_all_fit():# Build giant documents item@@ -74,23 +86,23 @@ def test_get_page_data_many_docs_all_fit():documents_len = 500page_size = 4096for x in range(documents_len):- txt = 'text{}'.format(x)- document = Document(title=txt,url=txt,extract=txt,score=x)+ txt = "text{}".format(x)+ document = Document(title=txt, url=txt, extract=txt, score=x)documents.append(document)items = [document.as_tuple() for document in documents]-+# Trim the itemscompressor = ZstdCompressor()- num_fitting,trimmed_data = _trim_items_to_page(compressor,page_size,items)-+ num_fitting, trimmed_data = _trim_items_to_page(compressor, page_size, items)+# We should be able to fit all itemsassert num_fitting == documents_len-+# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmed data, then it should be equal to the data we persistserialized_data, num_stored = _get_page_data(page_size, items)padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)-+assert serialized_data == padded_trimmed_dataassert num_stored == num_fitting@@ -101,59 +113,59 @@ def test_get_page_data_many_docs_subset_fit():documents_len = 5000page_size = 4096for x in range(documents_len):- txt = 'text{}'.format(x)- document = Document(title=txt,url=txt,extract=txt,score=x)+ txt = "text{}".format(x)+ document = Document(title=txt, url=txt, extract=txt, score=x)documents.append(document)items = [document.as_tuple() for document in documents]-+# Trim the itemscompressor = ZstdCompressor()- num_fitting,trimmed_data = _trim_items_to_page(compressor,page_size,items)-+ num_fitting, trimmed_data = _trim_items_to_page(compressor, page_size, items)+# We should be able to fit a subset of the items onto the pageassert num_fitting > 1assert num_fitting < documents_len-+# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmed data, then it should be equal to the data we persistserialized_data, num_stored = _get_page_data(page_size, items)padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)-+assert serialized_data == padded_trimmed_data# The count is what store() reports back, so it has to match what was actually kept.assert num_stored == num_fittingdef test_constructing_document_removes_none():- document = Document(title=None,url='url',extract=None,score=1.0)- assert document.title == ''- assert document.extract == ''+ document = Document(title=None, url="url", extract=None, score=1.0)+ assert document.title == ""+ assert document.extract == ""def test_as_tuple_with_new_fields():- doc = Document(title='t', url='u', extract='e', user_ids=[1, 2], last_crawled=1700000000)- assert doc.as_tuple() == ('t', 'u', 'e', None, None, None, [1, 2], 1700000000)+ doc = Document(title="t", url="u", extract="e", user_ids=[1, 2], last_crawled=1700000000)+ assert doc.as_tuple() == ("t", "u", "e", None, None, None, [1, 2], 1700000000)def test_as_tuple_strips_trailing_nones():- doc = Document(title='t', url='u', extract='e')- assert doc.as_tuple() == ('t', 'u', 'e')+ doc = Document(title="t", url="u", extract="e")+ assert doc.as_tuple() == ("t", "u", "e")def test_as_tuple_only_last_crawled_none_strips_it():- doc = Document(title='t', url='u', extract='e', user_ids=[1])- assert doc.as_tuple() == ('t', 'u', 'e', None, None, None, [1])+ doc = Document(title="t", url="u", extract="e", user_ids=[1])+ assert doc.as_tuple() == ("t", "u", "e", None, None, None, [1])def test_document_round_trip_with_new_fields():- doc = Document(title='t', url='u', extract='e', term='q', user_ids=[42], last_crawled=1700000000)+ doc = Document(title="t", url="u", extract="e", term="q", user_ids=[42], last_crawled=1700000000)restored = Document(*doc.as_tuple())assert restored.user_ids == [42]assert restored.last_crawled == 1700000000def test_document_backward_compat_old_six_element_tuple():- old_tuple = ('title', 'https://example.com', 'extract', None, 'term', None)+ old_tuple = ("title", "https://example.com", "extract", None, "term", None)doc = Document(*old_tuple)assert doc.user_ids is Noneassert doc.last_crawled is None@@ -161,8 +173,17 @@ def test_document_backward_compat_old_six_element_tuple():def test_state_index_matches_the_tuple_layout():"""get_page blanks the state by position, so the constant has to track as_tuple()."""- document = Document("Title", "https://example.com", "Extract", 1.0, "term",- DocumentState.FROM_WIKI, None, 123, DocumentSource.WIKIPEDIA)+ document = Document(+ "Title",+ "https://example.com",+ "Extract",+ 1.0,+ "term",+ DocumentState.FROM_WIKI,+ None,+ 123,+ DocumentSource.WIKIPEDIA,+ )assert document.as_tuple()[STATE_INDEX] == DocumentState.FROM_WIKI.value@@ -173,15 +194,14 @@ def test_a_document_with_an_unreadable_state_keeps_its_other_values():tuple - source follows it on external cache entries - so stripping the tail would takesource off instead, leave the bad state in place, and drop the document on the retry."""with TemporaryDirectory() as temp_dir:- index_path = str(Path(temp_dir) / 'temp-index.tinysearch')+ index_path = str(Path(temp_dir) / "temp-index.tinysearch")TinyIndex.create(Document, index_path, num_pages=2, page_size=4096)# title, url, extract, score, term, state, user_ids, last_crawled, source- item = ["Title", "https://example.com", "Extract", 1.0, "term", 99, None, 123,- DocumentSource.WIKIPEDIA.value]- with TinyIndex(Document, index_path, 'w') as index:+ item = ["Title", "https://example.com", "Extract", 1.0, "term", 99, None, 123, DocumentSource.WIKIPEDIA.value]+ with TinyIndex(Document, index_path, "w") as index:index._write_page([item], 0)- with TinyIndex(Document, index_path, 'r') as index:+ with TinyIndex(Document, index_path, "r") as index:documents = index.get_page(0)assert len(documents) == 1diff --git a/test/test_large_url_list.py b/test/test_large_url_list.pyindex 5b1aef7..56c3a98 100644--- a/test/test_large_url_list.py+++ b/test/test_large_url_list.py@@ -1,29 +1,20 @@-import pytestimport json-from django.contrib.auth import get_user_model++import pytestfrom allauth.account.models import EmailAddress+from django.contrib.auth import get_user_modelfrom ninja_jwt.tokens import RefreshTokenfrom mwmbl.models import SearchResultVote-User = get_user_model()@pytest.fixturedef verified_user():"""Create a user with verified email"""- user = User.objects.create_user(- username='testuser',- email='[email protected]',- password='testpass123'- )- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=True,- primary=True- )+ user = User.objects.create_user(username="testuser", email="[email protected]", password="testpass123")+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@@ -40,60 +31,52 @@ def test_large_url_list_post_method(client, verified_user, access_token):# Generate 200 URLs to test well beyond the previous threshold of ~160urls = []query = "test query"-+for i in range(200):# Generate URLs with varying lengths and some special characters- url = f'https://example{i}.com/path/to/resource?param=value&id={i}&data={"some,data,with,commas"}'+ url = f"https://example{i}.com/path/to/resource?param=value&id={i}&data={'some,data,with,commas'}"urls.append(url)-+# Create some votes for variety- vote_type = 'upvote' if i % 3 == 0 else 'downvote'+ vote_type = "upvote" if i % 3 == 0 else "downvote"if i % 2 == 0: # Create votes for every other URL- SearchResultVote.objects.create(- user=verified_user,- url=url,- query=query,- vote_type=vote_type- )-+ SearchResultVote.objects.create(user=verified_user, url=url, query=query, vote_type=vote_type)+# This would have failed with the old GET method due to URL length limits- vote_stats_request = {- "query": query,- "urls": urls- }-+ vote_stats_request = {"query": query, "urls": urls}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert 'votes' in response_data- assert len(response_data['votes']) == 200-+ assert "votes" in response_data+ assert len(response_data["votes"]) == 200+# Verify all URLs are present and have correct vote countsvotes_with_data = 0for i, url in enumerate(urls):- assert url in response_data['votes']+ assert url in response_data["votes"]if i % 2 == 0: # URLs that have votesvotes_with_data += 1- expected_vote_type = 'upvote' if i % 3 == 0 else 'downvote'- assert response_data['votes'][url]['user_vote'] == expected_vote_type- if expected_vote_type == 'upvote':- assert response_data['votes'][url]['upvotes'] == 1- assert response_data['votes'][url]['downvotes'] == 0+ expected_vote_type = "upvote" if i % 3 == 0 else "downvote"+ assert response_data["votes"][url]["user_vote"] == expected_vote_type+ if expected_vote_type == "upvote":+ assert response_data["votes"][url]["upvotes"] == 1+ assert response_data["votes"][url]["downvotes"] == 0else:- assert response_data['votes'][url]['upvotes'] == 0- assert response_data['votes'][url]['downvotes'] == 1+ assert response_data["votes"][url]["upvotes"] == 0+ assert response_data["votes"][url]["downvotes"] == 1else: # URLs without votes- assert response_data['votes'][url]['upvotes'] == 0- assert response_data['votes'][url]['downvotes'] == 0- assert response_data['votes'][url]['user_vote'] is None-+ assert response_data["votes"][url]["upvotes"] == 0+ assert response_data["votes"][url]["downvotes"] == 0+ assert response_data["votes"][url]["user_vote"] is None+assert votes_with_data == 100 # Half of 200 URLs should have votes@@ -102,97 +85,79 @@ def test_very_long_urls_in_large_list(client, verified_user, access_token):"""Test handling URLs with very long paths that would be problematic in query strings"""base_urls = []query = "complex query with spaces and special chars !@#$%"-+# Create URLs with paths and query params that would be problematic in a GET query stringfor i in range(50):path = "/very/long/path" + "/segment" * 5 + f"/file{i}"query_params = "&".join([f"param{j}=value{j}" for j in range(5)])- url = f'https://example{i}.com{path}?{query_params}'+ url = f"https://example{i}.com{path}?{query_params}"base_urls.append(url)-+# Create votes for some URLsif i % 3 == 0:- SearchResultVote.objects.create(- user=verified_user,- url=url,- query=query,- vote_type='upvote'- )-+ SearchResultVote.objects.create(user=verified_user, url=url, query=query, vote_type="upvote")+# Add the original problematic URLs from the bug report- problematic_urls = [- 'http://keybase.io/hello',- 'http://keybase.io/hello_',- 'https://github.com/hello'- ]+ problematic_urls = ["http://keybase.io/hello", "http://keybase.io/hello_", "https://github.com/hello"]base_urls.extend(problematic_urls)-- vote_stats_request = {- "query": query,- "urls": base_urls- }-++ vote_stats_request = {"query": query, "urls": base_urls}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert len(response_data['votes']) == len(base_urls)-+ assert len(response_data["votes"]) == len(base_urls)+# Verify the original problematic URLs are handled correctlyfor url in problematic_urls:- assert url in response_data['votes']- assert response_data['votes'][url]['upvotes'] == 0- assert response_data['votes'][url]['downvotes'] == 0- assert response_data['votes'][url]['user_vote'] is None+ assert url in response_data["votes"]+ assert response_data["votes"][url]["upvotes"] == 0+ assert response_data["votes"][url]["downvotes"] == 0+ assert response_data["votes"][url]["user_vote"] is Nonedef test_edge_case_empty_and_malformed_urls(client, verified_user, access_token):"""Test edge cases with empty strings and malformed URLs in the list"""urls = [- 'https://valid-url.com/path',- '', # Empty string- 'https://another-valid.com/path',- 'not-a-valid-url', # Malformed URL- 'https://third-valid.com/path',+ "https://valid-url.com/path",+ "", # Empty string+ "https://another-valid.com/path",+ "not-a-valid-url", # Malformed URL+ "https://third-valid.com/path",]query = "edge case test"-+# Create vote for the first valid URLSearchResultVote.objects.create(- user=verified_user,- url='https://valid-url.com/path',- query=query,- vote_type='upvote'+ user=verified_user, url="https://valid-url.com/path", query=query, vote_type="upvote")-- vote_stats_request = {- "query": query,- "urls": urls- }-++ vote_stats_request = {"query": query, "urls": urls}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()# Should include all URLs (even empty and malformed ones)- assert len(response_data['votes']) == len(urls)-+ assert len(response_data["votes"]) == len(urls)+# Valid URL should have vote data- assert response_data['votes']['https://valid-url.com/path']['user_vote'] == 'upvote'-+ assert response_data["votes"]["https://valid-url.com/path"]["user_vote"] == "upvote"+# Empty and malformed URLs should have no votes- assert response_data['votes']['']['upvotes'] == 0- assert response_data['votes']['not-a-valid-url']['upvotes'] == 0+ assert response_data["votes"][""]["upvotes"] == 0+ assert response_data["votes"]["not-a-valid-url"]["upvotes"] == 0diff --git a/test/test_marketing_consent.py b/test/test_marketing_consent.pyindex 71f4094..c44bff9 100644--- a/test/test_marketing_consent.py+++ b/test/test_marketing_consent.py@@ -22,9 +22,7 @@ def verified_user(db):email="[email protected]",password="correctpassword",)- EmailAddress.objects.create(- user=user, email="[email protected]", verified=True, primary=True- )+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@@ -43,6 +41,7 @@ def _register(client, **extra):# Registration records consent# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_register_with_opt_in_records_consent(client):response = _register(client, source="API", marketing_opt_in=True)@@ -77,6 +76,7 @@ def test_register_without_source_records_nothing(client):# Authenticated view + update# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_get_marketing_consent_returns_latest_per_source(client, verified_user, access_token):MarketingConsent.objects.create(user=verified_user, source=MarketingSource.API, opted_in=True)@@ -135,6 +135,7 @@ def test_marketing_consent_requires_auth(client):# One-click unsubscribe (RFC 8058)# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_unsubscribe_token_roundtrip_records_opt_out(client, verified_user):token = make_unsubscribe_token(verified_user, MarketingSource.API)diff --git a/test/test_openapi.py b/test/test_openapi.pyindex d160b67..a1f5708 100644--- a/test/test_openapi.py+++ b/test/test_openapi.py@@ -1,8 +1,7 @@import pytest-import json-from django.test import Client-from django.contrib.auth import get_user_modelfrom allauth.account.models import EmailAddress+from django.contrib.auth import get_user_model+from django.test import Clientfrom ninja_jwt.tokens import RefreshTokenUser = get_user_model()@@ -11,22 +10,13 @@ User = get_user_model()@pytest.fixturedef user():"""Create a test user"""- return User.objects.create_user(- username='testuser',- email='[email protected]',- password='testpass123'- )+ return User.objects.create_user(username="testuser", email="[email protected]", password="testpass123")@pytest.fixturedef verified_user(user):"""Create a user with verified email"""- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=True,- primary=True- )+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@@ -43,36 +33,36 @@ def test_openapi_spec_generation():client = Client()# The unified docs are now at /api/v1/openapi.json (not per-router)- response = client.get('/api/v1/openapi.json')+ response = client.get("/api/v1/openapi.json")assert response.status_code == 200# Parse the JSON to ensure it's validopenapi_spec = response.json()# Basic OpenAPI spec validation- assert 'openapi' in openapi_spec- assert 'info' in openapi_spec- assert 'paths' in openapi_spec- assert 'components' in openapi_spec+ assert "openapi" in openapi_spec+ assert "info" in openapi_spec+ assert "paths" in openapi_spec+ assert "components" in openapi_spec# Check that our voting endpoints are present (mounted under /platform/)- paths = openapi_spec['paths']- assert '/api/v1/platform/search-results/vote' in paths- assert '/api/v1/platform/search-results/votes' in paths- assert '/api/v1/platform/search-results/my-votes' in paths+ paths = openapi_spec["paths"]+ assert "/api/v1/platform/search-results/vote" in paths+ assert "/api/v1/platform/search-results/votes" in paths+ assert "/api/v1/platform/search-results/my-votes" in paths# Check that POST endpoint for voting exists- vote_endpoint = paths['/api/v1/platform/search-results/vote']- assert 'post' in vote_endpoint- assert 'delete' in vote_endpoint+ vote_endpoint = paths["/api/v1/platform/search-results/vote"]+ assert "post" in vote_endpoint+ assert "delete" in vote_endpoint# Check that POST endpoint for vote counts exists (uses POST to handle large URL lists)- votes_endpoint = paths['/api/v1/platform/search-results/votes']- assert 'post' in votes_endpoint+ votes_endpoint = paths["/api/v1/platform/search-results/votes"]+ assert "post" in votes_endpoint# Check that GET endpoint for user vote history exists- my_votes_endpoint = paths['/api/v1/platform/search-results/my-votes']- assert 'get' in my_votes_endpoint+ my_votes_endpoint = paths["/api/v1/platform/search-results/my-votes"]+ assert "get" in my_votes_endpoint@pytest.mark.django_db@@ -81,9 +71,9 @@ def test_openapi_docs_page():client = Client()# The unified docs page is now at /api/v1/docs- response = client.get('/api/v1/docs')+ response = client.get("/api/v1/docs")assert response.status_code == 200- assert 'text/html' in response['Content-Type']+ assert "text/html" in response["Content-Type"]@pytest.mark.django_db@@ -91,39 +81,39 @@ def test_voting_schemas_in_openapi():"""Test that our voting schemas are properly defined in the unified OpenAPI spec"""client = Client()- response = client.get('/api/v1/openapi.json')+ response = client.get("/api/v1/openapi.json")assert response.status_code == 200openapi_spec = response.json()- components = openapi_spec.get('components', {})- schemas = components.get('schemas', {})+ components = openapi_spec.get("components", {})+ schemas = components.get("schemas", {})# Check that our voting schemas are present- assert 'VoteRequest' in schemas- assert 'VoteStats' in schemas- assert 'VoteResponse' in schemas- assert 'UserVoteHistory' in schemas- assert 'VoteRemoveRequest' in schemas+ assert "VoteRequest" in schemas+ assert "VoteStats" in schemas+ assert "VoteResponse" in schemas+ assert "UserVoteHistory" in schemas+ assert "VoteRemoveRequest" in schemas# Check VoteRequest schema structure- vote_request_schema = schemas['VoteRequest']- assert 'properties' in vote_request_schema- properties = vote_request_schema['properties']+ vote_request_schema = schemas["VoteRequest"]+ assert "properties" in vote_request_schema+ properties = vote_request_schema["properties"]- assert 'url' in properties- assert 'query' in properties- assert 'vote_type' in properties+ assert "url" in properties+ assert "query" in properties+ assert "vote_type" in properties# Check that vote_type has the correct enum values- vote_type_property = properties['vote_type']- assert 'enum' in vote_type_property- assert 'upvote' in vote_type_property['enum']- assert 'downvote' in vote_type_property['enum']+ vote_type_property = properties["vote_type"]+ assert "enum" in vote_type_property+ assert "upvote" in vote_type_property["enum"]+ assert "downvote" in vote_type_property["enum"]# Check that descriptions are present- assert 'description' in vote_type_property- assert 'description' in properties['url']- assert 'description' in properties['query']+ assert "description" in vote_type_property+ assert "description" in properties["url"]+ assert "description" in properties["query"]@pytest.mark.django_db@@ -131,39 +121,39 @@ def test_voting_endpoint_documentation():"""Test that voting endpoints have proper documentation in the unified OpenAPI spec"""client = Client()- response = client.get('/api/v1/openapi.json')+ response = client.get("/api/v1/openapi.json")assert response.status_code == 200openapi_spec = response.json()- paths = openapi_spec['paths']+ paths = openapi_spec["paths"]# Check POST /search-results/vote endpoint documentation- vote_post = paths['/api/v1/platform/search-results/vote']['post']- assert 'summary' in vote_post- assert 'description' in vote_post- assert 'tags' in vote_post- assert 'Search Result Voting' in vote_post['tags']+ vote_post = paths["/api/v1/platform/search-results/vote"]["post"]+ assert "summary" in vote_post+ assert "description" in vote_post+ assert "tags" in vote_post+ assert "Search Result Voting" in vote_post["tags"]# Check POST /search-results/votes endpoint documentation- votes_post = paths['/api/v1/platform/search-results/votes']['post']- assert 'summary' in votes_post- assert 'description' in votes_post- assert 'tags' in votes_post- assert 'Search Result Voting' in votes_post['tags']+ votes_post = paths["/api/v1/platform/search-results/votes"]["post"]+ assert "summary" in votes_post+ assert "description" in votes_post+ assert "tags" in votes_post+ assert "Search Result Voting" in votes_post["tags"]# Check DELETE /search-results/vote endpoint documentation- vote_delete = paths['/api/v1/platform/search-results/vote']['delete']- assert 'summary' in vote_delete- assert 'description' in vote_delete- assert 'tags' in vote_delete- assert 'Search Result Voting' in vote_delete['tags']+ vote_delete = paths["/api/v1/platform/search-results/vote"]["delete"]+ assert "summary" in vote_delete+ assert "description" in vote_delete+ assert "tags" in vote_delete+ assert "Search Result Voting" in vote_delete["tags"]# Check GET /search-results/my-votes endpoint documentation- my_votes_get = paths['/api/v1/platform/search-results/my-votes']['get']- assert 'summary' in my_votes_get- assert 'description' in my_votes_get- assert 'tags' in my_votes_get- assert 'Search Result Voting' in my_votes_get['tags']+ my_votes_get = paths["/api/v1/platform/search-results/my-votes"]["get"]+ assert "summary" in my_votes_get+ assert "description" in my_votes_get+ assert "tags" in my_votes_get+ assert "Search Result Voting" in my_votes_get["tags"]@pytest.mark.django_db@@ -171,30 +161,30 @@ def test_schema_examples_in_openapi():"""Test that our schemas have proper examples in the unified OpenAPI spec"""client = Client()- response = client.get('/api/v1/openapi.json')+ response = client.get("/api/v1/openapi.json")assert response.status_code == 200openapi_spec = response.json()- schemas = openapi_spec['components']['schemas']+ schemas = openapi_spec["components"]["schemas"]# Check VoteRequest schema has examples- vote_request = schemas['VoteRequest']- properties = vote_request['properties']+ vote_request = schemas["VoteRequest"]+ properties = vote_request["properties"]# Check that examples are present- assert 'example' in properties['url']- assert 'example' in properties['query']- assert 'example' in properties['vote_type']+ assert "example" in properties["url"]+ assert "example" in properties["query"]+ assert "example" in properties["vote_type"]# Check example values are reasonable- assert 'https://' in properties['url']['example']- assert properties['vote_type']['example'] in ['upvote', 'downvote']+ assert "https://" in properties["url"]["example"]+ assert properties["vote_type"]["example"] in ["upvote", "downvote"]# Check VoteStats schema has examples- vote_stats = schemas['VoteStats']- vote_stats_properties = vote_stats['properties']+ vote_stats = schemas["VoteStats"]+ vote_stats_properties = vote_stats["properties"]- assert 'example' in vote_stats_properties['upvotes']- assert 'example' in vote_stats_properties['downvotes']- assert isinstance(vote_stats_properties['upvotes']['example'], int)- assert isinstance(vote_stats_properties['downvotes']['example'], int)+ assert "example" in vote_stats_properties["upvotes"]+ assert "example" in vote_stats_properties["downvotes"]+ assert isinstance(vote_stats_properties["upvotes"]["example"], int)+ assert isinstance(vote_stats_properties["downvotes"]["example"], int)diff --git a/test/test_platform_billing.py b/test/test_platform_billing.pyindex afbb88d..e80470a 100644--- a/test/test_platform_billing.py+++ b/test/test_platform_billing.py@@ -104,6 +104,7 @@ def test_cancel_subscription_success(api_client, access_token, verified_user_witdef test_cancel_subscription_already_canceled(api_client, access_token, verified_user_with_billing):"""Test that canceling an already canceled subscription returns 409."""from unittest.mock import Mock+from polar_sdk.models import AlreadyCanceledSubscriptionbilling = UserBilling.objects.get(user=verified_user_with_billing)@@ -112,12 +113,12 @@ def test_cancel_subscription_already_canceled(api_client, access_token, verifiedwith patch("mwmbl.platform.api.Polar") as MockPolar:mock_polar_instance = MockPolar.return_value.__enter__.return_value-+# Create a mock responsemock_response = Mock()mock_response.status_code = 409mock_response.text = "Subscription is already canceled"-+# Create AlreadyCanceledSubscription with proper argumentsmock_data = Mock()mock_data.detail = "Subscription is already canceled"@@ -226,7 +227,9 @@ def test_uncancel_subscription_unauthenticated(api_client):def test_update_spend_limit_requires_subscription_first(api_client, access_token, db):"""Raising the spend limit above $0 without an active subscription returns 409."""user = User.objects.create_user(- username="nobilling", email="[email protected]", password="testpass123",+ username="nobilling",+ email="[email protected]",+ password="testpass123",)EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)token = str(RefreshToken.for_user(user).access_token)@@ -296,8 +299,10 @@ def test_checkout_uses_single_usage_product(api_client, access_token, verified_u"""Checkout always targets the single usage product and always sets external_customer_id."""from django.conf import settings- with patch.object(settings, "POLAR_PRODUCT_ID_USAGE", "prod_usage123"), \- patch("mwmbl.platform.api.Polar") as MockPolar:+ with (+ patch.object(settings, "POLAR_PRODUCT_ID_USAGE", "prod_usage123"),+ patch("mwmbl.platform.api.Polar") as MockPolar,+ ):mock_polar_instance = MockPolar.return_value.__enter__.return_valuemock_polar_instance.checkouts.create.return_value.url = "https://polar.example/checkout/abc"@@ -339,6 +344,7 @@ def test_checkout_not_configured_returns_503(api_client, access_token, verified_def _mock_webhook_event(event_type, user_id, **data_overrides):from unittest.mock import Mock+event = Mock()event.TYPE = event_typeevent.data = Mock()@@ -356,7 +362,9 @@ def test_webhook_subscription_active_sets_billing_fields_not_spend_limit(api_clibilling.max_monthly_spend_cents = 2_500billing.save()- event = _mock_webhook_event("subscription.active", verified_user_with_billing.id, customer_id="cust_abc", subscription_id="sub_abc")+ event = _mock_webhook_event(+ "subscription.active", verified_user_with_billing.id, customer_id="cust_abc", subscription_id="sub_abc"+ )with patch("mwmbl.platform.api.validate_event", return_value=event):response = api_client.post("/api/v1/platform/billing/webhook",diff --git a/test/test_process_tasks_worker.py b/test/test_process_tasks_worker.pyindex d16c2db..0463744 100644--- a/test/test_process_tasks_worker.py+++ b/test/test_process_tasks_worker.py@@ -6,6 +6,7 @@ and django-background-tasks' answer is `manage.py process_tasks`. Until this worexisted there was no way to deploy one from the app image, and every periodic task inproduction had sat pending, unattempted, for months."""+import pytestfrom background_task.models import Taskfrom background_task.tasks import tasks@@ -23,6 +24,7 @@ def clear_schedule_lock():yieldcache.delete(MwmblConfig._SCHEDULE_LOCK_KEY)+# The names apps.py schedules Task rows under.SCHEDULED_TASK_NAMES = ["mwmbl.background.sync_search_counts",@@ -85,6 +87,7 @@ def stop_after_two_runs(monkeypatch, recorded_commands):assert seconds == main.TASK_QUEUE_RESTART_SECONDSif recorded_commands.count("process_tasks") >= 2:raise StopLoop+monkeypatch.setattr(main, "sleep", fake_sleep)@@ -101,6 +104,7 @@ def test_the_queue_is_restarted_if_it_ever_exits(recorded_commands, stop_after_tclass FakeGunicorn:"""Stands in for BaseApplication so the server branch does not start a real server."""+runs = 0def __init__(self, *args, **kwargs):@@ -116,8 +120,7 @@ def fake_server(monkeypatch):monkeypatch.setattr(main, "BaseApplication", FakeGunicorn)spawned = []- monkeypatch.setattr(main.multiprocessing, "get_context",- lambda method: _FakeContext(method, spawned))+ monkeypatch.setattr(main.multiprocessing, "get_context", lambda method: _FakeContext(method, spawned))monkeypatch.setenv("MWMBL_APP", "server")return spawneddiff --git a/test/test_purge_blacklisted.py b/test/test_purge_blacklisted.pyindex 7bc6d3f..7bdec17 100644--- a/test/test_purge_blacklisted.py+++ b/test/test_purge_blacklisted.py@@ -3,14 +3,22 @@ import pytestfrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.indexer.index_batches import index_pages, index_results_against_query, preprocess_documentsfrom mwmbl.indexer.purge_blacklisted import pages_for_document, purge_documents-from mwmbl.tinysearchengine.indexer import Document, PAGE_SIZE, TinyIndex+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndexNUM_PAGES = 32-BAD = Document(title="Bad page about apples", url="https://badsite.test/apples",- extract="apples and oranges and bananas", score=1.0)-GOOD = Document(title="Good page about apples", url="https://example.test/apples",- extract="apples and oranges and bananas", score=1.0)+BAD = Document(+ title="Bad page about apples",+ url="https://badsite.test/apples",+ extract="apples and oranges and bananas",+ score=1.0,+)+GOOD = Document(+ title="Good page about apples",+ url="https://example.test/apples",+ extract="apples and oranges and bananas",+ score=1.0,+)@pytest.fixture@@ -25,14 +33,14 @@ def index(index_path, documents):def urls_in_index(index_path):- with TinyIndex(Document, index_path, 'r') as index:+ with TinyIndex(Document, index_path, "r") as index:return [d.url for page in range(NUM_PAGES) for d in index.get_page(page)]def test_purge_removes_a_document_from_every_page_it_is_filed_under(index_path):index(index_path, [BAD, GOOD])- with TinyIndex(Document, index_path, 'w') as tiny_index:+ with TinyIndex(Document, index_path, "w") as tiny_index:# The document must be filed under several terms, or this proves nothing.assert len(pages_for_document(tiny_index, BAD)) > 1removed = purge_documents(tiny_index, [BAD])@@ -48,7 +56,7 @@ def test_purge_leaves_other_documents_on_shared_pages_alone(index_path):otherwise it sweeps up documents it was never asked to touch."""index(index_path, [BAD, GOOD])- with TinyIndex(Document, index_path, 'w') as tiny_index:+ with TinyIndex(Document, index_path, "w") as tiny_index:shared = pages_for_document(tiny_index, BAD) & pages_for_document(tiny_index, GOOD)assert shared, "expected these documents to share at least one page"purge_documents(tiny_index, [BAD])@@ -63,7 +71,7 @@ def test_purge_re_checks_the_blacklist_before_removing(index_path):survive."""index(index_path, [BAD, GOOD])- with TinyIndex(Document, index_path, 'w') as tiny_index:+ with TinyIndex(Document, index_path, "w") as tiny_index:removed = purge_documents(tiny_index, [BAD], is_blacklisted=lambda domain: False)assert removed == {}@@ -73,9 +81,8 @@ def test_purge_re_checks_the_blacklist_before_removing(index_path):def test_purge_removes_only_the_documents_that_are_still_blacklisted(index_path):index(index_path, [BAD, GOOD])- with TinyIndex(Document, index_path, 'w') as tiny_index:- removed = purge_documents(tiny_index, [BAD, GOOD],- is_blacklisted=lambda domain: domain == "badsite.test")+ with TinyIndex(Document, index_path, "w") as tiny_index:+ removed = purge_documents(tiny_index, [BAD, GOOD], is_blacklisted=lambda domain: domain == "badsite.test")assert set(removed) == {"badsite.test"}urls = urls_in_index(index_path)@@ -87,7 +94,7 @@ def test_purge_of_an_absent_document_is_a_no_op(index_path):index(index_path, [GOOD])absent = Document(title="Nowhere", url="https://badsite.test/nowhere", extract="nothing", score=1.0)- with TinyIndex(Document, index_path, 'w') as tiny_index:+ with TinyIndex(Document, index_path, "w") as tiny_index:assert purge_documents(tiny_index, [absent]) == {}assert "https://example.test/apples" in urls_in_index(index_path)@@ -96,7 +103,7 @@ def test_purge_of_an_absent_document_is_a_no_op(index_path):def test_pages_for_document_matches_the_indexer(index_path):"""purge_documents relies on tokenize_document being a pure function of the storedfields, so that recomputing it recovers exactly the pages the indexer wrote to."""- with TinyIndex(Document, index_path, 'r') as tiny_index:+ with TinyIndex(Document, index_path, "r") as tiny_index:tokens = tokenize_document(BAD.url, BAD.title, BAD.extract, BAD.score).tokensexpected = {tiny_index.get_key_page_index(token) for token in tokens}assert pages_for_document(tiny_index, BAD) == expected@@ -111,24 +118,27 @@ def test_pages_for_document_matches_the_indexer(index_path):# tokenize_document() does not find those pages: without the term on the document, the# purge removes nothing and the next query re-queues it, forever.+def test_purge_removes_the_copy_filed_under_a_query_term(index_path):# index_results_against_query matches a query term against the document's *full* token# set, but a document is filed under only the first 10 tokens (plus bigrams). "zebra"# is past that cut, so it names a page that recomputing tokenize_document never finds.- wordy = Document(title="Bad page about apples", url="https://badsite.test/apples",- extract="apples and oranges and bananas here are many more words "- "including zebra", score=1.0)+ wordy = Document(+ title="Bad page about apples",+ url="https://badsite.test/apples",+ extract="apples and oranges and bananas here are many more words including zebra",+ score=1.0,+ )assert "zebra" not in tokenize_document(wordy.url, wordy.title, wordy.extract, wordy.score).tokensindex_results_against_query([wordy], "zebra", index_path)- with TinyIndex(Document, index_path, 'r') as tiny_index:+ with TinyIndex(Document, index_path, "r") as tiny_index:term_page = tiny_index.get_key_page_index("zebra")assert [d.url for d in tiny_index.get_page(term_page)] == [wordy.url]- queued = Document(title=wordy.title, url=wordy.url, extract=wordy.extract,- score=wordy.score, term="zebra")- with TinyIndex(Document, index_path, 'w') as tiny_index:+ queued = Document(title=wordy.title, url=wordy.url, extract=wordy.extract, score=wordy.score, term="zebra")+ with TinyIndex(Document, index_path, "w") as tiny_index:assert purge_documents(tiny_index, [queued]) == {"badsite.test": 1}assert wordy.url not in urls_in_index(index_path)@@ -137,7 +147,7 @@ def test_purge_removes_the_copy_filed_under_a_query_term(index_path):def test_a_document_with_no_term_still_purges_by_its_tokens(index_path):index(index_path, [BAD, GOOD])- with TinyIndex(Document, index_path, 'w') as tiny_index:+ with TinyIndex(Document, index_path, "w") as tiny_index:assert purge_documents(tiny_index, [BAD]) == {"badsite.test": 1}assert BAD.url not in urls_in_index(index_path)@@ -148,17 +158,16 @@ def test_removal_counts_are_per_url_not_per_page(index_path):order of magnitude more documents purged than there were."""index(index_path, [BAD])- with TinyIndex(Document, index_path, 'w') as tiny_index:+ with TinyIndex(Document, index_path, "w") as tiny_index:assert len(pages_for_document(tiny_index, BAD)) > 1assert purge_documents(tiny_index, [BAD]) == {"badsite.test": 1}def test_pages_for_document_includes_the_term_page(index_path):- with TinyIndex(Document, index_path, 'r') as tiny_index:+ with TinyIndex(Document, index_path, "r") as tiny_index:without_term = pages_for_document(tiny_index, BAD)with_term = pages_for_document(- tiny_index,- Document(title=BAD.title, url=BAD.url, extract=BAD.extract, score=BAD.score,- term="wibble"))+ tiny_index, Document(title=BAD.title, url=BAD.url, extract=BAD.extract, score=BAD.score, term="wibble")+ )assert with_term == without_term | {tiny_index.get_key_page_index("wibble")}diff --git a/test/test_rank.py b/test/test_rank.pyindex 928a094..0788341 100644--- a/test/test_rank.py+++ b/test/test_rank.py@@ -10,14 +10,19 @@ from urllib3.exceptions import MaxRetryError, ResponseErrorfrom mwmbl.tinysearchengine import rankfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.ltr_rank import LTRRanker-from mwmbl.tinysearchengine.rank import (NUM_WIKI_RESULTS, HeuristicAndWikiRanker,- HeuristicRanker, get_wiki_results, wiki_score)+from mwmbl.tinysearchengine.rank import (+ NUM_WIKI_RESULTS,+ HeuristicAndWikiRanker,+ HeuristicRanker,+ get_wiki_results,+ wiki_score,+)def test_order_result():- doc1 = Document(title='title2', url='https://something.com', extract='extract2', score=2.0)- doc2 = Document(title='title3', url='https://something.com', extract='extract3', score=3.0)- doc3 = Document(title='Bananas and apples', url='https://something.com', extract='extract1', score=1.0)+ doc1 = Document(title="title2", url="https://something.com", extract="extract2", score=2.0)+ doc2 = Document(title="title3", url="https://something.com", extract="extract3", score=3.0)+ doc3 = Document(title="Bananas and apples", url="https://something.com", extract="extract1", score=1.0)documents = [doc1, doc2, doc3]@@ -26,7 +31,7 @@ def test_order_result():# Sort the documentsordered_results = ranker.order_results(["bananas"], documents, True)- assert ordered_results[0].title == 'Bananas and apples'+ assert ordered_results[0].title == "Bananas and apples"class _TrackingRanker(HeuristicRanker):@@ -77,8 +82,7 @@ def _reset_wiki_circuit():def _get_wiki_results_with_session(session):# The cache is off in these tests: they are about what the fetch does when Wikipedia# misbehaves, and a hit would return before the fetch ever ran.- with override_settings(EXTERNAL_CACHE_ENABLED=False), \- patch.object(rank.requests, "Session") as mock_session:+ with override_settings(EXTERNAL_CACHE_ENABLED=False), patch.object(rank.requests, "Session") as mock_session:mock_session.return_value.__enter__.return_value = sessionreturn get_wiki_results("query", 5)@@ -117,8 +121,7 @@ def test_wiki_query_text_containing_429_is_not_mistaken_for_rate_limit():def test_open_wiki_circuit_short_circuits_without_calling_wikipedia():rank._trip_wiki_circuit()- with override_settings(EXTERNAL_CACHE_ENABLED=False), \- patch.object(rank.requests, "Session") as mock_session:+ with override_settings(EXTERNAL_CACHE_ENABLED=False), patch.object(rank.requests, "Session") as mock_session:results = get_wiki_results("query", 5)mock_session.assert_not_called()@@ -137,6 +140,7 @@ def test_is_wiki_rate_limited_ignores_non_429_reason():# Keeping the wiki results the model is served the ones it was trained on# ---------------------------------------------------------------------------+def test_the_wiki_score_depends_on_rank_alone():"""`score` is a feature of the LTR model. It used to be max_wiki_results + 1 - i, so acaller asking for a different number of results moved a feature value and with it thediff --git a/test/test_rank_blacklist.py b/test/test_rank_blacklist.pyindex a00caba..b7d1cb5 100644--- a/test/test_rank_blacklist.py+++ b/test/test_rank_blacklist.py@@ -1,5 +1,6 @@"""Retrieval-time blacklist filtering: the guarantee that a blacklisted domain is nevershown, plus the enqueueing that lets the background purge clean the index up afterwards."""+from unittest.mock import patchimport fakeredis@@ -14,10 +15,8 @@ from mwmbl.tinysearchengine.rank import HeuristicRankerBLACKLISTED_DOMAIN = "badsite.test"-BAD = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/bananas",- extract="bananas and apples", score=1.0)-GOOD = Document(title="Bananas", url="https://example.test/bananas",- extract="bananas and apples", score=1.0)+BAD = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/bananas", extract="bananas and apples", score=1.0)+GOOD = Document(title="Bananas", url="https://example.test/bananas", extract="bananas and apples", score=1.0)class FakeIndex:@@ -53,15 +52,16 @@ def redis_client():def blacklist(redis_client):"""A snapshot-free blacklist: the built-in rules alone are enough to decide here, sothese tests exercise the ranker rather than the snapshot machinery."""- return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({BLACKLISTED_DOMAIN}),- redis_client=redis_client)+ return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({BLACKLISTED_DOMAIN}), redis_client=redis_client)@pytest.fixturedef ranker(blacklist, redis_client, request):documents = getattr(request, "param", [BAD, GOOD])- with patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist), \- patch.object(purge_queue, "get_redis", return_value=redis_client):+ with (+ patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist),+ patch.object(purge_queue, "get_redis", return_value=redis_client),+ ):yield HeuristicRanker(FakeIndex(documents), FakeCompleter())@@ -129,12 +129,19 @@ def test_complete_does_not_suggest_blacklisted_urls(ranker):def test_curated_blacklisted_documents_are_dropped(blacklist, redis_client):"""Curated items bypass order_results entirely - they are prepended straight intodeduplicate() - so they need filtering separately from the ranked candidates."""- curated = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/curated",- extract="bananas", score=1.0, term="bananas",- state=DocumentState.ORGANIC_APPROVED)-- with patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist), \- patch.object(purge_queue, "get_redis", return_value=redis_client):+ curated = Document(+ title="Bananas",+ url=f"https://{BLACKLISTED_DOMAIN}/curated",+ extract="bananas",+ score=1.0,+ term="bananas",+ state=DocumentState.ORGANIC_APPROVED,+ )++ with (+ patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist),+ patch.object(purge_queue, "get_redis", return_value=redis_client),+ ):ranker = HeuristicRanker(FakeIndex([curated, GOOD]), FakeCompleter())urls = [result.url for result in ranker.search("bananas", [])]@@ -145,11 +152,18 @@ def test_curated_blacklisted_documents_are_dropped(blacklist, redis_client):def test_additional_results_are_filtered_but_not_queued(blacklist, redis_client):"""Caller-supplied results (the Google-sourced documents the search page passes in)must be filtered too, but there is nothing in our index to purge for them."""- additional = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/from-google",- extract="bananas", score=1.0, state=DocumentState.FROM_GOOGLE)-- with patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist), \- patch.object(purge_queue, "get_redis", return_value=redis_client):+ additional = Document(+ title="Bananas",+ url=f"https://{BLACKLISTED_DOMAIN}/from-google",+ extract="bananas",+ score=1.0,+ state=DocumentState.FROM_GOOGLE,+ )++ with (+ patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist),+ patch.object(purge_queue, "get_redis", return_value=redis_client),+ ):ranker = HeuristicRanker(FakeIndex([GOOD]), FakeCompleter())urls = [result.url for result in ranker.search("bananas", [additional])]@@ -167,12 +181,15 @@ def test_filtering_can_be_switched_off(ranker, settings):def test_a_redis_failure_does_not_break_search(blacklist):"""Enqueueing is best-effort: losing an entry just means the next query re-queues it,but a search response must never fail because of it."""+class BrokenRedis:def scard(self, key):raise ConnectionError("redis is down")- with patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist), \- patch.object(purge_queue, "get_redis", return_value=BrokenRedis()):+ with (+ patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist),+ patch.object(purge_queue, "get_redis", return_value=BrokenRedis()),+ ):ranker = HeuristicRanker(FakeIndex([BAD, GOOD]), FakeCompleter())urls = [result.url for result in ranker.search("bananas", [])]@@ -184,6 +201,7 @@ def test_a_failed_enqueue_is_retried_rather_than_suppressed(blacklist, redis_cli"""The recently-queued record exists to stop re-queueing what is already on the queue.Recording a document before the Redis write lands would suppress it for the whole TTLon the strength of an attempt that failed, stalling the purge for it."""+class BrokenRedis:def scard(self, key):raise ConnectionError("redis is down")@@ -201,15 +219,19 @@ def test_a_failed_enqueue_is_retried_rather_than_suppressed(blacklist, redis_clidef test_a_full_queue_does_not_suppress_the_next_attempt(blacklist, redis_client):- with patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist), \- patch.object(purge_queue, "get_redis", return_value=redis_client), \- patch.object(purge_queue, "MAX_QUEUE_SIZE", 0):+ with (+ patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist),+ patch.object(purge_queue, "get_redis", return_value=redis_client),+ patch.object(purge_queue, "MAX_QUEUE_SIZE", 0),+ ):ranker = HeuristicRanker(FakeIndex([BAD, GOOD]), FakeCompleter())ranker.search("bananas", [])assert redis_client.scard(PURGE_QUEUE_KEY) == 0- with patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist), \- patch.object(purge_queue, "get_redis", return_value=redis_client):+ with (+ patch("mwmbl.tinysearchengine.rank.get_snapshot_blacklist", return_value=blacklist),+ patch.object(purge_queue, "get_redis", return_value=redis_client),+ ):ranker.search("bananas", [])assert [d.url for d in drain_purge_queue(10, redis_client)] == [BAD.url]diff --git a/test/test_rankeval_gold.py b/test/test_rankeval_gold.pyindex 6578202..a3a5092 100644--- a/test/test_rankeval_gold.py+++ b/test/test_rankeval_gold.py@@ -5,10 +5,10 @@ and stored as consecutive rows under the one query - 1,135 of the 5,969 test queEvery harness that scores against it goes through gold_scores_for, so this is where the"one query, one ranking" rule has to hold."""+import pandas as pd-from mwmbl.rankeval.evaluation.evaluate import (- CLICK_PROPORTIONS, gold_scores_for, latest_ranking)+from mwmbl.rankeval.evaluation.evaluate import CLICK_PROPORTIONS, gold_scores_for, latest_rankingdef _rankings(rows):@@ -16,21 +16,25 @@ def _rankings(rows):def test_latest_ranking_keeps_only_the_most_recent_scrape():- rankings = _rankings([- ("piles", "https://old-first", 1, "2025-07-20"),- ("piles", "https://old-second", 2, "2025-07-20"),- ("piles", "https://new-first", 1, "2026-01-29"),- ("piles", "https://new-second", 2, "2026-01-29"),- ])+ rankings = _rankings(+ [+ ("piles", "https://old-first", 1, "2025-07-20"),+ ("piles", "https://old-second", 2, "2025-07-20"),+ ("piles", "https://new-first", 1, "2026-01-29"),+ ("piles", "https://new-second", 2, "2026-01-29"),+ ]+ )assert list(latest_ranking(rankings)["url"]) == ["https://new-first", "https://new-second"]def test_latest_ranking_sorts_by_rank():- rankings = _rankings([- ("piles", "https://second", 2, "2026-01-29"),- ("piles", "https://first", 1, "2026-01-29"),- ])+ rankings = _rankings(+ [+ ("piles", "https://second", 2, "2026-01-29"),+ ("piles", "https://first", 1, "2026-01-29"),+ ]+ )assert list(latest_ranking(rankings)["url"]) == ["https://first", "https://second"]@@ -38,11 +42,13 @@ def test_latest_ranking_sorts_by_rank():def test_gold_scores_ignore_an_earlier_scrape():# Without this the first ten rows would be the older scrape's, and a URL appearing in# both would take the weight of whichever row came second - the *worse* rank.- rankings = _rankings([- ("piles", "https://nhs", 5, "2025-07-20"),- ("piles", "https://nhs", 1, "2026-01-29"),- ("piles", "https://mayo", 2, "2026-01-29"),- ])+ rankings = _rankings(+ [+ ("piles", "https://nhs", 5, "2025-07-20"),+ ("piles", "https://nhs", 1, "2026-01-29"),+ ("piles", "https://mayo", 2, "2026-01-29"),+ ]+ )assert gold_scores_for(rankings) == {"https://nhs": CLICK_PROPORTIONS[0],@@ -51,11 +57,13 @@ def test_gold_scores_ignore_an_earlier_scrape():def test_gold_scores_drop_a_duplicate_url_keeping_the_best_rank():- rankings = _rankings([- ("piles", "https://nhs", 1, "2026-01-29"),- ("piles", "https://mayo", 2, "2026-01-29"),- ("piles", "https://nhs", 3, "2026-01-29"),- ])+ rankings = _rankings(+ [+ ("piles", "https://nhs", 1, "2026-01-29"),+ ("piles", "https://mayo", 2, "2026-01-29"),+ ("piles", "https://nhs", 3, "2026-01-29"),+ ]+ )assert gold_scores_for(rankings) == {"https://nhs": CLICK_PROPORTIONS[0],@@ -64,8 +72,7 @@ def test_gold_scores_drop_a_duplicate_url_keeping_the_best_rank():def test_gold_scores_take_at_most_ten_results():- rankings = _rankings([("piles", f"https://result-{i}", i, "2026-01-29")- for i in range(1, 16)])+ rankings = _rankings([("piles", f"https://result-{i}", i, "2026-01-29") for i in range(1, 16)])scores = gold_scores_for(rankings)assert len(scores) == len(CLICK_PROPORTIONS)diff --git a/test/test_retrieve_extract.py b/test/test_retrieve_extract.pyindex d075a67..5d1373d 100644--- a/test/test_retrieve_extract.py+++ b/test/test_retrieve_extract.py@@ -5,6 +5,7 @@ justext yields no 'good' content for JS-first pages and for non-English pagesfalls back, in order, to Open Graph tags, the meta description, and the firstsubstantive paragraph. These tests cover the helpers and that fallback chain."""+import fakeredisfrom mwmbl.crawler import retrieve@@ -20,9 +21,7 @@ def _dom(html: str):def test_get_meta_description():- dom = _dom('<html><head>'- '<meta name="Description" content=" A short summary. ">'- '</head><body></body></html>')+ dom = _dom('<html><head><meta name="Description" content=" A short summary. "></head><body></body></html>')assert get_meta_description(dom) == "A short summary."@@ -31,26 +30,28 @@ def test_get_meta_description_absent():def test_get_first_paragraph_returns_first_substantive():- dom = _dom("<html><body>"- "<p>Hi</p>" # too short, skipped- "<p>This is the first real paragraph of body content.</p>"- "<p>A later paragraph.</p>"- "</body></html>")+ dom = _dom(+ "<html><body>"+ "<p>Hi</p>" # too short, skipped+ "<p>This is the first real paragraph of body content.</p>"+ "<p>A later paragraph.</p>"+ "</body></html>"+ )assert get_first_paragraph(dom) == "This is the first real paragraph of body content."def test_get_first_paragraph_skips_link_heavy():- dom = _dom("<html><body>"- '<p><a href="/a">Home</a> <a href="/b">Products</a> <a href="/c">About us</a></p>'- "<p>The actual descriptive prose for this page lives here.</p>"- "</body></html>")+ dom = _dom(+ "<html><body>"+ '<p><a href="/a">Home</a> <a href="/b">Products</a> <a href="/c">About us</a></p>'+ "<p>The actual descriptive prose for this page lives here.</p>"+ "</body></html>"+ )assert get_first_paragraph(dom) == "The actual descriptive prose for this page lives here."def test_get_first_paragraph_collapses_whitespace_and_nested_tags():- dom = _dom("<html><body>"- "<p>From <b>Proto-Finnic</b>\n *kic'as. Cognate with Finnish.</p>"- "</body></html>")+ dom = _dom("<html><body><p>From <b>Proto-Finnic</b>\n *kic'as. Cognate with Finnish.</p></body></html>")assert get_first_paragraph(dom) == "From Proto-Finnic *kic'as. Cognate with Finnish."@@ -68,26 +69,32 @@ def _crawl_html(monkeypatch, html: str) -> dict:def test_crawl_url_falls_back_to_first_paragraph(monkeypatch):# No justext 'good' content (single short non-English line) and no meta tags,# so the extract must come from the first substantive paragraph.- html = ("<html><head><title>kitsas</title></head><body>"- "<p>kitsas (genitive kitsa, partitive kitsast, comparative kitsam)</p>"- "</body></html>")+ html = (+ "<html><head><title>kitsas</title></head><body>"+ "<p>kitsas (genitive kitsa, partitive kitsast, comparative kitsam)</p>"+ "</body></html>"+ )content = _crawl_html(monkeypatch, html)["content"]assert content is not Noneassert content["extract"].startswith("kitsas (genitive kitsa")def test_crawl_url_prefers_meta_description_over_first_paragraph(monkeypatch):- html = ('<html><head><title>t</title>'- '<meta name="description" content="The meta summary.">'- "</head><body><p>Some other first paragraph text here.</p></body></html>")+ html = (+ "<html><head><title>t</title>"+ '<meta name="description" content="The meta summary.">'+ "</head><body><p>Some other first paragraph text here.</p></body></html>"+ )content = _crawl_html(monkeypatch, html)["content"]assert content["extract"] == "The meta summary."def test_crawl_url_prefers_og_description(monkeypatch):- html = ('<html><head><title>t</title>'- '<meta property="og:description" content="The OG summary.">'- '<meta name="description" content="The meta summary.">'- "</head><body><p>Some other first paragraph text here.</p></body></html>")+ html = (+ "<html><head><title>t</title>"+ '<meta property="og:description" content="The OG summary.">'+ '<meta name="description" content="The meta summary.">'+ "</head><body><p>Some other first paragraph text here.</p></body></html>"+ )content = _crawl_html(monkeypatch, html)["content"]assert content["extract"] == "The OG summary."diff --git a/test/test_rust_features.py b/test/test_rust_features.pyindex c1064ab..cf1fabf 100644--- a/test/test_rust_features.py+++ b/test/test_rust_features.py@@ -7,7 +7,9 @@ These tests require the mwmbl_rank Rust extension to be built:Run with:pytest test/test_rust_features.py -v"""+import math+import pytest# Skip the entire module if the Rust extension is not built@@ -21,9 +23,7 @@ ATOL = 1e-4def rust_get_features(terms, title, url, extract, score, is_complete):"""Call the Rust get_features_py function."""- return mwmbl_rank.get_features_py(- list(terms), title, url, extract, float(score), is_complete- )+ return mwmbl_rank.get_features_py(list(terms), title, url, extract, float(score), is_complete)# ---------------------------------------------------------------------------@@ -115,21 +115,15 @@ def test_feature_parity(terms, title, url, extract, score, is_complete):assert len(rust_feats) == mwmbl_rank.NUM_FEATURES, (f"Expected {mwmbl_rank.NUM_FEATURES} features, got {len(rust_feats)}")- assert len(py_feats) == len(rust_feats), (- f"Feature count mismatch: Python={len(py_feats)}, Rust={len(rust_feats)}"- )+ assert len(py_feats) == len(rust_feats), f"Feature count mismatch: Python={len(py_feats)}, Rust={len(rust_feats)}"feature_names = mwmbl_rank.FEATURE_NAMESmismatches = []for i, (py_val, rust_val) in enumerate(zip(py_feats.values(), rust_feats)):if not math.isclose(float(py_val), float(rust_val), abs_tol=ATOL, rel_tol=1e-3):- mismatches.append(- f" [{i}] {feature_names[i]}: Python={py_val:.6f}, Rust={rust_val:.6f}"- )+ mismatches.append(f" [{i}] {feature_names[i]}: Python={py_val:.6f}, Rust={rust_val:.6f}")- assert not mismatches, (- f"Feature mismatches for query={terms!r}, url={url!r}:\n" + "\n".join(mismatches)- )+ assert not mismatches, f"Feature mismatches for query={terms!r}, url={url!r}:\n" + "\n".join(mismatches)def test_feature_names_match():@@ -139,11 +133,7 @@ def test_feature_names_match():py_feats = py_get_features(["test"], "Test", "https://example.com/", "Extract", 1.0, True)py_names = list(py_feats.keys())- assert rust_names == py_names, (- f"Feature name mismatch.\n"- f"Rust: {rust_names}\n"- f"Python: {py_names}"- )+ assert rust_names == py_names, f"Feature name mismatch.\nRust: {rust_names}\nPython: {py_names}"def test_num_features():@@ -157,16 +147,13 @@ def test_no_nan_in_features():rust_feats = rust_get_features(terms, title, url, extract, score, is_complete)for i, val in enumerate(rust_feats):assert not math.isnan(val), (- f"NaN at feature {i} ({mwmbl_rank.FEATURE_NAMES[i]}) "- f"for query={terms!r}, url={url!r}"+ f"NaN at feature {i} ({mwmbl_rank.FEATURE_NAMES[i]}) for query={terms!r}, url={url!r}")def test_domain_score_known_domain():"""paulgraham.com is in the HN top domains list; domain_score should be > 0."""- rust_feats = rust_get_features(- ["paul"], "Paul Graham", "https://paulgraham.com/articles.html", "", 1.0, True- )+ rust_feats = rust_get_features(["paul"], "Paul Graham", "https://paulgraham.com/articles.html", "", 1.0, True)feature_names = list(mwmbl_rank.FEATURE_NAMES)domain_score_idx = feature_names.index("domain_score")assert rust_feats[domain_score_idx] > 0.0@@ -175,9 +162,7 @@ def test_domain_score_known_domain():@pytest.mark.skip("TODO: this test is failing because there's a bug")def test_wiki_score_zero_for_non_wiki():"""A non-Wikipedia URL should have wiki_score = 0."""- rust_feats = rust_get_features(- ["test"], "Test", "https://example.com/test", "", 1.0, True- )+ rust_feats = rust_get_features(["test"], "Test", "https://example.com/test", "", 1.0, True)feature_names = list(mwmbl_rank.FEATURE_NAMES)wiki_score_idx = feature_names.index("wiki_score")assert rust_feats[wiki_score_idx] == 0.0diff --git a/test/test_rust_pipeline.py b/test/test_rust_pipeline.pyindex f6713f7..e3f78b2 100644--- a/test/test_rust_pipeline.py+++ b/test/test_rust_pipeline.py@@ -7,6 +7,7 @@ These tests require the mwmbl_rank Rust extension to be built:Run with:pytest test/test_rust_pipeline.py -v"""+import osimport tempfile@@ -19,11 +20,11 @@ pytest.importorskip("mwmbl_rank", reason="mwmbl_rank Rust extension not built")from mwmbl.tinysearchengine.ltr import RustXGBPipeline-# ---------------------------------------------------------------------------# Fixtures# ---------------------------------------------------------------------------+def make_dataframe(n: int = 30, seed: int = 42) -> pd.DataFrame:"""Create a synthetic DataFrame with the columns expected by RustXGBPipeline."""rng = np.random.RandomState(seed)@@ -53,13 +54,15 @@ def make_dataframe(n: int = 30, seed: int = 42) -> pd.DataFrame:rows = []for i in range(n):idx = i % len(queries)- rows.append({- "query": queries[idx],- "url": urls[idx],- "title": titles[idx],- "extract": extracts[idx],- "score": float(rng.uniform(0, 2)),- })+ rows.append(+ {+ "query": queries[idx],+ "url": urls[idx],+ "title": titles[idx],+ "extract": extracts[idx],+ "score": float(rng.uniform(0, 2)),+ }+ )return pd.DataFrame(rows)@@ -72,6 +75,7 @@ def make_labels(n: int = 30, seed: int = 42) -> np.ndarray:# Tests# ---------------------------------------------------------------------------+class TestRustXGBPipelineBasic:def test_instantiation(self):pipeline = RustXGBPipeline()@@ -152,14 +156,14 @@ class TestRustXGBPipelineFitPredict:def test_predict_with_null_title(self, trained_pipeline):pipeline, X, _ = trained_pipelineX_null = X.copy()- X_null.loc[0, 'title'] = None+ X_null.loc[0, "title"] = Nonepreds = pipeline.predict(X_null)assert not np.any(np.isnan(preds))def test_predict_with_null_extract(self, trained_pipeline):pipeline, X, _ = trained_pipelineX_null = X.copy()- X_null.loc[0, 'extract'] = None+ X_null.loc[0, "extract"] = Nonepreds = pipeline.predict(X_null)assert not np.any(np.isnan(preds))@@ -186,8 +190,7 @@ class TestRustXGBPipelinePersistence:loaded_preds = loaded.predict(X)np.testing.assert_allclose(- original_preds, loaded_preds, atol=1e-5,- err_msg="Predictions differ after save/load"+ original_preds, loaded_preds, atol=1e-5, err_msg="Predictions differ after save/load")finally:os.unlink(model_path)@@ -246,15 +249,15 @@ class TestRustXGBPipelineNDCG:# Use the devdata CSV if available, otherwise use synthetic datadevdata_path = "devdata/rankeval/learning-to-rank.csv"if os.path.exists(devdata_path):- dataset = pd.read_csv(devdata_path, lineterminator='\n')- dataset['title'] = dataset['title'].fillna('')- dataset['extract'] = dataset['extract'].fillna('')- X = dataset[['query', 'url', 'title', 'extract', 'score']]- y = dataset['gold_standard_rank'].fillna(0).astype(float)+ dataset = pd.read_csv(devdata_path, lineterminator="\n")+ dataset["title"] = dataset["title"].fillna("")+ dataset["extract"] = dataset["extract"].fillna("")+ X = dataset[["query", "url", "title", "extract", "score"]]+ y = dataset["gold_standard_rank"].fillna(0).astype(float)else:# Synthetic: create data where higher score → higher relevanceX = make_dataframe(50)- y = X['score'].values.astype(np.float32)+ y = X["score"].values.astype(np.float32)pipeline = RustXGBPipeline(num_rounds=50)pipeline.fit(X, y)diff --git a/test/test_search_api_key.py b/test/test_search_api_key.pyindex bf14531..8a29e9e 100644--- a/test/test_search_api_key.py+++ b/test/test_search_api_key.py@@ -9,22 +9,21 @@ Covers:- flush_search_counts background task"""-from datetime import datetime, timezone as stdlib_timezone+from datetime import datetime+from datetime import timezone as stdlib_timezonefrom unittest.mock import patchimport pytestfrom allauth.account.models import EmailAddress+from django.conf import settingsfrom django.contrib.auth import get_user_modelfrom django.core.cache import cachefrom django.test import Client-from django.utils import timezonefrom ninja_jwt.tokens import RefreshToken-from django.conf import settings-from mwmbl import pricingfrom mwmbl.background import sync_search_counts-from mwmbl.models import ApiKey, AgreementType, MwmblUser, UsageBucket, UserAgreement, UserBilling, generate_api_key+from mwmbl.models import AgreementType, ApiKey, UsageBucket, UserAgreement, UserBilling, generate_api_keyfrom mwmbl.quota import RATE_LIMIT, _monthly_key, check_rate_limit, get_monthly_count, increment_monthlyUser = get_user_model()@@ -34,6 +33,7 @@ User = get_user_model()# Fixtures# ---------------------------------------------------------------------------+@pytest.fixturedef verified_user(db):user = User.objects.create_user(@@ -130,6 +130,7 @@ def api_key_header(key):# Helper: get the unified API client# ---------------------------------------------------------------------------+@pytest.fixturedef api_client():"""Return a Django test client pointed at the v1 API."""@@ -140,6 +141,7 @@ def api_client():# API key management — POST /api/v1/platform/api-keys/# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_create_api_key(api_client, access_token):response = api_client.post(@@ -209,14 +211,10 @@ def test_create_crawl_api_key(api_client, access_token):@pytest.mark.django_dbdef test_create_crawl_api_key_without_gui_tos_returns_403(api_client, db):"""A user who has only accepted the API TOS cannot create a crawl key."""- user = User.objects.create_user(- username="apionly", email="[email protected]", password="testpass123"- )+ user = User.objects.create_user(username="apionly", email="[email protected]", password="testpass123")EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)version_id = settings.CURRENT_AGREEMENT_VERSIONS.get(AgreementType.TERMS_OF_SERVICE_API)- UserAgreement.objects.create(- user=user, agreement_type=AgreementType.TERMS_OF_SERVICE_API, version_id=version_id- )+ UserAgreement.objects.create(user=user, agreement_type=AgreementType.TERMS_OF_SERVICE_API, version_id=version_id)token = str(RefreshToken.for_user(user).access_token)response = api_client.post("/api/v1/platform/api-keys/",@@ -231,6 +229,7 @@ def test_create_crawl_api_key_without_gui_tos_returns_403(api_client, db):# API key management — GET /api/v1/platform/api-keys/# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_list_api_keys_hides_raw_key(api_client, access_token, search_api_key):response = api_client.get(@@ -270,6 +269,7 @@ def test_list_api_keys_unauthenticated(api_client):# API key management — DELETE /api/v1/platform/api-keys/{id}# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_delete_api_key(api_client, access_token, search_api_key):key_id = search_api_key.id@@ -315,6 +315,7 @@ def test_delete_crawl_api_key(api_client, access_token, crawl_api_key):# Search endpoint — authentication# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_search_missing_key_returns_results_with_null_usage(api_client):"""Unauthenticated requests are allowed; usage fields are null."""@@ -345,11 +346,14 @@ def test_search_crawl_scoped_key_rejected(api_client, crawl_api_key):# Search endpoint — successful request with usage metadata# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_search_with_valid_key(api_client, search_api_key):- with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \- patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=0), \- patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=1):+ with (+ patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True),+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=0),+ patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=1),+ ):response = api_client.get("/api/v2/search/?q=python",**api_key_header(search_api_key.raw_key),@@ -360,10 +364,12 @@ def test_search_with_valid_key(api_client, search_api_key):@pytest.mark.django_dbdef test_search_response_includes_usage_fields(api_client, search_api_key):"""When quota helpers are mocked, the response includes monthly_usage and monthly_limit."""- with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \- patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=5), \- patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=6), \- patch("mwmbl.tinysearchengine.rank.HeuristicRanker.search", return_value=[]):+ with (+ patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True),+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=5),+ patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=6),+ patch("mwmbl.tinysearchengine.rank.HeuristicRanker.search", return_value=[]),+ ):response = api_client.get("/api/v2/search/?q=python",**api_key_header(search_api_key.raw_key),@@ -379,6 +385,7 @@ def test_search_response_includes_usage_fields(api_client, search_api_key):# Search endpoint — quota enforcement# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_search_rate_limit_exceeded(api_client, search_api_key):with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=False):@@ -393,8 +400,10 @@ def test_search_rate_limit_exceeded(api_client, search_api_key):@pytest.mark.django_dbdef test_search_monthly_quota_exceeded(api_client, search_api_key):limit = pricing.FREE_KEYED_MONTHLY_LIMIT- with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \- patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=limit):+ with (+ patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True),+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=limit),+ ):response = api_client.get("/api/v2/search/?q=python",**api_key_header(search_api_key.raw_key),@@ -408,9 +417,11 @@ def test_search_monthly_quota_exceeded(api_client, search_api_key):@pytest.mark.django_dbdef test_search_keyed_default_zero_spend_cap_is_free_allowance(api_client, search_api_key):"""A fresh account with no UserBilling row defaults to the free-only cap."""- with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \- patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=pricing.FREE_KEYED_MONTHLY_LIMIT - 1), \- patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=pricing.FREE_KEYED_MONTHLY_LIMIT):+ with (+ patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True),+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=pricing.FREE_KEYED_MONTHLY_LIMIT - 1),+ patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=pricing.FREE_KEYED_MONTHLY_LIMIT),+ ):response = api_client.get("/api/v2/search/?q=python",**api_key_header(search_api_key.raw_key),@@ -426,16 +437,20 @@ def test_search_keyed_with_spend_limit_raises_cap(api_client, search_api_key, vecap = pricing.effective_monthly_request_cap(1_000)assert cap == pricing.FREE_KEYED_MONTHLY_LIMIT + 2_000- with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \- patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=cap - 1):+ with (+ patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True),+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=cap - 1),+ ):response = api_client.get("/api/v2/search/?q=python",**api_key_header(search_api_key.raw_key),)assert response.status_code == 200- with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \- patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=cap):+ with (+ patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True),+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=cap),+ ):response = api_client.get("/api/v2/search/?q=python",**api_key_header(search_api_key.raw_key),@@ -447,6 +462,7 @@ def test_search_keyed_with_spend_limit_raises_cap(api_client, search_api_key, ve# Crawler /results — header vs body key and scope enforcement# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_post_results_no_key_returns_401(api_client):response = api_client.post(@@ -459,9 +475,11 @@ def test_post_results_no_key_returns_401(api_client):@pytest.mark.django_dbdef test_post_results_header_key_accepted(api_client, crawl_api_key):- with patch("mwmbl.crawler.app.index_documents"), \- patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"), \- patch("mwmbl.crawler.app.stats_manager"):+ with (+ patch("mwmbl.crawler.app.index_documents"),+ patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"),+ patch("mwmbl.crawler.app.stats_manager"),+ ):response = api_client.post("/api/v1/crawler/results",content_type="application/json",@@ -474,9 +492,11 @@ def test_post_results_header_key_accepted(api_client, crawl_api_key):@pytest.mark.django_dbdef test_post_results_body_key_deprecated_still_works(api_client, crawl_api_key):"""Body api_key field is deprecated but must still work for backward compatibility."""- with patch("mwmbl.crawler.app.index_documents"), \- patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"), \- patch("mwmbl.crawler.app.stats_manager"):+ with (+ patch("mwmbl.crawler.app.index_documents"),+ patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"),+ patch("mwmbl.crawler.app.stats_manager"),+ ):response = api_client.post("/api/v1/crawler/results",content_type="application/json",@@ -500,9 +520,11 @@ def test_post_results_search_scoped_key_rejected(api_client, search_api_key):@pytest.mark.django_dbdef test_post_results_header_takes_precedence_over_body(api_client, crawl_api_key, search_api_key):"""When both header and body key are present, the header key is used."""- with patch("mwmbl.crawler.app.index_documents"), \- patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"), \- patch("mwmbl.crawler.app.stats_manager"):+ with (+ patch("mwmbl.crawler.app.index_documents"),+ patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"),+ patch("mwmbl.crawler.app.stats_manager"),+ ):response = api_client.post("/api/v1/crawler/results",content_type="application/json",@@ -530,9 +552,11 @@ def test_post_results_uses_submitted_last_crawled(api_client, crawl_api_key):past_ts = int(datetime.now(stdlib_timezone.utc).timestamp()) - 60captured = []- with patch("mwmbl.crawler.app.index_documents", side_effect=lambda docs, path: captured.extend(docs)), \- patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"), \- patch("mwmbl.crawler.app.stats_manager"):+ with (+ patch("mwmbl.crawler.app.index_documents", side_effect=lambda docs, path: captured.extend(docs)),+ patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"),+ patch("mwmbl.crawler.app.stats_manager"),+ ):response = api_client.post("/api/v1/crawler/results",content_type="application/json",@@ -547,9 +571,11 @@ def test_post_results_uses_submitted_last_crawled(api_client, crawl_api_key):def test_post_results_sets_user_id(api_client, crawl_api_key, verified_user):captured = []- with patch("mwmbl.crawler.app.index_documents", side_effect=lambda docs, path: captured.extend(docs)), \- patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"), \- patch("mwmbl.crawler.app.stats_manager"):+ with (+ patch("mwmbl.crawler.app.index_documents", side_effect=lambda docs, path: captured.extend(docs)),+ patch("mwmbl.crawler.app.upload_object", return_value="fake/path.json.gz"),+ patch("mwmbl.crawler.app.stats_manager"),+ ):response = api_client.post("/api/v1/crawler/results",content_type="application/json",@@ -564,6 +590,7 @@ def test_post_results_sets_user_id(api_client, crawl_api_key, verified_user):# Background tasks# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_sync_search_counts_redis_to_postgres(verified_user):"""sync_search_counts should write live Redis counts into UsageBucket."""@@ -643,6 +670,7 @@ def test_sync_search_counts_keeps_redis_value_when_higher(verified_user):# Subscription endpoint — GET /api/v1/platform/billing/subscription# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_subscription_unauthenticated(api_client):response = api_client.get("/api/v1/platform/billing/subscription")@@ -729,6 +757,7 @@ def test_subscription_reflects_spend_limit(api_client, access_token, verified_us# Quota helper unit tests# ---------------------------------------------------------------------------+def test_rate_limit_allows_up_to_limit():user_id = 99999diff --git a/test/test_ssrf.py b/test/test_ssrf.pyindex ac4b31e..46d2edc 100644--- a/test/test_ssrf.py+++ b/test/test_ssrf.py@@ -1,4 +1,5 @@"""Tests for the SSRF guard and its wiring into fetch() and add_url()."""+import socketimport pytest@@ -17,17 +18,21 @@ User = get_user_model()# Unit: is_safe_host / validate_url# ---------------------------------------------------------------------------[email protected]("host", [- "127.0.0.1",- "127.0.0.2",- "0.0.0.0",- "10.0.0.1",- "192.168.1.1",- "172.16.0.1",- "169.254.169.254", # cloud metadata endpoint- "::1",- "::ffff:127.0.0.1", # IPv4-mapped loopback-])++ "host",+ [+ "127.0.0.1",+ "127.0.0.2",+ "0.0.0.0",+ "10.0.0.1",+ "192.168.1.1",+ "172.16.0.1",+ "169.254.169.254", # cloud metadata endpoint+ "::1",+ "::ffff:127.0.0.1", # IPv4-mapped loopback+ ],+)def test_internal_ip_literals_are_unsafe(host):assert is_safe_host(host) is False@@ -38,7 +43,8 @@ def test_public_ip_literal_is_safe():def test_localhost_resolving_to_loopback_is_unsafe(monkeypatch):monkeypatch.setattr(- socket, "getaddrinfo",+ socket,+ "getaddrinfo",lambda *a, **k: [(socket.AF_INET, None, None, "", ("127.0.0.1", 0))],)assert is_safe_host("localhost") is False@@ -46,7 +52,8 @@ def test_localhost_resolving_to_loopback_is_unsafe(monkeypatch):def test_hostname_resolving_to_private_ip_is_unsafe(monkeypatch):monkeypatch.setattr(- socket, "getaddrinfo",+ socket,+ "getaddrinfo",lambda *a, **k: [(socket.AF_INET, None, None, "", ("10.1.2.3", 0))],)assert is_safe_host("internal.corp") is False@@ -54,7 +61,8 @@ def test_hostname_resolving_to_private_ip_is_unsafe(monkeypatch):def test_hostname_resolving_to_public_ip_is_safe(monkeypatch):monkeypatch.setattr(- socket, "getaddrinfo",+ socket,+ "getaddrinfo",lambda *a, **k: [(socket.AF_INET, None, None, "", ("93.184.216.34", 0))],)assert is_safe_host("example.com") is True@@ -63,6 +71,7 @@ def test_hostname_resolving_to_public_ip_is_safe(monkeypatch):def test_resolution_failure_is_unsafe(monkeypatch):def boom(*a, **k):raise socket.gaierror("nope")+monkeypatch.setattr(socket, "getaddrinfo", boom)assert is_safe_host("does-not-resolve.invalid") is False@@ -83,6 +92,7 @@ def test_validate_url_rejects_internal():# fetch() integration# ---------------------------------------------------------------------------+class _FakeResponse:def __init__(self, status_code, headers=None, is_redirect=False, body=b"hello"):self.status_code = status_code@@ -100,8 +110,7 @@ class _FakeResponse:def test_fetch_blocks_internal_url_without_request(monkeypatch):calls = []- monkeypatch.setattr(retrieve.requests, "get",- lambda *a, **k: calls.append(a) or _FakeResponse(200))+ monkeypatch.setattr(retrieve.requests, "get", lambda *a, **k: calls.append(a) or _FakeResponse(200))with pytest.raises(UnsafeURLError):retrieve.fetch("http://127.0.0.1/secret")assert calls == [], "requests.get must not be called for an internal URL"@@ -122,8 +131,7 @@ def test_fetch_blocks_redirect_to_internal(monkeypatch):def test_fetch_returns_content_for_public_url(monkeypatch):- monkeypatch.setattr(retrieve.requests, "get",- lambda *a, **k: _FakeResponse(200, body=b"page body"))+ monkeypatch.setattr(retrieve.requests, "get", lambda *a, **k: _FakeResponse(200, body=b"page body"))status, content, resolved_url = retrieve.fetch("http://93.184.216.34/page")assert status == 200assert content == b"page body"@@ -133,10 +141,10 @@ def test_fetch_returns_content_for_public_url(monkeypatch):def test_fetch_reports_the_host_that_answered_not_the_one_we_asked(monkeypatch):"""The last hop is the only place a redirect is visible to a caller. Moderation rejectsdomains that redirect elsewhere, and cannot see one if fetch reports the requested URL."""+def fake_get(url, **kwargs):if url == "http://93.184.216.34/start":- return _FakeResponse(301, headers={"Location": "http://93.184.216.35/parked"},- is_redirect=True)+ return _FakeResponse(301, headers={"Location": "http://93.184.216.35/parked"}, is_redirect=True)return _FakeResponse(200, body=b"parking page")monkeypatch.setattr(retrieve.requests, "get", fake_get)@@ -149,6 +157,7 @@ def test_fetch_reports_the_host_that_answered_not_the_one_we_asked(monkeypatch):# add_url view# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_add_url_rejects_internal_url(monkeypatch):user = User.objects.create_user(username="ssrfuser", email="[email protected]", password="x")@@ -157,6 +166,7 @@ def test_add_url_rejects_internal_url(monkeypatch):# Fail loudly if the view ever reaches the network for an internal URL.def boom(*a, **k):raise AssertionError("requests.get must not be called for an internal URL")+monkeypatch.setattr("mwmbl.views.requests.get", boom)client = Client()diff --git a/test/test_stats_blacklisted_removed.py b/test/test_stats_blacklisted_removed.pyindex c8b5714..c489db2 100644--- a/test/test_stats_blacklisted_removed.py+++ b/test/test_stats_blacklisted_removed.py@@ -5,6 +5,7 @@ Retrieval filters blacklisted domains out of results whether or not the backgrouever removes them, so a broken purge loop is invisible from the search results alone.This counter is the only signal that the removals are actually happening."""+from datetime import datetimefrom unittest.mock import patch@@ -58,13 +59,16 @@ def test_the_purge_task_records_what_it_removed():redis = fakeredis.FakeRedis(decode_responses=True)queued = [object()]- with patch("mwmbl.background.drain_purge_queue", return_value=queued), \- patch("mwmbl.background.get_snapshot_blacklist"), \- patch("mwmbl.background.TinyIndex"), \- patch("mwmbl.background.queue_size", return_value=0), \- patch("mwmbl.background.purge_documents", return_value={"bad.test": 2, "worse.test": 3}), \- patch("mwmbl.background.stats_manager", StatsManager(redis)):+ with (+ patch("mwmbl.background.drain_purge_queue", return_value=queued),+ patch("mwmbl.background.get_snapshot_blacklist"),+ patch("mwmbl.background.TinyIndex"),+ patch("mwmbl.background.queue_size", return_value=0),+ patch("mwmbl.background.purge_documents", return_value={"bad.test": 2, "worse.test": 3}),+ patch("mwmbl.background.stats_manager", StatsManager(redis)),+ ):from mwmbl.background import purge_blacklisted_from_queue+purge_blacklisted_from_queue.now()key = BLACKLISTED_REMOVED_COUNT_KEY.format(date=datetime.utcnow().date())@@ -74,9 +78,12 @@ def test_the_purge_task_records_what_it_removed():def test_the_purge_task_records_nothing_when_the_queue_is_empty():redis = fakeredis.FakeRedis(decode_responses=True)- with patch("mwmbl.background.drain_purge_queue", return_value=[]), \- patch("mwmbl.background.stats_manager", StatsManager(redis)):+ with (+ patch("mwmbl.background.drain_purge_queue", return_value=[]),+ patch("mwmbl.background.stats_manager", StatsManager(redis)),+ ):from mwmbl.background import purge_blacklisted_from_queue+purge_blacklisted_from_queue.now()assert redis.get(BLACKLISTED_REMOVED_COUNT_KEY.format(date=datetime.utcnow().date())) is Nonediff --git a/test/test_super_search.py b/test/test_super_search.pyindex 1844a64..72e615b 100644--- a/test/test_super_search.py+++ b/test/test_super_search.py@@ -6,10 +6,10 @@ These tests:- Verify quota, auth, event ordering, and forbidden-symbol presence inthe orchestrator (architecture guard)."""+import asyncioimport jsonfrom pathlib import Path-from unittest.mock import patchfrom urllib.parse import urlparseimport pytest@@ -31,6 +31,7 @@ User = get_user_model()# Fixtures# ---------------------------------------------------------------------------+@pytest.fixturedef user(db):u = User.objects.create_user(username="ssuser", email="[email protected]", password="x")@@ -66,9 +67,9 @@ def _parse_sse(body: bytes) -> list[tuple[str, dict]]:data = Nonefor line in chunk.split(b"\n"):if line.startswith(b"event: "):- event_type = line[len(b"event: "):].decode()+ event_type = line[len(b"event: ") :].decode()elif line.startswith(b"data: "):- data = json.loads(line[len(b"data: "):].decode())+ data = json.loads(line[len(b"data: ") :].decode())if event_type is not None:events.append((event_type, data))return events@@ -77,8 +78,10 @@ def _parse_sse(body: bytes) -> list[tuple[str, dict]]:def _read_stream(response) -> bytes:content = response.streaming_contentif hasattr(content, "__aiter__"):+async def _collect():return b"".join([chunk async for chunk in content])+return async_to_sync(_collect)()return b"".join(content)@@ -89,8 +92,10 @@ def _stub_sources(monkeypatch, by_source: dict[str, list[Document]]):new_sources = {}for name, docs in by_source.items():+async def fake_search(client, query, limit, _docs=docs):return _docs+new_sources[name] = fake_searchmonkeypatch.setattr(ss, "SOURCES", new_sources)@@ -117,6 +122,7 @@ def _stub_scoring(monkeypatch, scores: list[float]):# Auth & quota# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_super_search_requires_auth(client):response = client.get("/api/v2/super-search/?q=python")@@ -165,18 +171,22 @@ def test_super_search_quota_enforced(client, api_key, monkeypatch):# Event-stream behaviour# ---------------------------------------------------------------------------+@pytest.mark.django_db@override_settings(SUPER_SEARCH_TOP_K=2)def test_promoted_results_use_top_k(client, api_key, monkeypatch):"""Exactly top-K docs are promoted for crawling; surplus docs are excluded."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [- Document(title="Best", url="https://best.example/", extract="x"),- Document(title="Second", url="https://second.example/", extract="x"),- Document(title="Third", url="https://third.example/", extract="x"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="Best", url="https://best.example/", extract="x"),+ Document(title="Second", url="https://second.example/", extract="x"),+ Document(title="Third", url="https://third.example/", extract="x"),+ ],+ },+ )# Scores in descending order — only the top-2 should be promoted._stub_scoring(monkeypatch, [0.9, 0.5, 0.1] + [0.0] * 20)@@ -210,14 +220,17 @@ def test_heap_replacement_promotes_better_late_doc(client, api_key, monkeypatch)"""When the heap is full, a later doc with a higher score replaces the minimum andis still promoted — the heap min-replacement path must work."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [- Document(title="Low A", url="https://low-a.example/", extract="x"),- Document(title="Low B", url="https://low-b.example/", extract="x"),- Document(title="Low C", url="https://low-c.example/", extract="x"),- Document(title="Best", url="https://best.example/", extract="x"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="Low A", url="https://low-a.example/", extract="x"),+ Document(title="Low B", url="https://low-b.example/", extract="x"),+ Document(title="Low C", url="https://low-c.example/", extract="x"),+ Document(title="Best", url="https://best.example/", extract="x"),+ ],+ },+ )# First two fill the heap at 0.1; Low C can't enter (0.1 > 0.1 is False);# Best (0.9) beats the minimum and must be promoted._stub_scoring(monkeypatch, [0.1, 0.1, 0.1, 0.9] + [0.0] * 20)@@ -245,13 +258,16 @@ def test_heap_equal_score_does_not_enter_full_heap(client, api_key, monkeypatch)"""A doc whose score equals the heap minimum must NOT be promoted — the check isstrictly greater-than, so ties don't displace existing entries."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [- Document(title="First", url="https://first.example/", extract="x"),- Document(title="Second", url="https://second.example/", extract="x"),- Document(title="Third", url="https://third.example/", extract="x"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="First", url="https://first.example/", extract="x"),+ Document(title="Second", url="https://second.example/", extract="x"),+ Document(title="Third", url="https://third.example/", extract="x"),+ ],+ },+ )_stub_scoring(monkeypatch, [0.5, 0.5, 0.5] + [0.0] * 20)def fake_crawl(url, redis):@@ -275,9 +291,12 @@ def test_heap_equal_score_does_not_enter_full_heap(client, api_key, monkeypatch)def test_final_results_event_emitted(client, api_key, monkeypatch):"""A 'results' event with the full ranked list is emitted before 'done'."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [Document(title="Python intro", url="https://py.example/", extract="A guide")],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [Document(title="Python intro", url="https://py.example/", extract="A guide")],+ },+ )_stub_scoring(monkeypatch, [0.5] + [0.0] * 20)def fake_crawl(url, redis):@@ -312,16 +331,16 @@ def test_final_results_are_mmr_diversified(client, api_key, monkeypatch):cache.delete(_super_search_monthly_key(api_key.user.id))# Relevance order a > b > c. a and b share a domain (and text); c is a fresh# domain. MMR should lift c above the second same-domain result b.- _stub_sources(monkeypatch, {- "hn": [- Document(title="Python alpha repo", url="https://github.com/x/alpha",- extract="python alpha project"),- Document(title="Python beta repo", url="https://github.com/x/beta",- extract="python beta project"),- Document(title="Python gamma site", url="https://example.org/gamma",- extract="python gamma guide"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="Python alpha repo", url="https://github.com/x/alpha", extract="python alpha project"),+ Document(title="Python beta repo", url="https://github.com/x/beta", extract="python beta project"),+ Document(title="Python gamma site", url="https://example.org/gamma", extract="python gamma guide"),+ ],+ },+ )# Promotion scores, then descending final-ranking scores (consumed once per# _emit_final_results call) all in the same a > b > c order._stub_scoring(monkeypatch, [0.9, 0.8, 0.7] * 4 + [0.0] * 30)@@ -354,9 +373,12 @@ def test_done_reports_pages_indexed(client, api_key, monkeypatch):import mwmbl.tinysearchengine.super_search as sscache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [Document(title="Python intro", url="https://py.example/", extract="A guide")],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [Document(title="Python intro", url="https://py.example/", extract="A guide")],+ },+ )_stub_scoring(monkeypatch, [0.5] + [0.0] * 20)monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url",@@ -453,21 +475,36 @@ def test_link_following_adds_followed_docs(client, api_key, monkeypatch):"""A promoted page is crawled, its outbound links followed, and the followeddocs appear in the final ranking — exercises page_fetched / link_followed."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [Document(title="Python parent", url="https://parent.example/",- extract="python parent text")],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [Document(title="Python parent", url="https://parent.example/", extract="python parent text")],+ },+ )_stub_scoring(monkeypatch, [0.9] + [0.0] * 20)def fake_crawl(url, redis):if url == "https://parent.example/":- return {"url": url, "status": 200, "content": {- "title": "Python parent", "extract": "python parent text",- "links": ["https://child.example/python-guide"], "extra_links": [],- }}- return {"url": url, "status": 200, "content": {- "title": "Python child", "extract": "about python", "links": [], "extra_links": [],- }}+ return {+ "url": url,+ "status": 200,+ "content": {+ "title": "Python parent",+ "extract": "python parent text",+ "links": ["https://child.example/python-guide"],+ "extra_links": [],+ },+ }+ return {+ "url": url,+ "status": 200,+ "content": {+ "title": "Python child",+ "extract": "about python",+ "links": [],+ "extra_links": [],+ },+ }monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -481,8 +518,9 @@ def test_link_following_adds_followed_docs(client, api_key, monkeypatch):assert "page_fetched" in typesfollowed = [d for t, d in events if t == "link_followed"]- assert any(d["url"] == "https://child.example/python-guide" and d["from"] == "https://parent.example/"- for d in followed)+ assert any(+ d["url"] == "https://child.example/python-guide" and d["from"] == "https://parent.example/" for d in followed+ )final = [d for t, d in events if t == "results"][-1]["results"]final_urls = {r["url"] for r in final}@@ -494,12 +532,15 @@ def test_link_following_adds_followed_docs(client, api_key, monkeypatch):def test_no_duplicate_consecutive_results_frames(client, api_key, monkeypatch):"""The dedup guard must never emit two consecutive identical `results` frames."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [- Document(title="Python one", url="https://one.example/", extract="python one"),- Document(title="Python two", url="https://two.example/", extract="python two"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="Python one", url="https://one.example/", extract="python one"),+ Document(title="Python two", url="https://two.example/", extract="python two"),+ ],+ },+ )_stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 40)def fake_crawl(url, redis):@@ -526,12 +567,15 @@ def test_title_only_source_result_retained(client, api_key, monkeypatch):(e.g. "olearia macrodonta", where two relevant pages have empty extracts)."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "mwmbl": [- Document(title="Python guide", url="https://py.example/", extract="A guide"),- Document(title="Python reference", url="https://noextract.example/", extract=""),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "mwmbl": [+ Document(title="Python guide", url="https://py.example/", extract="A guide"),+ Document(title="Python reference", url="https://noextract.example/", extract=""),+ ],+ },+ )_stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 40)def fake_crawl(url, redis):@@ -554,21 +598,36 @@ def test_followed_link_without_title_excluded(client, api_key, monkeypatch):(its only title would be URL-derived) and must not enter the final ranking,even though its extract matches the query."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [Document(title="Python parent", url="https://parent.example/",- extract="python parent text")],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [Document(title="Python parent", url="https://parent.example/", extract="python parent text")],+ },+ )_stub_scoring(monkeypatch, [0.9] + [0.0] * 20)def fake_crawl(url, redis):if url == "https://parent.example/":- return {"url": url, "status": 200, "content": {- "title": "Python parent", "extract": "python parent text",- "links": ["https://child.example/python-guide"], "extra_links": [],- }}- return {"url": url, "status": 200, "content": {- "title": "", "extract": "python child text", "links": [], "extra_links": [],- }}+ return {+ "url": url,+ "status": 200,+ "content": {+ "title": "Python parent",+ "extract": "python parent text",+ "links": ["https://child.example/python-guide"],+ "extra_links": [],+ },+ }+ return {+ "url": url,+ "status": 200,+ "content": {+ "title": "",+ "extract": "python child text",+ "links": [],+ "extra_links": [],+ },+ }monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -585,21 +644,36 @@ def test_followed_link_without_title_excluded(client, api_key, monkeypatch):def test_followed_link_with_title_and_no_extract_kept(client, api_key, monkeypatch):"""A followed link with a genuine crawled title but an empty extract is kept."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [Document(title="Python parent", url="https://parent.example/",- extract="python parent text")],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [Document(title="Python parent", url="https://parent.example/", extract="python parent text")],+ },+ )_stub_scoring(monkeypatch, [0.9] + [0.0] * 20)def fake_crawl(url, redis):if url == "https://parent.example/":- return {"url": url, "status": 200, "content": {- "title": "Python parent", "extract": "python parent text",- "links": ["https://child.example/python-guide"], "extra_links": [],- }}- return {"url": url, "status": 200, "content": {- "title": "Python child", "extract": "", "links": [], "extra_links": [],- }}+ return {+ "url": url,+ "status": 200,+ "content": {+ "title": "Python parent",+ "extract": "python parent text",+ "links": ["https://child.example/python-guide"],+ "extra_links": [],+ },+ }+ return {+ "url": url,+ "status": 200,+ "content": {+ "title": "Python child",+ "extract": "",+ "links": [],+ "extra_links": [],+ },+ }monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)@@ -615,6 +689,7 @@ def test_followed_link_with_title_and_no_extract_kept(client, api_key, monkeypat# Architecture guard: no forbidden sync symbols in the orchestrator# ---------------------------------------------------------------------------+def test_super_search_module_has_no_blocking_imports():"""A failsafe against accidentally importing the sync `requests` libor using `time.sleep` at module scope in the async orchestrator.@@ -633,6 +708,7 @@ def test_super_search_module_has_no_blocking_imports():# index_results_against_query rather than index_documents, and it emits its own result# frames rather than going through Ranker.get_results.+def _stub_blacklist(monkeypatch, domains: set[str]):import mwmbl.tinysearchengine.super_search as ss@@ -645,17 +721,21 @@ def _stub_blacklist(monkeypatch, domains: set[str]):@pytest.mark.django_dbdef test_blacklisted_source_results_are_not_shown(client, api_key, monkeypatch):cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [- Document(title="Bad", url="https://badsite.test/x", extract="bad"),- Document(title="Good", url="https://good.test/x", extract="good"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="Bad", url="https://badsite.test/x", extract="bad"),+ Document(title="Good", url="https://good.test/x", extract="good"),+ ],+ },+ )_stub_scoring(monkeypatch, [0.9, 0.5] + [0.0] * 20)_stub_blacklist(monkeypatch, {"badsite.test"})monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url",- lambda url, redis: {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None})+ lambda url, redis: {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None},+ )response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)events = _parse_sse(_read_stream(response))@@ -674,17 +754,21 @@ def test_blacklisted_results_are_not_indexed(client, api_key, monkeypatch):"""_index_results calls index_results_against_query directly, so the index-timefilter in index_documents never sees these documents."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [- Document(title="Bad", url="https://badsite.test/x", extract="bad"),- Document(title="Good", url="https://good.test/x", extract="good"),- ],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [+ Document(title="Bad", url="https://badsite.test/x", extract="bad"),+ Document(title="Good", url="https://good.test/x", extract="good"),+ ],+ },+ )_stub_scoring(monkeypatch, [0.9, 0.5] + [0.0] * 20)_stub_blacklist(monkeypatch, {"badsite.test"})monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url",- lambda url, redis: {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None})+ lambda url, redis: {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None},+ )indexed = []@@ -704,9 +788,12 @@ def test_blacklisted_results_are_not_indexed(client, api_key, monkeypatch):def test_blacklisted_links_are_not_crawled(client, api_key, monkeypatch):"""Links are filtered before the crawl, so a blacklisted page is never fetched."""cache.delete(_super_search_monthly_key(api_key.user.id))- _stub_sources(monkeypatch, {- "hn": [Document(title="Parent", url="https://good.test/", extract="parent")],- })+ _stub_sources(+ monkeypatch,+ {+ "hn": [Document(title="Parent", url="https://good.test/", extract="parent")],+ },+ )_stub_scoring(monkeypatch, [0.9] + [0.0] * 20)_stub_blacklist(monkeypatch, {"badsite.test"})@@ -715,12 +802,25 @@ def test_blacklisted_links_are_not_crawled(client, api_key, monkeypatch):def fake_crawl(url, redis):crawled.append(url)if url == "https://good.test/":- return {"url": url, "status": 200, "timestamp": 0, "error": None,- "content": {"title": "Parent", "extract": "parent",- "links": ["https://badsite.test/x", "https://other.test/y"],- "extra_links": []}}- return {"url": url, "status": 200, "timestamp": 0, "error": None,- "content": {"title": "Child", "extract": "child", "links": [], "extra_links": []}}+ return {+ "url": url,+ "status": 200,+ "timestamp": 0,+ "error": None,+ "content": {+ "title": "Parent",+ "extract": "parent",+ "links": ["https://badsite.test/x", "https://other.test/y"],+ "extra_links": [],+ },+ }+ return {+ "url": url,+ "status": 200,+ "timestamp": 0,+ "error": None,+ "content": {"title": "Child", "extract": "child", "links": [], "extra_links": []},+ }monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)diff --git a/test/test_super_search_eval.py b/test/test_super_search_eval.pyindex 6c68011..d38c7e4 100644--- a/test/test_super_search_eval.py+++ b/test/test_super_search_eval.py@@ -1,4 +1,5 @@"""Tests for the offline evaluation harness on synthetic reward matrices."""+import importlib.utilfrom pathlib import Path@@ -7,7 +8,9 @@ import pytestfrom mwmbl.tinysearchengine.super_search_select import evaluationfrom mwmbl.tinysearchengine.super_search_select.domains import (- host_of, registrable, source_domain_map,+ host_of,+ registrable,+ source_domain_map,)from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrixfrom mwmbl.tinysearchengine.super_search_select.features import FEATURE_NAMES@@ -30,13 +33,15 @@ def _synthetic_matrix(Q=120, S=30, seed=0) -> RewardMatrix:X[:, :, 0] = 1.0 # biascos_i = FEATURE_NAMES.index("cos_bow")pop_i = FEATURE_NAMES.index("popularity")- R = np.clip(0.8 * X[:, :, cos_i] + 0.2 * X[:, :, pop_i]- + 0.02 * rng.standard_normal((Q, S)), 0.0, 1.0)+ R = np.clip(0.8 * X[:, :, cos_i] + 0.2 * X[:, :, pop_i] + 0.02 * rng.standard_normal((Q, S)), 0.0, 1.0)mask = np.ones((Q, S), dtype=bool)return RewardMatrix(queries=[f"q{i}" for i in range(Q)],sources=[f"s{i}" for i in range(S)],- feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask,+ feature_names=list(FEATURE_NAMES),+ X=X,+ R=R,+ mask=mask,)@@ -68,7 +73,7 @@ def test_ts_tuning_beats_random_and_exploration_helps():best_nu = max(sweep, key=sweep.get)best = sweep[best_nu]assert best > base["random"]- assert best > sweep[0.0] # exploration helps vs greedy+ assert best > sweep[0.0] # exploration helps vs greedyassert best <= base["oracle"] + 1e-9@@ -103,8 +108,7 @@ def test_feature_selection_flags_cos_bow(tmp_path):# XGB contextual bandit: replay + holdout evaluation# ----------------------------------------------------------------------------FAST_XGB = {"n_estimators": 50, "max_depth": 6, "learning_rate": 0.1,- "subsample": 0.8, "random_state": 0}+FAST_XGB = {"n_estimators": 50, "max_depth": 6, "learning_rate": 0.1, "subsample": 0.8, "random_state": 0}def _routing_matrix(Q=160, S=16, seed=0):@@ -133,7 +137,10 @@ def _routing_matrix(Q=160, S=16, seed=0):matrix = RewardMatrix(queries=[f"q{i}" for i in range(Q)],sources=[f"s{i}" for i in range(S)],- feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask,+ feature_names=list(FEATURE_NAMES),+ X=X,+ R=R,+ mask=mask,)return matrix, home@@ -142,8 +149,7 @@ def test_simulate_xgb_beats_random():pytest.importorskip("xgboost")m = _synthetic_matrix()base = evaluation.simulate_baselines(m, k=10)- captured = evaluation.simulate_xgb(m, k=10, epsilon=0.1, refit_every=30,- min_rows=100, params=FAST_XGB)+ captured = evaluation.simulate_xgb(m, k=10, epsilon=0.1, refit_every=30, min_rows=100, params=FAST_XGB)assert captured > base["random"]assert captured <= base["oracle"] + 1e-9@@ -151,8 +157,7 @@ def test_simulate_xgb_beats_random():def test_evaluate_holdout_learns_identity_intent_routing():pytest.importorskip("xgboost")m, home = _routing_matrix()- result = evaluation.evaluate_holdout(m, k=3, test_frac=0.25,- home_by_query=home, params=FAST_XGB)+ result = evaluation.evaluate_holdout(m, k=3, test_frac=0.25, home_by_query=home, params=FAST_XGB)cov, recall = result["coverage_at_k"], result["home_recall_at_k"]# Only identity x intent explains the reward: xgb should crush the# identity-blind baselines on both metrics.@@ -174,11 +179,12 @@ def test_evaluate_holdout_default_home_is_oracle_best():# Domain matching (shared helpers behind the gold-grounded matrix)# ---------------------------------------------------------------------------+def test_registrable_folds_subdomains_and_multi_suffixes():assert registrable("www.github.com") == "github.com"assert registrable("m.example.org") == "example.org"assert registrable("news.ycombinator.com") == "ycombinator.com"- assert registrable("foo.bbc.co.uk") == "bbc.co.uk" # multi-label suffix kept+ assert registrable("foo.bbc.co.uk") == "bbc.co.uk" # multi-label suffix keptassert registrable("example.com") == "example.com"@@ -197,6 +203,7 @@ def test_source_domain_map_groups_sources_by_registrable_domain():# Gold-grounded matrix construction (build-gold-matrix core logic)# ---------------------------------------------------------------------------+def test_is_gold_treats_nan_none_blank_as_not_gold():mod = _load_eval_script()assert mod._is_gold(1) and mod._is_gold("3")@@ -211,9 +218,9 @@ def test_attribute_rows_binary_gold_and_in_coverage_filter():reg_map = {"github.com": ["github"], "stackoverflow.com": ["stackexchange"]}rows = [("q1", "https://github.com/a", "ta", "ea", float("nan")), # available, not gold- ("q1", "https://github.com/b", "tb", "eb", 3), # gold -> github True- ("q1", "https://stackoverflow.com/x", "tx", "ex", None), # available, not gold- ("q2", "https://example.com/none", "t", "e", 1), # off-source -> dropped+ ("q1", "https://github.com/b", "tb", "eb", 3), # gold -> github True+ ("q1", "https://stackoverflow.com/x", "tx", "ex", None), # available, not gold+ ("q2", "https://example.com/none", "t", "e", 1), # off-source -> dropped]per_query, prof_text = mod.attribute_rows(rows, reg_map)diff --git a/test/test_super_search_judge.py b/test/test_super_search_judge.pyindex 1d30e1d..464c024 100644--- a/test/test_super_search_judge.py+++ b/test/test_super_search_judge.py@@ -1,4 +1,5 @@"""Tests for the fine-tuned relevance judge (super_search_select/judge.py)."""+from pathlib import Pathimport pytest@@ -26,15 +27,17 @@ def test_get_judge_unavailable_returns_none(monkeypatch):assert ss_judge.get_judge() is None[email protected](not (MODEL_DIR / "model.onnx").exists(),- reason="judge model artifact not present")[email protected](not (MODEL_DIR / "model.onnx").exists(), reason="judge model artifact not present")def test_judge_scores_relevant_above_irrelevant():judge = ss_judge.Judge(MODEL_DIR)- scores = judge.score("python asyncio tutorial", [- ss_judge.doc_text("Async IO in Python: A Complete Walkthrough",- "A tutorial covering asyncio, coroutines and event loops."),- ss_judge.doc_text("Best chocolate cake recipes",- "Moist chocolate cake with simple ingredients."),- ])+ scores = judge.score(+ "python asyncio tutorial",+ [+ ss_judge.doc_text(+ "Async IO in Python: A Complete Walkthrough", "A tutorial covering asyncio, coroutines and event loops."+ ),+ ss_judge.doc_text("Best chocolate cake recipes", "Moist chocolate cake with simple ingredients."),+ ],+ )assert all(0.0 <= s <= 1.0 for s in scores)assert scores[0] > scores[1]diff --git a/test/test_super_search_recipes.py b/test/test_super_search_recipes.pyindex 66c3fc4..5635bf1 100644--- a/test/test_super_search_recipes.py+++ b/test/test_super_search_recipes.py@@ -5,6 +5,7 @@ url/title/extract coercion (templated URLs, relative-href resolution, HTMLstripping) and that an HTTP error returns [] rather than propagating. We alsoverify the shipped recipe files load and register into SOURCES."""+import refrom pathlib import Path@@ -50,6 +51,7 @@ def _recipe(name: str) -> Recipe:# Recipe loading / registration# ---------------------------------------------------------------------------+def test_shipped_recipes_load():assert {"wiktionary", "archive_org", "gutenberg"} <= set(RECIPES)assert _recipe("gutenberg").response_format == "html"@@ -57,6 +59,7 @@ def test_shipped_recipes_load():def test_recipes_registered_in_sources():from mwmbl.tinysearchengine.super_search_sources import SOURCES+assert "wiktionary" in SOURCES and "gutenberg" in SOURCES@@ -80,13 +83,17 @@ def test_recipe_has_smoke_block(recipe):# JSON: Wiktionary (MediaWiki) — templated URL + HTML-stripped snippet# ---------------------------------------------------------------------------+async def test_wiktionary_templates_url_and_strips_html(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://en\.wiktionary\.org/w/api\.php.*"),- json={"query": {"search": [- {"title": "serendipity",- "snippet": 'fortunate <span class="searchmatch">serendipity</span>'},- ]}},+ json={+ "query": {+ "search": [+ {"title": "serendipity", "snippet": 'fortunate <span class="searchmatch">serendipity</span>'},+ ]+ }+ },)async with httpx.AsyncClient() as client:docs = await search_with_recipe(client, _recipe("wiktionary"), "serendipity", 5)@@ -110,13 +117,18 @@ async def test_wiktionary_quotes_titles_with_spaces(httpx_mock):# JSON: archive.org — nested results path + templated URL from identifier# ---------------------------------------------------------------------------+async def test_archive_org_nested_results_and_template(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://archive\.org/advancedsearch\.php.*"),- json={"response": {"docs": [- {"identifier": "apollo11", "title": "Apollo 11", "description": "Moon landing"},- {"title": "no id"}, # no identifier -> no URL -> skipped- ]}},+ json={+ "response": {+ "docs": [+ {"identifier": "apollo11", "title": "Apollo 11", "description": "Moon landing"},+ {"title": "no id"}, # no identifier -> no URL -> skipped+ ]+ }+ },)async with httpx.AsyncClient() as client:docs = await search_with_recipe(client, _recipe("archive_org"), "apollo 11", 5)@@ -129,10 +141,16 @@ async def test_json_base_url_joins_relative_path(httpx_mock):"""A JSON API returning a site-relative url (e.g. gov.uk's "/contact-hmrc") isjoined onto base_url to the canonical absolute URL, without %-quoting slashes."""recipe = Recipe(- name="govuk", domain="www.gov.uk", field="law-government",+ name="govuk",+ domain="www.gov.uk",+ field="law-government",request={"url": "https://www.gov.uk/api/search.json", "params": {"q": "{query}"}},- response={"format": "json", "base_url": "https://www.gov.uk", "results": "results",- "fields": {"title": "title", "url": "link"}},+ response={+ "format": "json",+ "base_url": "https://www.gov.uk",+ "results": "results",+ "fields": {"title": "title", "url": "link"},+ },smoke={"query": "vat", "expect_title_contains": "VAT"},)httpx_mock.add_response(@@ -213,6 +231,7 @@ async def test_xml_attributes_and_template(httpx_mock):# Robustness: errors swallowed# ---------------------------------------------------------------------------+async def test_http_error_returns_empty(httpx_mock):httpx_mock.add_response(status_code=500)async with httpx.AsyncClient() as client:@@ -228,6 +247,7 @@ def test_recipes_dir_exists():# v2 engine enhancements: headers, POST json body, XML namespaces, list index# ---------------------------------------------------------------------------+async def test_request_headers_sent(httpx_mock):recipe = Recipe(name="hdr",@@ -265,6 +285,7 @@ async def test_post_json_body_with_substitution(httpx_mock):request = httpx_mock.get_requests()[0]assert request.method == "POST"import json as _json+body = _json.loads(request.content)assert body == {"query": "neutrino", "size": "7"}assert docs[0].url == "https://example.com/1"@@ -274,8 +295,7 @@ async def test_json_list_index_path(httpx_mock):recipe = Recipe(name="listidx",request={"url": "https://example.com/api", "params": {"q": "{query}"}},- response={"format": "json", "results": "data.0.items",- "fields": {"title": "name", "url": "link"}},+ response={"format": "json", "results": "data.0.items", "fields": {"title": "name", "url": "link"}},)httpx_mock.add_response(url=re.compile(r"https://example\.com/.*"),diff --git a/test/test_super_search_select.py b/test/test_super_search_select.pyindex 1d4aa94..57437e9 100644--- a/test/test_super_search_select.py+++ b/test/test_super_search_select.py@@ -1,11 +1,17 @@"""Tests for Super Search source selection: profiles, features, policy."""+import fakeredisimport numpy as npimport pytestfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.super_search_select import (- features, policy, profiles, rewards, rstats, vectors,+ features,+ policy,+ profiles,+ rewards,+ rstats,+ vectors,)from mwmbl.tinysearchengine.super_search_select.features import QueryContext, SiteStatsfrom mwmbl.tinysearchengine.super_search_select.registry import SiteMeta@@ -31,6 +37,7 @@ def _doc(title, extract=""):# Profiles# ---------------------------------------------------------------------------+def test_unseen_profile_is_none(fake_profile_redis):assert profiles.get_profile("nope") == (None, None)@@ -77,6 +84,7 @@ def test_batch_profiles(fake_profile_redis):# Features# ---------------------------------------------------------------------------+def _qctx(query):bow = vectors.project_bow(query, DIM)cng = vectors.project_char_ngrams(query, DIM)@@ -109,6 +117,7 @@ def test_feature_vector_uses_stats():# Policy# ---------------------------------------------------------------------------+def test_select_returns_all_when_fewer_than_k(fake_profile_redis):names = ["a", "b", "c"]assert set(policy.select_sources("q", names, k=10)) == set(names)@@ -135,6 +144,7 @@ def test_select_respects_budget_and_records_features(fake_profile_redis):# Rewards# ---------------------------------------------------------------------------+def test_compute_rewards_fraction_in_top_k():ctx = SelectionContext(selected=["a", "b", "c"], per_source_limit=10)ctx.record_results("a", ["u1", "u2", "u3"])diff --git a/test/test_super_search_smoke.py b/test/test_super_search_smoke.pyindex 21ae8f1..204fdfe 100644--- a/test/test_super_search_smoke.py+++ b/test/test_super_search_smoke.py@@ -13,6 +13,7 @@ search_with_recipe swallows transport/parse errors and returns [], so ablocked or reformatted site surfaces here as an empty result list — which theassertions below turn into a clear failure."""+import httpximport pytestdiff --git a/test/test_super_search_sources.py b/test/test_super_search_sources.pyindex c0cc5e0..1fb994c 100644--- a/test/test_super_search_sources.py+++ b/test/test_super_search_sources.py@@ -5,10 +5,10 @@ result coercion without hitting the network. We also assert that an HTTPerror returns an empty list rather than propagating — orchestrator robustnessdepends on this."""+import reimport httpx-import pytestfrom mwmbl.tinysearchengine.super_search_sources.arxiv import search as search_arxivfrom mwmbl.tinysearchengine.super_search_sources.github import search as search_github@@ -26,13 +26,16 @@ async def _client():# Hacker News (Algolia)# ---------------------------------------------------------------------------+async def test_hn_returns_documents(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://hn\.algolia\.com/api/v1/search\?.*query.*python.*"),- json={"hits": [- {"title": "Why Python", "url": "https://example.com/why", "objectID": "1"},- {"title": "Ask HN", "story_text": "Some text", "objectID": "2"}, # no url- ]},+ json={+ "hits": [+ {"title": "Why Python", "url": "https://example.com/why", "objectID": "1"},+ {"title": "Ask HN", "story_text": "Some text", "objectID": "2"}, # no url+ ]+ },)async with httpx.AsyncClient() as client:docs = await search_hn(client, "python", 5)@@ -52,14 +55,20 @@ async def test_hn_swallows_http_errors(httpx_mock):# GitHub# ---------------------------------------------------------------------------+async def test_github_returns_documents(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://api\.github\.com/search/repositories.*"),- json={"items": [- {"full_name": "psf/requests", "html_url": "https://github.com/psf/requests",- "description": "HTTP for humans"},- {"name": "noop", "html_url": ""}, # missing url -> skipped- ]},+ json={+ "items": [+ {+ "full_name": "psf/requests",+ "html_url": "https://github.com/psf/requests",+ "description": "HTTP for humans",+ },+ {"name": "noop", "html_url": ""}, # missing url -> skipped+ ]+ },)async with httpx.AsyncClient() as client:docs = await search_github(client, "requests", 5)@@ -78,13 +87,20 @@ async def test_github_swallows_errors(httpx_mock):# Stack Exchange# ---------------------------------------------------------------------------+async def test_stackexchange_returns_documents(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://api\.stackexchange\.com/.*"),- json={"items": [- {"title": "How do I & X?", "link": "https://stackoverflow.com/q/1",- "tags": ["python", "async"], "score": 12},- ]},+ json={+ "items": [+ {+ "title": "How do I & X?",+ "link": "https://stackoverflow.com/q/1",+ "tags": ["python", "async"],+ "score": 12,+ },+ ]+ },)async with httpx.AsyncClient() as client:docs = await search_stackexchange(client, "async python", 5)@@ -109,6 +125,7 @@ ARXIV_ATOM = """<?xml version="1.0" encoding="UTF-8"?></feed>"""+async def test_arxiv_returns_documents(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://export\.arxiv\.org/.*"),@@ -125,11 +142,17 @@ async def test_arxiv_returns_documents(httpx_mock):# PyPI# ---------------------------------------------------------------------------+async def test_pypi_returns_document(httpx_mock):httpx_mock.add_response(url="https://pypi.org/pypi/requests/json",- json={"info": {"name": "requests", "summary": "HTTP for humans",- "package_url": "https://pypi.org/project/requests/"}},+ json={+ "info": {+ "name": "requests",+ "summary": "HTTP for humans",+ "package_url": "https://pypi.org/project/requests/",+ }+ },)async with httpx.AsyncClient() as client:docs = await search_pypi(client, "requests", 5)@@ -156,14 +179,17 @@ async def test_pypi_rejects_invalid_name():# IMDb (autosuggest)# ---------------------------------------------------------------------------+async def test_imdb_maps_ids_to_canonical_urls(httpx_mock):httpx_mock.add_response(url=re.compile(r"https://v3\.sg\.media-imdb\.com/suggestion/.*"),- json={"d": [- {"id": "tt0468569", "l": "The Dark Knight", "q": "feature", "s": "Christian Bale"},- {"id": "nm0000288", "l": "Christian Bale", "s": "Actor"},- {"id": "ls123", "l": "a user list"}, # not a title/name -> skipped- ]},+ json={+ "d": [+ {"id": "tt0468569", "l": "The Dark Knight", "q": "feature", "s": "Christian Bale"},+ {"id": "nm0000288", "l": "Christian Bale", "s": "Actor"},+ {"id": "ls123", "l": "a user list"}, # not a title/name -> skipped+ ]+ },)async with httpx.AsyncClient() as client:docs = await search_imdb(client, "the dark knight", 5)diff --git a/test/test_super_search_vectors.py b/test/test_super_search_vectors.pyindex 777629c..c430885 100644--- a/test/test_super_search_vectors.py+++ b/test/test_super_search_vectors.py@@ -1,9 +1,9 @@"""Unit tests for Super Search vector utilities (no DB / Redis needed)."""+import numpy as npfrom mwmbl.tinysearchengine.super_search_select import vectors-DIM = 64diff --git a/test/test_super_search_xgb.py b/test/test_super_search_xgb.pyindex 393e16a..56dc432 100644--- a/test/test_super_search_xgb.py+++ b/test/test_super_search_xgb.py@@ -1,4 +1,5 @@"""Tests for the XGBoost contextual-bandit source model and the xgb policy."""+import jsonimport fakeredis@@ -55,6 +56,7 @@ def _trained_model(tmp_path, sources=("github", "recipes"), reward_kind="test"):# Encoding / training data# ---------------------------------------------------------------------------+def test_encode_shape_and_identity_block():vocab_index = {"a": 0, "b": 1}x = xgb_model.encode(_shared(), "b", vocab_index)@@ -95,23 +97,29 @@ def test_build_training_data_from_matrix():R = np.array([[1.0, 0.0], [0.5, 0.2], [0.0, 0.9]])mask = np.ones((Q, S), dtype=bool)mask[2, 0] = False- matrix = RewardMatrix(queries=["q1", "q2", "q3"], sources=["b", "a"],- feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)+ matrix = RewardMatrix(+ queries=["q1", "q2", "q3"], sources=["b", "a"], feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask+ )Xf, y, vocab, means = xgb_model.build_training_data_from_matrix(matrix)assert vocab == ["a", "b"]assert Xf.shape == (5, NUM_FEATURES + 2)assert set(y.tolist()) == {1.0, 0.0, 0.5, 0.2, 0.9}# Per-source mean reward over masked cells, injected into the ema slot.- assert means["b"] == pytest.approx(0.75) # (1.0 + 0.5) / 2+ assert means["b"] == pytest.approx(0.75) # (1.0 + 0.5) / 2assert means["a"] == pytest.approx((0.0 + 0.2 + 0.9) / 3)ema_i = FEATURE_NAMES.index("contribution_ema")assert set(np.round(Xf[:, ema_i], 6)) == {0.75, round((0.0 + 0.2 + 0.9) / 3, 6)}def test_matrix_with_mismatched_features_raises():- matrix = RewardMatrix(queries=["q"], sources=["a"], feature_names=["bias"],- X=np.zeros((1, 1, 1)), R=np.zeros((1, 1)),- mask=np.ones((1, 1), dtype=bool))+ matrix = RewardMatrix(+ queries=["q"],+ sources=["a"],+ feature_names=["bias"],+ X=np.zeros((1, 1, 1)),+ R=np.zeros((1, 1)),+ mask=np.ones((1, 1), dtype=bool),+ )with pytest.raises(ValueError, match="feature names"):xgb_model.build_training_data_from_matrix(matrix)@@ -120,6 +128,7 @@ def test_matrix_with_mismatched_features_raises():# Model learns identity x context interactions# ---------------------------------------------------------------------------+def test_model_learns_intent_source_interaction(tmp_path):_trained_model(tmp_path)loaded = xgb_model.load_artifact(tmp_path)@@ -133,6 +142,7 @@ def test_model_learns_intent_source_interaction(tmp_path):# Artifact save / load / get_model# ---------------------------------------------------------------------------+def test_artifact_roundtrip_meta(tmp_path):vocab = _trained_model(tmp_path, reward_kind="judge")loaded = xgb_model.load_artifact(tmp_path)@@ -179,15 +189,13 @@ def test_get_model_falls_back_to_bundle_when_runtime_empty(tmp_path, monkeypatchbundle = tmp_path / "bundle"_trained_model(bundle, sources=("bundled_source",))monkeypatch.setattr(xgb_model, "BUNDLED_DIR", bundle)- monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",- str(tmp_path / "empty_runtime"))+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(tmp_path / "empty_runtime"))assert xgb_model.get_model().vocab == ["bundled_source"]def test_get_model_no_artifact_anywhere_raises(tmp_path, monkeypatch):monkeypatch.setattr(xgb_model, "BUNDLED_DIR", tmp_path / "no_bundle")- monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",- str(tmp_path / "no_runtime"))+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(tmp_path / "no_runtime"))with pytest.raises(FileNotFoundError):xgb_model.get_model()@@ -204,6 +212,7 @@ def test_get_model_hot_reloads_after_retrain(tmp_path, monkeypatch):# ensure the mtime moves even on coarse-grained filesystemsstat = meta_path.stat()import os+os.utime(meta_path, (stat.st_atime, stat.st_mtime + 1))assert xgb_model.get_model().vocab == ["arxiv", "github", "recipes"]@@ -212,6 +221,7 @@ def test_get_model_hot_reloads_after_retrain(tmp_path, monkeypatch):# xgb policy# ---------------------------------------------------------------------------+@pytest.fixturedef xgb_policy_env(tmp_path, monkeypatch):"""Policy environment with a toy artifact trained over the policy's sources."""@@ -236,8 +246,7 @@ def test_policy_xgb_selects_and_records_features(xgb_policy_env, monkeypatch):def test_policy_xgb_greedy_is_deterministic(xgb_policy_env, monkeypatch):monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 0.0)- runs = {tuple(policy.select_sources("some query", xgb_policy_env, k=6))- for _ in range(5)}+ runs = {tuple(policy.select_sources("some query", xgb_policy_env, k=6)) for _ in range(5)}assert len(runs) == 1@@ -257,8 +266,7 @@ def test_policy_xgb_missing_artifact_raises(monkeypatch, tmp_path):r = fakeredis.FakeRedis()monkeypatch.setattr(profiles, "_redis", r)monkeypatch.setattr(rstats, "_redis", r)- monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",- str(tmp_path / "empty"))+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(tmp_path / "empty"))monkeypatch.setattr(xgb_model, "BUNDLED_DIR", tmp_path / "no_bundle")names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]with pytest.raises(FileNotFoundError):@@ -269,6 +277,7 @@ def test_policy_xgb_missing_artifact_raises(monkeypatch, tmp_path):# Online per-source reward EMA (rstats -> contribution_ema feature)# ---------------------------------------------------------------------------+@pytest.fixturedef fake_rstats_redis(monkeypatch):r = fakeredis.FakeRedis()@@ -306,11 +315,10 @@ def test_seed_profiles_never_clobbers_live_values(fake_rstats_redis, monkeypatchlive_before, _ = profiles.get_profile("live")seed_vec = np.ones(64, dtype=np.float32) / 8.0- seeded = profiles.seed_profiles({"live": (seed_vec, seed_vec),- "cold": (seed_vec, seed_vec)})+ seeded = profiles.seed_profiles({"live": (seed_vec, seed_vec), "cold": (seed_vec, seed_vec)})assert seeded == 1live_after, _ = profiles.get_profile("live")- assert np.allclose(live_after, live_before) # untouched+ assert np.allclose(live_after, live_before) # untouchedcold_bow, cold_cng = profiles.get_profile("cold")assert np.allclose(cold_bow, seed_vec) and np.allclose(cold_cng, seed_vec)@@ -321,6 +329,7 @@ def test_seed_online_state_from_bundled_artifact(fake_rstats_redis, tmp_path, mo_trained_model(tmp_path) # writes model.json + meta.json (no reward means)# add reward means + profiles to the artifactimport json as _json+meta_path = tmp_path / xgb_model.META_FILEmeta = _json.loads(meta_path.read_text())meta["source_reward_means"] = {"github": 0.6, "recipes": 0.3}@@ -359,6 +368,7 @@ def test_rstats_feeds_selection_features(fake_rstats_redis, monkeypatch):# Online retrain from the impression log# ---------------------------------------------------------------------------+@pytest.mark.django_dbdef test_retrain_from_impressions_writes_artifact(tmp_path, monkeypatch):from mwmbl.models import SuperSearchImpression@@ -366,15 +376,17 @@ def test_retrain_from_impressions_writes_artifact(tmp_path, monkeypatch):monkeypatch.setattr(xgb_model, "XGB_PARAMS", FAST_PARAMS)for features, rewards in _interaction_rows(30):SuperSearchImpression.objects.create(- candidates=list(features), selected=list(features),- features={k: list(v) for k, v in features.items()}, rewards=rewards)+ candidates=list(features),+ selected=list(features),+ features={k: list(v) for k, v in features.items()},+ rewards=rewards,+ )# A pre-intent row with a short (10-dim) vector must zero-pad, not break.SuperSearchImpression.objects.create(- candidates=["old"], selected=["old"],- features={"old": [1.0] * 10}, rewards={"old": 0.5})+ candidates=["old"], selected=["old"], features={"old": [1.0] * 10}, rewards={"old": 0.5}+ )- metrics = xgb_model.train_and_save_from_impressions(- window_days=7, min_rows=10, out_dir=tmp_path)+ metrics = xgb_model.train_and_save_from_impressions(window_days=7, min_rows=10, out_dir=tmp_path)assert metrics is not None and "train_rmse" in metricsloaded = xgb_model.load_artifact(tmp_path)assert {"github", "recipes", "old"} <= set(loaded.vocab)@@ -387,10 +399,9 @@ def test_retrain_skips_below_min_rows(tmp_path):from mwmbl.models import SuperSearchImpressionSuperSearchImpression.objects.create(- candidates=["a"], selected=["a"],- features={"a": [0.0] * NUM_FEATURES}, rewards={"a": 1.0})- assert xgb_model.train_and_save_from_impressions(- window_days=7, min_rows=100, out_dir=tmp_path) is None+ candidates=["a"], selected=["a"], features={"a": [0.0] * NUM_FEATURES}, rewards={"a": 1.0}+ )+ assert xgb_model.train_and_save_from_impressions(window_days=7, min_rows=100, out_dir=tmp_path) is Noneassert not (tmp_path / xgb_model.META_FILE).exists()@@ -398,6 +409,7 @@ def test_retrain_skips_below_min_rows(tmp_path):# Privacy tripwire: nothing vector-valued may enter the persisted features# ---------------------------------------------------------------------------+def test_persisted_features_are_scalar_and_reconstruction_safe():"""Impressions persist the feature vector per source; every entry must be anamed scalar that cannot reconstruct the query. A name matching the pattern@@ -406,7 +418,8 @@ def test_persisted_features_are_scalar_and_reconstruction_safe():import refrom mwmbl.tinysearchengine.super_search_select.features import (- QueryContext, feature_vector,+ QueryContext,+ feature_vector,)from mwmbl.tinysearchengine.super_search_select.registry import get_metadiff --git a/test/test_update_urls.py b/test/test_update_urls.pyindex ce58630..a94bcb2 100644--- a/test/test_update_urls.py+++ b/test/test_update_urls.py@@ -1,12 +1,11 @@from collections import defaultdictfrom mwmbl.crawler.batch import Link-from mwmbl.indexer.update_urls import process_linkfrom mwmbl.indexer.blacklist_providers import CombinedBlacklistProvider, StaticBlacklistProvider+from mwmbl.indexer.update_urls import process_linkdef test_process_link_normal():- url_scores = {"https://somesite.com/something.html": 0.0, "https://somesite.com/": 0.0}url_timestamps = {}url_users = {}domain_links = defaultdict(set)diff --git a/test/test_url_database.py b/test/test_url_database.pyindex d7373ce..26e987e 100644--- a/test/test_url_database.py+++ b/test/test_url_database.py@@ -8,6 +8,7 @@ again. The bug these tests guard against was that `update_found_urls` derivedfirst-time crawl came back with `last_crawled=None` and `queue_urls` treated itas never-crawled."""+import globimport osfrom datetime import datetime, timezone@@ -15,7 +16,7 @@ from datetime import datetime, timezoneimport fakeredisimport pytest-from mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatus+from mwmbl.crawler.urls import FoundURL, URLDatabase, URLStatusfrom mwmbl.indexer.blacklist_providers import StaticBlacklistProviderfrom mwmbl.indexer.update_urls import get_datetime_from_timestampfrom mwmbl.redis_url_queue import RedisURLQueue@@ -24,6 +25,7 @@ from mwmbl.redis_url_queue import RedisURLQueue@pytest.fixturedef clean_bloom_filters():"""Remove any bloom filter files so each test starts from a clean slate."""+def _remove():for path in glob.glob("/tmp/test_urls*.bloom"):os.remove(path)diff --git a/test/test_voting_api.py b/test/test_voting_api.pyindex c897e0e..d921586 100644--- a/test/test_voting_api.py+++ b/test/test_voting_api.py@@ -1,73 +1,46 @@-import pytestimport json-from django.contrib.auth import get_user_model++import pytestfrom allauth.account.models import EmailAddress+from django.contrib.auth import get_user_modelfrom ninja_jwt.tokens import RefreshTokenfrom mwmbl.models import SearchResultVote-User = get_user_model()@pytest.fixturedef user():"""Create a test user"""- return User.objects.create_user(- username='testuser',- email='[email protected]',- password='testpass123'- )+ return User.objects.create_user(username="testuser", email="[email protected]", password="testpass123")@pytest.fixturedef user2():"""Create a second test user"""- return User.objects.create_user(- username='testuser2',- email='[email protected]',- password='testpass123'- )+ return User.objects.create_user(username="testuser2", email="[email protected]", password="testpass123")@pytest.fixturedef verified_user(user):"""Create a user with verified email"""- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=True,- primary=True- )+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)return user@pytest.fixturedef verified_user2(user2):"""Create a second user with verified email"""- EmailAddress.objects.create(- user=user2,- email='[email protected]',- verified=True,- primary=True- )+ EmailAddress.objects.create(user=user2, email="[email protected]", verified=True, primary=True)return user2@pytest.fixturedef unverified_user():"""Create a user with unverified email"""- user = User.objects.create_user(- username='unverified',- email='[email protected]',- password='testpass123'- )- EmailAddress.objects.create(- user=user,- email='[email protected]',- verified=False,- primary=True- )+ user = User.objects.create_user(username="unverified", email="[email protected]", password="testpass123")+ EmailAddress.objects.create(user=user, email="[email protected]", verified=False, primary=True)return user@@ -95,36 +68,30 @@ def unverified_access_token(unverified_user):@pytest.fixturedef sample_vote_data():"""Sample vote data for testing"""- return {- "url": "https://example.com/test-page",- "query": "test search query",- "vote_type": "upvote"- }+ return {"url": "https://example.com/test-page", "query": "test search query", "vote_type": "upvote"}@pytest.mark.django_dbdef test_vote_on_search_result_success(client, verified_user, access_token, sample_vote_data):"""Test successfully voting on a search result"""response = client.post(- '/api/v1/platform/search-results/vote',+ "/api/v1/platform/search-results/vote",data=json.dumps(sample_vote_data),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert response_data['status'] == 'ok'- assert 'Vote created successfully' in response_data['message']-+ assert response_data["status"] == "ok"+ assert "Vote created successfully" in response_data["message"]+# Verify vote was created in databasevote = SearchResultVote.objects.get(- user=verified_user,- url=sample_vote_data['url'],- query=sample_vote_data['query']+ user=verified_user, url=sample_vote_data["url"], query=sample_vote_data["query"])- assert vote.vote_type == 'upvote'+ assert vote.vote_type == "upvote"@pytest.mark.django_db@@ -132,68 +99,61 @@ def test_vote_update_existing_vote(client, verified_user, access_token, sample_v"""Test updating an existing vote"""# Create initial voteSearchResultVote.objects.create(- user=verified_user,- url=sample_vote_data['url'],- query=sample_vote_data['query'],- vote_type='upvote'+ user=verified_user, url=sample_vote_data["url"], query=sample_vote_data["query"], vote_type="upvote")-+# Update to downvoteupdated_vote_data = sample_vote_data.copy()- updated_vote_data['vote_type'] = 'downvote'-+ updated_vote_data["vote_type"] = "downvote"+response = client.post(- '/api/v1/platform/search-results/vote',+ "/api/v1/platform/search-results/vote",data=json.dumps(updated_vote_data),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert response_data['status'] == 'ok'- assert 'Vote updated successfully' in response_data['message']-+ assert response_data["status"] == "ok"+ assert "Vote updated successfully" in response_data["message"]+# Verify vote was updatedvote = SearchResultVote.objects.get(- user=verified_user,- url=sample_vote_data['url'],- query=sample_vote_data['query']+ user=verified_user, url=sample_vote_data["url"], query=sample_vote_data["query"])- assert vote.vote_type == 'downvote'+ assert vote.vote_type == "downvote"@pytest.mark.django_dbdef test_vote_invalid_vote_type(client, access_token, sample_vote_data):"""Test voting with invalid vote type"""invalid_vote_data = sample_vote_data.copy()- invalid_vote_data['vote_type'] = 'INVALID'-+ invalid_vote_data["vote_type"] = "INVALID"+response = client.post(- '/api/v1/platform/search-results/vote',+ "/api/v1/platform/search-results/vote",data=json.dumps(invalid_vote_data),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 422-+response_data = response.json()# Django Ninja validation errors have a different format- assert 'detail' in response_data- assert any('upvote' in str(error) and 'downvote' in str(error) for error in response_data['detail'])+ assert "detail" in response_data+ assert any("upvote" in str(error) and "downvote" in str(error) for error in response_data["detail"])@pytest.mark.django_dbdef test_vote_without_authentication(client, sample_vote_data):"""Test voting without JWT token"""response = client.post(- '/api/v1/platform/search-results/vote',- data=json.dumps(sample_vote_data),- content_type='application/json'+ "/api/v1/platform/search-results/vote", data=json.dumps(sample_vote_data), content_type="application/json")-+assert response.status_code == 401@@ -201,17 +161,17 @@ def test_vote_without_authentication(client, sample_vote_data):def test_vote_with_unverified_email(client, unverified_access_token, sample_vote_data):"""Test voting with unverified email address"""response = client.post(- '/api/v1/platform/search-results/vote',+ "/api/v1/platform/search-results/vote",data=json.dumps(sample_vote_data),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {unverified_access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {unverified_access_token}",)-+assert response.status_code == 403-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'Email address is not verified' in response_data['message']+ assert response_data["status"] == "error"+ assert "Email address is not verified" in response_data["message"]@pytest.mark.django_db@@ -219,40 +179,31 @@ def test_get_vote_counts_single_url(client, verified_user, verified_user2, acces"""Test getting vote counts for a single URL"""# Create votes from different usersSearchResultVote.objects.create(- user=verified_user,- url=sample_vote_data['url'],- query=sample_vote_data['query'],- vote_type='upvote'+ user=verified_user, url=sample_vote_data["url"], query=sample_vote_data["query"], vote_type="upvote")SearchResultVote.objects.create(- user=verified_user2,- url=sample_vote_data['url'],- query=sample_vote_data['query'],- vote_type='downvote'+ user=verified_user2, url=sample_vote_data["url"], query=sample_vote_data["query"], vote_type="downvote")-- vote_stats_request = {- "query": sample_vote_data["query"],- "urls": [sample_vote_data["url"]]- }-++ vote_stats_request = {"query": sample_vote_data["query"], "urls": [sample_vote_data["url"]]}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert 'votes' in response_data- assert sample_vote_data['url'] in response_data['votes']-- vote_stats = response_data['votes'][sample_vote_data['url']]- assert vote_stats['upvotes'] == 1- assert vote_stats['downvotes'] == 1- assert vote_stats['user_vote'] == 'upvote' # Current user's vote+ assert "votes" in response_data+ assert sample_vote_data["url"] in response_data["votes"]++ vote_stats = response_data["votes"][sample_vote_data["url"]]+ assert vote_stats["upvotes"] == 1+ assert vote_stats["downvotes"] == 1+ assert vote_stats["user_vote"] == "upvote" # Current user's vote@pytest.mark.django_db@@ -261,64 +212,48 @@ def test_get_vote_counts_multiple_urls(client, verified_user, access_token):url1 = "https://example.com/page1"url2 = "https://example.com/page2"query = "test query"-+# Create votes for different URLs- SearchResultVote.objects.create(- user=verified_user,- url=url1,- query=query,- vote_type='upvote'- )- SearchResultVote.objects.create(- user=verified_user,- url=url2,- query=query,- vote_type='downvote'- )-- vote_stats_request = {- "query": query,- "urls": [url1, url2]- }-+ SearchResultVote.objects.create(user=verified_user, url=url1, query=query, vote_type="upvote")+ SearchResultVote.objects.create(user=verified_user, url=url2, query=query, vote_type="downvote")++ vote_stats_request = {"query": query, "urls": [url1, url2]}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert len(response_data['votes']) == 2- assert url1 in response_data['votes']- assert url2 in response_data['votes']-- assert response_data['votes'][url1]['user_vote'] == 'upvote'- assert response_data['votes'][url2]['user_vote'] == 'downvote'+ assert len(response_data["votes"]) == 2+ assert url1 in response_data["votes"]+ assert url2 in response_data["votes"]++ assert response_data["votes"][url1]["user_vote"] == "upvote"+ assert response_data["votes"][url2]["user_vote"] == "downvote"@pytest.mark.django_dbdef test_get_vote_counts_no_urls(client, access_token):"""Test getting vote counts without providing URLs"""- vote_stats_request = {- "query": "test",- "urls": []- }-+ vote_stats_request = {"query": "test", "urls": []}+response = client.post(- '/api/v1/platform/search-results/votes',+ "/api/v1/platform/search-results/votes",data=json.dumps(vote_stats_request),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 400-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'At least one URL must be provided' in response_data['message']+ assert response_data["status"] == "error"+ assert "At least one URL must be provided" in response_data["message"]@pytest.mark.django_db@@ -326,33 +261,25 @@ def test_remove_vote_success(client, verified_user, access_token, sample_vote_da"""Test successfully removing a vote"""# Create a vote firstSearchResultVote.objects.create(- user=verified_user,- url=sample_vote_data['url'],- query=sample_vote_data['query'],- vote_type='upvote'+ user=verified_user, url=sample_vote_data["url"], query=sample_vote_data["query"], vote_type="upvote")-+response = client.delete(- '/api/v1/platform/search-results/vote',- data=json.dumps({- 'url': sample_vote_data['url'],- 'query': sample_vote_data['query']- }),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ "/api/v1/platform/search-results/vote",+ data=json.dumps({"url": sample_vote_data["url"], "query": sample_vote_data["query"]}),+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 200-+response_data = response.json()- assert response_data['status'] == 'ok'- assert 'Vote removed successfully' in response_data['message']-+ assert response_data["status"] == "ok"+ assert "Vote removed successfully" in response_data["message"]+# Verify vote was deletedassert not SearchResultVote.objects.filter(- user=verified_user,- url=sample_vote_data['url'],- query=sample_vote_data['query']+ user=verified_user, url=sample_vote_data["url"], query=sample_vote_data["query"]).exists()@@ -360,20 +287,17 @@ def test_remove_vote_success(client, verified_user, access_token, sample_vote_dadef test_remove_nonexistent_vote(client, access_token, sample_vote_data):"""Test removing a vote that doesn't exist"""response = client.delete(- '/api/v1/platform/search-results/vote',- data=json.dumps({- 'url': sample_vote_data['url'],- 'query': sample_vote_data['query']- }),- content_type='application/json',- HTTP_AUTHORIZATION=f'Bearer {access_token}'+ "/api/v1/platform/search-results/vote",+ data=json.dumps({"url": sample_vote_data["url"], "query": sample_vote_data["query"]}),+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {access_token}",)-+assert response.status_code == 404-+response_data = response.json()- assert response_data['status'] == 'error'- assert 'No vote found to remove' in response_data['message']+ assert response_data["status"] == "error"+ assert "No vote found to remove" in response_data["message"]@pytest.mark.django_db@@ -385,56 +309,43 @@ def test_get_user_vote_history(client, verified_user, access_token):{"url": "https://example.com/page2", "query": "query2", "vote_type": "downvote"},{"url": "https://example.com/page3", "query": "query3", "vote_type": "upvote"},]-+for vote_data in votes_data:- SearchResultVote.objects.create(- user=verified_user,- **vote_data- )-- response = client.get(- '/api/v1/platform/search-results/my-votes',- HTTP_AUTHORIZATION=f'Bearer {access_token}'- )-+ SearchResultVote.objects.create(user=verified_user, **vote_data)++ response = client.get("/api/v1/platform/search-results/my-votes", HTTP_AUTHORIZATION=f"Bearer {access_token}")+assert response.status_code == 200-+response_data = response.json()- assert 'items' in response_data # Paginated response- assert len(response_data['items']) == 3-+ assert "items" in response_data # Paginated response+ assert len(response_data["items"]) == 3+# Verify the votes are ordered by timestamp (newest first)- for i, vote in enumerate(response_data['items']):- assert vote['url'] == votes_data[2-i]['url'] # Reverse order due to ordering by -timestamp- assert vote['query'] == votes_data[2-i]['query']- assert vote['vote_type'] == votes_data[2-i]['vote_type']+ for i, vote in enumerate(response_data["items"]):+ assert vote["url"] == votes_data[2 - i]["url"] # Reverse order due to ordering by -timestamp+ assert vote["query"] == votes_data[2 - i]["query"]+ assert vote["vote_type"] == votes_data[2 - i]["vote_type"]@pytest.mark.django_dbdef test_vote_unique_constraint(verified_user):"""Test that the unique constraint works (one vote per user per URL per query)"""from django.db import IntegrityError, transaction-- vote_data = {- "user": verified_user,- "url": "https://example.com/test",- "query": "test query",- "vote_type": "upvote"- }-++ vote_data = {"user": verified_user, "url": "https://example.com/test", "query": "test query", "vote_type": "upvote"}+# Create first vote- vote1 = SearchResultVote.objects.create(**vote_data)-+ SearchResultVote.objects.create(**vote_data)+# Try to create duplicate vote - should raise IntegrityErrorwith pytest.raises(IntegrityError):with transaction.atomic():SearchResultVote.objects.create(**vote_data)-+# Verify only one vote exists (in a new transaction)with transaction.atomic():count = SearchResultVote.objects.filter(- user=verified_user,- url=vote_data['url'],- query=vote_data['query']+ user=verified_user, url=vote_data["url"], query=vote_data["query"]).count()assert count == 1