An open source, non-profit web search engine
Swap bandit reward to judge scores; judge-rank the final results frame
Daoud Clarke18f677e1c4be
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 13be25d..b50cdf2 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -288,6 +288,13 @@ SUPER_SEARCH_MAX_LINKS_PER_PAGE = 3SUPER_SEARCH_RESULTS_PER_SOURCE = 10SUPER_SEARCH_FINAL_RESULTS_LIMIT = 100SUPER_SEARCH_CRAWL_WORKERS = 8 # threads in the dedicated crawl pool+# Fine-tuned relevance judge (dir with model.onnx + tokenizer.json; see+# devdata/judge_train/RESULTS.md). If the artifact is missing, Super Search+# falls back to LTR final ranking and top-K-survival bandit rewards.+SUPER_SEARCH_JUDGE_MODEL_DIR = os.environ.get(+ "SUPER_SEARCH_JUDGE_MODEL_DIR",+ str(Path(__file__).parent.parent / "devdata" / "judge_train" / "models"+ / "minilm-both-v1" / "onnx"))# Super Search source selection (bandit / cosine profiles)SUPER_SEARCH_SOURCES_TO_QUERY = 10 # max sources queried per searchdiff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.pyindex 89c88f7..4ef8cb5 100644--- a/mwmbl/tinysearchengine/super_search.py+++ b/mwmbl/tinysearchengine/super_search.py@@ -49,10 +49,12 @@ from mwmbl.tinysearchengine.mmr_rank import mmr_rerankfrom mwmbl.tinysearchengine.rank import score_result_wholefrom mwmbl.tinysearchengine.super_search_sources import SOURCESfrom mwmbl.tinysearchengine.super_search_select import bandit as ss_bandit+from mwmbl.tinysearchengine.super_search_select import judge as ss_judgefrom mwmbl.tinysearchengine.super_search_select import profiles as ss_profilesfrom mwmbl.tinysearchengine.super_search_select.policy import select_sourcesfrom mwmbl.tinysearchengine.super_search_select.rewards import (SelectionContext,+ compute_judge_rewards,compute_rewards,log_impression,record_source_provenance,@@ -435,9 +437,17 @@ async def _follow_links(await _emit_final_results(query, all_docs, emit, last_results_key, lock, ctx)+def _judge_score_docs(query: str, docs: list[Document]) -> list[float] | None:+ """Score docs with the fine-tuned relevance judge; None if it is unavailable."""+ judge = ss_judge.get_judge()+ if judge is None:+ return None+ return judge.score(query, [ss_judge.doc_text(d.title, d.extract) for d in docs])++async def _emit_final_results(query: str, all_docs: list[Document], emit, last_results_key: list, lock: asyncio.Lock,- ctx: SelectionContext,+ ctx: SelectionContext, use_judge: bool = False,) -> None:terms = tokenize(query)final_limit = getattr(settings, "SUPER_SEARCH_FINAL_RESULTS_LIMIT", 100)@@ -460,7 +470,17 @@ async def _emit_final_results(if not unique:return- final_scores = await asyncio.to_thread(score_documents, ltr_model, query, unique)+ # The end-of-search frame is re-ranked by the fine-tuned relevance+ # judge when available (the LTR model anti-correlates with human+ # curation — see devdata/judge_train/RESULTS.md); the per-URL judge+ # scores double as the bandit's per-source reward signal.+ final_scores = None+ if use_judge:+ final_scores = await asyncio.to_thread(_judge_score_docs, query, unique)+ if final_scores is not None:+ ctx.judge_scores = dict(zip((doc.url for doc in unique), final_scores))+ if final_scores is None:+ final_scores = await asyncio.to_thread(score_documents, ltr_model, query, unique)ranked = sorted(zip(unique, final_scores), key=lambda x: -x[1])[:final_limit]# Diversify with MMR (demotes, never drops, same-domain / near-duplicate# results) to match standard search — see MMRRanker in search_setup.py.@@ -515,12 +535,19 @@ def _enqueue_discovered_urls(ctx: SelectionContext) -> None:async def _record_rewards(query: str, ctx: SelectionContext, last_results_key: list) -> None:- """Compute implicit per-source rewards from the final top-K and log the impression."""+ """Compute per-source rewards and log the impression.++ Preferred reward: mean relevance-judge score of each source's final-pool+ documents (recorded by the judged final re-rank). Fallback when the judge+ is unavailable: fraction of a source's results surviving the final top-K.+ """if not ctx.selected:return- final_urls = list(last_results_key[0] or ())- top_k = getattr(settings, "SUPER_SEARCH_TOP_K", 10)- rewards = compute_rewards(ctx, final_urls[:top_k])+ rewards = compute_judge_rewards(ctx)+ if rewards is None:+ final_urls = list(last_results_key[0] or ())+ top_k = getattr(settings, "SUPER_SEARCH_TOP_K", 10)+ rewards = compute_rewards(ctx, final_urls[:top_k])try:await asyncio.to_thread(log_impression, query, ctx, rewards)except Exception:@@ -665,7 +692,8 @@ async def _sse_stream(query: str, monthly_usage: int, monthly_limit: int):finally:if reason in ("complete", "timed_out"):try:- await _emit_final_results(query, all_docs, emit, last_results_key, results_lock, selection_ctx)+ await _emit_final_results(query, all_docs, emit, last_results_key,+ results_lock, selection_ctx, use_judge=True)except Exception:logger.exception("super-search failed to emit final results")pages_indexed = await _index_results(query, all_docs)@@ -734,7 +762,7 @@ def init_router() -> None:"| `result_promoted` | result item (`origin=\"direct\"`) | A result entered the live top-K and will have its outbound links followed. |\n""| `page_fetched` | `{url, links}` | A promoted page was crawled; `links` is the number of outbound links found. |\n""| `link_followed` | `{url, from}` | An outbound link from a crawled page was fetched and added to the candidate pool. |\n"- "| `results` | `{results[], count}` | The current authoritative ranking (items have `origin=\"final\"`). Emitted progressively after each source and once more at the end — **replace** your displayed list on each. |\n"+ "| `results` | `{results[], count}` | The current authoritative ranking (items have `origin=\"final\"`). Emitted progressively after each source and once more at the end; the end-of-search frame is re-ranked by the fine-tuned relevance judge when available — **replace** your displayed list on each. |\n""| `error` | `{message}` | The pipeline crashed; the stream ends. |\n""| `done` | `{reason, elapsed_seconds, monthly_usage, monthly_limit, pages_indexed}` | Terminal event. `reason` is `complete`, `timed_out`, `cancelled` or `error`; `pages_indexed` is the number of new pages added to the Mwmbl index by this search. |\n\n""The exact JSON shape of every payload is given by the `oneOf` schema "diff --git a/mwmbl/tinysearchengine/super_search_select/judge.py b/mwmbl/tinysearchengine/super_search_select/judge.pynew file mode 100644index 0000000..ee9488e--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_select/judge.py@@ -0,0 +1,94 @@+"""Local relevance judge: fine-tuned cross-encoder served over ONNX on CPU.++Scores (query, title+extract) pairs in [0, 1] with the judge trained by+scripts/modal_judge_train.py (winner: multi-task MiniLM; see+devdata/judge_train/RESULTS.md). Used at the end of a Super Search to+re-rank the final results for the user and to compute per-source bandit+rewards — replacing the circular "survived the LTR top-K" reward.++The model is loaded lazily from ``settings.SUPER_SEARCH_JUDGE_MODEL_DIR``+(a directory holding ``model.onnx`` + ``tokenizer.json``). If the artifact+or onnxruntime is unavailable the judge is disabled — ``get_judge()``+returns None and callers fall back to LTR ranking / survival rewards — so+deploys without the model artifact keep working unchanged.++Queries pass through this module in memory only; nothing is persisted.+"""+from __future__ import annotations++import logging+import threading+from pathlib import Path++import numpy as np+from django.conf import settings++logger = logging.getLogger(__name__)++DOC_CHARS = 1000 # matches judge training (scripts/judge_bakeoff.py DOC_CHARS)+MAX_TOKENS = 256 # matches training max_length+BATCH_SIZE = 64+++def doc_text(title: str | None, extract: str | None) -> str:+ """Assemble the document text exactly as the judge was trained on."""+ return f"{title or ''}. {extract or ''}".strip()[:DOC_CHARS]+++class Judge:+ def __init__(self, model_dir: Path):+ import onnxruntime+ from tokenizers import Tokenizer++ self.tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))+ self.tokenizer.enable_truncation(max_length=MAX_TOKENS)+ self.tokenizer.enable_padding()+ self.session = onnxruntime.InferenceSession(+ str(model_dir / "model.onnx"), providers=["CPUExecutionProvider"])+ self.input_names = {i.name for i in self.session.get_inputs()}++ def score(self, query: str, doc_texts: list[str]) -> list[float]:+ """Relevance of each doc text to the query, each in [0, 1]."""+ scores: list[float] = []+ for start in range(0, len(doc_texts), BATCH_SIZE):+ batch = doc_texts[start:start + BATCH_SIZE]+ encodings = self.tokenizer.encode_batch([(query, text) for text in batch])+ feed = {+ "input_ids": np.array([e.ids for e in encodings], dtype=np.int64),+ "attention_mask": np.array([e.attention_mask for e in encodings],+ dtype=np.int64),+ }+ if "token_type_ids" in self.input_names:+ feed["token_type_ids"] = np.array([e.type_ids for e in encodings],+ dtype=np.int64)+ logits = self.session.run(None, feed)[0][:, 0].astype(np.float64)+ scores.extend(float(s) for s in 1 / (1 + np.exp(-logits)))+ return scores+++_judge: Judge | None = None+_load_attempted = False+_lock = threading.Lock()+++def get_judge() -> Judge | None:+ """Lazily load the shared judge; None (memoized) if unavailable."""+ global _judge, _load_attempted+ if _load_attempted:+ return _judge+ with _lock:+ if _load_attempted:+ return _judge+ model_dir = Path(getattr(settings, "SUPER_SEARCH_JUDGE_MODEL_DIR", ""))+ try:+ if (model_dir / "model.onnx").exists():+ _judge = Judge(model_dir)+ logger.info("relevance judge loaded from %s", model_dir)+ else:+ logger.warning(+ "relevance judge model not found at %s; falling back to "+ "LTR ranking and survival rewards", model_dir)+ except Exception:+ logger.exception("failed to load relevance judge from %s", model_dir)+ _load_attempted = True+ return _judgediff --git a/mwmbl/tinysearchengine/super_search_select/rewards.py b/mwmbl/tinysearchengine/super_search_select/rewards.pyindex 58791c3..5c53cc2 100644--- a/mwmbl/tinysearchengine/super_search_select/rewards.py+++ b/mwmbl/tinysearchengine/super_search_select/rewards.py@@ -1,11 +1,14 @@"""Implicit reward and impression logging for source selection.-The reward for a queried source is the fraction of its results that survive into-the final LTR-ranked top-K — a self-contained "did this source contribute"-signal needing no click logging. ``SelectionContext`` carries the per-request-state (which sources were selected, and which URL came from which source) from-the pipeline to the completion hook, where ``compute_rewards`` and-``log_impression`` run.+The preferred reward for a queried source is the mean relevance-judge score of+the documents it contributed to the final pool (``compute_judge_rewards``; see+super_search_select/judge.py) — an independent signal validated against LLM+labels and human curations. When the judge is unavailable the fallback is the+original survival reward: the fraction of a source's results in the final+LTR-ranked top-K (``compute_rewards``). ``SelectionContext`` carries the+per-request state (which sources were selected, which URL came from which+source, and the judge scores from the final re-rank) from the pipeline to the+completion hook, where reward computation and ``log_impression`` run."""from __future__ import annotations@@ -24,6 +27,7 @@ class SelectionContext:selected: list[str] = field(default_factory=list) # sources actually queriedsource_by_url: dict[str, str] = field(default_factory=dict) # url -> originating sourcefeatures: dict[str, list[float]] = field(default_factory=dict) # source -> feature vector+ judge_scores: dict[str, float] = field(default_factory=dict) # url -> judge score (final pool)per_source_limit: int = 0def record_results(self, source: str, urls: list[str]) -> None:@@ -48,6 +52,29 @@ def compute_rewards(ctx: SelectionContext, final_top_k_urls: list[str]) -> dict[return {name: min(r, 1.0) for name, r in rewards.items()}+def compute_judge_rewards(ctx: SelectionContext) -> dict[str, float] | None:+ """Reward per selected source = mean judge score of its final-pool documents.++ Uses the per-URL relevance-judge scores recorded during the final re-rank+ (``ctx.judge_scores``). The mean over a source's contributed documents is+ the aggregate the judge was validated on (source-level agreement in the+ bake-off). Every selected source gets an entry — 0.0 when nothing it+ returned reached the final pool, since the bandit needs the zeros too.+ Returns None when no judge scores exist (judge unavailable or no results);+ callers fall back to ``compute_rewards``.+ """+ if not ctx.judge_scores:+ return None+ totals = {name: [0.0, 0] for name in ctx.selected}+ for url, score in ctx.judge_scores.items():+ source = ctx.source_by_url.get(url)+ if source in totals:+ totals[source][0] += score+ totals[source][1] += 1+ return {name: total / count if count else 0.0+ for name, (total, count) in totals.items()}++def log_impression(query: str, ctx: SelectionContext, rewards: dict[str, float]) -> None:"""Persist a SuperSearchImpression row (no-op without a database)."""if not getattr(settings, "HAS_DATABASE", False):diff --git a/pyproject.toml b/pyproject.tomlindex 6f46ce5..df6a1f8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -51,6 +51,8 @@ dependencies = [# Used by the Super Search recipe engine (super_search_sources/recipe.py),# which is imported at app startup — must be a core dep, not indexer-only."beautifulsoup4==4.10.0",+ "onnxruntime>=1.27.0",+ "tokenizers>=0.23.1",][project.optional-dependencies]diff --git a/test/test_super_search_judge.py b/test/test_super_search_judge.pynew file mode 100644index 0000000..1d30e1d--- /dev/null+++ b/test/test_super_search_judge.py@@ -0,0 +1,40 @@+"""Tests for the fine-tuned relevance judge (super_search_select/judge.py)."""+from pathlib import Path++import pytest+from django.conf import settings++from mwmbl.tinysearchengine.super_search_select import judge as ss_judge++MODEL_DIR = Path(getattr(settings, "SUPER_SEARCH_JUDGE_MODEL_DIR", ""))+++def test_doc_text_matches_training_format():+ assert ss_judge.doc_text("Title", "Extract") == "Title. Extract"+ assert ss_judge.doc_text(None, "Extract") == ". Extract"+ assert ss_judge.doc_text("Title", None) == "Title."+ assert len(ss_judge.doc_text("t" * 2000, "e")) == ss_judge.DOC_CHARS+++def test_get_judge_unavailable_returns_none(monkeypatch):+ monkeypatch.setattr(ss_judge, "_judge", None)+ monkeypatch.setattr(ss_judge, "_load_attempted", False)+ monkeypatch.setattr(settings, "SUPER_SEARCH_JUDGE_MODEL_DIR", "/nonexistent")+ assert ss_judge.get_judge() is None+ # memoized: second call must not retry the load+ assert ss_judge._load_attempted+ assert ss_judge.get_judge() is None++[email protected](not (MODEL_DIR / "model.onnx").exists(),+ reason="judge model artifact not present")+def test_judge_scores_relevant_above_irrelevant():+ judge = ss_judge.Judge(MODEL_DIR)+ scores = judge.score("python asyncio tutorial", [+ ss_judge.doc_text("Async IO in Python: A Complete Walkthrough",+ "A tutorial covering asyncio, coroutines and event loops."),+ ss_judge.doc_text("Best chocolate cake recipes",+ "Moist chocolate cake with simple ingredients."),+ ])+ assert all(0.0 <= s <= 1.0 for s in scores)+ assert scores[0] > scores[1]diff --git a/test/test_super_search_select.py b/test/test_super_search_select.pyindex c35e5f6..b8fbca0 100644--- a/test/test_super_search_select.py+++ b/test/test_super_search_select.py@@ -169,6 +169,29 @@ def test_record_results_first_source_wins():assert ctx.source_by_url["shared"] == "a"+def test_compute_judge_rewards_mean_per_source():+ ctx = SelectionContext(selected=["a", "b", "c"])+ ctx.record_results("a", ["u1", "u2"])+ ctx.record_results("b", ["u3"])+ # c returned nothing; u2 never reached the final pool so has no score+ ctx.judge_scores = {"u1": 0.8, "u3": 0.4}+ r = rewards.compute_judge_rewards(ctx)+ assert r == {"a": pytest.approx(0.8), "b": pytest.approx(0.4), "c": 0.0}+++def test_compute_judge_rewards_averages_multiple_docs():+ ctx = SelectionContext(selected=["a"])+ ctx.record_results("a", ["u1", "u2"])+ ctx.judge_scores = {"u1": 1.0, "u2": 0.0}+ assert rewards.compute_judge_rewards(ctx) == {"a": pytest.approx(0.5)}+++def test_compute_judge_rewards_none_without_scores():+ ctx = SelectionContext(selected=["a"])+ ctx.record_results("a", ["u1"])+ assert rewards.compute_judge_rewards(ctx) is None++def test_log_impression_noop_without_database(monkeypatch):monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)# Should not raise even though no DB is configured.diff --git a/uv.lock b/uv.lockindex 3254076..aeae93d 100644--- a/uv.lock+++ b/uv.lock@@ -176,14 +176,14 @@ wheels = [[[package]]name = "click"-version = "8.3.2"+version = "8.4.2"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "colorama", marker = "sys_platform == 'win32'" },]-sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" }+sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" },+ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" },][[package]]@@ -449,12 +449,38 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/6f/27/b8b057a23f7777177e92d3a602fd866751b6b45014964548997e92e048fd/fakeredis-2.35.1-py3-none-any.whl", hash = "sha256:67d97e11f562b7870e11e5c30cf182270bfb2dd37f6707dba47cc6d91628d1b9", size = 129678, upload-time = "2026-04-12T17:05:56.86Z" },]+[[package]]+name = "filelock"+version = "3.29.5"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/e3/ee/29c668c50888588c432a702f7c2e8ee8a0c9e5286028d91f170308d6b2e9/filelock-3.29.5.tar.gz", hash = "sha256:6e6034c57a00a020e767f2614a5539863f056de7e7991d6d1473aef7ff73f156", size = 68927, upload-time = "2026-07-03T03:50:31.818Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/4a/e3/f1fae3647d170919c2cf2a898e77e7d1a4e5c7cae0aed7bb4bd3f5ebff6f/filelock-3.29.5-py3-none-any.whl", hash = "sha256:8af830889ba3a0ffcefbd6c7d2af8a54012058103771f2e10848222f476a1693", size = 45073, upload-time = "2026-07-03T03:50:30.445Z" },+]++[[package]]+name = "flatbuffers"+version = "25.12.19"+source = { registry = "https://pypi.org/simple" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" },+]+[[package]]name = "forbiddenfruit"version = "0.1.4"source = { registry = "https://pypi.org/simple" }sdist = { url = "https://files.pythonhosted.org/packages/e6/79/d4f20e91327c98096d605646bdc6a5ffedae820f38d378d3515c42ec5e60/forbiddenfruit-0.1.4.tar.gz", hash = "sha256:e3f7e66561a29ae129aac139a85d610dbf3dd896128187ed5454b6421f624253", size = 43756, upload-time = "2021-01-16T21:03:35.401Z" }+[[package]]+name = "fsspec"+version = "2026.6.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" },+]+[[package]]name = "gunicorn"version = "25.3.0"@@ -476,6 +502,22 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },]+[[package]]+name = "hf-xet"+version = "1.5.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" },+ { url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" },+ { url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" },+ { url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" },+ { url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" },+ { url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" },+ { url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" },+ { url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" },+]+[[package]]name = "hiredis"version = "3.3.1"@@ -525,6 +567,26 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },]+[[package]]+name = "huggingface-hub"+version = "1.22.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "click" },+ { name = "filelock" },+ { name = "fsspec" },+ { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },+ { name = "httpx" },+ { name = "packaging" },+ { name = "pyyaml" },+ { name = "tqdm" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/77/ea/dc54b4dda5841cb3a7812a178695be776e7c15c597887c2ed892f17d015a/huggingface_hub-1.22.0.tar.gz", hash = "sha256:e2dfe5fe1ec3b87ba2709aa34555b23e3f3f6ad4d7255238e13ddb8348e6bbfa", size = 914232, upload-time = "2026-07-03T09:46:44.685Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/64/9c/a1a377265abd8b823a2c661c665028ccb6b9fba1ca9d08e52ff679c20ecd/huggingface_hub-1.22.0-py3-none-any.whl", hash = "sha256:b09e19309ae09ee0a71892701c4fe70af39ab4e00817321dc62f2289a977249b", size = 765085, upload-time = "2026-07-03T09:46:42.832Z" },+]+[[package]]name = "idna"version = "3.3"@@ -729,6 +791,7 @@ dependencies = [{ name = "mmh3" },{ name = "numpy" },{ name = "objgraph" },+ { name = "onnxruntime" },{ name = "pandas" },{ name = "polar-sdk" },{ name = "psutil" },@@ -744,6 +807,7 @@ dependencies = [{ name = "scikit-learn" },{ name = "scipy" },{ name = "sentry-sdk" },+ { name = "tokenizers" },{ name = "uvicorn" },{ name = "wasmtime" },{ name = "xgboost" },@@ -800,6 +864,7 @@ requires-dist = [{ name = "mmh3", specifier = ">=3.0.0" },{ name = "numpy", specifier = "==1.23.2" },{ name = "objgraph", specifier = ">=3.6.1" },+ { name = "onnxruntime", specifier = ">=1.27.0" },{ name = "pandas", specifier = ">=1.3.5,<2.0.0" },{ name = "polar-sdk", specifier = "==0.31.7" },{ name = "psutil", specifier = ">=6.0.0" },@@ -817,6 +882,7 @@ requires-dist = [{ name = "scikit-learn", specifier = ">=1.0.2" },{ name = "scipy", specifier = ">=1.8.0" },{ name = "sentry-sdk", specifier = ">=2.13.0" },+ { name = "tokenizers", specifier = ">=0.23.1" },{ name = "ujson", marker = "extra == 'indexer'", specifier = "==4.3.0" },{ name = "uvicorn", specifier = ">=0.16.0" },{ name = "warcio", marker = "extra == 'indexer'", specifier = "==1.7.4" },@@ -880,6 +946,24 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/8e/67/7bffbb861cb8a0a62b7df50738d35812bf40dc8bcc1559c04bdf593f1164/objgraph-3.6.2-py3-none-any.whl", hash = "sha256:8114c97712291c3ba30d882406a384d0a7651b307ea9a06e0d83836ccde85e15", size = 17667, upload-time = "2024-10-10T12:00:41.581Z" },]+[[package]]+name = "onnxruntime"+version = "1.27.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "flatbuffers" },+ { name = "numpy" },+ { name = "packaging" },+ { name = "protobuf" },+]+wheels = [+ { url = "https://files.pythonhosted.org/packages/d4/e4/5353d7e09ced4a8f473f843223fc75d726b2b5519dcefc12f22a6c92852d/onnxruntime-1.27.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:8ba14a38c570087f3cdb8cfba33f7a38a1e826c1e5b29e17c28ceda0cc910016", size = 18416484, upload-time = "2026-06-15T22:43:43.894Z" },+ { url = "https://files.pythonhosted.org/packages/ed/1f/a2117aa3f144fce88774efa37440d0ca72d0c9144854dfc0961f2b04c6fc/onnxruntime-1.27.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2eb083321af8a236a84c7c140a7f4cecbfa2a987a18c07c78db471c20cd390ef", size = 16419330, upload-time = "2026-06-15T22:42:37.58Z" },+ { url = "https://files.pythonhosted.org/packages/e0/cd/74bb804170ceb622fda9111df31a07b3024f7491472256d3a90b5391a4d2/onnxruntime-1.27.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4f7b0e90d2d212e2c2deaa6c8291616183ab815d3ec558ea12d3ac8b26d36f4", size = 18636930, upload-time = "2026-06-15T22:43:01.584Z" },+ { url = "https://files.pythonhosted.org/packages/fe/8f/5b8e2b85e81735696887175dbaf6409f215683f5ca9d4928fbb038211d32/onnxruntime-1.27.0-cp311-cp311-win_amd64.whl", hash = "sha256:ff050e4f6bf7f12918fa14dcb047c0b02e295f35e86d42532552be4b3d54e977", size = 13356110, upload-time = "2026-06-15T22:43:32.172Z" },+ { url = "https://files.pythonhosted.org/packages/b0/3a/4f568de678126b6a371a93862f015a82138359decd97fcac61fc84b5b774/onnxruntime-1.27.0-cp311-cp311-win_arm64.whl", hash = "sha256:75fbc1e1fb43a39a856c8209c544cca7817b5de7ac16b15b1bdf55d1cc67b9df", size = 13098635, upload-time = "2026-06-15T22:43:19.607Z" },+]+[[package]]name = "packaging"version = "26.0"@@ -970,6 +1054,21 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/dc/e0/a88026b9432c7e2c0c096911ddc97cb06f483036a00a8afd5254efe1c16a/polar_sdk-0.31.7-py3-none-any.whl", hash = "sha256:acff8f3405046d45fa31130674f269314586fee1d3d2fec221eed21cb8697340", size = 902336, upload-time = "2026-06-12T06:51:01.002Z" },]+[[package]]+name = "protobuf"+version = "7.35.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" },+ { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" },+ { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" },+ { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" },+ { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" },+ { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" },+ { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" },+]+[[package]]name = "psutil"version = "7.2.2"@@ -1495,6 +1594,45 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" },]+[[package]]+name = "tokenizers"+version = "0.23.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "huggingface-hub" },+]+sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" },+ { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" },+ { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" },+ { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" },+ { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" },+ { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" },+ { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" },+ { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" },+ { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" },+ { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" },+ { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" },+ { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" },+ { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" },+ { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" },+ { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" },+ { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" },+]++[[package]]+name = "tqdm"+version = "4.68.3"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "colorama", marker = "sys_platform == 'win32'" },+]+sdist = { url = "https://files.pythonhosted.org/packages/87/d7/0535a28b1f5f24f6612fb3ff1e89fb1a8d160fee0f976e0aa6803862134b/tqdm-4.68.3.tar.gz", hash = "sha256:00dfa48452b6b6cfae3dd9885636c23d3422d1ec97c66d96818cbd5e0821d482", size = 170596, upload-time = "2026-06-17T07:36:52.105Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl", hash = "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03", size = 78337, upload-time = "2026-06-17T07:36:50.132Z" },+]+[[package]]name = "types-deprecated"version = "1.3.1.20260408"