web search done right
increase max politeness, faster setup and some documentation
Mikkel Denker1af94339bbcd
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 39c4ab39..0bce76dd 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -39,7 +39,7 @@ use super::{const MIN_CRAWL_DELAY_MS: u64 = 200;const MAX_CRAWL_DELAY_MS: u64 = 30_000;-const MAX_POLITENESS_FACTOR: f32 = 128.0;+const MAX_POLITENESS_FACTOR: f32 = 1024.0;const MIN_POLITENESS_FACTOR: f32 = 1.0;pub struct Worker {diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex a8362ea0..429e74fc 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -997,6 +997,13 @@ impl SignalAggregator {.unwrap_or_default()}+ /// Computes the scored signals for a given document.+ ///+ /// Important: This function assues that the docs a scored in ascending order of docid+ /// within their segment. If this invariant is not upheld, the documents will not have+ /// scores calculated for their text related signals. The wrong ranking will most likely+ /// be returned.+ /// This function also assumes that the segment reader has been set.pub fn compute_signals(&mut self,doc: DocId,diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdxindex 2f15f7cd..e1292e77 100644--- a/frontend/src/pages/crawler.mdx+++ b/frontend/src/pages/crawler.mdx@@ -9,7 +9,7 @@ StractSearch is the name of our crawler that is used to collect pages from the w## PolitenessStractSearch is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds.-This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 128).+This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 1024).The crawler looks for the token `StractSearch` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.diff --git a/justfile b/justfileindex 7a9dd550..1bbd03a1 100644--- a/justfile+++ b/justfile@@ -24,8 +24,7 @@just cargo run --release --all-features -- configure {{ARGS}}@setup_python_env:- rm -rf .venv- python3 -m venv .venv+ python3 -m venv .venv || true.venv/bin/pip install -r scripts/requirements.txt@download_libtorch *ARGS:diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_modelindex 09e3ea5d..bb584ead 100755--- a/scripts/export_abstractive_summary_model+++ b/scripts/export_abstractive_summary_model@@ -3,8 +3,13 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizerimport urllib.requestimport torchimport os+import sys-os.makedirs("data/summarizer/abstractive", exist_ok=True)+if os.path.exists("data/summarizer/abstractive"):+ print("data/summarizer/abstractive already exists. Exiting...")+ sys.exit()++os.makedirs("data/summarizer/abstractive")MODEL = "lidiya/bart-large-xsum-samsum"diff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex 15f5db3b..d372a82b 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -2,11 +2,17 @@import osimport transformersimport torch+import sysMODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"+if os.path.exists("data/cross_encoder/"):+ print("data/cross_encoder/ already exists. Exiting...")+ sys.exit()+os.system("mkdir data/cross_encoder")+model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)model.eval()diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoderindex 0462bca8..393f5407 100755--- a/scripts/export_dual_encoder+++ b/scripts/export_dual_encoder@@ -3,6 +3,7 @@ import osimport transformersimport torchimport torch.nn.functional as F+import sys# MODEL = "nreimers/MiniLM-L6-H384-uncased"MODEL = "sentence-transformers/all-MiniLM-L6-v2"@@ -15,6 +16,10 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tractof tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""+if os.path.exists("data/summarizer/dual_encoder/"):+ print("data/summarizer/dual_encoder/ already exists. Exiting...")+ sys.exit()+os.system("mkdir data/summarizer/dual_encoder")model = transformers.AutoModel.from_pretrained(MODEL)diff --git a/scripts/export_fact_model b/scripts/export_fact_modelindex e14dd385..a23164a8 100755--- a/scripts/export_fact_model+++ b/scripts/export_fact_model@@ -2,9 +2,14 @@import osimport transformersimport torch+import sysMODEL = "Dzeniks/roberta-fact-check"+if os.path.exists("data/fact_model/"):+ print("data/fact_model/ already exists. Exiting...")+ sys.exit()+os.system("mkdir data/fact_model")model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)diff --git a/scripts/export_qa_model b/scripts/export_qa_modelindex cfafbddf..0a474ef8 100755--- a/scripts/export_qa_model+++ b/scripts/export_qa_model@@ -2,6 +2,7 @@import osimport transformersimport torch+import sysMODEL = "deepset/tinyroberta-squad2"@@ -13,6 +14,10 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tractof tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""+if os.path.exists("data/qa_model/"):+ print("data/qa_model/ already exists. Exiting...")+ sys.exit()+os.system("mkdir data/qa_model")model = transformers.AutoModelForQuestionAnswering.from_pretrained(MODEL)