Skip to content

web search done right

increase max politeness, faster setup and some documentation

Mikkel Denker1af94339bbcd
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 39c4ab39..0bce76dd 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -39,7 +39,7 @@ use super::{
const MIN_CRAWL_DELAY_MS: u64 = 200;
const MAX_CRAWL_DELAY_MS: u64 = 30_000;
-const MAX_POLITENESS_FACTOR: f32 = 128.0;
+const MAX_POLITENESS_FACTOR: f32 = 1024.0;
const MIN_POLITENESS_FACTOR: f32 = 1.0;
pub struct Worker {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index a8362ea0..429e74fc 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -997,6 +997,13 @@ impl SignalAggregator {
.unwrap_or_default()
}
+ /// Computes the scored signals for a given document.
+ ///
+ /// Important: This function assues that the docs a scored in ascending order of docid
+ /// within their segment. If this invariant is not upheld, the documents will not have
+ /// scores calculated for their text related signals. The wrong ranking will most likely
+ /// be returned.
+ /// This function also assumes that the segment reader has been set.
pub fn compute_signals(
&mut self,
doc: DocId,
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index 2f15f7cd..e1292e77 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -9,7 +9,7 @@ StractSearch is the name of our crawler that is used to collect pages from the w
## Politeness
StractSearch is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.
It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 30 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 30 seconds.
-This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 128).
+This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 1024).
The crawler looks for the token `StractSearch` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.
diff --git a/justfile b/justfile
index 7a9dd550..1bbd03a1 100644
--- a/justfile
+++ b/justfile
@@ -24,8 +24,7 @@
just cargo run --release --all-features -- configure {{ARGS}}
@setup_python_env:
- rm -rf .venv
- python3 -m venv .venv
+ python3 -m venv .venv || true
.venv/bin/pip install -r scripts/requirements.txt
@download_libtorch *ARGS:
diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_model
index 09e3ea5d..bb584ead 100755
--- a/scripts/export_abstractive_summary_model
+++ b/scripts/export_abstractive_summary_model
@@ -3,8 +3,13 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
import urllib.request
import torch
import os
+import sys
-os.makedirs("data/summarizer/abstractive", exist_ok=True)
+if os.path.exists("data/summarizer/abstractive"):
+ print("data/summarizer/abstractive already exists. Exiting...")
+ sys.exit()
+
+os.makedirs("data/summarizer/abstractive")
MODEL = "lidiya/bart-large-xsum-samsum"
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index 15f5db3b..d372a82b 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -2,11 +2,17 @@
import os
import transformers
import torch
+import sys
MODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
+if os.path.exists("data/cross_encoder/"):
+ print("data/cross_encoder/ already exists. Exiting...")
+ sys.exit()
+
os.system("mkdir data/cross_encoder")
+
model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)
model.eval()
diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoder
index 0462bca8..393f5407 100755
--- a/scripts/export_dual_encoder
+++ b/scripts/export_dual_encoder
@@ -3,6 +3,7 @@ import os
import transformers
import torch
import torch.nn.functional as F
+import sys
# MODEL = "nreimers/MiniLM-L6-H384-uncased"
MODEL = "sentence-transformers/all-MiniLM-L6-v2"
@@ -15,6 +16,10 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana
"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract
of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""
+if os.path.exists("data/summarizer/dual_encoder/"):
+ print("data/summarizer/dual_encoder/ already exists. Exiting...")
+ sys.exit()
+
os.system("mkdir data/summarizer/dual_encoder")
model = transformers.AutoModel.from_pretrained(MODEL)
diff --git a/scripts/export_fact_model b/scripts/export_fact_model
index e14dd385..a23164a8 100755
--- a/scripts/export_fact_model
+++ b/scripts/export_fact_model
@@ -2,9 +2,14 @@
import os
import transformers
import torch
+import sys
MODEL = "Dzeniks/roberta-fact-check"
+if os.path.exists("data/fact_model/"):
+ print("data/fact_model/ already exists. Exiting...")
+ sys.exit()
+
os.system("mkdir data/fact_model")
model = transformers.AutoModelForSequenceClassification.from_pretrained(MODEL)
diff --git a/scripts/export_qa_model b/scripts/export_qa_model
index cfafbddf..0a474ef8 100755
--- a/scripts/export_qa_model
+++ b/scripts/export_qa_model
@@ -2,6 +2,7 @@
import os
import transformers
import torch
+import sys
MODEL = "deepset/tinyroberta-squad2"
@@ -13,6 +14,10 @@ minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana
"Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract
of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."""
+if os.path.exists("data/qa_model/"):
+ print("data/qa_model/ already exists. Exiting...")
+ sys.exit()
+
os.system("mkdir data/qa_model")
model = transformers.AutoModelForQuestionAnswering.from_pretrained(MODEL)