web search done right
Fixed bug where crossencoder scores didn't make sense. For some reason, the ouput of the model needs to be f32 and can then later be cast to f64. No idea why
Mikkel Denkerd9e84c311f8a
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex 609877d5..07d613e6 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -174,7 +174,10 @@ pub async fn route(SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),},Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),- Err(_) => panic!("Search failed"), // TODO: show 500 status to user here+ Err(_err) => {+ dbg!(_err);+ panic!("Search failed")+ } // TODO: show 500 status to user here}}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 5f678ef7..7ae2c156 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -70,7 +70,7 @@ fn proximity_queries(let tantivy_field = schema.get_field(field.name()).unwrap();let tantivy_entry = schema.get_field_entry(tantivy_field);- for (boost, slop) in [(6, 0), (5, 1), (4, 2), (3, 4), (2, 16), (1, 32)] {+ for (boost, slop) in [(32, 0), (16, 1), (8, 2), (4, 4), (2, 16), (1, 32)] {let mut terms = Vec::new();let mut num_terms = 0;diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rsindex 3a498bd0..7c0bd050 100644--- a/core/src/ranking/models/cross_encoder.rs+++ b/core/src/ranking/models/cross_encoder.rs@@ -60,17 +60,24 @@ impl CrossEncoder for CrossEncoderModel {fn run(&self, query: &str, body: &str) -> f64 {let encoded = self.tokenizer.encode((query, body), true).unwrap();- let ids = encoded.get_ids().iter().map(|i| *i as i64).collect_vec();+ let ids = encoded+ .get_ids()+ .iter()+ .map(|i| *i as i64)+ .take(512)+ .collect_vec();let attention_mask = encoded.get_attention_mask().iter()+ .take(512).map(|i| *i as i64).collect_vec();let type_ids = encoded.get_type_ids().iter()+ .take(512).map(|i| *i as i64).collect_vec();@@ -78,7 +85,7 @@ impl CrossEncoder for CrossEncoderModel {let mut sess = self.session.lock().unwrap();- let res: Vec<OrtOwnedTensor<f64, _>> = sess+ let res: Vec<OrtOwnedTensor<f32, _>> = sess.run(vec![ArrayBase::from_vec(ids).into_shape((1, num_tokens))@@ -92,7 +99,10 @@ impl CrossEncoder for CrossEncoderModel {]).unwrap();- res[0][[0, 0]]+ let res = res[0][[0, 0]] as f64;++ let s = res.exp();+ s / (s + 1.0)}}@@ -104,6 +114,21 @@ mod tests {fn sanity_check() {match CrossEncoderModel::open("../data/cross_encoder") {Ok(model) => {+ let s = model.run(+ "how many people live in paris",+ "there are currently 1234 people living in paris",+ );++ for _ in 0..10 {+ assert_eq!(+ s,+ model.run(+ "how many people live in paris",+ "there are currently 1234 people living in paris"+ )+ );+ }+assert!(model.run("how many people live in paris",@@ -115,7 +140,7 @@ mod tests {);}Err(_err) => {- // dbg!(err);+ // dbg!(_err);// panic!();}}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex b35b2c42..87887012 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -94,7 +94,7 @@ impl RankingWebsite {Field::Text(TextField::Title) => {res.title = value.value.as_text().unwrap().to_string()}- Field::Text(TextField::CleanBody) => {+ Field::Text(TextField::StemmedCleanBody) => {res.clean_body = value.value.as_text().unwrap().to_string()}_ => {}@@ -121,7 +121,7 @@ struct ReRanker<M: CrossEncoder> {impl<M: CrossEncoder> ReRanker<M> {fn new(model: Arc<M>) -> Self {Self {- coefficient: 100.0,+ coefficient: 10000.0,model,prev_score: 1.0,query: String::new(),@@ -164,7 +164,7 @@ struct PrioritizeText {impl Default for PrioritizeText {fn default() -> Self {Self {- bm25: 1.0,+ bm25: 10.0,prev_score: 1.0,}}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 3eda45ad..387aa13a 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -395,7 +395,7 @@ impl Field {}Field::Text(TextField::CleanBody) => Some(4.0),Field::Text(TextField::DmozDescription) => Some(3.0),- Field::Text(TextField::Title) => Some(10.0),+ Field::Text(TextField::Title) => Some(15.0),Field::Text(TextField::Url) => Some(1.0),Field::Text(TextField::Domain) => Some(1.0),Field::Text(TextField::AllBody) => Some(0.01),diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 030b180a..1b78b937 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -49,7 +49,7 @@ use super::{InitialWebsiteResult, SearchQuery, SearchResult};type Result<T> = std::result::Result<T, Error>;const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;-const DISCUSSIONS_WIDGET_THRESHOLD: f64 = 10.0;+const DISCUSSIONS_WIDGET_THRESHOLD: f64 = 100.0;struct RemoteSearcher {addr: SocketAddr,diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 3aab829f..bde2b18f 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -336,8 +336,7 @@ impl<S: Store> Webgraph<S> {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),live_segment: None,segments,- // executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),- executor: Arc::new(Executor::single_thread()),+ executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),meta,diff --git a/optics/testcases b/optics/testcasesindex 2bfb741a..8c44c1c7 160000--- a/optics/testcases+++ b/optics/testcases@@ -1 +1 @@-Subproject commit 2bfb741a62a69bd5256f8924348cd2d52429dcd5+Subproject commit 8c44c1c73904ea44a4b8485790b0a7afcd3bf94ddiff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex 83316637..2555584e 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -2,7 +2,7 @@import osimport transformers-MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"+MODEL = "cross-encoder/ms-marco-TinyBERT-L-2-v2"os.system("pip3 install transformers[onnx] torch")os.system("mkdir data/cross_encoder")diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.shindex 0a8564b1..7650b011 100644--- a/scripts/run_frontend.sh+++ b/scripts/run_frontend.sh@@ -1,4 +1,4 @@#!/bin/bashtrap 'kill $(jobs -p)' EXIT-cargo run -- search-server configs/search_server.toml &-cargo run -- frontend configs/frontend.toml+cargo run --release -- search-server configs/search_server.toml &+cargo run --release -- frontend configs/frontend.toml