web search done right
Fixed bug where monogram term was not defined. This can happen if a monogram is not part of top_n but the same term in a bigram was
Mikkel Denker693800f817f0
diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rsindex b3e55b9e..5b913e03 100644--- a/core/src/spell/dictionary.rs+++ b/core/src/spell/dictionary.rs@@ -291,8 +291,12 @@ impl InnerDictionary {#[inline]pub fn score(&self, before: &[&str], term: &str, after: &[&str]) -> Option<f64> {self.terms.get(term).map(|id| {- let mut d = 1;- let mut total_score = self.monograms[&Monogram(*id)] as f64 / self.total_freq as f64;+ let mut d = 0;+ let mut total_score = 0.0;+ if let Some(mono) = self.monograms.get(&Monogram(*id)) {+ total_score += *mono as f64 / self.total_freq as f64;+ d += 1;+ }if !before.is_empty() {if let Some(id_before) = self.terms.get(before[before.len() - 1]) {@@ -357,7 +361,11 @@ impl InnerDictionary {}}- total_score / d as f64+ if d == 0 {+ 0.0+ } else {+ total_score / d as f64+ }})}diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rsindex b6f03f02..3b624387 100644--- a/core/src/spell/mod.rs+++ b/core/src/spell/mod.rs@@ -119,7 +119,7 @@ impl Spell {let searcher = index.inverted_index.tv_searcher();let schema = searcher.schema();let mut dict = DictionaryBuilder::new(100_000);- #[allow(unused_assignments)]+ #[allow(unused_assignments, unused_mut)]let mut limit_terms: Option<usize> = None;#[cfg(debug_assertions)]{diff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex 1e2144cb..11ca6ae3 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -7,8 +7,8 @@ import pandas as pdfrom tqdm import tqdmSTRACT_ENDPOINT = "localhost:3000"-NUM_QUERIES = 1-TOP_N_RESULTS = 1+NUM_QUERIES = 10+TOP_N_RESULTS = 10CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'@@ -17,8 +17,10 @@ openai.api_key = os.getenv("OPENAI_API_KEY")def search(q, top_n, optic=None):url = f"http://{STRACT_ENDPOINT}/beta/api/search"+ print(q)r = requests.post(url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})+ print(r)search_result = r.json()['webpages']res = []