Skip to content

web search done right

Fixed bug where monogram term was not defined. This can happen if a monogram is not part of top_n but the same term in a bigram was

Mikkel Denker693800f817f0
diff --git a/core/src/spell/dictionary.rs b/core/src/spell/dictionary.rs
index b3e55b9e..5b913e03 100644
--- a/core/src/spell/dictionary.rs
+++ b/core/src/spell/dictionary.rs
@@ -291,8 +291,12 @@ impl InnerDictionary {
#[inline]
pub fn score(&self, before: &[&str], term: &str, after: &[&str]) -> Option<f64> {
self.terms.get(term).map(|id| {
- let mut d = 1;
- let mut total_score = self.monograms[&Monogram(*id)] as f64 / self.total_freq as f64;
+ let mut d = 0;
+ let mut total_score = 0.0;
+ if let Some(mono) = self.monograms.get(&Monogram(*id)) {
+ total_score += *mono as f64 / self.total_freq as f64;
+ d += 1;
+ }
if !before.is_empty() {
if let Some(id_before) = self.terms.get(before[before.len() - 1]) {
@@ -357,7 +361,11 @@ impl InnerDictionary {
}
}
- total_score / d as f64
+ if d == 0 {
+ 0.0
+ } else {
+ total_score / d as f64
+ }
})
}
diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rs
index b6f03f02..3b624387 100644
--- a/core/src/spell/mod.rs
+++ b/core/src/spell/mod.rs
@@ -119,7 +119,7 @@ impl Spell {
let searcher = index.inverted_index.tv_searcher();
let schema = searcher.schema();
let mut dict = DictionaryBuilder::new(100_000);
- #[allow(unused_assignments)]
+ #[allow(unused_assignments, unused_mut)]
let mut limit_terms: Option<usize> = None;
#[cfg(debug_assertions)]
{
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index 1e2144cb..11ca6ae3 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -7,8 +7,8 @@ import pandas as pd
from tqdm import tqdm
STRACT_ENDPOINT = "localhost:3000"
-NUM_QUERIES = 1
-TOP_N_RESULTS = 1
+NUM_QUERIES = 10
+TOP_N_RESULTS = 10
CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'
@@ -17,8 +17,10 @@ openai.api_key = os.getenv("OPENAI_API_KEY")
def search(q, top_n, optic=None):
url = f"http://{STRACT_ENDPOINT}/beta/api/search"
+ print(q)
r = requests.post(
url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})
+ print(r)
search_result = r.json()['webpages']
res = []