Skip to content

web search done right

return ranking signals from api

Mikkel Denker80ac0bd4bc16
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index d92b4969..9635a816 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -138,6 +138,7 @@ pub async fn route(
page: skip_pages,
site_rankings,
num_results: NUM_RESULTS_PER_PAGE,
+ return_ranking_signals: false,
})
.await
{
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index 650a4c78..12a21e7f 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -160,6 +160,7 @@ pub struct DisplayedWebpage {
pub snippet: Snippet,
pub body: String,
pub primary_image_uuid: Option<String>,
+ pub ranking_signals: Option<Vec<Option<f64>>>,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -197,6 +198,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
snippet,
body: webpage.body,
primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
+ ranking_signals: None,
}
}
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 40a1706a..8c5f71c5 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -27,6 +27,7 @@ use crate::{
ranking::{
models::cross_encoder::CrossEncoderModel,
pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},
+ ALL_SIGNALS,
},
search_prettifier::{
create_stackoverflow_sidebar, DisplayedAnswer, DisplayedWebpage,
@@ -506,9 +507,9 @@ impl DistributedSearcher {
return Ok(None);
}
- let results = self.combine_results(initial_results, pipeline);
+ let top_websites = self.combine_results(initial_results, pipeline);
- let scores: Vec<_> = results
+ let scores: Vec<_> = top_websites
.iter()
.map(|pointer| pointer.website.score)
.collect();
@@ -523,8 +524,8 @@ impl DistributedSearcher {
return Ok(None);
}
- let result = self
- .retrieve_webpages(&results, &query.query)
+ let result: Vec<DisplayedWebpage> = self
+ .retrieve_webpages(&top_websites, &query.query)
.await
.into_iter()
.map(DisplayedWebpage::from)
@@ -576,10 +577,22 @@ impl DistributedSearcher {
.map(DisplayedWebpage::from)
.collect();
- if retrieved_webpages.is_empty() && !top_websites.is_empty() {
+ if retrieved_webpages.len() != top_websites.len() {
return Err(Error::SearchFailed.into());
}
+ if query.return_ranking_signals {
+ for (website, pointer) in retrieved_webpages.iter_mut().zip(top_websites.iter()) {
+ let mut signals = Vec::with_capacity(ALL_SIGNALS.len());
+
+ for signal in ALL_SIGNALS {
+ signals.push(pointer.website.signals.get(signal).copied());
+ }
+
+ website.ranking_signals = Some(signals);
+ }
+ }
+
let direct_answer = self.answer(&query.query, &mut retrieved_webpages);
let search_duration_ms = start.elapsed().as_millis();
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 6bce9594..67ff6ce2 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -62,6 +62,7 @@ pub struct SearchQuery {
pub selected_region: Option<Region>,
pub optic_program: Option<String>,
pub site_rankings: Option<SiteRankings>,
+ pub return_ranking_signals: bool,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -85,6 +86,7 @@ impl Default for SearchQuery {
selected_region: Default::default(),
optic_program: Default::default(),
site_rankings: Default::default(),
+ return_ranking_signals: false,
}
}
}
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index f8be72fc..df84b831 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -7,9 +7,9 @@ import pandas as pd
from tqdm import tqdm
STRACT_ENDPOINT = "localhost:3000"
-NUM_SHUFFLES = 5
+NUM_SHUFFLES = 3
NUM_QUERIES = 100
-TOP_N_RESULTS = 20
+TOP_N_RESULTS = 10
openai.api_key = os.getenv("OPENAI_API_KEY")
@@ -17,7 +17,7 @@ openai.api_key = os.getenv("OPENAI_API_KEY")
def search(q):
url = f"http://{STRACT_ENDPOINT}/beta/api/search"
r = requests.post(
- url, json={"query": q, "offset": 0, "num_results": TOP_N_RESULTS})
+ url, json={"query": q, "page": 0, "num_results": TOP_N_RESULTS, "return_ranking_signals": True})
return r.json()['webpages']
@@ -47,44 +47,57 @@ def score(query, results):
queries = requests.get(
"https://s3.trystract.com/public/queries_us_big.csv").text.splitlines()
random.shuffle(queries)
-queries = queries[:NUM_QUERIES]
+ranking_signals = {}
scores = {'query': [], 'url': [], 'score': []}
-for query in tqdm(queries):
- search_result = search(query)
+queries_taken = 0
- res = []
+with tqdm(total=NUM_QUERIES) as pbar:
+ for query in queries:
+ if queries_taken >= NUM_QUERIES:
+ break
- for r in search_result:
- if 'Normal' not in r['snippet']:
- continue
+ search_result = search(query)
+
+ res = []
- snip = r['snippet']['Normal']
- res.append({"domain": r['domain'], "title": r['title'], "url": r['url'],
- "snippet": snip['text']})
+ for r in search_result:
+ if 'Normal' not in r['snippet']:
+ continue
- if len(res) == 0:
- continue
+ snip = r['snippet']['Normal']
+ res.append({"domain": r['domain'], "title": r['title'], "url": r['url'], "ranking_signals": r['ranking_signals'],
+ "snippet": snip['text']})
- for _ in range(NUM_SHUFFLES):
- random.shuffle(res)
- try:
- s = score(query, res)
- for i, s in enumerate(s.split(',')):
- s = float(s)
- url = res[i]['url']
+ if len(res) < TOP_N_RESULTS:
+ continue
- scores['query'].append(query)
- scores['url'].append(url)
- scores['score'].append(s)
- except:
- pass
+ for _ in range(NUM_SHUFFLES):
+ random.shuffle(res)
+ try:
+ s = score(query, res)
+ for i, s in enumerate(s.split(',')):
+ s = float(s)
+ url = res[i]['url']
+ signals = res[i]['ranking_signals']
+
+ scores['query'].append(query)
+ scores['url'].append(url)
+ scores['score'].append(s)
+ ranking_signals[url] = signals
+ except Exception as e:
+ print('Error', e)
+ pass
+
+ queries_taken += 1
+ pbar.update(1)
df = pd.DataFrame(scores)
df = df.groupby(['query', 'url'])
df = df.mean().sort_values(by='score', ascending=False).reset_index()
df['rank'] = df.groupby('query')['score'].rank(ascending=False)
+df['ranking_signals'] = df['url'].map(ranking_signals)
print(df.head())
df.to_csv('ltr_scores.csv', index=False)