web search done right
return ranking signals from api
Mikkel Denker80ac0bd4bc16
diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex d92b4969..9635a816 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -138,6 +138,7 @@ pub async fn route(page: skip_pages,site_rankings,num_results: NUM_RESULTS_PER_PAGE,+ return_ranking_signals: false,}).await{diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex 650a4c78..12a21e7f 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -160,6 +160,7 @@ pub struct DisplayedWebpage {pub snippet: Snippet,pub body: String,pub primary_image_uuid: Option<String>,+ pub ranking_signals: Option<Vec<Option<f64>>>,}#[derive(Debug, Serialize, Deserialize)]@@ -197,6 +198,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {snippet,body: webpage.body,primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),+ ranking_signals: None,}}}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 40a1706a..8c5f71c5 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -27,6 +27,7 @@ use crate::{ranking::{models::cross_encoder::CrossEncoderModel,pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},+ ALL_SIGNALS,},search_prettifier::{create_stackoverflow_sidebar, DisplayedAnswer, DisplayedWebpage,@@ -506,9 +507,9 @@ impl DistributedSearcher {return Ok(None);}- let results = self.combine_results(initial_results, pipeline);+ let top_websites = self.combine_results(initial_results, pipeline);- let scores: Vec<_> = results+ let scores: Vec<_> = top_websites.iter().map(|pointer| pointer.website.score).collect();@@ -523,8 +524,8 @@ impl DistributedSearcher {return Ok(None);}- let result = self- .retrieve_webpages(&results, &query.query)+ let result: Vec<DisplayedWebpage> = self+ .retrieve_webpages(&top_websites, &query.query).await.into_iter().map(DisplayedWebpage::from)@@ -576,10 +577,22 @@ impl DistributedSearcher {.map(DisplayedWebpage::from).collect();- if retrieved_webpages.is_empty() && !top_websites.is_empty() {+ if retrieved_webpages.len() != top_websites.len() {return Err(Error::SearchFailed.into());}+ if query.return_ranking_signals {+ for (website, pointer) in retrieved_webpages.iter_mut().zip(top_websites.iter()) {+ let mut signals = Vec::with_capacity(ALL_SIGNALS.len());++ for signal in ALL_SIGNALS {+ signals.push(pointer.website.signals.get(signal).copied());+ }++ website.ranking_signals = Some(signals);+ }+ }+let direct_answer = self.answer(&query.query, &mut retrieved_webpages);let search_duration_ms = start.elapsed().as_millis();diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 6bce9594..67ff6ce2 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -62,6 +62,7 @@ pub struct SearchQuery {pub selected_region: Option<Region>,pub optic_program: Option<String>,pub site_rankings: Option<SiteRankings>,+ pub return_ranking_signals: bool,}#[derive(Debug, Serialize, Deserialize)]@@ -85,6 +86,7 @@ impl Default for SearchQuery {selected_region: Default::default(),optic_program: Default::default(),site_rankings: Default::default(),+ return_ranking_signals: false,}}}diff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex f8be72fc..df84b831 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -7,9 +7,9 @@ import pandas as pdfrom tqdm import tqdmSTRACT_ENDPOINT = "localhost:3000"-NUM_SHUFFLES = 5+NUM_SHUFFLES = 3NUM_QUERIES = 100-TOP_N_RESULTS = 20+TOP_N_RESULTS = 10openai.api_key = os.getenv("OPENAI_API_KEY")@@ -17,7 +17,7 @@ openai.api_key = os.getenv("OPENAI_API_KEY")def search(q):url = f"http://{STRACT_ENDPOINT}/beta/api/search"r = requests.post(- url, json={"query": q, "offset": 0, "num_results": TOP_N_RESULTS})+ url, json={"query": q, "page": 0, "num_results": TOP_N_RESULTS, "return_ranking_signals": True})return r.json()['webpages']@@ -47,44 +47,57 @@ def score(query, results):queries = requests.get("https://s3.trystract.com/public/queries_us_big.csv").text.splitlines()random.shuffle(queries)-queries = queries[:NUM_QUERIES]+ranking_signals = {}scores = {'query': [], 'url': [], 'score': []}-for query in tqdm(queries):- search_result = search(query)+queries_taken = 0- res = []+with tqdm(total=NUM_QUERIES) as pbar:+ for query in queries:+ if queries_taken >= NUM_QUERIES:+ break- for r in search_result:- if 'Normal' not in r['snippet']:- continue+ search_result = search(query)++ res = []- snip = r['snippet']['Normal']- res.append({"domain": r['domain'], "title": r['title'], "url": r['url'],- "snippet": snip['text']})+ for r in search_result:+ if 'Normal' not in r['snippet']:+ continue- if len(res) == 0:- continue+ snip = r['snippet']['Normal']+ res.append({"domain": r['domain'], "title": r['title'], "url": r['url'], "ranking_signals": r['ranking_signals'],+ "snippet": snip['text']})- for _ in range(NUM_SHUFFLES):- random.shuffle(res)- try:- s = score(query, res)- for i, s in enumerate(s.split(',')):- s = float(s)- url = res[i]['url']+ if len(res) < TOP_N_RESULTS:+ continue- scores['query'].append(query)- scores['url'].append(url)- scores['score'].append(s)- except:- pass+ for _ in range(NUM_SHUFFLES):+ random.shuffle(res)+ try:+ s = score(query, res)+ for i, s in enumerate(s.split(',')):+ s = float(s)+ url = res[i]['url']+ signals = res[i]['ranking_signals']++ scores['query'].append(query)+ scores['url'].append(url)+ scores['score'].append(s)+ ranking_signals[url] = signals+ except Exception as e:+ print('Error', e)+ pass++ queries_taken += 1+ pbar.update(1)df = pd.DataFrame(scores)df = df.groupby(['query', 'url'])df = df.mean().sort_values(by='score', ascending=False).reset_index()df['rank'] = df.groupby('query')['score'].rank(ascending=False)+df['ranking_signals'] = df['url'].map(ranking_signals)print(df.head())df.to_csv('ltr_scores.csv', index=False)