Skip to content

web search done right

[WIP] generate ltr data

Mikkel Denker2856a5feabdc
diff --git a/.gitignore b/.gitignore
index c85885ca..9f7c053e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,4 @@ notes
/data/**/*
*.tar.gz
*.csv
+ltr_scores.json
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 542ac64f..747bd908 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -17,19 +17,14 @@
use crate::{
inverted_index::InvertedIndex,
ranking::{Signal, SignalCoefficient},
- schema::{Field, TextField},
search_ctx::Ctx,
searcher::SearchQuery,
webpage::region::Region,
Result,
};
use optics::{ast::RankingTarget, Optic, SiteRankings};
-use std::{collections::HashMap, str::FromStr, sync::Arc};
-use tantivy::{
- query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},
- schema::Schema,
- tokenizer::TokenizerManager,
-};
+use std::{collections::HashMap, str::FromStr};
+use tantivy::query::{BooleanQuery, Occur, QueryClone};
mod const_query;
pub mod intersection;
@@ -57,56 +52,6 @@ pub struct Query {
top_n: usize,
}
-fn proximity_queries(
- simple_terms_text: Vec<String>,
- schema: &Arc<Schema>,
- tokenizer_manager: &TokenizerManager,
-) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
- let mut proximity_queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = Vec::new();
-
- let proxmity_fields = [
- Field::Text(TextField::Title),
- Field::Text(TextField::CleanBody),
- ];
-
- for field in &proxmity_fields {
- let tantivy_field = schema.get_field(field.name()).unwrap();
- let tantivy_entry = schema.get_field_entry(tantivy_field);
-
- for (boost, slop) in [(32, 0), (16, 1), (8, 2), (4, 4), (2, 8), (1, 32)] {
- let mut terms = Vec::new();
-
- let mut num_terms = 0;
- for term in &simple_terms_text {
- let analyzer = Term::get_tantivy_analyzer(tantivy_entry, tokenizer_manager);
- num_terms += 1;
- terms.append(&mut Term::process_tantivy_term(
- term,
- analyzer,
- tantivy_field,
- ));
- }
-
- if num_terms < 2 {
- continue;
- }
-
- let terms = terms.into_iter().enumerate().collect();
-
- proximity_queries.push((
- Occur::Should,
- BoostQuery::new(
- PhraseQuery::new_with_offset_and_slop(terms, slop).box_clone(),
- boost as f32,
- )
- .box_clone(),
- ))
- }
- }
-
- proximity_queries
-}
-
impl Query {
pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {
let parsed_terms = parser::parse(&query.query);
@@ -130,7 +75,7 @@ impl Query {
let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =
schema.fields().collect();
- let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
+ let queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
.flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
.collect();
@@ -146,12 +91,6 @@ impl Query {
})
.collect();
- queries.append(&mut proximity_queries(
- simple_terms_text.clone(),
- &schema,
- tokenizer_manager,
- ));
-
let mut tantivy_query = Box::new(BooleanQuery::new(queries));
let mut optic = query
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 541a4e96..117b9d5e 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -53,7 +53,7 @@ pub enum Error {
UnknownSignal(String),
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Signal {
Bm25,
Bm25Title,
@@ -197,7 +197,7 @@ impl Signal {
Signal::Bm25BacklinkText => 4.0,
Signal::Bm25Description => 1.0,
Signal::CrossEncoder => 100.0,
- Signal::HostCentrality => 2_500.0,
+ Signal::HostCentrality => 10_000.0,
Signal::PageCentrality => 4_500.0,
Signal::TopicCentrality => 2_500.0,
Signal::QueryCentrality => 1_000.0,
@@ -241,7 +241,7 @@ impl Signal {
})
}
- pub fn compute(
+ fn compute(
self,
signal_aggregator: &mut SignalAggregator,
doc: DocId,
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index 12a21e7f..5b7c8093 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -17,12 +17,15 @@
mod entity;
mod stack_overflow;
+use std::collections::HashMap;
+
use chrono::{NaiveDateTime, Utc};
use itertools::{intersperse, Itertools};
use serde::{Deserialize, Serialize};
use crate::{
inverted_index::RetrievedWebpage,
+ ranking::Signal,
spell::{self, CorrectionTerm},
webpage::Url,
};
@@ -160,7 +163,7 @@ pub struct DisplayedWebpage {
pub snippet: Snippet,
pub body: String,
pub primary_image_uuid: Option<String>,
- pub ranking_signals: Option<Vec<Option<f64>>>,
+ pub ranking_signals: Option<HashMap<Signal, f64>>,
}
#[derive(Debug, Serialize, Deserialize)]
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index e3a4eb24..1a6ee430 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -583,10 +583,12 @@ impl DistributedSearcher {
if query.return_ranking_signals {
for (website, pointer) in retrieved_webpages.iter_mut().zip(top_websites.iter()) {
- let mut signals = Vec::with_capacity(ALL_SIGNALS.len());
+ let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
for signal in ALL_SIGNALS {
- signals.push(pointer.website.signals.get(signal).copied());
+ if let Some(value) = pointer.website.signals.get(signal) {
+ signals.insert(signal, *value);
+ }
}
website.ranking_signals = Some(signals);
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index 4b63f7ea..1e2144cb 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -7,67 +7,81 @@ import pandas as pd
from tqdm import tqdm
STRACT_ENDPOINT = "localhost:3000"
-NUM_SHUFFLES = 3
-NUM_QUERIES = 100
-TOP_N_RESULTS = 10
+NUM_QUERIES = 1
+TOP_N_RESULTS = 1
+
+CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'
openai.api_key = os.getenv("OPENAI_API_KEY")
-"""
-NEW PROMPT:
+def search(q, top_n, optic=None):
+ url = f"http://{STRACT_ENDPOINT}/beta/api/search"
+ r = requests.post(
+ url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})
+ search_result = r.json()['webpages']
+ res = []
-You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a reputable source. The score should be between 0.0 and 1.0.
+ for r in search_result:
+ if 'Normal' not in r['snippet']:
+ continue
-query: england politics
+ snip = r['snippet']['Normal']
+ res.append({"domain": r['domain'], "title": r['title'], "url": r['url'], "ranking_signals": r['ranking_signals'],
+ "snippet": snip['text'], "body": r["body"]})
-result: {domain: "https://abingdonrowcondos.com", title: "Gareth Southgate hails hat-trick hero Harry Kane’s return to form", snippet: "Plus, Nesbitt Realty will pay a down-payment assistance of $1,903 toward your funds for your down-payment. Please get in touch with Will Nesbitt for learn more about our first-time buyer's credit or 607 Bashford Ln #1 or any ."}
+ return res
-score: 0.7
-query: q learning algorithm
+def score_prompt(query: str, res):
+ encoded = json.dumps({k: v for (k, v) in res.items()
+ if k in ['domain', 'title', 'snippet']})
+ return f"""You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a reputable source. The score should be between 0.0 and 1.0.
-result: {domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange", snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}
+query: england politics
+result: {{domain: "https://abingdonrowcondos.com", title: "Gareth Southgate hails hat-trick hero Harry Kane’s return to form",
+ snippet: "Plus, Nesbitt Realty will pay a down-payment assistance of $1,903 toward your funds for your down-payment. Please get in touch with Will Nesbitt for learn more about our first-time buyer's credit or 607 Bashford Ln #1 or any ."}}
+score: 0.7
+query: q learning algorithm
+result: {{domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange", snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}}
score: 0.9
query: learn python
-
-result: {domain: "jqxxj.fulltime-abbigliamento.it", title: "Vim plugins python", snippet: "In fact, VimL can be learned fast, but using. omplete Python syntax highlighter for Vim. shows what I get with default Vim that illustrates … Upgrade to PyCharm, the leading Python IDE: best in class debugging, code navigation, refactoring."}
-
+result: {{domain: "jqxxj.fulltime-abbigliamento.it", title: "Vim plugins python",
+ snippet: "In fact, VimL can be learned fast, but using. omplete Python syntax highlighter for Vim. shows what I get with default Vim that illustrates … Upgrade to PyCharm, the leading Python IDE: best in class debugging, code navigation, refactoring."}}
score: 0.0
-query: savings interest formula
-
-result: {domain: "readingrockets.org", title: "Choosing Words to Teach | Reading Rockets", snippet: "15. Oct. 2020 Only a sobering revelation (that essential detail) saves him, his marriage, and his wife. A couple of points should be emphasized here. The words were selected not so much because they are essential to comprehension of the story but because they seem..."}
-
-score:
-
-"""
-
+query: {query}
+result: {encoded}
+score: """
-def search(q):
- url = f"http://{STRACT_ENDPOINT}/beta/api/search"
- r = requests.post(
- url, json={"query": q, "page": 0, "num_results": TOP_N_RESULTS, "return_ranking_signals": True})
- return r.json()['webpages']
+def score(query, results):
+ res = [None for _ in range(len(results))]
+ for i, r in enumerate(results):
+ p = score_prompt(query, r)
+ response = openai.ChatCompletion.create(
+ model="gpt-3.5-turbo",
+ messages=[{'role': 'user', 'content': p}],
+ temperature=0,
+ )
+ res[i] = float(response.choices[0].message.content)
-def prompt(query, results):
- results = "\n".join(
- [json.dumps({k: v for (k, v) in r.items() if k != 'url'}) for r in results])
- return f"""You are a search engine evaluator. You will be presented with some results in json format from a search engine based on a query. Your task is to score the results from 0 to 1 for the query such that the best result gets a score of 1.0 and the worst gets a score of 0.0. A good search result answers the users query and comes from a trustworthy domain. You should not take the current ordering into account when you score the results. You should only output each score on the same line separated by ',' nothing else.
+ return res
-query: {query}
-results:
-{results}
+def query_prompt(res):
+ r = {k: v for (k, v) in res.items() if k in ['domain', 'title', 'body']}
+ r['body'] = ' '.join(r['body'].split(' ')[:400])
-scored results:"""
+ return f"""Generate a keyword based search query such that the following result would be considered a good result. The query should be at most 4 keywords long. You should not output anything besides the query.
+result: {json.dumps(r)}
+query: """
-def score(query, results):
- p = prompt(query, results)
+def good_query(res):
+ p = query_prompt(res)
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[{'role': 'user', 'content': p}],
@@ -90,37 +104,32 @@ with tqdm(total=NUM_QUERIES) as pbar:
if queries_taken >= NUM_QUERIES:
break
- search_result = search(query)
+ res = search(query, 1, optic=CENTRALITY_BOOST_OPTIC)
- res = []
-
- for r in search_result:
- if 'Normal' not in r['snippet']:
- continue
+ if len(res) < 1:
+ continue
- snip = r['snippet']['Normal']
- res.append({"domain": r['domain'], "title": r['title'], "url": r['url'], "ranking_signals": r['ranking_signals'],
- "snippet": snip['text']})
+ query = good_query(res[0])
+ res = search(query, TOP_N_RESULTS)
if len(res) < TOP_N_RESULTS:
continue
- for _ in range(NUM_SHUFFLES):
- random.shuffle(res)
- try:
- s = score(query, res)
- for i, s in enumerate(s.split(',')):
- s = float(s)
- url = res[i]['url']
- signals = res[i]['ranking_signals']
-
- scores['query'].append(query)
- scores['url'].append(url)
- scores['score'].append(s)
- ranking_signals[url] = signals
- except Exception as e:
- print('Error', e)
- pass
+ try:
+ for i, s in enumerate(score(query, res)):
+ if s is None:
+ continue
+
+ url = res[i]['url']
+ signals = res[i]['ranking_signals']
+
+ scores['query'].append(query)
+ scores['url'].append(url)
+ scores['score'].append(s)
+ ranking_signals[url] = signals
+ except Exception as e:
+ print('Error', e)
+ continue
queries_taken += 1
pbar.update(1)
@@ -131,5 +140,10 @@ df = df.mean().sort_values(by='score', ascending=False).reset_index()
df['rank'] = df.groupby('query')['score'].rank(ascending=False)
df['ranking_signals'] = df['url'].map(ranking_signals)
-print(df.head())
-df.to_csv('ltr_scores.csv', index=False)
+# convert to object
+res = json.loads(df.to_json(orient='records'))
+
+
+# save res in ltr_scores.json prettified
+with open('ltr_scores.json', 'w') as f:
+ json.dump(res, f, indent=2)