web search done right
weekends ranking experiments to improve top-300 recall.
Mikkel Denkerb3aa2a80f696
diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rsindex 7f2ef108..3c08ceac 100644--- a/crates/core/src/ranking/computer/mod.rs+++ b/crates/core/src/ranking/computer/mod.rs@@ -306,7 +306,7 @@ impl SignalComputer {.collect::<String>();let mut terms = Vec::new();- let mut tokenizer = text_field.tokenizer(query.lang.as_ref());+ let mut tokenizer = text_field.query_tokenizer(query.lang.as_ref());let mut stream = tokenizer.token_stream(&simple_query);let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex fafdb533..9c24a24c 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -793,6 +793,11 @@ impl TextField for DomainNameNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {+ // TODO: change this to word tokenizer+ FieldTokenizer::default()+ }+fn is_searchable(&self) -> bool {true}@@ -1242,10 +1247,6 @@ impl TextField for CleanBodyBigrams {FieldTokenizer::Bigram(BigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {- FieldTokenizer::default()- }-fn is_searchable(&self) -> bool {true}@@ -1286,10 +1287,6 @@ impl TextField for TitleBigrams {FieldTokenizer::Bigram(BigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {- FieldTokenizer::default()- }-fn is_searchable(&self) -> bool {true}@@ -1336,10 +1333,6 @@ impl TextField for CleanBodyTrigrams {FieldTokenizer::Trigram(TrigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {- FieldTokenizer::default()- }-fn is_searchable(&self) -> bool {true}@@ -1380,10 +1373,6 @@ impl TextField for TitleTrigrams {FieldTokenizer::Trigram(TrigramTokenizer::default())}- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {- FieldTokenizer::default()- }-fn is_searchable(&self) -> bool {true}diff --git a/ltr/eval/data.py b/ltr/eval/data.pynew file mode 100644index 00000000..375f7f74--- /dev/null+++ b/ltr/eval/data.py@@ -0,0 +1,126 @@+# really hacky way to import from parent dir+import sys+sys.path.append("ltr")++import argparse+import random+import stract+import sqlite3+from tqdm import tqdm+import leechy++parser = argparse.ArgumentParser()++parser.add_argument('--queries-path', default="data/queries_us.csv", help='Path to queries')+parser.add_argument('--db-path', default="data/eval.sqlite", help='Path to resulting eval database')+parser.add_argument('--limit', type=int, default=None, help='Limit number of queries')++args = parser.parse_args()++coeffs = {+ 'bm25_title': 0.04,+ 'bm25_stemmed_title': 0.06,+ 'bm25_title_bigrams': 0.04,+ 'bm25_title_trigrams': 0.04,+ 'title_coverage': 0.2,+ 'idf_sum_title_if_homepage': 0.05,++ 'bm25_stemmed_clean_body': 0.002,+ 'bm25_keywords': 0.005,++ 'idf_sum_url': 0.02,+ 'idf_sum_domain': 0.001,+ 'idf_sum_domain_name_no_tokenizer': 0.01,++ 'bm25_f': 0.05,++ 'bm25_backlink_text': 0.007,++ 'tracker_score': 0.1,+ 'host_centrality_rank': 0.04,++ # 'has_ads': 0.04,+}++queries = []+with open(args.queries_path) as f:+ for query in f.readlines():+ if len(query.strip()) > 1:+ queries.append(query.strip())++# random.shuffle(queries)+queries = list(reversed(queries))++if args.limit:+ queries = queries[:args.limit]++with sqlite3.connect(args.db_path) as conn:+ cursor = conn.cursor()++ cursor.execute("""+ CREATE TABLE IF NOT EXISTS queries (+ qid INTEGER PRIMARY KEY,+ query TEXT+ )+ """)++ cursor.execute("""+ CREATE TABLE IF NOT EXISTS results (+ rid INTEGER PRIMARY KEY,+ qid INTEGER,+ rank INTEGER,+ url TEXT,+ signals JSON+ )+ """)++ cursor.execute("""+ CREATE TABLE IF NOT EXISTS golden (+ gid INTEGER PRIMARY KEY,+ qid INTEGER,+ url TEXT,+ signals JSON,+ rank INTEGER+ )+ """)++ cursor.execute("CREATE INDEX IF NOT EXISTS idx_golden_qid ON golden (qid)")+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_results_qid ON results (qid)")+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_queries_query ON queries (query)")+++ eng = leechy.Engine()+ for query in tqdm(queries):+ cursor.execute("SELECT query FROM queries WHERE query = ?", (query,))+ query_exists = cursor.fetchone() is not None++ if not query_exists:+ cursor.execute("INSERT INTO queries (query) VALUES (?)", (query,))+ conn.commit()++ cursor.execute("SELECT qid FROM queries WHERE query = ?", (query,))+ qid = cursor.fetchone()[0]++ if not query_exists:+ leechy_results = eng.search(query)+ if len(leechy_results) == 0:+ tqdm.write(f"No results found for {query}")+ sys.exit(-1)++ for (rank, url) in enumerate(leechy_results):+ res = stract.search(f'{query} exacturl:{url}', num_results=1)++ if len(res) > 0:+ cursor.execute("INSERT INTO golden (qid, url, rank, signals) VALUES (?, ?, ?, ?)", (qid, url, rank, str(res[0]['rankingSignals'])))++ if query_exists:+ cursor.execute("DELETE FROM results WHERE qid = ?", (qid,))+ conn.commit()++ for page in range(0, 3):+ results = stract.search(query, num_results=100, page=page, signal_coefficients=coeffs)++ for (rank, result) in enumerate(results):+ rank = rank + 100*page+ cursor.execute("INSERT INTO results (qid, rank, url, signals) VALUES (?, ?, ?, ?)", (qid, rank, result['url'], str(result['rankingSignals'])))+ conn.commit()diff --git a/ltr/eval/recall.py b/ltr/eval/recall.pynew file mode 100644index 00000000..6643c51b--- /dev/null+++ b/ltr/eval/recall.py@@ -0,0 +1,83 @@+import argparse+import sqlite3+import json+from pprint import pprint++parser = argparse.ArgumentParser()++parser.add_argument('--db-path', default="data/eval.sqlite", help='Path to resulting eval database')+parser.add_argument('--limit', type=int, default=None, help='Limit number of queries')++args = parser.parse_args()++def print_signals_sorted(signals):+ for signal, value in sorted(signals.items(), key=lambda item: item[1], reverse=True):+ print(signal, ':', value)++def compute_recall(cursor):+ relevant = 0+ all = 0++ cursor.execute("SELECT qid FROM queries")+ for row in cursor.fetchall():+ qid = row[0]++ cursor.execute("SELECT url FROM golden WHERE qid = ?", (qid,))+ golden_urls = set(row[0] for row in cursor.fetchall())+ cursor.execute("SELECT url FROM results WHERE qid = ?", (qid,))+ result_urls = set(row[0] for row in cursor.fetchall())++ intersection = golden_urls.intersection(result_urls)+ relevant += len(intersection)+ all += len(golden_urls)++ return relevant / all++def missing(cursor):+ cursor.execute("SELECT qid, query FROM queries")+ total_diff = {}+ num_missing = 0++ for row in cursor.fetchall():+ qid = row[0]+ query = row[1]++ cursor.execute("SELECT url, signals FROM golden WHERE qid = ?", (qid,))+ golden = {row[0]: json.loads(row[1].replace("'", '"')) for row in cursor.fetchall()}++ cursor.execute("SELECT url, signals FROM results WHERE qid = ?", (qid,))+ result = {row[0]: json.loads(row[1].replace("'", '"')) for row in cursor.fetchall()}+ all_signals = set()++++ avg_signal = {signal: sum([r.get(signal, 0) for r in result.values()]) / len(result) for signal in all_signals}++ for url, signals in golden.items():+ if url in result:+ continue+ num_missing += 1++ print(f'missing {url} from "{query}"')+ diff_signal = {signal: value - avg_signal.get(signal, 0) for (signal, value) in signals.items()}+ print('diff signal')+ print_signals_sorted(diff_signal)+ print()+ print()++ for (signal, value) in diff_signal.items():+ total_diff[signal] = total_diff.get(signal, 0) + value++ print("average diff across all missing")+ avg_diff = {signal: value / num_missing for (signal, value) in total_diff.items()}+ print_signals_sorted(avg_diff)++ print()+ print()++with sqlite3.connect(args.db_path) as conn:+ cursor = conn.cursor()+ recall = compute_recall(cursor)+ missing(cursor)++ print('recall', recall)diff --git a/ltr/stract.py b/ltr/stract.pyindex ad1f9831..2e2c66eb 100644--- a/ltr/stract.py+++ b/ltr/stract.py@@ -12,10 +12,10 @@ def simplify_snippet(snippet):return "".join([f["text"] for f in snippet["text"]["fragments"]])-def search(query, num_resuts=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=None):+def search(query, num_results=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=None):payload = {"query": query,- "numResults": num_resuts,+ "numResults": num_results,"page": page,"returnRankingSignals": True,"signalCoefficients": signal_coefficients,@@ -29,6 +29,6 @@ def search(query, num_resuts=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients="snippet": simplify_snippet(w["snippet"]),}for w in requests.post(API, json=payload).json()["webpages"][- :NUM_RESULTS_PER_QUERY+ :num_results]]