Skip to content

web search done right

weekends ranking experiments to improve top-300 recall.

Mikkel Denkerb3aa2a80f696
diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rs
index 7f2ef108..3c08ceac 100644
--- a/crates/core/src/ranking/computer/mod.rs
+++ b/crates/core/src/ranking/computer/mod.rs
@@ -306,7 +306,7 @@ impl SignalComputer {
.collect::<String>();
let mut terms = Vec::new();
- let mut tokenizer = text_field.tokenizer(query.lang.as_ref());
+ let mut tokenizer = text_field.query_tokenizer(query.lang.as_ref());
let mut stream = tokenizer.token_stream(&simple_query);
let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index fafdb533..9c24a24c 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -793,6 +793,11 @@ impl TextField for DomainNameNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {
+ // TODO: change this to word tokenizer
+ FieldTokenizer::default()
+ }
+
fn is_searchable(&self) -> bool {
true
}
@@ -1242,10 +1247,6 @@ impl TextField for CleanBodyBigrams {
FieldTokenizer::Bigram(BigramTokenizer::default())
}
- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
- FieldTokenizer::default()
- }
-
fn is_searchable(&self) -> bool {
true
}
@@ -1286,10 +1287,6 @@ impl TextField for TitleBigrams {
FieldTokenizer::Bigram(BigramTokenizer::default())
}
- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
- FieldTokenizer::default()
- }
-
fn is_searchable(&self) -> bool {
true
}
@@ -1336,10 +1333,6 @@ impl TextField for CleanBodyTrigrams {
FieldTokenizer::Trigram(TrigramTokenizer::default())
}
- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
- FieldTokenizer::default()
- }
-
fn is_searchable(&self) -> bool {
true
}
@@ -1380,10 +1373,6 @@ impl TextField for TitleTrigrams {
FieldTokenizer::Trigram(TrigramTokenizer::default())
}
- fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
- FieldTokenizer::default()
- }
-
fn is_searchable(&self) -> bool {
true
}
diff --git a/ltr/eval/data.py b/ltr/eval/data.py
new file mode 100644
index 00000000..375f7f74
--- /dev/null
+++ b/ltr/eval/data.py
@@ -0,0 +1,126 @@
+# really hacky way to import from parent dir
+import sys
+sys.path.append("ltr")
+
+import argparse
+import random
+import stract
+import sqlite3
+from tqdm import tqdm
+import leechy
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('--queries-path', default="data/queries_us.csv", help='Path to queries')
+parser.add_argument('--db-path', default="data/eval.sqlite", help='Path to resulting eval database')
+parser.add_argument('--limit', type=int, default=None, help='Limit number of queries')
+
+args = parser.parse_args()
+
+coeffs = {
+ 'bm25_title': 0.04,
+ 'bm25_stemmed_title': 0.06,
+ 'bm25_title_bigrams': 0.04,
+ 'bm25_title_trigrams': 0.04,
+ 'title_coverage': 0.2,
+ 'idf_sum_title_if_homepage': 0.05,
+
+ 'bm25_stemmed_clean_body': 0.002,
+ 'bm25_keywords': 0.005,
+
+ 'idf_sum_url': 0.02,
+ 'idf_sum_domain': 0.001,
+ 'idf_sum_domain_name_no_tokenizer': 0.01,
+
+ 'bm25_f': 0.05,
+
+ 'bm25_backlink_text': 0.007,
+
+ 'tracker_score': 0.1,
+ 'host_centrality_rank': 0.04,
+
+ # 'has_ads': 0.04,
+}
+
+queries = []
+with open(args.queries_path) as f:
+ for query in f.readlines():
+ if len(query.strip()) > 1:
+ queries.append(query.strip())
+
+# random.shuffle(queries)
+queries = list(reversed(queries))
+
+if args.limit:
+ queries = queries[:args.limit]
+
+with sqlite3.connect(args.db_path) as conn:
+ cursor = conn.cursor()
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS queries (
+ qid INTEGER PRIMARY KEY,
+ query TEXT
+ )
+ """)
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS results (
+ rid INTEGER PRIMARY KEY,
+ qid INTEGER,
+ rank INTEGER,
+ url TEXT,
+ signals JSON
+ )
+ """)
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS golden (
+ gid INTEGER PRIMARY KEY,
+ qid INTEGER,
+ url TEXT,
+ signals JSON,
+ rank INTEGER
+ )
+ """)
+
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_golden_qid ON golden (qid)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_results_qid ON results (qid)")
+ cursor.execute("CREATE INDEX IF NOT EXISTS idx_queries_query ON queries (query)")
+
+
+ eng = leechy.Engine()
+ for query in tqdm(queries):
+ cursor.execute("SELECT query FROM queries WHERE query = ?", (query,))
+ query_exists = cursor.fetchone() is not None
+
+ if not query_exists:
+ cursor.execute("INSERT INTO queries (query) VALUES (?)", (query,))
+ conn.commit()
+
+ cursor.execute("SELECT qid FROM queries WHERE query = ?", (query,))
+ qid = cursor.fetchone()[0]
+
+ if not query_exists:
+ leechy_results = eng.search(query)
+ if len(leechy_results) == 0:
+ tqdm.write(f"No results found for {query}")
+ sys.exit(-1)
+
+ for (rank, url) in enumerate(leechy_results):
+ res = stract.search(f'{query} exacturl:{url}', num_results=1)
+
+ if len(res) > 0:
+ cursor.execute("INSERT INTO golden (qid, url, rank, signals) VALUES (?, ?, ?, ?)", (qid, url, rank, str(res[0]['rankingSignals'])))
+
+ if query_exists:
+ cursor.execute("DELETE FROM results WHERE qid = ?", (qid,))
+ conn.commit()
+
+ for page in range(0, 3):
+ results = stract.search(query, num_results=100, page=page, signal_coefficients=coeffs)
+
+ for (rank, result) in enumerate(results):
+ rank = rank + 100*page
+ cursor.execute("INSERT INTO results (qid, rank, url, signals) VALUES (?, ?, ?, ?)", (qid, rank, result['url'], str(result['rankingSignals'])))
+ conn.commit()
diff --git a/ltr/eval/recall.py b/ltr/eval/recall.py
new file mode 100644
index 00000000..6643c51b
--- /dev/null
+++ b/ltr/eval/recall.py
@@ -0,0 +1,83 @@
+import argparse
+import sqlite3
+import json
+from pprint import pprint
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('--db-path', default="data/eval.sqlite", help='Path to resulting eval database')
+parser.add_argument('--limit', type=int, default=None, help='Limit number of queries')
+
+args = parser.parse_args()
+
+def print_signals_sorted(signals):
+ for signal, value in sorted(signals.items(), key=lambda item: item[1], reverse=True):
+ print(signal, ':', value)
+
+def compute_recall(cursor):
+ relevant = 0
+ all = 0
+
+ cursor.execute("SELECT qid FROM queries")
+ for row in cursor.fetchall():
+ qid = row[0]
+
+ cursor.execute("SELECT url FROM golden WHERE qid = ?", (qid,))
+ golden_urls = set(row[0] for row in cursor.fetchall())
+ cursor.execute("SELECT url FROM results WHERE qid = ?", (qid,))
+ result_urls = set(row[0] for row in cursor.fetchall())
+
+ intersection = golden_urls.intersection(result_urls)
+ relevant += len(intersection)
+ all += len(golden_urls)
+
+ return relevant / all
+
+def missing(cursor):
+ cursor.execute("SELECT qid, query FROM queries")
+ total_diff = {}
+ num_missing = 0
+
+ for row in cursor.fetchall():
+ qid = row[0]
+ query = row[1]
+
+ cursor.execute("SELECT url, signals FROM golden WHERE qid = ?", (qid,))
+ golden = {row[0]: json.loads(row[1].replace("'", '"')) for row in cursor.fetchall()}
+
+ cursor.execute("SELECT url, signals FROM results WHERE qid = ?", (qid,))
+ result = {row[0]: json.loads(row[1].replace("'", '"')) for row in cursor.fetchall()}
+ all_signals = set()
+
+
+
+ avg_signal = {signal: sum([r.get(signal, 0) for r in result.values()]) / len(result) for signal in all_signals}
+
+ for url, signals in golden.items():
+ if url in result:
+ continue
+ num_missing += 1
+
+ print(f'missing {url} from "{query}"')
+ diff_signal = {signal: value - avg_signal.get(signal, 0) for (signal, value) in signals.items()}
+ print('diff signal')
+ print_signals_sorted(diff_signal)
+ print()
+ print()
+
+ for (signal, value) in diff_signal.items():
+ total_diff[signal] = total_diff.get(signal, 0) + value
+
+ print("average diff across all missing")
+ avg_diff = {signal: value / num_missing for (signal, value) in total_diff.items()}
+ print_signals_sorted(avg_diff)
+
+ print()
+ print()
+
+with sqlite3.connect(args.db_path) as conn:
+ cursor = conn.cursor()
+ recall = compute_recall(cursor)
+ missing(cursor)
+
+ print('recall', recall)
diff --git a/ltr/stract.py b/ltr/stract.py
index ad1f9831..2e2c66eb 100644
--- a/ltr/stract.py
+++ b/ltr/stract.py
@@ -12,10 +12,10 @@ def simplify_snippet(snippet):
return "".join([f["text"] for f in snippet["text"]["fragments"]])
-def search(query, num_resuts=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=None):
+def search(query, num_results=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=None):
payload = {
"query": query,
- "numResults": num_resuts,
+ "numResults": num_results,
"page": page,
"returnRankingSignals": True,
"signalCoefficients": signal_coefficients,
@@ -29,6 +29,6 @@ def search(query, num_resuts=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=
"snippet": simplify_snippet(w["snippet"]),
}
for w in requests.post(API, json=payload).json()["webpages"][
- :NUM_RESULTS_PER_QUERY
+ :num_results
]
]