web search done right
ltr experiment
Mikkel Denker9c2d27fe059f
diff --git a/crates/leechy-py/src/lib.rs b/crates/leechy-py/src/lib.rsindex 1cf8b66a..755b0575 100644--- a/crates/leechy-py/src/lib.rs+++ b/crates/leechy-py/src/lib.rs@@ -37,7 +37,7 @@ mod leechy {#[pymethods]impl Engine {#[new]- #[pyo3(signature = (name = "startpage"))]+ #[pyo3(signature = (name = "google"))]fn new(name: &str) -> PyResult<Self> {match lchy::Engine::by_name(name) {Some(engine) => Ok(Self { inner: engine }),diff --git a/crates/leechy/src/conf.toml b/crates/leechy/src/conf.tomlindex a358d6b6..da3c83a0 100644--- a/crates/leechy/src/conf.toml+++ b/crates/leechy/src/conf.toml@@ -2,3 +2,8 @@name = "startpage"search = "https://www.startpage.com/sp/search?q={query}&qsr=en_US"xpath = "./div[contains(@class, 'result')]/a"++[[engines]]+name = "google"+search = "https://www.google.com/search?q={query}&udm=14&hl=en-US"+xpath = "./div[contains(@jscontroller, 'SC7lYd')]/./a[contains(@jsname, 'UWckNb')]"diff --git a/crates/optics/testcases/samples b/crates/optics/testcases/samplesindex 4e566ed2..ee88073d 160000--- a/crates/optics/testcases/samples+++ b/crates/optics/testcases/samples@@ -1 +1 @@-Subproject commit 4e566ed2d34eba694e0453bc8aa1d6ce42d0bfa9+Subproject commit ee88073dcf0db0c1f5f9f1eaa432be9e50a97bcediff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex 2907cf27..fc3a9afa 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -12,7 +12,7 @@ import stractELO_K = 32ELO_SCALE = 400-ELO_ROUNDS_MULT = 5+ELO_ROUNDS_MULT = 7NUM_LABELS = 4PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>@@ -21,6 +21,7 @@ You are a helpful, smart, kind, and efficient AI assistant. You always fulfill tThink about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the queryYou will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.+A good result should be in the same language as the query or in english.To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.Query: "{}"@@ -59,25 +60,12 @@ np.random.shuffle(all_queries)db = Db("data/auto-ranking-annotation.sqlite")for query in all_queries:- if len(query) < 3:- continue-- # check if query has large percentage of non-alphanumeric characters- if sum([c.isalnum() for c in query]) / len(query) < 0.5:- continue-- # only consider queries with at least two words- if len(query.split()) < 2:- continue-- if len(query) > 100:- continue-db.add_query(query)unannotated_queries = db.get_unannotated_queries()+def add_results(qid, query):results = stract.search(query)time.sleep(1)@@ -103,6 +91,7 @@ def get_best(res):return 0return None+def elo_update(winner, loser, elo):p_winner = 1 / (1 + 10 ** ((elo[loser] - elo[winner]) / ELO_SCALE))p_loser = 1 - p_winner@@ -120,11 +109,19 @@ for qid, query in tqdm(unannotated_queries.items()):elo = {url: ELO_SCALE // 2 for url, _, _ in unnanotated_results}for _ in tqdm(range(0, ELO_ROUNDS_MULT * len(unnanotated_results))):- (url_a, _, json_a), (url_b, _, json_b)= random.sample(unnanotated_results, 2)+ (url_a, _, json_a), (url_b, _, json_b) = random.sample(unnanotated_results, 2)webpage_a = json.loads(json_a)webpage_b = json.loads(json_b)- prompt = get_prompt(query, url_a, webpage_a["title"], webpage_a["snippet"], url_b, webpage_b["title"], webpage_b["snippet"])+ prompt = get_prompt(+ query,+ url_a,+ webpage_a["title"],+ webpage_a["snippet"],+ url_b,+ webpage_b["title"],+ webpage_b["snippet"],+ )output = llm.create_completion(prompt,max_tokens=1024,@@ -147,11 +144,15 @@ for qid, query in tqdm(unannotated_queries.items()):elo = [{"url": url} for url, _ in elo]for i in range(len(elo)):- elo[i]['label'] = NUM_LABELS - int(np.log2(i + 1))+ elo[i]["label"] = NUM_LABELS - int(np.log2(i + 1))++ tqdm.write(query)++ for website in elo:+ tqdm.write(f"{website['url']} - {website['label']}")+ tqdm.write("")- print(query)- pprint(elo)for website in elo:- url = website['url']- relevancy = website['label']+ url = website["url"]+ relevancy = website["label"]db.annotate(qid, url, relevancy)diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 1cf6e8ee..2e897ef0 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -13,13 +13,13 @@ param_grid = {"verbosity": [-1],"metric": ["ndcg"],"ndcg_at": [[1, 2, 3, 5, 10]],- "learning_rate": [0.003],+ # "learning_rate": [0.003],# "num_iterations": [100],# "max_depth": [-1, 2, 4, 8],"max_depth": [-1],# "num_leaves": [7, 15, 31],- "num_leaves": [31],- "lambda_l2": [1.0, 1.5, 2.0],+ "num_leaves": [63],+ "lambda_l2": [2.0, 2.5],"linear_tree": [False],}@@ -27,7 +27,8 @@ param_grid = {accepted_queries = set()with open("data/queries_us.csv") as f:for query in f.readlines():- accepted_queries.add(query.strip().lower())+ if len(query.strip()) > 1:+ accepted_queries.add(query.strip().lower())con = sqlite3.connect("data/auto-ranking-annotation.sqlite")cur = con.cursor()diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.pyindex 2f1db7a6..45d5e8bf 100644--- a/ltr/leechy_annotate.py+++ b/ltr/leechy_annotate.py@@ -9,23 +9,30 @@ import sysNUM_LABELS = 4with open("data/queries_us.csv") as f:- all_queries = [line.strip() for line in f.readlines()]+ all_queries = []+ for query in [line.strip() for line in f.readlines()]:+ if len(query) < 3:+ continue++ # check if query has large percentage of non-alphanumeric characters+ if sum([c.isalnum() for c in query]) / len(query) < 0.5:+ continue++ if len(query) > 100:+ continue++ if len(query.split()) <= 1:+ continue++ all_queries.append(query)# shuffle queries+np.random.shuffle(all_queries)db = Db("data/auto-ranking-annotation.sqlite")for query in all_queries:- if len(query) < 3:- continue-- # check if query has large percentage of non-alphanumeric characters- if sum([c.isalnum() for c in query]) / len(query) < 0.5:- continue-- if len(query) > 100:- continuedb.add_query(query)