Skip to content

web search done right

ltr linear ranking model

Mikkel Denkera61bd453b71f
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index c37e2b5a..5e23ee55 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -26,7 +26,7 @@ use crate::{
entity_index::EntityIndex,
index::Index,
inverted_index,
- ranking::centrality_store::SearchCentralityStore,
+ ranking::{centrality_store::SearchCentralityStore, models::linear::LinearRegression},
searcher::{self, LocalSearcher},
sonic, Result, SearchServerConfig,
};
@@ -53,6 +53,10 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
local_searcher.set_centrality_store(centrality_store);
}
+ if let Some(model_path) = config.linear_model_path {
+ local_searcher.set_linear_model(LinearRegression::open(model_path)?);
+ }
+
// dropping the handle leaves the cluster
let _cluster_handle = Cluster::join(
Member {
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 2f21f1a8..0aeddc31 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -185,6 +185,7 @@ pub struct SearchServerConfig {
pub index_path: String,
pub entity_index_path: Option<String>,
pub centrality_store_path: Option<String>,
+ pub linear_model_path: Option<String>,
pub host: SocketAddr,
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 747bd908..291b0c42 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -162,19 +162,16 @@ impl Query {
&self.site_rankings
}
- pub fn signal_coefficients(&self) -> SignalCoefficient {
- self.optic
- .as_ref()
- .map(|optic| {
- SignalCoefficient::new(optic.rankings.iter().filter_map(|coeff| {
- match &coeff.target {
- RankingTarget::Signal(signal) => Signal::from_str(signal)
- .ok()
- .map(|signal| (signal, coeff.value)),
- }
- }))
- })
- .unwrap_or_default()
+ pub fn signal_coefficients(&self) -> Option<SignalCoefficient> {
+ self.optic.as_ref().map(|optic| {
+ SignalCoefficient::new(optic.rankings.iter().filter_map(|coeff| {
+ match &coeff.target {
+ RankingTarget::Signal(signal) => Signal::from_str(signal)
+ .ok()
+ .map(|signal| (signal, coeff.value)),
+ }
+ }))
+ })
}
}
diff --git a/core/src/ranking/models/linear.rs b/core/src/ranking/models/linear.rs
new file mode 100644
index 00000000..81ed458e
--- /dev/null
+++ b/core/src/ranking/models/linear.rs
@@ -0,0 +1,54 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::fs::File;
+use std::io::BufReader;
+use std::{collections::HashMap, path::Path};
+
+use crate::enum_map::EnumMap;
+use crate::ranking::Signal;
+use crate::Result;
+
+#[derive(serde::Serialize, serde::Deserialize, Debug)]
+struct SerialziedLinearRegression {
+ weights: HashMap<Signal, f64>,
+}
+
+impl From<SerialziedLinearRegression> for LinearRegression {
+ fn from(model: SerialziedLinearRegression) -> Self {
+ let mut weights = EnumMap::new();
+
+ for (signal, weight) in model.weights {
+ weights.insert(signal, weight);
+ }
+
+ Self { weights }
+ }
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Debug)]
+pub struct LinearRegression {
+ pub weights: EnumMap<Signal, f64>,
+}
+
+impl LinearRegression {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let file = File::open(path)?;
+ let reader = BufReader::new(file);
+ let model: SerialziedLinearRegression = serde_json::from_reader(reader)?;
+ Ok(model.into())
+ }
+}
diff --git a/core/src/ranking/models/mod.rs b/core/src/ranking/models/mod.rs
index 8638a905..2756dd23 100644
--- a/core/src/ranking/models/mod.rs
+++ b/core/src/ranking/models/mod.rs
@@ -15,3 +15,4 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
pub mod cross_encoder;
+pub mod linear;
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index de795b97..fea71702 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -46,6 +46,7 @@ use crate::{
use super::bm25::Bm25Weight;
use super::inbound_similarity;
+use super::models::linear::LinearRegression;
#[derive(Debug, Error)]
pub enum Error {
@@ -390,7 +391,7 @@ impl Signal {
value.map(|value| ComputedSignal {
signal: self,
- coefficient: signal_aggregator.coefficients().get(&self),
+ coefficient: signal_aggregator.coefficient(&self),
value,
})
}
@@ -472,7 +473,7 @@ impl Signal {
value.map(|value| ComputedSignal {
signal: self,
- coefficient: signal_aggregator.coefficients().get(&self),
+ coefficient: signal_aggregator.coefficient(&self),
value,
})
}
@@ -593,32 +594,24 @@ fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
}
}
-#[derive(Debug, Clone, Default)]
-pub struct SignalCoefficient(Vec<Option<f64>>);
+#[derive(Debug, Clone)]
+pub struct SignalCoefficient {
+ map: EnumMap<Signal, f64>,
+}
impl SignalCoefficient {
- pub fn get(&self, signal: &Signal) -> f64 {
- self.0
- .get((*signal) as usize)
- .copied()
- .flatten()
- .unwrap_or_else(|| signal.default_coefficient())
+ pub fn get(&self, signal: &Signal) -> Option<f64> {
+ self.map.get(*signal).copied()
}
pub fn new(coefficients: impl Iterator<Item = (Signal, f64)>) -> Self {
- let mut fast_coefficients = Vec::new();
+ let mut map = EnumMap::default();
for (signal, coefficient) in coefficients {
- let idx = signal as usize;
-
- while idx >= fast_coefficients.len() {
- fast_coefficients.push(None);
- }
-
- fast_coefficients[idx] = Some(coefficient);
+ map.insert(signal, coefficient);
}
- Self(fast_coefficients)
+ Self { map }
}
}
@@ -651,7 +644,7 @@ struct SegmentReader {
pub struct SignalAggregator {
query: Option<Query>,
- signal_coefficients: SignalCoefficient,
+ query_signal_coefficients: Option<SignalCoefficient>,
segment_reader: Option<SegmentReader>,
personal_centrality: Option<Arc<online_harmonic::Scorer>>,
inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,
@@ -662,13 +655,14 @@ pub struct SignalAggregator {
region_count: Option<Arc<RegionCount>>,
selected_region: Option<Region>,
current_timestamp: Option<usize>,
+ linear_regression: Option<Arc<LinearRegression>>,
}
impl Clone for SignalAggregator {
fn clone(&self) -> Self {
Self {
query: self.query.clone(),
- signal_coefficients: self.signal_coefficients.clone(),
+ query_signal_coefficients: self.query_signal_coefficients.clone(),
segment_reader: None,
personal_centrality: self.personal_centrality.clone(),
inbound_similariy: self.inbound_similariy.clone(),
@@ -679,6 +673,7 @@ impl Clone for SignalAggregator {
region_count: self.region_count.clone(),
selected_region: self.selected_region,
current_timestamp: self.current_timestamp,
+ linear_regression: self.linear_regression.clone(),
}
}
}
@@ -693,10 +688,7 @@ impl std::fmt::Debug for SignalAggregator {
impl SignalAggregator {
pub fn new(query: Option<Query>) -> Self {
- let signal_coefficients = query
- .as_ref()
- .map(|q| q.signal_coefficients())
- .unwrap_or_default();
+ let query_signal_coefficients = query.as_ref().and_then(|q| q.signal_coefficients());
let fetch_time_ms_cache: Vec<_> = (0..1000)
.map(|fetch_time| 1.0 / (fetch_time as f64 + 1.0))
@@ -710,7 +702,7 @@ impl SignalAggregator {
segment_reader: None,
personal_centrality: None,
inbound_similariy: None,
- signal_coefficients,
+ query_signal_coefficients,
fetch_time_ms_cache,
update_time_cache,
topic_scorer: None,
@@ -718,6 +710,7 @@ impl SignalAggregator {
region_count: None,
selected_region: None,
current_timestamp: None,
+ linear_regression: None,
query,
}
}
@@ -921,6 +914,10 @@ impl SignalAggregator {
self.current_timestamp = Some(current_timestamp);
}
+ pub fn set_linear_model(&mut self, linear_model: Arc<LinearRegression>) {
+ self.linear_regression = Some(linear_model);
+ }
+
pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {
self.topic_scorer
.as_ref()
@@ -992,8 +989,16 @@ impl SignalAggregator {
.sum()
}
- fn coefficients(&self) -> &SignalCoefficient {
- &self.signal_coefficients
+ fn coefficient(&self, signal: &Signal) -> f64 {
+ self.query_signal_coefficients
+ .as_ref()
+ .and_then(|coefficients| coefficients.get(signal))
+ .or_else(|| {
+ self.linear_regression
+ .as_ref()
+ .and_then(|model| model.weights.get(*signal).copied())
+ })
+ .unwrap_or(signal.default_coefficient())
}
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 99218376..0da90704 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -548,9 +548,10 @@ impl DistributedSearcher {
let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
for signal in ALL_SIGNALS {
- if let Some(value) = pointer.website.signals.get(signal) {
- signals.insert(signal, *value);
- }
+ signals.insert(
+ signal,
+ pointer.website.signals.get(signal).copied().unwrap_or(0.0),
+ );
}
website.ranking_signals = Some(signals);
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 5fc7ab4f..3d91f2ca 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -16,6 +16,7 @@
use std::collections::HashMap;
use std::str::FromStr;
+use std::sync::Arc;
use uuid::Uuid;
@@ -25,6 +26,7 @@ use crate::index::Index;
use crate::inverted_index::RetrievedWebpage;
use crate::query::Query;
use crate::ranking::centrality_store::SearchCentralityStore;
+use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator, ALL_SIGNALS};
use crate::schema::TextField;
@@ -46,6 +48,7 @@ pub struct LocalSearcher {
entity_index: Option<EntityIndex>,
centrality_store: Option<SearchCentralityStore>,
topic_centrality: Option<TopicCentrality>,
+ linear_regression: Option<Arc<LinearRegression>>,
}
impl From<Index> for LocalSearcher {
@@ -69,6 +72,7 @@ impl LocalSearcher {
entity_index: None,
centrality_store: None,
topic_centrality: None,
+ linear_regression: None,
}
}
@@ -84,6 +88,10 @@ impl LocalSearcher {
self.topic_centrality = Some(topic_centrality);
}
+ pub fn set_linear_model(&mut self, model: LinearRegression) {
+ self.linear_regression = Some(Arc::new(model));
+ }
+
fn parse_query(&self, ctx: &Ctx, query: &SearchQuery) -> Result<Query> {
let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;
@@ -185,6 +193,10 @@ impl LocalSearcher {
aggregator.set_region_count(self.index.region_count.clone());
+ if let Some(model) = self.linear_regression.as_ref() {
+ aggregator.set_linear_model(model.clone());
+ }
+
let ranker = self.ranker(&parsed_query, ctx, de_rank_similar, aggregator)?;
let res = self.index.inverted_index.search_initial(
@@ -281,7 +293,7 @@ impl LocalSearcher {
/// This function is mainly used for tests and benchmarks
pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
- use std::{sync::Arc, time::Instant};
+ use std::time::Instant;
use crate::{
ranking::models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},
diff --git a/frontend/src/pages/about.mdx b/frontend/src/pages/about.mdx
index f568eee3..56a9d2bd 100644
--- a/frontend/src/pages/about.mdx
+++ b/frontend/src/pages/about.mdx
@@ -5,7 +5,7 @@ title: About us
# About Us
-Stract is an open source search engine where the user has the ability to see exactly what is going on and customize almost everything about their search results. It's a search engine made for hackers just like us. No more searches where some of the terms in the query arent used, and the engine tries to guess what you _really_ meant. You get what you search for.
+Stract is an open source search engine where the user has the ability to see exactly what is going on and customize almost everything about their search results. It's a search engine made for hackers and tinkerers just like ourselves. No more searches where some of the terms in the query arent used, and the engine tries to guess what you _really_ meant. You get what you search for.
## Our Values
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index 5b425411..46cf5a4a 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -7,8 +7,8 @@ import pandas as pd
from tqdm import tqdm
STRACT_ENDPOINT = "localhost:3000"
-NUM_QUERIES = 1
-TOP_N_RESULTS = 2
+NUM_QUERIES = 100
+TOP_N_RESULTS = 10
CENTRALITY_BOOST_OPTIC = 'Ranking(Signal("host_centrality"), 100000000);'
@@ -34,22 +34,34 @@ def search(q, top_n, optic=None):
def score_prompt(query: str, res):
- encoded = json.dumps({k: v for (k, v) in res.items()
- if k in ['domain', 'title', 'snippet']})
- return f"""You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a reputable source. The score should be between 0.0 and 1.0.
-
-query: england politics
-result: {{domain: "https://abingdonrowcondos.com", title: "Gareth Southgate hails hat-trick hero Harry Kane’s return to form",
- snippet: "Plus, Nesbitt Realty will pay a down-payment assistance of $1,903 toward your funds for your down-payment. Please get in touch with Will Nesbitt for learn more about our first-time buyer's credit or 607 Bashford Ln #1 or any ."}}
-score: 0.7
+ r = {k: v for (k, v) in res.items()
+ if k in ['domain', 'title', 'snippet']}
+ r['domain_score'] = '{:f}'.format(res['ranking_signals']['HostCentrality'])
+ encoded = json.dumps(r)
+ return f"""You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a domain with a high domain_score. The domain_score is between 0.0 and 1.0. The score for the search result should be between 0.0 and 1.0.
+
+query: england
+result: {{domain: "theguardian.com", title: "Hungary fans fight with police inside Wembley at start of England match | World Cup 2022 qualifiers | The Guardian",
+ snippet: "Tyrone Mings said that England's players were not fazed by Hungary's fans booing the knee. “We've faced criticism for taking the knee and we have collectively stood passionately together,” the <b>England</b> defender said.",
+ domain_score: 0.028118234}}
+score: 0.8
+
+query: vim plugins
+result: {{domain: "revelry.co", title: "My VIM Setup - Coding Creativity At Revelry",
+ snippet: "Here are the plugins in my Vim setup that I really like: sessionman. im is a Vim session manager, meaning it will save your open buffers to a file for easy re-opening. It's useful in cases when you've been working on a ticket and you need to have one set of files open and then need to switch to another ticket that requires",
+ domain_score: 0.0}}
+score: 0.2
query: q learning algorithm
-result: {{domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange", snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}}
-score: 0.9
-
-query: learn python
-result: {{domain: "jqxxj.fulltime-abbigliamento.it", title: "Vim plugins python",
- snippet: "In fact, VimL can be learned fast, but using. omplete Python syntax highlighter for Vim. shows what I get with default Vim that illustrates … Upgrade to PyCharm, the leading Python IDE: best in class debugging, code navigation, refactoring."}}
+result: {{domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange",
+ snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently",
+ domain_score: 0.017928008}}
+score: 1.0
+
+query: mutton curry variations
+result: {{domain: "earthspice.in", title: "NADAN KOZHI CHICKEN CURRY - EarthSpice",
+ snippet: "VARIATIONS AND SUBSTITUTES You can make this dish without coconut milk for a thinner gravy and different taste. Roasted chicken can also be used Chicken stock can be added along with the coconut milk Earthspice Organic Spice Provide Company in all India, Likes, Garam Masala, Tea Masala, Batthi Masala and Etc.",
+ domain_score: 0.0}}
score: 0.0
query: {query}
@@ -73,7 +85,7 @@ def score(query, results):
def query_prompt(res):
r = {k: v for (k, v) in res.items() if k in ['domain', 'title', 'body']}
- r['body'] = ' '.join(r['body'].split(' ')[:400])
+ r['body'] = ' '.join(r['body'].split(' ')[:200])[:1000]
return f"""Generate a keyword based search query such that the following result would be considered a good result. The query should be at most 4 keywords long. You should not output anything besides the query.
result: {json.dumps(r)}
diff --git a/ltr/linear_model.py b/ltr/linear_model.py
new file mode 100644
index 00000000..b35319b0
--- /dev/null
+++ b/ltr/linear_model.py
@@ -0,0 +1,56 @@
+import sklearn
+import numpy as np
+import json
+import random
+from sklearn.linear_model import LinearRegression
+
+TRAIN_PERCENT = 0.8
+
+with open("ltr_scores.json", "r") as f:
+ scores = json.load(f)
+
+random.shuffle(scores)
+
+feature2id = {k: i for i, (k, _) in enumerate(
+ scores[0]['ranking_signals'].items())}
+id2feature = {v: k for k, v in feature2id.items()}
+
+train = scores[:int(len(scores) * TRAIN_PERCENT)]
+test = scores[int(len(scores) * TRAIN_PERCENT):]
+
+X_train = []
+y_train = []
+for score in train:
+ X_train.append([score['ranking_signals'][k] for k in feature2id])
+ y_train.append(score['score'])
+
+X_test = []
+y_test = []
+for score in test:
+ X_test.append([score['ranking_signals'][k] for k in feature2id])
+ y_test.append(score['score'])
+
+X_train = np.array(X_train)
+y_train = np.array(y_train)
+X_test = np.array(X_test)
+y_test = np.array(y_test)
+
+model = LinearRegression(
+ fit_intercept=False, positive=True)
+model.fit(X_train, y_train)
+
+weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}
+linear_model = {'weights': weights}
+with open("data/linear_model.json", "w") as f:
+ json.dump(linear_model, f, indent=2)
+
+print("TRAIN")
+print("score: ", model.score(X_train, y_train))
+print("Mean squared error: ", sklearn.metrics.mean_squared_error(
+ y_train, model.predict(X_train)))
+
+print()
+print("TEST")
+print("score: ", model.score(X_test, y_test))
+print("Mean squared error: ", sklearn.metrics.mean_squared_error(
+ y_test, model.predict(X_test)))