web search done right
tld id as numerical field so we can potentially use it during ranking
Mikkel Denkerabfc90eaebff
diff --git a/crates/core/src/entrypoint/admin/mod.rs b/crates/core/src/entrypoint/admin/mod.rsindex f0b1dcde..625e2eaf 100644--- a/crates/core/src/entrypoint/admin/mod.rs+++ b/crates/core/src/entrypoint/admin/mod.rs@@ -89,7 +89,7 @@ pub async fn status() -> Result<()> {let config = Config::load()?;let mut conn = sonic::service::Connection::create(config.host).await?;- let status = conn.send(api::ClusterStatus).await?;+ let status = conn.send_without_timeout(api::ClusterStatus).await?;println!("Members:");for member in status.members {@@ -103,7 +103,9 @@ pub async fn top_keyphrases(top: usize) -> Result<()> {let config = Config::load()?;let mut conn = sonic::service::Connection::create(config.host).await?;- let keyphrases = conn.send(api::TopKeyphrases { top }).await?;+ let keyphrases = conn+ .send_without_timeout(api::TopKeyphrases { top })+ .await?;println!("id,text,score");for (i, keyphrase) in keyphrases.iter().enumerate() {diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rsindex 533b5076..71564989 100644--- a/crates/core/src/entrypoint/api.rs+++ b/crates/core/src/entrypoint/api.rs@@ -141,7 +141,10 @@ impl sonic::service::Message<ManagementService> for ClusterStatus {async fn run_management(addr: SocketAddr, cluster: Arc<Cluster>) -> Result<()> {let server = ManagementService::new(cluster).await?.bind(addr).await?;- info!("search server is ready to accept requests on {}", addr);+ info!(+ "management interface is ready to accept requests on {}",+ addr+ );loop {if let Err(e) = server.accept().await {diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex dc9826c8..35be0344 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -148,7 +148,7 @@ impl Signal for HostCentrality {pub struct HostCentralityRank;impl Signal for HostCentralityRank {fn default_coefficient(&self) -> f64 {- 0.4+ 0.02}fn as_field(&self) -> Option<Field> {@@ -228,7 +228,7 @@ impl Signal for PageCentrality {pub struct PageCentralityRank;impl Signal for PageCentralityRank {fn default_coefficient(&self) -> f64 {- 0.4+ 0.02}fn as_field(&self) -> Option<Field> {diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rsindex b0987e6b..09c407a6 100644--- a/crates/core/src/schema/numerical_field.rs+++ b/crates/core/src/schema/numerical_field.rs@@ -16,6 +16,7 @@use bitflags::bitflags;use enum_dispatch::enum_dispatch;+use rustc_hash::FxHashMap;use strum::{EnumDiscriminants, VariantArray};use tantivy::{schema::{BytesOptions, NumericOptions},@@ -26,7 +27,7 @@ use crate::{enum_dispatch_from_discriminant,enum_map::InsertEnumMapKey,simhash,- webpage::{html::FnCache, Html, Webpage},+ webpage::{html::FnCache, url_ext::UrlExt, Html, Webpage},Result,};@@ -170,6 +171,7 @@ pub enum NumericalFieldEnum {LinkDensity,TitleEmbeddings,KeywordEmbeddings,+ SuffixId,}enum_dispatch_from_discriminant!(NumericalFieldEnumDiscriminants => NumericalFieldEnum,@@ -213,6 +215,7 @@ enum_dispatch_from_discriminant!(NumericalFieldEnumDiscriminants => NumericalFieLinkDensity,TitleEmbeddings,KeywordEmbeddings,+ SuffixId,]);impl NumericalFieldEnum {@@ -1392,3 +1395,38 @@ impl NumericalField for KeywordEmbeddings {Ok(())}}++static SUFFIX_ID: std::sync::LazyLock<FxHashMap<String, u32>> = std::sync::LazyLock::new(|| {+ include_str!("../../public_suffix_list.dat")+ .lines()+ .filter(|l| !l.starts_with("//") && !l.chars().all(|c| c.is_whitespace()) && !l.is_empty())+ .enumerate()+ .map(|(i, l)| (l.to_string(), i as u32))+ .collect()+});++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct SuffixId;+impl NumericalField for SuffixId {+ fn name(&self) -> &str {+ "suffix_id"+ }++ fn is_stored(&self) -> bool {+ true+ }++ fn add_html_tantivy(+ &self,+ html: &Html,+ _: &mut FnCache,+ doc: &mut TantivyDocument,+ index: &crate::inverted_index::InvertedIndex,+ ) -> Result<()> {+ let tld = html.url().tld().map(|s| s.to_string()).unwrap_or_default();+ let suffix_id = SUFFIX_ID.get(&tld).copied().unwrap_or(u32::MAX);+ doc.add_u64(self.tantivy_field(index.schema_ref()), suffix_id as u64);++ Ok(())+ }+}diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex 4275df32..59118eeb 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -79,8 +79,16 @@ impl Html {)}+ pub fn root_domain(&self) -> String {+ self.url().root_domain().unwrap_or_default().to_string()+ }++ pub fn icann_domain(&self) -> String {+ self.url().icann_domain().unwrap_or_default().to_string()+ }+pub fn pretokenize_domain(&self) -> PreTokenizedString {- let domain = self.url().root_domain().unwrap_or_default().to_string();+ let domain = self.root_domain();self.pretokenize_string(domain, text_field::Domain.into())}@@ -133,10 +141,9 @@ impl Html {}pub fn domain_name(&self) -> String {- let domain = self.url().domain().unwrap_or_default();- self.url()- .root_domain()- .unwrap_or_default()+ let domain = &self.root_domain();++ domain.find('.').map(|index| &domain[..ceil_char_boundary(domain, index).min(domain.len())]).unwrap_or_default()@@ -214,3 +221,43 @@ impl Html {Ok(doc)}}++#[cfg(test)]+mod tests {+ use url::Url;++ use super::*;++ #[test]+ fn test_domain_name() {+ let url = Url::parse("https://www.example.com").unwrap();+ let html = Html::parse_without_text("", url.as_str()).unwrap();++ assert_eq!(html.domain_name(), "example");+ assert_eq!(html.root_domain(), "example.com");++ let url = Url::parse("https://example.com").unwrap();+ let html = Html::parse_without_text("", url.as_str()).unwrap();++ assert_eq!(html.domain_name(), "example");+ assert_eq!(html.root_domain(), "example.com");++ let url = Url::parse("https://example.co.uk").unwrap();+ let html = Html::parse_without_text("", url.as_str()).unwrap();++ assert_eq!(html.domain_name(), "example");+ assert_eq!(html.root_domain(), "example.co.uk");++ let url = Url::parse("https://this.is.a.test.example.co.uk").unwrap();+ let html = Html::parse_without_text("", url.as_str()).unwrap();++ assert_eq!(html.domain_name(), "example");+ assert_eq!(html.root_domain(), "example.co.uk");++ let url = Url::parse("https://example").unwrap();+ let html = Html::parse_without_text("", url.as_str()).unwrap();++ assert_eq!(html.domain_name(), "");+ assert_eq!(html.root_domain(), "");+ }+}diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex 61c1d054..718e42ea 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -8,7 +8,8 @@ import timeimport requestsimport random-API = "https://stract.com/beta/api/search"+# API = "https://stract.com/beta/api/search"+API = "http://localhost:3000/beta/api/search"NUM_RESULTS_PER_QUERY = 20ELO_K = 32@@ -160,6 +161,10 @@ def get_search_results(query):"query": query,"numResults": NUM_RESULTS_PER_QUERY,"returnRankingSignals": True,+ 'signalCoefficients': {+ 'host_centrality_rank': 0.022,+ 'page_centrality_rank': 0.022,+ }}return [diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex c819bda9..ed3026eb 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -17,6 +17,7 @@ param_grid = {"max_depth": [-1, 2, 4, 8],"num_leaves": [7, 15, 31],"lambda_l2": [0.1, 0.5, 1.0],+ "linear_tree": [False],}diff --git a/ltr/linear_model.py b/ltr/linear_model.pyindex 64c3252d..ec2e8346 100644--- a/ltr/linear_model.py+++ b/ltr/linear_model.py@@ -2,8 +2,11 @@ import sklearnimport numpy as npimport jsonimport random-from sklearn.linear_model import LinearRegression+from sklearn.linear_model import LinearRegression, LogisticRegressionfrom pprint import pprint+import jax+import jax.numpy as jnp+import raxTRAIN_PERCENT = 0.8@@ -74,12 +77,13 @@ random.shuffle(scores)sorted_features = sorted(feature2id.items(), key=lambda x: x[1])for qid, data in queries.items():+ sum_scores = sum([score for _, score, _, _ in data["urls"]])for i, (url, score, _, signals) in enumerate(data["urls"]):signals = {feature2id[k]: v for k, v in signals.items()}- signals = [signals[k] for _, k in sorted_features]+ signals = [signals.get(k, 0) for _, k in sorted_features]- scores.append({"ranking_signals": signals, "score": 20 / (i + 1)})+ scores.append({"ranking_signals": signals, "score": score})train = scores[: int(len(scores) * TRAIN_PERCENT)]test = scores[int(len(scores) * TRAIN_PERCENT) :]@@ -96,33 +100,56 @@ for score in test:X_test.append(score["ranking_signals"])y_test.append(score["score"])-X_train = np.array(X_train)-y_train = np.array(y_train)-X_test = np.array(X_test)-y_test = np.array(y_test)+X_train = jnp.array(X_train)+y_train = jnp.array(y_train)+X_test = jnp.array(X_test)+y_test = jnp.array(y_test)-model = LinearRegression(fit_intercept=False, positive=True)+# jax model+w = jnp.zeros(X_train.shape[1])++def model(w, X):+ return jnp.dot(X, w)++def loss(w, batch):+ features, labels, mask = batch+ scores = model(w, features)++ return rax.approx_t12n(rax.mrr_metric)(scores, labels)++grad_fn = jax.jit(jax.grad(loss))++# train+for i in range(128):+ batch_size = 20+ for j in range(0, len(X_train), batch_size):+ batch = (X_train[j : j + batch_size], y_train[j : j + batch_size], jnp.ones(batch_size))+ w = w - grad_fn(w, batch) * 3e-4+ w = w.clip(0, 10)++++# model = LinearRegression(fit_intercept=False, positive=True)# model = SVR(kernel="linear")# model = DecisionTreeRegressor()-model.fit(X_train, y_train)+# model.fit(X_train, y_train)print("TRAIN")-print("score: ", model.score(X_train, y_train))-print(- "Mean squared error: ",- sklearn.metrics.mean_squared_error(y_train, model.predict(X_train)),-)+for k in [1, 2, 5, 10]:+ print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k))print()print("TEST")-print("score: ", model.score(X_test, y_test))-print(- "Mean squared error: ",- sklearn.metrics.mean_squared_error(y_test, model.predict(X_test)),-)+for k in [1, 2, 5, 10]:+ print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_test], [model(w, X_test)], k=k))-weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}+++weights = {id2feature[i]: float(v) for (i, v) in enumerate(w)}+print()+print("Weights:")+for k, v in sorted(weights.items(), key=lambda x: -x[1]):+ print(k, v)linear_model = {"weights": weights}-pprint(linear_model)with open("data/linear_model.json", "w") as f:json.dump(linear_model, f, indent=2)