Skip to content

web search done right

tld id as numerical field so we can potentially use it during ranking

Mikkel Denkerabfc90eaebff
diff --git a/crates/core/src/entrypoint/admin/mod.rs b/crates/core/src/entrypoint/admin/mod.rs
index f0b1dcde..625e2eaf 100644
--- a/crates/core/src/entrypoint/admin/mod.rs
+++ b/crates/core/src/entrypoint/admin/mod.rs
@@ -89,7 +89,7 @@ pub async fn status() -> Result<()> {
let config = Config::load()?;
let mut conn = sonic::service::Connection::create(config.host).await?;
- let status = conn.send(api::ClusterStatus).await?;
+ let status = conn.send_without_timeout(api::ClusterStatus).await?;
println!("Members:");
for member in status.members {
@@ -103,7 +103,9 @@ pub async fn top_keyphrases(top: usize) -> Result<()> {
let config = Config::load()?;
let mut conn = sonic::service::Connection::create(config.host).await?;
- let keyphrases = conn.send(api::TopKeyphrases { top }).await?;
+ let keyphrases = conn
+ .send_without_timeout(api::TopKeyphrases { top })
+ .await?;
println!("id,text,score");
for (i, keyphrase) in keyphrases.iter().enumerate() {
diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rs
index 533b5076..71564989 100644
--- a/crates/core/src/entrypoint/api.rs
+++ b/crates/core/src/entrypoint/api.rs
@@ -141,7 +141,10 @@ impl sonic::service::Message<ManagementService> for ClusterStatus {
async fn run_management(addr: SocketAddr, cluster: Arc<Cluster>) -> Result<()> {
let server = ManagementService::new(cluster).await?.bind(addr).await?;
- info!("search server is ready to accept requests on {}", addr);
+ info!(
+ "management interface is ready to accept requests on {}",
+ addr
+ );
loop {
if let Err(e) = server.accept().await {
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rs
index dc9826c8..35be0344 100644
--- a/crates/core/src/ranking/signal/non_text.rs
+++ b/crates/core/src/ranking/signal/non_text.rs
@@ -148,7 +148,7 @@ impl Signal for HostCentrality {
pub struct HostCentralityRank;
impl Signal for HostCentralityRank {
fn default_coefficient(&self) -> f64 {
- 0.4
+ 0.02
}
fn as_field(&self) -> Option<Field> {
@@ -228,7 +228,7 @@ impl Signal for PageCentrality {
pub struct PageCentralityRank;
impl Signal for PageCentralityRank {
fn default_coefficient(&self) -> f64 {
- 0.4
+ 0.02
}
fn as_field(&self) -> Option<Field> {
diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rs
index b0987e6b..09c407a6 100644
--- a/crates/core/src/schema/numerical_field.rs
+++ b/crates/core/src/schema/numerical_field.rs
@@ -16,6 +16,7 @@
use bitflags::bitflags;
use enum_dispatch::enum_dispatch;
+use rustc_hash::FxHashMap;
use strum::{EnumDiscriminants, VariantArray};
use tantivy::{
schema::{BytesOptions, NumericOptions},
@@ -26,7 +27,7 @@ use crate::{
enum_dispatch_from_discriminant,
enum_map::InsertEnumMapKey,
simhash,
- webpage::{html::FnCache, Html, Webpage},
+ webpage::{html::FnCache, url_ext::UrlExt, Html, Webpage},
Result,
};
@@ -170,6 +171,7 @@ pub enum NumericalFieldEnum {
LinkDensity,
TitleEmbeddings,
KeywordEmbeddings,
+ SuffixId,
}
enum_dispatch_from_discriminant!(NumericalFieldEnumDiscriminants => NumericalFieldEnum,
@@ -213,6 +215,7 @@ enum_dispatch_from_discriminant!(NumericalFieldEnumDiscriminants => NumericalFie
LinkDensity,
TitleEmbeddings,
KeywordEmbeddings,
+ SuffixId,
]);
impl NumericalFieldEnum {
@@ -1392,3 +1395,38 @@ impl NumericalField for KeywordEmbeddings {
Ok(())
}
}
+
+static SUFFIX_ID: std::sync::LazyLock<FxHashMap<String, u32>> = std::sync::LazyLock::new(|| {
+ include_str!("../../public_suffix_list.dat")
+ .lines()
+ .filter(|l| !l.starts_with("//") && !l.chars().all(|c| c.is_whitespace()) && !l.is_empty())
+ .enumerate()
+ .map(|(i, l)| (l.to_string(), i as u32))
+ .collect()
+});
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SuffixId;
+impl NumericalField for SuffixId {
+ fn name(&self) -> &str {
+ "suffix_id"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _: &mut FnCache,
+ doc: &mut TantivyDocument,
+ index: &crate::inverted_index::InvertedIndex,
+ ) -> Result<()> {
+ let tld = html.url().tld().map(|s| s.to_string()).unwrap_or_default();
+ let suffix_id = SUFFIX_ID.get(&tld).copied().unwrap_or(u32::MAX);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), suffix_id as u64);
+
+ Ok(())
+ }
+}
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index 4275df32..59118eeb 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -79,8 +79,16 @@ impl Html {
)
}
+ pub fn root_domain(&self) -> String {
+ self.url().root_domain().unwrap_or_default().to_string()
+ }
+
+ pub fn icann_domain(&self) -> String {
+ self.url().icann_domain().unwrap_or_default().to_string()
+ }
+
pub fn pretokenize_domain(&self) -> PreTokenizedString {
- let domain = self.url().root_domain().unwrap_or_default().to_string();
+ let domain = self.root_domain();
self.pretokenize_string(domain, text_field::Domain.into())
}
@@ -133,10 +141,9 @@ impl Html {
}
pub fn domain_name(&self) -> String {
- let domain = self.url().domain().unwrap_or_default();
- self.url()
- .root_domain()
- .unwrap_or_default()
+ let domain = &self.root_domain();
+
+ domain
.find('.')
.map(|index| &domain[..ceil_char_boundary(domain, index).min(domain.len())])
.unwrap_or_default()
@@ -214,3 +221,43 @@ impl Html {
Ok(doc)
}
}
+
+#[cfg(test)]
+mod tests {
+ use url::Url;
+
+ use super::*;
+
+ #[test]
+ fn test_domain_name() {
+ let url = Url::parse("https://www.example.com").unwrap();
+ let html = Html::parse_without_text("", url.as_str()).unwrap();
+
+ assert_eq!(html.domain_name(), "example");
+ assert_eq!(html.root_domain(), "example.com");
+
+ let url = Url::parse("https://example.com").unwrap();
+ let html = Html::parse_without_text("", url.as_str()).unwrap();
+
+ assert_eq!(html.domain_name(), "example");
+ assert_eq!(html.root_domain(), "example.com");
+
+ let url = Url::parse("https://example.co.uk").unwrap();
+ let html = Html::parse_without_text("", url.as_str()).unwrap();
+
+ assert_eq!(html.domain_name(), "example");
+ assert_eq!(html.root_domain(), "example.co.uk");
+
+ let url = Url::parse("https://this.is.a.test.example.co.uk").unwrap();
+ let html = Html::parse_without_text("", url.as_str()).unwrap();
+
+ assert_eq!(html.domain_name(), "example");
+ assert_eq!(html.root_domain(), "example.co.uk");
+
+ let url = Url::parse("https://example").unwrap();
+ let html = Html::parse_without_text("", url.as_str()).unwrap();
+
+ assert_eq!(html.domain_name(), "");
+ assert_eq!(html.root_domain(), "");
+ }
+}
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index 61c1d054..718e42ea 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -8,7 +8,8 @@ import time
import requests
import random
-API = "https://stract.com/beta/api/search"
+# API = "https://stract.com/beta/api/search"
+API = "http://localhost:3000/beta/api/search"
NUM_RESULTS_PER_QUERY = 20
ELO_K = 32
@@ -160,6 +161,10 @@ def get_search_results(query):
"query": query,
"numResults": NUM_RESULTS_PER_QUERY,
"returnRankingSignals": True,
+ 'signalCoefficients': {
+ 'host_centrality_rank': 0.022,
+ 'page_centrality_rank': 0.022,
+ }
}
return [
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index c819bda9..ed3026eb 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -17,6 +17,7 @@ param_grid = {
"max_depth": [-1, 2, 4, 8],
"num_leaves": [7, 15, 31],
"lambda_l2": [0.1, 0.5, 1.0],
+ "linear_tree": [False],
}
diff --git a/ltr/linear_model.py b/ltr/linear_model.py
index 64c3252d..ec2e8346 100644
--- a/ltr/linear_model.py
+++ b/ltr/linear_model.py
@@ -2,8 +2,11 @@ import sklearn
import numpy as np
import json
import random
-from sklearn.linear_model import LinearRegression
+from sklearn.linear_model import LinearRegression, LogisticRegression
from pprint import pprint
+import jax
+import jax.numpy as jnp
+import rax
TRAIN_PERCENT = 0.8
@@ -74,12 +77,13 @@ random.shuffle(scores)
sorted_features = sorted(feature2id.items(), key=lambda x: x[1])
for qid, data in queries.items():
+ sum_scores = sum([score for _, score, _, _ in data["urls"]])
for i, (url, score, _, signals) in enumerate(data["urls"]):
signals = {feature2id[k]: v for k, v in signals.items()}
- signals = [signals[k] for _, k in sorted_features]
+ signals = [signals.get(k, 0) for _, k in sorted_features]
- scores.append({"ranking_signals": signals, "score": 20 / (i + 1)})
+ scores.append({"ranking_signals": signals, "score": score})
train = scores[: int(len(scores) * TRAIN_PERCENT)]
test = scores[int(len(scores) * TRAIN_PERCENT) :]
@@ -96,33 +100,56 @@ for score in test:
X_test.append(score["ranking_signals"])
y_test.append(score["score"])
-X_train = np.array(X_train)
-y_train = np.array(y_train)
-X_test = np.array(X_test)
-y_test = np.array(y_test)
+X_train = jnp.array(X_train)
+y_train = jnp.array(y_train)
+X_test = jnp.array(X_test)
+y_test = jnp.array(y_test)
-model = LinearRegression(fit_intercept=False, positive=True)
+# jax model
+w = jnp.zeros(X_train.shape[1])
+
+def model(w, X):
+ return jnp.dot(X, w)
+
+def loss(w, batch):
+ features, labels, mask = batch
+ scores = model(w, features)
+
+ return rax.approx_t12n(rax.mrr_metric)(scores, labels)
+
+grad_fn = jax.jit(jax.grad(loss))
+
+# train
+for i in range(128):
+ batch_size = 20
+ for j in range(0, len(X_train), batch_size):
+ batch = (X_train[j : j + batch_size], y_train[j : j + batch_size], jnp.ones(batch_size))
+ w = w - grad_fn(w, batch) * 3e-4
+ w = w.clip(0, 10)
+
+
+
+# model = LinearRegression(fit_intercept=False, positive=True)
# model = SVR(kernel="linear")
# model = DecisionTreeRegressor()
-model.fit(X_train, y_train)
+# model.fit(X_train, y_train)
print("TRAIN")
-print("score: ", model.score(X_train, y_train))
-print(
- "Mean squared error: ",
- sklearn.metrics.mean_squared_error(y_train, model.predict(X_train)),
-)
+for k in [1, 2, 5, 10]:
+ print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k))
print()
print("TEST")
-print("score: ", model.score(X_test, y_test))
-print(
- "Mean squared error: ",
- sklearn.metrics.mean_squared_error(y_test, model.predict(X_test)),
-)
+for k in [1, 2, 5, 10]:
+ print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_test], [model(w, X_test)], k=k))
-weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}
+
+
+weights = {id2feature[i]: float(v) for (i, v) in enumerate(w)}
+print()
+print("Weights:")
+for k, v in sorted(weights.items(), key=lambda x: -x[1]):
+ print(k, v)
linear_model = {"weights": weights}
-pprint(linear_model)
with open("data/linear_model.json", "w") as f:
json.dump(linear_model, f, indent=2)