Skip to content

web search done right

number of slashes and digits in url as ranking signal

Mikkel Denker1e2ee6b8ea59
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 4c36c797..9ec10c24 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -235,7 +235,13 @@ impl InvertedIndex {
let mut top_websites = Vec::new();
let mut pointers: Vec<_> = pointers.into_iter().enumerate().collect();
- pointers.sort_by(|a, b| a.1.address.segment.cmp(&b.1.address.segment));
+ pointers.sort_by(|a, b| {
+ a.1.address
+ .segment
+ .cmp(&b.1.address.segment)
+ .then_with(|| a.1.address.doc_id.cmp(&b.1.address.doc_id))
+ });
+
let mut prev_segment = None;
for (orig_index, pointer) in pointers {
let update_segment = match prev_segment {
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 2071858c..09b77ea0 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -135,7 +135,7 @@ mod tests {
use crate::{
index::Index,
searcher::{LocalSearcher, SearchQuery},
- webpage::{Html, Webpage},
+ webpage::{Html, Webpage, Link},
};
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
@@ -216,253 +216,153 @@ mod tests {
assert_eq!(result.webpages[1].url, "https://www.a.com");
}
- // #[test]
- // fn page_centrality_ranking() {
- // let mut index = Index::temporary().expect("Unable to open index");
-
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>Website A</title>
- // </head>
- // <body>
- // {CONTENT} {}
- // example example example
- // </body>
- // </html>
- // "#,
- // crate::rand_words(100)
- // ),
- // "https://www.a.com",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.0,
- // fetch_time_ms: 500,
- // page_centrality: 0.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>Website B</title>
- // </head>
- // <body>
- // {CONTENT} {}
- // </body>
- // </html>
- // "#,
- // crate::rand_words(100)
- // ),
- // "https://www.b.com",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.0,
- // fetch_time_ms: 500,
- // page_centrality: 5.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
-
- // index.commit().expect("failed to commit index");
- // let searcher = LocalSearcher::from(index);
- // let result = searcher
- // .search(&SearchQuery {
- // query: "example".to_string(),
- // ..Default::default()
- // })
- // .expect("Search failed");
- // assert_eq!(result.webpages.len(), 2);
- // assert_eq!(result.webpages[0].url, "https://www.b.com");
- // assert_eq!(result.webpages[1].url, "https://www.a.com");
- // }
-
- // #[test]
- // fn navigational_search() {
- // let mut index = Index::temporary().expect("Unable to open index");
-
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>DR Homepage</title>
- // </head>
- // <body>
- // {CONTENT} {}
- // </body>
- // </html>
- // "#,
- // crate::rand_words(100)
- // ),
- // "https://www.dr.dk",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.0,
- // fetch_time_ms: 500,
- // page_centrality: 0.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>Subsite dr</title>
- // </head>
- // <body>
- // {CONTENT} {}
- // </body>
- // </html>
- // "#,
- // crate::rand_words(100)
- // ),
- // "https://www.dr.dk/whatever",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.0,
- // fetch_time_ms: 500,
- // page_centrality: 0.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>Website B</title>
- // </head>
- // dr and some other text {CONTENT} dk {}
- // </html>
- // "#,
- // crate::rand_words(100)
- // ),
- // "https://www.b.com",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.003,
- // fetch_time_ms: 500,
- // page_centrality: 0.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
-
- // index.commit().expect("failed to commit index");
- // let searcher = LocalSearcher::from(index);
- // let result = searcher
- // .search(&SearchQuery {
- // query: "dr dk".to_string(),
- // ..Default::default()
- // })
- // .expect("Search failed");
-
- // assert_eq!(result.webpages.len(), 3);
- // assert_eq!(result.webpages[0].url, "https://www.dr.dk");
- // }
-
- // #[test]
- // fn freshness_ranking() {
- // let mut index = Index::temporary().expect("Unable to open index");
-
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>Title</title>
- // <meta property="og:updated_time" content="1999-06-22T19:37:34+00:00" />
- // </head>
- // <body>
- // {CONTENT}
- // </body>
- // </html>
- // "#
- // ),
- // "https://www.old.com",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.0091,
- // fetch_time_ms: 500,
- // page_centrality: 0.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // &format!(
- // r#"
- // <html>
- // <head>
- // <title>Title</title>
- // <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
- // </head>
- // <body>
- // {CONTENT}
- // </body>
- // </html>
- // "#
- // ),
- // "https://www.new.com",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.009,
- // fetch_time_ms: 500,
- // pre_computed_score: 0.0,
- // page_centrality: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
-
- // index.commit().expect("failed to commit index");
- // let searcher = LocalSearcher::from(index);
- // let result = searcher
- // .search(&SearchQuery {
- // query: "title".to_string(),
- // ..Default::default()
- // })
- // .expect("Search failed");
-
- // assert_eq!(result.webpages[0].url, "https://www.new.com");
- // }
+ #[test]
+ fn page_centrality_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ example example example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.a.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.b.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 5.0,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+ let result = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ ..Default::default()
+ })
+ .expect("Search failed");
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.b.com");
+ assert_eq!(result.webpages[1].url, "https://www.a.com");
+ }
+
+ #[test]
+ fn freshness_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Title</title>
+ <meta property="og:updated_time" content="1999-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.old.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 499,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Title</title>
+ <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.new.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ page_centrality: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+ let result = searcher
+ .search(&SearchQuery {
+ query: "title".to_string(),
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages[0].url, "https://www.new.com");
+ }
#[test]
fn derank_trackers() {
@@ -544,77 +444,77 @@ mod tests {
assert_eq!(result.webpages[0].url, "https://www.first.com");
}
- // #[test]
- // fn backlink_text() {
- // let mut index = Index::temporary().expect("Unable to open index");
-
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // r#"
- // <html>
- // <head>
- // <title>Test site</title>
- // </head>
- // <body>
- // test
- // </body>
- // </html>
- // "#,
- // "https://www.first.com",
- // ),
- // backlinks: vec![Link {
- // source: "https://www.second.com".to_string().into(),
- // destination: "https://www.first.com".to_string().into(),
- // text: "test this is the best test site".to_string(),
- // }],
- // host_centrality: 0.0,
- // fetch_time_ms: 500,
- // page_centrality: 0.0,
- // pre_computed_score: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
- // index
- // .insert(Webpage {
- // html: Html::parse(
- // r#"
- // <html>
- // <head>
- // <title>Second test site</title>
- // </head>
- // <body>
- // test test test test test test test
- // </body>
- // </html>
- // "#,
- // "https://www.second.com",
- // ),
- // backlinks: vec![],
- // host_centrality: 0.00003,
- // fetch_time_ms: 500,
- // pre_computed_score: 0.0,
- // page_centrality: 0.0,
- //
- // node_id: None,
- // dmoz_description: None,
- // })
- // .expect("failed to insert webpage");
-
- // index.commit().expect("failed to commit index");
- // let searcher = LocalSearcher::from(index);
- // let result = searcher
- // .search(&SearchQuery {
- // query: "test".to_string(),
- // ..Default::default()
- // })
- // .expect("Search failed");
-
- // assert_eq!(result.webpages.len(), 2);
- // assert_eq!(result.webpages[0].url, "https://www.first.com");
- // }
+ #[test]
+ fn backlink_text() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "https://www.first.com",
+ ),
+ backlinks: vec![Link {
+ source: "https://www.second.com".to_string().into(),
+ destination: "https://www.first.com".to_string().into(),
+ text: "test this is the best test site".to_string(),
+ }],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>Second test site</title>
+ </head>
+ <body>
+ test test test test test test test
+ </body>
+ </html>
+ "#,
+ "https://www.second.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.00003,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ page_centrality: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+ }
#[test]
fn custom_signal_aggregation() {
@@ -932,4 +832,110 @@ mod tests {
assert_eq!(result.webpages[0].url, "https://www.first.com");
assert_eq!(result.webpages[1].url, "https://www.second.com");
}
+
+
+ #[test]
+ fn num_slashes_and_digits() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.first.com/one",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 2,
+ pre_computed_score: 0.0,
+ page_centrality: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.second.com/one/two",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 1,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.third.com/one/two123",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 0,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+
+ node_id: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::new(index);
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.num_hits, 3);
+ assert_eq!(result.webpages.len(), 3);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/one");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/one/two");
+ assert_eq!(result.webpages[2].url, "https://www.third.com/one/two123");
+ }
}
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index d5b05e11..967d8b4a 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -64,6 +64,16 @@ mod tests {
Node::from("https://www.third.com").into_host(),
String::new(),
);
+ graph.insert(
+ Node::from("https://www.extra.com").into_host(),
+ Node::from("https://www.first.com").into_host(),
+ String::new(),
+ );
+ graph.insert(
+ Node::from("https://www.second.com").into_host(),
+ Node::from("https://www.extra.com").into_host(),
+ String::new(),
+ );
graph.commit();
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 52fd98a0..7d4e6492 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -129,6 +129,10 @@ pub enum Signal {
LambdaMART,
#[serde(rename = "linear_regression")]
LinearRegression,
+ #[serde(rename = "url_digits")]
+ UrlDigits,
+ #[serde(rename = "url_slashes")]
+ UrlSlashes,
}
impl From<Signal> for usize {
@@ -137,7 +141,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 36] = [
+pub const ALL_SIGNALS: [Signal; 38] = [
Signal::Bm25,
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
@@ -174,6 +178,8 @@ pub const ALL_SIGNALS: [Signal; 36] = [
Signal::QueryCentrality,
Signal::InboundSimilarity,
Signal::LambdaMART,
+ Signal::UrlDigits,
+ Signal::UrlSlashes,
];
fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f64 {
@@ -195,6 +201,31 @@ fn score_trackers(num_trackers: f64) -> f64 {
1.0 / (num_trackers + 1.0)
}
+fn score_digits(num_digits: f64) -> f64 {
+ 1.0 / (num_digits + 1.0)
+}
+
+fn score_slashes(num_slashes: f64) -> f64 {
+ 1.0 / (num_slashes + 1.0)
+}
+
+fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
+ match aggregator.region_count.as_ref() {
+ Some(region_count) => {
+ let boost = aggregator.selected_region.map_or(0.0, |region| {
+ if region == webpage_region {
+ 50.0
+ } else {
+ 0.0
+ }
+ });
+
+ boost + region_count.score(&webpage_region)
+ }
+ None => 0.0,
+ }
+}
+
fn bm25(field: &mut TextFieldData, doc: DocId) -> f64 {
let mut term_freq = 0;
for posting in &mut field.postings {
@@ -236,7 +267,7 @@ impl Signal {
Signal::Bm25DomainIfHomepage => 0.0,
Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0,
Signal::Bm25TitleIfHomepage => 0.0,
- Signal::Bm25BacklinkText => 0.0,
+ Signal::Bm25BacklinkText => 0.001,
Signal::Bm25Description => 5.99679347128802e-05,
Signal::ProximitySlop0 => 0.0,
Signal::ProximitySlop1 => 0.0,
@@ -244,17 +275,19 @@ impl Signal {
Signal::ProximitySlop4 => 0.0,
Signal::ProximitySlop8 => 0.003435930677879231,
Signal::CrossEncoder => 0.16801589371906178,
- Signal::HostCentrality => 15.233546236366237,
- Signal::PageCentrality => 0.0,
+ Signal::HostCentrality => 1.0,
+ Signal::PageCentrality => 2.0,
Signal::QueryCentrality => 0.0,
Signal::IsHomepage => 0.0,
- Signal::FetchTimeMs => 0.0,
- Signal::UpdateTimestamp => 0.0,
+ Signal::FetchTimeMs => 0.001,
+ Signal::UpdateTimestamp => 0.001,
Signal::TrackerScore => 0.019542120533715634,
Signal::Region => 0.06437975586036043,
Signal::InboundSimilarity => 10.0,
Signal::LambdaMART => 10.0,
Signal::LinearRegression => 1.0,
+ Signal::UrlSlashes => 0.01,
+ Signal::UrlDigits => 0.01,
}
}
@@ -335,6 +368,20 @@ impl Signal {
field_value.map(|num_trackers| score_trackers(num_trackers as f64))
}
+ Signal::UrlDigits => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|num_digits| score_digits(num_digits as f64))
+ }
+ Signal::UrlSlashes => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|num_slashes| score_slashes(num_slashes as f64))
+ }
Signal::Region => {
let field_value: Option<u64> = self
.fastfield_value(signal_aggregator, doc)
@@ -481,6 +528,8 @@ impl Signal {
| Signal::ProximitySlop2
| Signal::ProximitySlop4
| Signal::ProximitySlop8
+ | Signal::UrlDigits
+ | Signal::UrlSlashes
| Signal::CrossEncoder
| Signal::InboundSimilarity
| Signal::LambdaMART
@@ -518,6 +567,8 @@ impl Signal {
Signal::UpdateTimestamp => Some(FastField::LastUpdated),
Signal::TrackerScore => Some(FastField::TrackerScore),
Signal::Region => Some(FastField::Region),
+ Signal::UrlSlashes => Some(FastField::NumPathAndQuerySlashes),
+ Signal::UrlDigits => Some(FastField::NumPathAndQueryDigits),
_ => None,
}
}
@@ -560,23 +611,6 @@ impl FromStr for Signal {
}
}
-fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
- match aggregator.region_count.as_ref() {
- Some(region_count) => {
- let boost = aggregator.selected_region.map_or(0.0, |region| {
- if region == webpage_region {
- 50.0
- } else {
- 0.0
- }
- });
-
- boost + region_count.score(&webpage_region)
- }
- None => 0.0,
- }
-}
-
#[derive(Debug, Clone, Default)]
pub struct SignalCoefficient {
map: EnumMap<Signal, f64>,
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 671a15ef..4bf72b86 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -185,6 +185,8 @@ pub enum FastField {
HostNodeID,
SimHash,
NumFlattenedSchemaTokens,
+ NumPathAndQuerySlashes,
+ NumPathAndQueryDigits,
}
impl FastField {
@@ -217,6 +219,8 @@ impl FastField {
FastField::PreComputedScore => "pre_computed_score",
FastField::HostNodeID => "host_node_id",
FastField::SimHash => "sim_hash",
+ FastField::NumPathAndQuerySlashes => "num_path_and_query_slashes",
+ FastField::NumPathAndQueryDigits => "num_path_and_query_digits",
}
}
}
@@ -233,7 +237,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 51] = [
+pub static ALL_FIELDS: [Field; 53] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -286,6 +290,8 @@ pub static ALL_FIELDS: [Field; 51] = [
Field::Fast(FastField::PreComputedScore),
Field::Fast(FastField::HostNodeID),
Field::Fast(FastField::SimHash),
+ Field::Fast(FastField::NumPathAndQuerySlashes),
+ Field::Fast(FastField::NumPathAndQueryDigits),
];
impl Field {
@@ -493,6 +499,18 @@ impl Field {
.set_indexed()
.set_stored(),
),
+ Field::Fast(FastField::NumPathAndQuerySlashes) => IndexingOption::Integer(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed()
+ .set_stored(),
+ ),
+ Field::Fast(FastField::NumPathAndQueryDigits) => IndexingOption::Integer(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed()
+ .set_stored(),
+ ),
}
}
@@ -589,6 +607,8 @@ impl FastField {
FastField::PreComputedScore => DataType::F64,
FastField::HostNodeID => DataType::U64,
FastField::SimHash => DataType::U64,
+ FastField::NumPathAndQuerySlashes => DataType::U64,
+ FastField::NumPathAndQueryDigits => DataType::U64,
}
}
}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index c31de376..162ae1ca 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -913,6 +913,24 @@ impl Html {
};
doc.add_u64(tantivy_field, hash);
}
+ Field::Fast(FastField::NumPathAndQuerySlashes) => {
+ let num_slashes = self
+ .url()
+ .path_and_query()
+ .chars()
+ .filter(|c| *c == '/')
+ .count();
+ doc.add_u64(tantivy_field, num_slashes as u64);
+ }
+ Field::Fast(FastField::NumPathAndQueryDigits) => {
+ let num_digits = self
+ .url()
+ .path_and_query()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count();
+ doc.add_u64(tantivy_field, num_digits as u64);
+ }
Field::Text(TextField::BacklinkText)
| Field::Fast(FastField::HostCentrality)
| Field::Fast(FastField::PageCentrality)
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 5f17b615..9214f473 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -291,6 +291,13 @@ impl Url {
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
+
+ pub fn path_and_query(&self) -> &str {
+ let without_prot = self.without_protocol();
+ without_prot
+ .find('/')
+ .map_or("", |idx| without_prot.get(idx..).map_or("", |path| path))
+ }
}
#[cfg(test)]
@@ -464,4 +471,22 @@ mod tests {
let url: Url = "https://test.example.com/test.png".to_string().into();
assert!(url.path_ends_with(".png"));
}
+
+ #[test]
+ fn path_and_query() {
+ let url: Url = "https://example.com".to_string().into();
+ assert!(url.path_and_query().is_empty());
+
+ let url: Url = "https://example.com/".to_string().into();
+ assert!(url.path_and_query().is_empty());
+
+ let url: Url = "https://example.com/test".to_string().into();
+ assert_eq!(url.path_and_query(), "/test");
+
+ let url: Url = "https://example.com/test?a=b".to_string().into();
+ assert_eq!(url.path_and_query(), "/test?a=b");
+
+ let url: Url = "example.com/test?a=b".to_string().into();
+ assert_eq!(url.path_and_query(), "/test?a=b");
+ }
}