web search done right
added 'DomainNameNoTokenizer' field to better rank navigational searches
Mikkel Denkerc71727dedcb6
diff --git a/core/benches/build-similarity.rs b/core/benches/build-similarity.rsindex 14d201be..fbac03c1 100644--- a/core/benches/build-similarity.rs+++ b/core/benches/build-similarity.rs@@ -1,18 +1,14 @@use criterion::{criterion_group, criterion_main, Criterion};-use stract::{- ranking::{centrality_store::CentralityStore, inbound_similarity::InboundSimilarity},- webgraph::WebgraphBuilder,-};+use stract::{ranking::inbound_similarity::InboundSimilarity, webgraph::WebgraphBuilder};const WEBGRAPH_PATH: &str = "data/webgraph";pub fn criterion_benchmark(c: &mut Criterion) {let graph = WebgraphBuilder::new(WEBGRAPH_PATH).open();- let store = CentralityStore::build_harmonic(&graph, stract::gen_temp_path());c.bench_function("Inbound similarity creation", |b| {b.iter(|| {- InboundSimilarity::build(&graph, &store.harmonic);+ InboundSimilarity::build(&graph);})});}diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex 1e9755f3..8512466e 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -83,7 +83,9 @@ impl Modify for ApiModifier {fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {openapi.info.description = Some(r#"Stract is an open source web search engine. The API is totally free while in beta, but some endpoints will most likely be paid by consumption in the future.-The API might also change quite a bit during the beta period, but we will try to keep it as stable as possible. We look forward to see what you will build!"#.to_string(),+The API might also change quite a bit during the beta period, but we will try to keep it as stable as possible. We look forward to see what you will build!++Remember to always give proper attributions to the sources you use from the search results."#.to_string(),);}}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 31fa7979..61e58825 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -89,6 +89,8 @@ pub enum Signal {Bm25SiteNoTokenizer,#[serde(rename = "bm25_domain_no_tokenizer")]Bm25DomainNoTokenizer,+ #[serde(rename = "bm25_domain_name_no_tokenizer")]+ Bm25DomainNameNoTokenizer,#[serde(rename = "bm25_domain_if_homepage")]Bm25DomainIfHomepage,#[serde(rename = "bm25_domain_name_if_homepage_no_tokenizer")]@@ -114,6 +116,7 @@ pub enum Signal {/// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_2")]ProximitySlop2,+/// All proximity signals are disabled for now./// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_4")]@@ -158,7 +161,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 38] = [+pub const ALL_SIGNALS: [Signal; 39] = [Signal::Bm25,Signal::Bm25Title,Signal::Bm25TitleBigrams,@@ -174,6 +177,7 @@ pub const ALL_SIGNALS: [Signal; 38] = [Signal::Bm25Domain,Signal::Bm25SiteNoTokenizer,Signal::Bm25DomainNoTokenizer,+ Signal::Bm25DomainNameNoTokenizer,Signal::Bm25DomainIfHomepage,Signal::Bm25DomainNameIfHomepageNoTokenizer,Signal::Bm25TitleIfHomepage,@@ -281,6 +285,7 @@ impl Signal {Signal::Bm25Domain => 0.00005,Signal::Bm25SiteNoTokenizer => 0.00005,Signal::Bm25DomainNoTokenizer => 0.00005,+ Signal::Bm25DomainNameNoTokenizer => 0.00005,Signal::Bm25DomainIfHomepage => 0.0002,Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0002,Signal::Bm25TitleIfHomepage => 0.00002,@@ -450,6 +455,7 @@ impl Signal {| Signal::Bm25Domain| Signal::Bm25SiteNoTokenizer| Signal::Bm25DomainNoTokenizer+ | Signal::Bm25DomainNameNoTokenizer| Signal::Bm25DomainIfHomepage| Signal::Bm25DomainNameIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage@@ -572,6 +578,7 @@ impl Signal {| Signal::Bm25Domain| Signal::Bm25SiteNoTokenizer| Signal::Bm25DomainNoTokenizer+ | Signal::Bm25DomainNameNoTokenizer| Signal::Bm25DomainIfHomepage| Signal::Bm25DomainNameIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage@@ -642,6 +649,7 @@ impl Signal {Signal::Bm25Domain => Some(TextField::Domain),Signal::Bm25SiteNoTokenizer => Some(TextField::SiteNoTokenizer),Signal::Bm25DomainNoTokenizer => Some(TextField::DomainNoTokenizer),+ Signal::Bm25DomainNameNoTokenizer => Some(TextField::DomainNameNoTokenizer),Signal::Bm25DomainIfHomepage => Some(TextField::DomainIfHomepage),Signal::Bm25DomainNameIfHomepageNoTokenizer => {Some(TextField::DomainNameIfHomepageNoTokenizer)diff --git a/core/src/schema.rs b/core/src/schema.rsindex 8de93764..8d335821 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -36,6 +36,7 @@ pub enum TextField {SiteNoTokenizer,SiteIfHomepageNoTokenizer,DomainNoTokenizer,+ DomainNameNoTokenizer,/// this field is only set if the webpage is the homepage for the site. Allows us to boostDomainIfHomepage,/// this field is only set if the webpage is the homepage for the site. Allows us to boost@@ -87,6 +88,7 @@ impl TextField {TextField::SiteNoTokenizer => Tokenizer::Identity(Identity {}),TextField::SiteIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),TextField::DomainNoTokenizer => Tokenizer::Identity(Identity {}),+ TextField::DomainNameNoTokenizer => Tokenizer::Identity(Identity {}),TextField::DomainIfHomepage => Tokenizer::default(),TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),TextField::TitleIfHomepage => Tokenizer::default(),@@ -126,6 +128,7 @@ impl TextField {TextField::SiteNoTokenizer => false,TextField::SiteIfHomepageNoTokenizer => false,TextField::DomainNoTokenizer => false,+ TextField::DomainNameNoTokenizer => false,TextField::DomainIfHomepage => false,TextField::DomainNameIfHomepageNoTokenizer => false,TextField::TitleIfHomepage => false,@@ -154,6 +157,7 @@ impl TextField {TextField::SiteNoTokenizer => "site_no_tokenizer",TextField::SiteIfHomepageNoTokenizer => "site_if_homepage_no_tokenizer",TextField::DomainNoTokenizer => "domain_no_tokenizer",+ TextField::DomainNameNoTokenizer => "domain_name_no_tokenizer",TextField::BacklinkText => "backlink_text",TextField::StemmedTitle => "stemmed_title",TextField::StemmedCleanBody => "stemmed_body",@@ -260,7 +264,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 57] = [+pub static ALL_FIELDS: [Field; 58] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -273,6 +277,7 @@ pub static ALL_FIELDS: [Field; 57] = [Field::Text(TextField::SiteNoTokenizer),Field::Text(TextField::SiteIfHomepageNoTokenizer),Field::Text(TextField::DomainNoTokenizer),+ Field::Text(TextField::DomainNameNoTokenizer),Field::Text(TextField::DomainIfHomepage),Field::Text(TextField::DomainNameIfHomepageNoTokenizer),Field::Text(TextField::TitleIfHomepage),@@ -365,6 +370,9 @@ impl Field {Field::Text(TextField::DomainNoTokenizer) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::DomainNameNoTokenizer) => {+ IndexingOption::Text(self.default_text_options())+ }Field::Text(TextField::AllBody) => IndexingOption::Text(self.default_text_options()),Field::Text(TextField::DomainIfHomepage) => {IndexingOption::Text(self.default_text_options())diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex f03f16ad..e691186d 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -931,6 +931,17 @@ impl Html {let description = self.pretokenize_description();let microformats = self.pretokenize_microformats();+ let domain_name = self+ .url()+ .root_domain()+ .unwrap_or_default()+ .find('.')+ .map(|index| {+ &domain.text[..ceil_char_boundary(&domain.text, index).min(domain.text.len())]+ })+ .unwrap_or_default()+ .to_string();+let schemas: Vec<_> = self.schema_org();let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();@@ -1109,19 +1120,23 @@ impl Html {doc.add_text(tantivy_field, "");}}+ Field::Text(TextField::DomainNameNoTokenizer) => {+ doc.add_pre_tokenized_text(+ tantivy_field,+ PreTokenizedString {+ text: domain_name.to_string(),+ tokens: vec![tantivy::tokenizer::Token {+ offset_from: 0,+ offset_to: domain_name.len(),+ position: 0,+ text: domain_name.to_string(),+ position_length: 1,+ }],+ },+ );+ }Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {if self.is_homepage() {- let domain_name = self- .url()- .root_domain()- .unwrap_or_default()- .find('.')- .map(|index| {- &domain.text[..ceil_char_boundary(&domain.text, index)- .min(domain.text.len())]- })- .unwrap_or_default();-doc.add_pre_tokenized_text(tantivy_field,PreTokenizedString {