Skip to content

web search done right

added 'DomainNameNoTokenizer' field to better rank navigational searches

Mikkel Denkerc71727dedcb6
diff --git a/core/benches/build-similarity.rs b/core/benches/build-similarity.rs
index 14d201be..fbac03c1 100644
--- a/core/benches/build-similarity.rs
+++ b/core/benches/build-similarity.rs
@@ -1,18 +1,14 @@
use criterion::{criterion_group, criterion_main, Criterion};
-use stract::{
- ranking::{centrality_store::CentralityStore, inbound_similarity::InboundSimilarity},
- webgraph::WebgraphBuilder,
-};
+use stract::{ranking::inbound_similarity::InboundSimilarity, webgraph::WebgraphBuilder};
const WEBGRAPH_PATH: &str = "data/webgraph";
pub fn criterion_benchmark(c: &mut Criterion) {
let graph = WebgraphBuilder::new(WEBGRAPH_PATH).open();
- let store = CentralityStore::build_harmonic(&graph, stract::gen_temp_path());
c.bench_function("Inbound similarity creation", |b| {
b.iter(|| {
- InboundSimilarity::build(&graph, &store.harmonic);
+ InboundSimilarity::build(&graph);
})
});
}
diff --git a/core/src/api/docs.rs b/core/src/api/docs.rs
index 1e9755f3..8512466e 100644
--- a/core/src/api/docs.rs
+++ b/core/src/api/docs.rs
@@ -83,7 +83,9 @@ impl Modify for ApiModifier {
fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
openapi.info.description = Some(
r#"Stract is an open source web search engine. The API is totally free while in beta, but some endpoints will most likely be paid by consumption in the future.
-The API might also change quite a bit during the beta period, but we will try to keep it as stable as possible. We look forward to see what you will build!"#.to_string(),
+The API might also change quite a bit during the beta period, but we will try to keep it as stable as possible. We look forward to see what you will build!
+
+Remember to always give proper attributions to the sources you use from the search results."#.to_string(),
);
}
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 31fa7979..61e58825 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -89,6 +89,8 @@ pub enum Signal {
Bm25SiteNoTokenizer,
#[serde(rename = "bm25_domain_no_tokenizer")]
Bm25DomainNoTokenizer,
+ #[serde(rename = "bm25_domain_name_no_tokenizer")]
+ Bm25DomainNameNoTokenizer,
#[serde(rename = "bm25_domain_if_homepage")]
Bm25DomainIfHomepage,
#[serde(rename = "bm25_domain_name_if_homepage_no_tokenizer")]
@@ -114,6 +116,7 @@ pub enum Signal {
/// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_2")]
ProximitySlop2,
+
/// All proximity signals are disabled for now.
/// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_4")]
@@ -158,7 +161,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 38] = [
+pub const ALL_SIGNALS: [Signal; 39] = [
Signal::Bm25,
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
@@ -174,6 +177,7 @@ pub const ALL_SIGNALS: [Signal; 38] = [
Signal::Bm25Domain,
Signal::Bm25SiteNoTokenizer,
Signal::Bm25DomainNoTokenizer,
+ Signal::Bm25DomainNameNoTokenizer,
Signal::Bm25DomainIfHomepage,
Signal::Bm25DomainNameIfHomepageNoTokenizer,
Signal::Bm25TitleIfHomepage,
@@ -281,6 +285,7 @@ impl Signal {
Signal::Bm25Domain => 0.00005,
Signal::Bm25SiteNoTokenizer => 0.00005,
Signal::Bm25DomainNoTokenizer => 0.00005,
+ Signal::Bm25DomainNameNoTokenizer => 0.00005,
Signal::Bm25DomainIfHomepage => 0.0002,
Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0002,
Signal::Bm25TitleIfHomepage => 0.00002,
@@ -450,6 +455,7 @@ impl Signal {
| Signal::Bm25Domain
| Signal::Bm25SiteNoTokenizer
| Signal::Bm25DomainNoTokenizer
+ | Signal::Bm25DomainNameNoTokenizer
| Signal::Bm25DomainIfHomepage
| Signal::Bm25DomainNameIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
@@ -572,6 +578,7 @@ impl Signal {
| Signal::Bm25Domain
| Signal::Bm25SiteNoTokenizer
| Signal::Bm25DomainNoTokenizer
+ | Signal::Bm25DomainNameNoTokenizer
| Signal::Bm25DomainIfHomepage
| Signal::Bm25DomainNameIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
@@ -642,6 +649,7 @@ impl Signal {
Signal::Bm25Domain => Some(TextField::Domain),
Signal::Bm25SiteNoTokenizer => Some(TextField::SiteNoTokenizer),
Signal::Bm25DomainNoTokenizer => Some(TextField::DomainNoTokenizer),
+ Signal::Bm25DomainNameNoTokenizer => Some(TextField::DomainNameNoTokenizer),
Signal::Bm25DomainIfHomepage => Some(TextField::DomainIfHomepage),
Signal::Bm25DomainNameIfHomepageNoTokenizer => {
Some(TextField::DomainNameIfHomepageNoTokenizer)
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 8de93764..8d335821 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -36,6 +36,7 @@ pub enum TextField {
SiteNoTokenizer,
SiteIfHomepageNoTokenizer,
DomainNoTokenizer,
+ DomainNameNoTokenizer,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
DomainIfHomepage,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
@@ -87,6 +88,7 @@ impl TextField {
TextField::SiteNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::SiteIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::DomainNoTokenizer => Tokenizer::Identity(Identity {}),
+ TextField::DomainNameNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::DomainIfHomepage => Tokenizer::default(),
TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::TitleIfHomepage => Tokenizer::default(),
@@ -126,6 +128,7 @@ impl TextField {
TextField::SiteNoTokenizer => false,
TextField::SiteIfHomepageNoTokenizer => false,
TextField::DomainNoTokenizer => false,
+ TextField::DomainNameNoTokenizer => false,
TextField::DomainIfHomepage => false,
TextField::DomainNameIfHomepageNoTokenizer => false,
TextField::TitleIfHomepage => false,
@@ -154,6 +157,7 @@ impl TextField {
TextField::SiteNoTokenizer => "site_no_tokenizer",
TextField::SiteIfHomepageNoTokenizer => "site_if_homepage_no_tokenizer",
TextField::DomainNoTokenizer => "domain_no_tokenizer",
+ TextField::DomainNameNoTokenizer => "domain_name_no_tokenizer",
TextField::BacklinkText => "backlink_text",
TextField::StemmedTitle => "stemmed_title",
TextField::StemmedCleanBody => "stemmed_body",
@@ -260,7 +264,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 57] = [
+pub static ALL_FIELDS: [Field; 58] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -273,6 +277,7 @@ pub static ALL_FIELDS: [Field; 57] = [
Field::Text(TextField::SiteNoTokenizer),
Field::Text(TextField::SiteIfHomepageNoTokenizer),
Field::Text(TextField::DomainNoTokenizer),
+ Field::Text(TextField::DomainNameNoTokenizer),
Field::Text(TextField::DomainIfHomepage),
Field::Text(TextField::DomainNameIfHomepageNoTokenizer),
Field::Text(TextField::TitleIfHomepage),
@@ -365,6 +370,9 @@ impl Field {
Field::Text(TextField::DomainNoTokenizer) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::DomainNameNoTokenizer) => {
+ IndexingOption::Text(self.default_text_options())
+ }
Field::Text(TextField::AllBody) => IndexingOption::Text(self.default_text_options()),
Field::Text(TextField::DomainIfHomepage) => {
IndexingOption::Text(self.default_text_options())
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index f03f16ad..e691186d 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -931,6 +931,17 @@ impl Html {
let description = self.pretokenize_description();
let microformats = self.pretokenize_microformats();
+ let domain_name = self
+ .url()
+ .root_domain()
+ .unwrap_or_default()
+ .find('.')
+ .map(|index| {
+ &domain.text[..ceil_char_boundary(&domain.text, index).min(domain.text.len())]
+ })
+ .unwrap_or_default()
+ .to_string();
+
let schemas: Vec<_> = self.schema_org();
let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
@@ -1109,19 +1120,23 @@ impl Html {
doc.add_text(tantivy_field, "");
}
}
+ Field::Text(TextField::DomainNameNoTokenizer) => {
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: domain_name.to_string(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain_name.len(),
+ position: 0,
+ text: domain_name.to_string(),
+ position_length: 1,
+ }],
+ },
+ );
+ }
Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
if self.is_homepage() {
- let domain_name = self
- .url()
- .root_domain()
- .unwrap_or_default()
- .find('.')
- .map(|index| {
- &domain.text[..ceil_char_boundary(&domain.text, index)
- .min(domain.text.len())]
- })
- .unwrap_or_default();
-
doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {