web search done right
'Bm25DomainIfHomepageNoTokenizer' field and signal. better navigational search for 'terms example.com' queries
Mikkel Denkerec053134291b
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rsindex 41511229..0c7a900c 100644--- a/core/src/alice/raw_model.rs+++ b/core/src/alice/raw_model.rs@@ -266,7 +266,7 @@ struct TimeMix {block_idx: i64,}impl TimeMix {- fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {+ fn forward(&self, x: &Tensor, state: &Tensor) -> Tensor {let x = x.squeeze();let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_k);let xv = &x * &self.time_mix_v + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_v);@@ -345,7 +345,7 @@ struct ChannelMix {block_idx: i64,}impl ChannelMix {- fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {+ fn forward(&self, x: &Tensor, state: &Tensor) -> Tensor {let x = x.squeeze();let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx) * (1 - &self.time_mix_k);let xr = &x * &self.time_mix_r + state.i(5 * self.block_idx) * (1 - &self.time_mix_r);@@ -473,11 +473,11 @@ impl Block {x += self.att- .forward(&self.ln1.forward_t(&x, false), &mut state)+ .forward(&self.ln1.forward_t(&x, false), &state).reshape([1, -1]);x += self.ffn- .forward(&self.ln2.forward_t(&x, false), &mut state)+ .forward(&self.ln2.forward_t(&x, false), &state).reshape([1, -1]);(x, state)diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 146d8e51..8474f9d7 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -88,7 +88,7 @@ impl CentralityStore {fn store_harmonic<P: AsRef<Path>>(output_path: P,- store: &mut CentralityStore,+ store: &CentralityStore,harmonic_centrality: HarmonicCentrality,graph: &Webgraph,) {@@ -127,10 +127,10 @@ impl CentralityStore {}pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {- let mut store = CentralityStore::open(output_path.as_ref());+ let store = CentralityStore::open(output_path.as_ref());let harmonic_centrality = HarmonicCentrality::calculate(graph);- Self::store_harmonic(&output_path, &mut store, harmonic_centrality, graph);+ Self::store_harmonic(&output_path, &store, harmonic_centrality, graph);store.flush();storediff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex d5465f95..263583a2 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -129,10 +129,7 @@ impl InboundSimilarity {let adjacency: DashMap<NodeID, IntSet<NodeID>> = DashMap::new();graph.par_edges().for_each(|edge| {- adjacency- .entry(edge.to)- .or_insert_with(IntSet::new)- .insert(edge.from);+ adjacency.entry(edge.to).or_default().insert(edge.from);});for (node_id, inbound) in adjacency {diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 94b5fa73..3c757982 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -95,6 +95,8 @@ pub enum Signal {Bm25DomainIfHomepage,#[serde(rename = "bm25_domain_name_if_homepage_no_tokenizer")]Bm25DomainNameIfHomepageNoTokenizer,+ #[serde(rename = "bm25_domain_if_homepage_no_tokenizer")]+ Bm25DomainIfHomepageNoTokenizer,#[serde(rename = "bm25_title_if_homepage")]Bm25TitleIfHomepage,#[serde(rename = "bm25_backlink_text")]@@ -161,7 +163,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 39] = [+pub const ALL_SIGNALS: [Signal; 40] = [Signal::Bm25,Signal::Bm25Title,Signal::Bm25TitleBigrams,@@ -180,6 +182,7 @@ pub const ALL_SIGNALS: [Signal; 39] = [Signal::Bm25DomainNameNoTokenizer,Signal::Bm25DomainIfHomepage,Signal::Bm25DomainNameIfHomepageNoTokenizer,+ Signal::Bm25DomainIfHomepageNoTokenizer,Signal::Bm25TitleIfHomepage,Signal::Bm25BacklinkText,Signal::Bm25Description,@@ -288,6 +291,7 @@ impl Signal {Signal::Bm25DomainNameNoTokenizer => 0.00005,Signal::Bm25DomainIfHomepage => 0.0002,Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.007,+ Signal::Bm25DomainIfHomepageNoTokenizer => 0.007,Signal::Bm25TitleIfHomepage => 0.00002,Signal::Bm25BacklinkText => 0.003,Signal::Bm25Description => 0.00001,@@ -458,6 +462,7 @@ impl Signal {| Signal::Bm25DomainNameNoTokenizer| Signal::Bm25DomainIfHomepage| Signal::Bm25DomainNameIfHomepageNoTokenizer+ | Signal::Bm25DomainIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage| Signal::Bm25BacklinkText| Signal::Bm25Description => signal_aggregator.segment_reader.as_mut().map(|reader| {@@ -581,6 +586,7 @@ impl Signal {| Signal::Bm25DomainNameNoTokenizer| Signal::Bm25DomainIfHomepage| Signal::Bm25DomainNameIfHomepageNoTokenizer+ | Signal::Bm25DomainIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage| Signal::Bm25BacklinkText| Signal::Bm25Description@@ -657,6 +663,7 @@ impl Signal {Signal::Bm25TitleIfHomepage => Some(TextField::TitleIfHomepage),Signal::Bm25BacklinkText => Some(TextField::BacklinkText),Signal::Bm25Description => Some(TextField::Description),+ Signal::Bm25DomainIfHomepageNoTokenizer => Some(TextField::DomainIfHomepageNoTokenizer),_ => None,}}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 8d335821..7dd2b511 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -34,14 +34,17 @@ pub enum TextField {SiteWithout,Domain,SiteNoTokenizer,- SiteIfHomepageNoTokenizer,DomainNoTokenizer,DomainNameNoTokenizer,/// this field is only set if the webpage is the homepage for the site. Allows us to boost+ SiteIfHomepageNoTokenizer,+ /// this field is only set if the webpage is the homepage for the site. Allows us to boostDomainIfHomepage,/// this field is only set if the webpage is the homepage for the site. Allows us to boostDomainNameIfHomepageNoTokenizer,/// this field is only set if the webpage is the homepage for the site. Allows us to boost+ DomainIfHomepageNoTokenizer,+ /// this field is only set if the webpage is the homepage for the site. Allows us to boostTitleIfHomepage,BacklinkText,Description,@@ -91,6 +94,7 @@ impl TextField {TextField::DomainNameNoTokenizer => Tokenizer::Identity(Identity {}),TextField::DomainIfHomepage => Tokenizer::default(),TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),+ TextField::DomainIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),TextField::TitleIfHomepage => Tokenizer::default(),TextField::BacklinkText => Tokenizer::default(),TextField::Description => Tokenizer::default(),@@ -131,6 +135,7 @@ impl TextField {TextField::DomainNameNoTokenizer => false,TextField::DomainIfHomepage => false,TextField::DomainNameIfHomepageNoTokenizer => false,+ TextField::DomainIfHomepageNoTokenizer => false,TextField::TitleIfHomepage => false,TextField::BacklinkText => false,TextField::Description => true,@@ -163,6 +168,7 @@ impl TextField {TextField::StemmedCleanBody => "stemmed_body",TextField::DomainIfHomepage => "domain_if_homepage",TextField::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",+ TextField::DomainIfHomepageNoTokenizer => "domain_if_homepage_no_tokenizer",TextField::Description => "description",TextField::TitleIfHomepage => "title_if_homepage",TextField::AllBody => "all_body",@@ -264,7 +270,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 58] = [+pub static ALL_FIELDS: [Field; 59] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -280,6 +286,7 @@ pub static ALL_FIELDS: [Field; 58] = [Field::Text(TextField::DomainNameNoTokenizer),Field::Text(TextField::DomainIfHomepage),Field::Text(TextField::DomainNameIfHomepageNoTokenizer),+ Field::Text(TextField::DomainIfHomepageNoTokenizer),Field::Text(TextField::TitleIfHomepage),Field::Text(TextField::BacklinkText),Field::Text(TextField::Description),@@ -383,6 +390,9 @@ impl Field {Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::DomainIfHomepageNoTokenizer) => {+ IndexingOption::Text(self.default_text_options())+ }Field::Text(TextField::BacklinkText) => {IndexingOption::Text(self.default_text_options())}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex f63c3c91..43806816 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -154,7 +154,7 @@ impl StoredSegment {})}- fn merge_adjacency<F1>(edges_fn: F1, packet: &mut MergePacket, segments: &[Self])+ fn merge_adjacency<F1>(edges_fn: F1, packet: &MergePacket, segments: &[Self])whereF1: Fn(&Self, &NodeID) -> Option<HashSet<FullStoredEdge>>,{@@ -201,14 +201,14 @@ impl StoredSegment {let new_full_adjacency = Store::open(new_path.join(FULL_ADJACENCY_STORE));let new_small_adjacency = Store::open(new_path.join(SMALL_ADJACENCY_STORE));- let mut packet = MergePacket {+ let packet = MergePacket {new_full_adjacency,new_small_adjacency,};Self::merge_adjacency(|segment: &StoredSegment, node_id: &NodeID| segment.full_adjacency.get(node_id),- &mut packet,+ &packet,&segments,);let new_adjacency = packet.new_full_adjacency;@@ -218,7 +218,7 @@ impl StoredSegment {let new_small_reversed_adjacency =Store::open(new_path.join(SMALL_REVERSED_ADJACENCY_STORE));- let mut packet = MergePacket {+ let packet = MergePacket {new_full_adjacency: new_full_reversed_adjacency,new_small_adjacency: new_small_reversed_adjacency,};@@ -227,7 +227,7 @@ impl StoredSegment {|segment: &StoredSegment, node_id: &NodeID| {segment.full_reversed_adjacency.get(node_id)},- &mut packet,+ &packet,&segments,);diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 46531e8b..2da5f894 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -1154,6 +1154,13 @@ impl Html {doc.add_text(tantivy_field, "");}}+ Field::Text(TextField::DomainIfHomepageNoTokenizer) => {+ if self.is_homepage() {+ doc.add_pre_tokenized_text(tantivy_field, domain.clone());+ } else {+ doc.add_text(tantivy_field, "");+ }+ }Field::Text(TextField::AllBody) => {doc.add_pre_tokenized_text(tantivy_field, all_text.clone())}