Skip to content

web search done right

'Bm25DomainIfHomepageNoTokenizer' field and signal. better navigational search for 'terms example.com' queries

Mikkel Denkerec053134291b
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rs
index 41511229..0c7a900c 100644
--- a/core/src/alice/raw_model.rs
+++ b/core/src/alice/raw_model.rs
@@ -266,7 +266,7 @@ struct TimeMix {
block_idx: i64,
}
impl TimeMix {
- fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {
+ fn forward(&self, x: &Tensor, state: &Tensor) -> Tensor {
let x = x.squeeze();
let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_k);
let xv = &x * &self.time_mix_v + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_v);
@@ -345,7 +345,7 @@ struct ChannelMix {
block_idx: i64,
}
impl ChannelMix {
- fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {
+ fn forward(&self, x: &Tensor, state: &Tensor) -> Tensor {
let x = x.squeeze();
let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx) * (1 - &self.time_mix_k);
let xr = &x * &self.time_mix_r + state.i(5 * self.block_idx) * (1 - &self.time_mix_r);
@@ -473,11 +473,11 @@ impl Block {
x += self
.att
- .forward(&self.ln1.forward_t(&x, false), &mut state)
+ .forward(&self.ln1.forward_t(&x, false), &state)
.reshape([1, -1]);
x += self
.ffn
- .forward(&self.ln2.forward_t(&x, false), &mut state)
+ .forward(&self.ln2.forward_t(&x, false), &state)
.reshape([1, -1]);
(x, state)
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 146d8e51..8474f9d7 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -88,7 +88,7 @@ impl CentralityStore {
fn store_harmonic<P: AsRef<Path>>(
output_path: P,
- store: &mut CentralityStore,
+ store: &CentralityStore,
harmonic_centrality: HarmonicCentrality,
graph: &Webgraph,
) {
@@ -127,10 +127,10 @@ impl CentralityStore {
}
pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
- let mut store = CentralityStore::open(output_path.as_ref());
+ let store = CentralityStore::open(output_path.as_ref());
let harmonic_centrality = HarmonicCentrality::calculate(graph);
- Self::store_harmonic(&output_path, &mut store, harmonic_centrality, graph);
+ Self::store_harmonic(&output_path, &store, harmonic_centrality, graph);
store.flush();
store
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index d5465f95..263583a2 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -129,10 +129,7 @@ impl InboundSimilarity {
let adjacency: DashMap<NodeID, IntSet<NodeID>> = DashMap::new();
graph.par_edges().for_each(|edge| {
- adjacency
- .entry(edge.to)
- .or_insert_with(IntSet::new)
- .insert(edge.from);
+ adjacency.entry(edge.to).or_default().insert(edge.from);
});
for (node_id, inbound) in adjacency {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 94b5fa73..3c757982 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -95,6 +95,8 @@ pub enum Signal {
Bm25DomainIfHomepage,
#[serde(rename = "bm25_domain_name_if_homepage_no_tokenizer")]
Bm25DomainNameIfHomepageNoTokenizer,
+ #[serde(rename = "bm25_domain_if_homepage_no_tokenizer")]
+ Bm25DomainIfHomepageNoTokenizer,
#[serde(rename = "bm25_title_if_homepage")]
Bm25TitleIfHomepage,
#[serde(rename = "bm25_backlink_text")]
@@ -161,7 +163,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 39] = [
+pub const ALL_SIGNALS: [Signal; 40] = [
Signal::Bm25,
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
@@ -180,6 +182,7 @@ pub const ALL_SIGNALS: [Signal; 39] = [
Signal::Bm25DomainNameNoTokenizer,
Signal::Bm25DomainIfHomepage,
Signal::Bm25DomainNameIfHomepageNoTokenizer,
+ Signal::Bm25DomainIfHomepageNoTokenizer,
Signal::Bm25TitleIfHomepage,
Signal::Bm25BacklinkText,
Signal::Bm25Description,
@@ -288,6 +291,7 @@ impl Signal {
Signal::Bm25DomainNameNoTokenizer => 0.00005,
Signal::Bm25DomainIfHomepage => 0.0002,
Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.007,
+ Signal::Bm25DomainIfHomepageNoTokenizer => 0.007,
Signal::Bm25TitleIfHomepage => 0.00002,
Signal::Bm25BacklinkText => 0.003,
Signal::Bm25Description => 0.00001,
@@ -458,6 +462,7 @@ impl Signal {
| Signal::Bm25DomainNameNoTokenizer
| Signal::Bm25DomainIfHomepage
| Signal::Bm25DomainNameIfHomepageNoTokenizer
+ | Signal::Bm25DomainIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
| Signal::Bm25BacklinkText
| Signal::Bm25Description => signal_aggregator.segment_reader.as_mut().map(|reader| {
@@ -581,6 +586,7 @@ impl Signal {
| Signal::Bm25DomainNameNoTokenizer
| Signal::Bm25DomainIfHomepage
| Signal::Bm25DomainNameIfHomepageNoTokenizer
+ | Signal::Bm25DomainIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
| Signal::Bm25BacklinkText
| Signal::Bm25Description
@@ -657,6 +663,7 @@ impl Signal {
Signal::Bm25TitleIfHomepage => Some(TextField::TitleIfHomepage),
Signal::Bm25BacklinkText => Some(TextField::BacklinkText),
Signal::Bm25Description => Some(TextField::Description),
+ Signal::Bm25DomainIfHomepageNoTokenizer => Some(TextField::DomainIfHomepageNoTokenizer),
_ => None,
}
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 8d335821..7dd2b511 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -34,14 +34,17 @@ pub enum TextField {
SiteWithout,
Domain,
SiteNoTokenizer,
- SiteIfHomepageNoTokenizer,
DomainNoTokenizer,
DomainNameNoTokenizer,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ SiteIfHomepageNoTokenizer,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
DomainIfHomepage,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
DomainNameIfHomepageNoTokenizer,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ DomainIfHomepageNoTokenizer,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
TitleIfHomepage,
BacklinkText,
Description,
@@ -91,6 +94,7 @@ impl TextField {
TextField::DomainNameNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::DomainIfHomepage => Tokenizer::default(),
TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
+ TextField::DomainIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::TitleIfHomepage => Tokenizer::default(),
TextField::BacklinkText => Tokenizer::default(),
TextField::Description => Tokenizer::default(),
@@ -131,6 +135,7 @@ impl TextField {
TextField::DomainNameNoTokenizer => false,
TextField::DomainIfHomepage => false,
TextField::DomainNameIfHomepageNoTokenizer => false,
+ TextField::DomainIfHomepageNoTokenizer => false,
TextField::TitleIfHomepage => false,
TextField::BacklinkText => false,
TextField::Description => true,
@@ -163,6 +168,7 @@ impl TextField {
TextField::StemmedCleanBody => "stemmed_body",
TextField::DomainIfHomepage => "domain_if_homepage",
TextField::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",
+ TextField::DomainIfHomepageNoTokenizer => "domain_if_homepage_no_tokenizer",
TextField::Description => "description",
TextField::TitleIfHomepage => "title_if_homepage",
TextField::AllBody => "all_body",
@@ -264,7 +270,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 58] = [
+pub static ALL_FIELDS: [Field; 59] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -280,6 +286,7 @@ pub static ALL_FIELDS: [Field; 58] = [
Field::Text(TextField::DomainNameNoTokenizer),
Field::Text(TextField::DomainIfHomepage),
Field::Text(TextField::DomainNameIfHomepageNoTokenizer),
+ Field::Text(TextField::DomainIfHomepageNoTokenizer),
Field::Text(TextField::TitleIfHomepage),
Field::Text(TextField::BacklinkText),
Field::Text(TextField::Description),
@@ -383,6 +390,9 @@ impl Field {
Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
+ IndexingOption::Text(self.default_text_options())
+ }
Field::Text(TextField::BacklinkText) => {
IndexingOption::Text(self.default_text_options())
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index f63c3c91..43806816 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -154,7 +154,7 @@ impl StoredSegment {
})
}
- fn merge_adjacency<F1>(edges_fn: F1, packet: &mut MergePacket, segments: &[Self])
+ fn merge_adjacency<F1>(edges_fn: F1, packet: &MergePacket, segments: &[Self])
where
F1: Fn(&Self, &NodeID) -> Option<HashSet<FullStoredEdge>>,
{
@@ -201,14 +201,14 @@ impl StoredSegment {
let new_full_adjacency = Store::open(new_path.join(FULL_ADJACENCY_STORE));
let new_small_adjacency = Store::open(new_path.join(SMALL_ADJACENCY_STORE));
- let mut packet = MergePacket {
+ let packet = MergePacket {
new_full_adjacency,
new_small_adjacency,
};
Self::merge_adjacency(
|segment: &StoredSegment, node_id: &NodeID| segment.full_adjacency.get(node_id),
- &mut packet,
+ &packet,
&segments,
);
let new_adjacency = packet.new_full_adjacency;
@@ -218,7 +218,7 @@ impl StoredSegment {
let new_small_reversed_adjacency =
Store::open(new_path.join(SMALL_REVERSED_ADJACENCY_STORE));
- let mut packet = MergePacket {
+ let packet = MergePacket {
new_full_adjacency: new_full_reversed_adjacency,
new_small_adjacency: new_small_reversed_adjacency,
};
@@ -227,7 +227,7 @@ impl StoredSegment {
|segment: &StoredSegment, node_id: &NodeID| {
segment.full_reversed_adjacency.get(node_id)
},
- &mut packet,
+ &packet,
&segments,
);
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 46531e8b..2da5f894 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -1154,6 +1154,13 @@ impl Html {
doc.add_text(tantivy_field, "");
}
}
+ Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
+ if self.is_homepage() {
+ doc.add_pre_tokenized_text(tantivy_field, domain.clone());
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
Field::Text(TextField::AllBody) => {
doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
}