web search done right
group backlinks by the host centrality rank of the linking site so we can take this into account during ranking.
Mikkel Denkere71c9716c052
diff --git a/crates/core/src/backlink_grouper.rs b/crates/core/src/backlink_grouper.rsnew file mode 100644index 00000000..d8c6b634--- /dev/null+++ b/crates/core/src/backlink_grouper.rs@@ -0,0 +1,213 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{log_group::HarmonicRankGroup, webgraph::FullEdge};++/// Number of groups to divide the backlinks into.+/// If this is changed, also change the grouped backlink fields in the schema.+const NUM_GROUPS: u64 = 10;++#[derive(Debug)]+pub struct Group {+ group: u64,+ backlinks: Vec<FullEdge>,+}++impl Group {+ fn new(group: u64) -> Self {+ Self {+ group,+ backlinks: Vec::new(),+ }+ }++ fn insert(&mut self, backlink: FullEdge) {+ self.backlinks.push(backlink);+ }++ pub fn group(&self) -> u64 {+ self.group+ }++ pub fn backlinks(&self) -> &[FullEdge] {+ &self.backlinks+ }+}++#[derive(Debug)]+pub struct GroupedBacklinks {+ groups: Vec<Group>,+}++impl GroupedBacklinks {+ pub fn empty() -> Self {+ let groups = (0..NUM_GROUPS).map(Group::new).collect();+ Self { groups }+ }++ pub fn all(&self) -> &[Group] {+ &self.groups+ }++ pub fn get(&self, group: u64) -> Option<&Group> {+ self.groups.get(group as usize)+ }++ fn add(&mut self, group: u64, backlink: FullEdge) {+ if let Some(group) = self.groups.get_mut(group as usize) {+ group.insert(backlink)+ }+ }+}++impl Default for GroupedBacklinks {+ fn default() -> Self {+ Self::empty()+ }+}++/// Groups backlinks by their harmonic centrality rank.+pub struct BacklinkGrouper {+ grouper: HarmonicRankGroup,+ groups: GroupedBacklinks,+}++impl BacklinkGrouper {+ pub fn new(num_hosts: u64) -> Self {+ Self {+ grouper: HarmonicRankGroup::new(num_hosts, NUM_GROUPS),+ groups: GroupedBacklinks::empty(),+ }+ }++ pub fn add(&mut self, backlink: FullEdge, rank: u64) {+ let group = self.grouper.group(rank);+ self.groups.add(group, backlink);+ }++ pub fn groups(self) -> GroupedBacklinks {+ self.groups+ }+}++#[cfg(test)]+mod tests {+ use crate::webgraph::Node;++ use super::*;++ #[test]+ fn test_grouped_backlinks() {+ let mut grouper = BacklinkGrouper::new(10);+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 0,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 1,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 2,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 3,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 4,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 5,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 6,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 7,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 8,+ );+ grouper.add(+ FullEdge {+ from: Node::from("https://a.com"),+ to: Node::from("https://b.com"),+ label: String::new(),+ },+ 9,+ );++ {+ let groups = grouper.groups();+ assert_eq!(groups.all().len(), 10);+ assert_eq!(+ groups+ .all()+ .iter()+ .map(|g| g.backlinks().len())+ .sum::<usize>(),+ 10+ );++ for (i, group) in groups.all().iter().enumerate() {+ assert_eq!(group.group(), i as u64);+ }+ }+ }+}diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsindex c1455e68..1eb507b3 100644--- a/crates/core/src/entrypoint/indexer/job.rs+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -96,7 +96,7 @@ impl Job {has_page_centrality = true;}- if !webpage.backlinks.is_empty() {+ if !webpage.backlinks().is_empty() {has_backlinks = true;}trace!("inserting webpage: {:?}", webpage.html.url());diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex e01cece8..c4c4fa66 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -23,6 +23,7 @@ use tracing::debug;pub use super::indexable_webpage::IndexableWebpage;pub use super::job::{Job, JobSettings};+use crate::backlink_grouper::BacklinkGrouper;use crate::config::{IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LiveIndexConfig,WebgraphGranularity,@@ -176,15 +177,17 @@ impl IndexingWorker {{let config = Config::from(config);+ let host_centrality_rank_store = speedy_kv::Db::open_or_create(+ Path::new(&config.host_centrality_store_path).join("harmonic_rank"),+ )+ .unwrap();+Self {host_centrality_store: speedy_kv::Db::open_or_create(Path::new(&config.host_centrality_store_path).join("harmonic"),).unwrap(),- host_centrality_rank_store: speedy_kv::Db::open_or_create(- Path::new(&config.host_centrality_store_path).join("harmonic_rank"),- )- .unwrap(),+ host_centrality_rank_store,page_centrality_store: config.page_centrality_store_path.as_ref()@@ -386,7 +389,7 @@ impl IndexingWorker {}}- pub fn set_backlink_labels(&self, pages: &mut [Webpage]) {+ pub fn set_backlinks(&self, pages: &mut [Webpage]) {if let Some(graph) = self.page_webgraph() {let ids = pages.iter()@@ -396,7 +399,27 @@ impl IndexingWorker {let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS);for (page, backlinks) in pages.iter_mut().zip_eq(backlinks) {- page.backlinks = backlinks;+ page.set_backlinks(backlinks.clone());++ let mut grouper =+ BacklinkGrouper::new(self.host_centrality_rank_store.len() as u64);++ for backlink in backlinks {+ let node = backlink.from.clone().into_host().id();++ if Some(node) == page.node_id {+ continue;+ }++ let rank = self+ .host_centrality_rank_store+ .get(&node)+ .unwrap()+ .unwrap_or(u64::MAX);+ grouper.add(backlink, rank);+ }++ page.set_grouped_backlinks(grouper.groups())}}}@@ -436,40 +459,21 @@ impl IndexingWorker {let mut signal_computer = SignalComputer::new(None);for page in batch {- let mut prepared = match self.prepare(page) {+ let mut webpage = match self.prepare(page) {Ok(html) => html,Err(err) => {debug!("skipping webpage: {}", err);continue;}};- if let Err(e) = self.set_host_centrality(&mut prepared) {+ if let Err(e) = self.set_host_centrality(&mut webpage) {debug!("skipping webpage: {}", e);continue;}- self.set_page_centralities(&mut prepared);- self.set_keywords(&mut prepared);- self.set_safety_classification(&mut prepared);-- // make sure we remember to set everything- let mut webpage = Webpage {- html: prepared.html,- backlinks: prepared.backlinks,- page_centrality: prepared.page_centrality,- page_centrality_rank: prepared.page_centrality_rank,- host_centrality: prepared.host_centrality,- host_centrality_rank: prepared.host_centrality_rank,- fetch_time_ms: page.fetch_time_ms,- pre_computed_score: 0.0,- node_id: prepared.node_id,- dmoz_description: prepared.dmoz_description,- safety_classification: prepared.safety_classification,- inserted_at: Utc::now(),- keywords: prepared.keywords,- title_embedding: None, // set later- keyword_embedding: None, // set later- };+ self.set_page_centralities(&mut webpage);+ self.set_keywords(&mut webpage);+ self.set_safety_classification(&mut webpage);signal_computer.set_current_timestamp(Utc::now().timestamp().max(0) as usize);webpage.pre_computed_score = signal_computer.precompute_score(&webpage);@@ -479,7 +483,7 @@ impl IndexingWorker {self.set_title_embeddings(&mut res);self.set_keyword_embeddings(&mut res);- self.set_backlink_labels(&mut res);+ self.set_backlinks(&mut res);res}diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex 410d7cf5..60f2a7df 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -115,7 +115,7 @@ impl InvertedIndex {self.writer.as_ref().expect("writer has not been prepared")- .add_document(webpage.as_tantivy(&self.schema)?)?;+ .add_document(webpage.as_tantivy(self)?)?;Ok(())}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex e0df2cec..863ed9ac 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -198,6 +198,10 @@ impl InvertedIndex {Arc::clone(&self.schema)}+ pub fn schema_ref(&self) -> &Schema {+ &self.schema+ }+pub fn num_segments(&self) -> usize {self.tantivy_index.searchable_segments().unwrap().len()}@@ -485,11 +489,10 @@ mod tests {.unwrap(),).expect("failed to insert webpage");- index- .insert(&Webpage {- html: Html::parse(- &format!(- r#"+ let mut webpage = Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -499,20 +502,22 @@ mod tests {</body></html>"#- ),- "https://www.b.com",- )- .unwrap(),- backlinks: vec![FullEdge {- from: Node::from("https://www.a.com"),- to: Node::from("https://www.b.com"),- label: "B site is great".to_string(),- }],- host_centrality: 1.0,- fetch_time_ms: 500,- ..Default::default()- })- .expect("failed to insert webpage");+ ),+ "https://www.b.com",+ )+ .unwrap(),+ host_centrality: 1.0,+ fetch_time_ms: 500,+ ..Default::default()+ };++ webpage.set_backlinks(vec![FullEdge {+ from: Node::from("https://www.a.com"),+ to: Node::from("https://www.b.com"),+ label: "B site is great".to_string(),+ }]);++ index.insert(&webpage).expect("failed to insert webpage");index.commit().expect("failed to commit index");diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 610d2392..924eb167 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -43,6 +43,7 @@ pub mod ampc;mod api;pub mod autosuggest;+mod backlink_grouper;pub mod bangs;mod bincode_utils;pub mod canon_index;@@ -64,6 +65,7 @@ mod intmap;mod kahan_sum;mod leaky_queue;mod live_index;+mod log_group;mod metrics;mod models;pub mod naive_bayes;diff --git a/crates/core/src/log_group.rs b/crates/core/src/log_group.rsnew file mode 100644index 00000000..96e11b47--- /dev/null+++ b/crates/core/src/log_group.rs@@ -0,0 +1,93 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub struct LogGroup {+ base: f64,+ groups: u64,+}++impl LogGroup {+ pub fn new(num_items: u64, groups: u64) -> Self {+ let base = ((num_items as f64).ln() / groups as f64).exp();++ Self { base, groups }+ }++ pub fn group(&self, item: u64) -> u64 {+ (((item + 1) as f64).log(self.base) as u64).min(self.groups - 1)+ }++ pub fn num_groups(&self) -> u64 {+ self.groups+ }+}++pub struct HarmonicRankGroup {+ log_group: LogGroup,+}++impl HarmonicRankGroup {+ pub fn new(num_hosts: u64, groups: u64) -> Self {+ Self {+ log_group: LogGroup::new(num_hosts, groups),+ }+ }++ pub fn group(&self, rank: u64) -> u64 {+ self.log_group.num_groups() - self.log_group.group(rank) - 1+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_log_group() {+ let lg = LogGroup::new(100, 10);++ assert_eq!(lg.group(0), 0);+ assert_eq!(lg.group(1), 1);+ assert_eq!(lg.group(2), 2);+ assert_eq!(lg.group(10), 5);+ assert_eq!(lg.group(11), 5);+ assert_eq!(lg.group(90), 9);+ assert_eq!(lg.group(99), 9);++ // items over num_items are grouped together with the last group+ assert_eq!(lg.group(100), 9);+ assert_eq!(lg.group(1000), 9);++ let lg = LogGroup::new(40_000_000, 10);++ for i in 0..5 {+ assert_eq!(lg.group(i), 0, "i = {}", i);+ }+ }++ #[test]+ fn test_harmonic_rank_group() {+ let hrg = HarmonicRankGroup::new(100, 10);++ assert_eq!(hrg.group(0), 9);+ assert_eq!(hrg.group(1), 8);+ assert_eq!(hrg.group(10), 4);+ assert_eq!(hrg.group(11), 4);+ assert_eq!(hrg.group(90), 0);+ assert_eq!(hrg.group(99), 0);+ assert_eq!(hrg.group(100), 0);+ }+}diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex a7a30c0c..868deac3 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -660,11 +660,11 @@ mod tests {).unwrap();- webpage.backlinks = vec![FullEdge {+ webpage.set_backlinks(vec![FullEdge {from: Node::from("https://www.second.com/example/abc"),to: Node::from("https://www.first.com"),label: String::new(),- }];+ }]);index.insert(&webpage).expect("failed to insert webpage");let mut webpage = Webpage::test_parse(@@ -682,7 +682,7 @@ mod tests {"https://www.second.com/example/abc",).unwrap();- webpage.backlinks = vec![];+ webpage.set_backlinks(vec![]);index.insert(&webpage).expect("failed to insert webpage");index.commit().expect("failed to commit index");diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex b14f6840..26dc6c2b 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -250,7 +250,6 @@ mod tests {"https://www.b.com",).unwrap(),- backlinks: vec![],fetch_time_ms: 500,page_centrality: 5.0,..Default::default()@@ -415,10 +414,9 @@ mod tests {fn backlink_text() {let mut index = Index::temporary().expect("Unable to open index");- index- .insert(&Webpage {- html: Html::parse(- r#"+ let mut webpage = Webpage {+ html: Html::parse(+ r#"<html><head><title>Test site</title>@@ -428,18 +426,20 @@ mod tests {</body></html>"#,- "https://www.first.com",- )- .unwrap(),- backlinks: vec![FullEdge {- from: Node::from("https://www.backlink.com"),- to: Node::from("https://www.first.com"),- label: "test this is the best test site".to_string(),- }],- fetch_time_ms: 500,- ..Default::default()- })- .expect("failed to insert webpage");+ "https://www.first.com",+ )+ .unwrap(),+ fetch_time_ms: 500,+ ..Default::default()+ };++ webpage.set_backlinks(vec![FullEdge {+ from: Node::from("https://www.backlink.com"),+ to: Node::from("https://www.first.com"),+ label: "test this is the best test site".to_string(),+ }]);++ index.insert(&webpage).expect("failed to insert webpage");index.insert(&Webpage {html: Html::parse(diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 2775c39a..6f1494c5 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -162,7 +162,6 @@ mod tests {"https://www.third.com",).unwrap(),- backlinks: vec![],host_centrality: 1.0,node_id: Some(Node::from("https://www.third.com").into_host().id()),..Default::default()diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rsindex c7e62c30..b0987e6b 100644--- a/crates/core/src/schema/numerical_field.rs+++ b/crates/core/src/schema/numerical_field.rs@@ -52,14 +52,14 @@ pub trait NumericalField:html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()>;fn add_webpage_tantivy(&self,_webpage: &crate::webpage::Webpage,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -264,9 +264,9 @@ impl NumericalField for IsHomepage {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_bool(self.tantivy_field(schema), html.is_homepage());+ doc.add_bool(self.tantivy_field(index.schema_ref()), html.is_homepage());Ok(())}@@ -284,7 +284,7 @@ impl NumericalField for HostCentrality {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -297,9 +297,12 @@ impl NumericalField for HostCentrality {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_f64(self.tantivy_field(schema), webpage.host_centrality);+ doc.add_f64(+ self.tantivy_field(index.schema_ref()),+ webpage.host_centrality,+ );Ok(())}@@ -321,7 +324,7 @@ impl NumericalField for HostCentralityRank {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -330,9 +333,12 @@ impl NumericalField for HostCentralityRank {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), webpage.host_centrality_rank);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ webpage.host_centrality_rank,+ );Ok(())}@@ -353,7 +359,7 @@ impl NumericalField for PageCentrality {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -366,9 +372,12 @@ impl NumericalField for PageCentrality {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_f64(self.tantivy_field(schema), webpage.page_centrality);+ doc.add_f64(+ self.tantivy_field(index.schema_ref()),+ webpage.page_centrality,+ );Ok(())}@@ -389,7 +398,7 @@ impl NumericalField for PageCentralityRank {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -398,9 +407,12 @@ impl NumericalField for PageCentralityRank {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), webpage.page_centrality_rank);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ webpage.page_centrality_rank,+ );Ok(())}@@ -421,7 +433,7 @@ impl NumericalField for FetchTimeMs {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -430,9 +442,12 @@ impl NumericalField for FetchTimeMs {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), webpage.fetch_time_ms);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ webpage.fetch_time_ms,+ );Ok(())}@@ -458,10 +473,10 @@ impl NumericalField for LastUpdated {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),html.updated_time().map_or(0, |time| time.timestamp().max(0) as u64),);@@ -486,9 +501,12 @@ impl NumericalField for TrackerScore {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), html.trackers().len() as u64);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ html.trackers().len() as u64,+ );Ok(())}@@ -514,7 +532,7 @@ impl NumericalField for Region {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -523,14 +541,14 @@ impl NumericalField for Region {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let region = crate::webpage::region::Region::guess_from(webpage);if let Ok(region) = region {- doc.add_u64(self.tantivy_field(schema), region.id());+ doc.add_u64(self.tantivy_field(index.schema_ref()), region.id());} else {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),crate::webpage::region::Region::All.id(),);}@@ -555,10 +573,10 @@ impl NumericalField for NumUrlTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_url().tokens.len() as u64,);@@ -578,10 +596,10 @@ impl NumericalField for NumTitleTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_title().as_ref()@@ -605,10 +623,10 @@ impl NumericalField for NumCleanBodyTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_clean_text().tokens.len() as u64,);Ok(())@@ -627,10 +645,10 @@ impl NumericalField for NumDescriptionTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_description().tokens.len() as u64,);@@ -650,10 +668,10 @@ impl NumericalField for NumUrlForSiteOperatorTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_url_for_site_operator().tokens.len() as u64,);@@ -673,10 +691,10 @@ impl NumericalField for NumDomainTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_domain().tokens.len() as u64,);@@ -696,10 +714,10 @@ impl NumericalField for NumMicroformatTagsTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenize_microformats().tokens.len() as u64,);@@ -719,9 +737,9 @@ impl NumericalField for SiteHash1 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.site_hash()[0]);+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.site_hash()[0]);Ok(())}@@ -739,9 +757,9 @@ impl NumericalField for SiteHash2 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.site_hash()[1]);+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.site_hash()[1]);Ok(())}@@ -759,10 +777,10 @@ impl NumericalField for UrlWithoutQueryHash1 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.url_without_query_hash()[0],);@@ -782,10 +800,10 @@ impl NumericalField for UrlWithoutQueryHash2 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.url_without_query_hash()[1],);@@ -805,9 +823,12 @@ impl NumericalField for TitleHash1 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.title_hash()[0]);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ cache.title_hash()[0],+ );Ok(())}@@ -825,9 +846,12 @@ impl NumericalField for TitleHash2 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.title_hash()[1]);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ cache.title_hash()[1],+ );Ok(())}@@ -845,9 +869,9 @@ impl NumericalField for UrlHash1 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.url_hash()[0]);+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.url_hash()[0]);Ok(())}@@ -865,9 +889,9 @@ impl NumericalField for UrlHash2 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.url_hash()[1]);+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.url_hash()[1]);Ok(())}@@ -885,9 +909,12 @@ impl NumericalField for DomainHash1 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.domain_hash()[0]);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ cache.domain_hash()[0],+ );Ok(())}@@ -905,9 +932,12 @@ impl NumericalField for DomainHash2 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.domain_hash()[1]);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ cache.domain_hash()[1],+ );Ok(())}@@ -925,9 +955,12 @@ impl NumericalField for UrlWithoutTldHash1 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.url_without_tld_hash()[0]);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ cache.url_without_tld_hash()[0],+ );Ok(())}@@ -945,9 +978,12 @@ impl NumericalField for UrlWithoutTldHash2 {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_u64(self.tantivy_field(schema), cache.url_without_tld_hash()[1]);+ doc.add_u64(+ self.tantivy_field(index.schema_ref()),+ cache.url_without_tld_hash()[1],+ );Ok(())}@@ -969,7 +1005,7 @@ impl NumericalField for PreComputedScore {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -982,9 +1018,12 @@ impl NumericalField for PreComputedScore {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_f64(self.tantivy_field(schema), webpage.pre_computed_score);+ doc.add_f64(+ self.tantivy_field(index.schema_ref()),+ webpage.pre_computed_score,+ );Ok(())}@@ -1006,7 +1045,7 @@ impl NumericalField for HostNodeID {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1015,14 +1054,14 @@ impl NumericalField for HostNodeID {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {match &webpage.node_id {Some(node_id) => {- doc.add_u64(self.tantivy_field(schema), node_id.as_u64());+ doc.add_u64(self.tantivy_field(index.schema_ref()), node_id.as_u64());}None => {- doc.add_u64(self.tantivy_field(schema), u64::MAX);+ doc.add_u64(self.tantivy_field(index.schema_ref()), u64::MAX);}}@@ -1050,7 +1089,7 @@ impl NumericalField for SimHash {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let clean_text = cache.pretokenize_clean_text();@@ -1059,7 +1098,7 @@ impl NumericalField for SimHash {} else {0};- doc.add_u64(self.tantivy_field(schema), hash);+ doc.add_u64(self.tantivy_field(index.schema_ref()), hash);Ok(())}@@ -1077,10 +1116,10 @@ impl NumericalField for NumFlattenedSchemaTokens {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_u64(- self.tantivy_field(schema),+ self.tantivy_field(index.schema_ref()),cache.pretokenized_schema_json().tokens.len() as u64,);@@ -1104,7 +1143,7 @@ impl NumericalField for NumPathAndQuerySlashes {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let num_slashes = html.url()@@ -1112,7 +1151,7 @@ impl NumericalField for NumPathAndQuerySlashes {.map(|segments| segments.count()).unwrap_or(0);- doc.add_u64(self.tantivy_field(schema), num_slashes as u64);+ doc.add_u64(self.tantivy_field(index.schema_ref()), num_slashes as u64);Ok(())}@@ -1138,7 +1177,7 @@ impl NumericalField for NumPathAndQueryDigits {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let num_digits = html.url()@@ -1154,7 +1193,7 @@ impl NumericalField for NumPathAndQueryDigits {.filter(|c| c.is_ascii_digit()).count();- doc.add_u64(self.tantivy_field(schema), num_digits as u64);+ doc.add_u64(self.tantivy_field(index.schema_ref()), num_digits as u64);Ok(())}@@ -1184,9 +1223,12 @@ impl NumericalField for LikelyHasAds {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_bool(self.tantivy_field(schema), html.likely_has_ads());+ doc.add_bool(+ self.tantivy_field(index.schema_ref()),+ html.likely_has_ads(),+ );Ok(())}@@ -1216,9 +1258,12 @@ impl NumericalField for LikelyHasPaywall {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_bool(self.tantivy_field(schema), html.likely_has_paywall());+ doc.add_bool(+ self.tantivy_field(index.schema_ref()),+ html.likely_has_paywall(),+ );Ok(())}@@ -1248,9 +1293,9 @@ impl NumericalField for LinkDensity {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {- doc.add_f64(self.tantivy_field(schema), html.link_density());+ doc.add_f64(self.tantivy_field(index.schema_ref()), html.link_density());Ok(())}@@ -1280,7 +1325,7 @@ impl NumericalField for TitleEmbeddings {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1289,15 +1334,15 @@ impl NumericalField for TitleEmbeddings {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {if let Some(emb) = &webpage.title_embedding {let mut serialized = Vec::new();emb.write_bytes(&mut serialized)?;- doc.add_bytes(self.tantivy_field(schema), &serialized);+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &serialized);} else {- doc.add_bytes(self.tantivy_field(schema), &[]);+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &[]);}Ok(())@@ -1324,7 +1369,7 @@ impl NumericalField for KeywordEmbeddings {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1333,15 +1378,15 @@ impl NumericalField for KeywordEmbeddings {&self,webpage: &Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {if let Some(emb) = &webpage.keyword_embedding {let mut serialized = Vec::new();emb.write_bytes(&mut serialized)?;- doc.add_bytes(self.tantivy_field(schema), &serialized);+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &serialized);} else {- doc.add_bytes(self.tantivy_field(schema), &[]);+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &[]);}Ok(())diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 7177050b..b540ba51 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -54,7 +54,7 @@ pub trait TextField:html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()>;#[allow(unused_variables)]@@ -62,7 +62,7 @@ pub trait TextField:&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -200,6 +200,16 @@ pub enum TextFieldEnum {KeyPhrases,Links,Backlinks,+ BacklinkLabelsGroup0,+ BacklinkLabelsGroup1,+ BacklinkLabelsGroup2,+ BacklinkLabelsGroup3,+ BacklinkLabelsGroup4,+ BacklinkLabelsGroup5,+ BacklinkLabelsGroup6,+ BacklinkLabelsGroup7,+ BacklinkLabelsGroup8,+ BacklinkLabelsGroup9,}enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,@@ -239,6 +249,16 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,KeyPhrases,Links,Backlinks,+ BacklinkLabelsGroup0,+ BacklinkLabelsGroup1,+ BacklinkLabelsGroup2,+ BacklinkLabelsGroup3,+ BacklinkLabelsGroup4,+ BacklinkLabelsGroup5,+ BacklinkLabelsGroup6,+ BacklinkLabelsGroup7,+ BacklinkLabelsGroup8,+ BacklinkLabelsGroup9,]);impl TextFieldEnum {@@ -329,10 +349,10 @@ impl TextField for Title {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenize_title()@@ -365,10 +385,10 @@ impl TextField for CleanBody {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenize_clean_text().clone(),);@@ -400,7 +420,7 @@ impl TextField for StemmedTitle {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let title = cache.pretokenize_title()@@ -411,7 +431,7 @@ impl TextField for StemmedTitle {stem_tokens(&mut tokens, html.lang().copied().unwrap_or(Lang::Eng));doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: title.text.clone(),@@ -450,14 +470,14 @@ impl TextField for StemmedCleanBody {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let clean_text = cache.pretokenize_clean_text();let mut tokens = clean_text.tokens.clone();stem_tokens(&mut tokens, html.lang().copied().unwrap_or(Lang::Eng));doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: clean_text.text.clone(),@@ -489,7 +509,7 @@ impl TextField for AllBody {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let all_text = cache.pretokenize_all_text()@@ -498,7 +518,7 @@ impl TextField for AllBody {.map_err(|e| anyhow::anyhow!("{}", e))?;doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),all_text.clone(),);@@ -535,11 +555,11 @@ impl TextField for Url {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let url = cache.pretokenize_url();doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),url.clone(),);@@ -568,12 +588,12 @@ impl TextField for UrlNoTokenizer {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let url = html.url().to_string();doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: url.clone(),@@ -611,10 +631,10 @@ impl TextField for UrlForSiteOperator {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenize_url_for_site_operator().clone(),);@@ -639,10 +659,10 @@ impl TextField for SiteWithout {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenize_site().clone(),);@@ -666,10 +686,10 @@ impl TextField for Domain {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenize_domain().clone(),);@@ -698,12 +718,12 @@ impl TextField for SiteNoTokenizer {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let site = cache.pretokenize_site();doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: site.text.clone(),@@ -741,12 +761,12 @@ impl TextField for DomainNoTokenizer {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let domain = cache.pretokenize_domain();doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: domain.text.clone(),@@ -784,12 +804,12 @@ impl TextField for DomainNameNoTokenizer {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let domain_name = cache.domain_name();doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: domain_name.clone(),@@ -823,13 +843,13 @@ impl TextField for SiteIfHomepageNoTokenizer {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let site = cache.pretokenize_site();if html.is_homepage() {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: site.text.clone(),@@ -844,7 +864,7 @@ impl TextField for SiteIfHomepageNoTokenizer {);} else {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),"",);@@ -866,18 +886,18 @@ impl TextField for DomainIfHomepage {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let domain = cache.pretokenize_domain();if html.is_homepage() {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),domain.text.clone(),);} else {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),"",);@@ -903,13 +923,13 @@ impl TextField for DomainNameIfHomepageNoTokenizer {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let domain_name = cache.domain_name();if html.is_homepage() {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: domain_name.clone(),@@ -924,7 +944,7 @@ impl TextField for DomainNameIfHomepageNoTokenizer {);} else {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),"",);@@ -950,13 +970,13 @@ impl TextField for DomainIfHomepageNoTokenizer {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let domain = cache.pretokenize_domain();if html.is_homepage() {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),PreTokenizedString {text: domain.text.clone(),@@ -971,7 +991,7 @@ impl TextField for DomainIfHomepageNoTokenizer {);} else {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),"",);@@ -993,7 +1013,7 @@ impl TextField for TitleIfHomepage {html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let title = cache.pretokenize_title()@@ -1003,13 +1023,13 @@ impl TextField for TitleIfHomepage {if html.is_homepage() {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),title,);} else {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),"",);@@ -1031,7 +1051,7 @@ impl TextField for BacklinkText {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1040,12 +1060,16 @@ impl TextField for BacklinkText {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),- webpage.backlinks.iter().map(|e| e.label.clone()).join("\n"),+ webpage+ .backlinks()+ .iter()+ .map(|e| e.label.clone())+ .join("\n"),);Ok(())@@ -1076,11 +1100,11 @@ impl TextField for Description {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let description = cache.pretokenize_description();doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),description.clone(),);@@ -1109,7 +1133,7 @@ impl TextField for DmozDescription {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1118,10 +1142,10 @@ impl TextField for DmozDescription {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),webpage.dmoz_description().unwrap_or_default(),);@@ -1150,10 +1174,10 @@ impl TextField for SchemaOrgJson {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.schema_json(),);@@ -1182,10 +1206,10 @@ impl TextField for FlattenedSchemaOrgJson {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenized_schema_json().clone(),);@@ -1226,10 +1250,10 @@ impl TextField for CleanBodyBigrams {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),html.clean_text().cloned().unwrap_or_default(),);@@ -1270,7 +1294,7 @@ impl TextField for TitleBigrams {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let title = cache.pretokenize_title()@@ -1279,7 +1303,7 @@ impl TextField for TitleBigrams {.map_err(|e| anyhow::anyhow!("{}", e))?;doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),title.text.clone(),);@@ -1320,10 +1344,10 @@ impl TextField for CleanBodyTrigrams {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),html.clean_text().cloned().unwrap_or_default(),);@@ -1364,7 +1388,7 @@ impl TextField for TitleTrigrams {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let title = cache.pretokenize_title()@@ -1373,7 +1397,7 @@ impl TextField for TitleTrigrams {.map_err(|e| anyhow::anyhow!("{}", e))?;doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),title.text.clone(),);@@ -1398,10 +1422,10 @@ impl TextField for MicroformatTags {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_pre_tokenized_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.pretokenize_microformats().clone(),);@@ -1426,7 +1450,7 @@ impl TextField for SafetyClassification {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1435,7 +1459,7 @@ impl TextField for SafetyClassification {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {let safety = webpage.safety_classification@@ -1443,7 +1467,7 @@ impl TextField for SafetyClassification {.unwrap_or_default();doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),safety,);@@ -1472,7 +1496,7 @@ impl TextField for InsertionTimestamp {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1481,10 +1505,10 @@ impl TextField for InsertionTimestamp {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_date(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),tantivy::DateTime::from_utc(OffsetDateTime::from_unix_timestamp(webpage.inserted_at.timestamp(),@@ -1515,10 +1539,10 @@ impl TextField for RecipeFirstIngredientTagId {_html: &Html,cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),cache.first_ingredient_tag_id().cloned().unwrap_or_default(),);@@ -1547,10 +1571,10 @@ impl TextField for Links {html: &Html,_cache: &mut FnCache,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),html.anchor_links().into_iter()@@ -1581,13 +1605,13 @@ impl TextField for Backlinks {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),webpage- .backlinks+ .backlinks().iter().map(|l| l.from.as_str().to_string()).join("\n"),@@ -1601,7 +1625,7 @@ impl TextField for Backlinks {_: &Html,_: &mut FnCache,_: &mut TantivyDocument,- _: &tantivy::schema::Schema,+ _: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1623,7 +1647,7 @@ impl TextField for Keywords {_html: &Html,_cache: &mut FnCache,_doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ _index: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1632,10 +1656,10 @@ impl TextField for Keywords {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),webpage.keywords.join("\n"),);@@ -1659,10 +1683,10 @@ impl TextField for KeyPhrases {&self,webpage: &crate::webpage::Webpage,doc: &mut TantivyDocument,- schema: &tantivy::schema::Schema,+ index: &crate::inverted_index::InvertedIndex,) -> Result<()> {doc.add_text(- self.tantivy_field(schema)+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),webpage.keywords.join("\n"),);@@ -1675,7 +1699,7 @@ impl TextField for KeyPhrases {_: &Html,_: &mut FnCache,_: &mut TantivyDocument,- _: &tantivy::schema::Schema,+ _: &crate::inverted_index::InvertedIndex,) -> Result<()> {Ok(())}@@ -1684,3 +1708,58 @@ impl TextField for KeyPhrases {FieldTokenizer::Newline(NewlineTokenizer::default())}}++macro_rules! impl_backlink_group {+ ($name:ident, $index:expr) => {+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+ pub struct $name;+ impl TextField for $name {+ fn name(&self) -> &str {+ concat!("backlink_group_", $index)+ }++ fn add_html_tantivy(+ &self,+ _: &Html,+ _: &mut FnCache,+ _: &mut TantivyDocument,+ _: &crate::inverted_index::InvertedIndex,+ ) -> Result<()> {+ Ok(())+ }++ fn add_webpage_tantivy(+ &self,+ webpage: &crate::webpage::Webpage,+ doc: &mut TantivyDocument,+ index: &crate::inverted_index::InvertedIndex,+ ) -> Result<()> {+ let backlinks = webpage+ .grouped_backlinks()+ .get($index)+ .map(|group| group.backlinks().iter().map(|l| &l.label).join("\n"))+ .unwrap_or_default();++ doc.add_text(+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| {+ panic!("could not find field '{}' in index", self.name())+ }),+ backlinks,+ );++ Ok(())+ }+ }+ };+}++impl_backlink_group!(BacklinkLabelsGroup0, 0);+impl_backlink_group!(BacklinkLabelsGroup1, 1);+impl_backlink_group!(BacklinkLabelsGroup2, 2);+impl_backlink_group!(BacklinkLabelsGroup3, 3);+impl_backlink_group!(BacklinkLabelsGroup4, 4);+impl_backlink_group!(BacklinkLabelsGroup5, 5);+impl_backlink_group!(BacklinkLabelsGroup6, 6);+impl_backlink_group!(BacklinkLabelsGroup7, 7);+impl_backlink_group!(BacklinkLabelsGroup8, 8);+impl_backlink_group!(BacklinkLabelsGroup9, 9);diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex f3a7100b..4275df32 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -193,17 +193,21 @@ impl Html {split_u128(hash(self.title().unwrap_or_default()).0)}- pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {+ pub fn as_tantivy(+ &self,+ index: &crate::inverted_index::InvertedIndex,+ ) -> Result<TantivyDocument> {let mut doc = TantivyDocument::new();let mut cache = FnCache::new(self);- for field in schema+ for field in index+ .schema_ref().fields().filter_map(|(field, _)| Field::get(field.field_id() as usize)){match field {- Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,- Field::Numerical(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,+ Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, index)?,+ Field::Numerical(f) => f.add_html_tantivy(self, &mut cache, &mut doc, index)?,}}diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex a7bf0282..1759b646 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -489,7 +489,7 @@ mod tests {use tests::links::RelFlags;- use crate::{schema::create_schema, webpage::Link};+ use crate::webpage::Link;use super::*;@@ -734,8 +734,9 @@ mod tests {assert!(webpage.all_text().is_some());assert!(!webpage.all_text().unwrap().is_empty());- let schema = create_schema();- webpage.as_tantivy(&schema).unwrap();+ let index =+ crate::inverted_index::InvertedIndex::temporary().expect("Unable to open index");+ webpage.as_tantivy(&index).unwrap();}#[test]diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex b6c58712..acf807c6 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -15,6 +15,8 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ backlink_grouper::GroupedBacklinks,+ inverted_index::InvertedIndex,schema::{numerical_field::NumericalField, text_field::TextField, Field},webgraph::{FullEdge, NodeID},Result,@@ -41,7 +43,15 @@ pub use region::Region;#[derive(Debug)]pub struct Webpage {pub html: Html,++ #[cfg(test)]pub backlinks: Vec<FullEdge>,++ #[cfg(not(test))]+ backlinks: Vec<FullEdge>,++ pub grouped_backlinks: GroupedBacklinks,+pub host_centrality: f64,pub host_centrality_rank: u64,pub page_centrality: f64,@@ -63,6 +73,7 @@ impl Default for Webpage {Self {html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),backlinks: Default::default(),+ grouped_backlinks: GroupedBacklinks::empty(),host_centrality: Default::default(),host_centrality_rank: u64::MAX,page_centrality: Default::default(),@@ -84,6 +95,7 @@ impl From<Html> for Webpage {fn from(html: Html) -> Self {Self {html,+ grouped_backlinks: GroupedBacklinks::empty(),backlinks: Default::default(),host_centrality: Default::default(),host_centrality_rank: u64::MAX,@@ -129,13 +141,29 @@ impl Webpage {})}- pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {- let mut doc = self.html.as_tantivy(schema)?;+ pub fn backlinks(&self) -> &[FullEdge] {+ &self.backlinks+ }++ pub fn set_backlinks(&mut self, backlinks: Vec<FullEdge>) {+ self.backlinks = backlinks;+ }++ pub fn set_grouped_backlinks(&mut self, grouped_backlinks: GroupedBacklinks) {+ self.grouped_backlinks = grouped_backlinks;+ }++ pub fn grouped_backlinks(&self) -> &GroupedBacklinks {+ &self.grouped_backlinks+ }++ pub fn as_tantivy(&self, index: &InvertedIndex) -> Result<TantivyDocument> {+ let mut doc = self.html.as_tantivy(index)?;for field in Field::all() {match field {- Field::Numerical(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,- Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,+ Field::Numerical(f) => f.add_webpage_tantivy(self, &mut doc, index)?,+ Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, index)?,}}