Skip to content

web search done right

group backlinks by the host centrality rank of the linking site so we can take this into account during ranking.

Mikkel Denkere71c9716c052
diff --git a/crates/core/src/backlink_grouper.rs b/crates/core/src/backlink_grouper.rs
new file mode 100644
index 00000000..d8c6b634
--- /dev/null
+++ b/crates/core/src/backlink_grouper.rs
@@ -0,0 +1,213 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{log_group::HarmonicRankGroup, webgraph::FullEdge};
+
+/// Number of groups to divide the backlinks into.
+/// If this is changed, also change the grouped backlink fields in the schema.
+const NUM_GROUPS: u64 = 10;
+
+#[derive(Debug)]
+pub struct Group {
+ group: u64,
+ backlinks: Vec<FullEdge>,
+}
+
+impl Group {
+ fn new(group: u64) -> Self {
+ Self {
+ group,
+ backlinks: Vec::new(),
+ }
+ }
+
+ fn insert(&mut self, backlink: FullEdge) {
+ self.backlinks.push(backlink);
+ }
+
+ pub fn group(&self) -> u64 {
+ self.group
+ }
+
+ pub fn backlinks(&self) -> &[FullEdge] {
+ &self.backlinks
+ }
+}
+
+#[derive(Debug)]
+pub struct GroupedBacklinks {
+ groups: Vec<Group>,
+}
+
+impl GroupedBacklinks {
+ pub fn empty() -> Self {
+ let groups = (0..NUM_GROUPS).map(Group::new).collect();
+ Self { groups }
+ }
+
+ pub fn all(&self) -> &[Group] {
+ &self.groups
+ }
+
+ pub fn get(&self, group: u64) -> Option<&Group> {
+ self.groups.get(group as usize)
+ }
+
+ fn add(&mut self, group: u64, backlink: FullEdge) {
+ if let Some(group) = self.groups.get_mut(group as usize) {
+ group.insert(backlink)
+ }
+ }
+}
+
+impl Default for GroupedBacklinks {
+ fn default() -> Self {
+ Self::empty()
+ }
+}
+
+/// Groups backlinks by their harmonic centrality rank.
+pub struct BacklinkGrouper {
+ grouper: HarmonicRankGroup,
+ groups: GroupedBacklinks,
+}
+
+impl BacklinkGrouper {
+ pub fn new(num_hosts: u64) -> Self {
+ Self {
+ grouper: HarmonicRankGroup::new(num_hosts, NUM_GROUPS),
+ groups: GroupedBacklinks::empty(),
+ }
+ }
+
+ pub fn add(&mut self, backlink: FullEdge, rank: u64) {
+ let group = self.grouper.group(rank);
+ self.groups.add(group, backlink);
+ }
+
+ pub fn groups(self) -> GroupedBacklinks {
+ self.groups
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::webgraph::Node;
+
+ use super::*;
+
+ #[test]
+ fn test_grouped_backlinks() {
+ let mut grouper = BacklinkGrouper::new(10);
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 0,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 1,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 2,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 3,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 4,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 5,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 6,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 7,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 8,
+ );
+ grouper.add(
+ FullEdge {
+ from: Node::from("https://a.com"),
+ to: Node::from("https://b.com"),
+ label: String::new(),
+ },
+ 9,
+ );
+
+ {
+ let groups = grouper.groups();
+ assert_eq!(groups.all().len(), 10);
+ assert_eq!(
+ groups
+ .all()
+ .iter()
+ .map(|g| g.backlinks().len())
+ .sum::<usize>(),
+ 10
+ );
+
+ for (i, group) in groups.all().iter().enumerate() {
+ assert_eq!(group.group(), i as u64);
+ }
+ }
+ }
+}
diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rs
index c1455e68..1eb507b3 100644
--- a/crates/core/src/entrypoint/indexer/job.rs
+++ b/crates/core/src/entrypoint/indexer/job.rs
@@ -96,7 +96,7 @@ impl Job {
has_page_centrality = true;
}
- if !webpage.backlinks.is_empty() {
+ if !webpage.backlinks().is_empty() {
has_backlinks = true;
}
trace!("inserting webpage: {:?}", webpage.html.url());
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index e01cece8..c4c4fa66 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -23,6 +23,7 @@ use tracing::debug;
pub use super::indexable_webpage::IndexableWebpage;
pub use super::job::{Job, JobSettings};
+use crate::backlink_grouper::BacklinkGrouper;
use crate::config::{
IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LiveIndexConfig,
WebgraphGranularity,
@@ -176,15 +177,17 @@ impl IndexingWorker {
{
let config = Config::from(config);
+ let host_centrality_rank_store = speedy_kv::Db::open_or_create(
+ Path::new(&config.host_centrality_store_path).join("harmonic_rank"),
+ )
+ .unwrap();
+
Self {
host_centrality_store: speedy_kv::Db::open_or_create(
Path::new(&config.host_centrality_store_path).join("harmonic"),
)
.unwrap(),
- host_centrality_rank_store: speedy_kv::Db::open_or_create(
- Path::new(&config.host_centrality_store_path).join("harmonic_rank"),
- )
- .unwrap(),
+ host_centrality_rank_store,
page_centrality_store: config
.page_centrality_store_path
.as_ref()
@@ -386,7 +389,7 @@ impl IndexingWorker {
}
}
- pub fn set_backlink_labels(&self, pages: &mut [Webpage]) {
+ pub fn set_backlinks(&self, pages: &mut [Webpage]) {
if let Some(graph) = self.page_webgraph() {
let ids = pages
.iter()
@@ -396,7 +399,27 @@ impl IndexingWorker {
let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS);
for (page, backlinks) in pages.iter_mut().zip_eq(backlinks) {
- page.backlinks = backlinks;
+ page.set_backlinks(backlinks.clone());
+
+ let mut grouper =
+ BacklinkGrouper::new(self.host_centrality_rank_store.len() as u64);
+
+ for backlink in backlinks {
+ let node = backlink.from.clone().into_host().id();
+
+ if Some(node) == page.node_id {
+ continue;
+ }
+
+ let rank = self
+ .host_centrality_rank_store
+ .get(&node)
+ .unwrap()
+ .unwrap_or(u64::MAX);
+ grouper.add(backlink, rank);
+ }
+
+ page.set_grouped_backlinks(grouper.groups())
}
}
}
@@ -436,40 +459,21 @@ impl IndexingWorker {
let mut signal_computer = SignalComputer::new(None);
for page in batch {
- let mut prepared = match self.prepare(page) {
+ let mut webpage = match self.prepare(page) {
Ok(html) => html,
Err(err) => {
debug!("skipping webpage: {}", err);
continue;
}
};
- if let Err(e) = self.set_host_centrality(&mut prepared) {
+ if let Err(e) = self.set_host_centrality(&mut webpage) {
debug!("skipping webpage: {}", e);
continue;
}
- self.set_page_centralities(&mut prepared);
- self.set_keywords(&mut prepared);
- self.set_safety_classification(&mut prepared);
-
- // make sure we remember to set everything
- let mut webpage = Webpage {
- html: prepared.html,
- backlinks: prepared.backlinks,
- page_centrality: prepared.page_centrality,
- page_centrality_rank: prepared.page_centrality_rank,
- host_centrality: prepared.host_centrality,
- host_centrality_rank: prepared.host_centrality_rank,
- fetch_time_ms: page.fetch_time_ms,
- pre_computed_score: 0.0,
- node_id: prepared.node_id,
- dmoz_description: prepared.dmoz_description,
- safety_classification: prepared.safety_classification,
- inserted_at: Utc::now(),
- keywords: prepared.keywords,
- title_embedding: None, // set later
- keyword_embedding: None, // set later
- };
+ self.set_page_centralities(&mut webpage);
+ self.set_keywords(&mut webpage);
+ self.set_safety_classification(&mut webpage);
signal_computer.set_current_timestamp(Utc::now().timestamp().max(0) as usize);
webpage.pre_computed_score = signal_computer.precompute_score(&webpage);
@@ -479,7 +483,7 @@ impl IndexingWorker {
self.set_title_embeddings(&mut res);
self.set_keyword_embeddings(&mut res);
- self.set_backlink_labels(&mut res);
+ self.set_backlinks(&mut res);
res
}
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index 410d7cf5..60f2a7df 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -115,7 +115,7 @@ impl InvertedIndex {
self.writer
.as_ref()
.expect("writer has not been prepared")
- .add_document(webpage.as_tantivy(&self.schema)?)?;
+ .add_document(webpage.as_tantivy(self)?)?;
Ok(())
}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index e0df2cec..863ed9ac 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -198,6 +198,10 @@ impl InvertedIndex {
Arc::clone(&self.schema)
}
+ pub fn schema_ref(&self) -> &Schema {
+ &self.schema
+ }
+
pub fn num_segments(&self) -> usize {
self.tantivy_index.searchable_segments().unwrap().len()
}
@@ -485,11 +489,10 @@ mod tests {
.unwrap(),
)
.expect("failed to insert webpage");
- index
- .insert(&Webpage {
- html: Html::parse(
- &format!(
- r#"
+ let mut webpage = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -499,20 +502,22 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.b.com",
- )
- .unwrap(),
- backlinks: vec![FullEdge {
- from: Node::from("https://www.a.com"),
- to: Node::from("https://www.b.com"),
- label: "B site is great".to_string(),
- }],
- host_centrality: 1.0,
- fetch_time_ms: 500,
- ..Default::default()
- })
- .expect("failed to insert webpage");
+ ),
+ "https://www.b.com",
+ )
+ .unwrap(),
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ ..Default::default()
+ };
+
+ webpage.set_backlinks(vec![FullEdge {
+ from: Node::from("https://www.a.com"),
+ to: Node::from("https://www.b.com"),
+ label: "B site is great".to_string(),
+ }]);
+
+ index.insert(&webpage).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 610d2392..924eb167 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -43,6 +43,7 @@ pub mod ampc;
mod api;
pub mod autosuggest;
+mod backlink_grouper;
pub mod bangs;
mod bincode_utils;
pub mod canon_index;
@@ -64,6 +65,7 @@ mod intmap;
mod kahan_sum;
mod leaky_queue;
mod live_index;
+mod log_group;
mod metrics;
mod models;
pub mod naive_bayes;
diff --git a/crates/core/src/log_group.rs b/crates/core/src/log_group.rs
new file mode 100644
index 00000000..96e11b47
--- /dev/null
+++ b/crates/core/src/log_group.rs
@@ -0,0 +1,93 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub struct LogGroup {
+ base: f64,
+ groups: u64,
+}
+
+impl LogGroup {
+ pub fn new(num_items: u64, groups: u64) -> Self {
+ let base = ((num_items as f64).ln() / groups as f64).exp();
+
+ Self { base, groups }
+ }
+
+ pub fn group(&self, item: u64) -> u64 {
+ (((item + 1) as f64).log(self.base) as u64).min(self.groups - 1)
+ }
+
+ pub fn num_groups(&self) -> u64 {
+ self.groups
+ }
+}
+
+pub struct HarmonicRankGroup {
+ log_group: LogGroup,
+}
+
+impl HarmonicRankGroup {
+ pub fn new(num_hosts: u64, groups: u64) -> Self {
+ Self {
+ log_group: LogGroup::new(num_hosts, groups),
+ }
+ }
+
+ pub fn group(&self, rank: u64) -> u64 {
+ self.log_group.num_groups() - self.log_group.group(rank) - 1
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_log_group() {
+ let lg = LogGroup::new(100, 10);
+
+ assert_eq!(lg.group(0), 0);
+ assert_eq!(lg.group(1), 1);
+ assert_eq!(lg.group(2), 2);
+ assert_eq!(lg.group(10), 5);
+ assert_eq!(lg.group(11), 5);
+ assert_eq!(lg.group(90), 9);
+ assert_eq!(lg.group(99), 9);
+
+ // items over num_items are grouped together with the last group
+ assert_eq!(lg.group(100), 9);
+ assert_eq!(lg.group(1000), 9);
+
+ let lg = LogGroup::new(40_000_000, 10);
+
+ for i in 0..5 {
+ assert_eq!(lg.group(i), 0, "i = {}", i);
+ }
+ }
+
+ #[test]
+ fn test_harmonic_rank_group() {
+ let hrg = HarmonicRankGroup::new(100, 10);
+
+ assert_eq!(hrg.group(0), 9);
+ assert_eq!(hrg.group(1), 8);
+ assert_eq!(hrg.group(10), 4);
+ assert_eq!(hrg.group(11), 4);
+ assert_eq!(hrg.group(90), 0);
+ assert_eq!(hrg.group(99), 0);
+ assert_eq!(hrg.group(100), 0);
+ }
+}
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index a7a30c0c..868deac3 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -660,11 +660,11 @@ mod tests {
)
.unwrap();
- webpage.backlinks = vec![FullEdge {
+ webpage.set_backlinks(vec![FullEdge {
from: Node::from("https://www.second.com/example/abc"),
to: Node::from("https://www.first.com"),
label: String::new(),
- }];
+ }]);
index.insert(&webpage).expect("failed to insert webpage");
let mut webpage = Webpage::test_parse(
@@ -682,7 +682,7 @@ mod tests {
"https://www.second.com/example/abc",
)
.unwrap();
- webpage.backlinks = vec![];
+ webpage.set_backlinks(vec![]);
index.insert(&webpage).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index b14f6840..26dc6c2b 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -250,7 +250,6 @@ mod tests {
"https://www.b.com",
)
.unwrap(),
- backlinks: vec![],
fetch_time_ms: 500,
page_centrality: 5.0,
..Default::default()
@@ -415,10 +414,9 @@ mod tests {
fn backlink_text() {
let mut index = Index::temporary().expect("Unable to open index");
- index
- .insert(&Webpage {
- html: Html::parse(
- r#"
+ let mut webpage = Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Test site</title>
@@ -428,18 +426,20 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com",
- )
- .unwrap(),
- backlinks: vec![FullEdge {
- from: Node::from("https://www.backlink.com"),
- to: Node::from("https://www.first.com"),
- label: "test this is the best test site".to_string(),
- }],
- fetch_time_ms: 500,
- ..Default::default()
- })
- .expect("failed to insert webpage");
+ "https://www.first.com",
+ )
+ .unwrap(),
+ fetch_time_ms: 500,
+ ..Default::default()
+ };
+
+ webpage.set_backlinks(vec![FullEdge {
+ from: Node::from("https://www.backlink.com"),
+ to: Node::from("https://www.first.com"),
+ label: "test this is the best test site".to_string(),
+ }]);
+
+ index.insert(&webpage).expect("failed to insert webpage");
index
.insert(&Webpage {
html: Html::parse(
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 2775c39a..6f1494c5 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -162,7 +162,6 @@ mod tests {
"https://www.third.com",
)
.unwrap(),
- backlinks: vec![],
host_centrality: 1.0,
node_id: Some(Node::from("https://www.third.com").into_host().id()),
..Default::default()
diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rs
index c7e62c30..b0987e6b 100644
--- a/crates/core/src/schema/numerical_field.rs
+++ b/crates/core/src/schema/numerical_field.rs
@@ -52,14 +52,14 @@ pub trait NumericalField:
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()>;
fn add_webpage_tantivy(
&self,
_webpage: &crate::webpage::Webpage,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -264,9 +264,9 @@ impl NumericalField for IsHomepage {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_bool(self.tantivy_field(schema), html.is_homepage());
+ doc.add_bool(self.tantivy_field(index.schema_ref()), html.is_homepage());
Ok(())
}
@@ -284,7 +284,7 @@ impl NumericalField for HostCentrality {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -297,9 +297,12 @@ impl NumericalField for HostCentrality {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_f64(self.tantivy_field(schema), webpage.host_centrality);
+ doc.add_f64(
+ self.tantivy_field(index.schema_ref()),
+ webpage.host_centrality,
+ );
Ok(())
}
@@ -321,7 +324,7 @@ impl NumericalField for HostCentralityRank {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -330,9 +333,12 @@ impl NumericalField for HostCentralityRank {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), webpage.host_centrality_rank);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ webpage.host_centrality_rank,
+ );
Ok(())
}
@@ -353,7 +359,7 @@ impl NumericalField for PageCentrality {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -366,9 +372,12 @@ impl NumericalField for PageCentrality {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_f64(self.tantivy_field(schema), webpage.page_centrality);
+ doc.add_f64(
+ self.tantivy_field(index.schema_ref()),
+ webpage.page_centrality,
+ );
Ok(())
}
@@ -389,7 +398,7 @@ impl NumericalField for PageCentralityRank {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -398,9 +407,12 @@ impl NumericalField for PageCentralityRank {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), webpage.page_centrality_rank);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ webpage.page_centrality_rank,
+ );
Ok(())
}
@@ -421,7 +433,7 @@ impl NumericalField for FetchTimeMs {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -430,9 +442,12 @@ impl NumericalField for FetchTimeMs {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), webpage.fetch_time_ms);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ webpage.fetch_time_ms,
+ );
Ok(())
}
@@ -458,10 +473,10 @@ impl NumericalField for LastUpdated {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
html.updated_time()
.map_or(0, |time| time.timestamp().max(0) as u64),
);
@@ -486,9 +501,12 @@ impl NumericalField for TrackerScore {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), html.trackers().len() as u64);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ html.trackers().len() as u64,
+ );
Ok(())
}
@@ -514,7 +532,7 @@ impl NumericalField for Region {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -523,14 +541,14 @@ impl NumericalField for Region {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let region = crate::webpage::region::Region::guess_from(webpage);
if let Ok(region) = region {
- doc.add_u64(self.tantivy_field(schema), region.id());
+ doc.add_u64(self.tantivy_field(index.schema_ref()), region.id());
} else {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
crate::webpage::region::Region::All.id(),
);
}
@@ -555,10 +573,10 @@ impl NumericalField for NumUrlTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenize_url().tokens.len() as u64,
);
@@ -578,10 +596,10 @@ impl NumericalField for NumTitleTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache
.pretokenize_title()
.as_ref()
@@ -605,10 +623,10 @@ impl NumericalField for NumCleanBodyTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenize_clean_text().tokens.len() as u64,
);
Ok(())
@@ -627,10 +645,10 @@ impl NumericalField for NumDescriptionTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenize_description().tokens.len() as u64,
);
@@ -650,10 +668,10 @@ impl NumericalField for NumUrlForSiteOperatorTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenize_url_for_site_operator().tokens.len() as u64,
);
@@ -673,10 +691,10 @@ impl NumericalField for NumDomainTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenize_domain().tokens.len() as u64,
);
@@ -696,10 +714,10 @@ impl NumericalField for NumMicroformatTagsTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenize_microformats().tokens.len() as u64,
);
@@ -719,9 +737,9 @@ impl NumericalField for SiteHash1 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.site_hash()[0]);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.site_hash()[0]);
Ok(())
}
@@ -739,9 +757,9 @@ impl NumericalField for SiteHash2 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.site_hash()[1]);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.site_hash()[1]);
Ok(())
}
@@ -759,10 +777,10 @@ impl NumericalField for UrlWithoutQueryHash1 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.url_without_query_hash()[0],
);
@@ -782,10 +800,10 @@ impl NumericalField for UrlWithoutQueryHash2 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.url_without_query_hash()[1],
);
@@ -805,9 +823,12 @@ impl NumericalField for TitleHash1 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.title_hash()[0]);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ cache.title_hash()[0],
+ );
Ok(())
}
@@ -825,9 +846,12 @@ impl NumericalField for TitleHash2 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.title_hash()[1]);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ cache.title_hash()[1],
+ );
Ok(())
}
@@ -845,9 +869,9 @@ impl NumericalField for UrlHash1 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.url_hash()[0]);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.url_hash()[0]);
Ok(())
}
@@ -865,9 +889,9 @@ impl NumericalField for UrlHash2 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.url_hash()[1]);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), cache.url_hash()[1]);
Ok(())
}
@@ -885,9 +909,12 @@ impl NumericalField for DomainHash1 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.domain_hash()[0]);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ cache.domain_hash()[0],
+ );
Ok(())
}
@@ -905,9 +932,12 @@ impl NumericalField for DomainHash2 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.domain_hash()[1]);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ cache.domain_hash()[1],
+ );
Ok(())
}
@@ -925,9 +955,12 @@ impl NumericalField for UrlWithoutTldHash1 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.url_without_tld_hash()[0]);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ cache.url_without_tld_hash()[0],
+ );
Ok(())
}
@@ -945,9 +978,12 @@ impl NumericalField for UrlWithoutTldHash2 {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_u64(self.tantivy_field(schema), cache.url_without_tld_hash()[1]);
+ doc.add_u64(
+ self.tantivy_field(index.schema_ref()),
+ cache.url_without_tld_hash()[1],
+ );
Ok(())
}
@@ -969,7 +1005,7 @@ impl NumericalField for PreComputedScore {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -982,9 +1018,12 @@ impl NumericalField for PreComputedScore {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_f64(self.tantivy_field(schema), webpage.pre_computed_score);
+ doc.add_f64(
+ self.tantivy_field(index.schema_ref()),
+ webpage.pre_computed_score,
+ );
Ok(())
}
@@ -1006,7 +1045,7 @@ impl NumericalField for HostNodeID {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1015,14 +1054,14 @@ impl NumericalField for HostNodeID {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
match &webpage.node_id {
Some(node_id) => {
- doc.add_u64(self.tantivy_field(schema), node_id.as_u64());
+ doc.add_u64(self.tantivy_field(index.schema_ref()), node_id.as_u64());
}
None => {
- doc.add_u64(self.tantivy_field(schema), u64::MAX);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), u64::MAX);
}
}
@@ -1050,7 +1089,7 @@ impl NumericalField for SimHash {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let clean_text = cache.pretokenize_clean_text();
@@ -1059,7 +1098,7 @@ impl NumericalField for SimHash {
} else {
0
};
- doc.add_u64(self.tantivy_field(schema), hash);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), hash);
Ok(())
}
@@ -1077,10 +1116,10 @@ impl NumericalField for NumFlattenedSchemaTokens {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_u64(
- self.tantivy_field(schema),
+ self.tantivy_field(index.schema_ref()),
cache.pretokenized_schema_json().tokens.len() as u64,
);
@@ -1104,7 +1143,7 @@ impl NumericalField for NumPathAndQuerySlashes {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let num_slashes = html
.url()
@@ -1112,7 +1151,7 @@ impl NumericalField for NumPathAndQuerySlashes {
.map(|segments| segments.count())
.unwrap_or(0);
- doc.add_u64(self.tantivy_field(schema), num_slashes as u64);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), num_slashes as u64);
Ok(())
}
@@ -1138,7 +1177,7 @@ impl NumericalField for NumPathAndQueryDigits {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let num_digits = html
.url()
@@ -1154,7 +1193,7 @@ impl NumericalField for NumPathAndQueryDigits {
.filter(|c| c.is_ascii_digit())
.count();
- doc.add_u64(self.tantivy_field(schema), num_digits as u64);
+ doc.add_u64(self.tantivy_field(index.schema_ref()), num_digits as u64);
Ok(())
}
@@ -1184,9 +1223,12 @@ impl NumericalField for LikelyHasAds {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_bool(self.tantivy_field(schema), html.likely_has_ads());
+ doc.add_bool(
+ self.tantivy_field(index.schema_ref()),
+ html.likely_has_ads(),
+ );
Ok(())
}
@@ -1216,9 +1258,12 @@ impl NumericalField for LikelyHasPaywall {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_bool(self.tantivy_field(schema), html.likely_has_paywall());
+ doc.add_bool(
+ self.tantivy_field(index.schema_ref()),
+ html.likely_has_paywall(),
+ );
Ok(())
}
@@ -1248,9 +1293,9 @@ impl NumericalField for LinkDensity {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
- doc.add_f64(self.tantivy_field(schema), html.link_density());
+ doc.add_f64(self.tantivy_field(index.schema_ref()), html.link_density());
Ok(())
}
@@ -1280,7 +1325,7 @@ impl NumericalField for TitleEmbeddings {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1289,15 +1334,15 @@ impl NumericalField for TitleEmbeddings {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
if let Some(emb) = &webpage.title_embedding {
let mut serialized = Vec::new();
emb.write_bytes(&mut serialized)?;
- doc.add_bytes(self.tantivy_field(schema), &serialized);
+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &serialized);
} else {
- doc.add_bytes(self.tantivy_field(schema), &[]);
+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &[]);
}
Ok(())
@@ -1324,7 +1369,7 @@ impl NumericalField for KeywordEmbeddings {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1333,15 +1378,15 @@ impl NumericalField for KeywordEmbeddings {
&self,
webpage: &Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
if let Some(emb) = &webpage.keyword_embedding {
let mut serialized = Vec::new();
emb.write_bytes(&mut serialized)?;
- doc.add_bytes(self.tantivy_field(schema), &serialized);
+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &serialized);
} else {
- doc.add_bytes(self.tantivy_field(schema), &[]);
+ doc.add_bytes(self.tantivy_field(index.schema_ref()), &[]);
}
Ok(())
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 7177050b..b540ba51 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -54,7 +54,7 @@ pub trait TextField:
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()>;
#[allow(unused_variables)]
@@ -62,7 +62,7 @@ pub trait TextField:
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -200,6 +200,16 @@ pub enum TextFieldEnum {
KeyPhrases,
Links,
Backlinks,
+ BacklinkLabelsGroup0,
+ BacklinkLabelsGroup1,
+ BacklinkLabelsGroup2,
+ BacklinkLabelsGroup3,
+ BacklinkLabelsGroup4,
+ BacklinkLabelsGroup5,
+ BacklinkLabelsGroup6,
+ BacklinkLabelsGroup7,
+ BacklinkLabelsGroup8,
+ BacklinkLabelsGroup9,
}
enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
@@ -239,6 +249,16 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
KeyPhrases,
Links,
Backlinks,
+ BacklinkLabelsGroup0,
+ BacklinkLabelsGroup1,
+ BacklinkLabelsGroup2,
+ BacklinkLabelsGroup3,
+ BacklinkLabelsGroup4,
+ BacklinkLabelsGroup5,
+ BacklinkLabelsGroup6,
+ BacklinkLabelsGroup7,
+ BacklinkLabelsGroup8,
+ BacklinkLabelsGroup9,
]);
impl TextFieldEnum {
@@ -329,10 +349,10 @@ impl TextField for Title {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache
.pretokenize_title()
@@ -365,10 +385,10 @@ impl TextField for CleanBody {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.pretokenize_clean_text().clone(),
);
@@ -400,7 +420,7 @@ impl TextField for StemmedTitle {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let title = cache
.pretokenize_title()
@@ -411,7 +431,7 @@ impl TextField for StemmedTitle {
stem_tokens(&mut tokens, html.lang().copied().unwrap_or(Lang::Eng));
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: title.text.clone(),
@@ -450,14 +470,14 @@ impl TextField for StemmedCleanBody {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let clean_text = cache.pretokenize_clean_text();
let mut tokens = clean_text.tokens.clone();
stem_tokens(&mut tokens, html.lang().copied().unwrap_or(Lang::Eng));
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: clean_text.text.clone(),
@@ -489,7 +509,7 @@ impl TextField for AllBody {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let all_text = cache
.pretokenize_all_text()
@@ -498,7 +518,7 @@ impl TextField for AllBody {
.map_err(|e| anyhow::anyhow!("{}", e))?;
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
all_text.clone(),
);
@@ -535,11 +555,11 @@ impl TextField for Url {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let url = cache.pretokenize_url();
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
url.clone(),
);
@@ -568,12 +588,12 @@ impl TextField for UrlNoTokenizer {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let url = html.url().to_string();
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: url.clone(),
@@ -611,10 +631,10 @@ impl TextField for UrlForSiteOperator {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.pretokenize_url_for_site_operator().clone(),
);
@@ -639,10 +659,10 @@ impl TextField for SiteWithout {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.pretokenize_site().clone(),
);
@@ -666,10 +686,10 @@ impl TextField for Domain {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.pretokenize_domain().clone(),
);
@@ -698,12 +718,12 @@ impl TextField for SiteNoTokenizer {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let site = cache.pretokenize_site();
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: site.text.clone(),
@@ -741,12 +761,12 @@ impl TextField for DomainNoTokenizer {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let domain = cache.pretokenize_domain();
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: domain.text.clone(),
@@ -784,12 +804,12 @@ impl TextField for DomainNameNoTokenizer {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let domain_name = cache.domain_name();
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: domain_name.clone(),
@@ -823,13 +843,13 @@ impl TextField for SiteIfHomepageNoTokenizer {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let site = cache.pretokenize_site();
if html.is_homepage() {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: site.text.clone(),
@@ -844,7 +864,7 @@ impl TextField for SiteIfHomepageNoTokenizer {
);
} else {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
"",
);
@@ -866,18 +886,18 @@ impl TextField for DomainIfHomepage {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let domain = cache.pretokenize_domain();
if html.is_homepage() {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
domain.text.clone(),
);
} else {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
"",
);
@@ -903,13 +923,13 @@ impl TextField for DomainNameIfHomepageNoTokenizer {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let domain_name = cache.domain_name();
if html.is_homepage() {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: domain_name.clone(),
@@ -924,7 +944,7 @@ impl TextField for DomainNameIfHomepageNoTokenizer {
);
} else {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
"",
);
@@ -950,13 +970,13 @@ impl TextField for DomainIfHomepageNoTokenizer {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let domain = cache.pretokenize_domain();
if html.is_homepage() {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
PreTokenizedString {
text: domain.text.clone(),
@@ -971,7 +991,7 @@ impl TextField for DomainIfHomepageNoTokenizer {
);
} else {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
"",
);
@@ -993,7 +1013,7 @@ impl TextField for TitleIfHomepage {
html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let title = cache
.pretokenize_title()
@@ -1003,13 +1023,13 @@ impl TextField for TitleIfHomepage {
if html.is_homepage() {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
title,
);
} else {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
"",
);
@@ -1031,7 +1051,7 @@ impl TextField for BacklinkText {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1040,12 +1060,16 @@ impl TextField for BacklinkText {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
- webpage.backlinks.iter().map(|e| e.label.clone()).join("\n"),
+ webpage
+ .backlinks()
+ .iter()
+ .map(|e| e.label.clone())
+ .join("\n"),
);
Ok(())
@@ -1076,11 +1100,11 @@ impl TextField for Description {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let description = cache.pretokenize_description();
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
description.clone(),
);
@@ -1109,7 +1133,7 @@ impl TextField for DmozDescription {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1118,10 +1142,10 @@ impl TextField for DmozDescription {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
webpage.dmoz_description().unwrap_or_default(),
);
@@ -1150,10 +1174,10 @@ impl TextField for SchemaOrgJson {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.schema_json(),
);
@@ -1182,10 +1206,10 @@ impl TextField for FlattenedSchemaOrgJson {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.pretokenized_schema_json().clone(),
);
@@ -1226,10 +1250,10 @@ impl TextField for CleanBodyBigrams {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
html.clean_text().cloned().unwrap_or_default(),
);
@@ -1270,7 +1294,7 @@ impl TextField for TitleBigrams {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let title = cache
.pretokenize_title()
@@ -1279,7 +1303,7 @@ impl TextField for TitleBigrams {
.map_err(|e| anyhow::anyhow!("{}", e))?;
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
title.text.clone(),
);
@@ -1320,10 +1344,10 @@ impl TextField for CleanBodyTrigrams {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
html.clean_text().cloned().unwrap_or_default(),
);
@@ -1364,7 +1388,7 @@ impl TextField for TitleTrigrams {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let title = cache
.pretokenize_title()
@@ -1373,7 +1397,7 @@ impl TextField for TitleTrigrams {
.map_err(|e| anyhow::anyhow!("{}", e))?;
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
title.text.clone(),
);
@@ -1398,10 +1422,10 @@ impl TextField for MicroformatTags {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_pre_tokenized_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.pretokenize_microformats().clone(),
);
@@ -1426,7 +1450,7 @@ impl TextField for SafetyClassification {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1435,7 +1459,7 @@ impl TextField for SafetyClassification {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
let safety = webpage
.safety_classification
@@ -1443,7 +1467,7 @@ impl TextField for SafetyClassification {
.unwrap_or_default();
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
safety,
);
@@ -1472,7 +1496,7 @@ impl TextField for InsertionTimestamp {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1481,10 +1505,10 @@ impl TextField for InsertionTimestamp {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_date(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
tantivy::DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
webpage.inserted_at.timestamp(),
@@ -1515,10 +1539,10 @@ impl TextField for RecipeFirstIngredientTagId {
_html: &Html,
cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
cache.first_ingredient_tag_id().cloned().unwrap_or_default(),
);
@@ -1547,10 +1571,10 @@ impl TextField for Links {
html: &Html,
_cache: &mut FnCache,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
html.anchor_links()
.into_iter()
@@ -1581,13 +1605,13 @@ impl TextField for Backlinks {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
webpage
- .backlinks
+ .backlinks()
.iter()
.map(|l| l.from.as_str().to_string())
.join("\n"),
@@ -1601,7 +1625,7 @@ impl TextField for Backlinks {
_: &Html,
_: &mut FnCache,
_: &mut TantivyDocument,
- _: &tantivy::schema::Schema,
+ _: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1623,7 +1647,7 @@ impl TextField for Keywords {
_html: &Html,
_cache: &mut FnCache,
_doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ _index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1632,10 +1656,10 @@ impl TextField for Keywords {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
webpage.keywords.join("\n"),
);
@@ -1659,10 +1683,10 @@ impl TextField for KeyPhrases {
&self,
webpage: &crate::webpage::Webpage,
doc: &mut TantivyDocument,
- schema: &tantivy::schema::Schema,
+ index: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
doc.add_text(
- self.tantivy_field(schema)
+ self.tantivy_field(index.schema_ref())
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
webpage.keywords.join("\n"),
);
@@ -1675,7 +1699,7 @@ impl TextField for KeyPhrases {
_: &Html,
_: &mut FnCache,
_: &mut TantivyDocument,
- _: &tantivy::schema::Schema,
+ _: &crate::inverted_index::InvertedIndex,
) -> Result<()> {
Ok(())
}
@@ -1684,3 +1708,58 @@ impl TextField for KeyPhrases {
FieldTokenizer::Newline(NewlineTokenizer::default())
}
}
+
+macro_rules! impl_backlink_group {
+ ($name:ident, $index:expr) => {
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+ pub struct $name;
+ impl TextField for $name {
+ fn name(&self) -> &str {
+ concat!("backlink_group_", $index)
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _: &Html,
+ _: &mut FnCache,
+ _: &mut TantivyDocument,
+ _: &crate::inverted_index::InvertedIndex,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ index: &crate::inverted_index::InvertedIndex,
+ ) -> Result<()> {
+ let backlinks = webpage
+ .grouped_backlinks()
+ .get($index)
+ .map(|group| group.backlinks().iter().map(|l| &l.label).join("\n"))
+ .unwrap_or_default();
+
+ doc.add_text(
+ self.tantivy_field(index.schema_ref()).unwrap_or_else(|| {
+ panic!("could not find field '{}' in index", self.name())
+ }),
+ backlinks,
+ );
+
+ Ok(())
+ }
+ }
+ };
+}
+
+impl_backlink_group!(BacklinkLabelsGroup0, 0);
+impl_backlink_group!(BacklinkLabelsGroup1, 1);
+impl_backlink_group!(BacklinkLabelsGroup2, 2);
+impl_backlink_group!(BacklinkLabelsGroup3, 3);
+impl_backlink_group!(BacklinkLabelsGroup4, 4);
+impl_backlink_group!(BacklinkLabelsGroup5, 5);
+impl_backlink_group!(BacklinkLabelsGroup6, 6);
+impl_backlink_group!(BacklinkLabelsGroup7, 7);
+impl_backlink_group!(BacklinkLabelsGroup8, 8);
+impl_backlink_group!(BacklinkLabelsGroup9, 9);
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index f3a7100b..4275df32 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -193,17 +193,21 @@ impl Html {
split_u128(hash(self.title().unwrap_or_default()).0)
}
- pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+ pub fn as_tantivy(
+ &self,
+ index: &crate::inverted_index::InvertedIndex,
+ ) -> Result<TantivyDocument> {
let mut doc = TantivyDocument::new();
let mut cache = FnCache::new(self);
- for field in schema
+ for field in index
+ .schema_ref()
.fields()
.filter_map(|(field, _)| Field::get(field.field_id() as usize))
{
match field {
- Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
- Field::Numerical(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
+ Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, index)?,
+ Field::Numerical(f) => f.add_html_tantivy(self, &mut cache, &mut doc, index)?,
}
}
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index a7bf0282..1759b646 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -489,7 +489,7 @@ mod tests {
use tests::links::RelFlags;
- use crate::{schema::create_schema, webpage::Link};
+ use crate::webpage::Link;
use super::*;
@@ -734,8 +734,9 @@ mod tests {
assert!(webpage.all_text().is_some());
assert!(!webpage.all_text().unwrap().is_empty());
- let schema = create_schema();
- webpage.as_tantivy(&schema).unwrap();
+ let index =
+ crate::inverted_index::InvertedIndex::temporary().expect("Unable to open index");
+ webpage.as_tantivy(&index).unwrap();
}
#[test]
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index b6c58712..acf807c6 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -15,6 +15,8 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ backlink_grouper::GroupedBacklinks,
+ inverted_index::InvertedIndex,
schema::{numerical_field::NumericalField, text_field::TextField, Field},
webgraph::{FullEdge, NodeID},
Result,
@@ -41,7 +43,15 @@ pub use region::Region;
#[derive(Debug)]
pub struct Webpage {
pub html: Html,
+
+ #[cfg(test)]
pub backlinks: Vec<FullEdge>,
+
+ #[cfg(not(test))]
+ backlinks: Vec<FullEdge>,
+
+ pub grouped_backlinks: GroupedBacklinks,
+
pub host_centrality: f64,
pub host_centrality_rank: u64,
pub page_centrality: f64,
@@ -63,6 +73,7 @@ impl Default for Webpage {
Self {
html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),
backlinks: Default::default(),
+ grouped_backlinks: GroupedBacklinks::empty(),
host_centrality: Default::default(),
host_centrality_rank: u64::MAX,
page_centrality: Default::default(),
@@ -84,6 +95,7 @@ impl From<Html> for Webpage {
fn from(html: Html) -> Self {
Self {
html,
+ grouped_backlinks: GroupedBacklinks::empty(),
backlinks: Default::default(),
host_centrality: Default::default(),
host_centrality_rank: u64::MAX,
@@ -129,13 +141,29 @@ impl Webpage {
})
}
- pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
- let mut doc = self.html.as_tantivy(schema)?;
+ pub fn backlinks(&self) -> &[FullEdge] {
+ &self.backlinks
+ }
+
+ pub fn set_backlinks(&mut self, backlinks: Vec<FullEdge>) {
+ self.backlinks = backlinks;
+ }
+
+ pub fn set_grouped_backlinks(&mut self, grouped_backlinks: GroupedBacklinks) {
+ self.grouped_backlinks = grouped_backlinks;
+ }
+
+ pub fn grouped_backlinks(&self) -> &GroupedBacklinks {
+ &self.grouped_backlinks
+ }
+
+ pub fn as_tantivy(&self, index: &InvertedIndex) -> Result<TantivyDocument> {
+ let mut doc = self.html.as_tantivy(index)?;
for field in Field::all() {
match field {
- Field::Numerical(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
- Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
+ Field::Numerical(f) => f.add_webpage_tantivy(self, &mut doc, index)?,
+ Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, index)?,
}
}