Skip to content

web search done right

Ftr/page centrality (#55)

Mikkel Denker8c9ffede3068
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index c492c6bf..d4aa3b48 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,5 +1,5 @@
warc_paths_file = "warc.paths"
-centrality_store_path = "./data/centrality/host"
+centrality_store_path = "./data/centrality"
limit_warc_files = 100
download_images = false
batch_size = 10
diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.toml
index 1ef3c6f3..9d5ca99c 100644
--- a/configs/indexer/profile.toml
+++ b/configs/indexer/profile.toml
@@ -1,9 +1,8 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
-limit_warc_files = 3
download_images = false
[warc_source]
type = "Local"
folder = "./data/warc_files"
-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz", "CC-MAIN-20220116093137-20220116123137-00049_1.warc.gz", "CC-MAIN-20220116093137-20220116123137-00049_2.warc.gz"]
+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
diff --git a/data.tar.gz b/data.tar.gz
index bc1f8c04..9ec479b1 100644
--- a/data.tar.gz
+++ b/data.tar.gz
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:4aa671bf426371399436ef6753e0ac4c3b398967df2c9487690ed8b4a6bd9208
-size 896036141
+oid sha256:d770c6f965cdc6dbc2bac3c97bb413b9c820b718eebfee04053b579fd451e8e4
+size 1132332917
diff --git a/justfile b/justfile
index ce18a18d..d6a4ae07 100644
--- a/justfile
+++ b/justfile
@@ -2,9 +2,9 @@
rm -rf data
tar -zxvf data.tar.gz
-@flamegraph:
+@profile-indexer:
sudo rm -rf data/index
- cargo flamegraph --root -- indexer local configs/indexer/local.toml
+ cargo flamegraph --root -- indexer local configs/indexer/profile.toml
@worker:
cargo run --release -- webgraph worker configs/webgraph/worker.toml
diff --git a/src/bangs.rs b/src/bangs.rs
index 8d959b9b..21a86c22 100644
--- a/src/bangs.rs
+++ b/src/bangs.rs
@@ -125,7 +125,7 @@ mod tests {
use tantivy::tokenizer::TokenizerManager;
- use crate::{ranking::goggles::SignalAggregator, schema::create_schema};
+ use crate::{ranking::SignalAggregator, schema::create_schema};
use super::*;
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 6a151e2e..cc095839 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -49,14 +49,19 @@ struct Job {
}
struct IndexingWorker {
- centrality_store: CentralityStore,
+ host_centrality_store: CentralityStore,
+ page_centrality_store: CentralityStore,
webgraph: Option<Webgraph>,
}
impl IndexingWorker {
fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {
+ let host_centrality_path = Path::new(&centrality_store_path).join("host");
+ let page_centrality_path = Path::new(&centrality_store_path).join("full");
+
Self {
- centrality_store: CentralityStore::new(centrality_store_path),
+ host_centrality_store: CentralityStore::new(host_centrality_path),
+ page_centrality_store: CentralityStore::new(page_centrality_path),
webgraph: webgraph_path.map(|path| {
WebgraphBuilder::new(path)
.with_full_graph()
@@ -113,10 +118,17 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
.collect()
})
.unwrap_or_else(Vec::new);
- let centrality = worker
- .centrality_store
+
+ let host_centrality = worker
+ .host_centrality_store
.get(html.url().host_without_specific_subdomains())
.unwrap_or_default();
+
+ let page_centrality = worker
+ .page_centrality_store
+ .get(html.url().raw())
+ .unwrap_or_default();
+
let fetch_time_ms = record.metadata.fetch_time_ms as u64;
trace!("inserting webpage: {:?}", html.url());
@@ -127,7 +139,8 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
let webpage = Webpage {
html,
backlinks,
- centrality,
+ page_centrality,
+ host_centrality,
fetch_time_ms,
primary_image: None,
};
diff --git a/src/index.rs b/src/index.rs
index 0021831f..06484075 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -156,7 +156,7 @@ impl Index {
fn maybe_insert_primary_image(&mut self, webpage: &mut Webpage) {
match webpage
- .centrality
+ .host_centrality
.partial_cmp(&IMAGE_WEBPAGE_CENTRALITY_THRESHOLD)
{
None | Some(Ordering::Greater) => {}
@@ -291,7 +291,7 @@ impl From<Index> for FrozenIndex {
#[cfg(test)]
mod tests {
- use crate::ranking::{goggles::SignalAggregator, Ranker};
+ use crate::ranking::{Ranker, SignalAggregator};
use super::*;
@@ -316,11 +316,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().unwrap();
@@ -373,11 +370,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().unwrap();
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 47f5bbf8..77cf2741 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -363,7 +363,8 @@ impl From<Document> for RetrievedWebpage {
}
}
Field::BacklinkText
- | Field::Centrality
+ | Field::HostCentrality
+ | Field::PageCentrality
| Field::Site
| Field::StemmedTitle
| Field::CleanBody
@@ -388,8 +389,8 @@ mod tests {
use maplit::hashset;
use crate::{
- ranking::{goggles::SignalAggregator, Ranker},
- webpage::{region::RegionCount, Link},
+ ranking::{Ranker, SignalAggregator},
+ webpage::{region::RegionCount, Html, Link},
};
use super::*;
@@ -429,11 +430,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -471,11 +469,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -512,11 +507,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -553,11 +545,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -593,15 +582,13 @@ mod tests {
"#
),
"https://www.a.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -611,17 +598,20 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.b.com",
),
- "https://www.b.com",
- vec![Link {
+ backlinks: vec![Link {
source: "https://www.a.com".to_string().into(),
destination: "https://www.b.com".to_string().into(),
text: "B site is great".to_string(),
}],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ host_centrality: 1.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -665,11 +655,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
}
index.commit().expect("failed to commit index");
@@ -707,11 +694,8 @@ mod tests {
"#
),
"https://www.dr.dk",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -740,11 +724,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
let mut index2 = InvertedIndex::temporary().expect("Unable to open index");
@@ -763,11 +744,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
let mut index = index1.merge(index2);
index.commit().unwrap();
@@ -823,11 +801,8 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -838,69 +813,6 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
}
- #[test]
- fn fetch_time_ranking() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse(
- "test",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
-
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- ),
- "https://www.first.com",
- vec![],
- 1.0,
- 0,
- ))
- .expect("failed to parse webpage");
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- ),
- "https://www.second.com",
- vec![],
- 1.0,
- 5000,
- ))
- .expect("failed to parse webpage");
- index.commit().expect("failed to commit index");
-
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
- assert_eq!(result.num_docs, 2);
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
- }
-
#[test]
fn only_show_primary_images_when_relevant() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
@@ -923,14 +835,11 @@ mod tests {
"#
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
);
let uuid = uuid::uuid!("00000000-0000-0000-0000-ffff00000000");
webpage.set_primary_image(uuid, webpage.html.primary_image().unwrap());
- index.insert(webpage).expect("failed to parse webpage");
+ index.insert(webpage).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let query = Query::parse(
diff --git a/src/query/mod.rs b/src/query/mod.rs
index da9c07fb..1d943ae8 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -14,11 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{
- ranking::goggles::{Goggle, SignalAggregator},
- schema::Field,
- Result,
-};
+use crate::{ranking::goggles::Goggle, ranking::SignalAggregator, schema::Field, Result};
use std::{collections::HashMap, sync::Arc};
use tantivy::{
query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},
@@ -312,11 +308,8 @@ mod tests {
</html>
"#,
"https://www.first.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index
.insert(Webpage::new(
r#"
@@ -330,11 +323,8 @@ mod tests {
</html>
"#,
"https://www.second.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let result = index
@@ -362,11 +352,8 @@ mod tests {
</html>
"#,
"https://www.first.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index
.insert(Webpage::new(
r#"
@@ -380,11 +367,8 @@ mod tests {
</html>
"#,
"https://www.second.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let query = Query::parse(
@@ -450,11 +434,8 @@ mod tests {
</html>
"#,
"https://www.first.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index
.insert(Webpage::new(
r#"
@@ -468,11 +449,8 @@ mod tests {
</html>
"#,
"https://www.second.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let query = Query::parse(
@@ -508,11 +486,8 @@ mod tests {
</html>
"#,
"https://www.first.com/forum",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index
.insert(Webpage::new(
r#"
@@ -526,11 +501,8 @@ mod tests {
</html>
"#,
"https://www.second.com",
- vec![],
- 1.0,
- 0,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let query = Query::parse(
diff --git a/src/query/parser.rs b/src/query/parser.rs
index 2cbccfa9..879f9fa1 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -22,7 +22,7 @@ use tantivy::{
use crate::{
bangs::BANG_PREFIX,
- ranking::goggles::FieldBoost,
+ ranking::FieldBoost,
schema::{Field, ALL_FIELDS},
};
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 08776159..40505a7a 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -14,9 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-mod ast;
+pub mod ast;
mod pattern_query;
-mod signal;
use std::convert::TryFrom;
@@ -28,12 +27,13 @@ use tantivy::{
tokenizer::TextAnalyzer,
};
-pub use self::signal::*;
use self::{
ast::{RawAction, RawGoggle, RawInstruction, RawPatternOption, RawPatternPart},
pattern_query::PatternQuery,
};
+use super::signal::SignalAggregator;
+
pub fn parse(goggle: &str) -> Result<Goggle> {
let raw_goggle = ast::parse(goggle)?;
@@ -302,7 +302,12 @@ impl Instruction {
#[cfg(test)]
mod tests {
- use crate::{index::Index, schema::create_schema, searcher::Searcher, webpage::Webpage};
+ use crate::{
+ index::Index,
+ schema::create_schema,
+ searcher::Searcher,
+ webpage::{Html, Webpage},
+ };
use super::*;
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
@@ -312,9 +317,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website A</title>
@@ -325,17 +331,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.a.com",
),
- "https://www.a.com",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -345,13 +355,16 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.b.com",
),
- "https://www.b.com",
- vec![],
- 0.0001,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0001,
+ page_centrality: 0.0,
+ primary_image: None,
+ fetch_time_ms: 500,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -424,9 +437,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website A</title>
@@ -437,17 +451,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.a.com/this/is/a/pattern",
),
- "https://www.a.com/this/is/a/pattern",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -457,13 +475,16 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.b.com/this/is/b/pattern",
),
- "https://www.b.com/this/is/b/pattern",
- vec![],
- 0.0001,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0001,
+ page_centrality: 0.0,
+ primary_image: None,
+ fetch_time_ms: 500,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -515,9 +536,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website A</title>
@@ -528,17 +550,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.a.com/this/is/a/pattern",
),
- "https://www.a.com/this/is/a/pattern",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -548,17 +574,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.b.com/this/is/b/pattern",
),
- "https://www.b.com/this/is/b/pattern",
- vec![],
- 0.0001,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0001,
+ page_centrality: 0.0,
+ primary_image: None,
+ fetch_time_ms: 500,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -568,13 +598,16 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.c.com/this/is/c/pattern",
),
- "https://www.c.com/this/is/c/pattern",
- vec![],
- 0.0001,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0001,
+ page_centrality: 0.0,
+ primary_image: None,
+ fetch_time_ms: 500,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 79c1312c..2ec08a5a 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -21,7 +21,7 @@ use chrono::Utc;
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
use tantivy::{DocId, Score, SegmentReader};
-use super::goggles::SignalAggregator;
+use super::SignalAggregator;
pub(crate) struct InitialScoreTweaker {
region_count: Arc<RegionCount>,
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index ef508c02..92971a81 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -18,6 +18,7 @@ mod bm25;
pub mod centrality_store;
pub mod goggles;
mod initial;
+pub mod signal;
use std::sync::Arc;
@@ -29,7 +30,7 @@ use crate::{
webpage::region::{Region, RegionCount},
};
-use self::goggles::SignalAggregator;
+pub use self::signal::*;
pub struct Ranker {
region_count: Arc<RegionCount>,
@@ -80,19 +81,20 @@ mod tests {
use crate::{
index::Index,
searcher::Searcher,
- webpage::{Link, Webpage},
+ webpage::{Html, Link, Webpage},
};
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
#[test]
- fn harmonic_ranking() {
+ fn host_centrality_ranking() {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website A</title>
@@ -103,17 +105,87 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.a.com",
),
- "https://www.a.com",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.b.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 5.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::from(index);
+ let result = searcher
+ .search("example", None, None, None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
+ assert_eq!(result.webpages.documents.len(), 2);
+ assert_eq!(result.webpages.documents[0].url, "https://www.b.com");
+ assert_eq!(result.webpages.documents[1].url, "https://www.a.com");
+ }
+
+ #[test]
+ fn page_centrality_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <body>
+ {CONTENT}
+ example example example
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.a.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -123,13 +195,16 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.b.com",
),
- "https://www.b.com",
- vec![],
- 5.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 5.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -148,9 +223,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>DR Homepage</title>
@@ -160,17 +236,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.dr.dk",
),
- "https://www.dr.dk",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Subsite dr</title>
@@ -180,17 +260,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.dr.dk/whatever",
),
- "https://www.dr.dk/whatever",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Website B</title>
@@ -198,13 +282,16 @@ mod tests {
dr and some other text {CONTENT} dk
</html>
"#
+ ),
+ "https://www.b.com",
),
- "https://www.b.com",
- vec![],
- 0.003,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.003,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -223,9 +310,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Title</title>
@@ -236,17 +324,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.old.com",
),
- "https://www.old.com",
- vec![],
- 0.092,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.092,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Title</title>
@@ -257,13 +349,16 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.new.com",
),
- "https://www.new.com",
- vec![],
- 0.09,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 0.09,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -281,8 +376,9 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Test site</title>
@@ -292,15 +388,18 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com",
- vec![],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ "https://www.first.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(r#"
<html>
<head>
<script>
@@ -324,12 +423,14 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com",
- vec![],
- 0.003,
- 500,
- ))
- .expect("failed to parse webpage");
+ "https://www.second.com"),
+ backlinks: vec![],
+ host_centrality: 0.003,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -348,8 +449,9 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Test site</title>
@@ -359,19 +461,23 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com",
- vec![Link {
+ "https://www.first.com",
+ ),
+ backlinks: vec![Link {
source: "https://www.second.com".to_string().into(),
destination: "https://www.first.com".to_string().into(),
text: "test this is the best test site".to_string(),
}],
- 0.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Second test site</title>
@@ -381,12 +487,15 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com",
- vec![],
- 0.003,
- 500,
- ))
- .expect("failed to parse webpage");
+ "https://www.second.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.003,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -405,8 +514,9 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -416,16 +526,20 @@ mod tests {
</body>
</html>
"#,
- "https://www.body.com",
- vec![],
- 1.0,
- 20,
- ))
- .expect("failed to parse webpage");
+ "https://www.body.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 20,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Example website</title>
@@ -435,16 +549,20 @@ mod tests {
</body>
</html>
"#,
- "https://www.title.com",
- vec![],
- 1.0,
- 20,
- ))
- .expect("failed to parse webpage");
+ "https://www.title.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 20,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Example website</title>
@@ -454,12 +572,15 @@ mod tests {
</body>
</html>
"#,
- "https://www.centrality.com",
- vec![],
- 1.0002,
- 500,
- ))
- .expect("failed to parse webpage");
+ "https://www.centrality.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0002,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().unwrap();
@@ -524,9 +645,10 @@ mod tests {
fn term_proximity_ranking() {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -536,17 +658,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.first.com",
),
- "https://www.first.com",
- vec![],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -556,17 +682,21 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.third.com",
),
- "https://www.third.com",
- vec![],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -576,13 +706,16 @@ mod tests {
</body>
</html>
"#
+ ),
+ "https://www.second.com",
),
- "https://www.second.com",
- vec![],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::new(index, None, None);
@@ -599,4 +732,72 @@ mod tests {
assert_eq!(result.documents[1].url, "https://www.second.com");
assert_eq!(result.documents[2].url, "https://www.third.com");
}
+
+ #[test]
+ fn fetch_time_ranking() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.first.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 0,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.second.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 5000,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::new(index, None, None);
+
+ let result = searcher
+ .search("test", None, None, None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ assert_eq!(result.documents[1].url, "https://www.second.com");
+ }
}
diff --git a/src/ranking/goggles/signal.rs b/src/ranking/signal.rs
similarity index 96%
rename from src/ranking/goggles/signal.rs
rename to src/ranking/signal.rs
index b3958857..531b35bf 100644
--- a/src/ranking/goggles/signal.rs
+++ b/src/ranking/signal.rs
@@ -27,12 +27,13 @@ use crate::{
webpage::region::{Region, RegionCount},
};
-use super::ast::{RawAlteration, Target};
+use crate::ranking::goggles::ast::{RawAlteration, Target};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Signal {
Bm25,
HostCentrality,
+ PageCentrality,
IsHomepage,
FetchTimeMs,
UpdateTimestamp,
@@ -40,9 +41,10 @@ pub enum Signal {
Region,
}
-const ALL_SIGNALS: [Signal; 7] = [
+const ALL_SIGNALS: [Signal; 8] = [
Signal::Bm25,
Signal::HostCentrality,
+ Signal::PageCentrality,
Signal::IsHomepage,
Signal::FetchTimeMs,
Signal::UpdateTimestamp,
@@ -54,7 +56,8 @@ impl Signal {
fn from_field(field: &Field) -> Option<Self> {
match field {
Field::IsHomepage => Some(Signal::IsHomepage),
- Field::Centrality => Some(Signal::HostCentrality),
+ Field::HostCentrality => Some(Signal::HostCentrality),
+ Field::PageCentrality => Some(Signal::PageCentrality),
Field::FetchTimeMs => Some(Signal::FetchTimeMs),
Field::LastUpdated => Some(Signal::UpdateTimestamp),
Field::NumTrackers => Some(Signal::NumTrackers),
@@ -80,7 +83,7 @@ impl Signal {
) -> f64 {
match self {
Signal::Bm25 => bm25 as f64,
- Signal::HostCentrality => {
+ Signal::HostCentrality | Signal::PageCentrality => {
fastfield.unwrap().get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64
}
Signal::IsHomepage => fastfield.unwrap().get_val(doc as u64) as f64,
@@ -130,7 +133,8 @@ impl Signal {
fn default_coefficient(&self) -> f64 {
match self {
Signal::Bm25 => 1.0,
- Signal::HostCentrality => 1024.0,
+ Signal::HostCentrality => 512.0,
+ Signal::PageCentrality => 1024.0,
Signal::IsHomepage => 0.1,
Signal::FetchTimeMs => 0.1,
Signal::UpdateTimestamp => 80.0,
diff --git a/src/schema.rs b/src/schema.rs
index c4dc03d2..85ce550a 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -32,10 +32,12 @@ pub enum Field {
Url,
Site,
Domain,
- DomainIfHomepage, // this field is only set if the webpage is the homepage for the site. Allows us to boost
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ DomainIfHomepage,
IsHomepage,
BacklinkText,
- Centrality,
+ HostCentrality,
+ PageCentrality,
FetchTimeMs,
PrimaryImage,
LastUpdated,
@@ -48,7 +50,7 @@ pub enum Field {
NumDescriptionTokens,
}
-pub static ALL_FIELDS: [Field; 22] = [
+pub static ALL_FIELDS: [Field; 23] = [
Field::Title,
Field::CleanBody,
Field::StemmedTitle,
@@ -60,7 +62,8 @@ pub static ALL_FIELDS: [Field; 22] = [
Field::DomainIfHomepage,
Field::IsHomepage,
Field::BacklinkText,
- Field::Centrality,
+ Field::HostCentrality,
+ Field::PageCentrality,
Field::FetchTimeMs,
Field::PrimaryImage,
Field::LastUpdated,
@@ -104,7 +107,12 @@ impl Field {
.set_indexed(),
),
Field::BacklinkText => IndexingOption::Text(self.default_text_options()),
- Field::Centrality => IndexingOption::Numeric(
+ Field::HostCentrality => IndexingOption::Numeric(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed(),
+ ),
+ Field::PageCentrality => IndexingOption::Numeric(
NumericOptions::default()
.set_fast(Cardinality::SingleValue)
.set_indexed(),
@@ -170,7 +178,8 @@ impl Field {
Field::Url => "url",
Field::Site => "site",
Field::BacklinkText => "backlink_text",
- Field::Centrality => "centrality",
+ Field::HostCentrality => "host_centrality",
+ Field::PageCentrality => "page_centrality",
Field::StemmedTitle => "stemmed_title",
Field::StemmedCleanBody => "stemmed_body",
Field::Domain => "domain",
@@ -201,7 +210,8 @@ impl Field {
Field::Domain => Some(1.0),
Field::AllBody => Some(0.01),
Field::BacklinkText => Some(4.0),
- Field::Centrality
+ Field::HostCentrality
+ | Field::PageCentrality
| Field::IsHomepage
| Field::PrimaryImage
| Field::FetchTimeMs
@@ -224,7 +234,7 @@ impl Field {
matches!(
self,
Field::IsHomepage
- | Field::Centrality
+ | Field::HostCentrality
| Field::FetchTimeMs
| Field::LastUpdated
| Field::NumTrackers
@@ -243,7 +253,8 @@ impl Field {
"url" => Some(Field::Url),
"site" => Some(Field::Site),
"backlink_text" => Some(Field::BacklinkText),
- "centrality" => Some(Field::Centrality),
+ "host_centrality" => Some(Field::HostCentrality),
+ "page_centrality" => Some(Field::PageCentrality),
"stemmed_title" => Some(Field::StemmedTitle),
"stemmed_body" => Some(Field::StemmedCleanBody),
"domain" => Some(Field::Domain),
diff --git a/src/searcher.rs b/src/searcher.rs
index 79293ba8..b58c3149 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -26,8 +26,8 @@ use crate::image_store::Image;
use crate::index::Index;
use crate::inverted_index::InvertedIndexSearchResult;
use crate::query::Query;
-use crate::ranking::goggles::{self, SignalAggregator};
-use crate::ranking::Ranker;
+use crate::ranking::goggles;
+use crate::ranking::{Ranker, SignalAggregator};
use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{Error, Result};
@@ -173,7 +173,7 @@ impl SearchResult {
#[cfg(test)]
mod tests {
- use crate::webpage::Webpage;
+ use crate::webpage::{Html, Webpage};
use super::*;
@@ -186,8 +186,9 @@ mod tests {
for i in 0..NUM_WEBSITES {
index
- .insert(Webpage::new(
- r#"
+ .insert(Webpage {
+ html: Html::parse(
+ r#"
<html>
<head>
<title>Example website</title>
@@ -197,12 +198,15 @@ mod tests {
</body>
</html>
"#,
- &format!("https://www.{i}.com"),
- vec![],
- (NUM_WEBSITES - i) as f64,
- 500,
- ))
- .expect("failed to parse webpage");
+ &format!("https://www.{i}.com"),
+ ),
+ backlinks: vec![],
+ host_centrality: (NUM_WEBSITES - i) as f64,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ })
+ .expect("failed to insert webpage");
}
index.commit().unwrap();
diff --git a/src/snippet.rs b/src/snippet.rs
index 59b2d455..209fc730 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -356,11 +356,8 @@ Survey in 2016, 2017, and 2018."#;
TEST_TEXT
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -396,11 +393,8 @@ Survey in 2016, 2017, and 2018."#;
TEST_TEXT
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
@@ -591,11 +585,8 @@ Survey in 2016, 2017, and 2018."#;
TEST_TEXT
),
"https://www.example.com",
- vec![],
- 1.0,
- 500,
))
- .expect("failed to parse webpage");
+ .expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 0dd85919..b9f1d891 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -118,27 +118,23 @@ pub struct StoredPrimaryImage {
pub struct Webpage {
pub html: Html,
pub backlinks: Vec<Link>,
- pub centrality: f64,
+ pub host_centrality: f64,
+ pub page_centrality: f64,
pub fetch_time_ms: u64,
pub primary_image: Option<StoredPrimaryImage>,
}
impl Webpage {
#[cfg(test)]
- pub fn new(
- html: &str,
- url: &str,
- backlinks: Vec<Link>,
- centrality: f64,
- fetch_time_ms: u64,
- ) -> Self {
+ pub fn new(html: &str, url: &str) -> Self {
let html = Html::parse(html, url);
Self {
html,
- backlinks,
- centrality,
- fetch_time_ms,
+ backlinks: Vec::new(),
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 0,
primary_image: None,
}
}
@@ -179,9 +175,16 @@ impl Webpage {
doc.add_u64(
schema
- .get_field(Field::Centrality.as_str())
- .expect("Failed to get centrality field"),
- (self.centrality * CENTRALITY_SCALING as f64) as u64,
+ .get_field(Field::HostCentrality.as_str())
+ .expect("Failed to get host_centrality field"),
+ (self.host_centrality * CENTRALITY_SCALING as f64) as u64,
+ );
+
+ doc.add_u64(
+ schema
+ .get_field(Field::PageCentrality.as_str())
+ .expect("Failed to get page_centrality field"),
+ (self.page_centrality * CENTRALITY_SCALING as f64) as u64,
);
doc.add_u64(
@@ -577,7 +580,8 @@ impl Html {
doc.add_u64(tantivy_field, description.tokens.len() as u64)
}
Field::BacklinkText
- | Field::Centrality
+ | Field::HostCentrality
+ | Field::PageCentrality
| Field::FetchTimeMs
| Field::Region
| Field::PrimaryImage => {}