web search done right
Ftr/page centrality (#55)
Mikkel Denker8c9ffede3068
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex c492c6bf..d4aa3b48 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,5 +1,5 @@warc_paths_file = "warc.paths"-centrality_store_path = "./data/centrality/host"+centrality_store_path = "./data/centrality"limit_warc_files = 100download_images = falsebatch_size = 10diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.tomlindex 1ef3c6f3..9d5ca99c 100644--- a/configs/indexer/profile.toml+++ b/configs/indexer/profile.toml@@ -1,9 +1,8 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"-limit_warc_files = 3download_images = false[warc_source]type = "Local"folder = "./data/warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz", "CC-MAIN-20220116093137-20220116123137-00049_1.warc.gz", "CC-MAIN-20220116093137-20220116123137-00049_2.warc.gz"]+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]diff --git a/data.tar.gz b/data.tar.gzindex bc1f8c04..9ec479b1 100644--- a/data.tar.gz+++ b/data.tar.gz@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:4aa671bf426371399436ef6753e0ac4c3b398967df2c9487690ed8b4a6bd9208-size 896036141+oid sha256:d770c6f965cdc6dbc2bac3c97bb413b9c820b718eebfee04053b579fd451e8e4+size 1132332917diff --git a/justfile b/justfileindex ce18a18d..d6a4ae07 100644--- a/justfile+++ b/justfile@@ -2,9 +2,9 @@rm -rf datatar -zxvf data.tar.gz-@flamegraph:+@profile-indexer:sudo rm -rf data/index- cargo flamegraph --root -- indexer local configs/indexer/local.toml+ cargo flamegraph --root -- indexer local configs/indexer/profile.toml@worker:cargo run --release -- webgraph worker configs/webgraph/worker.tomldiff --git a/src/bangs.rs b/src/bangs.rsindex 8d959b9b..21a86c22 100644--- a/src/bangs.rs+++ b/src/bangs.rs@@ -125,7 +125,7 @@ mod tests {use tantivy::tokenizer::TokenizerManager;- use crate::{ranking::goggles::SignalAggregator, schema::create_schema};+ use crate::{ranking::SignalAggregator, schema::create_schema};use super::*;diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 6a151e2e..cc095839 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -49,14 +49,19 @@ struct Job {}struct IndexingWorker {- centrality_store: CentralityStore,+ host_centrality_store: CentralityStore,+ page_centrality_store: CentralityStore,webgraph: Option<Webgraph>,}impl IndexingWorker {fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {+ let host_centrality_path = Path::new(¢rality_store_path).join("host");+ let page_centrality_path = Path::new(¢rality_store_path).join("full");+Self {- centrality_store: CentralityStore::new(centrality_store_path),+ host_centrality_store: CentralityStore::new(host_centrality_path),+ page_centrality_store: CentralityStore::new(page_centrality_path),webgraph: webgraph_path.map(|path| {WebgraphBuilder::new(path).with_full_graph()@@ -113,10 +118,17 @@ impl Map<IndexingWorker, FrozenIndex> for Job {.collect()}).unwrap_or_else(Vec::new);- let centrality = worker- .centrality_store++ let host_centrality = worker+ .host_centrality_store.get(html.url().host_without_specific_subdomains()).unwrap_or_default();++ let page_centrality = worker+ .page_centrality_store+ .get(html.url().raw())+ .unwrap_or_default();+let fetch_time_ms = record.metadata.fetch_time_ms as u64;trace!("inserting webpage: {:?}", html.url());@@ -127,7 +139,8 @@ impl Map<IndexingWorker, FrozenIndex> for Job {let webpage = Webpage {html,backlinks,- centrality,+ page_centrality,+ host_centrality,fetch_time_ms,primary_image: None,};diff --git a/src/index.rs b/src/index.rsindex 0021831f..06484075 100644--- a/src/index.rs+++ b/src/index.rs@@ -156,7 +156,7 @@ impl Index {fn maybe_insert_primary_image(&mut self, webpage: &mut Webpage) {match webpage- .centrality+ .host_centrality.partial_cmp(&IMAGE_WEBPAGE_CENTRALITY_THRESHOLD){None | Some(Ordering::Greater) => {}@@ -291,7 +291,7 @@ impl From<Index> for FrozenIndex {#[cfg(test)]mod tests {- use crate::ranking::{goggles::SignalAggregator, Ranker};+ use crate::ranking::{Ranker, SignalAggregator};use super::*;@@ -316,11 +316,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().unwrap();@@ -373,11 +370,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().unwrap();diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 47f5bbf8..77cf2741 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -363,7 +363,8 @@ impl From<Document> for RetrievedWebpage {}}Field::BacklinkText- | Field::Centrality+ | Field::HostCentrality+ | Field::PageCentrality| Field::Site| Field::StemmedTitle| Field::CleanBody@@ -388,8 +389,8 @@ mod tests {use maplit::hashset;use crate::{- ranking::{goggles::SignalAggregator, Ranker},- webpage::{region::RegionCount, Link},+ ranking::{Ranker, SignalAggregator},+ webpage::{region::RegionCount, Html, Link},};use super::*;@@ -429,11 +430,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -471,11 +469,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -512,11 +507,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -553,11 +545,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -593,15 +582,13 @@ mod tests {"#),"https://www.a.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -611,17 +598,20 @@ mod tests {</body></html>"#+ ),+ "https://www.b.com",),- "https://www.b.com",- vec![Link {+ backlinks: vec![Link {source: "https://www.a.com".to_string().into(),destination: "https://www.b.com".to_string().into(),text: "B site is great".to_string(),}],- 1.0,- 500,- ))- .expect("failed to parse webpage");+ host_centrality: 1.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");@@ -665,11 +655,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");}index.commit().expect("failed to commit index");@@ -707,11 +694,8 @@ mod tests {"#),"https://www.dr.dk",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -740,11 +724,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");let mut index2 = InvertedIndex::temporary().expect("Unable to open index");@@ -763,11 +744,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");let mut index = index1.merge(index2);index.commit().unwrap();@@ -823,11 +801,8 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -838,69 +813,6 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");}- #[test]- fn fetch_time_ranking() {- let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse(- "test",- index.schema(),- index.tokenizers(),- &SignalAggregator::default(),- )- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());-- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT}- </body>- </html>- "#- ),- "https://www.first.com",- vec![],- 1.0,- 0,- ))- .expect("failed to parse webpage");- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT}- </body>- </html>- "#- ),- "https://www.second.com",- vec![],- 1.0,- 5000,- ))- .expect("failed to parse webpage");- index.commit().expect("failed to commit index");-- let result = index- .search(&query, ranker.collector())- .expect("Search failed");- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");- }-#[test]fn only_show_primary_images_when_relevant() {let mut index = InvertedIndex::temporary().expect("Unable to open index");@@ -923,14 +835,11 @@ mod tests {"#),"https://www.example.com",- vec![],- 1.0,- 500,);let uuid = uuid::uuid!("00000000-0000-0000-0000-ffff00000000");webpage.set_primary_image(uuid, webpage.html.primary_image().unwrap());- index.insert(webpage).expect("failed to parse webpage");+ index.insert(webpage).expect("failed to insert webpage");index.commit().expect("failed to commit index");let query = Query::parse(diff --git a/src/query/mod.rs b/src/query/mod.rsindex da9c07fb..1d943ae8 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -14,11 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{- ranking::goggles::{Goggle, SignalAggregator},- schema::Field,- Result,-};+use crate::{ranking::goggles::Goggle, ranking::SignalAggregator, schema::Field, Result};use std::{collections::HashMap, sync::Arc};use tantivy::{query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone},@@ -312,11 +308,8 @@ mod tests {</html>"#,"https://www.first.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.insert(Webpage::new(r#"@@ -330,11 +323,8 @@ mod tests {</html>"#,"https://www.second.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let result = index@@ -362,11 +352,8 @@ mod tests {</html>"#,"https://www.first.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.insert(Webpage::new(r#"@@ -380,11 +367,8 @@ mod tests {</html>"#,"https://www.second.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let query = Query::parse(@@ -450,11 +434,8 @@ mod tests {</html>"#,"https://www.first.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.insert(Webpage::new(r#"@@ -468,11 +449,8 @@ mod tests {</html>"#,"https://www.second.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let query = Query::parse(@@ -508,11 +486,8 @@ mod tests {</html>"#,"https://www.first.com/forum",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.insert(Webpage::new(r#"@@ -526,11 +501,8 @@ mod tests {</html>"#,"https://www.second.com",- vec![],- 1.0,- 0,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let query = Query::parse(diff --git a/src/query/parser.rs b/src/query/parser.rsindex 2cbccfa9..879f9fa1 100644--- a/src/query/parser.rs+++ b/src/query/parser.rs@@ -22,7 +22,7 @@ use tantivy::{use crate::{bangs::BANG_PREFIX,- ranking::goggles::FieldBoost,+ ranking::FieldBoost,schema::{Field, ALL_FIELDS},};diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 08776159..40505a7a 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -14,9 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-mod ast;+pub mod ast;mod pattern_query;-mod signal;use std::convert::TryFrom;@@ -28,12 +27,13 @@ use tantivy::{tokenizer::TextAnalyzer,};-pub use self::signal::*;use self::{ast::{RawAction, RawGoggle, RawInstruction, RawPatternOption, RawPatternPart},pattern_query::PatternQuery,};+use super::signal::SignalAggregator;+pub fn parse(goggle: &str) -> Result<Goggle> {let raw_goggle = ast::parse(goggle)?;@@ -302,7 +302,12 @@ impl Instruction {#[cfg(test)]mod tests {- use crate::{index::Index, schema::create_schema, searcher::Searcher, webpage::Webpage};+ use crate::{+ index::Index,+ schema::create_schema,+ searcher::Searcher,+ webpage::{Html, Webpage},+ };use super::*;const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -312,9 +317,10 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website A</title>@@ -325,17 +331,21 @@ mod tests {</body></html>"#+ ),+ "https://www.a.com",),- "https://www.a.com",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -345,13 +355,16 @@ mod tests {</body></html>"#+ ),+ "https://www.b.com",),- "https://www.b.com",- vec![],- 0.0001,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0001,+ page_centrality: 0.0,+ primary_image: None,+ fetch_time_ms: 500,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -424,9 +437,10 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website A</title>@@ -437,17 +451,21 @@ mod tests {</body></html>"#+ ),+ "https://www.a.com/this/is/a/pattern",),- "https://www.a.com/this/is/a/pattern",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -457,13 +475,16 @@ mod tests {</body></html>"#+ ),+ "https://www.b.com/this/is/b/pattern",),- "https://www.b.com/this/is/b/pattern",- vec![],- 0.0001,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0001,+ page_centrality: 0.0,+ primary_image: None,+ fetch_time_ms: 500,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -515,9 +536,10 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website A</title>@@ -528,17 +550,21 @@ mod tests {</body></html>"#+ ),+ "https://www.a.com/this/is/a/pattern",),- "https://www.a.com/this/is/a/pattern",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -548,17 +574,21 @@ mod tests {</body></html>"#+ ),+ "https://www.b.com/this/is/b/pattern",),- "https://www.b.com/this/is/b/pattern",- vec![],- 0.0001,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0001,+ page_centrality: 0.0,+ primary_image: None,+ fetch_time_ms: 500,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -568,13 +598,16 @@ mod tests {</body></html>"#+ ),+ "https://www.c.com/this/is/c/pattern",),- "https://www.c.com/this/is/c/pattern",- vec![],- 0.0001,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0001,+ page_centrality: 0.0,+ primary_image: None,+ fetch_time_ms: 500,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 79c1312c..2ec08a5a 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -21,7 +21,7 @@ use chrono::Utc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};use tantivy::{DocId, Score, SegmentReader};-use super::goggles::SignalAggregator;+use super::SignalAggregator;pub(crate) struct InitialScoreTweaker {region_count: Arc<RegionCount>,diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex ef508c02..92971a81 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -18,6 +18,7 @@ mod bm25;pub mod centrality_store;pub mod goggles;mod initial;+pub mod signal;use std::sync::Arc;@@ -29,7 +30,7 @@ use crate::{webpage::region::{Region, RegionCount},};-use self::goggles::SignalAggregator;+pub use self::signal::*;pub struct Ranker {region_count: Arc<RegionCount>,@@ -80,19 +81,20 @@ mod tests {use crate::{index::Index,searcher::Searcher,- webpage::{Link, Webpage},+ webpage::{Html, Link, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";#[test]- fn harmonic_ranking() {+ fn host_centrality_ranking() {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website A</title>@@ -103,17 +105,87 @@ mod tests {</body></html>"#+ ),+ "https://www.a.com",),- "https://www.a.com",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.b.com",+ ),+ backlinks: vec![],+ host_centrality: 5.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");++ index.commit().expect("failed to commit index");+ let searcher = Searcher::from(index);+ let result = searcher+ .search("example", None, None, None)+ .expect("Search failed")+ .into_websites()+ .unwrap();+ assert_eq!(result.webpages.documents.len(), 2);+ assert_eq!(result.webpages.documents[0].url, "https://www.b.com");+ assert_eq!(result.webpages.documents[1].url, "https://www.a.com");+ }++ #[test]+ fn page_centrality_ranking() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ <body>+ {CONTENT}+ example example example+ </body>+ </html>+ "#+ ),+ "https://www.a.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -123,13 +195,16 @@ mod tests {</body></html>"#+ ),+ "https://www.b.com",),- "https://www.b.com",- vec![],- 5.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 5.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -148,9 +223,10 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>DR Homepage</title>@@ -160,17 +236,21 @@ mod tests {</body></html>"#+ ),+ "https://www.dr.dk",),- "https://www.dr.dk",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Subsite dr</title>@@ -180,17 +260,21 @@ mod tests {</body></html>"#+ ),+ "https://www.dr.dk/whatever",),- "https://www.dr.dk/whatever",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Website B</title>@@ -198,13 +282,16 @@ mod tests {dr and some other text {CONTENT} dk</html>"#+ ),+ "https://www.b.com",),- "https://www.b.com",- vec![],- 0.003,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.003,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -223,9 +310,10 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Title</title>@@ -236,17 +324,21 @@ mod tests {</body></html>"#+ ),+ "https://www.old.com",),- "https://www.old.com",- vec![],- 0.092,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.092,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Title</title>@@ -257,13 +349,16 @@ mod tests {</body></html>"#+ ),+ "https://www.new.com",),- "https://www.new.com",- vec![],- 0.09,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 0.09,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -281,8 +376,9 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Test site</title>@@ -292,15 +388,18 @@ mod tests {</body></html>"#,- "https://www.first.com",- vec![],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ "https://www.first.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(r#"<html><head><script>@@ -324,12 +423,14 @@ mod tests {</body></html>"#,- "https://www.second.com",- vec![],- 0.003,- 500,- ))- .expect("failed to parse webpage");+ "https://www.second.com"),+ backlinks: vec![],+ host_centrality: 0.003,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -348,8 +449,9 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Test site</title>@@ -359,19 +461,23 @@ mod tests {</body></html>"#,- "https://www.first.com",- vec![Link {+ "https://www.first.com",+ ),+ backlinks: vec![Link {source: "https://www.second.com".to_string().into(),destination: "https://www.first.com".to_string().into(),text: "test this is the best test site".to_string(),}],- 0.0,- 500,- ))- .expect("failed to parse webpage");+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Second test site</title>@@ -381,12 +487,15 @@ mod tests {</body></html>"#,- "https://www.second.com",- vec![],- 0.003,- 500,- ))- .expect("failed to parse webpage");+ "https://www.second.com",+ ),+ backlinks: vec![],+ host_centrality: 0.003,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -405,8 +514,9 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Test website</title>@@ -416,16 +526,20 @@ mod tests {</body></html>"#,- "https://www.body.com",- vec![],- 1.0,- 20,- ))- .expect("failed to parse webpage");+ "https://www.body.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 20,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Example website</title>@@ -435,16 +549,20 @@ mod tests {</body></html>"#,- "https://www.title.com",- vec![],- 1.0,- 20,- ))- .expect("failed to parse webpage");+ "https://www.title.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 20,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Example website</title>@@ -454,12 +572,15 @@ mod tests {</body></html>"#,- "https://www.centrality.com",- vec![],- 1.0002,- 500,- ))- .expect("failed to parse webpage");+ "https://www.centrality.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0002,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().unwrap();@@ -524,9 +645,10 @@ mod tests {fn term_proximity_ranking() {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Test website</title>@@ -536,17 +658,21 @@ mod tests {</body></html>"#+ ),+ "https://www.first.com",),- "https://www.first.com",- vec![],- 1.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Test website</title>@@ -556,17 +682,21 @@ mod tests {</body></html>"#+ ),+ "https://www.third.com",),- "https://www.third.com",- vec![],- 1.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index- .insert(Webpage::new(- &format!(- r#"+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Test website</title>@@ -576,13 +706,16 @@ mod tests {</body></html>"#+ ),+ "https://www.second.com",),- "https://www.second.com",- vec![],- 1.0,- 500,- ))- .expect("failed to parse webpage");+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::new(index, None, None);@@ -599,4 +732,72 @@ mod tests {assert_eq!(result.documents[1].url, "https://www.second.com");assert_eq!(result.documents[2].url, "https://www.third.com");}++ #[test]+ fn fetch_time_ranking() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.first.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 0,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.second.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 5000,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = Searcher::new(index, None, None);++ let result = searcher+ .search("test", None, None, None)+ .expect("Search failed")+ .into_websites()+ .unwrap()+ .webpages;++ assert_eq!(result.num_docs, 2);+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.first.com");+ assert_eq!(result.documents[1].url, "https://www.second.com");+ }}diff --git a/src/ranking/goggles/signal.rs b/src/ranking/signal.rssimilarity index 96%rename from src/ranking/goggles/signal.rsrename to src/ranking/signal.rsindex b3958857..531b35bf 100644--- a/src/ranking/goggles/signal.rs+++ b/src/ranking/signal.rs@@ -27,12 +27,13 @@ use crate::{webpage::region::{Region, RegionCount},};-use super::ast::{RawAlteration, Target};+use crate::ranking::goggles::ast::{RawAlteration, Target};#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub enum Signal {Bm25,HostCentrality,+ PageCentrality,IsHomepage,FetchTimeMs,UpdateTimestamp,@@ -40,9 +41,10 @@ pub enum Signal {Region,}-const ALL_SIGNALS: [Signal; 7] = [+const ALL_SIGNALS: [Signal; 8] = [Signal::Bm25,Signal::HostCentrality,+ Signal::PageCentrality,Signal::IsHomepage,Signal::FetchTimeMs,Signal::UpdateTimestamp,@@ -54,7 +56,8 @@ impl Signal {fn from_field(field: &Field) -> Option<Self> {match field {Field::IsHomepage => Some(Signal::IsHomepage),- Field::Centrality => Some(Signal::HostCentrality),+ Field::HostCentrality => Some(Signal::HostCentrality),+ Field::PageCentrality => Some(Signal::PageCentrality),Field::FetchTimeMs => Some(Signal::FetchTimeMs),Field::LastUpdated => Some(Signal::UpdateTimestamp),Field::NumTrackers => Some(Signal::NumTrackers),@@ -80,7 +83,7 @@ impl Signal {) -> f64 {match self {Signal::Bm25 => bm25 as f64,- Signal::HostCentrality => {+ Signal::HostCentrality | Signal::PageCentrality => {fastfield.unwrap().get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64}Signal::IsHomepage => fastfield.unwrap().get_val(doc as u64) as f64,@@ -130,7 +133,8 @@ impl Signal {fn default_coefficient(&self) -> f64 {match self {Signal::Bm25 => 1.0,- Signal::HostCentrality => 1024.0,+ Signal::HostCentrality => 512.0,+ Signal::PageCentrality => 1024.0,Signal::IsHomepage => 0.1,Signal::FetchTimeMs => 0.1,Signal::UpdateTimestamp => 80.0,diff --git a/src/schema.rs b/src/schema.rsindex c4dc03d2..85ce550a 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -32,10 +32,12 @@ pub enum Field {Url,Site,Domain,- DomainIfHomepage, // this field is only set if the webpage is the homepage for the site. Allows us to boost+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost+ DomainIfHomepage,IsHomepage,BacklinkText,- Centrality,+ HostCentrality,+ PageCentrality,FetchTimeMs,PrimaryImage,LastUpdated,@@ -48,7 +50,7 @@ pub enum Field {NumDescriptionTokens,}-pub static ALL_FIELDS: [Field; 22] = [+pub static ALL_FIELDS: [Field; 23] = [Field::Title,Field::CleanBody,Field::StemmedTitle,@@ -60,7 +62,8 @@ pub static ALL_FIELDS: [Field; 22] = [Field::DomainIfHomepage,Field::IsHomepage,Field::BacklinkText,- Field::Centrality,+ Field::HostCentrality,+ Field::PageCentrality,Field::FetchTimeMs,Field::PrimaryImage,Field::LastUpdated,@@ -104,7 +107,12 @@ impl Field {.set_indexed(),),Field::BacklinkText => IndexingOption::Text(self.default_text_options()),- Field::Centrality => IndexingOption::Numeric(+ Field::HostCentrality => IndexingOption::Numeric(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed(),+ ),+ Field::PageCentrality => IndexingOption::Numeric(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed(),@@ -170,7 +178,8 @@ impl Field {Field::Url => "url",Field::Site => "site",Field::BacklinkText => "backlink_text",- Field::Centrality => "centrality",+ Field::HostCentrality => "host_centrality",+ Field::PageCentrality => "page_centrality",Field::StemmedTitle => "stemmed_title",Field::StemmedCleanBody => "stemmed_body",Field::Domain => "domain",@@ -201,7 +210,8 @@ impl Field {Field::Domain => Some(1.0),Field::AllBody => Some(0.01),Field::BacklinkText => Some(4.0),- Field::Centrality+ Field::HostCentrality+ | Field::PageCentrality| Field::IsHomepage| Field::PrimaryImage| Field::FetchTimeMs@@ -224,7 +234,7 @@ impl Field {matches!(self,Field::IsHomepage- | Field::Centrality+ | Field::HostCentrality| Field::FetchTimeMs| Field::LastUpdated| Field::NumTrackers@@ -243,7 +253,8 @@ impl Field {"url" => Some(Field::Url),"site" => Some(Field::Site),"backlink_text" => Some(Field::BacklinkText),- "centrality" => Some(Field::Centrality),+ "host_centrality" => Some(Field::HostCentrality),+ "page_centrality" => Some(Field::PageCentrality),"stemmed_title" => Some(Field::StemmedTitle),"stemmed_body" => Some(Field::StemmedCleanBody),"domain" => Some(Field::Domain),diff --git a/src/searcher.rs b/src/searcher.rsindex 79293ba8..b58c3149 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -26,8 +26,8 @@ use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::InvertedIndexSearchResult;use crate::query::Query;-use crate::ranking::goggles::{self, SignalAggregator};-use crate::ranking::Ranker;+use crate::ranking::goggles;+use crate::ranking::{Ranker, SignalAggregator};use crate::webpage::region::Region;use crate::webpage::Url;use crate::{Error, Result};@@ -173,7 +173,7 @@ impl SearchResult {#[cfg(test)]mod tests {- use crate::webpage::Webpage;+ use crate::webpage::{Html, Webpage};use super::*;@@ -186,8 +186,9 @@ mod tests {for i in 0..NUM_WEBSITES {index- .insert(Webpage::new(- r#"+ .insert(Webpage {+ html: Html::parse(+ r#"<html><head><title>Example website</title>@@ -197,12 +198,15 @@ mod tests {</body></html>"#,- &format!("https://www.{i}.com"),- vec![],- (NUM_WEBSITES - i) as f64,- 500,- ))- .expect("failed to parse webpage");+ &format!("https://www.{i}.com"),+ ),+ backlinks: vec![],+ host_centrality: (NUM_WEBSITES - i) as f64,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ })+ .expect("failed to insert webpage");}index.commit().unwrap();diff --git a/src/snippet.rs b/src/snippet.rsindex 59b2d455..209fc730 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -356,11 +356,8 @@ Survey in 2016, 2017, and 2018."#;TEST_TEXT),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -396,11 +393,8 @@ Survey in 2016, 2017, and 2018."#;TEST_TEXT),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);@@ -591,11 +585,8 @@ Survey in 2016, 2017, and 2018."#;TEST_TEXT),"https://www.example.com",- vec![],- 1.0,- 500,))- .expect("failed to parse webpage");+ .expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = Searcher::from(index);diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 0dd85919..b9f1d891 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -118,27 +118,23 @@ pub struct StoredPrimaryImage {pub struct Webpage {pub html: Html,pub backlinks: Vec<Link>,- pub centrality: f64,+ pub host_centrality: f64,+ pub page_centrality: f64,pub fetch_time_ms: u64,pub primary_image: Option<StoredPrimaryImage>,}impl Webpage {#[cfg(test)]- pub fn new(- html: &str,- url: &str,- backlinks: Vec<Link>,- centrality: f64,- fetch_time_ms: u64,- ) -> Self {+ pub fn new(html: &str, url: &str) -> Self {let html = Html::parse(html, url);Self {html,- backlinks,- centrality,- fetch_time_ms,+ backlinks: Vec::new(),+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 0,primary_image: None,}}@@ -179,9 +175,16 @@ impl Webpage {doc.add_u64(schema- .get_field(Field::Centrality.as_str())- .expect("Failed to get centrality field"),- (self.centrality * CENTRALITY_SCALING as f64) as u64,+ .get_field(Field::HostCentrality.as_str())+ .expect("Failed to get host_centrality field"),+ (self.host_centrality * CENTRALITY_SCALING as f64) as u64,+ );++ doc.add_u64(+ schema+ .get_field(Field::PageCentrality.as_str())+ .expect("Failed to get page_centrality field"),+ (self.page_centrality * CENTRALITY_SCALING as f64) as u64,);doc.add_u64(@@ -577,7 +580,8 @@ impl Html {doc.add_u64(tantivy_field, description.tokens.len() as u64)}Field::BacklinkText- | Field::Centrality+ | Field::HostCentrality+ | Field::PageCentrality| Field::FetchTimeMs| Field::Region| Field::PrimaryImage => {}