web search done right
Ftr/trust centrality (#63)
Mikkel Denkerfd97641eb864
diff --git a/Cargo.lock b/Cargo.lockindex 5d383a45..42fb854b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -294,6 +294,19 @@ dependencies = ["crunchy",]+[[package]]+name = "boomphf"+version = "0.5.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4e8fb7f38fef59c32549861151d63a6190865e60cf690340c13e3d7178b42a2f"+dependencies = [+ "crossbeam-utils",+ "log",+ "rayon",+ "serde",+ "wyhash",+]+[[package]]name = "bstr"version = "0.2.17"@@ -679,6 +692,7 @@ dependencies = ["axum-extra","base64","bincode",+ "boomphf","bzip2","chrono","clap 3.1.18",@@ -3529,6 +3543,15 @@ dependencies = ["winapi",]+[[package]]+name = "wyhash"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295"+dependencies = [+ "rand_core 0.6.3",+]+[[package]]name = "zstd-sys"version = "2.0.1+zstd.1.5.2"diff --git a/Cargo.toml b/Cargo.tomlindex fa4d1402..8cb74bd4 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -20,7 +20,7 @@ itertools = "0.10.3"lru = "0.7.6"rayon = "1.5.3"reqwest = { version = "0.11.10", features = ["blocking", "stream"] }-serde = "1.0.137"+serde = { version = "1.0.137", features = ["rc"] }serde_json = "1.0.81"tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "a451f6d60dc4cb751457d11a3cc9d038d9b63713" }thiserror = "1.0.31"@@ -58,6 +58,7 @@ rust-stemmers = "1.2.0"min-max-heap = "1.3.0"base64 = "0.13.0"async-trait = "0.1.57"+boomphf = { version = "0.5.9", features = ["serde"] }[dev-dependencies]criterion = "0.3.6"diff --git a/README.md b/README.mdindex b95de1b8..3a9ed67f 100644--- a/README.md+++ b/README.md@@ -23,7 +23,7 @@ Cuely is an open source web search engine hosted at [cuely.io](https://cuely.io/* Regional search* Customize how signals are combined during search for the final search result* Use [goggles](https://brave.com/static-assets/files/goggles.pdf) to almost endlessly customize your search results.-* (Coming soon) Prioritize links (centrality) from the sites you trust.+* Prioritize links (centrality) from the sites you trust.# π©βπ» SetupWe recommend everyone to use the hosted version at [cuely.io](https://cuely.io/), but you can also follow the steps outlined in [CONTRIBUTING.md](CONTRIBUTING.md) to setup the engine locally.diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rsindex 7d85dc5d..7ad20961 100644--- a/benches/search-preindexed-goggle.rs+++ b/benches/search-preindexed-goggle.rs@@ -30,7 +30,7 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();- let searcher = LocalSearcher::new(index, None, None);+ let searcher = LocalSearcher::new(index);let goggle = include_str!("../testcases/goggles/hacker_news.goggle");// for _ in 0..10 {diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rsindex bfd70f3f..1a845564 100644--- a/benches/search-preindexed.rs+++ b/benches/search-preindexed.rs@@ -29,7 +29,7 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();- let searcher = LocalSearcher::new(index, None, None);+ let searcher = LocalSearcher::new(index);// for _ in 0..100 {bench!("the", searcher, c);diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 4e7c19e9..81f75d4d 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,13 +1,11 @@-warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"-limit_warc_files = 3-host_centrality_threshold = 0.0000000001+limit_warc_files = 1download_images = false[warc_source]-type = "Local"-folder = "./data/warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]-# type = "HTTP"-# base_url = "https://data.commoncrawl.org/"-# warc_paths_file = "warc.paths"+# type = "Local"+# folder = "./data/warc_files"+# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "warc.paths"diff --git a/examples/print_approx_harmonic_centrality.rs b/examples/print_approx_harmonic_centrality.rsnew file mode 100644index 00000000..078e1a8f--- /dev/null+++ b/examples/print_approx_harmonic_centrality.rs@@ -0,0 +1,165 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ cmp,+ collections::{BinaryHeap, HashMap},+};++use cuely::{+ ranking::centrality_store::CentralityStore,+ webgraph::{centrality::approximate_harmonic::Scorer, Node, WebgraphBuilder},+ webpage::Url,+};++struct ScoredNode {+ score: f64,+ id: u64,+}++impl PartialOrd for ScoredNode {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.score.partial_cmp(&other.score)+ }+}++impl PartialEq for ScoredNode {+ fn eq(&self, other: &Self) -> bool {+ self.score == other.score+ }+}++impl Ord for ScoredNode {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)+ }+}++impl Eq for ScoredNode {}++fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[u64]) -> Vec<ScoredNode> {+ let mut top_nodes = BinaryHeap::with_capacity(top_n);++ for node in nodes {+ top_nodes.push(cmp::Reverse(ScoredNode {+ score: scorer.score(*node),+ id: *node,+ }));++ while top_nodes.len() > top_n {+ top_nodes.pop();+ }+ }++ let mut nodes = Vec::new();+ while let Some(node) = top_nodes.pop() {+ nodes.push(node.0);+ }++ nodes+}++fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<u64, Node>) {+ for (i, node) in nodes.iter().rev().enumerate() {+ println!(+ "{i} \t {:.3} \t {} ",+ node.score,+ id2node.get(&node.id).unwrap().name+ );+ }+}++fn print_top_nodes(+ liked_sites: &[&str],+ top_n: usize,+ store: &CentralityStore,+ id2node: &HashMap<u64, Node>,+ nodes: &[u64],+) {+ let liked_nodes: Vec<_> = liked_sites+ .iter()+ .map(|host| Node::from_url(&Url::from(host.to_string())))+ .collect();++ let scorer = store.approx_harmonic.scorer(&liked_nodes, &[]);+ let top_nodes = get_top_nodes(scorer, top_n, nodes);+ println!(+ "top {} sites for these liked sites {:?}",+ top_n, liked_sites+ );+ print_nodes(&top_nodes, id2node);++ println!();+ println!();+}++pub fn main() {+ const TOP_N: usize = 50;+ let graph = WebgraphBuilder::new("data/webgraph")+ .with_host_graph()+ .read_only(true)+ .open();+ let nodes: Vec<_> = graph.host.as_ref().unwrap().nodes().collect();+ let id2node: HashMap<_, _> = nodes+ .iter()+ .map(|id| (*id, graph.host.as_ref().unwrap().id2node(id).unwrap()))+ .collect();++ let store = CentralityStore::open("data/centrality");++ let mut proxy_nodes = Vec::new();+ for node in &store.approx_harmonic.proxy_nodes {+ proxy_nodes.push(id2node.get(&node.id).unwrap().name.clone());+ }++ print_top_nodes(+ &[+ "dr.dk",+ "berlingske.dk",+ "ekstrabladet.dk",+ "politikken.dk",+ "tv2.dk",+ "seoghoer.dk",+ "zetland.dk",+ ],+ TOP_N,+ &store,+ &id2node,+ &nodes,+ );++ print_top_nodes(+ &[+ "stackoverflow.com",+ "github.com",+ "arxiv.org",+ "news.ycombinator.com",+ ],+ TOP_N,+ &store,+ &id2node,+ &nodes,+ );++ print_top_nodes(&["nature.com", "who.int"], TOP_N, &store, &id2node, &nodes);+ print_top_nodes(+ &["webmd.com", "medlineplus.gov"],+ TOP_N,+ &store,+ &id2node,+ &nodes,+ );+}diff --git a/frontend/src/pages/settings/sites.astro b/frontend/src/pages/settings/sites.astroindex c9b6dadf..16047016 100644--- a/frontend/src/pages/settings/sites.astro+++ b/frontend/src/pages/settings/sites.astro@@ -13,7 +13,7 @@ import SettingsMenu from "../../components/SettingsMenu.astro";<SettingsMenu /><div class="flex w-full flex-col max-w-2xl space-y-10"><div>- <h1 class="font-medium text-2xl mb-2">Preffered Sites</h1>+ <h1 class="font-medium text-2xl mb-2">Liked Sites</h1><div class="text-sm">These sites receive a boost during search. Results from these sitesare more likely to appear in your search results.@@ -21,7 +21,7 @@ import SettingsMenu from "../../components/SettingsMenu.astro";<div class="mt-5"><divclass="grid grid-cols-[auto_1fr] w-full space-y-5"- id="preffered-list"+ id="liked-list"><div class="w-10"></div><div class="font-medium">Site Name</div>@@ -30,7 +30,7 @@ import SettingsMenu from "../../components/SettingsMenu.astro";</div><div>- <h1 class="font-medium text-2xl mb-2">Junk Sites</h1>+ <h1 class="font-medium text-2xl mb-2">Disliked Sites</h1><div class="text-sm">These sites get de-prioritized during search. Results from thesesites are less likely to appear in your search results.@@ -38,7 +38,7 @@ import SettingsMenu from "../../components/SettingsMenu.astro";<div class="mt-5"><divclass="grid grid-cols-[auto_1fr] w-full space-y-5"- id="junk-list"+ id="disliked-list"><div class="w-10"></div><div class="font-medium">Site Name</div>@@ -74,8 +74,8 @@ import SettingsMenu from "../../components/SettingsMenu.astro";window.localStorage.getItem("sites-ranking-adjustment") || "{}");- const prefferedList = document.getElementById("preffered-list");- const junkList = document.getElementById("junk-list");+ const likedList = document.getElementById("liked-list");+ const dislikedList = document.getElementById("disliked-list");const blockedList = document.getElementById("blocked-list");const BLOCKED = 0;@@ -83,11 +83,11 @@ import SettingsMenu from "../../components/SettingsMenu.astro";const MORE = 2;function addMoreRow(site) {- addRow(site, prefferedList);+ addRow(site, likedList);}function addLessRow(site) {- addRow(site, junkList);+ addRow(site, dislikedList);}function addBlockedRow(site) {diff --git a/src/entrypoint/centrality.rs b/src/entrypoint/centrality.rsindex d3e19278..76d5975d 100644--- a/src/entrypoint/centrality.rs+++ b/src/entrypoint/centrality.rs@@ -14,73 +14,20 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::HashMap, fs::File, path::Path};+use std::path::Path;-use crate::{- ranking::centrality_store::CentralityStore,- webgraph::{Webgraph, WebgraphBuilder},-};+use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};pub struct Centrality {}impl Centrality {- fn host_centrality(graph: &Webgraph) -> HashMap<String, f64> {- graph- .host_harmonic_centrality()- .into_iter()- .map(|(node, centrality)| (node.name, centrality))- .collect()- }-- fn full_centrality(graph: &Webgraph) -> HashMap<String, f64> {- graph- .harmonic_centrality()- .into_iter()- .map(|(node, centrality)| (node.name, centrality))- .collect()- }-- fn save<P: AsRef<Path>>(centrality: HashMap<String, f64>, output_path: P) {- let mut centrality_store = CentralityStore::new(output_path.as_ref());-- centrality_store.append(centrality.clone().into_iter());-- let mut centralities: Vec<_> = centrality.into_iter().collect();- centralities- .sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));-- let csv_file = File::options()- .write(true)- .create(true)- .truncate(true)- .open(output_path.as_ref().join("data.csv"))- .unwrap();- let mut wtr = csv::Writer::from_writer(csv_file);-- for (host, centrality) in centralities {- wtr.write_record(&[host, centrality.to_string()]).unwrap();- }-- wtr.flush().unwrap();- }-- fn host<P: AsRef<Path>>(graph: &Webgraph, output_path: P) {- let centrality = Self::host_centrality(graph);- Self::save(centrality, output_path);- }-- fn full<P: AsRef<Path>>(graph: &Webgraph, output_path: P) {- let centrality = Self::full_centrality(graph);- Self::save(centrality, output_path);- }-pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {let graph = WebgraphBuilder::new(webgraph_path)+ .read_only(true).with_host_graph().with_full_graph().open();- Self::host(&graph, output_path.as_ref().join("host"));- Self::full(&graph, output_path.as_ref().join("full"));+ CentralityStore::build(&graph, output_path);}}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 4861efe2..5bac3be0 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -53,19 +53,14 @@ struct Job {}struct IndexingWorker {- host_centrality_store: CentralityStore,- page_centrality_store: CentralityStore,+ centrality_store: CentralityStore,webgraph: Option<Webgraph>,}impl IndexingWorker {fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {- let host_centrality_path = Path::new(¢rality_store_path).join("host");- let page_centrality_path = Path::new(¢rality_store_path).join("full");-Self {- host_centrality_store: CentralityStore::new(host_centrality_path),- page_centrality_store: CentralityStore::new(page_centrality_path),+ centrality_store: CentralityStore::open(centrality_store_path),webgraph: webgraph_path.map(|path| {WebgraphBuilder::new(path).with_full_graph()@@ -98,7 +93,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {let name = file.split('/').last().unwrap();let path = Path::new(&job.base_path).join("warc_files").join(name);- if let Ok(file) = WarcFile::open(path) {+ if let Ok(file) = WarcFile::open(&path) {for record infile.records().flatten()@@ -110,12 +105,15 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut html = Html::parse_without_text(&record.response.body, &record.request.url);let host_centrality = worker- .host_centrality_store- .get(html.url().host_without_specific_subdomains())+ .centrality_store+ .harmonic+ .host+ .get(&html.url().host_without_specific_subdomains().to_string()).unwrap_or_default();if let Some(host_centrality_threshold) = job.host_centrality_threshold {if host_centrality < host_centrality_threshold {+ debug!("skipping due to low host_centrality value");continue;}}@@ -139,8 +137,10 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {.unwrap_or_else(Vec::new);let page_centrality = worker- .page_centrality_store- .get(html.url().raw())+ .centrality_store+ .harmonic+ .full+ .get(&html.url().raw().to_string()).unwrap_or_default();let fetch_time_ms = record.metadata.fetch_time_ms as u64;@@ -150,6 +150,13 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {trace!("title = {:?}", html.title());trace!("text = {:?}", html.clean_text());+ let node_id = worker+ .centrality_store+ .approx_harmonic+ .node2id+ .get(&Node::from_url(html.url()).into_host())+ .copied();+let mut webpage = Webpage {html,backlinks,@@ -158,6 +165,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {fetch_time_ms,primary_image: None,pre_computed_score: 0.0,+ node_id,};webpage.pre_computed_score =@@ -175,7 +183,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {index.commit().unwrap();- std::fs::remove_file(file).ok();+ std::fs::remove_file(path).unwrap();}info!("{} done", name);diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex 0102062a..c642ccb0 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -60,16 +60,14 @@ async fn async_download_all_warc_files<'a>(.unwrap();debug!("downloading warc file {}", &warc_path);let res = WarcFile::download_into_buf(source, &warc_path, &mut file).await;++ if let Err(err) = res {+ debug!("error while downloading: {:?}", err);+ }+debug!("finished downloading");- (warc_path, res)+ warc_path})).buffer_unordered(num_files)- .filter_map(|(path, res)| async move {- if res.is_ok() {- Some(path.clone())- } else {- None- }- })}diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rsindex 97fb1a00..9de385ca 100644--- a/src/entrypoint/search_server.rs+++ b/src/entrypoint/search_server.rs@@ -21,6 +21,7 @@ use crate::{entity_index::EntityIndex,index::Index,inverted_index,+ ranking::centrality_store::CentralityStore,search_prettifier::{self},searcher::{self, LocalSearcher},sonic, Result, SearchServerConfig,@@ -35,9 +36,24 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {.entity_index_path.map(|path| EntityIndex::open(path).unwrap());let bangs = config.bangs_path.map(Bangs::from_path);+ let centrality_store = config+ .centrality_store_path+ .map(|path| CentralityStore::open(path));let search_index = Index::open(config.index_path)?;- let local_searcher = LocalSearcher::new(search_index, entity_index, bangs);+ let mut local_searcher = LocalSearcher::new(search_index);++ if let Some(entity_index) = entity_index {+ local_searcher.set_entity_index(entity_index);+ }++ if let Some(bangs) = bangs {+ local_searcher.set_bangs(bangs);+ }++ if let Some(centrality_store) = centrality_store {+ local_searcher.set_centrality_store(centrality_store);+ }loop {if let Ok(req) = server.accept::<searcher::distributed::Request>().await {diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex f7885465..06f42f68 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -75,7 +75,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {.join("warc_files").join(name);- if let Ok(file) = WarcFile::open(path) {+ if let Ok(file) = WarcFile::open(&path) {for record in file.records().flatten() {let webpage = Html::parse_without_text(&record.response.body, &record.request.url);for link in webpage@@ -94,10 +94,10 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {}}- std::fs::remove_file(warc_path).ok();- }+ graph.flush();- graph.flush();+ std::fs::remove_file(path).unwrap();+ }info!("{} done", name);diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex a90752f8..5e15fd1d 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -711,6 +711,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");diff --git a/src/kv/mod.rs b/src/kv/mod.rsindex 386a7028..3844bc37 100644--- a/src/kv/mod.rs+++ b/src/kv/mod.rs@@ -14,6 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::hash::Hash;+use std::{collections::HashMap, sync::RwLock};+use serde::{de::DeserializeOwned, Serialize};pub mod rocksdb_store;@@ -42,4 +45,47 @@ whereself.insert_raw(key_bytes, val_bytes);}++ fn load_in_memory(&self) -> Memory<K, V>+ where+ K: Eq + Hash,+ {+ let mut map = HashMap::new();++ for (k, v) in self.iter() {+ map.insert(k, v);+ }++ Memory(RwLock::new(map))+ }+}++pub struct Memory<K, V>(RwLock<HashMap<K, V>>);++impl<K, V> Kv<K, V> for Memory<K, V>+where+ K: Serialize + DeserializeOwned + Send + Sync + Eq + Hash + Clone,+ V: Serialize + DeserializeOwned + Send + Sync + Clone,+{+ fn get_raw(&self, _key: &[u8]) -> Option<Vec<u8>> {+ unimplemented!()+ }++ fn insert_raw(&self, _key: Vec<u8>, _value: Vec<u8>) {+ unimplemented!()+ }++ fn flush(&self) {}++ fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {+ todo!();+ }++ fn get(&self, key: &K) -> Option<V> {+ self.0.read().unwrap().get(key).cloned()+ }++ fn insert(&self, key: K, val: V) {+ self.0.write().unwrap().insert(key, val);+ }}diff --git a/src/lib.rs b/src/lib.rsindex 42868b36..a89d57d0 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -46,7 +46,7 @@ pub mod index;mod kv;pub mod prehashed;mod query;-mod ranking;+pub mod ranking;mod schema;mod schema_org;mod search_prettifier;@@ -59,8 +59,8 @@ mod tokenizer;#[allow(unused)]mod ttl_cache;mod warc;-mod webgraph;-mod webpage;+pub mod webgraph;+pub mod webpage;#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {@@ -154,6 +154,7 @@ pub struct SearchServerConfig {pub index_path: String,pub entity_index_path: Option<String>,pub bangs_path: Option<String>,+ pub centrality_store_path: Option<String>,pub host: String,}diff --git a/src/query/mod.rs b/src/query/mod.rsindex 7ae521b5..b8527b4d 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -150,12 +150,10 @@ impl Query {})}- pub fn set_goggles(&mut self, goggles: &[Goggle], schema: &Schema) {+ pub fn set_goggle(&mut self, goggle: &Goggle, schema: &Schema) {let mut subqueries = vec![(Occur::Must, self.tantivy_query.box_clone())];- for goggle in goggles {- subqueries.append(&mut goggle.as_tantivy(schema));- }+ subqueries.append(&mut goggle.as_tantivy(schema));self.tantivy_query = Box::new(BooleanQuery::new(subqueries))}diff --git a/src/ranking/centrality_store.rs b/src/ranking/centrality_store.rsindex f2c10677..b7b2a776 100644--- a/src/ranking/centrality_store.rs+++ b/src/ranking/centrality_store.rs@@ -14,39 +14,107 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;+use std::{fs::File, path::Path};-use crate::kv::{rocksdb_store::RocksDbStore, Kv};+use crate::{+ kv::{rocksdb_store::RocksDbStore, Kv},+ webgraph::{+ centrality::{+ approximate_harmonic::ApproximatedHarmonicCentrality, harmonic::HarmonicCentrality,+ },+ Webgraph,+ },+};-pub struct CentralityStore {- inner: Box<dyn Kv<String, f64>>,+pub struct HarmonicCentralityStore {+ pub host: Box<dyn Kv<String, f64>>,+ pub full: Box<dyn Kv<String, f64>>,}-impl CentralityStore {- pub fn new<P: AsRef<Path>>(path: P) -> Self {+impl HarmonicCentralityStore {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- inner: RocksDbStore::open(path),+ host: RocksDbStore::open(path.as_ref().join("host")),+ full: RocksDbStore::open(path.as_ref().join("full")),}}- pub fn insert(&mut self, key: String, centrality: f64) {- self.inner.insert(key, centrality);+ fn flush(&self) {+ self.host.flush();+ self.full.flush();}+}++pub struct CentralityStore {+ pub harmonic: HarmonicCentralityStore,+ pub approx_harmonic: ApproximatedHarmonicCentrality,+ pub base_path: String,+}- pub fn get(&self, key: &str) -> Option<f64> {- self.inner.get(&key.to_string())+impl CentralityStore {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ Self {+ harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),+ approx_harmonic: ApproximatedHarmonicCentrality::load(+ path.as_ref().join("approx_harmonic"),+ )+ .ok()+ .unwrap_or_default(),+ base_path: path.as_ref().to_str().unwrap().to_string(),+ }}- pub fn append(&mut self, it: impl Iterator<Item = (String, f64)>) {- it.filter(|(_, value)| *value != 0.0)- .for_each(|(key, value)| {- self.insert(key, value);- });+ pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {+ let mut store = CentralityStore::open(output_path.as_ref());++ store.approx_harmonic = ApproximatedHarmonicCentrality::new(graph);+ let harmonic_centrality = HarmonicCentrality::calculate(graph);- self.flush();+ let csv_file = File::options()+ .write(true)+ .create(true)+ .truncate(true)+ .open(output_path.as_ref().join("harmonic_full.csv"))+ .unwrap();+ let mut wtr = csv::Writer::from_writer(csv_file);++ let mut full: Vec<_> = harmonic_centrality.full.into_iter().collect();+ full.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ for (node, centrality) in full {+ store.harmonic.full.insert(node.name.clone(), centrality);+ wtr.write_record(&[node.name, centrality.to_string()])+ .unwrap();+ }++ wtr.flush().unwrap();++ let csv_file = File::options()+ .write(true)+ .create(true)+ .truncate(true)+ .open(output_path.as_ref().join("harmonic_host.csv"))+ .unwrap();++ let mut host: Vec<_> = harmonic_centrality.host.into_iter().collect();+ host.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ let mut wtr = csv::Writer::from_writer(csv_file);+ for (node, centrality) in host {+ store.harmonic.host.insert(node.name.clone(), centrality);+ wtr.write_record(&[node.name, centrality.to_string()])+ .unwrap();+ }+ wtr.flush().unwrap();++ store.flush();++ store}pub fn flush(&self) {- self.inner.flush();+ self.harmonic.flush();++ self.approx_harmonic+ .save(Path::new(&self.base_path).join("approx_harmonic"))+ .unwrap();}}diff --git a/src/ranking/goggles/ast.rs b/src/ranking/goggles/ast.rsindex cfabaeff..09b63fe3 100644--- a/src/ranking/goggles/ast.rs+++ b/src/ranking/goggles/ast.rs@@ -41,6 +41,7 @@ pub struct RawGoggle {pub comments: Vec<Comment>,pub instructions: Vec<RawInstruction>,pub alterations: Vec<RawAlteration>,+ pub site_preferences: Vec<RawSitePreference>,}impl From<Vec<GoggleBlock>> for RawGoggle {@@ -48,12 +49,14 @@ impl From<Vec<GoggleBlock>> for RawGoggle {let mut alterations = Vec::new();let mut comments = Vec::new();let mut instructions = Vec::new();+ let mut site_preferences = Vec::new();for block in blocks {match block {GoggleBlock::Comment(comment) => comments.push(comment),GoggleBlock::Instruction(instruction) => instructions.push(instruction),GoggleBlock::Alteration(alteration) => alterations.push(alteration),+ GoggleBlock::SitePreference(pref) => site_preferences.push(pref),}}@@ -61,6 +64,7 @@ impl From<Vec<GoggleBlock>> for RawGoggle {comments,instructions,alterations,+ site_preferences,}}}@@ -70,6 +74,7 @@ pub enum GoggleBlock {Comment(Comment),Instruction(RawInstruction),Alteration(RawAlteration),+ SitePreference(RawSitePreference),}#[derive(Debug, PartialEq, Eq)]@@ -84,6 +89,12 @@ pub struct RawInstruction {pub options: Vec<RawPatternOption>,}+#[derive(Debug, PartialEq, Eq)]+pub enum RawSitePreference {+ Liked(Vec<String>),+ Disliked(Vec<String>),+}+#[derive(Debug, PartialEq, Eq, Clone)]pub enum RawPatternPart {Raw(String),diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex b3662a67..567998e5 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -20,11 +20,12 @@ pub mod ast;mod const_query;mod pattern_query;-use std::convert::TryFrom;+use std::{collections::HashMap, convert::TryFrom, sync::Arc};use crate::{query::union::UnionQuery,schema::{Field, TextField},+ webgraph::centrality::approximate_harmonic::ApproximatedHarmonicCentrality,Result,};use itertools::Itertools;@@ -34,12 +35,12 @@ use tantivy::{};use self::{- ast::{RawAction, RawGoggle, RawInstruction, RawPatternOption, RawPatternPart},+ ast::{RawAction, RawGoggle, RawInstruction, RawPatternOption, RawPatternPart, Target},const_query::ConstQuery,pattern_query::PatternQuery,};-use super::signal::SignalAggregator;+use super::{signal::SignalAggregator, site_rankings::SiteRankings, Alteration, Signal};pub fn parse(goggle: &str) -> Result<Goggle> {let raw_goggle = ast::parse(goggle)?;@@ -57,9 +58,46 @@ impl TryFrom<RawGoggle> for Goggle {instructions.push(Instruction::try_from(inst)?);}+ let mut boosts = HashMap::new();+ let mut coefficients = HashMap::new();++ for alteration in raw.alterations {+ let alteration = Alteration::try_from(alteration)?;+ match alteration.target {+ Target::Signal(name) => {+ if let Some(signal) = Signal::from_string(name) {+ coefficients.insert(signal, alteration.score);+ }+ }+ Target::Field(name) => {+ if let Some(field) = Field::from_name(name) {+ if let Some(text_field) = field.as_text() {+ boosts.insert(text_field, alteration.score);+ }+ }+ }+ }+ }++ let mut liked_sites = Vec::new();+ let mut disliked_sites = Vec::new();++ for pref in raw.site_preferences {+ match pref {+ ast::RawSitePreference::Liked(mut sites) => liked_sites.append(&mut sites),+ ast::RawSitePreference::Disliked(mut sites) => disliked_sites.append(&mut sites),+ }+ }+Ok(Self {- aggregator: SignalAggregator::try_from(raw.alterations)?,instructions,+ coefficients,+ boosts,+ site_rankings: SiteRankings {+ liked: liked_sites,+ disliked: disliked_sites,+ blocked: Vec::new(), // blocked sites are handled by `$discard` syntax.+ },})}}@@ -158,7 +196,9 @@ pub enum Action {#[derive(Debug, Default, Clone)]pub struct Goggle {- pub aggregator: SignalAggregator,+ pub coefficients: HashMap<Signal, f64>,+ pub boosts: HashMap<TextField, f64>,+ pub site_rankings: SiteRankings,pub instructions: Vec<Instruction>,}@@ -174,6 +214,7 @@ impl Goggle {UnionQuery::from(self.instructions.iter()+ .chain(self.site_rankings.instructions().iter()).filter_map(|instruction| instruction.as_tantivy(schema)).map(|query| BooleanQuery::from(vec![query]).box_clone()).collect_vec(),@@ -183,10 +224,46 @@ impl Goggle {} else {self.instructions.iter()+ .chain(self.site_rankings.instructions().iter()).filter_map(|instruction| instruction.as_tantivy(schema)).collect()}}++ pub fn merge(mut self, mut other: Self) -> Self {+ self.instructions.append(&mut other.instructions);+ self.coefficients.extend(other.coefficients.into_iter());+ self.boosts.extend(other.boosts.into_iter());++ self.site_rankings+ .liked+ .append(&mut other.site_rankings.liked);++ self.site_rankings+ .disliked+ .append(&mut other.site_rankings.disliked);++ self.site_rankings+ .blocked+ .append(&mut other.site_rankings.blocked);++ self+ }++ pub fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator {+ let mut aggregator = SignalAggregator::new(+ self.coefficients.clone().into_iter(),+ self.boosts.clone().into_iter(),+ );++ if let Some(approx) = approx {+ aggregator+ .personal_harmonic+ .push(Arc::new(self.site_rankings.centrality_scorer(approx)));+ }++ aggregator+ }}fn process_site(site: &str, field: tantivy::schema::Field) -> Box<dyn tantivy::query::Query> {@@ -361,6 +438,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -386,6 +464,7 @@ mod tests {primary_image: None,pre_computed_score: 0.0,fetch_time_ms: 500,+ node_id: None,}).expect("failed to insert webpage");@@ -491,6 +570,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -516,6 +596,7 @@ mod tests {primary_image: None,pre_computed_score: 0.0,fetch_time_ms: 500,+ node_id: None,}).expect("failed to insert webpage");@@ -599,6 +680,7 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -624,6 +706,7 @@ mod tests {primary_image: None,pre_computed_score: 0.0,fetch_time_ms: 500,+ node_id: None,}).expect("failed to insert webpage");index@@ -649,6 +732,7 @@ mod tests {primary_image: None,pre_computed_score: 0.0,fetch_time_ms: 500,+ node_id: None,}).expect("failed to insert webpage");diff --git a/src/ranking/goggles/parser.lalrpop b/src/ranking/goggles/parser.lalrpopindex 8f061e10..2a18d99e 100644--- a/src/ranking/goggles/parser.lalrpop+++ b/src/ranking/goggles/parser.lalrpop@@ -9,12 +9,28 @@ Sep<S, T>: Vec<T> = {}};+SepNewline<S, T>: Vec<T> = {+ <mut xs:(<T> S ";"?)*> <x:T?> => match x {+ Some(x) => {+ xs.push(x);+ xs+ }+ None => xs+ }+};+pub Blocks: Vec<GoggleBlock> = <Sep<";", Block>>;Block: GoggleBlock = {<RawAlteration> => GoggleBlock::Alteration(<>),<Comment> => GoggleBlock::Comment(<>),<RawInstruction> => GoggleBlock::Instruction(<>),+ <RawSitePreference> => GoggleBlock::SitePreference(<>),+}++RawSitePreference: RawSitePreference = {+ "liked" "=" "[" ";"? <SepNewline<",", Ident>> "]" => RawSitePreference::Liked(<>),+ "disliked" "=" "[" ";"? <SepNewline<",", Ident>> "]" => RawSitePreference::Disliked(<>),}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 53c1691d..8c167a28 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -150,6 +150,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -175,6 +176,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -224,6 +226,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -249,6 +252,7 @@ mod tests {page_centrality: 5.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -297,6 +301,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -322,6 +327,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -345,6 +351,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -394,6 +401,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -420,6 +428,7 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -465,6 +474,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -500,6 +510,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -550,6 +561,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -573,6 +585,7 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -619,6 +632,7 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -643,6 +657,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");@@ -668,12 +683,13 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index.commit().unwrap();- let searcher = LocalSearcher::new(index, None, None);+ let searcher = LocalSearcher::new(index);let res = searcher.search(&SearchQuery {@@ -764,6 +780,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -789,6 +806,7 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -814,10 +832,11 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index, None, None);+ let searcher = LocalSearcher::new(index);let result = searcher.search(&SearchQuery {@@ -866,6 +885,7 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index@@ -891,10 +911,11 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index, None, None);+ let searcher = LocalSearcher::new(index);let result = searcher.search(&SearchQuery {diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex f22ae4ee..b38dd886 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -17,6 +17,7 @@use crate::{fastfield_cache,schema::{FastField, TextField},+ webgraph::centrality::approximate_harmonic,webpage::Webpage,Result,};@@ -42,9 +43,10 @@ pub enum Signal {UpdateTimestamp,NumTrackers,Region,+ PersonalCentrality,}-pub const ALL_SIGNALS: [Signal; 8] = [+pub const ALL_SIGNALS: [Signal; 9] = [Signal::Bm25,Signal::HostCentrality,Signal::PageCentrality,@@ -53,30 +55,37 @@ pub const ALL_SIGNALS: [Signal; 8] = [Signal::UpdateTimestamp,Signal::NumTrackers,Signal::Region,+ Signal::PersonalCentrality,];+struct SignalValue {+ bm25: Score,+ fastfield_value: Option<u64>,+ current_timestamp: usize,+ selected_region: Option<Region>,+ personal_centrality: f64,+}+impl Signal {fn is_computable_before_search(&self) -> bool {- !matches!(self, Signal::Bm25)+ self.as_fastfield().is_some()}fn value(&self,- bm25: Score,- fastfield_value: Option<u64>,region_count: &impl Deref<Target = RegionCount>,- current_timestamp: usize,- selected_region: Option<Region>,aggregator: &SignalAggregator,+ value: SignalValue,) -> f64 {match self {- Signal::Bm25 => bm25 as f64,+ Signal::Bm25 => value.bm25 as f64,Signal::HostCentrality | Signal::PageCentrality => {- fastfield_value.unwrap() as f64 / CENTRALITY_SCALING as f64+ value.fastfield_value.unwrap() as f64 / CENTRALITY_SCALING as f64}- Signal::IsHomepage => fastfield_value.unwrap() as f64,+ Signal::PersonalCentrality => value.personal_centrality,+ Signal::IsHomepage => value.fastfield_value.unwrap() as f64,Signal::FetchTimeMs => {- let fetch_time_ms = fastfield_value.unwrap() as usize;+ let fetch_time_ms = value.fastfield_value.unwrap() as usize;if fetch_time_ms >= aggregator.fetch_time_ms_cache.len() {0.0@@ -85,14 +94,14 @@ impl Signal {}}Signal::UpdateTimestamp => {- let update_timestamp = fastfield_value.unwrap() as i64;+ let update_timestamp = value.fastfield_value.unwrap() as i64;- if current_timestamp as i64 - update_timestamp <= 0 {+ if value.current_timestamp as i64 - update_timestamp <= 0 {return 0.0;}let hours_since_update =- ((current_timestamp as i64 - update_timestamp).max(1) / 3600) as usize;+ ((value.current_timestamp as i64 - update_timestamp).max(1) / 3600) as usize;if hours_since_update < aggregator.update_time_cache.len() {aggregator.update_time_cache[hours_since_update]@@ -101,17 +110,19 @@ impl Signal {}}Signal::NumTrackers => {- let num_trackers = fastfield_value.unwrap() as f64;+ let num_trackers = value.fastfield_value.unwrap() as f64;1.0 / (num_trackers + 1.0)}Signal::Region => {- let webpage_region = Region::from_id(fastfield_value.unwrap());+ let webpage_region = Region::from_id(value.fastfield_value.unwrap());- let boost =- selected_region.map_or(- 0.0,- |region| if region == webpage_region { 50.0 } else { 0.0 },- );+ let boost = value.selected_region.map_or(0.0, |region| {+ if region == webpage_region {+ 50.0+ } else {+ 0.0+ }+ });boost + region_count.score(&webpage_region)}@@ -128,13 +139,21 @@ impl Signal {Signal::UpdateTimestamp => 80.0,Signal::NumTrackers => 20.0,Signal::Region => 60.0,+ Signal::PersonalCentrality => 1024.0,}}- fn from_string(name: String) -> Option<Signal> {+ pub fn from_string(name: String) -> Option<Signal> {match name.as_str() {"bm25" => Some(Signal::Bm25),"host_centrality" => Some(Signal::HostCentrality),+ "page_centrality" => Some(Signal::PageCentrality),+ "is_homepage" => Some(Signal::IsHomepage),+ "fetch_time_ms" => Some(Signal::FetchTimeMs),+ "update_timestamp" => Some(Signal::UpdateTimestamp),+ "num_trackers" => Some(Signal::NumTrackers),+ "region" => Some(Signal::Region),+ "personal_centrality" => Some(Signal::PersonalCentrality),_ => None,}}@@ -149,6 +168,7 @@ impl Signal {Signal::UpdateTimestamp => Some(FastField::LastUpdated),Signal::NumTrackers => Some(FastField::NumTrackers),Signal::Region => Some(FastField::Region),+ Signal::PersonalCentrality => None,}}}@@ -216,6 +236,7 @@ impl FieldBoost {pub struct SignalAggregator {fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,signal_coefficients: SignalCoefficient,+ pub personal_harmonic: Vec<Arc<approximate_harmonic::Scorer>>,field_boost: FieldBoost,fetch_time_ms_cache: [f64; 1000],update_time_cache: Vec<f64>,@@ -252,6 +273,7 @@ impl SignalAggregator {Self {fastfield_cache: None,+ personal_harmonic: Vec::new(),signal_coefficients,field_boost,fetch_time_ms_cache,@@ -280,14 +302,23 @@ impl SignalAggregator {.and_then(|cache| cache.get_doc_cache(&field).get_u64(&doc))});+ let personal_centrality = self+ .personal_harmonic+ .iter()+ .map(|scorer| scorer.score(doc as u64))+ .sum();+self.coefficients().get(&signal)* signal.value(- bm25,- fastfield_value,region_count,- current_timestamp,- selected_region,self,+ SignalValue {+ bm25,+ fastfield_value,+ current_timestamp,+ selected_region,+ personal_centrality,+ },)}).sum()@@ -321,12 +352,15 @@ impl SignalAggregator {self.coefficients().get(&signal)* signal.value(- 0.0,- Some(fastfield_value),®ion_count,- current_timestamp,- None,self,+ SignalValue {+ bm25: 0.0,+ fastfield_value: Some(fastfield_value),+ current_timestamp,+ selected_region: None,+ personal_centrality: 0.0,+ },)}).sum()@@ -356,32 +390,3 @@ impl TryFrom<RawAlteration> for Alteration {})}}--impl TryFrom<Vec<RawAlteration>> for SignalAggregator {- type Error = crate::Error;-- fn try_from(alterations: Vec<RawAlteration>) -> Result<Self> {- let mut coefficients = Vec::new();- let mut boosts = Vec::new();-- for alteration in alterations {- let alteration = Alteration::try_from(alteration)?;- match alteration.target {- Target::Signal(name) => {- if let Some(signal) = Signal::from_string(name) {- coefficients.push((signal, alteration.score));- }- }- Target::Field(name) => {- if let Some(field) = Field::from_name(name) {- if let Some(text_field) = field.as_text() {- boosts.push((text_field, alteration.score));- }- }- }- }- }-- Ok(Self::new(coefficients.into_iter(), boosts.into_iter()))- }-}diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rsindex b78ce7d6..36cc9cc3 100644--- a/src/ranking/site_rankings.rs+++ b/src/ranking/site_rankings.rs@@ -16,55 +16,79 @@use serde::{Deserialize, Serialize};-use super::{- goggles::{Action, Goggle, Instruction, PatternOption},- SignalAggregator,+use crate::{+ webgraph::{+ centrality::approximate_harmonic::{ApproximatedHarmonicCentrality, Scorer},+ Node,+ },+ webpage::Url,};-#[derive(Debug, Serialize, Deserialize, Clone)]+use super::goggles::{Action, Goggle, Instruction, PatternOption};++#[derive(Debug, Default, Serialize, Deserialize, Clone)]pub struct SiteRankings {- pub preferred: Vec<String>,+ pub liked: Vec<String>,pub disliked: Vec<String>,pub blocked: Vec<String>,}impl SiteRankings {- pub fn into_goggle(self) -> Goggle {+ pub fn instructions(&self) -> Vec<Instruction> {let mut instructions = Vec::new();- for site in self.preferred {+ for site in &self.liked {instructions.push(Instruction {patterns: Vec::new(),options: vec![- PatternOption::Site(site),+ PatternOption::Site(site.clone()),PatternOption::Action(Action::Boost(5)),],});}- for site in self.disliked {+ for site in &self.disliked {instructions.push(Instruction {patterns: Vec::new(),options: vec![- PatternOption::Site(site),+ PatternOption::Site(site.clone()),PatternOption::Action(Action::Downrank(5)),],});}- for site in self.blocked {+ for site in &self.blocked {instructions.push(Instruction {patterns: Vec::new(),options: vec![- PatternOption::Site(site),+ PatternOption::Site(site.clone()),PatternOption::Action(Action::Discard),],});}+ instructions+ }++ pub fn centrality_scorer(&self, approx_harmonic: &ApproximatedHarmonicCentrality) -> Scorer {+ let mut liked_nodes = Vec::new();+ let mut disliked_nodes = Vec::new();++ for site in &self.liked {+ liked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());+ }++ for site in &self.disliked {+ disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());+ }++ approx_harmonic.scorer(&liked_nodes, &disliked_nodes)+ }++ pub fn into_goggle(self) -> Goggle {Goggle {- aggregator: SignalAggregator::default(),- instructions,+ site_rankings: self,+ ..Default::default()}}}@@ -72,9 +96,11 @@ impl SiteRankings {#[cfg(test)]mod tests {use crate::{+ gen_temp_path,index::Index,- ranking::site_rankings::SiteRankings,+ ranking::{centrality_store::CentralityStore, site_rankings::SiteRankings},searcher::{LocalSearcher, SearchQuery},+ webgraph::{Node, WebgraphBuilder},webpage::{Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -83,6 +109,24 @@ mod tests {fn site_rankings() {let mut index = Index::temporary().expect("Unable to open index");+ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(+ Node::from("https://www.first.com").into_host(),+ Node::from("https://www.second.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("https://www.third.com").into_host(),+ Node::from("https://www.third.com").into_host(),+ String::new(),+ );++ graph.flush();+index.insert(Webpage {html: Html::parse(@@ -106,6 +150,14 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ node_id: Some(+ graph+ .host+ .as_ref()+ .unwrap()+ .node2id(&Node::from("https://www.first.com").into_host())+ .unwrap(),+ ),}).expect("failed to insert webpage");index@@ -127,14 +179,57 @@ mod tests {),backlinks: vec![],host_centrality: 1.0,+ fetch_time_ms: 2000,+ page_centrality: 0.0,+ pre_computed_score: 0.0,+ primary_image: None,+ node_id: Some(+ graph+ .host+ .as_ref()+ .unwrap()+ .node2id(&Node::from("https://www.second.com").into_host())+ .unwrap(),+ ),+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.third.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,fetch_time_ms: 0,page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: Some(+ graph+ .host+ .as_ref()+ .unwrap()+ .node2id(&Node::from("https://www.third.com").into_host())+ .unwrap(),+ ),}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index, None, None);+ let mut searcher = LocalSearcher::new(index);++ searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()));let result = searcher.search(&SearchQuery {@@ -143,8 +238,8 @@ mod tests {goggle_program: None,skip_pages: None,site_rankings: Some(SiteRankings {- preferred: vec!["first.com".to_string()],- disliked: vec!["second.com".to_string()],+ liked: vec!["first.com".to_string()],+ disliked: vec!["third.com".to_string()],blocked: vec![],}),})@@ -153,10 +248,11 @@ mod tests {.unwrap().webpages;- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);+ assert_eq!(result.num_docs, 3);+ assert_eq!(result.documents.len(), 3);assert_eq!(result.documents[0].url, "https://www.first.com");assert_eq!(result.documents[1].url, "https://www.second.com");+ assert_eq!(result.documents[2].url, "https://www.third.com");let result = searcher.search(&SearchQuery {@@ -165,7 +261,30 @@ mod tests {goggle_program: None,skip_pages: None,site_rankings: Some(SiteRankings {- preferred: vec![],+ liked: vec!["first.com".to_string()],+ disliked: vec![],+ blocked: vec![],+ }),+ })+ .expect("Search failed")+ .into_websites()+ .unwrap()+ .webpages;++ assert_eq!(result.num_docs, 3);+ assert_eq!(result.documents.len(), 3);+ assert_eq!(result.documents[0].url, "https://www.first.com");+ assert_eq!(result.documents[1].url, "https://www.second.com");+ assert_eq!(result.documents[2].url, "https://www.third.com");++ let result = searcher+ .search(&SearchQuery {+ original: "test".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ site_rankings: Some(SiteRankings {+ liked: vec![],disliked: vec!["second.com".to_string()],blocked: vec!["first.com".to_string()],}),@@ -175,8 +294,9 @@ mod tests {.unwrap().webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.second.com");+ assert_eq!(result.num_docs, 2);+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.third.com");+ assert_eq!(result.documents[1].url, "https://www.second.com");}}diff --git a/src/schema.rs b/src/schema.rsindex e2b9e544..e545d3a3 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -64,6 +64,7 @@ pub enum FastField {UrlHash,DomainHash,PreComputedScore,+ HostNodeID,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -72,7 +73,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 33] = [+pub static ALL_FIELDS: [Field; 34] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -107,6 +108,7 @@ pub static ALL_FIELDS: [Field; 33] = [Field::Fast(FastField::UrlHash),Field::Fast(FastField::DomainHash),Field::Fast(FastField::PreComputedScore),+ Field::Fast(FastField::HostNodeID),];impl Field {@@ -248,6 +250,12 @@ impl Field {.set_indexed().set_stored(),),+ Field::Fast(FastField::HostNodeID) => IndexingOption::Integer(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed()+ .set_stored(),+ ),}}@@ -288,6 +296,7 @@ impl Field {Field::Fast(FastField::TitleHash) => "title_hash",Field::Fast(FastField::UrlHash) => "url_hash",Field::Fast(FastField::DomainHash) => "domain_hash",+ Field::Fast(FastField::HostNodeID) => "host_node_id",}}@@ -356,6 +365,7 @@ impl Field {"url_hash" => Some(Field::Fast(FastField::UrlHash)),"domain_hash" => Some(Field::Fast(FastField::DomainHash)),"title_hash" => Some(Field::Fast(FastField::TitleHash)),+ "host_node_id" => Some(Field::Fast(FastField::HostNodeID)),_ => None,}}@@ -423,6 +433,7 @@ impl FastField {FastField::UrlHash => DataType::U64s,FastField::DomainHash => DataType::U64s,FastField::PreComputedScore => DataType::F64,+ FastField::HostNodeID => DataType::U64,}}}diff --git a/src/searcher/local.rs b/src/searcher/local.rsindex 00133172..191a5712 100644--- a/src/searcher/local.rs+++ b/src/searcher/local.rs@@ -25,7 +25,8 @@ use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;use crate::query::Query;-use crate::ranking::goggles;+use crate::ranking::centrality_store::CentralityStore;+use crate::ranking::goggles::{self, Goggle};use crate::ranking::{Ranker, SignalAggregator};use crate::webpage::region::Region;use crate::webpage::Url;@@ -37,30 +38,37 @@ pub struct LocalSearcher {index: Index,entity_index: Option<EntityIndex>,bangs: Option<Bangs>,+ centrality_store: Option<CentralityStore>,}impl From<Index> for LocalSearcher {fn from(index: Index) -> Self {- Self::new(index, None, None)+ Self::new(index)}}-#[derive(Debug, Serialize, Deserialize)]-pub struct InitialWebsiteResult {- pub spell_corrected_query: Option<String>,- pub websites: inverted_index::InitialSearchResult,- pub entity: Option<StoredEntity>,-}-impl LocalSearcher {- pub fn new(index: Index, entity_index: Option<EntityIndex>, bangs: Option<Bangs>) -> Self {+ pub fn new(index: Index) -> Self {LocalSearcher {index,- entity_index,- bangs,+ entity_index: None,+ bangs: None,+ centrality_store: None,}}+ pub fn set_entity_index(&mut self, entity_index: EntityIndex) {+ self.entity_index = Some(entity_index);+ }++ pub fn set_bangs(&mut self, bangs: Bangs) {+ self.bangs = Some(bangs);+ }++ pub fn set_centrality_store(&mut self, centrality_store: CentralityStore) {+ self.centrality_store = Some(centrality_store);+ }+pub fn search_initial(&self,query: &SearchQuery,@@ -72,14 +80,22 @@ impl LocalSearcher {.as_ref().and_then(|program| goggles::parse(program).ok());+ let query_aggregator = goggle+ .as_ref()+ .map(|goggle| {+ goggle.aggregator(+ self.centrality_store+ .as_ref()+ .map(|centrality_store| ¢rality_store.approx_harmonic),+ )+ })+ .unwrap_or_default();+let mut parsed_query = Query::parse(&query.original,self.index.schema(),self.index.tokenizers(),- goggle- .as_ref()- .map(|goggle| &goggle.aggregator)- .unwrap_or(&SignalAggregator::default()),+ &query_aggregator,)?;if parsed_query.is_empty() {@@ -102,11 +118,19 @@ impl LocalSearcher {goggles.push(site_rankings.clone().into_goggle())}- parsed_query.set_goggles(&goggles, &self.index.schema());+ let goggle = goggles+ .into_iter()+ .fold(Goggle::default(), |acc, elem| acc.merge(elem));++ parsed_query.set_goggle(&goggle, &self.index.schema());let mut ranker = Ranker::new(self.index.region_count.clone(),- goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),+ goggle.aggregator(+ self.centrality_store+ .as_ref()+ .map(|centrality_store| ¢rality_store.approx_harmonic),+ ),self.index.inverted_index.fastfield_cache(),);@@ -207,6 +231,14 @@ impl LocalSearcher {.and_then(|index| index.get_attribute_occurrence(attribute))}}++#[derive(Debug, Serialize, Deserialize)]+pub struct InitialWebsiteResult {+ pub spell_corrected_query: Option<String>,+ pub websites: inverted_index::InitialSearchResult,+ pub entity: Option<StoredEntity>,+}+impl SearchResult {#[cfg(test)]pub fn into_websites(self) -> Option<WebsitesResult> {@@ -253,13 +285,14 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}).expect("failed to insert webpage");}index.commit().unwrap();- let searcher = LocalSearcher::new(index, None, None);+ let searcher = LocalSearcher::new(index);for p in 0..NUM_PAGES {let urls: Vec<_> = searcherdiff --git a/src/warc.rs b/src/warc.rsindex ec5fd27f..eecb7b01 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -110,7 +110,8 @@ impl WarcFile {return Ok(());}- debug!("warc download failed, retrying in {} ms", dur.as_millis());+ debug!("warc download failed: {:?}", res.err().unwrap());+ debug!("retrying in {} ms", dur.as_millis());sleep(dur).await;}diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rsnew file mode 100644index 00000000..e9ba5b75--- /dev/null+++ b/src/webgraph/centrality/approximate_harmonic.rs@@ -0,0 +1,546 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! Algorithm in broad strokes+//! We first calculate the betweenness centrality for all nodes an choose top k as proxy nodes.+//! The distances from every node to every proxy node and reverse is then calculated and stored.+//!+//! During search the user can choose a set of liked and disliked nodes. For each user node, we find the best k proxy nodes.+//! For all proxy nodes we then estimate the harmonic centrality from the liked nodes - disliked nodes by estimating+//! the shortest distance as the distance through their best proxy node. This distance estimate is very fast since+//! all distances are pre-computed for the proxy nodes.++use std::collections::{BinaryHeap, HashMap};+use std::fs::File;+use std::io::{BufReader, Read, Write};+use std::path::Path;+use std::sync::{Arc, Mutex};++use boomphf::hashmap::BoomHashMap;+use itertools::Itertools;+use rayon::prelude::*;+use serde::{Deserialize, Serialize};++use crate::webgraph::Node;+use crate::webgraph::{centrality::betweenness::Betweenness, NodeID, Webgraph};+use crate::Result;++const NUM_PROXY_NODES: usize = 10_000;+const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;+const USER_NODES_LIMIT: usize = 20; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing++#[derive(Serialize, Deserialize)]+pub struct ProxyNode {+ pub id: NodeID,+ dist_from_node: BoomHashMap<NodeID, usize>, // from node to proxy node+ dist_to_node: BoomHashMap<NodeID, usize>, // from proxy node to other node+}++impl ProxyNode {+ fn dist(&self, from: &NodeID, to: &NodeID) -> Option<usize> {+ if let Some(from_node_to_proxy) = self.dist_from_node.get(from) {+ if let Some(from_proxy_to_node) = self.dist_to_node.get(to) {+ return Some(from_node_to_proxy + from_proxy_to_node);+ }+ }++ None+ }+}++struct WeightedProxyNode {+ node: Arc<ProxyNode>,+ dist: usize,+}++impl PartialOrd for WeightedProxyNode {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.dist.partial_cmp(&other.dist)+ }+}+impl Ord for WeightedProxyNode {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)+ }+}++impl PartialEq for WeightedProxyNode {+ fn eq(&self, other: &Self) -> bool {+ self.dist == other.dist+ }+}++impl Eq for WeightedProxyNode {}++pub struct Scorer {+ fixed_scores: HashMap<NodeID, f64>,+ liked_nodes: Vec<UserNode>,+ disliked_nodes: Vec<UserNode>,+ cache: Mutex<HashMap<NodeID, f64>>,+ num_liked_nodes: usize,+}++fn create_user_nodes(+ nodes: &[NodeID],+ proxy_nodes: &[Arc<ProxyNode>],+ betweenness: &HashMap<NodeID, f64>,+) -> Vec<UserNode> {+ let mut res = Vec::new();++ let mut nodes = nodes.to_vec();+ nodes.sort_by(|a, b| {+ betweenness+ .get(b)+ .unwrap_or(&0.0)+ .partial_cmp(betweenness.get(a).unwrap_or(&0.0))+ .unwrap_or(std::cmp::Ordering::Equal)+ });++ for id in nodes {+ let user_node = UserNode::new(id, proxy_nodes);+ if res.len() < USER_NODES_LIMIT {+ res.push(user_node);+ } else {+ let mut best = res+ .iter_mut()+ .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(1_000_000))+ .unwrap();++ best.weight += 1;+ }+ }++ res+}++impl Scorer {+ fn new(+ proxy_nodes: &[Arc<ProxyNode>],+ liked_nodes: &[NodeID],+ disliked_nodes: &[NodeID],+ fixed_scores: HashMap<NodeID, f64>,+ betweenness: &HashMap<NodeID, f64>,+ ) -> Self {+ let num_liked_nodes = liked_nodes.len();++ Self {+ fixed_scores,+ num_liked_nodes,+ liked_nodes: create_user_nodes(liked_nodes, proxy_nodes, betweenness),+ disliked_nodes: create_user_nodes(disliked_nodes, proxy_nodes, betweenness),+ cache: Mutex::new(HashMap::new()),+ }+ }++ pub fn score(&self, node: NodeID) -> f64 {+ if let Some(score) = self.fixed_scores.get(&node) {+ *score+ } else {+ if let Some(cached) = self.cache.lock().unwrap().get(&node) {+ return *cached;+ }++ let res = ((self+ .liked_nodes+ .iter()+ .filter_map(|liked_node| liked_node.best_dist(&node).map(|dist| (dist, liked_node)))+ .map(|(dist, liked_node)| liked_node.weight as f64 / dist as f64)+ .sum::<f64>()+ - self+ .disliked_nodes+ .iter()+ .filter_map(|disliked_node| {+ disliked_node+ .best_dist(&node)+ .map(|dist| (dist, disliked_node))+ })+ .map(|(dist, disliked_node)| disliked_node.weight as f64 / dist as f64)+ .sum::<f64>())+ / self.num_liked_nodes as f64)+ .max(0.0);++ self.cache.lock().unwrap().insert(node, res);++ res+ }+ }+}++struct UserNode {+ id: NodeID,+ proxy_nodes: Vec<Arc<ProxyNode>>,+ weight: usize,+}++impl UserNode {+ fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {+ let mut heap = BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);+ for proxy in proxy_nodes {+ if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id) {+ let weighted_node = WeightedProxyNode {+ node: Arc::clone(proxy),+ dist: *dist_to_proxy,+ };++ if heap.len() == BEST_PROXY_NODES_PER_USER_NODE {+ if let Some(mut worst) = heap.peek_mut() {+ *worst = weighted_node;+ }+ } else {+ heap.push(weighted_node);+ }+ }+ }++ Self {+ id,+ weight: 1,+ proxy_nodes: heap.into_iter().map(|weighted| weighted.node).collect(),+ }+ }++ fn best_dist(&self, node: &NodeID) -> Option<usize> {+ let mut best = None;++ for proxy in &self.proxy_nodes {+ if let Some(dist) = proxy.dist(&self.id, node) {+ best = match best {+ Some((best_dist, best_proxy)) => {+ if dist < best_dist {+ Some((dist, proxy))+ } else {+ Some((best_dist, best_proxy))+ }+ }+ None => Some((dist, proxy)),+ }+ }+ }++ best.map(|(dist, _)| dist)+ }+}++#[derive(Serialize, Deserialize, Default)]+pub struct ApproximatedHarmonicCentrality {+ pub node2id: HashMap<Node, NodeID>,+ pub proxy_nodes: Vec<Arc<ProxyNode>>,+ betweenness: HashMap<NodeID, f64>,+}++impl ApproximatedHarmonicCentrality {+ pub fn new(graph: &Webgraph) -> Self {+ Self::new_with_num_proxy(graph, NUM_PROXY_NODES)+ }++ fn new_with_num_proxy(graph: &Webgraph, num_proxy_nodes: usize) -> Self {+ let betweenness = Betweenness::calculate_with_progress(graph);++ let mut nodes = betweenness.centrality.iter().collect_vec();+ nodes.sort_by(|a, b| b.1.partial_cmp(a.1).unwrap_or(std::cmp::Ordering::Equal));++ let mut node2id: HashMap<Node, NodeID> = HashMap::new();++ if let Some(store) = graph.host.as_ref() {+ for id in store.nodes() {+ if let Some(node) = store.id2node(&id) {+ node2id.insert(node, id);+ }+ }+ }++ let proxy_nodes: Vec<_> = nodes+ .into_par_iter()+ .take(num_proxy_nodes)+ .map(|(node, _)| node)+ .filter(|node| node2id.contains_key(node))+ .map(|node| {+ let (ids, dist) = graph.raw_host_distances(node.clone()).into_iter().unzip();+ let (rev_ids, rev_dist) = graph+ .raw_host_reversed_distances(node.clone())+ .into_iter()+ .unzip();++ Arc::new(ProxyNode {+ id: *node2id.get(node).unwrap(),+ dist_to_node: BoomHashMap::new(ids, dist),+ dist_from_node: BoomHashMap::new(rev_ids, rev_dist),+ })+ })+ .collect();++ Self {+ proxy_nodes,+ betweenness: betweenness+ .centrality+ .into_iter()+ .filter_map(|(node, centrality)| node2id.get(&node).map(|id| (*id, centrality)))+ .collect(),+ node2id,+ }+ }++ pub fn scorer(&self, liked_nodes: &[Node], disliked_nodes: &[Node]) -> Scorer {+ let liked_nodes = liked_nodes+ .iter()+ .map(|node| node.clone().into_host())+ .filter_map(|node| self.node2id.get(&node).copied())+ .collect_vec();++ let disliked_nodes = disliked_nodes+ .iter()+ .map(|node| node.clone().into_host())+ .filter_map(|node| self.node2id.get(&node).copied())+ .collect_vec();++ let fixed_scores: HashMap<_, _> = liked_nodes+ .iter()+ .map(|node| (*node, 1.0))+ .chain(disliked_nodes.iter().map(|node| (*node, 0.0)))+ .collect();++ Scorer::new(+ &self.proxy_nodes,+ &liked_nodes,+ &disliked_nodes,+ fixed_scores,+ &self.betweenness,+ )+ }++ pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {+ let mut file = File::options()+ .create(true)+ .truncate(true)+ .write(true)+ .open(path)?;++ let buf = bincode::serialize(&self)?;+ file.write_all(&buf)?;++ Ok(())+ }++ pub fn load<P: AsRef<Path>>(path: P) -> Result<Self> {+ let file = File::open(path)?;+ let mut reader = BufReader::new(file);++ let mut buf = Vec::new();+ reader.read_to_end(&mut buf)?;++ Ok(bincode::deserialize(&buf)?)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::webgraph::WebgraphBuilder;++ fn test_graph() -> Webgraph {+ /*+ G β---- F+ | |+ | βΌ+ ------βΊ E -------βΊ H+ β² |+ A -- | |+ | | βΌ+ B ----βΊ D β------- C+ */++ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("A"), Node::from("D"), String::new());+ graph.insert(Node::from("B"), Node::from("D"), String::new());+ graph.insert(Node::from("C"), Node::from("D"), String::new());+ graph.insert(Node::from("D"), Node::from("E"), String::new());+ graph.insert(Node::from("E"), Node::from("H"), String::new());+ graph.insert(Node::from("H"), Node::from("C"), String::new());+ graph.insert(Node::from("F"), Node::from("E"), String::new());+ graph.insert(Node::from("F"), Node::from("G"), String::new());+ graph.insert(Node::from("G"), Node::from("E"), String::new());++ graph.flush();++ graph+ }++ #[test]+ fn proxy_nodes_selection() {+ // Liked nodes: E, C+ let graph = test_graph();+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);+ let id2node: HashMap<NodeID, Node> = centrality+ .node2id+ .clone()+ .into_iter()+ .map(|(node, id)| (id, node))+ .collect();++ assert_eq!(centrality.proxy_nodes.len(), 3);+ let mut proxy_nodes = centrality+ .proxy_nodes+ .iter()+ .map(|node| id2node.get(&node.id).unwrap().clone())+ .collect_vec();+ proxy_nodes.sort();++ assert_eq!(+ proxy_nodes,+ vec![+ Node::from("D".to_string()),+ Node::from("E".to_string()),+ Node::from("H".to_string())+ ]+ );+ }++ #[test]+ fn liked_nodes_centrality() {+ let graph = test_graph();+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);++ let liked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];++ let scorer = centrality.scorer(&liked_nodes, &[]);++ for node in &liked_nodes {+ assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 1.0);+ }+ }++ #[test]+ fn ordering() {+ let graph = test_graph();+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);++ let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];++ let scorer = centrality.scorer(&liked_nodes, &[]);++ assert!(+ scorer.score(+ *centrality+ .node2id+ .get(&Node::from("E".to_string()))+ .unwrap()+ ) > scorer.score(+ *centrality+ .node2id+ .get(&Node::from("H".to_string()))+ .unwrap()+ )+ );+ assert!(+ scorer.score(+ *centrality+ .node2id+ .get(&Node::from("H".to_string()))+ .unwrap()+ ) > scorer.score(+ *centrality+ .node2id+ .get(&Node::from("C".to_string()))+ .unwrap()+ )+ );++ assert!(+ scorer.score(+ *centrality+ .node2id+ .get(&Node::from("C".to_string()))+ .unwrap()+ ) > scorer.score(+ *centrality+ .node2id+ .get(&Node::from("A".to_string()))+ .unwrap()+ )+ );+ }++ #[test]+ fn disliked_nodes_centrality() {+ let graph = test_graph();+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);++ let disliked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];++ let scorer = centrality.scorer(&[], &disliked_nodes);++ for node in &disliked_nodes {+ assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 0.0);+ }+ }++ #[test]+ fn ordering_with_dislikes() {+ let graph = test_graph();+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);++ let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];+ let disliked_nodes = vec![Node::from("F".to_string())];++ let scorer = centrality.scorer(&liked_nodes, &disliked_nodes);++ assert!(+ scorer.score(+ *centrality+ .node2id+ .get(&Node::from("E".to_string()))+ .unwrap()+ ) > scorer.score(+ *centrality+ .node2id+ .get(&Node::from("H".to_string()))+ .unwrap()+ )+ );++ assert!(+ scorer.score(+ *centrality+ .node2id+ .get(&Node::from("H".to_string()))+ .unwrap()+ ) > scorer.score(+ *centrality+ .node2id+ .get(&Node::from("C".to_string()))+ .unwrap()+ )+ );++ assert!(+ scorer.score(+ *centrality+ .node2id+ .get(&Node::from("C".to_string()))+ .unwrap()+ ) > scorer.score(+ *centrality+ .node2id+ .get(&Node::from("A".to_string()))+ .unwrap()+ )+ );+ }+}diff --git a/src/webgraph/centrality/betweenness.rs b/src/webgraph/centrality/betweenness.rsnew file mode 100644index 00000000..b43b1243--- /dev/null+++ b/src/webgraph/centrality/betweenness.rs@@ -0,0 +1,240 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! this is an implementation of the algorithm+//! described in "A Faster Algorithm for Betweenness Centrality"++use std::collections::{HashMap, VecDeque};++use indicatif::{ProgressBar, ProgressStyle};+use serde::{Deserialize, Serialize};++use crate::webgraph::{graph_store::GraphStore, Node, NodeID, Store, Webgraph};++fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<Node, f64>, i32) {+ let mut centrality: HashMap<NodeID, f64> = HashMap::new();+ let mut n = 0;+ let mut max_dist = 0;++ let nodes: Vec<_> = store.nodes().collect();++ let pb =+ if with_progress {+ let pb = ProgressBar::new(nodes.len() as u64);+ pb.set_style(+ ProgressStyle::default_bar()+ .template("{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})")+ .progress_chars("#>-"),+ );+ Some(pb)+ } else {+ None+ };++ for s in nodes.into_iter() {+ if let Some(pb) = &pb {+ pb.inc(1);+ }++ n += 1;+ centrality.entry(s).or_default();++ let mut stack = Vec::new();+ let mut predecessors: Vec<Vec<u64>> = Vec::new();++ let mut sigma = Vec::new();++ while s as usize >= sigma.len() {+ sigma.push(0);+ }++ sigma[s as usize] = 1;++ let mut distances = Vec::new();+ while s as usize >= distances.len() {+ distances.push(-1);+ }+ distances[s as usize] = 0;++ let mut q = VecDeque::new();+ q.push_back(s);++ while let Some(v) = q.pop_front() {+ stack.push(v);+ for edge in store.outgoing_edges(v) {+ let w = edge.to;++ while w as usize >= distances.len() {+ distances.push(-1);+ }++ if distances[w as usize] == -1 {+ let dist_v = distances[v as usize];+ q.push_back(w);+ distances[w as usize] = dist_v + 1;+ }++ if distances[w as usize] == distances[v as usize] + 1 {+ let sigma_v = *sigma.get(v as usize).unwrap_or(&0);++ while w as usize >= sigma.len() {+ sigma.push(0);+ }++ sigma[w as usize] += sigma_v;++ while w as usize >= predecessors.len() {+ predecessors.push(Vec::new());+ }++ predecessors[w as usize].push(v);+ }+ }+ }++ max_dist = max_dist.max(*distances.iter().max().unwrap_or(&0));++ let mut delta = Vec::new();+ while let Some(w) = stack.pop() {+ if let Some(pred) = predecessors.get(w as usize) {+ for v in pred {+ while *v as usize >= delta.len() {+ delta.push(0.0);+ }++ delta[*v as usize] += (sigma[*v as usize] as f64 / sigma[w as usize] as f64)+ * (1.0 + delta.get(w as usize).unwrap_or(&0.0));+ }+ }++ if w != s {+ *centrality.entry(w).or_insert(0.0) += *delta.get(w as usize).unwrap_or(&0.0);+ }+ }+ }++ if let Some(pb) = &pb {+ pb.finish();+ }++ let n = n as f64;+ let norm = n * (n - 1.0);++ (+ centrality+ .into_iter()+ .map(|(id, centrality)| (store.id2node(&id).unwrap(), centrality / norm))+ .collect(),+ max_dist,+ )+}++#[derive(Debug, Default, Serialize, Deserialize)]+pub struct Betweenness {+ pub centrality: HashMap<Node, f64>,+ pub max_dist: usize,+}++impl Betweenness {+ #[allow(unused)]+ pub fn calculate(graph: &Webgraph) -> Self {+ match &graph.host {+ Some(store) => {+ let (host, max_dist) = calculate(store, false);+ Self {+ centrality: host,+ max_dist: max_dist.max(0) as usize,+ }+ }+ None => Self {+ centrality: HashMap::new(),+ max_dist: 0,+ },+ }+ }++ pub fn calculate_with_progress(graph: &Webgraph) -> Self {+ match &graph.host {+ Some(store) => {+ let (host, max_dist) = calculate(store, true);+ Self {+ centrality: host,+ max_dist: max_dist.max(0) as usize,+ }+ }+ None => Self {+ centrality: HashMap::new(),+ max_dist: 0,+ },+ }+ }+}++#[cfg(test)]+mod tests {+ use maplit::hashmap;++ use crate::webgraph::WebgraphBuilder;++ use super::*;++ fn create_path_graph(n: usize) -> Webgraph {+ // ββββββ+ // β β+ // ββββAβββ β+ // β β β+ // βΌ β β+ // BββββββΊCββ+ // β²+ // β+ // β+ // D++ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ for i in 0..n - 1 {+ graph.insert(+ Node::from(i.to_string()),+ Node::from((i + 1).to_string()),+ String::new(),+ );+ }++ graph.flush();++ graph+ }++ #[test]+ fn path() {+ let p = create_path_graph(5);+ let centrality = Betweenness::calculate(&p);++ assert_eq!(+ centrality.centrality,+ hashmap! {+ Node::from("0".to_string()) => 0.0,+ Node::from("1".to_string()) => 0.15,+ Node::from("2".to_string()) => 0.2,+ Node::from("3".to_string()) => 0.15,+ Node::from("4".to_string()) => 0.0,+ }+ );+ }+}diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rsnew file mode 100644index 00000000..c446a577--- /dev/null+++ b/src/webgraph/centrality/harmonic.rs@@ -0,0 +1,198 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::collections::HashMap;++use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};+use tracing::info;++use crate::webgraph::{graph_store::GraphStore, Node, NodeID, Store, Webgraph};++pub struct HarmonicCentrality {+ pub full: HashMap<Node, f64>,+ pub host: HashMap<Node, f64>,+}++fn calculate_centrality<S, F>(graph: &GraphStore<S>, node_distances: F) -> HashMap<Node, f64>+where+ S: Store,+ F: Fn(Node) -> HashMap<NodeID, usize>,+{+ let nodes: Vec<_> = graph.nodes().collect();+ info!("Found {} nodes in the graph", nodes.len());+ let pb = ProgressBar::new(nodes.len() as u64);+ pb.set_style(+ ProgressStyle::default_bar()+ .template("{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})")+ .progress_chars("#>-"),+ );+ let norm_factor = (nodes.len() - 1) as f64;+ nodes+ .iter()+ .progress_with(pb)+ .map(|node_id| {+ let node = graph.id2node(node_id).expect("unknown node");+ let centrality_values: HashMap<NodeID, f64> = node_distances(node.clone())+ .into_iter()+ .filter(|(other_id, _)| *other_id != *node_id)+ .map(|(other_node, dist)| (other_node, 1f64 / dist as f64))+ .collect();++ let centrality = centrality_values+ .into_iter()+ .map(|(_, val)| val)+ .sum::<f64>()+ / norm_factor;++ (node, centrality)+ })+ .filter(|(_, centrality)| *centrality > 0.0)+ .collect()+}++fn calculate_full(graph: &Webgraph) -> HashMap<Node, f64> {+ graph+ .full+ .as_ref()+ .map(|full_graph| {+ calculate_centrality(full_graph, |node| graph.raw_reversed_distances(node))+ })+ .unwrap_or_default()+}++fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {+ graph+ .host+ .as_ref()+ .map(|host_graph| {+ calculate_centrality(host_graph, |node| graph.raw_host_reversed_distances(node))+ })+ .unwrap_or_default()+}++impl HarmonicCentrality {+ pub fn calculate(graph: &Webgraph) -> Self {+ Self {+ host: calculate_host(graph),+ full: calculate_full(graph),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::webgraph::WebgraphBuilder;++ fn test_graph() -> Webgraph {+ // ββββββ+ // β β+ // ββββAβββ β+ // β β β+ // βΌ β β+ // BββββββΊCββ+ // β²+ // β+ // β+ // D++ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("B"), Node::from("C"), String::new());+ graph.insert(Node::from("A"), Node::from("C"), String::new());+ graph.insert(Node::from("C"), Node::from("A"), String::new());+ graph.insert(Node::from("D"), Node::from("C"), String::new());++ graph.flush();++ graph+ }++ #[test]+ fn host_harmonic_centrality() {+ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());+ graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());+ graph.insert(Node::from("A.com/1"), Node::from("A.com/4"), String::new());+ graph.insert(Node::from("A.com/2"), Node::from("A.com/1"), String::new());+ graph.insert(Node::from("A.com/2"), Node::from("A.com/3"), String::new());+ graph.insert(Node::from("A.com/2"), Node::from("A.com/4"), String::new());+ graph.insert(Node::from("A.com/3"), Node::from("A.com/1"), String::new());+ graph.insert(Node::from("A.com/3"), Node::from("A.com/2"), String::new());+ graph.insert(Node::from("A.com/3"), Node::from("A.com/4"), String::new());+ graph.insert(Node::from("A.com/4"), Node::from("A.com/1"), String::new());+ graph.insert(Node::from("A.com/4"), Node::from("A.com/2"), String::new());+ graph.insert(Node::from("A.com/4"), Node::from("A.com/3"), String::new());+ graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());+ graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());++ graph.flush();++ let centrality = HarmonicCentrality::calculate(&graph);+ assert!(+ centrality.full.get(&Node::from("A.com/1")).unwrap()+ > centrality.full.get(&Node::from("B.com")).unwrap()+ );++ assert!(+ centrality.host.get(&Node::from("B.com")).unwrap()+ > centrality.host.get(&Node::from("A.com")).unwrap_or(&0.0)+ );+ }++ #[test]+ fn harmonic_centrality() {+ let graph = test_graph();++ let centrality = HarmonicCentrality::calculate(&graph);++ assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);+ assert_eq!(centrality.full.get(&Node::from("D")), None);+ assert_eq!(+ (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,+ 0.67+ );+ assert_eq!(+ (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,+ 0.61+ );+ }++ #[test]+ fn www_subdomain_ignored() {+ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());+ graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());++ graph.flush();+ let centrality = HarmonicCentrality::calculate(&graph);++ assert_eq!(centrality.host.get(&Node::from("A.com")), Some(&1.0));+ assert_eq!(centrality.host.get(&Node::from("www.A.com")), None);+ }+}diff --git a/src/webgraph/centrality/mod.rs b/src/webgraph/centrality/mod.rsnew file mode 100644index 00000000..885a3ea5--- /dev/null+++ b/src/webgraph/centrality/mod.rs@@ -0,0 +1,19 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod approximate_harmonic;+pub mod betweenness;+pub mod harmonic;diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rsindex e5420797..4140382f 100644--- a/src/webgraph/graph_store.rs+++ b/src/webgraph/graph_store.rs@@ -124,9 +124,13 @@ wherefn insert(&mut self, key: K, value: V) {if self.cache.len() == self.cache.cap() {- if let Some((key, value)) = self.cache.pop_lru() {- self.store.insert(key, value);+ while self.cache.len() > self.cache.cap() / 2 {+ if let Some((key, value)) = self.cache.pop_lru() {+ self.store.insert(key, value);+ }}++ self.store.flush();}self.cache.push(key, value);@@ -325,9 +329,12 @@ impl Store for RocksDbStore {}fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self> {- let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));- let reversed_adjacency =- RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));+ let adjacency = Box::new(+ RocksDbStore::open_read_only(path.as_ref().join("adjacency")).load_in_memory(),+ );+ let reversed_adjacency = Box::new(+ RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency")).load_in_memory(),+ );let node2id = RocksDbStore::open_read_only(path.as_ref().join("node2id"));let id2node = RocksDbStore::open_read_only(path.as_ref().join("id2node"));let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex ce776e1a..3cc2a9e4 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -15,13 +15,11 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.mod graph_store;-use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};use serde::{Deserialize, Serialize};use std::collections::{BinaryHeap, HashMap};use std::path::Path;use std::sync::Mutex;use std::{cmp, fs};-use tracing::info;use graph_store::GraphStore;@@ -31,6 +29,8 @@ use crate::webpage::Url;use self::graph_store::Adjacency;use crate::kv::rocksdb_store::RocksDbStore;+pub mod centrality;+type NodeID = u64;#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]@@ -45,7 +45,7 @@ pub struct Node {}impl Node {- fn into_host(self) -> Node {+ pub fn into_host(self) -> Node {let url = Url::from(self.name);let host = url.host_without_specific_subdomains();@@ -54,6 +54,10 @@ impl Node {name: host.to_string(),}}++ pub fn from_url(url: &Url) -> Self {+ Node::from(url.full())+ }}impl From<String> for Node {@@ -211,14 +215,14 @@ impl WebgraphBuilder {pub fn open(self) -> Webgraph {if self.read_only {Webgraph {- full_graph: self.full_graph_path.map(GraphStore::open_read_only),- host_graph: self.host_graph_path.map(GraphStore::open_read_only),+ full: self.full_graph_path.map(GraphStore::open_read_only),+ host: self.host_graph_path.map(GraphStore::open_read_only),path: self.path.to_str().unwrap().to_string(),}} else {Webgraph {- full_graph: self.full_graph_path.map(GraphStore::open),- host_graph: self.host_graph_path.map(GraphStore::open),+ full: self.full_graph_path.map(GraphStore::open),+ host: self.host_graph_path.map(GraphStore::open),path: self.path.to_str().unwrap().to_string(),}}@@ -239,31 +243,31 @@ wherepub struct Webgraph<S: Store = RocksDbStore> {pub path: String,- full_graph: Option<GraphStore<S>>,- host_graph: Option<GraphStore<S>>,+ pub full: Option<GraphStore<S>>,+ pub host: Option<GraphStore<S>>,}impl<S: Store> Webgraph<S> {pub fn insert(&mut self, from: Node, to: Node, label: String) {- if let Some(full_graph) = &mut self.full_graph {+ if let Some(full_graph) = &mut self.full {full_graph.insert(from.clone(), to.clone(), label.clone());}- if let Some(host_graph) = &mut self.host_graph {+ if let Some(host_graph) = &mut self.host {host_graph.insert(from.into_host(), to.into_host(), label);}}pub fn merge(&mut self, other: Webgraph<S>) {- match (&mut self.full_graph, other.full_graph) {+ match (&mut self.full, other.full) {(Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),- (None, Some(other_graph)) => self.full_graph = Some(other_graph),+ (None, Some(other_graph)) => self.full = Some(other_graph),(Some(_), None) | (None, None) => {}}- match (&mut self.host_graph, other.host_graph) {+ match (&mut self.host, other.host) {(Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),- (None, Some(other_graph)) => self.host_graph = Some(other_graph),+ (None, Some(other_graph)) => self.host = Some(other_graph),(Some(_), None) | (None, None) => {}}@@ -315,7 +319,7 @@ impl<S: Store> Webgraph<S> {#[allow(unused)]pub fn distances(&self, source: Node) -> HashMap<Node, usize> {- self.full_graph+ self.full.as_ref().map(|full_graph| {let distances = Webgraph::dijkstra(@@ -335,7 +339,7 @@ impl<S: Store> Webgraph<S> {#[allow(unused)]fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {- self.full_graph+ self.full.as_ref().map(|full_graph| {Webgraph::dijkstra(@@ -350,7 +354,7 @@ impl<S: Store> Webgraph<S> {#[allow(unused)]pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {- self.full_graph+ self.full.as_ref().map(|full_graph| {self.raw_reversed_distances(source)@@ -363,26 +367,33 @@ impl<S: Store> Webgraph<S> {#[allow(unused)]pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {- self.host_graph+ self.host.as_ref().map(|host_graph| {- let distances = Webgraph::dijkstra(+ self.raw_host_distances(source)+ .into_iter()+ .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))+ .collect()+ })+ .unwrap_or_default()+ }++ pub fn raw_host_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ self.host+ .as_ref()+ .map(|host_graph| {+ Webgraph::dijkstra(source,|node| host_graph.outgoing_edges(node),|edge| edge.to,host_graph,- );-- distances- .into_iter()- .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))- .collect()+ )}).unwrap_or_default()}- fn raw_host_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {- self.host_graph+ pub fn raw_host_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ self.host.as_ref().map(|host_graph| {Webgraph::dijkstra(@@ -397,7 +408,7 @@ impl<S: Store> Webgraph<S> {#[allow(unused)]pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {- self.host_graph+ self.host.as_ref().map(|host_graph| {self.raw_host_reversed_distances(source)@@ -408,76 +419,17 @@ impl<S: Store> Webgraph<S> {.unwrap_or_default()}- fn calculate_centrality<F>(graph: &GraphStore<S>, node_distances: F) -> HashMap<Node, f64>- where- F: Fn(Node) -> HashMap<NodeID, usize>,- {- let nodes: Vec<_> = graph.nodes().collect();- info!("Found {} nodes in the graph", nodes.len());- let pb = ProgressBar::new(nodes.len() as u64);- pb.set_style(- ProgressStyle::default_bar()- .template(- "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",- )- .progress_chars("#>-"),- );- let norm_factor = (nodes.len() - 1) as f64;- nodes- .iter()- .progress_with(pb)- .map(|node_id| {- let node = graph.id2node(node_id).expect("unknown node");- let centrality_values: HashMap<NodeID, f64> = node_distances(node.clone())- .into_iter()- .filter(|(other_id, _)| *other_id != *node_id)- .map(|(other_node, dist)| (other_node, 1f64 / dist as f64))- .collect();-- let centrality = centrality_values- .into_iter()- .map(|(_, val)| val)- .sum::<f64>()- / norm_factor;-- (node, centrality)- })- .filter(|(_, centrality)| *centrality > 0.0)- .collect()- }-- #[allow(unused)]- pub fn harmonic_centrality(&self) -> HashMap<Node, f64> {- self.full_graph- .as_ref()- .map(|full_graph| {- Webgraph::calculate_centrality(full_graph, |node| self.raw_reversed_distances(node))- })- .unwrap_or_default()- }-- pub fn host_harmonic_centrality(&self) -> HashMap<Node, f64> {- self.host_graph- .as_ref()- .map(|host_graph| {- Webgraph::calculate_centrality(host_graph, |node| {- self.raw_host_reversed_distances(node)- })- })- .unwrap_or_default()- }-pub fn flush(&self) {- if let Some(full_graph) = &self.full_graph {+ if let Some(full_graph) = &self.full {full_graph.flush();}- if let Some(host_graph) = &self.host_graph {+ if let Some(host_graph) = &self.host {host_graph.flush();}}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {- if let Some(graph) = &self.full_graph {+ if let Some(graph) = &self.full {if let Some(node_id) = graph.node2id(&node) {graph.ingoing_edges(node_id)@@ -533,8 +485,8 @@ impl From<Webgraph> for FrozenWebgraph {fn from(graph: Webgraph) -> Self {graph.flush();let path = graph.path.clone();- let has_full = graph.full_graph.is_some();- let has_host = graph.host_graph.is_some();+ let has_full = graph.full.is_some();+ let has_host = graph.host.is_some();drop(graph);let root = directory::scan_folder(path).unwrap();@@ -620,78 +572,6 @@ mod test {assert_eq!(distances.get(&Node::from("B")), Some(&2));}- #[test]- fn harmonic_centrality() {- let graph = test_graph();-- let centrality = graph.harmonic_centrality();-- assert_eq!(centrality.get(&Node::from("C")).unwrap(), &1.0);- assert_eq!(centrality.get(&Node::from("D")), None);- assert_eq!(- (*centrality.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,- 0.67- );- assert_eq!(- (*centrality.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,- 0.61- );- }-- #[test]- fn host_harmonic_centrality() {- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();-- graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());- graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());- graph.insert(Node::from("A.com/1"), Node::from("A.com/4"), String::new());- graph.insert(Node::from("A.com/2"), Node::from("A.com/1"), String::new());- graph.insert(Node::from("A.com/2"), Node::from("A.com/3"), String::new());- graph.insert(Node::from("A.com/2"), Node::from("A.com/4"), String::new());- graph.insert(Node::from("A.com/3"), Node::from("A.com/1"), String::new());- graph.insert(Node::from("A.com/3"), Node::from("A.com/2"), String::new());- graph.insert(Node::from("A.com/3"), Node::from("A.com/4"), String::new());- graph.insert(Node::from("A.com/4"), Node::from("A.com/1"), String::new());- graph.insert(Node::from("A.com/4"), Node::from("A.com/2"), String::new());- graph.insert(Node::from("A.com/4"), Node::from("A.com/3"), String::new());- graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());- graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());-- graph.flush();-- let centrality = graph.harmonic_centrality();- assert!(- centrality.get(&Node::from("A.com/1")).unwrap()- > centrality.get(&Node::from("B.com")).unwrap()- );-- let host_centrality = graph.host_harmonic_centrality();- assert!(- host_centrality.get(&Node::from("B.com")).unwrap()- > host_centrality.get(&Node::from("A.com")).unwrap_or(&0.0)- );- }-- #[test]- fn www_subdomain_ignored() {- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();-- graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());- graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());-- graph.flush();-- let centrality = graph.host_harmonic_centrality();- assert_eq!(centrality.get(&Node::from("A.com")), Some(&1.0));- assert_eq!(centrality.get(&Node::from("www.A.com")), None);- }-#[test]fn merge() {let mut graph1 = WebgraphBuilder::new_memory()diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 483eebbb..39c91d47 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -128,6 +128,7 @@ pub struct Webpage {pub fetch_time_ms: u64,pub pre_computed_score: f64,pub primary_image: Option<StoredPrimaryImage>,+ pub node_id: Option<u64>,}impl Webpage {@@ -143,6 +144,7 @@ impl Webpage {fetch_time_ms: 0,pre_computed_score: 0.0,primary_image: None,+ node_id: None,}}@@ -216,6 +218,13 @@ impl Webpage {image,);+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::HostNodeID).name())+ .expect("Failed to get node_id field"),+ self.node_id.unwrap_or(u64::MAX),+ );+Ok(doc)}@@ -697,6 +706,7 @@ impl Html {| Field::Fast(FastField::FetchTimeMs)| Field::Fast(FastField::PreComputedScore)| Field::Fast(FastField::Region)+ | Field::Fast(FastField::HostNodeID)| Field::Text(TextField::PrimaryImage) => {}}}diff --git a/testcases/goggles/quickstart.goggle b/testcases/goggles/quickstart.goggleindex 2b23842e..37d146ae 100644--- a/testcases/goggles/quickstart.goggle+++ b/testcases/goggles/quickstart.goggle@@ -69,7 +69,7 @@ $site=example.com! web3$indescription! web3$incontent-! Finally, you can specify an 'action', which indicates how the ranking of a+! You can specify an 'action', which indicates how the ranking of a! matched result should be changed by your instruction. This is the mechanism! through which you can customize the ranking of results to your liking. You can! use one of three possible actions in your instructions, and by default, any@@ -89,10 +89,40 @@ $site=example.com/r/google/$downrank=2/r/google/$downrank=3-! Tou can also discard results completely:+! You can also discard results completely:$discard,site=idontwanttobepartoftheresults.com/this/is/spam/$discard+! Most search engines analyze how websites link to eachother to figure out, which+! sites are better than others. Google popularized this with their+! [PageRank](https://en.wikipedia.org/wiki/PageRank) algorithm which is often+! credited for their great search results when they launched. At Cuely we+! calculate the [Harmonic Centrality](https://en.wikipedia.org/wiki/Centrality#Harmonic_centrality)+! and use this to rank websites (along with many other signals).+! The thing is, people tend to like and dislike different websites. What+! Harmonic Centrality (or PageRank for that matter) finds to be good websites might not+! be what you think of as good websites. You should be able to personalize this value.+! You can therefore specify a set of sites that you like, and a set of sites that you+! dislike. A Harmonic Centrality value is then approxmiated from the sites you choose+! during search and used in the ranking.++! The liked and disliked sites are specified as follows+liked = [+ ilikethissite.com,+ example.com,+]++disliked = [+ evilwebsite.com,+ githubspam.com,+]++! It is important to note, that the sites you like and dislike here adjusts the ranking for all+! sites, not just the ones specified. We calculate the distance from your specified sites+! to all sites in the search results, and essentially prioritize sites that are closely+! linked with your liked sites while not being closely linked to your disliked sites.++! Last but not least, you can control exactly how the signals gets prioritized for! the final ranking. We countinously add more signals used during ranking,! but you can e.g. heavily prioritize webpages with a high centrality by specifying