web search done right
normalize graph nodes whenver a node struct i constructed
Mikkel Denker5f737943cc15
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex ea9f1fb9..2b3afc41 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -125,13 +125,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}){let mut html = Html::parse_without_text(&record.response.body, &record.request.url);- let node_id = worker.host_centrality_store.node2id.get(- &html- .url()- .host_without_specific_subdomains_and_query()- .to_string()- .into(),- );+ let node_id = worker.host_centrality_store.node2id.get(&html.url().into());let host_centrality = node_id.and_then(|node_id| worker.host_centrality_store.harmonic.get(&node_id))@@ -175,9 +169,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut page_centrality = 0.0;if let Some(store) = worker.page_centrality_store.as_ref() {- let node_id = store- .node2id- .get(&html.url().without_protocol().to_lowercase().into());+ let node_id = store.node2id.get(&html.url().into());page_centrality = node_id.and_then(|node_id| store.harmonic.get(&node_id))@@ -299,7 +291,7 @@ impl Indexer {let worker = IndexingWorker::new(config.host_centrality_store_path.clone(),config.page_centrality_store_path.clone(),- config.webgraph_path.clone(),+ config.page_webgraph_path.clone(),config.topics_path.clone(),);diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex ce1ce6e7..3a9b8646 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -123,10 +123,8 @@ impl WebgraphWorker {trace!("inserting link {:?}", link);let mut source = Node::from(source);- source.remove_protocol();let mut destination = Node::from(destination);- destination.remove_protocol();if let WebgraphLevel::Host = job.level {source = source.into_host();diff --git a/core/src/lib.rs b/core/src/lib.rsindex f746c6c2..b60ca554 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -90,7 +90,7 @@ pub struct IndexingLocalConfig {skip_warc_files: Option<usize>,warc_source: WarcSource,batch_size: Option<usize>,- webgraph_path: Option<String>,+ page_webgraph_path: Option<String>,output_path: Option<String>,host_centrality_threshold: Option<f64>,topics_path: Option<String>,diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 16473596..ecc49e2d 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -67,15 +67,13 @@ impl Node {pub fn from_url(url: &Url) -> Self {Node::from(url.full())}-- pub fn remove_protocol(&mut self) {- let url = Url::from(self.name.clone());- self.name = url.without_protocol().to_string();- }}impl From<String> for Node {fn from(name: String) -> Self {+ let url = Url::from(name);+ let name = url.without_protocol();+Self {name: name.to_lowercase(),}@@ -84,15 +82,13 @@ impl From<String> for Node {impl From<&Url> for Node {fn from(url: &Url) -> Self {- Self {- name: url.without_protocol().to_lowercase(),- }+ url.full().into()}}impl From<&str> for Node {fn from(name: &str) -> Self {- Self::from(name.to_lowercase())+ name.to_string().into()}}@@ -782,8 +778,7 @@ mod test {#[test]fn remove_protocol() {- let mut n = Node::from("https://www.example.com?test");- n.remove_protocol();+ let n = Node::from("https://www.example.com?test");assert_eq!(&n.name, "www.example.com?test");}