Skip to content

web search done right

normalize graph nodes whenver a node struct i constructed

Mikkel Denker5f737943cc15
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index ea9f1fb9..2b3afc41 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -125,13 +125,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
})
{
let mut html = Html::parse_without_text(&record.response.body, &record.request.url);
- let node_id = worker.host_centrality_store.node2id.get(
- &html
- .url()
- .host_without_specific_subdomains_and_query()
- .to_string()
- .into(),
- );
+ let node_id = worker.host_centrality_store.node2id.get(&html.url().into());
let host_centrality = node_id
.and_then(|node_id| worker.host_centrality_store.harmonic.get(&node_id))
@@ -175,9 +169,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut page_centrality = 0.0;
if let Some(store) = worker.page_centrality_store.as_ref() {
- let node_id = store
- .node2id
- .get(&html.url().without_protocol().to_lowercase().into());
+ let node_id = store.node2id.get(&html.url().into());
page_centrality = node_id
.and_then(|node_id| store.harmonic.get(&node_id))
@@ -299,7 +291,7 @@ impl Indexer {
let worker = IndexingWorker::new(
config.host_centrality_store_path.clone(),
config.page_centrality_store_path.clone(),
- config.webgraph_path.clone(),
+ config.page_webgraph_path.clone(),
config.topics_path.clone(),
);
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index ce1ce6e7..3a9b8646 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -123,10 +123,8 @@ impl WebgraphWorker {
trace!("inserting link {:?}", link);
let mut source = Node::from(source);
- source.remove_protocol();
let mut destination = Node::from(destination);
- destination.remove_protocol();
if let WebgraphLevel::Host = job.level {
source = source.into_host();
diff --git a/core/src/lib.rs b/core/src/lib.rs
index f746c6c2..b60ca554 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -90,7 +90,7 @@ pub struct IndexingLocalConfig {
skip_warc_files: Option<usize>,
warc_source: WarcSource,
batch_size: Option<usize>,
- webgraph_path: Option<String>,
+ page_webgraph_path: Option<String>,
output_path: Option<String>,
host_centrality_threshold: Option<f64>,
topics_path: Option<String>,
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 16473596..ecc49e2d 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -67,15 +67,13 @@ impl Node {
pub fn from_url(url: &Url) -> Self {
Node::from(url.full())
}
-
- pub fn remove_protocol(&mut self) {
- let url = Url::from(self.name.clone());
- self.name = url.without_protocol().to_string();
- }
}
impl From<String> for Node {
fn from(name: String) -> Self {
+ let url = Url::from(name);
+ let name = url.without_protocol();
+
Self {
name: name.to_lowercase(),
}
@@ -84,15 +82,13 @@ impl From<String> for Node {
impl From<&Url> for Node {
fn from(url: &Url) -> Self {
- Self {
- name: url.without_protocol().to_lowercase(),
- }
+ url.full().into()
}
}
impl From<&str> for Node {
fn from(name: &str) -> Self {
- Self::from(name.to_lowercase())
+ name.to_string().into()
}
}
@@ -782,8 +778,7 @@ mod test {
#[test]
fn remove_protocol() {
- let mut n = Node::from("https://www.example.com?test");
- n.remove_protocol();
+ let n = Node::from("https://www.example.com?test");
assert_eq!(&n.name, "www.example.com?test");
}