Skip to content

web search done right

remove fragments from page graph nodes

Mikkel Denker90653c51cc5d
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rs
index 9cddf89a..855c2dc4 100644
--- a/crates/core/src/api/webgraph.rs
+++ b/crates/core/src/api/webgraph.rs
@@ -157,7 +157,9 @@ pub mod host {
)
.await
{
- Ok(Some(node)) => Ok(Json(KnowsHost::Known { host: node.name })),
+ Ok(Some(node)) => Ok(Json(KnowsHost::Known {
+ host: node.as_str().to_string(),
+ })),
Err(err) => {
tracing::error!("Failed to send request to webgraph: {}", err);
Ok(Json(KnowsHost::Unknown))
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 08d434d2..94416ce6 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -83,7 +83,7 @@ impl CrawlPlanner {
for host in hosts {
let node = self.host_graph.id2node(host).unwrap();
- if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {
+ if let Ok(url) = Url::parse(&format!("http://{}", node.as_str())) {
let domain = Domain::from(url);
domains.entry(domain).or_default().push(*host);
}
@@ -156,14 +156,14 @@ impl CrawlPlanner {
total_wander_budget += wander_budget;
total_schedule_budget += schedule_budget;
- let host_name = self.host_graph.id2node(host).unwrap().name.clone();
+ let host_name = self.host_graph.id2node(host).unwrap().as_str().to_string();
let before = urls.len();
urls.extend(
pages
.into_iter()
.filter_map(|(id, score)| self.page_graph.id2node(&id).map(|n| (n, score)))
- .map(|(n, score)| (n.name, score))
+ .map(|(n, score)| (n.as_str().to_string(), score))
.filter_map(|(n, score)| {
Url::parse(&format!("http://{n}")).ok().map(|u| (u, score))
})
diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rs
index b9e3a893..d3f7bdd0 100644
--- a/crates/core/src/entrypoint/centrality.rs
+++ b/crates/core/src/entrypoint/centrality.rs
@@ -40,7 +40,7 @@ fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {
data.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
let mut wtr = csv::Writer::from_writer(csv_file);
for (node, centrality) in data {
- wtr.write_record(&[node.name, centrality.to_string()])
+ wtr.write_record(&[node.as_str().to_string(), centrality.to_string()])
.unwrap();
}
wtr.flush().unwrap();
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index d9a17cd4..db87f7c0 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -92,7 +92,7 @@ impl Message<WebGraphService> for SimilarHosts {
let urls = similar_hosts
.iter()
- .filter_map(|s| Url::parse(&("http://".to_string() + s.node.name.as_str())).ok())
+ .filter_map(|s| Url::parse(&("http://".to_string() + s.node.as_str())).ok())
.collect_vec();
let descriptions = server.searcher.get_homepage_descriptions(&urls).await;
@@ -100,12 +100,12 @@ impl Message<WebGraphService> for SimilarHosts {
let similar_hosts = similar_hosts
.into_iter()
.map(|site| {
- let description = Url::parse(&("http://".to_string() + site.node.name.as_str()))
+ let description = Url::parse(&("http://".to_string() + site.node.as_str()))
.ok()
.and_then(|url| descriptions.get(&url).cloned());
ScoredHost {
- host: site.node.name,
+ host: site.node.as_str().to_string(),
score: site.score,
description,
}
diff --git a/crates/core/src/feed/scheduler.rs b/crates/core/src/feed/scheduler.rs
index 6100da9a..d3598bae 100644
--- a/crates/core/src/feed/scheduler.rs
+++ b/crates/core/src/feed/scheduler.rs
@@ -115,7 +115,7 @@ pub fn schedule(
for host in top_hosts {
let host = host_graph.id2node(&host).unwrap();
- let url = Url::parse(&format!("http://{}", host.name));
+ let url = Url::parse(&format!("http://{}", host.as_str()));
if url.is_err() {
continue;
@@ -127,7 +127,7 @@ pub fn schedule(
all_feeds
.entry(domain)
.or_insert(HashSet::new())
- .extend(index.search(&host.name).unwrap().into_iter());
+ .extend(index.search(host.as_str()).unwrap().into_iter());
}
let mut splits = Vec::new();
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 54797472..a5d46ad0 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -89,7 +89,7 @@ impl SimilarHostsFinder {
let domains = nodes
.iter()
.filter_map(|node| {
- Url::parse(&format!("http://{}", &node.name))
+ Url::parse(&format!("http://{}", &node.as_str()))
.ok()
.and_then(|url| url.root_domain().map(|d| d.to_string()))
})
@@ -161,7 +161,7 @@ impl SimilarHostsFinder {
})
.filter_map(|ScoredNodeID { node_id, score }| {
let node = self.webgraph.id2node(&node_id).unwrap();
- match Url::parse(&format!("http://{}", &node.name))
+ match Url::parse(&format!("http://{}", &node.as_str()))
.ok()
.and_then(|url| url.root_domain().map(|s| s.to_string()))
{
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 7cc19220..25526f93 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -171,7 +171,7 @@ pub struct FullEdge {
)]
#[serde(rename_all = "camelCase")]
pub struct Node {
- pub name: String,
+ name: String,
}
impl Node {
@@ -193,6 +193,10 @@ impl Node {
}
}
+ pub fn as_str(&self) -> &str {
+ self.name.as_str()
+ }
+
pub fn id(&self) -> NodeID {
let digest = md5::compute(self.name.as_bytes());
u128::from_le_bytes(*digest).into()
@@ -232,29 +236,7 @@ impl From<Url> for Node {
pub fn normalize_url(url: &Url) -> String {
let mut url = url.clone();
- let allowed_queries: Vec<_> = url
- .query_pairs()
- .filter(|(key, _)| {
- !key.starts_with("utm_")
- && !key.starts_with("fbclid")
- && !key.starts_with("gclid")
- && !key.starts_with("msclkid")
- })
- .map(|(key, value)| (key.to_string(), value.to_string()))
- .collect();
-
- {
- let mut queries = url.query_pairs_mut();
- queries.clear();
-
- if !allowed_queries.is_empty() {
- queries.extend_pairs(allowed_queries);
- }
- }
-
- if url.query().unwrap_or_default().is_empty() {
- url.set_query(None);
- }
+ url.normalize();
let scheme = url.scheme();
let mut normalized = url
@@ -1137,4 +1119,13 @@ mod test {
]
);
}
+
+ #[test]
+ fn test_node_normalized() {
+ let n = Node::from("http://www.example.com/abc");
+ assert_eq!(n.as_str(), "example.com/abc");
+
+ let n = Node::from("http://www.example.com/abc#123");
+ assert_eq!(n.as_str(), "example.com/abc");
+ }
}
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 59175ad4..0b93b31e 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -65,8 +65,7 @@ impl Html {
pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {
let root = kuchiki::parse_html().one(html);
- let mut url = Url::parse(url)?;
- url.set_fragment(None); // remove fragment (e.g. #comments
+ let url = Url::parse(url)?;
let mut res = Self {
root,
@@ -77,31 +76,13 @@ impl Html {
robots: None,
};
- let queries: Vec<_> = res
- .url
- .query_pairs()
- .filter(|(key, _)| !key.starts_with("utm_"))
- .map(|(key, value)| (key.to_string(), value.to_string()))
- .collect();
-
- {
- let mut query_mut = res.url.query_pairs_mut();
- query_mut.clear();
- if !queries.is_empty() {
- query_mut.extend_pairs(queries);
- }
- }
-
if let Some(canonical) = res.canonical_url() {
if canonical.root_domain() == res.url.root_domain() {
res.url = canonical;
}
}
- if res.url.query().unwrap_or_default().is_empty() {
- res.url.set_query(None);
- }
-
+ res.url.normalize();
res.robots = res.parse_robots_meta();
Ok(res)
@@ -1126,7 +1107,7 @@ mod tests {
<title data-rh="true">Russian Ships Near Data Cables Are Too Close for U.S. Comfort - The New York Times</title>
<meta data-rh="true" name="robots" content="noarchive, max-image-preview:large"/><meta data-rh="true" name="description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="og:url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Russian Ships Near Data Cables Are Too Close for U.S. Comfort (Published 2015)"/><meta data-rh="true" property="og:image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-facebookJumbo.jpg?year=2015&amp;h=550&amp;w=1050&amp;s=ceeb5555ae80dd2e2fe05553973d3948298bcba76c8d62a24907ae21e61140a7&amp;k=ZQJBKqZ0VN"/><meta data-rh="true" property="og:image:alt" content="Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year."/><meta data-rh="true" property="og:description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="twitter:url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="twitter:title" content="Russian Ships Near Data Cables Are Too Close for U.S. Comfort (Published 2015)"/><meta data-rh="true" property="twitter:description" content="Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S."/><meta data-rh="true" property="twitter:image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-articleLarge.jpg?year=2015&amp;h=400&amp;w=600&amp;s=1c25983f231db27a32cbc8063f067135d143de5f7c2d671b5798f0b76507c68f&amp;k=ZQJBKqZ0VN&amp;tw=1"/><meta data-rh="true" property="twitter:image:alt" content="Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year."/><meta data-rh="true" property="twitter:card" content="summary_large_image"/> <link data-rh="true" rel="canonical" href="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><link data-rh="true" rel="alternate" href="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><link data-rh="true" rel="alternate" type="application/json+oembed" href="https://www.nytimes.com/svc/oembed/json/?url=https%3A%2F%2Fwww.nytimes.com%2F2015%2F10%2F26%2Fworld%2Feurope%2Frussian-presence-near-undersea-cables-concerns-us.html" title="Russian Ships Near Data Cables Are Too Close for U.S. Comfort"/> <script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"NewsArticle","description":"Russian submarines are aggressively operating near the undersea cables that carry almost all global Internet communications, raising concerns in the U.S.","image":{"@context":"http://schema.org","@type":"ImageObject","url":"https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-superJumbo.jpg","height":1365,"width":2048,"contentUrl":"https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-superJumbo.jpg","caption":"Adm. Mark Ferguson said the intensity of Russian submarine patrols had risen by almost 50 percent over the last year.","creditText":"Brendan Smialowski/Agence France-Presse — Getty Images"},"mainEntityOfPage":"https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html","url":"https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html","author":[{"@context":"http://schema.org","@type":"Person","url":"http://www.nytimes.com/by/david-e-sanger","name":"David E. Sanger"},{"@context":"http://schema.org","@type":"Person","url":"http://www.nytimes.com/by/eric-schmitt","name":"Eric Schmitt"}],"dateModified":"2015-10-25T17:02:34.000Z","datePublished":"2015-10-25T17:02:35.000Z","headline":"Russian Ships Near Data Cables Are Too Close for U.S. Comfort","alternativeHeadline":"Russian Ships Near Data Cables Are Too Close for U.S. Comfort","publisher":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"hasPart":{"@type":"WebPageElement","isAccessibleForFree":false,"cssSelector":".meteredContent"},"copyrightHolder":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"sourceOrganization":{"@id":"https://www.nytimes.com/#publisher","name":"The New York Times"},"copyrightYear":2024,"isAccessibleForFree":false,"isPartOf":{"@type":["CreativeWork","Product"],"name":"The New York Times","productID":"nytimes.com:basic"}}</script><script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"NewsMediaOrganization","name":"The New York Times","logo":{"@context":"http://schema.org","@type":"ImageObject","url":"https://static01.nyt.com/images/icons/t_logo_291_black.png","height":291,"width":291,"contentUrl":"https://static01.nyt.com/images/icons/t_logo_291_black.png","creditText":"The New York Times"},"url":"https://www.nytimes.com/","@id":"https://www.nytimes.com/#publisher","diversityPolicy":"https://www.nytco.com/company/diversity-and-inclusion/","ethicsPolicy":"https://www.nytco.com/company/standards-ethics/","masthead":"https://www.nytimes.com/interactive/2023/01/28/admin/the-new-york-times-masthead.html","foundingDate":"1851-09-18","sameAs":"https://en.wikipedia.org/wiki/The_New_York_Times"}</script><script data-rh="true" type="application/ld+json">{"@context":"http://schema.org","@type":"BreadcrumbList","itemListElement":[{"@context":"http://schema.org","@type":"ListItem","name":"World","position":1,"item":"https://www.nytimes.com/section/world"},{"@context":"http://schema.org","@type":"ListItem","name":"Europe","position":2,"item":"https://www.nytimes.com/section/world/europe"}]}</script>
<meta data-rh="true" property="article:published_time" content="2015-10-25T17:02:35.000Z"/><meta data-rh="true" property="article:modified_time" content="2015-10-25T17:02:34.000Z"/><meta data-rh="true" http-equiv="Content-Language" content="en"/><meta data-rh="true" name="articleid" content="100000003995797"/><meta data-rh="true" name="nyt_uri" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" name="pubp_event_id" content="pubp://event/7219283749a6486e8ec2777e0840ab41"/><meta data-rh="true" name="image" content="https://static01.nyt.com/images/2015/10/26/world/JP-SUBS/JP-SUBS-facebookJumbo.jpg?year=2015&amp;h=550&amp;w=1050&amp;s=ceeb5555ae80dd2e2fe05553973d3948298bcba76c8d62a24907ae21e61140a7&amp;k=ZQJBKqZ0VN"/><meta data-rh="true" name="byl" content="By David E. Sanger and Eric Schmitt"/><meta data-rh="true" name="news_keywords" content=",Russia,,Submarines and Submersibles,Wire and Cable,Computers and the Internet,Tech Industry,US,Homeland Security,Spying and Intelligence Agencies,US Military,Military,Mark Ferguson"/><meta data-rh="true" name="pdate" content="20151025"/><meta data-rh="true" property="article:section" content="World"/><meta data-rh="true" property="article:author" content="http://www.nytimes.com/by/david-e-sanger"/><meta data-rh="true" property="article:author" content="http://www.nytimes.com/by/eric-schmitt"/><meta data-rh="true" property="article:tag" content="Schmitt, Eric"/><meta data-rh="true" property="article:tag" content="Russia"/><meta data-rh="true" property="article:tag" content="Sanger, David E"/><meta data-rh="true" property="article:tag" content="Submarines and Submersibles"/><meta data-rh="true" property="article:tag" content="Wire and Cable"/><meta data-rh="true" property="article:tag" content="Computers and the Internet"/><meta data-rh="true" property="article:tag" content="United States"/><meta data-rh="true" property="article:tag" content="Homeland Security Department"/><meta data-rh="true" property="article:tag" content="Espionage and Intelligence Services"/><meta data-rh="true" property="article:tag" content="United States Defense and Military Forces"/><meta data-rh="true" property="article:tag" content="Defense and Military Forces"/><meta data-rh="true" property="article:tag" content="Ferguson, Mark E III"/><meta data-rh="true" property="article:opinion" content="false"/><meta data-rh="true" property="article:content_tier" content="metered"/><meta data-rh="true" name="CG" content="world"/><meta data-rh="true" name="SCG" content="europe"/><meta data-rh="true" name="CN" content=""/><meta data-rh="true" name="CT" content=""/><meta data-rh="true" name="PT" content="article"/><meta data-rh="true" name="PST" content="News"/><meta data-rh="true" name="url" content="https://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" name="msapplication-starturl" content="https://www.nytimes.com"/><meta data-rh="true" property="al:android:url" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" property="al:android:package" content="com.nytimes.android"/><meta data-rh="true" property="al:android:app_name" content="NYTimes"/><meta data-rh="true" name="twitter:app:name:googleplay" content="NYTimes"/><meta data-rh="true" name="twitter:app:id:googleplay" content="com.nytimes.android"/><meta data-rh="true" name="twitter:app:url:googleplay" content="nyt://article/a608af24-ea77-5f40-b5ee-e830ccd0f1fd"/><meta data-rh="true" property="al:iphone:url" content="nytimes://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="al:iphone:app_store_id" content="284862083"/><meta data-rh="true" property="al:iphone:app_name" content="NYTimes"/><meta data-rh="true" property="al:ipad:url" content="nytimes://www.nytimes.com/2015/10/26/world/europe/russian-presence-near-undersea-cables-concerns-us.html"/><meta data-rh="true" property="al:ipad:app_store_id" content="357066198"/><meta data-rh="true" property="al:ipad:app_name" content="NYTimes"/>
-
+
</head>
<body>
TEST
diff --git a/crates/core/src/webpage/url_ext.rs b/crates/core/src/webpage/url_ext.rs
index 48bd5034..83c2281c 100644
--- a/crates/core/src/webpage/url_ext.rs
+++ b/crates/core/src/webpage/url_ext.rs
@@ -32,6 +32,7 @@ pub trait UrlExt {
fn icann_domain(&self) -> Option<&str>;
fn root_domain(&self) -> Option<&str>;
fn normalized_host(&self) -> Option<&str>;
+ fn normalize(&mut self);
fn subdomain(&self) -> Option<&str>;
fn is_homepage(&self) -> bool;
fn tld(&self) -> Option<&str>;
@@ -55,6 +56,33 @@ impl UrlExt for url::Url {
self.host_str().map(|host| host.trim_start_matches("www."))
}
+ fn normalize(&mut self) {
+ self.set_fragment(None); // remove fragment (e.g. #comments
+
+ let queries: Vec<_> = self
+ .query_pairs()
+ .filter(|(key, _)| {
+ !key.starts_with("utm_")
+ && !key.starts_with("fbclid")
+ && !key.starts_with("gclid")
+ && !key.starts_with("msclkid")
+ })
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect();
+
+ {
+ let mut query_mut = self.query_pairs_mut();
+ query_mut.clear();
+ if !queries.is_empty() {
+ query_mut.extend_pairs(queries);
+ }
+ }
+
+ if self.query().unwrap_or_default().is_empty() {
+ self.set_query(None);
+ }
+ }
+
fn subdomain(&self) -> Option<&str> {
let domain = self.root_domain()?;
let host = self.host_str()?;