Skip to content

web search done right

remove 'utm_*' parameters from urls when normalizing

Mikkel Denkere755068dadc7
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 4c37f154..b8edac77 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -797,7 +797,7 @@ impl Html {
};
let site_hash = split_u128(hash(self.url().site()).0);
- let url_without_query_hash = split_u128(hash(self.url().without_query()).0);
+ let url_without_query_hash = split_u128(hash(self.url().strip_query()).0);
let url_hash = split_u128(hash(self.url().full()).0);
let domain_hash = split_u128(hash(self.url().domain()).0);
let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 4bfef9f2..39531320 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{fmt::Display, time::Duration};
+use std::{fmt::Display, ops::Range, time::Duration};
use publicsuffix::Psl;
use serde::{Deserialize, Serialize};
@@ -65,12 +65,59 @@ impl From<Url> for String {
impl Url {
pub fn normalize(&self) -> Self {
- let res: Url = self.strip_protocol().to_lowercase().into();
+ let mut res: Url = self.strip_protocol().to_lowercase().into();
if let Some(stripped) = res.0.strip_prefix("www.") {
- Url(stripped.to_string())
- } else {
- res
+ res = Url(stripped.to_string())
+ }
+
+ let mut new_url = res.0.clone();
+
+ if let Some(queries) = res.queries() {
+ let mut new_queries = Vec::new();
+ for (key, value) in queries {
+ if !key.starts_with("utm_") {
+ new_queries.push((key, value));
+ }
+ }
+
+ new_url = res.strip_query().to_string();
+ if !new_queries.is_empty() {
+ new_url.push('?');
+ for (key, value) in new_queries {
+ new_url.push_str(key);
+
+ if let Some(value) = value {
+ new_url.push('=');
+ new_url.push_str(value);
+ }
+
+ new_url.push('&');
+ }
+ new_url.pop();
+ }
+ }
+
+ res.0 = new_url;
+
+ res
+ }
+
+ pub fn queries(&self) -> Option<impl Iterator<Item = (&str, Option<&str>)> + '_> {
+ match self.query_range() {
+ Some(range) => {
+ // remove the '?'
+ let query = &self.0[ceil_char_boundary(&self.0, range.start + 1)
+ ..ceil_char_boundary(&self.0, range.end)];
+
+ Some(query.split('&').filter_map(|query| {
+ let mut split = query.split('=');
+ let key = split.next()?;
+ let value = split.next();
+ Some((key, value))
+ }))
+ }
+ None => None,
}
}
@@ -88,15 +135,32 @@ impl Url {
&url[start_host..]
}
- pub fn strip_query(&self) -> &str {
+ fn query_range(&self) -> Option<Range<usize>> {
let url = &self.0;
- let mut start_query = url.len();
- if url.contains('?') {
- start_query = url.find('?').expect("The url contains atleast 1 '?'");
+
+ match url.find('?') {
+ Some(start_query) => {
+ let start_query = ceil_char_boundary(url, start_query);
+ let mut end_query = url.len();
+
+ if let Some(hash) = url.find('#') {
+ end_query = ceil_char_boundary(url, hash);
+ }
+
+ Some(start_query..end_query)
+ }
+ None => None,
}
+ }
+
+ pub fn strip_query(&self) -> &str {
+ let url = &self.0;
- let start_query = ceil_char_boundary(url, start_query);
- &url[..start_query]
+ if let Some(query_range) = self.query_range() {
+ &url[..query_range.start]
+ } else {
+ url
+ }
}
pub fn site(&self) -> &str {
@@ -278,15 +342,6 @@ impl Url {
}
}
- pub fn without_query(&self) -> &str {
- if let Some(query_begin) = self.0.find('?') {
- let idx = floor_char_boundary(&self.0, query_begin);
- &self.0[..idx]
- } else {
- &self.0
- }
- }
-
pub fn full_without_id_tags(&self) -> String {
let full = self.full();
if let Some(id_begin) = full.find('#') {
@@ -311,7 +366,7 @@ impl Url {
if self.is_homepage() {
false
} else {
- self.without_query().ends_with(ending)
+ self.strip_query().ends_with(ending)
}
}
@@ -435,7 +490,7 @@ mod tests {
.to_string()
.into();
- assert_eq!(url.without_query(), "https://test.example.com");
+ assert_eq!(url.strip_query(), "https://test.example.com");
}
#[test]
@@ -516,4 +571,26 @@ mod tests {
let url: Url = "example.com/test?a=b".to_string().into();
assert_eq!(url.path_and_query(), "/test?a=b");
}
+
+ #[test]
+ fn utm_removed_after_normalization() {
+ let url: Url = "https://example.com/test?a=b&utm_source=google"
+ .to_string()
+ .into();
+ let url = url.normalize();
+ assert_eq!(url.path_and_query(), "/test?a=b");
+
+ let url: Url = "https://example.com/test?a=b&utm_source=google&utm_medium=google"
+ .to_string()
+ .into();
+ let url = url.normalize();
+ assert_eq!(url.path_and_query(), "/test?a=b");
+
+ let url: Url =
+ "https://example.com/test?a=b&utm_source=google&utm_medium=google&utm_campaign=google"
+ .to_string()
+ .into();
+ let url = url.normalize();
+ assert_eq!(url.path_and_query(), "/test?a=b");
+ }
}