Skip to content

web search done right

Fix url root domain parsing and faster memory mapped crawldb.

Mikkel Denkerce1e15a11b6f
diff --git a/Cargo.lock b/Cargo.lock
index 1783a2aa..93c84b5b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2039,6 +2039,16 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+[[package]]
+name = "idna"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
[[package]]
name = "idna"
version = "0.4.0"
@@ -3366,6 +3376,12 @@ dependencies = [
"syn 0.15.44",
]
+[[package]]
+name = "psl-types"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+
[[package]]
name = "ptr_meta"
version = "0.1.4"
@@ -3386,6 +3402,16 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "publicsuffix"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457"
+dependencies = [
+ "idna 0.3.0",
+ "psl-types",
+]
+
[[package]]
name = "qoi"
version = "0.4.1"
@@ -4343,6 +4369,7 @@ dependencies = [
"parse_wiki_text",
"proptest",
"proptest-derive",
+ "publicsuffix",
"quick-xml 0.23.1",
"rand 0.8.5",
"rayon",
@@ -5168,7 +5195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
dependencies = [
"form_urlencoded",
- "idna",
+ "idna 0.4.0",
"percent-encoding",
"serde",
]
diff --git a/Cargo.toml b/Cargo.toml
index 6d90f3dc..2eaf4614 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -98,4 +98,5 @@ proptest-derive = "0.3.0"
utoipa = "3.4.4"
utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
tikv-jemallocator = "0.5"
+publicsuffix = "2.2.3"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index bd69d28a..06fe676f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -95,6 +95,7 @@ anyhow = { workspace = true }
bytemuck = { workspace = true }
utoipa = { workspace = true }
utoipa-swagger-ui = { workspace = true }
+publicsuffix = { workspace = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { workspace = true }
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 4e4fa9ac..3e229aac 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -18,13 +18,18 @@ use std::{
cmp::Ordering,
collections::{BTreeMap, BinaryHeap, HashMap, VecDeque},
hash::Hash,
+ io::Write,
num::NonZeroUsize,
+ ops::Range,
path::Path,
+ sync::Mutex,
};
+use dashmap::DashMap;
use itertools::Itertools;
use lru::LruCache;
use rand::Rng;
+use rayon::prelude::*;
use rocksdb::BlockBasedOptions;
use url::Url;
@@ -44,173 +49,195 @@ pub enum DomainStatus {
CrawlInProgress,
}
-struct IdTable<T> {
- t2id: rocksdb::DB,
- id2t: rocksdb::DB,
-
- next_id: u64,
-
- t2id_cache: LruCache<T, u64>,
- id2t_cache: LruCache<u64, T>,
-
- _marker: std::marker::PhantomData<T>,
+/// This is a simple key-value store that uses memory mapping to store the data on disk.
+/// It is not thread safe, so it should be wrapped in a mutex.
+///
+/// The keys are stored in memory in a map from `K` to `Range<usize>`, where the range is the
+/// position of the value in the file.
+///
+/// This DB should only be used for data that is acceptable to lose in case of a crash.
+/// Currently the database doesn't even save the key -> range map to disk, so it will be lost.
+struct MemmapDb<K, V> {
+ inner: Option<memmap::MmapMut>,
+ file: std::fs::File,
+ ranges: HashMap<K, Range<usize>>,
+ len: usize,
+
+ write_batch: Vec<u8>,
+
+ _phantom: std::marker::PhantomData<V>,
}
-impl<T> IdTable<T>
+impl<K, V> MemmapDb<K, V>
where
- T: serde::Serialize + serde::de::DeserializeOwned + Hash + Eq + Clone,
+ K: serde::Serialize + serde::de::DeserializeOwned + Hash + Eq + Clone,
+ V: serde::Serialize + serde::de::DeserializeOwned + Clone,
{
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mut options = rocksdb::Options::default();
- options.create_if_missing(true);
- options.increase_parallelism(8);
- options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable
- options.set_max_write_buffer_number(8);
+ fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ std::fs::create_dir_all(path.as_ref())?;
- let mut block_options = BlockBasedOptions::default();
- block_options.set_ribbon_filter(10.0);
+ let file = std::fs::OpenOptions::new()
+ .read(true)
+ .create(true)
+ .write(true)
+ .truncate(true)
+ .open(path.as_ref().join("db"))?;
- options.set_block_based_table_factory(&block_options);
+ let inner = if file.metadata()?.len() > 0 {
+ Some(unsafe { memmap::MmapOptions::new().map_mut(&file)? })
+ } else {
+ None
+ };
- // create dir if not exists
- std::fs::create_dir_all(path.as_ref())?;
+ let ranges = HashMap::new();
- let _ = rocksdb::DB::destroy(&options, path.as_ref().join("t2id"));
- let _ = rocksdb::DB::destroy(&options, path.as_ref().join("id2t"));
+ Ok(Self {
+ inner,
+ ranges,
+ file,
+ len: 0,
+ write_batch: Vec::new(),
+ _phantom: std::marker::PhantomData,
+ })
+ }
- let t2id = rocksdb::DB::open(&options, path.as_ref().join("t2id"))?;
- let id2t = rocksdb::DB::open(&options, path.as_ref().join("id2t"))?;
+ fn get(&self, key: &K) -> Option<V> {
+ let range = self.ranges.get(key)?;
- Ok(Self {
- t2id,
- id2t,
- next_id: 0,
+ if range.start >= self.inner.as_ref()?.len() || range.end > self.inner.as_ref()?.len() {
+ return None;
+ }
- t2id_cache: LruCache::new(NonZeroUsize::new(500_000).unwrap()),
- id2t_cache: LruCache::new(NonZeroUsize::new(500_000).unwrap()),
+ let value_bytes = &self.inner.as_ref()?[range.clone()];
+ let value: V = bincode::deserialize(value_bytes).ok()?;
- _marker: std::marker::PhantomData,
- })
+ Some(value)
}
- pub fn bulk_ids<'a>(&'a mut self, items: impl Iterator<Item = &'a T>) -> Result<Vec<u64>> {
- let mut ids = Vec::new();
+ fn put(&mut self, key: &K, value: &V) -> Result<()> {
+ let value_bytes = bincode::serialize(value)?;
- let mut batch_id2t = rocksdb::WriteBatch::default();
- let mut batch_t2id = rocksdb::WriteBatch::default();
+ let range = self.len..self.len + value_bytes.len();
- let mut batch_assignments = HashMap::new();
+ self.write_batch.write_all(&value_bytes)?;
+ self.len += value_bytes.len();
- for item in items {
- // check cache
- if let Some(id) = self.t2id_cache.get(item) {
- ids.push(*id);
- continue;
- }
+ self.ranges.insert(key.clone(), range);
- // check if item exists
- let item_bytes = bincode::serialize(item)?;
- let id = self.t2id.get(&item_bytes)?;
- if let Some(id) = id {
- let id = bincode::deserialize(&id)?;
+ Ok(())
+ }
- // update cache
- self.t2id_cache.put(item.clone(), id);
+ fn commit(&mut self) -> Result<()> {
+ self.file.write_all(&self.write_batch)?;
+ self.write_batch.clear();
- ids.push(id);
- continue;
- }
+ if let Some(inner) = &self.inner {
+ inner.flush()?;
+ }
- // insert item
- let assigned_id = batch_assignments.get(item).copied();
+ self.file.flush()?;
- let id = assigned_id.unwrap_or_else(|| {
- let id = self.next_id;
- self.next_id += 1;
- id
- });
+ self.inner = if self.file.metadata()?.len() > 0 {
+ Some(unsafe { memmap::MmapOptions::new().map_mut(&self.file)? })
+ } else {
+ None
+ };
+
+ Ok(())
+ }
- let id_bytes = bincode::serialize(&id)?;
- batch_t2id.put(&item_bytes, &id_bytes);
- batch_id2t.put(&id_bytes, &item_bytes);
+ fn contains(&self, key: &K) -> bool {
+ self.ranges.contains_key(key)
+ }
+}
- if assigned_id.is_none() {
- batch_assignments.insert(item.clone(), id);
- }
+type Id = u128;
+trait AsId {
+ fn as_id(&self) -> Id;
+}
- // update cache
- self.t2id_cache.put(item.clone(), id);
+impl AsId for Domain {
+ fn as_id(&self) -> Id {
+ self.id().0
+ }
+}
- ids.push(id);
- }
+impl AsId for Url {
+ fn as_id(&self) -> Id {
+ let digest = md5::compute(self.as_str());
+ u128::from_be_bytes(digest.0)
+ }
+}
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.set_sync(false);
- write_options.disable_wal(true);
+struct IdTable<T> {
+ db: Mutex<MemmapDb<Id, T>>,
+ cache: Mutex<LruCache<Id, T>>,
+}
+
+impl<T> IdTable<T>
+where
+ T: serde::Serialize + serde::de::DeserializeOwned + Hash + Eq + Clone + AsId,
+{
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ // create dir if not exists
+ std::fs::create_dir_all(path.as_ref())?;
- self.id2t.write_opt(batch_id2t, &write_options)?;
- self.t2id.write_opt(batch_t2id, &write_options)?;
+ let db = Mutex::new(MemmapDb::open(path.as_ref().join("db"))?);
- Ok(ids)
+ Ok(Self {
+ db,
+ cache: Mutex::new(LruCache::new(NonZeroUsize::new(500_000).unwrap())),
+ })
}
- pub fn id(&mut self, item: T) -> Result<u64> {
- // check cache
- if let Some(id) = self.t2id_cache.get(&item) {
- return Ok(*id);
- }
+ pub fn bulk_ids<'a>(&'a self, items: impl Iterator<Item = &'a T>) -> Result<Vec<Id>> {
+ let mut ids = Vec::new();
- // check if item exists
- let item_bytes = bincode::serialize(&item)?;
- let id = self.t2id.get(&item_bytes)?;
- if let Some(id) = id {
- let id = bincode::deserialize(&id)?;
+ let mut db = self.db.lock().unwrap();
+ for item in items {
+ let id = item.as_id();
- // update cache
- self.t2id_cache.put(item.clone(), id);
- self.id2t_cache.put(id, item);
+ if !db.contains(&id) {
+ db.put(&id, item)?;
+ }
- return Ok(id);
+ ids.push(id);
}
- // insert item
- let id = self.next_id;
- self.next_id += 1;
- let id_bytes = bincode::serialize(&id)?;
+ db.commit()?;
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.set_sync(false);
- write_options.disable_wal(true);
+ Ok(ids)
+ }
- self.t2id.put_opt(&item_bytes, &id_bytes, &write_options)?;
- self.id2t.put_opt(&id_bytes, &item_bytes, &write_options)?;
+ pub fn id(&self, item: &T) -> Result<Id> {
+ let id = item.as_id();
- // update cache
- self.t2id_cache.put(item.clone(), id);
- self.id2t_cache.put(id, item);
+ let mut db = self.db.lock().unwrap();
+
+ // check if item exists
+ if !db.contains(&id) {
+ db.put(&id, item)?;
+ }
Ok(id)
}
- pub fn value(&mut self, id: u64) -> Result<Option<T>> {
+ pub fn value(&mut self, id: Id) -> Result<Option<T>> {
+ let mut cache = self.cache.lock().unwrap();
+
// check cache
- if let Some(value) = self.id2t_cache.get(&id) {
+ if let Some(value) = cache.get(&id) {
return Ok(Some(value.clone()));
}
- let id_bytes = bincode::serialize(&id)?;
- let value_bytes = self.id2t.get(id_bytes)?;
- if let Some(value_bytes) = value_bytes {
- let value: T = bincode::deserialize(&value_bytes)?;
-
- // update cache
- self.t2id_cache.put(value.clone(), id);
- self.id2t_cache.put(id, value.clone());
+ // check db
+ let value = self.db.lock().unwrap().get(&id);
- return Ok(Some(value));
+ if let Some(value) = &value {
+ cache.put(id, value.clone());
}
- Ok(None)
+ Ok(value)
}
}
@@ -276,19 +303,19 @@ struct DomainState {
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-pub struct DomainId(u64);
+pub struct DomainId(u128);
-impl From<u64> for DomainId {
- fn from(id: u64) -> Self {
+impl From<u128> for DomainId {
+ fn from(id: u128) -> Self {
Self(id)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
-struct UrlId(u64);
+struct UrlId(u128);
-impl From<u64> for UrlId {
- fn from(id: u64) -> Self {
+impl From<u128> for UrlId {
+ fn from(id: u128) -> Self {
Self(id)
}
}
@@ -348,9 +375,9 @@ pub struct CrawlDb {
redirects: RedirectDb,
- domain_state: BTreeMap<DomainId, DomainState>,
+ domain_state: DashMap<DomainId, DomainState>,
- urls: BTreeMap<DomainId, BTreeMap<UrlId, UrlState>>,
+ urls: DashMap<DomainId, BTreeMap<UrlId, UrlState>>,
}
impl CrawlDb {
@@ -363,15 +390,15 @@ impl CrawlDb {
url_ids,
domain_ids,
redirects,
- domain_state: BTreeMap::new(),
- urls: BTreeMap::new(),
+ domain_state: DashMap::new(),
+ urls: DashMap::new(),
})
}
pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {
for url in urls {
- let domain_id = self.domain_ids.id(url.into())?.into();
- let url_id = self.url_ids.id(url.clone())?.into();
+ let domain_id = self.domain_ids.id(&Domain::from(url))?.into();
+ let url_id = self.url_ids.id(url)?.into();
self.domain_state
.entry(domain_id)
@@ -395,7 +422,7 @@ impl CrawlDb {
pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {
let mut domains: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();
- for res in responses {
+ responses.iter().for_each(|res| {
for url in &res.discovered_urls {
let domain = Domain::from(url);
let different_domain = res.domain != domain;
@@ -405,7 +432,7 @@ impl CrawlDb {
different_domain,
});
}
- }
+ });
let domain_ids: Vec<DomainId> = self
.domain_ids
@@ -417,34 +444,46 @@ impl CrawlDb {
self.url_ids
.bulk_ids(domains.values().flatten().map(|u| &u.url))?;
- for (domain_id, urls) in domain_ids.into_iter().zip_eq(domains.values()) {
- let domain_state = self
- .domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
+ domain_ids
+ .into_iter()
+ .zip_eq(domains.values())
+ .par_bridge()
+ .for_each(|(domain_id, urls)| {
+ let mut domain_state =
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
- let url_states = self.urls.entry(domain_id).or_default();
+ let mut url_states = self.urls.entry(domain_id).or_default();
- for url in urls {
- let url_id: UrlId = self.url_ids.id(url.url.clone())?.into();
+ for url in urls {
+ let id = self.url_ids.id(&url.url);
- let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
+ if id.is_err() {
+ continue;
+ }
- if url.different_domain {
- url_state.weight += 1.0;
- }
+ let id = id.unwrap();
+
+ let url_id: UrlId = id.into();
+
+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
- if url_state.weight > domain_state.weight {
- domain_state.weight = url_state.weight;
+ if url.different_domain {
+ url_state.weight += 1.0;
+ }
+
+ if url_state.weight > domain_state.weight {
+ domain_state.weight = url_state.weight;
+ }
}
- }
- }
+ });
Ok(())
}
@@ -497,53 +536,70 @@ impl CrawlDb {
// bulk register domains
self.domain_ids.bulk_ids(url_responses.keys())?;
- for (domain, responses) in url_responses {
- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
-
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
-
- let url_states = self.urls.entry(domain_id).or_default();
-
- for response in responses {
- match response {
- UrlResponse::Success { url } => {
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
-
- let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- url_state.status = UrlStatus::Done;
- }
- UrlResponse::Failed { url, status_code } => {
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
-
- let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- url_state.status = UrlStatus::Failed { status_code };
- }
- UrlResponse::Redirected { url, new_url } => {
- self.redirects.put(&url, &new_url)?;
+ url_responses
+ .into_par_iter()
+ .filter_map(|(domain, responses)| {
+ let domain_id: DomainId = self.domain_ids.id(&domain).ok()?.into();
+ Some((domain_id, responses))
+ })
+ .for_each(|(domain_id, responses)| {
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+
+ let mut url_states = self.urls.entry(domain_id).or_default();
+
+ for response in responses {
+ match response {
+ UrlResponse::Success { url } => {
+ let id = self.url_ids.id(&url);
+
+ if id.is_err() {
+ continue;
+ }
+
+ let url_id: UrlId = id.unwrap().into();
+
+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ url_state.status = UrlStatus::Done;
+ }
+ UrlResponse::Failed { url, status_code } => {
+ let id = self.url_ids.id(&url);
+
+ if id.is_err() {
+ continue;
+ }
+
+ let url_id: UrlId = id.unwrap().into();
+
+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ url_state.status = UrlStatus::Failed { status_code };
+ }
+ UrlResponse::Redirected { url, new_url } => {
+ self.redirects.put(&url, &new_url).ok();
+ }
}
}
- }
- }
+ });
+
Ok(())
}
pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {
- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let domain_id: DomainId = self.domain_ids.id(domain)?.into();
- let domain_state = self
+ let mut domain_state = self
.domain_state
.entry(domain_id)
.or_insert_with(|| DomainState {
@@ -557,14 +613,21 @@ impl CrawlDb {
}
pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<DomainId>> {
- let sampled = weighted_sample(
- self.domain_state.iter().filter_map(|(id, state)| {
+ let available_domains: Vec<_> = self
+ .domain_state
+ .iter()
+ .filter_map(|r| {
+ let (id, state) = r.pair();
if state.status == DomainStatus::Pending {
- Some((id, state.weight))
+ Some((*id, state.weight))
} else {
None
}
- }),
+ })
+ .collect();
+
+ let sampled = weighted_sample(
+ available_domains.iter().map(|(id, weight)| (id, *weight)),
num_jobs,
)
.into_iter()
@@ -572,7 +635,7 @@ impl CrawlDb {
.collect();
for id in &sampled {
- let state = self.domain_state.get_mut(id).unwrap();
+ let mut state = self.domain_state.get_mut(id).unwrap();
state.status = DomainStatus::CrawlInProgress;
}
@@ -582,7 +645,7 @@ impl CrawlDb {
pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {
let mut jobs = Vec::with_capacity(domains.len());
for domain_id in domains {
- let urls = self.urls.entry(*domain_id).or_default();
+ let mut urls = self.urls.entry(*domain_id).or_default();
let sampled: Vec<_> = weighted_sample(
urls.iter_mut().filter_map(|(id, state)| {
@@ -603,7 +666,7 @@ impl CrawlDb {
state.status = UrlStatus::Crawling;
}
- let domain_state = self.domain_state.get_mut(domain_id).unwrap();
+ let mut domain_state = self.domain_state.get_mut(domain_id).unwrap();
domain_state.weight = urls
.iter()
@@ -637,6 +700,8 @@ impl CrawlDb {
#[cfg(test)]
mod tests {
+ use crate::gen_temp_path;
+
use super::*;
#[test]
@@ -658,4 +723,28 @@ mod tests {
assert_eq!(sampled.len(), 1);
assert_eq!(*sampled[0], 0);
}
+
+ #[test]
+ fn memmap_db() {
+ let mut db: MemmapDb<u128, String> = MemmapDb::open(gen_temp_path()).unwrap();
+
+ assert!(!db.contains(&123));
+ assert!(db.get(&123).is_none());
+
+ db.put(&123, &"hello".to_string()).unwrap();
+ db.commit().unwrap();
+
+ assert!(db.contains(&123));
+ assert_eq!(db.get(&123).unwrap(), "hello");
+
+ db.put(&321, &"world".to_string()).unwrap();
+
+ assert!(db.contains(&321));
+ assert!(db.get(&321).is_none());
+
+ db.commit().unwrap();
+
+ assert!(db.contains(&321));
+ assert_eq!(db.get(&321).unwrap(), "world");
+ }
}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 8163b43e..a0ca9579 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -23,9 +23,9 @@ use std::{
use tokio::sync::Mutex;
use url::Url;
-use crate::config::CrawlerConfig;
+use crate::{config::CrawlerConfig, webpage::url_ext::UrlExt};
-use self::{warc_writer::WarcWriter, worker::WorkerThread};
+use self::{crawl_db::DomainId, warc_writer::WarcWriter, worker::WorkerThread};
pub mod coordinator;
pub mod crawl_db;
@@ -60,7 +60,7 @@ pub struct Domain(String);
impl From<&Url> for Domain {
fn from(url: &Url) -> Self {
- Self(url.domain().unwrap_or_default().to_string())
+ Self(url.root_domain().unwrap_or_default().to_string())
}
}
@@ -70,6 +70,13 @@ impl From<String> for Domain {
}
}
+impl Domain {
+ pub fn id(&self) -> DomainId {
+ let digest = md5::compute(self.0.as_bytes());
+ u128::from_be_bytes(digest.0).into()
+ }
+}
+
/// All urls in a job must be from the same domain and only one job per domain.
/// at a time. This ensures that we stay polite when crawling.
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 7f7cb8f2..6601a77e 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -54,11 +54,21 @@ impl RobotsTxtManager {
}
async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {
- let res = self
+ let mut res = self
.client
.get(&format!("http://{}/robots.txt", site.0))
.send()
- .await?;
+ .await;
+
+ if res.is_err() {
+ res = self
+ .client
+ .get(&format!("https://{}/robots.txt", site.0))
+ .send()
+ .await;
+ }
+
+ let res = res?;
if res.status() != reqwest::StatusCode::OK {
return Err(Error::FetchFailed(res.status()).into());
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 9676bc6a..5434c2db 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -21,7 +21,7 @@ use crate::{
entrypoint::download_all_warc_files,
mapreduce::Worker,
webgraph::{self, Node, WebgraphBuilder},
- webpage::Html,
+ webpage::{url_ext::UrlExt, Html},
Result,
};
use itertools::Itertools;
@@ -107,9 +107,10 @@ impl WebgraphWorker {
.anchor_links()
.into_iter()
.filter(|link| matches!(link.destination.scheme(), "http" | "https"))
- .filter(|link| link.source.domain() != link.destination.domain())
+ .filter(|link| link.source.root_domain() != link.destination.root_domain())
.filter(|link| {
- link.source.domain().is_some() && link.destination.domain().is_some()
+ link.source.root_domain().is_some()
+ && link.destination.root_domain().is_some()
})
{
let source = link.source.clone();
@@ -122,7 +123,7 @@ impl WebgraphWorker {
}
}
- if source.domain() == destination.domain() {
+ if source.root_domain() == destination.root_domain() {
continue;
}
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index c530e7a0..60911a05 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -28,6 +28,7 @@ use crate::{
inverted_index::RetrievedWebpage,
ranking::Signal,
spell::{self, CorrectionTerm},
+ webpage::url_ext::UrlExt,
};
pub use self::stack_overflow::{create_stackoverflow_sidebar, CodeOrText};
@@ -141,7 +142,7 @@ fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {
let url = Url::parse(&webpage.url).unwrap();
- if url.domain().unwrap_or_default() == "stackoverflow.com"
+ if url.root_domain().unwrap_or_default() == "stackoverflow.com"
&& webpage
.schema_org
.iter()
@@ -185,7 +186,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
let snippet = generate_snippet(&webpage);
let url = Url::parse(&webpage.url).unwrap();
- let domain = url.domain().unwrap_or_default().to_string();
+ let domain = url.root_domain().unwrap_or_default().to_string();
let pretty_url = prettify_url(&url);
let title = html_escape(&webpage.title);
diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rs
index 8a0edd1d..26171229 100644
--- a/core/src/subdomain_count.rs
+++ b/core/src/subdomain_count.rs
@@ -15,9 +15,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- ceil_char_boundary,
kv::{rocksdb_store::RocksDbStore, Kv},
prehashed::{hash, Prehashed},
+ webpage::url_ext::UrlExt,
};
use std::{collections::HashSet, path::Path};
use url::Url;
@@ -34,16 +34,9 @@ impl SubdomainCounter {
}
pub fn increment(&mut self, url: Url) {
- let scheme = url.scheme();
- let domain = url.domain().unwrap_or_default();
+ let domain = url.root_domain().unwrap_or_default();
- let mut url = url.as_str().strip_prefix(scheme).unwrap_or_default();
-
- if let Some(slash) = url.find('/') {
- url = &url[..ceil_char_boundary(url, slash)];
- }
-
- if let Some(subdomain) = url.strip_suffix(domain) {
+ if let Some(subdomain) = url.subdomain() {
let domain = hash(domain);
let subdomain = subdomain.to_string();
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index b8460725..9f194a1f 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -34,12 +34,14 @@ use whatlang::Lang;
mod just_text;
pub mod region;
pub mod schema_org;
+pub mod url_ext;
use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};
use self::{
just_text::{JustText, Paragraph},
region::Region,
+ url_ext::UrlExt,
};
pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
@@ -617,7 +619,7 @@ impl Html {
.or_else(|_| self.url().join(url.as_str()))
.ok()?;
- if script_url.domain() != self.url().domain() {
+ if script_url.root_domain() != self.url().root_domain() {
Some(Link {
source: self.url().clone(),
destination: script_url,
@@ -778,7 +780,7 @@ impl Html {
}
fn pretokenize_domain(&self) -> PreTokenizedString {
- let mut domain = self.url().domain().unwrap_or_default().to_string();
+ let mut domain = self.url().root_domain().unwrap_or_default().to_string();
if let Some(stripped) = domain.strip_prefix("www.") {
domain = stripped.to_string();
@@ -856,7 +858,7 @@ impl Html {
let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
let url_hash = split_u128(hash(self.url().as_str()).0);
- let domain_hash = split_u128(hash(self.url().domain().unwrap_or_default()).0);
+ let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);
let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
for field in &ALL_FIELDS {
@@ -1005,7 +1007,7 @@ impl Html {
if self.is_homepage() {
let domain_name = self
.url()
- .domain()
+ .root_domain()
.unwrap_or_default()
.find('.')
.map(|index| {
@@ -1417,7 +1419,7 @@ pub type Meta = HashMap<String, String>;
mod tests {
// TODO: make test macro to test both dom parsers
- use crate::schema::create_schema;
+ use crate::{schema::create_schema, webpage::url_ext::UrlExt};
use super::*;
@@ -1465,7 +1467,6 @@ mod tests {
"https://www.example.com/whatever"
);
assert_eq!(webpage.url().host_str().unwrap(), "www.example.com");
- assert_eq!(webpage.url().domain().unwrap(), "www.example.com");
}
#[test]
@@ -1567,8 +1568,8 @@ mod tests {
let webpage = Html::parse(raw, "https://www.domain.co.uk").unwrap();
assert_eq!(
- webpage.url().domain().unwrap_or_default(),
- "www.domain.co.uk"
+ webpage.url().root_domain().unwrap_or_default(),
+ "domain.co.uk"
);
}
@@ -1749,7 +1750,10 @@ mod tests {
#[test]
fn domain_from_domain_url() {
let url: Url = Url::parse("http://example.com").unwrap();
- assert_eq!(url.domain().unwrap(), "example.com");
+ assert_eq!(url.root_domain().unwrap(), "example.com");
+
+ let url: Url = Url::parse("http://test.example.com").unwrap();
+ assert_eq!(url.root_domain().unwrap(), "example.com");
}
#[test]
diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rs
new file mode 100644
index 00000000..5d09f632
--- /dev/null
+++ b/core/src/webpage/url_ext.rs
@@ -0,0 +1,45 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use publicsuffix::Psl;
+
+pub static LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {
+ include_str!("../../public_suffix_list.dat")
+ .parse()
+ .expect("Failed to parse public suffix list")
+});
+
+pub trait UrlExt {
+ fn root_domain(&self) -> Option<&str>;
+
+ fn subdomain(&self) -> Option<&str>;
+}
+
+impl UrlExt for url::Url {
+ fn root_domain(&self) -> Option<&str> {
+ let host = self.host_str()?;
+ let suffix = std::str::from_utf8(LIST.domain(host.as_bytes())?.as_bytes()).ok()?;
+ Some(suffix)
+ }
+
+ fn subdomain(&self) -> Option<&str> {
+ let host = self.host_str()?;
+ let suffix = std::str::from_utf8(LIST.domain(host.as_bytes())?.as_bytes()).ok()?;
+ let suffix = suffix.strip_prefix(".")?;
+ let subdomain = host.strip_suffix(suffix)?;
+ Some(subdomain)
+ }
+}
diff --git a/frontend/src/pages/about.mdx b/frontend/src/pages/about.mdx
index ede2e75b..fa324b97 100644
--- a/frontend/src/pages/about.mdx
+++ b/frontend/src/pages/about.mdx
@@ -15,7 +15,7 @@ Stract is an open source search engine where the user has the ability to see exa
Oh, and if we ever become evil (maybe by changing our motto) please take our code and start a competitor. The fact that you have this ability will make sure that our values will always be aligned with our users.
-## How do you make money?
+## How do we make money?
Well, currently we don't. We are bootstraped and trying to keep costs low. In the future we will have contextual ads based on your current search query and a subscription option without ads. Just to re-iterate; we will only use your current search to match ads and will never track you across searches.