web search done right
Revert store url states in crawldb on disk.
Mikkel Denkeraaf5bd064f54
diff --git a/Cargo.lock b/Cargo.lockindex aeb54bcb..c39c741f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -459,11 +459,10 @@ dependencies = [[[package]]name = "bincode"-version = "1.2.1"+version = "1.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf"+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"dependencies = [- "byteorder","serde",]diff --git a/Cargo.toml b/Cargo.tomlindex 8d4e0c25..322db5be 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -14,7 +14,7 @@ debug = true[workspace.dependencies]aes-gcm = "0.10.2"-bincode = "=1.2.1" # https://github.com/dtolnay/cargo-expand/commit/234519982e1275d8a21c1048962ffe0a32a4b1cd+bincode = "1.3.3"clap = { version = "3.1.18", features = ["derive"] }encoding_rs = "0.8.31"flate2 = "1.0.23"@@ -78,7 +78,7 @@ tokio-stream = "0.1.11"scylla = "0.7.0"chitchat = "0.5.0"memmap2 = "0.5.10"-rkyv = { version = "0.7.41", features = ["validation"] }+rkyv = "0.7.41"dashmap = "5.4.0"tch = "0.13.0"torch-sys = "0.13.0"diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 9f6290c9..98e9c490 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -24,15 +24,13 @@ use std::{use lru::LruCache;use rand::Rng;-use rkyv::Deserialize;use rocksdb::BlockBasedOptions;use crate::webpage::Url;use super::{Domain, Job, Result, UrlResponse};-#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Clone, PartialEq, Eq)]-#[archive(check_bytes)]+#[derive(Clone, PartialEq, Eq)]pub enum UrlStatus {Pending,Crawling,@@ -257,8 +255,6 @@ fn weighted_sample<'a, T: 'a>(sampled_items.into_iter().map(|s| s.0.item).collect()}-#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)]-#[archive(check_bytes)]struct UrlState {weight: f64,status: UrlStatus,@@ -268,20 +264,7 @@ struct DomainState {status: DomainStatus,}-#[derive(- serde::Serialize,- serde::Deserialize,- rkyv::Serialize,- rkyv::Deserialize,- rkyv::Archive,- Clone,- Copy,- Debug,- PartialEq,- Eq,- Hash,-)]-#[archive(check_bytes)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]pub struct DomainId(u64);impl From<u64> for DomainId {@@ -290,11 +273,7 @@ impl From<u64> for DomainId {}}-#[derive(- rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Clone, Copy, Debug, PartialEq, Eq, Hash,-)]-#[archive(check_bytes)]-#[archive_attr(derive(PartialEq, Eq, Hash))]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]struct UrlId(u64);impl From<u64> for UrlId {@@ -309,8 +288,6 @@ struct RedirectDb {impl RedirectDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- rocksdb::DB::destroy(&rocksdb::Options::default(), path.as_ref())?;-let inner = rocksdb::DB::open_default(path.as_ref())?;Ok(Self { inner })@@ -330,62 +307,6 @@ impl RedirectDb {}}-struct UrlMap {- inner: rocksdb::DB, // DomainId -> HashMap<UrlId, UrlState>-}--impl UrlMap {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- rocksdb::DB::destroy(&rocksdb::Options::default(), path.as_ref())?;-- let mut options = rocksdb::Options::default();- options.create_if_missing(true);- options.increase_parallelism(8);- options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable- options.set_max_write_buffer_number(8);-- let mut block_options = BlockBasedOptions::default();- block_options.set_bloom_filter(64.0, true);-- options.set_block_based_table_factory(&block_options);-- let inner = rocksdb::DB::open(&options, path.as_ref())?;-- Ok(Self { inner })- }-- pub fn put(&self, domain_id: DomainId, map: HashMap<UrlId, UrlState>) -> Result<()> {- let domain_bytes = rkyv::to_bytes::<_, 256>(&domain_id).expect("failed to serialize");- let map_bytes = rkyv::to_bytes::<_, 256>(&map).expect("failed to serialize");-- let mut write_options = rocksdb::WriteOptions::default();- write_options.set_sync(false);- write_options.disable_wal(true);-- self.inner- .put_opt(domain_bytes, map_bytes, &write_options)?;-- Ok(())- }-- pub fn get(&self, domain_id: DomainId) -> Result<Option<HashMap<UrlId, UrlState>>> {- let domain_bytes = rkyv::to_bytes::<_, 256>(&domain_id).expect("failed to serialize");-- let map_bytes = self.inner.get(domain_bytes)?;-- match map_bytes {- Some(bytes) => {- let map = rkyv::check_archived_root::<HashMap<UrlId, UrlState>>(&bytes)- .expect("failed to deserialize")- .deserialize(&mut rkyv::Infallible)- .expect("failed to deserialize");- Ok(Some(map))- }- None => Ok(None),- }- }-}-pub struct CrawlDb {url_ids: IdTable<Url>,domain_ids: IdTable<Domain>,@@ -394,14 +315,13 @@ pub struct CrawlDb {domain_state: HashMap<DomainId, DomainState>,- urls: UrlMap,+ urls: HashMap<DomainId, HashMap<UrlId, UrlState>>,}impl CrawlDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let url_ids = IdTable::open(path.as_ref().join("url_ids"))?;- let domain_ids = IdTable::open(path.as_ref().join("domain_ids"))?;- let url_map = UrlMap::open(path.as_ref().join("url_map"))?;+ let url_ids = IdTable::open(path.as_ref().join("urls"))?;+ let domain_ids = IdTable::open(path.as_ref().join("domains"))?;let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;Ok(Self {@@ -409,44 +329,29 @@ impl CrawlDb {domain_ids,redirects,domain_state: HashMap::new(),- urls: url_map,+ urls: HashMap::new(),})}pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {- let mut bundled: HashMap<Domain, Vec<Url>> = HashMap::new();-for url in urls {- bundled- .entry(url.domain().to_string().into())- .or_default()- .push(url.clone());- }-- for (domain, urls) in bundled {- let domain_id = self.domain_ids.id(domain)?.into();- let mut map = self.urls.get(domain_id)?.unwrap_or_default();- for url in urls {- let domain_id = self.domain_ids.id(url.clone().into())?.into();- let url_id = self.url_ids.id(url.clone())?.into();-- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });+ let domain_id = self.domain_ids.id(url.into())?.into();+ let url_id = self.url_ids.id(url.clone())?.into();- map.insert(- url_id,- UrlState {- weight: 0.0,- status: UrlStatus::Pending,- },- );- }+ self.domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });- self.urls.put(domain_id, map)?;+ self.urls.entry(domain_id).or_default().insert(+ url_id,+ UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ },+ );}Ok(())@@ -471,8 +376,6 @@ impl CrawlDb {self.url_ids.bulk_ids(domains.values().flatten().cloned())?;for (domain_id, (domain, urls)) in domain_ids.into_iter().zip(domains.into_iter()) {- let mut map = self.urls.get(domain_id)?.unwrap_or_default();-for url in urls {let url_id: UrlId = self.url_ids.id(url.clone())?.into();@@ -486,10 +389,15 @@ impl CrawlDb {domain_state.status = DomainStatus::Pending;- let url_state = map.entry(url_id).or_insert_with(|| UrlState {- weight: 0.0,- status: UrlStatus::Pending,- });+ let url_state = self+ .urls+ .entry(domain_id)+ .or_default()+ .entry(url_id)+ .or_insert_with(|| UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ });if &domain != crawled_domain {url_state.weight += 1.0;@@ -499,39 +407,12 @@ impl CrawlDb {domain_state.weight = url_state.weight;}}-- self.urls.put(domain_id, map)?;}Ok(())}pub fn update_url_status(&mut self, url_responses: &[UrlResponse]) -> Result<()> {- let mut bundled: HashMap<Domain, Vec<UrlResponse>> = HashMap::new();-- for res in url_responses {- match res {- UrlResponse::Success { url } => {- let domain: Domain = url.domain().to_string().into();- bundled.entry(domain).or_default().push(res.clone());- }- UrlResponse::Failed {- url,- status_code: _,- } => {- let domain: Domain = url.domain().to_string().into();- bundled.entry(domain).or_default().push(res.clone());- }- UrlResponse::Redirected { url, new_url: _ } => {- let domain: Domain = url.domain().to_string().into();- bundled.entry(domain).or_default().push(res.clone());- }- }- }-- // bulk register domains- self.domain_ids.bulk_ids(bundled.keys().cloned())?;-// bulk register urlsself.url_ids.bulk_ids(url_responses@@ -547,61 +428,66 @@ impl CrawlDb {.cloned(),)?;- for (domain, url_responses) in bundled {- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();- let mut map = self.urls.get(domain_id)?.unwrap_or_default();-- for response in url_responses {- match response {- UrlResponse::Success { url } => {- let domain: Domain = url.domain().to_string().into();- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();- let url_id: UrlId = self.url_ids.id(url.clone())?.into();-- let domain_state =- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });- domain_state.status = DomainStatus::Pending;-- let url_state = map.entry(url_id).or_insert_with(|| UrlState {+ for response in url_responses {+ match response {+ UrlResponse::Success { url } => {+ let domain: Domain = url.domain().to_string().into();+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();++ let domain_state =+ self.domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });+ domain_state.status = DomainStatus::Pending;++ let url_state = self+ .urls+ .entry(domain_id)+ .or_default()+ .entry(url_id)+ .or_insert_with(|| UrlState {weight: 0.0,status: UrlStatus::Pending,});- url_state.status = UrlStatus::Done;- }- UrlResponse::Failed { url, status_code } => {- let domain: Domain = url.domain().to_string().into();- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();- let url_id: UrlId = self.url_ids.id(url.clone())?.into();-- let domain_state =- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });- domain_state.status = DomainStatus::Pending;-- let url_state = map.entry(url_id).or_insert_with(|| UrlState {+ url_state.status = UrlStatus::Done;+ }+ UrlResponse::Failed { url, status_code } => {+ let domain: Domain = url.domain().to_string().into();+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();++ let domain_state =+ self.domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });+ domain_state.status = DomainStatus::Pending;++ let url_state = self+ .urls+ .entry(domain_id)+ .or_default()+ .entry(url_id)+ .or_insert_with(|| UrlState {weight: 0.0,status: UrlStatus::Pending,});- url_state.status = UrlStatus::Failed { status_code };- }- UrlResponse::Redirected { url, new_url } => {- self.redirects.put(&url, &new_url)?;- }+ url_state.status = UrlStatus::Failed {+ status_code: *status_code,+ };+ }+ UrlResponse::Redirected { url, new_url } => {+ self.redirects.put(url, new_url)?;}}-- self.urls.put(domain_id, map)?;}Ok(())}@@ -648,7 +534,7 @@ impl CrawlDb {pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {let mut jobs = Vec::with_capacity(domains.len());for domain_id in domains {- let mut urls = self.urls.get(*domain_id)?.unwrap_or_default();+ let urls = self.urls.entry(*domain_id).or_default();let sampled: Vec<_> = weighted_sample(urls.iter_mut().filter_map(|(id, state)| {@@ -680,8 +566,6 @@ impl CrawlDb {job.urls.push_back(url);}- self.urls.put(*domain_id, urls)?;-jobs.push(job);}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex c31e88d8..a6715f79 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -92,12 +92,6 @@ impl From<&Url> for Domain {}}-impl From<Url> for Domain {- fn from(url: Url) -> Self {- Self::from(&url)- }-}-impl From<String> for Domain {fn from(s: String) -> Self {Self(s)