web search done right
simplify crawldb by removing all ids and storing urls/domains directly
Mikkel Denker586811904825
diff --git a/Cargo.lock b/Cargo.lockindex 93c84b5b..1fcbcf6f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -104,6 +104,12 @@ dependencies = ["memchr",]+[[package]]+name = "allocator-api2"+version = "0.2.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"+[[package]]name = "android_system_properties"version = "0.1.5"@@ -580,28 +586,6 @@ version = "3.12.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8"-[[package]]-name = "bytecheck"-version = "0.6.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f"-dependencies = [- "bytecheck_derive",- "ptr_meta",- "simdutf8",-]--[[package]]-name = "bytecheck_derive"-version = "0.6.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5"-dependencies = [- "proc-macro2 1.0.56",- "quote 1.0.26",- "syn 1.0.109",-]-[[package]]name = "bytemuck"version = "1.13.1"@@ -1836,6 +1820,11 @@ name = "hashbrown"version = "0.14.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"+dependencies = [+ "ahash 0.8.3",+ "allocator-api2",+ "serde",+][[package]]name = "heck"@@ -3382,26 +3371,6 @@ version = "2.0.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"-[[package]]-name = "ptr_meta"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"-dependencies = [- "ptr_meta_derive",-]--[[package]]-name = "ptr_meta_derive"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"-dependencies = [- "proc-macro2 1.0.56",- "quote 1.0.26",- "syn 1.0.109",-]-[[package]]name = "publicsuffix"version = "2.2.3"@@ -3660,15 +3629,6 @@ version = "0.7.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"-[[package]]-name = "rend"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab"-dependencies = [- "bytecheck",-]-[[package]]name = "reqwest"version = "0.11.16"@@ -3724,31 +3684,6 @@ dependencies = ["thiserror",]-[[package]]-name = "rkyv"-version = "0.7.41"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "21499ed91807f07ae081880aabb2ccc0235e9d88011867d984525e9a4c3cfa3e"-dependencies = [- "bytecheck",- "hashbrown 0.12.3",- "ptr_meta",- "rend",- "rkyv_derive",- "seahash",-]--[[package]]-name = "rkyv_derive"-version = "0.7.41"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ac1c672430eb41556291981f45ca900a0239ad007242d1cb4b4167af842db666"-dependencies = [- "proc-macro2 1.0.56",- "quote 1.0.26",- "syn 1.0.109",-]-[[package]]name = "robotstxt-with-cache"version = "0.4.0"@@ -4039,12 +3974,6 @@ dependencies = ["syn 1.0.109",]-[[package]]-name = "seahash"-version = "4.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"-[[package]]name = "security-framework"version = "2.8.2"@@ -4227,12 +4156,6 @@ version = "0.3.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f"-[[package]]-name = "simdutf8"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"-[[package]]name = "siphasher"version = "0.3.10"@@ -4346,6 +4269,7 @@ dependencies = ["fst","futures","half 2.2.1",+ "hashbrown 0.14.0","html-escape","http","image",@@ -4376,7 +4300,6 @@ dependencies = ["regex","reqwest","reqwest-eventsource",- "rkyv","robotstxt-with-cache","rocksdb","rust-s3",diff --git a/Cargo.toml b/Cargo.tomlindex 2eaf4614..e14b88d1 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -78,7 +78,6 @@ tokio-stream = "0.1.11"scylla = "0.7.0"chitchat = "0.5.0"memmap2 = "0.5.10"-rkyv = "0.7.41"dashmap = { version = "5.4.0", features = ["rayon"] }tch = "0.13.0"torch-sys = "0.13.0"@@ -99,4 +98,5 @@ utoipa = "3.4.4"utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }tikv-jemallocator = "0.5"publicsuffix = "2.2.3"+hashbrown = { version = "0.14.0", features = ["serde"] }diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 06fe676f..99c53d93 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -74,7 +74,6 @@ tokio-stream = { workspace = true }scylla = { workspace = true }chitchat = { workspace = true }memmap2 = { workspace = true }-rkyv = { workspace = true }dashmap = { workspace = true }tch = { workspace = true }torch-sys = { workspace = true }@@ -96,6 +95,7 @@ bytemuck = { workspace = true }utoipa = { workspace = true }utoipa-swagger-ui = { workspace = true }publicsuffix = { workspace = true }+hashbrown = { workspace = true }[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = { workspace = true }diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 665074ca..fadd267c 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -14,22 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use hashbrown::HashMap;+use rand::Rng;+use serde::{Deserialize, Serialize};use std::{cmp::Ordering,- collections::{BinaryHeap, HashMap, HashSet, VecDeque},- hash::Hash,+ collections::{BinaryHeap, VecDeque},path::Path,};--use itertools::Itertools;-use rand::Rng;-use rayon::prelude::IntoParallelRefIterator;-use rayon::prelude::*;-use serde::{Deserialize, Serialize};use url::Url;-use crate::intmap::{self, IntMap};-use super::{Domain, Job, JobResponse, Result, UrlResponse};#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]@@ -46,88 +40,6 @@ pub enum DomainStatus {CrawlInProgress,}-type Id = u128;-trait AsId {- fn as_id(&self) -> Id;-}--impl AsId for Domain {- fn as_id(&self) -> Id {- self.id().0- }-}--impl AsId for Url {- fn as_id(&self) -> Id {- let digest = md5::compute(self.as_str());- u128::from_be_bytes(digest.0)- }-}--struct IdTable<T> {- db: rocksdb::DB,- _marker: std::marker::PhantomData<T>,-}--impl<T> IdTable<T>-where- T: serde::Serialize + serde::de::DeserializeOwned + Hash + Eq + Clone + AsId,-{- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- // create dir if not exists- std::fs::create_dir_all(path.as_ref())?;-- let mut options = rocksdb::Options::default();- options.create_if_missing(true);-- let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_ribbon_filter(5.0);-- options.set_block_based_table_factory(&block_options);- options.set_optimize_filters_for_hits(true);- options.set_max_background_jobs(8);- options.set_write_buffer_size(512 * 1024 * 1024);- options.set_allow_mmap_reads(true);- options.set_allow_mmap_writes(true);- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);- options.optimize_for_point_lookup(512);-- let db = rocksdb::DB::open(&options, path.as_ref())?;-- Ok(Self {- db,- _marker: std::marker::PhantomData,- })- }-- pub fn bulk_insert_ids(&self, items: &[(T, Id)]) -> Result<()> {- let mut batch = rocksdb::WriteBatch::default();-- for (item, id) in items {- batch.put(bincode::serialize(id)?, bincode::serialize(item)?);- }-- let mut write_options = rocksdb::WriteOptions::default();- write_options.disable_wal(true);-- self.db.write_opt(batch, &write_options)?;-- Ok(())- }-- pub fn value(&self, id: Id) -> Result<Option<T>> {- let id_bytes = bincode::serialize(&id)?;-- // check db- let value = self.db.get(id_bytes)?;-- match value {- Some(value) => Ok(Some(bincode::deserialize(&value)?)),- None => Ok(None),- }- }-}-struct SampledItem<T> {item: T,priority: f64,@@ -188,24 +100,6 @@ struct DomainState {status: DomainStatus,}-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]-pub struct DomainId(u128);--impl From<u128> for DomainId {- fn from(id: u128) -> Self {- Self(id)- }-}--#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]-struct UrlId(u128);--impl From<u128> for UrlId {- fn from(id: u128) -> Self {- Self(id)- }-}-pub struct RedirectDb {inner: rocksdb::DB,}@@ -271,7 +165,7 @@ whereoptions.create_if_missing(true);let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_ribbon_filter(5.0);+ block_options.set_ribbon_filter(10.0);options.set_block_based_table_factory(&block_options);options.set_optimize_filters_for_hits(true);@@ -290,8 +184,8 @@ where})}- pub fn get(&mut self, key: K) -> Result<Option<V>> {- let key_bytes = bincode::serialize(&key)?;+ pub fn get(&mut self, key: &K) -> Result<Option<V>> {+ let key_bytes = bincode::serialize(key)?;let value_bytes = self.db.get(key_bytes)?;match value_bytes {@@ -300,9 +194,9 @@ where}}- pub fn put(&mut self, key: K, value: V) -> Result<()> {- let key_bytes = bincode::serialize(&key)?;- let value_bytes = bincode::serialize(&value)?;+ pub fn put(&mut self, key: &K, value: &V) -> Result<()> {+ let key_bytes = bincode::serialize(key)?;+ let value_bytes = bincode::serialize(value)?;let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);@@ -338,9 +232,9 @@ impl DomainStateDb {Ok(Self { db })}- fn get(&self, domain_id: DomainId) -> Result<Option<DomainState>> {- let domain_id_bytes = bincode::serialize(&domain_id)?;- let value_bytes = self.db.get(domain_id_bytes)?;+ fn get(&self, domain: &Domain) -> Result<Option<DomainState>> {+ let domain_bytes = bincode::serialize(domain)?;+ let value_bytes = self.db.get(domain_bytes)?;if let Some(value_bytes) = &value_bytes {let value: DomainState = bincode::deserialize(value_bytes)?;@@ -350,60 +244,57 @@ impl DomainStateDb {Ok(None)}- fn put(&self, domain_id: DomainId, state: DomainState) -> Result<()> {- let domain_id_bytes = bincode::serialize(&domain_id)?;- let state_bytes = bincode::serialize(&state)?;+ fn put(&self, domain: &Domain, state: &DomainState) -> Result<()> {+ let domain_bytes = bincode::serialize(domain)?;+ let state_bytes = bincode::serialize(state)?;let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);- self.db- .put_opt(domain_id_bytes, state_bytes, &write_options)?;+ self.db.put_opt(domain_bytes, state_bytes, &write_options)?;Ok(())}- fn iter(&self) -> impl Iterator<Item = (DomainId, DomainState)> + '_ {+ fn iter(&self) -> impl Iterator<Item = (Domain, DomainState)> + '_ {let iter = self.db.iterator(rocksdb::IteratorMode::Start);iter.filter_map(|r| {let (key, value) = r.ok()?;- let domain_id: DomainId = bincode::deserialize(&key).unwrap();+ let domain: Domain = bincode::deserialize(&key).unwrap();let state: DomainState = bincode::deserialize(&value).unwrap();- Some((domain_id, state))+ Some((domain, state))})}}-impl intmap::Key for UrlId {- const BIG_PRIME: Self = Self(335579573203413586826293107669396558523);-- fn wrapping_mul(self, rhs: Self) -> Self {- Self(self.0.wrapping_mul(rhs.0))- }+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]+struct UrlString(String);- fn bit_and(self, rhs: Self) -> Self {- Self(self.0 & rhs.0)+impl From<&Url> for UrlString {+ fn from(url: &Url) -> Self {+ Self(url.as_str().to_string())}+}- fn from_usize(val: usize) -> Self {- Self(val as u128)+impl From<Url> for UrlString {+ fn from(url: Url) -> Self {+ Self(url.as_str().to_string())}+}- fn as_usize(self) -> usize {- self.0 as usize+impl From<&UrlString> for Url {+ fn from(url: &UrlString) -> Self {+ Url::parse(&url.0).unwrap()}}pub struct CrawlDb {- url_ids: IdTable<Url>,- domain_ids: IdTable<Domain>,-redirects: RedirectDb,domain_state: DomainStateDb,- urls: PointDb<DomainId, IntMap<UrlId, UrlState>>,+ urls: PointDb<Domain, HashMap<UrlString, UrlState>>,}impl CrawlDb {@@ -414,63 +305,38 @@ impl CrawlDb {));}- let url_ids = IdTable::open(path.as_ref().join("urls").join("ids"))?;- let domain_ids = IdTable::open(path.as_ref().join("domains").join("ids"))?;let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;Ok(Self {- url_ids,- domain_ids,redirects,- domain_state: DomainStateDb::open(path.as_ref().join("domains").join("states"))?,- urls: PointDb::open(path.as_ref().join("urls").join("states"))?,+ domain_state: DomainStateDb::open(path.as_ref().join("domains"))?,+ urls: PointDb::open(path.as_ref().join("urls"))?,})}pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {- let domain_ids: HashSet<_> = urls- .par_iter()- .map(Domain::from)- .map(|domain| {- let id = domain.as_id();- (domain, id)- })- .collect();-- let domain_ids: Vec<_> = domain_ids.into_iter().collect();- self.domain_ids.bulk_insert_ids(&domain_ids)?;-- let url_ids: HashSet<_> = urls- .par_iter()- .map(|url| (url.clone(), url.as_id()))- .collect();-- let url_ids: Vec<_> = url_ids.into_iter().collect();- self.url_ids.bulk_insert_ids(&url_ids)?;-for url in urls {- let domain_id = Domain::from(url).as_id().into();- let url_id = url.as_id().into();+ let domain = Domain::from(url);self.domain_state.put(- domain_id,- DomainState {+ &domain,+ &DomainState {weight: 0.0,status: DomainStatus::Pending,},)?;- let mut urls = self.urls.get(domain_id)?.unwrap_or_default();+ let mut urls = self.urls.get(&domain)?.unwrap_or_default();urls.insert(- url_id,+ url.into(),UrlState {weight: 0.0,status: UrlStatus::Pending,},);- self.urls.put(domain_id, urls)?;+ self.urls.put(&domain, &urls)?;}Ok(())@@ -491,51 +357,30 @@ impl CrawlDb {}});- let domain_ids: Vec<_> = domains- .par_iter()- .map(|(domain, _)| (domain.clone(), domain.as_id()))- .collect();- self.domain_ids.bulk_insert_ids(&domain_ids)?;-- let url_ids: Vec<_> = domains- .par_iter()- .flat_map(|(_, urls)| {- urls.iter()- .map(|url| (url.url.clone(), url.url.as_id()))- .collect_vec()- })- .collect();- self.url_ids.bulk_insert_ids(&url_ids)?;-- for (domain_id, urls) in domain_ids- .into_iter()- .map(|(_, id)| id)- .zip_eq(domains.values())- {- let domain_id = domain_id.into();-- let mut domain_state = match self.domain_state.get(domain_id)? {+ for (domain, urls) in domains.into_iter() {+ let mut domain_state = match self.domain_state.get(&domain)? {Some(state) => state,None => {let state = DomainState {weight: 0.0,status: DomainStatus::Pending,};- self.domain_state.put(domain_id, state.clone())?;+ self.domain_state.put(&domain, &state)?;state}};- let mut url_states = self.urls.get(domain_id)?.unwrap_or_default();+ let mut url_states = self.urls.get(&domain)?.unwrap_or_default();for url in urls {- let url_id: UrlId = url.url.as_id().into();-- let mut url_state = url_states.get(&url_id).cloned().unwrap_or(UrlState {- weight: 0.0,- status: UrlStatus::Pending,- });+ let mut url_state = url_states+ .get(&UrlString::from(&url.url))+ .cloned()+ .unwrap_or(UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ });if url.different_domain {url_state.weight += 1.0;@@ -545,12 +390,12 @@ impl CrawlDb {domain_state.weight = url_state.weight;}- url_states.insert(url_id, url_state);+ url_states.insert(url.url.into(), url_state);}- self.domain_state.put(domain_id, domain_state)?;+ self.domain_state.put(&domain, &domain_state)?;- self.urls.put(domain_id, url_states)?;+ self.urls.put(&domain, &url_states)?;}Ok(())@@ -590,73 +435,44 @@ impl CrawlDb {}}- // bulk register urls- let url_ids: Vec<_> = url_responses- .par_iter()- .flat_map(|(_, responses)| {- responses- .iter()- .flat_map(|res| match res {- UrlResponse::Success { url } => vec![url],- UrlResponse::Failed {- url,- status_code: _,- } => vec![url],- UrlResponse::Redirected { url, new_url } => vec![url, new_url],- })- .map(|url| (url.clone(), url.as_id()))- .collect_vec()- })- .collect();-- self.url_ids.bulk_insert_ids(&url_ids)?;-- // bulk register domains- let domain_ids: Vec<_> = url_responses- .par_iter()- .map(|(domain, _)| (domain.clone(), domain.as_id()))- .collect();- self.domain_ids.bulk_insert_ids(&domain_ids)?;-- for (domain_id, responses) in url_responses.into_iter().map(|(domain, responses)| {- let domain_id = domain.as_id().into();- (domain_id, responses)- }) {- if self.domain_state.get(domain_id)?.is_none() {+ for (domain, responses) in url_responses {+ if self.domain_state.get(&domain)?.is_none() {self.domain_state.put(- domain_id,- DomainState {+ &domain,+ &DomainState {weight: 0.0,status: DomainStatus::Pending,},)?;}- let mut url_states = self.urls.get(domain_id)?.unwrap_or_default();+ let mut url_states = self.urls.get(&domain)?.unwrap_or_default();for response in responses {match response {UrlResponse::Success { url } => {- let url_id: UrlId = url.as_id().into();-- let mut url_state = url_states.get(&url_id).cloned().unwrap_or(UrlState {- weight: 0.0,- status: UrlStatus::Pending,- });+ let mut url_state = url_states+ .get(&UrlString::from(&url))+ .cloned()+ .unwrap_or(UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ });url_state.status = UrlStatus::Done;- url_states.insert(url_id, url_state);+ url_states.insert(UrlString::from(&url), url_state);}UrlResponse::Failed { url, status_code } => {- let url_id: UrlId = url.as_id().into();-- let mut url_state = url_states.get(&url_id).cloned().unwrap_or(UrlState {- weight: 0.0,- status: UrlStatus::Pending,- });+ let mut url_state = url_states+ .get(&UrlString::from(&url))+ .cloned()+ .unwrap_or(UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ });url_state.status = UrlStatus::Failed { status_code };- url_states.insert(url_id, url_state);+ url_states.insert(url.into(), url_state);}UrlResponse::Redirected { url, new_url } => {self.redirects.put(&url, &new_url).ok();@@ -664,34 +480,30 @@ impl CrawlDb {}}- self.urls.put(domain_id, url_states)?;+ self.urls.put(&domain, &url_states)?;}Ok(())}pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {- self.domain_ids- .bulk_insert_ids(&[(domain.clone(), domain.as_id())])?;- let domain_id = domain.id();-- let mut domain_state = self.domain_state.get(domain_id)?.unwrap_or(DomainState {+ let mut domain_state = self.domain_state.get(domain)?.unwrap_or(DomainState {weight: 0.0,status,});domain_state.status = status;- self.domain_state.put(domain_id, domain_state)?;+ self.domain_state.put(domain, &domain_state)?;Ok(())}- pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<DomainId>> {+ pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<Domain>> {let sampled = weighted_sample(- self.domain_state.iter().filter_map(|(id, state)| {+ self.domain_state.iter().filter_map(|(domain, state)| {if state.status == DomainStatus::Pending {- Some((id, state.weight))+ Some((domain, state.weight))} else {None}@@ -699,40 +511,41 @@ impl CrawlDb {num_jobs,);- for id in sampled.iter() {- let mut state = self.domain_state.get(*id)?.unwrap();+ for domain in sampled.iter() {+ let mut state = self.domain_state.get(domain)?.unwrap();state.status = DomainStatus::CrawlInProgress;- self.domain_state.put(*id, state)?;+ self.domain_state.put(domain, &state)?;}Ok(sampled)}- pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {+ pub fn prepare_jobs(&mut self, domains: &[Domain], urls_per_job: usize) -> Result<Vec<Job>> {let mut jobs = Vec::with_capacity(domains.len());- for domain_id in domains {- let mut urls = self.urls.get(*domain_id)?.unwrap_or_default();-- let sampled: Vec<_> = weighted_sample(- urls.iter().filter_map(|(id, state)| {+ for domain in domains {+ let mut urls = self.urls.get(domain)?.unwrap_or_default();+ let available_urls: Vec<_> = urls+ .iter()+ .filter_map(|(url, state)| {if state.status == UrlStatus::Pending {- Some((id, state.weight))+ Some((url.clone(), state.weight))} else {None}- }),+ })+ .collect();++ let sampled: Vec<_> = weighted_sample(+ available_urls.iter().map(|(url, weight)| (url, *weight)),urls_per_job,- )- .into_iter()- .copied()- .collect();+ );- for id in &sampled {- let state = urls.get_mut(id).unwrap();+ for url in &sampled {+ let state = urls.get_mut(*url).unwrap();state.status = UrlStatus::Crawling;}- let mut domain_state = self.domain_state.get(*domain_id)?.unwrap();+ let mut domain_state = self.domain_state.get(domain)?.unwrap();domain_state.weight = urls.iter()@@ -746,22 +559,21 @@ impl CrawlDb {.max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)).unwrap_or(0.0);- self.domain_state.put(*domain_id, domain_state.clone())?;+ self.domain_state.put(domain, &domain_state)?;let mut job = Job {- domain: self.domain_ids.value(domain_id.0)?.unwrap(),+ domain: domain.clone(),fetch_sitemap: false, // todo: fetch for new sitesurls: VecDeque::with_capacity(urls_per_job),};- for url_id in sampled {- let url = self.url_ids.value(url_id.0)?.unwrap();- job.urls.push_back(url);+ for url in sampled {+ job.urls.push_back(url.into());}jobs.push(job);- self.urls.put(*domain_id, urls)?;+ self.urls.put(domain, &urls)?;}Ok(jobs)@@ -802,14 +614,13 @@ mod tests {.unwrap();let domain = Domain::from(&Url::parse("https://example.com").unwrap());- let domain_id = domain.id();let sample = db.sample_domains(128).unwrap();assert_eq!(sample.len(), 1);- assert_eq!(&sample[0], &domain_id);+ assert_eq!(&sample[0], &domain);assert_eq!(- db.domain_state.get(domain_id).unwrap().unwrap().status,+ db.domain_state.get(&domain).unwrap().unwrap().status,DomainStatus::CrawlInProgress);diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex bd2c1e8b..84349e77 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -25,7 +25,7 @@ use url::Url;use crate::{config::CrawlerConfig, webpage::url_ext::UrlExt};-use self::{crawl_db::DomainId, warc_writer::WarcWriter, worker::WorkerThread};+use self::{warc_writer::WarcWriter, worker::WorkerThread};pub mod coordinator;pub mod crawl_db;@@ -74,11 +74,6 @@ impl From<String> for Domain {}impl Domain {- pub fn id(&self) -> DomainId {- let digest = md5::compute(self.0.as_bytes());- u128::from_be_bytes(digest.0).into()- }-pub fn as_str(&self) -> &str {&self.0}