web search done right
try rkyv serialization in crawl coordinator to hopefully fix oom error
Mikkel Denkerbdd9d415d73f
diff --git a/Cargo.toml b/Cargo.tomlindex 3dd0a5fc..8d4e0c25 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -78,7 +78,7 @@ tokio-stream = "0.1.11"scylla = "0.7.0"chitchat = "0.5.0"memmap2 = "0.5.10"-rkyv = "0.7.41"+rkyv = { version = "0.7.41", features = ["validation"] }dashmap = "5.4.0"tch = "0.13.0"torch-sys = "0.13.0"diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 8637f496..9f6290c9 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -24,13 +24,15 @@ use std::{use lru::LruCache;use rand::Rng;+use rkyv::Deserialize;use rocksdb::BlockBasedOptions;use crate::webpage::Url;use super::{Domain, Job, Result, UrlResponse};-#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq)]+#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Clone, PartialEq, Eq)]+#[archive(check_bytes)]pub enum UrlStatus {Pending,Crawling,@@ -255,7 +257,8 @@ fn weighted_sample<'a, T: 'a>(sampled_items.into_iter().map(|s| s.0.item).collect()}-#[derive(serde::Serialize, serde::Deserialize)]+#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)]+#[archive(check_bytes)]struct UrlState {weight: f64,status: UrlStatus,@@ -265,7 +268,20 @@ struct DomainState {status: DomainStatus,}-#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[derive(+ serde::Serialize,+ serde::Deserialize,+ rkyv::Serialize,+ rkyv::Deserialize,+ rkyv::Archive,+ Clone,+ Copy,+ Debug,+ PartialEq,+ Eq,+ Hash,+)]+#[archive(check_bytes)]pub struct DomainId(u64);impl From<u64> for DomainId {@@ -274,7 +290,11 @@ impl From<u64> for DomainId {}}-#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[derive(+ rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Clone, Copy, Debug, PartialEq, Eq, Hash,+)]+#[archive(check_bytes)]+#[archive_attr(derive(PartialEq, Eq, Hash))]struct UrlId(u64);impl From<u64> for UrlId {@@ -335,8 +355,8 @@ impl UrlMap {}pub fn put(&self, domain_id: DomainId, map: HashMap<UrlId, UrlState>) -> Result<()> {- let domain_bytes = bincode::serialize(&domain_id)?;- let map_bytes = bincode::serialize(&map)?;+ let domain_bytes = rkyv::to_bytes::<_, 256>(&domain_id).expect("failed to serialize");+ let map_bytes = rkyv::to_bytes::<_, 256>(&map).expect("failed to serialize");let mut write_options = rocksdb::WriteOptions::default();write_options.set_sync(false);@@ -349,13 +369,16 @@ impl UrlMap {}pub fn get(&self, domain_id: DomainId) -> Result<Option<HashMap<UrlId, UrlState>>> {- let domain_bytes = bincode::serialize(&domain_id)?;+ let domain_bytes = rkyv::to_bytes::<_, 256>(&domain_id).expect("failed to serialize");let map_bytes = self.inner.get(domain_bytes)?;match map_bytes {Some(bytes) => {- let map = bincode::deserialize(&bytes)?;+ let map = rkyv::check_archived_root::<HashMap<UrlId, UrlState>>(&bytes)+ .expect("failed to deserialize")+ .deserialize(&mut rkyv::Infallible)+ .expect("failed to deserialize");Ok(Some(map))}None => Ok(None),