Skip to content

web search done right

rkyv serialization in crawl-db to increase performance quite a bit

Mikkel Denker62264700fa66
diff --git a/Cargo.lock b/Cargo.lock
index 1fcbcf6f..c1473f4f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -104,12 +104,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "allocator-api2"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
-
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -586,6 +580,28 @@ version = "3.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8"
+[[package]]
+name = "bytecheck"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627"
+dependencies = [
+ "bytecheck_derive",
+ "ptr_meta",
+ "simdutf8",
+]
+
+[[package]]
+name = "bytecheck_derive"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61"
+dependencies = [
+ "proc-macro2 1.0.56",
+ "quote 1.0.26",
+ "syn 1.0.109",
+]
+
[[package]]
name = "bytemuck"
version = "1.13.1"
@@ -1820,11 +1836,6 @@ name = "hashbrown"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
-dependencies = [
- "ahash 0.8.3",
- "allocator-api2",
- "serde",
-]
[[package]]
name = "heck"
@@ -3371,6 +3382,26 @@ version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+[[package]]
+name = "ptr_meta"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
+dependencies = [
+ "ptr_meta_derive",
+]
+
+[[package]]
+name = "ptr_meta_derive"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
+dependencies = [
+ "proc-macro2 1.0.56",
+ "quote 1.0.26",
+ "syn 1.0.109",
+]
+
[[package]]
name = "publicsuffix"
version = "2.2.3"
@@ -3629,6 +3660,15 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
+[[package]]
+name = "rend"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab"
+dependencies = [
+ "bytecheck",
+]
+
[[package]]
name = "reqwest"
version = "0.11.16"
@@ -3684,6 +3724,34 @@ dependencies = [
"thiserror",
]
+[[package]]
+name = "rkyv"
+version = "0.7.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58"
+dependencies = [
+ "bitvec",
+ "bytecheck",
+ "hashbrown 0.12.3",
+ "ptr_meta",
+ "rend",
+ "rkyv_derive",
+ "seahash",
+ "tinyvec",
+ "uuid",
+]
+
+[[package]]
+name = "rkyv_derive"
+version = "0.7.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d"
+dependencies = [
+ "proc-macro2 1.0.56",
+ "quote 1.0.26",
+ "syn 1.0.109",
+]
+
[[package]]
name = "robotstxt-with-cache"
version = "0.4.0"
@@ -3974,6 +4042,12 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "seahash"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+
[[package]]
name = "security-framework"
version = "2.8.2"
@@ -4156,6 +4230,12 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "238abfbb77c1915110ad968465608b68e869e0772622c9656714e73e5a1a522f"
+[[package]]
+name = "simdutf8"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
+
[[package]]
name = "siphasher"
version = "0.3.10"
@@ -4269,7 +4349,6 @@ dependencies = [
"fst",
"futures",
"half 2.2.1",
- "hashbrown 0.14.0",
"html-escape",
"http",
"image",
@@ -4300,6 +4379,7 @@ dependencies = [
"regex",
"reqwest",
"reqwest-eventsource",
+ "rkyv",
"robotstxt-with-cache",
"rocksdb",
"rust-s3",
diff --git a/Cargo.toml b/Cargo.toml
index e14b88d1..559e419f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -98,5 +98,5 @@ utoipa = "3.4.4"
utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
tikv-jemallocator = "0.5"
publicsuffix = "2.2.3"
-hashbrown = { version = "0.14.0", features = ["serde"] }
+rkyv = { version = "0.7.42", features = ["validation"] }
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 99c53d93..6ade96bb 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -95,7 +95,7 @@ bytemuck = { workspace = true }
utoipa = { workspace = true }
utoipa-swagger-ui = { workspace = true }
publicsuffix = { workspace = true }
-hashbrown = { workspace = true }
+rkyv = { workspace = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { workspace = true }
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index fadd267c..6e76fdc7 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -14,9 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use hashbrown::HashMap;
use rand::Rng;
-use serde::{Deserialize, Serialize};
+use rkyv::{Archive, Deserialize, Serialize};
+use std::collections::{BTreeMap, HashMap};
use std::{
cmp::Ordering,
collections::{BinaryHeap, VecDeque},
@@ -26,7 +26,8 @@ use url::Url;
use super::{Domain, Job, JobResponse, Result, UrlResponse};
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Archive)]
+#[archive(check_bytes)]
pub enum UrlStatus {
Pending,
Crawling,
@@ -34,7 +35,8 @@ pub enum UrlStatus {
Done,
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Archive)]
+#[archive(check_bytes)]
pub enum DomainStatus {
Pending,
CrawlInProgress,
@@ -89,12 +91,15 @@ fn weighted_sample<T>(items: impl Iterator<Item = (T, f64)>, num_items: usize) -
sampled_items.into_iter().map(|s| s.item).collect()
}
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize, Archive)]
+#[archive(check_bytes)]
struct UrlState {
weight: f64,
status: UrlStatus,
}
-#[derive(Clone, Serialize, Deserialize)]
+
+#[derive(Clone, Serialize, Deserialize, Archive)]
+#[archive(check_bytes)]
struct DomainState {
weight: f64,
status: DomainStatus,
@@ -149,16 +154,11 @@ struct UrlToInsert {
different_domain: bool,
}
-struct PointDb<K, V> {
+struct UrlStateDb {
db: rocksdb::DB,
- _marker: std::marker::PhantomData<(K, V)>,
}
-impl<K, V> PointDb<K, V>
-where
- K: serde::Serialize + serde::de::DeserializeOwned,
- V: serde::Serialize + serde::de::DeserializeOwned,
-{
+impl UrlStateDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let mut options = rocksdb::Options::default();
@@ -173,30 +173,35 @@ where
options.set_write_buffer_size(512 * 1024 * 1024);
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
options.optimize_for_point_lookup(512);
+ options.set_max_subcompactions(8);
+ options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
+ options.set_compression_type(rocksdb::DBCompressionType::None);
let db = rocksdb::DB::open(&options, path.as_ref())?;
- Ok(Self {
- db,
- _marker: std::marker::PhantomData,
- })
+ Ok(Self { db })
}
- pub fn get(&mut self, key: &K) -> Result<Option<V>> {
- let key_bytes = bincode::serialize(key)?;
+ pub fn get(&mut self, key: &Domain) -> Result<Option<BTreeMap<UrlString, UrlState>>> {
+ let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;
let value_bytes = self.db.get(key_bytes)?;
match value_bytes {
- Some(value_bytes) => Ok(Some(bincode::deserialize(&value_bytes)?)),
+ Some(value_bytes) => {
+ let archived =
+ rkyv::check_archived_root::<BTreeMap<UrlString, UrlState>>(&value_bytes[..])
+ .unwrap();
+ let value = archived.deserialize(&mut rkyv::Infallible).unwrap();
+ Ok(Some(value))
+ }
None => Ok(None),
}
}
- pub fn put(&mut self, key: &K, value: &V) -> Result<()> {
- let key_bytes = bincode::serialize(key)?;
- let value_bytes = bincode::serialize(value)?;
+ pub fn put(&mut self, key: &Domain, value: &BTreeMap<UrlString, UrlState>) -> Result<()> {
+ let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;
+ let value_bytes = rkyv::to_bytes::<_, 1024>(value)?;
let mut write_options = rocksdb::WriteOptions::default();
write_options.disable_wal(true);
@@ -225,7 +230,7 @@ impl DomainStateDb {
options.set_write_buffer_size(512 * 1024 * 1024);
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
- options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
+ options.set_max_subcompactions(8);
let db = rocksdb::DB::open(&options, path.as_ref())?;
@@ -233,11 +238,12 @@ impl DomainStateDb {
}
fn get(&self, domain: &Domain) -> Result<Option<DomainState>> {
- let domain_bytes = bincode::serialize(domain)?;
+ let domain_bytes = rkyv::to_bytes::<_, 1024>(domain)?;
let value_bytes = self.db.get(domain_bytes)?;
if let Some(value_bytes) = &value_bytes {
- let value: DomainState = bincode::deserialize(value_bytes)?;
+ let archived = rkyv::check_archived_root::<DomainState>(&value_bytes[..]).unwrap();
+ let value = archived.deserialize(&mut rkyv::Infallible).unwrap();
return Ok(Some(value));
}
@@ -245,8 +251,8 @@ impl DomainStateDb {
}
fn put(&self, domain: &Domain, state: &DomainState) -> Result<()> {
- let domain_bytes = bincode::serialize(domain)?;
- let state_bytes = bincode::serialize(state)?;
+ let domain_bytes = rkyv::to_bytes::<_, 1024>(domain)?;
+ let state_bytes = rkyv::to_bytes::<_, 1024>(state)?;
let mut write_options = rocksdb::WriteOptions::default();
write_options.disable_wal(true);
@@ -260,15 +266,20 @@ impl DomainStateDb {
iter.filter_map(|r| {
let (key, value) = r.ok()?;
- let domain: Domain = bincode::deserialize(&key).unwrap();
- let state: DomainState = bincode::deserialize(&value).unwrap();
+ let domain_archive = rkyv::check_archived_root::<Domain>(&key[..]).ok()?;
+ let state_archive = rkyv::check_archived_root::<DomainState>(&value[..]).ok()?;
+
+ let domain: Domain = domain_archive.deserialize(&mut rkyv::Infallible).ok()?;
+ let state: DomainState = state_archive.deserialize(&mut rkyv::Infallible).ok()?;
Some((domain, state))
})
}
}
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
+#[derive(Clone, Serialize, Deserialize, Archive, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[archive(check_bytes)]
+#[archive_attr(derive(Hash, PartialEq, Eq, PartialOrd, Ord))]
struct UrlString(String);
impl From<&Url> for UrlString {
@@ -294,23 +305,23 @@ pub struct CrawlDb {
domain_state: DomainStateDb,
- urls: PointDb<Domain, HashMap<UrlString, UrlState>>,
+ urls: UrlStateDb,
}
impl CrawlDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- if Path::new(path.as_ref()).exists() {
- return Err(anyhow::anyhow!(
- "crawl db already exists and might be in incorrect state"
- ));
- }
+ // if Path::new(path.as_ref()).exists() {
+ // return Err(anyhow::anyhow!(
+ // "crawl db already exists and might be in incorrect state"
+ // ));
+ // }
let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;
Ok(Self {
redirects,
domain_state: DomainStateDb::open(path.as_ref().join("domains"))?,
- urls: PointDb::open(path.as_ref().join("urls"))?,
+ urls: UrlStateDb::open(path.as_ref().join("urls"))?,
})
}
@@ -541,8 +552,10 @@ impl CrawlDb {
);
for url in &sampled {
- let state = urls.get_mut(*url).unwrap();
+ let mut state = urls.get(url).unwrap().clone();
state.status = UrlStatus::Crawling;
+
+ urls.insert((*url).clone(), state);
}
let mut domain_state = self.domain_state.get(domain)?.unwrap();
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 84349e77..afca1247 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -58,7 +58,19 @@ type Result<T, E = anyhow::Error> = std::result::Result<T, E>;
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct Site(String);
-#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
+#[derive(
+ Clone,
+ Debug,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ rkyv::Serialize,
+ rkyv::Deserialize,
+ rkyv::Archive,
+)]
+#[archive(check_bytes)]
pub struct Domain(String);
impl From<&Url> for Domain {
diff --git a/docs/src/optics.md b/docs/src/optics.md
index c6725c2a..caa41a3d 100644
--- a/docs/src/optics.md
+++ b/docs/src/optics.md
@@ -1,5 +1,5 @@
# Optics
Optics is a domain specific language that is intended to give the user full control over which search results gets returned.
+The syntax for optics is described in the [quickstart guide](https://github.com/StractOrg/sample-optics/blob/main/quickstart.optic), but the overall idea is that the user specifies a set of rules where each rule contains a set of matches that must all match the search result and an action that is then applied to the search result. The result can either be boosted, downranked or discarded.
-- If the user has specified an optic url, we grab the file from the url and parses it.
-- Compiles down to tantivy queries that are matched against the inverted index.
+When an optic is applied to a search query, the optic is fetched at the provided url after which it is parsed and compiled into an additional set of tantivy queries. These queries are then combined with the actual search query that can be evaluated against the inverted index.
\ No newline at end of file