Skip to content

web search done right

Search optimizations and prevent rocksdb write stall. The searches should now be a tiny bit faster by not having to search in unecesarry fields where all the text is allready covered in other fields. This is primarily all the '*IfHomepage' fields. They are still used for scoring.

Mikkel Denker3677f3d80730
diff --git a/Cargo.lock b/Cargo.lock
index e8fa8e7f..5e494217 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -440,9 +440,9 @@ dependencies = [
[[package]]
name = "bindgen"
-version = "0.64.0"
+version = "0.65.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
+checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5"
dependencies = [
"bitflags 1.3.2",
"cexpr",
@@ -450,12 +450,13 @@ dependencies = [
"lazy_static",
"lazycell",
"peeking_take_while",
+ "prettyplease",
"proc-macro2 1.0.66",
"quote 1.0.33",
"regex",
"rustc-hash",
"shlex",
- "syn 1.0.109",
+ "syn 2.0.29",
]
[[package]]
@@ -2206,9 +2207,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
[[package]]
name = "librocksdb-sys"
-version = "0.8.3+7.4.4"
+version = "0.11.0+8.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "557b255ff04123fcc176162f56ed0c9cd42d8f357cf55b3fabeb60f7413741b3"
+checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e"
dependencies = [
"bindgen",
"bzip2-sys",
@@ -2216,6 +2217,7 @@ dependencies = [
"glob",
"libc",
"libz-sys",
+ "lz4-sys",
"pkg-config",
"tikv-jemalloc-sys",
"zstd-sys",
@@ -2312,6 +2314,16 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39f3d72d77227090eed75ea331285a53726e78374a1f357ff5757702c23c70cc"
+[[package]]
+name = "lz4-sys"
+version = "1.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900"
+dependencies = [
+ "cc",
+ "libc",
+]
+
[[package]]
name = "lz4_flex"
version = "0.9.5"
@@ -3131,6 +3143,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+[[package]]
+name = "prettyplease"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
+dependencies = [
+ "proc-macro2 1.0.66",
+ "syn 2.0.29",
+]
+
[[package]]
name = "proc-macro-crate"
version = "1.3.1"
@@ -3616,9 +3638,9 @@ checksum = "413592c1c83b4657fd53561be2e58dc4d739caa90c954d7799b827312f6a3fe8"
[[package]]
name = "rocksdb"
-version = "0.19.0"
+version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"
+checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe"
dependencies = [
"libc",
"librocksdb-sys",
diff --git a/Cargo.toml b/Cargo.toml
index 391a92ec..55be4625 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,7 +37,7 @@ axum-macros = "0.3.4"
logos = "0.12.1"
csv = "1.1.6"
fst = { version = "0.4.7", features = ["levenshtein"] }
-rocksdb = { version = "0.19.0", features = ["default", "jemalloc", "io-uring"] }
+rocksdb = { version = "0.21.0", features = ["default", "jemalloc", "io-uring"] }
image = "0.24.3"
chrono = { version = "0.4.23", features = ["serde"] }
uuid = "1.1.2"
diff --git a/core/src/collector.rs b/core/src/collector.rs
index e5e38d50..8e715f4f 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -404,7 +404,7 @@ where
}
fn requires_scoring(&self) -> bool {
- true
+ false
}
fn merge_fruits(
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index d5a962c8..d9b57ca1 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -249,11 +249,16 @@ impl UrlStateDbShard {
options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
options.set_target_file_size_multiplier(10);
+ options.set_max_write_buffer_number(4);
+ options.set_min_write_buffer_number_to_merge(2);
+ options.set_level_zero_slowdown_writes_trigger(-1);
+ options.set_level_zero_stop_writes_trigger(-1);
+
// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
options.set_level_compaction_dynamic_level_bytes(true);
options.set_bytes_per_sync(1048576);
let mut block_options = rocksdb::BlockBasedOptions::default();
- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GB
+ let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024); // 1GB
block_options.set_block_cache(&cache);
block_options.set_ribbon_filter(10.0);
block_options.set_format_version(5);
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 7cf6baa9..7c2653b9 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -609,6 +609,16 @@ impl Field {
| Field::Text(TextField::SafetyClassification)
| Field::Text(TextField::FlattenedSchemaOrgJson)
| Field::Text(TextField::UrlForSiteOperator)
+ | Field::Text(TextField::Description)
+ | Field::Text(TextField::DmozDescription)
+ | Field::Text(TextField::SiteIfHomepageNoTokenizer)
+ | Field::Text(TextField::DomainIfHomepage)
+ | Field::Text(TextField::DomainNameIfHomepageNoTokenizer)
+ | Field::Text(TextField::DomainIfHomepageNoTokenizer)
+ | Field::Text(TextField::TitleIfHomepage)
+ | Field::Text(TextField::SiteWithout) // will match url
+ | Field::Text(TextField::Domain) // will match url
+ | Field::Text(TextField::CleanBody) // will match all_body
) && !self.is_fast()
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 2cea3b83..dc1f96c4 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -36,7 +36,6 @@ mod store;
use self::segment::StoredSegment;
pub const MAX_LABEL_LENGTH: usize = 1024;
-const MAX_BATCH_SIZE: usize = 50_000;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(u128);
@@ -521,7 +520,7 @@ impl Webgraph {
segments,
executor: Arc::new(executor),
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
- insert_batch: Vec::with_capacity(MAX_BATCH_SIZE),
+ insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),
meta,
}
}
@@ -547,7 +546,7 @@ impl Webgraph {
self.insert_batch.push(edge);
- if self.insert_batch.len() >= MAX_BATCH_SIZE {
+ if self.insert_batch.len() >= store::MAX_BATCH_SIZE {
self.commit();
}
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 11a0d22f..00ba2a84 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -21,7 +21,7 @@ use rocksdb::BlockBasedOptions;
use super::{Edge, EdgeLabel, NodeID};
-const MAX_BATCH_SIZE: usize = 50_000;
+pub const MAX_BATCH_SIZE: usize = 100_000;
pub struct EdgeStore {
reversed: bool,
@@ -44,6 +44,11 @@ impl EdgeStore {
options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
options.set_target_file_size_multiplier(10);
+ options.set_max_write_buffer_number(4);
+ options.set_min_write_buffer_number_to_merge(2);
+ options.set_level_zero_slowdown_writes_trigger(-1);
+ options.set_level_zero_stop_writes_trigger(-1);
+
// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
options.set_level_compaction_dynamic_level_bytes(true);
options.set_bytes_per_sync(1048576);