web search done right
Search optimizations and prevent rocksdb write stall. The searches should now be a tiny bit faster by not having to search in unecesarry fields where all the text is allready covered in other fields. This is primarily all the '*IfHomepage' fields. They are still used for scoring.
Mikkel Denker3677f3d80730
diff --git a/Cargo.lock b/Cargo.lockindex e8fa8e7f..5e494217 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -440,9 +440,9 @@ dependencies = [[[package]]name = "bindgen"-version = "0.64.0"+version = "0.65.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"+checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5"dependencies = ["bitflags 1.3.2","cexpr",@@ -450,12 +450,13 @@ dependencies = ["lazy_static","lazycell","peeking_take_while",+ "prettyplease","proc-macro2 1.0.66","quote 1.0.33","regex","rustc-hash","shlex",- "syn 1.0.109",+ "syn 2.0.29",][[package]]@@ -2206,9 +2207,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"[[package]]name = "librocksdb-sys"-version = "0.8.3+7.4.4"+version = "0.11.0+8.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "557b255ff04123fcc176162f56ed0c9cd42d8f357cf55b3fabeb60f7413741b3"+checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e"dependencies = ["bindgen","bzip2-sys",@@ -2216,6 +2217,7 @@ dependencies = ["glob","libc","libz-sys",+ "lz4-sys","pkg-config","tikv-jemalloc-sys","zstd-sys",@@ -2312,6 +2314,16 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39f3d72d77227090eed75ea331285a53726e78374a1f357ff5757702c23c70cc"+[[package]]+name = "lz4-sys"+version = "1.9.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900"+dependencies = [+ "cc",+ "libc",+]+[[package]]name = "lz4_flex"version = "0.9.5"@@ -3131,6 +3143,16 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"+[[package]]+name = "prettyplease"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"+dependencies = [+ "proc-macro2 1.0.66",+ "syn 2.0.29",+]+[[package]]name = "proc-macro-crate"version = "1.3.1"@@ -3616,9 +3638,9 @@ checksum = "413592c1c83b4657fd53561be2e58dc4d739caa90c954d7799b827312f6a3fe8"[[package]]name = "rocksdb"-version = "0.19.0"+version = "0.21.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"+checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe"dependencies = ["libc","librocksdb-sys",diff --git a/Cargo.toml b/Cargo.tomlindex 391a92ec..55be4625 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -37,7 +37,7 @@ axum-macros = "0.3.4"logos = "0.12.1"csv = "1.1.6"fst = { version = "0.4.7", features = ["levenshtein"] }-rocksdb = { version = "0.19.0", features = ["default", "jemalloc", "io-uring"] }+rocksdb = { version = "0.21.0", features = ["default", "jemalloc", "io-uring"] }image = "0.24.3"chrono = { version = "0.4.23", features = ["serde"] }uuid = "1.1.2"diff --git a/core/src/collector.rs b/core/src/collector.rsindex e5e38d50..8e715f4f 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -404,7 +404,7 @@ where}fn requires_scoring(&self) -> bool {- true+ false}fn merge_fruits(diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex d5a962c8..d9b57ca1 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -249,11 +249,16 @@ impl UrlStateDbShard {options.set_target_file_size_base(512 * 1024 * 1024); // 512 MBoptions.set_target_file_size_multiplier(10);+ options.set_max_write_buffer_number(4);+ options.set_min_write_buffer_number_to_merge(2);+ options.set_level_zero_slowdown_writes_trigger(-1);+ options.set_level_zero_stop_writes_trigger(-1);+// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);options.set_bytes_per_sync(1048576);let mut block_options = rocksdb::BlockBasedOptions::default();- let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GB+ let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024); // 1GBblock_options.set_block_cache(&cache);block_options.set_ribbon_filter(10.0);block_options.set_format_version(5);diff --git a/core/src/schema.rs b/core/src/schema.rsindex 7cf6baa9..7c2653b9 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -609,6 +609,16 @@ impl Field {| Field::Text(TextField::SafetyClassification)| Field::Text(TextField::FlattenedSchemaOrgJson)| Field::Text(TextField::UrlForSiteOperator)+ | Field::Text(TextField::Description)+ | Field::Text(TextField::DmozDescription)+ | Field::Text(TextField::SiteIfHomepageNoTokenizer)+ | Field::Text(TextField::DomainIfHomepage)+ | Field::Text(TextField::DomainNameIfHomepageNoTokenizer)+ | Field::Text(TextField::DomainIfHomepageNoTokenizer)+ | Field::Text(TextField::TitleIfHomepage)+ | Field::Text(TextField::SiteWithout) // will match url+ | Field::Text(TextField::Domain) // will match url+ | Field::Text(TextField::CleanBody) // will match all_body) && !self.is_fast()}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 2cea3b83..dc1f96c4 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -36,7 +36,6 @@ mod store;use self::segment::StoredSegment;pub const MAX_LABEL_LENGTH: usize = 1024;-const MAX_BATCH_SIZE: usize = 50_000;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(u128);@@ -521,7 +520,7 @@ impl Webgraph {segments,executor: Arc::new(executor),id2node: Id2NodeDb::open(path.as_ref().join("id2node")),- insert_batch: Vec::with_capacity(MAX_BATCH_SIZE),+ insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),meta,}}@@ -547,7 +546,7 @@ impl Webgraph {self.insert_batch.push(edge);- if self.insert_batch.len() >= MAX_BATCH_SIZE {+ if self.insert_batch.len() >= store::MAX_BATCH_SIZE {self.commit();}}diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 11a0d22f..00ba2a84 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -21,7 +21,7 @@ use rocksdb::BlockBasedOptions;use super::{Edge, EdgeLabel, NodeID};-const MAX_BATCH_SIZE: usize = 50_000;+pub const MAX_BATCH_SIZE: usize = 100_000;pub struct EdgeStore {reversed: bool,@@ -44,6 +44,11 @@ impl EdgeStore {options.set_target_file_size_base(512 * 1024 * 1024); // 512 MBoptions.set_target_file_size_multiplier(10);+ options.set_max_write_buffer_number(4);+ options.set_min_write_buffer_number_to_merge(2);+ options.set_level_zero_slowdown_writes_trigger(-1);+ options.set_level_zero_stop_writes_trigger(-1);+// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);options.set_bytes_per_sync(1048576);