web search done right
approximate number of hits by assuming that each term is independent this allows us to short-cirquit the query by default which significantly improves performance as we therefore don't have to iterate the non-scored results simply to count them
Mikkel Denker3c94cb7f8145
diff --git a/Cargo.lock b/Cargo.lockindex 76638ecc..3fb4dd83 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2919,6 +2919,17 @@ dependencies = ["winapi",]+[[package]]+name = "num-bigint"+version = "0.4.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"+dependencies = [+ "autocfg",+ "num-integer",+ "num-traits",+]+[[package]]name = "num-complex"version = "0.4.5"@@ -2935,6 +2946,27 @@ version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"+[[package]]+name = "num-integer"+version = "0.1.46"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"+dependencies = [+ "num-traits",+]++[[package]]+name = "num-rational"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"+dependencies = [+ "autocfg",+ "num-bigint",+ "num-integer",+ "num-traits",+]+[[package]]name = "num-traits"version = "0.2.18"@@ -4623,6 +4655,8 @@ dependencies = ["mime","min-max-heap","nom",+ "num-bigint",+ "num-rational","num_cpus","once_cell","openraft",diff --git a/Cargo.toml b/Cargo.tomlindex e5001b64..a2fd4326 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,14 +1,14 @@[workspace]members = [- "crates/bloom",- "crates/client-wasm",- "crates/client-wasm",- "crates/core",- "crates/file-store",- "crates/kuchiki",- "crates/optics",- "crates/speedy-kv",- "crates/zimba",+ "crates/bloom",+ "crates/client-wasm",+ "crates/client-wasm",+ "crates/core",+ "crates/file-store",+ "crates/kuchiki",+ "crates/optics",+ "crates/speedy-kv",+ "crates/zimba",]resolver = "2"@@ -20,61 +20,56 @@ lto = "thin"debug = true[workspace.dependencies]-ahash = "0.8.11"aes-gcm = "0.10.2"-anyhow = { version = "1.0.72", features = ["backtrace"] }+ahash = "0.8.11"+anyhow = {version = "1.0.72", features = ["backtrace"]}async-channel = "1.8.0"async-stream = "0.3.3"axum = "0.7.2"-axum-extra = { version = "0.9.0" }+axum-extra = {version = "0.9.0"}axum-macros = "0.4.0"base64 = "0.21.4"-bincode = { git = "https://github.com/bincode-org/bincode", rev = "8f65c60", features = [- "serde",-] }+bincode = {git = "https://github.com/bincode-org/bincode", rev = "8f65c60", features = [+ "serde",+]}bindgen = "0.69.2"-bitvec = { version = "1.0.1", features = ["serde"] }+bitvec = {version = "1.0.1", features = ["serde"]}bytecount = "0.6.7"-bytemuck = { version = "1.13.1", features = ["derive"] }+bytemuck = {version = "1.13.1", features = ["derive"]}byteorder = "1.4.3"bzip2 = "0.4.4"candle-core = "0.3.3"candle-nn = "0.3.3"-openraft = { version = "0.9.1", features = [- "storage-v2",- "serde",- "tracing-log",-] }candle-transformers = "0.3.3"-cc = { version = "1", features = ["parallel"] }+cc = {version = "1", features = ["parallel"]}chardetng = "0.1.17"chitchat = "0.5.0"-chrono = { version = "0.4.23", features = ["serde"] }-clap = { version = "4.4.6", features = ["derive"] }+chrono = {version = "0.4.23", features = ["serde"]}+clap = {version = "4.4.6", features = ["derive"]}cmake = "0.1"criterion = "0.5.1"crossbeam-channel = "0.5.6"csv = "1.1.6"-dashmap = { version = "5.4.0", features = ["rayon"] }+dashmap = {version = "5.4.0", features = ["rayon"]}encoding_rs = "0.8.31"enum_dispatch = "0.3.12"eventsource-stream = "0.2.3"fend-core = "1.4.8"flate2 = "1.0.28"fnv = "1.0.3"-fst = { version = "0.4.7", features = ["levenshtein"] }+fst = {version = "0.4.7", features = ["levenshtein"]}futures = "0.3.21"-half = { version = "2.2.1", features = ["serde"] }-hashbrown = { version = "0.14.0", features = ["serde"] }+half = {version = "2.2.1", features = ["serde"]}+hashbrown = {version = "0.14.0", features = ["serde"]}http = "1.0.0"image = "0.24.3"-indicatif = { version = "0.17.7", features = ["rayon"] }+indicatif = {version = "0.17.7", features = ["rayon"]}insta = "1.31"itertools = "0.11.0"-lalrpop = { version = "0.20.0", features = ["lexer"] }-lalrpop-util = { version = "0.20.0", features = ["lexer"] }+lalrpop = {version = "0.20.0", features = ["lexer"]}+lalrpop-util = {version = "0.20.0", features = ["lexer"]}libc = "0.2.142"-log = { version = "0.4", features = ["release_max_level_info"] }+log = {version = "0.4", features = ["release_max_level_info"]}logos = "0.13.0"lz-str = "0.2.1"lz4_flex = "0.11.1"@@ -85,9 +80,16 @@ memmap2 = "0.9.0"mime = "0.3.17"min-max-heap = "1.3.0"nom = "7.1.3"+num-bigint = "0.4.1"+num-rational = "0.4.1"num_cpus = "1.15.0"once_cell = "1.13.1"-postcard = { version = "1.0.8", features = ["experimental-derive", "alloc"] }+openraft = {version = "0.9.1", features = [+ "storage-v2",+ "serde",+ "tracing-log",+]}+postcard = {version = "1.0.8", features = ["experimental-derive", "alloc"]}proptest = "1.2.0"proptest-derive = "0.4.0"publicsuffix = "2.2.3"@@ -96,46 +98,46 @@ rand = "0.8.5"rayon = "1.5.3"redb = "2.0.0"regex = "1.6.0"-reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }-rkyv = { version = "0.7.44", features = ["validation", "bytecheck"] }+reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}ring = "0.17.3"rio_api = "0.8.4"rio_turtle = "0.8.4"+rkyv = {version = "0.7.44", features = ["validation", "bytecheck"]}robotstxt-with-cache = "0.4.0"-rusqlite = { version = "0.29.0", features = [- "bundled",- "modern-full",- "buildtime_bindgen",-] }-rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }+rusqlite = {version = "0.29.0", features = [+ "bundled",+ "modern-full",+ "buildtime_bindgen",+]}+rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}rust-stemmers = "1.2.0"safetensors = "0.3.1"-scylla = { version = "0.12.0", features = ["chrono"] }-serde = { version = "1.0.137", features = ["rc", "derive"] }+scylla = {version = "0.12.0", features = ["chrono"]}+serde = {version = "1.0.137", features = ["rc", "derive"]}serde_json = "1.0.81"serde_urlencoded = "0.7.1"stable_deref_trait = "1.2.0"-strum = { version = "0.26.2", features = ["derive"] }-tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "74940e9" }+strum = {version = "0.26.2", features = ["derive"]}+tantivy = {git = "https://github.com/quickwit-oss/tantivy", rev = "74940e9"}thiserror = "1.0.31"tikv-jemallocator = "0.5"tokenizers = "0.13.2"-tokio = { version = "1.23.1", features = ["full"] }+tokio = {version = "1.23.1", features = ["full"]}tokio-stream = "0.1.11"toml = "0.8.2"-tower = { version = "0.4.13", features = ["limit"] }-tower-http = { version = "0.5.0", features = ["compression-gzip", "cors"] }-tracing = { version = "0.1.34", features = ["release_max_level_info"] }-tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }-url = { version = "2.4.0", features = ["serde"] }+tower = {version = "0.4.13", features = ["limit"]}+tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}+tracing = {version = "0.1.34", features = ["release_max_level_info"]}+tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}+tracing-test = "0.2.4"+url = {version = "2.4.0", features = ["serde"]}urlencoding = "2.1.2"-utoipa = { version = "4.0.0", features = ["axum_extras"] }-utoipa-swagger-ui = { version = "5.0.0", features = ["axum"] }+utoipa = {version = "4.0.0", features = ["axum_extras"]}+utoipa-swagger-ui = {version = "5.0.0", features = ["axum"]}uuid = "1.1.2"-whatlang = { version = "0.16.0", features = ["serde"] }+whatlang = {version = "0.16.0", features = ["serde"]}+xxhash-rust = {version = "0.8.10", features = ["xxh3", "const_xxh3"]}zstd = "0.13"-tracing-test = "0.2.4"-xxhash-rust = { version = "0.8.10", features = ["xxh3", "const_xxh3"] }[profile.test.package]flate2.opt-level = 3diff --git a/assets/licenses.html b/assets/licenses.htmlindex 666ebc23..b0c12ad8 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,7 +44,7 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (388)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (391)</li><li><a href="#MIT">MIT License</a> (192)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (8)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (7)</li>@@ -7524,7 +7524,10 @@ limitations under the License.</pre><li><a href=" https://github.com/servo/html5ever ">markup5ever 0.11.0</a></li><li><a href=" https://github.com/hyperium/mime ">mime 0.3.17</a></li><li><a href=" https://github.com/tov/min-max-heap-rs ">min-max-heap 1.3.0</a></li>+ <li><a href=" https://github.com/rust-num/num-bigint ">num-bigint 0.4.4</a></li><li><a href=" https://github.com/rust-num/num-complex ">num-complex 0.4.5</a></li>+ <li><a href=" https://github.com/rust-num/num-integer ">num-integer 0.1.46</a></li>+ <li><a href=" https://github.com/rust-num/num-rational ">num-rational 0.4.1</a></li><li><a href=" https://github.com/rust-num/num-traits ">num-traits 0.2.18</a></li><li><a href=" https://github.com/seanmonstar/num_cpus ">num_cpus 1.16.0</a></li><li><a href=" https://github.com/gimli-rs/object ">object 0.32.2</a></li>diff --git a/configs/search_server.toml b/configs/search_server.tomlindex 51cfa902..0453c739 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -1,14 +1,17 @@cluster_id = "dev_search"+dual_encoder_model_path = "data/summarizer/dual_encoder"entity_index_path = "data/entity"gossip_addr = "0.0.0.0:3006"gossip_seed_nodes = ["0.0.0.0:3005"]host = "0.0.0.0:3002"index_path = "data/index"shard = 0-dual_encoder_model_path = "data/summarizer/dual_encoder"# linear_model_path = "data/linear_model.json"# lambda_model_path = "data/lambdamart.txt"[snippet]max_considered_words = 10_000num_words_for_lang_detection = 1_000++[collector]+max_docs_considered = 1000diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 215c2bae..2e6e07bf 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -20,113 +20,115 @@ name = "stract"path = "src/main.rs"[dependencies]-ahash = { workspace = true }-aes-gcm = { workspace = true }-anyhow = { workspace = true }-async-channel = { workspace = true }-async-stream = { workspace = true }-axum = { workspace = true }-axum-extra = { workspace = true }-axum-macros = { workspace = true }-base64 = { workspace = true }-bincode = { workspace = true }-bitvec = { workspace = true }-bloom = { path = "../bloom" }-bytecount = { workspace = true }-bytemuck = { workspace = true }-byteorder = { workspace = true }-bzip2 = { workspace = true }-candle-core = { workspace = true }-candle-nn = { workspace = true }-candle-transformers = { workspace = true }-chardetng = { workspace = true }-chitchat = { workspace = true }-chrono = { workspace = true }-clap = { workspace = true }-crossbeam-channel = { workspace = true }-csv = { workspace = true }-dashmap = { workspace = true }-encoding_rs = { workspace = true }-enum_dispatch = { workspace = true }-eventsource-stream = { workspace = true }-fend-core = { workspace = true }-file_store = { path = "../file-store" }-flate2 = { workspace = true }-fnv = { workspace = true }-fst = { workspace = true }-futures = { workspace = true }-half = { workspace = true }-hashbrown = { workspace = true }-http = { workspace = true }-image = { workspace = true }-indicatif = { workspace = true }-itertools = { workspace = true }-kuchiki = { path = "../kuchiki" }-libc = { workspace = true }-log = { workspace = true }-logos = { workspace = true }-lz-str = { workspace = true }-lz4_flex = { workspace = true }-md5 = { workspace = true }-memmap2 = { workspace = true }-mime = { workspace = true }-min-max-heap = { workspace = true }-nom = { workspace = true }-num_cpus = { workspace = true }-once_cell = { workspace = true }-openraft = { workspace = true }-optics = { path = "../optics" }-publicsuffix = { workspace = true }-postcard = { workspace = true }-quick-xml = { workspace = true }-rand = { workspace = true }-rayon = { workspace = true }-redb = { workspace = true }-regex = { workspace = true }-reqwest = { workspace = true }-rkyv = { workspace = true }-ring = { workspace = true }-rio_api = { workspace = true }-rio_turtle = { workspace = true }-robotstxt-with-cache = { workspace = true }-rust-s3 = { workspace = true }-rust-stemmers = { workspace = true }-safetensors = { workspace = true }-scylla = { workspace = true }-serde = { workspace = true }-serde_json = { workspace = true }-serde_urlencoded = { workspace = true }-speedy_kv = { path = "../speedy-kv" }-strum = { workspace = true }-tantivy = { workspace = true }-thiserror = { workspace = true }-tokenizers = { workspace = true }-tokio = { workspace = true }-tokio-stream = { workspace = true }-toml = { workspace = true }-tower = { workspace = true }-tower-http = { workspace = true }-tracing = { workspace = true }-tracing-subscriber = { workspace = true }-url = { workspace = true }-urlencoding = { workspace = true }-utoipa = { workspace = true }-utoipa-swagger-ui = { workspace = true }-uuid = { workspace = true }-whatlang = { workspace = true }-zimba = { path = "../zimba" }-xxhash-rust = { workspace = true }+aes-gcm = {workspace = true}+ahash = {workspace = true}+anyhow = {workspace = true}+async-channel = {workspace = true}+async-stream = {workspace = true}+axum = {workspace = true}+axum-extra = {workspace = true}+axum-macros = {workspace = true}+base64 = {workspace = true}+bincode = {workspace = true}+bitvec = {workspace = true}+bloom = {path = "../bloom"}+bytecount = {workspace = true}+bytemuck = {workspace = true}+byteorder = {workspace = true}+bzip2 = {workspace = true}+candle-core = {workspace = true}+candle-nn = {workspace = true}+candle-transformers = {workspace = true}+chardetng = {workspace = true}+chitchat = {workspace = true}+chrono = {workspace = true}+clap = {workspace = true}+crossbeam-channel = {workspace = true}+csv = {workspace = true}+dashmap = {workspace = true}+encoding_rs = {workspace = true}+enum_dispatch = {workspace = true}+eventsource-stream = {workspace = true}+fend-core = {workspace = true}+file_store = {path = "../file-store"}+flate2 = {workspace = true}+fnv = {workspace = true}+fst = {workspace = true}+futures = {workspace = true}+half = {workspace = true}+hashbrown = {workspace = true}+http = {workspace = true}+image = {workspace = true}+indicatif = {workspace = true}+itertools = {workspace = true}+kuchiki = {path = "../kuchiki"}+libc = {workspace = true}+log = {workspace = true}+logos = {workspace = true}+lz-str = {workspace = true}+lz4_flex = {workspace = true}+md5 = {workspace = true}+memmap2 = {workspace = true}+mime = {workspace = true}+min-max-heap = {workspace = true}+nom = {workspace = true}+num-bigint = {workspace = true}+num-rational = {workspace = true}+num_cpus = {workspace = true}+once_cell = {workspace = true}+openraft = {workspace = true}+optics = {path = "../optics"}+postcard = {workspace = true}+publicsuffix = {workspace = true}+quick-xml = {workspace = true}+rand = {workspace = true}+rayon = {workspace = true}+redb = {workspace = true}+regex = {workspace = true}+reqwest = {workspace = true}+ring = {workspace = true}+rio_api = {workspace = true}+rio_turtle = {workspace = true}+rkyv = {workspace = true}+robotstxt-with-cache = {workspace = true}+rust-s3 = {workspace = true}+rust-stemmers = {workspace = true}+safetensors = {workspace = true}+scylla = {workspace = true}+serde = {workspace = true}+serde_json = {workspace = true}+serde_urlencoded = {workspace = true}+speedy_kv = {path = "../speedy-kv"}+strum = {workspace = true}+tantivy = {workspace = true}+thiserror = {workspace = true}+tokenizers = {workspace = true}+tokio = {workspace = true}+tokio-stream = {workspace = true}+toml = {workspace = true}+tower = {workspace = true}+tower-http = {workspace = true}+tracing = {workspace = true}+tracing-subscriber = {workspace = true}+url = {workspace = true}+urlencoding = {workspace = true}+utoipa = {workspace = true}+utoipa-swagger-ui = {workspace = true}+uuid = {workspace = true}+whatlang = {workspace = true}+xxhash-rust = {workspace = true}+zimba = {path = "../zimba"}[target.'cfg(not(target_env = "msvc"))'.dependencies]-tikv-jemallocator = { workspace = true }+tikv-jemallocator = {workspace = true}[dev-dependencies]-criterion = { workspace = true }-insta = { workspace = true }-maplit = { workspace = true }-proptest = { workspace = true }-proptest-derive = { workspace = true }-tracing-test = { workspace = true }+criterion = {workspace = true}+insta = {workspace = true}+maplit = {workspace = true}+proptest = {workspace = true}+proptest-derive = {workspace = true}+tracing-test = {workspace = true}[[bench]]harness = falsediff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex f8ebd65a..f2dc87d5 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -98,6 +98,8 @@ use utoipa_swagger_ui::SwaggerUi;crate::search_prettifier::OneOrManyString,crate::search_prettifier::OneOrManyProperty,crate::search_prettifier::Property,++ crate::collector::approx_count::Count,),),modifiers(&ApiModifier),diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex 8a2b1180..1a4d20b1 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -69,8 +69,8 @@ pub struct ApiSearchQuery {#[serde(default = "defaults::SearchQuery::flatten_response")]pub flatten_response: bool,- #[serde(default = "defaults::SearchQuery::count_results")]- pub count_results: bool,+ #[serde(default = "defaults::SearchQuery::count_results_exact")]+ pub count_results_exact: bool,#[serde(default = "defaults::SearchQuery::return_structured_data")]pub return_structured_data: bool,@@ -99,7 +99,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {host_rankings: api.host_rankings,return_ranking_signals: api.return_ranking_signals,safe_search: api.safe_search.unwrap_or(default.safe_search),- count_results: api.count_results,+ count_results_exact: api.count_results_exact,return_body: api.return_body,return_structured_data: api.return_structured_data,})diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rsnew file mode 100644index 00000000..cb67f11e--- /dev/null+++ b/crates/core/src/collector/approx_count.rs@@ -0,0 +1,218 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::ops::Add;++use itertools::Itertools;+use tantivy::{+ collector::{Collector, SegmentCollector},+ tokenizer::Tokenizer,+};++use crate::schema::text_field::{self, TextField};++#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ Copy,+ PartialEq,+ Eq,+ utoipa::ToSchema,+)]+#[serde(rename_all = "camelCase", tag = "_type", content = "value")]+pub enum Count {+ Exact(u64),+ Approximate(u64),+}++impl Count {+ pub fn is_exact(&self) -> bool {+ match self {+ Count::Exact(_) => true,+ Count::Approximate(_) => false,+ }+ }++ pub fn is_approximate(&self) -> bool {+ match self {+ Count::Exact(_) => false,+ Count::Approximate(_) => true,+ }+ }++ pub fn compose(&self, other: &Self) -> Self {+ match (self, other) {+ (Count::Exact(a), Count::Exact(b)) => Count::Exact(a + b),+ (Count::Exact(a), Count::Approximate(b))+ | (Count::Approximate(a), Count::Exact(b))+ | (Count::Approximate(a), Count::Approximate(b)) => Count::Approximate(a + b),+ }+ }+}++impl Add for Count {+ type Output = Self;++ fn add(self, other: Self) -> Self {+ self.compose(&other)+ }+}++pub struct ApproxCount {+ max_docs_per_segment: u64,+ terms: Vec<String>,+}++impl ApproxCount {+ pub fn new(max_docs_per_segment: u64, terms: Vec<String>) -> Self {+ Self {+ max_docs_per_segment,+ terms: terms+ .into_iter()+ .map(|term| term.to_lowercase())+ .sorted()+ .dedup()+ .collect(),+ }+ }+}++impl Collector for ApproxCount {+ type Fruit = Count;+ type Child = SegmentApproxCount;++ fn for_segment(+ &self,+ _: tantivy::SegmentOrdinal,+ segment: &tantivy::SegmentReader,+ ) -> tantivy::Result<Self::Child> {+ let field = text_field::AllBody.tantivy_field(segment.schema()).unwrap();+ let mut tokenizer = text_field::AllBody.query_tokenizer();++ let inverted_index = segment.inverted_index(field)?;+ let num_docs = segment.max_doc() as u64;+ let mut term_freqs = Vec::with_capacity(self.terms.len());++ for term in &self.terms {+ let mut term_freq = 0;+ let mut stream = tokenizer.token_stream(term.as_str());++ while let Some(term) = stream.next() {+ term_freq += inverted_index+ .doc_freq(&tantivy::Term::from_field_text(field, term.text.as_str()))?;+ }++ term_freqs.push(term_freq as u64);+ }++ let approx_count = approx_results_assuming_term_independence(&term_freqs, num_docs);++ Ok(SegmentApproxCount {+ max_docs_per_segment: self.max_docs_per_segment,+ approx_count,+ exact_count: 0,+ })+ }++ fn requires_scoring(&self) -> bool {+ false+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> tantivy::Result<Self::Fruit> {+ Ok(segment_fruits+ .into_iter()+ .fold(Count::Exact(0), |acc, fruit| acc.compose(&fruit)))+ }+}++pub struct SegmentApproxCount {+ max_docs_per_segment: u64,+ approx_count: u64,+ exact_count: u64,+}++impl SegmentCollector for SegmentApproxCount {+ type Fruit = Count;++ fn collect(&mut self, _doc: u32, _score: f32) {+ self.exact_count += 1;+ }++ fn harvest(self) -> Self::Fruit {+ if self.exact_count >= self.max_docs_per_segment {+ if self.exact_count > self.approx_count {+ Count::Approximate(self.exact_count)+ } else {+ Count::Approximate(self.approx_count)+ }+ } else {+ Count::Exact(self.exact_count)+ }+ }+}++/// Approximate the number of results for a query assuming each term is independent.+/// The idea is to estiate the probability that a document contains all terms in the query+/// as the product of the probabilities that it contains each term.+///+/// P(A and B) = P(A) * P(B) if A and B are independent.+///+/// returns 0 if term_freqs is empty or num_docs is 0.+fn approx_results_assuming_term_independence(term_freqs: &[u64], num_docs: u64) -> u64 {+ if term_freqs.is_empty() || num_docs == 0 {+ return 0;+ }++ let mut res = None;++ for term_freq in term_freqs {+ let prob = num_rational::BigRational::new((*term_freq).into(), num_docs.into());++ res = Some(res.map_or(prob.clone(), move |res| res * prob));+ }++ let approx_results = res.unwrap() * num_rational::BigRational::new(num_docs.into(), 1.into());++ let (sign, digits) = approx_results.to_integer().to_u64_digits();++ if matches!(sign, num_bigint::Sign::Minus | num_bigint::Sign::NoSign) || digits.is_empty() {+ 0+ } else {+ digits[0]+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_approx_results_assuming_term_independence() {+ let term_freqs = vec![40, 25];+ let num_docs = 100;++ let approx_results = approx_results_assuming_term_independence(&term_freqs, num_docs);++ assert_eq!(approx_results, 10);+ }+}diff --git a/crates/core/src/collector/mod.rs b/crates/core/src/collector/mod.rsnew file mode 100644index 00000000..31412558--- /dev/null+++ b/crates/core/src/collector/mod.rs@@ -0,0 +1,52 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{prehashed::Prehashed, ranking::initial::InitialScoreTweaker, simhash};++pub mod approx_count;+mod top_docs;++pub use top_docs::{BucketCollector, TopDocs};+pub type MainCollector = top_docs::TweakedScoreTopCollector<InitialScoreTweaker>;++#[derive(Clone, Debug)]+pub struct MaxDocsConsidered {+ pub total_docs: usize,+ pub segments: usize,+}++#[derive(+ Clone,+ Copy,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+)]+pub struct Hashes {+ pub site: Prehashed,+ pub title: Prehashed,+ pub url: Prehashed,+ pub url_without_tld: Prehashed,+ pub simhash: simhash::HashType,+}++pub trait Doc: Clone {+ fn score(&self) -> f64;+ fn hashes(&self) -> Hashes;+}diff --git a/crates/core/src/collector.rs b/crates/core/src/collector/top_docs.rssimilarity index 96%rename from crates/core/src/collector.rsrename to crates/core/src/collector/top_docs.rsindex 9e87d440..0bbe6595 100644--- a/crates/core/src/collector.rs+++ b/crates/core/src/collector/top_docs.rs@@ -12,7 +12,7 @@// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.+// along with this program. If not, see <https://www.gnu.org/licenses/>use std::{collections::HashMap, sync::Arc};@@ -34,36 +34,7 @@ use crate::{simhash,};-pub type MainCollector = TweakedScoreTopCollector<InitialScoreTweaker>;--#[derive(Clone, Debug)]-pub struct MaxDocsConsidered {- pub total_docs: usize,- pub segments: usize,-}--#[derive(- Clone,- Copy,- Debug,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialEq,-)]-pub struct Hashes {- pub site: Prehashed,- pub title: Prehashed,- pub url: Prehashed,- pub url_without_tld: Prehashed,- pub simhash: simhash::HashType,-}--pub trait Doc: Clone {- fn score(&self) -> f64;- fn hashes(&self) -> Hashes;-}+use super::{Doc, Hashes, MainCollector, MaxDocsConsidered};pub struct TopDocs {top_n: usize,@@ -111,7 +82,7 @@ impl TopDocs {}pub fn main_collector(self, score_tweaker: InitialScoreTweaker) -> MainCollector {- TweakedScoreTopCollector::new(score_tweaker, self)+ MainCollector::new(score_tweaker, self)}}diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex ed4d9824..e690be5e 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -141,7 +141,7 @@ impl SearchQuery {false}- pub fn count_results() -> bool {+ pub fn count_results_exact() -> bool {false}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex c4c39a97..0ebb93de 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -39,7 +39,7 @@ use tantivy::schema::{Schema, Value};use tantivy::tokenizer::TokenizerManager;use tantivy::{IndexReader, IndexWriter, TantivyDocument};-use crate::collector::Hashes;+use crate::collector::{approx_count, Hashes};use crate::config::SnippetConfig;use crate::fastfield_reader::FastFieldReader;@@ -62,7 +62,7 @@ use std::sync::Arc;#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct InitialSearchResult {- pub num_websites: Option<usize>,+ pub num_websites: approx_count::Count,pub top_websites: Vec<WebpagePointer>,}@@ -214,7 +214,7 @@ impl InvertedIndex {#[derive(Debug, serde::Serialize, bincode::Encode)]pub struct SearchResult {- pub num_docs: Option<usize>,+ pub num_docs: approx_count::Count,pub documents: Vec<RetrievedWebpage>,}@@ -1264,4 +1264,62 @@ mod tests {assert!(ranking_websites[0].title_embedding().is_some());assert!(ranking_websites[1].title_embedding().is_none());}++ #[test]+ fn test_approximate_count() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ let webpage = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} test+ </body>+ </html>+ "#,+ CONTENT = crate::rand_words(100)+ ),+ "https://www.a.com",+ )+ .unwrap();++ for _ in 0..1_000 {+ index.insert(&webpage).unwrap();+ }++ index.commit().expect("failed to commit index");++ let ctx = index.local_search_ctx();++ let query = Query::parse(+ &ctx,+ &SearchQuery {+ query: "test".to_string(),+ ..Default::default()+ },+ &index,+ )+ .expect("Failed to parse query");++ let collector_config = CollectorConfig {+ max_docs_considered: 100,+ ..Default::default()+ };++ let ranker = Ranker::new(+ SignalComputer::new(Some(&query)),+ ctx.fastfield_reader.clone(),+ collector_config,+ );++ let res = index+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))+ .unwrap();++ assert_eq!(res.num_websites, approx_count::Count::Approximate(1_000));+ }}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 5d3cbb62..5d5cc1b8 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -23,7 +23,8 @@ use tantivy::schema::Value;use tantivy::TantivyDocument;use url::Url;-use crate::collector::MainCollector;+use crate::collector::approx_count::ApproxCount;+use crate::collector::{approx_count, MainCollector};use crate::fastfield_reader::FastFieldReader;use crate::highlighted::HighlightedFragment;@@ -47,29 +48,40 @@ impl InvertedIndex {ctx: &Ctx,collector: MainCollector,) -> Result<InitialSearchResult> {- if !query.count_results() {- let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());-- if let Some(limit) = collector.top_docs().max_docs() {- let docs_per_segment = limit.total_docs / limit.segments;- query = Box::new(ShortCircuitQuery::new(query, docs_per_segment as u64));- }-- let pointers = ctx.tv_searcher.search(&query, &collector)?;+ if query.count_results_exact() {+ let collector = (Count, collector);+ let (count, pointers) = ctx.tv_searcher.search(query, &collector)?;return Ok(InitialSearchResult {- num_websites: None,+ num_websites: approx_count::Count::Exact(count as u64),top_websites: pointers,});}- let collector = (Count, collector);- let (count, pointers) = ctx.tv_searcher.search(query, &collector)?;+ let simple_terms = query.simple_terms().to_vec();+ let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());++ if let Some(limit) = collector.top_docs().max_docs().cloned() {+ let docs_per_segment = (limit.total_docs / limit.segments) as u64;+ query = Box::new(ShortCircuitQuery::new(query, docs_per_segment));++ let (count, pointers) = ctx.tv_searcher.search(+ &query,+ &(ApproxCount::new(docs_per_segment, simple_terms), collector),+ )?;- Ok(InitialSearchResult {- num_websites: Some(count),- top_websites: pointers,- })+ Ok(InitialSearchResult {+ num_websites: count,+ top_websites: pointers,+ })+ } else {+ let (count, pointers) = ctx.tv_searcher.search(&query, &(Count, collector))?;++ Ok(InitialSearchResult {+ num_websites: approx_count::Count::Approximate(count as u64),+ top_websites: pointers,+ })+ }}pub fn local_search_ctx(&self) -> Ctx {diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 992f40b7..ce2c8abb 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -52,7 +52,7 @@ pub struct Query {region: Option<Region>,optics: Vec<Optic>,top_n: usize,- count_results: bool,+ count_results_exact: bool,}impl Clone for Query {@@ -65,7 +65,7 @@ impl Clone for Query {region: self.region,optics: self.optics.clone(),top_n: self.top_n,- count_results: self.count_results,+ count_results_exact: self.count_results_exact,}}}@@ -142,12 +142,12 @@ impl Query {offset: query.num_results * query.page,region: query.selected_region,top_n: query.num_results,- count_results: query.count_results,+ count_results_exact: query.count_results_exact,})}- pub fn count_results(&self) -> bool {- self.count_results+ pub fn count_results_exact(&self) -> bool {+ self.count_results_exact}pub fn simple_terms(&self) -> &[String] {diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 77a6c000..a0c37032 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -27,7 +27,7 @@ use url::Url;use ahash::AHashMap as HashMap;use crate::bangs::{Bang, BangHit};-use crate::collector::{self, Doc};+use crate::collector::{self, approx_count, Doc};use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};use crate::enum_map::EnumMap;use crate::image_store::Image;@@ -599,7 +599,7 @@ wherelet num_docs = initial_results.iter().map(|result| result.local_result.num_websites)- .sum();+ .fold(approx_count::Count::Exact(0), |acc, count| acc + count);let (top_websites, has_more_results) = self.combine_results(diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 99934c28..e3ce10f8 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -20,6 +20,7 @@ use std::sync::{Arc, RwLockReadGuard};use itertools::Itertools;use url::Url;+use crate::collector::approx_count;use crate::config::{CollectorConfig, SnippetConfig};use crate::index::Index;use crate::inverted_index::{InvertedIndex, RetrievedWebpage};@@ -119,7 +120,7 @@ wherestruct InvertedIndexResult {webpages: Vec<LocalRecallRankingWebpage>,- num_hits: Option<usize>,+ num_hits: approx_count::Count,has_more: bool,}diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex 47b96c51..76622f01 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -28,6 +28,7 @@ use utoipa::ToSchema;use crate::{api::search::ReturnBody,bangs::BangHit,+ collector::approx_count::Count,config::defaults,ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficient},search_prettifier::DisplayedWebpage,@@ -59,7 +60,7 @@ impl SearchResult {#[serde(rename_all = "camelCase")]pub struct WebsitesResult {pub webpages: Vec<DisplayedWebpage>,- pub num_hits: Option<usize>,+ pub num_hits: Count,pub search_duration_ms: u128,pub has_more_results: bool,}@@ -74,14 +75,14 @@ pub struct SearchQuery {pub host_rankings: Option<HostRankings>,pub return_ranking_signals: bool,pub safe_search: bool,- pub count_results: bool,+ pub count_results_exact: bool,pub return_body: Option<ReturnBody>,pub return_structured_data: bool,}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct InitialWebsiteResult {- pub num_websites: Option<usize>,+ pub num_websites: Count,pub websites: Vec<LocalRecallRankingWebpage>,pub has_more: bool,}@@ -100,7 +101,7 @@ impl Default for SearchQuery {host_rankings: Default::default(),return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),safe_search: defaults::SearchQuery::safe_search(),- count_results: defaults::SearchQuery::count_results(),+ count_results_exact: defaults::SearchQuery::count_results_exact(),return_body: None,return_structured_data: defaults::SearchQuery::return_structured_data(),}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex c37e22fe..f227ef46 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -189,7 +189,7 @@ export const api = {};export type ApiSearchQuery = {- countResults?: boolean;+ countResultsExact?: boolean;flattenResponse?: boolean;hostRankings?: HostRankings;numResults?: number;@@ -235,6 +235,15 @@ export type CodeOrText =_type: 'text';value: string;};+export type Count =+ | {+ _type: 'exact';+ value: number;+ }+ | {+ _type: 'approximate';+ value: number;+ };export type Definition = string;export type DisplayedAnswer = {answer: string;@@ -411,7 +420,7 @@ export type ThesaurusWidget = {export type UrlWrapper = string;export type WebsitesResult = {hasMoreResults: boolean;- numHits?: number;+ numHits: Count;searchDurationMs: number;webpages: DisplayedWebpage[];};diff --git a/frontend/src/lib/search.ts b/frontend/src/lib/search.tsindex 4ad2ba79..4519c0f2 100644--- a/frontend/src/lib/search.ts+++ b/frontend/src/lib/search.ts@@ -106,7 +106,6 @@ export const search = async (params: SearchParams, options: ApiOptions) => {optic: params.optic && (await fetchRemoteOptic({ opticUrl: params.optic, fetch })),selectedRegion: params.selectedRegion,hostRankings: params.host_rankings,- countResults: true,},options,);@@ -141,7 +140,6 @@ export const search = async (params: SearchParams, options: ApiOptions) => {safeSearch: params.safeSearch,selectedRegion: params.selectedRegion,hostRankings: params.host_rankings,- countResults: false,},options,)diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex 828feb03..6b5f438c 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -12,6 +12,8 @@import { search } from '$lib/search';import { Keybind, searchCb, type Refs } from '$lib/keybind';import SpellCorrection from './SpellCorrection.svelte';+ import type { Count } from '$lib/api';+ import { match } from 'ts-pattern';export let data: PageData;$: results = data.results;@@ -101,6 +103,17 @@keybind.onKeyDown(event, $useKeyboardShortcuts, context);}};++ const prettyprintCount = (count: Count): string => {+ return match(count)+ .with({ _type: 'exact' }, () => {+ return count.value.toLocaleString();+ })+ .with({ _type: 'approximate' }, () => {+ return `~${count.value.toLocaleString()}`;+ })+ .exhaustive();+ };</script><svelte:window on:keydown={onKeyDown} />@@ -127,13 +140,9 @@<div class="mx-auto flex w-full justify-end sm:justify-between"><div class="hidden h-full flex-col space-x-2 text-xs text-neutral sm:flex"><p class="h-fit">- {#if results && results.numHits != null}- Found <span class="font-medium">{results.numHits.toLocaleString()}</span> results in+ {#if results}+ Found <span class="font-medium">{prettyprintCount(results.numHits)}</span> results in<span class="font-medium">{((results.searchDurationMs ?? 0) / 1000).toFixed(2)}s</span>- {:else if results}- Search took <span class="font-medium"- >{((results.searchDurationMs ?? 0) / 1000).toFixed(2)}s</span- >{/if}</p></div>