Skip to content

web search done right

approximate number of hits by assuming that each term is independent this allows us to short-cirquit the query by default which significantly improves performance as we therefore don't have to iterate the non-scored results simply to count them

Mikkel Denker3c94cb7f8145
diff --git a/Cargo.lock b/Cargo.lock
index 76638ecc..3fb4dd83 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2919,6 +2919,17 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "num-bigint"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "num-complex"
version = "0.4.5"
@@ -2935,6 +2946,27 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
+dependencies = [
+ "autocfg",
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "num-traits"
version = "0.2.18"
@@ -4623,6 +4655,8 @@ dependencies = [
"mime",
"min-max-heap",
"nom",
+ "num-bigint",
+ "num-rational",
"num_cpus",
"once_cell",
"openraft",
diff --git a/Cargo.toml b/Cargo.toml
index e5001b64..a2fd4326 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,14 +1,14 @@
[workspace]
members = [
- "crates/bloom",
- "crates/client-wasm",
- "crates/client-wasm",
- "crates/core",
- "crates/file-store",
- "crates/kuchiki",
- "crates/optics",
- "crates/speedy-kv",
- "crates/zimba",
+ "crates/bloom",
+ "crates/client-wasm",
+ "crates/client-wasm",
+ "crates/core",
+ "crates/file-store",
+ "crates/kuchiki",
+ "crates/optics",
+ "crates/speedy-kv",
+ "crates/zimba",
]
resolver = "2"
@@ -20,61 +20,56 @@ lto = "thin"
debug = true
[workspace.dependencies]
-ahash = "0.8.11"
aes-gcm = "0.10.2"
-anyhow = { version = "1.0.72", features = ["backtrace"] }
+ahash = "0.8.11"
+anyhow = {version = "1.0.72", features = ["backtrace"]}
async-channel = "1.8.0"
async-stream = "0.3.3"
axum = "0.7.2"
-axum-extra = { version = "0.9.0" }
+axum-extra = {version = "0.9.0"}
axum-macros = "0.4.0"
base64 = "0.21.4"
-bincode = { git = "https://github.com/bincode-org/bincode", rev = "8f65c60", features = [
- "serde",
-] }
+bincode = {git = "https://github.com/bincode-org/bincode", rev = "8f65c60", features = [
+ "serde",
+]}
bindgen = "0.69.2"
-bitvec = { version = "1.0.1", features = ["serde"] }
+bitvec = {version = "1.0.1", features = ["serde"]}
bytecount = "0.6.7"
-bytemuck = { version = "1.13.1", features = ["derive"] }
+bytemuck = {version = "1.13.1", features = ["derive"]}
byteorder = "1.4.3"
bzip2 = "0.4.4"
candle-core = "0.3.3"
candle-nn = "0.3.3"
-openraft = { version = "0.9.1", features = [
- "storage-v2",
- "serde",
- "tracing-log",
-] }
candle-transformers = "0.3.3"
-cc = { version = "1", features = ["parallel"] }
+cc = {version = "1", features = ["parallel"]}
chardetng = "0.1.17"
chitchat = "0.5.0"
-chrono = { version = "0.4.23", features = ["serde"] }
-clap = { version = "4.4.6", features = ["derive"] }
+chrono = {version = "0.4.23", features = ["serde"]}
+clap = {version = "4.4.6", features = ["derive"]}
cmake = "0.1"
criterion = "0.5.1"
crossbeam-channel = "0.5.6"
csv = "1.1.6"
-dashmap = { version = "5.4.0", features = ["rayon"] }
+dashmap = {version = "5.4.0", features = ["rayon"]}
encoding_rs = "0.8.31"
enum_dispatch = "0.3.12"
eventsource-stream = "0.2.3"
fend-core = "1.4.8"
flate2 = "1.0.28"
fnv = "1.0.3"
-fst = { version = "0.4.7", features = ["levenshtein"] }
+fst = {version = "0.4.7", features = ["levenshtein"]}
futures = "0.3.21"
-half = { version = "2.2.1", features = ["serde"] }
-hashbrown = { version = "0.14.0", features = ["serde"] }
+half = {version = "2.2.1", features = ["serde"]}
+hashbrown = {version = "0.14.0", features = ["serde"]}
http = "1.0.0"
image = "0.24.3"
-indicatif = { version = "0.17.7", features = ["rayon"] }
+indicatif = {version = "0.17.7", features = ["rayon"]}
insta = "1.31"
itertools = "0.11.0"
-lalrpop = { version = "0.20.0", features = ["lexer"] }
-lalrpop-util = { version = "0.20.0", features = ["lexer"] }
+lalrpop = {version = "0.20.0", features = ["lexer"]}
+lalrpop-util = {version = "0.20.0", features = ["lexer"]}
libc = "0.2.142"
-log = { version = "0.4", features = ["release_max_level_info"] }
+log = {version = "0.4", features = ["release_max_level_info"]}
logos = "0.13.0"
lz-str = "0.2.1"
lz4_flex = "0.11.1"
@@ -85,9 +80,16 @@ memmap2 = "0.9.0"
mime = "0.3.17"
min-max-heap = "1.3.0"
nom = "7.1.3"
+num-bigint = "0.4.1"
+num-rational = "0.4.1"
num_cpus = "1.15.0"
once_cell = "1.13.1"
-postcard = { version = "1.0.8", features = ["experimental-derive", "alloc"] }
+openraft = {version = "0.9.1", features = [
+ "storage-v2",
+ "serde",
+ "tracing-log",
+]}
+postcard = {version = "1.0.8", features = ["experimental-derive", "alloc"]}
proptest = "1.2.0"
proptest-derive = "0.4.0"
publicsuffix = "2.2.3"
@@ -96,46 +98,46 @@ rand = "0.8.5"
rayon = "1.5.3"
redb = "2.0.0"
regex = "1.6.0"
-reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
-rkyv = { version = "0.7.44", features = ["validation", "bytecheck"] }
+reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}
ring = "0.17.3"
rio_api = "0.8.4"
rio_turtle = "0.8.4"
+rkyv = {version = "0.7.44", features = ["validation", "bytecheck"]}
robotstxt-with-cache = "0.4.0"
-rusqlite = { version = "0.29.0", features = [
- "bundled",
- "modern-full",
- "buildtime_bindgen",
-] }
-rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
+rusqlite = {version = "0.29.0", features = [
+ "bundled",
+ "modern-full",
+ "buildtime_bindgen",
+]}
+rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}
rust-stemmers = "1.2.0"
safetensors = "0.3.1"
-scylla = { version = "0.12.0", features = ["chrono"] }
-serde = { version = "1.0.137", features = ["rc", "derive"] }
+scylla = {version = "0.12.0", features = ["chrono"]}
+serde = {version = "1.0.137", features = ["rc", "derive"]}
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
stable_deref_trait = "1.2.0"
-strum = { version = "0.26.2", features = ["derive"] }
-tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "74940e9" }
+strum = {version = "0.26.2", features = ["derive"]}
+tantivy = {git = "https://github.com/quickwit-oss/tantivy", rev = "74940e9"}
thiserror = "1.0.31"
tikv-jemallocator = "0.5"
tokenizers = "0.13.2"
-tokio = { version = "1.23.1", features = ["full"] }
+tokio = {version = "1.23.1", features = ["full"]}
tokio-stream = "0.1.11"
toml = "0.8.2"
-tower = { version = "0.4.13", features = ["limit"] }
-tower-http = { version = "0.5.0", features = ["compression-gzip", "cors"] }
-tracing = { version = "0.1.34", features = ["release_max_level_info"] }
-tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
-url = { version = "2.4.0", features = ["serde"] }
+tower = {version = "0.4.13", features = ["limit"]}
+tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}
+tracing = {version = "0.1.34", features = ["release_max_level_info"]}
+tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}
+tracing-test = "0.2.4"
+url = {version = "2.4.0", features = ["serde"]}
urlencoding = "2.1.2"
-utoipa = { version = "4.0.0", features = ["axum_extras"] }
-utoipa-swagger-ui = { version = "5.0.0", features = ["axum"] }
+utoipa = {version = "4.0.0", features = ["axum_extras"]}
+utoipa-swagger-ui = {version = "5.0.0", features = ["axum"]}
uuid = "1.1.2"
-whatlang = { version = "0.16.0", features = ["serde"] }
+whatlang = {version = "0.16.0", features = ["serde"]}
+xxhash-rust = {version = "0.8.10", features = ["xxh3", "const_xxh3"]}
zstd = "0.13"
-tracing-test = "0.2.4"
-xxhash-rust = { version = "0.8.10", features = ["xxh3", "const_xxh3"] }
[profile.test.package]
flate2.opt-level = 3
diff --git a/assets/licenses.html b/assets/licenses.html
index 666ebc23..b0c12ad8 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,7 +44,7 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (388)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (391)</li>
<li><a href="#MIT">MIT License</a> (192)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (8)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (7)</li>
@@ -7524,7 +7524,10 @@ limitations under the License.</pre>
<li><a href=" https://github.com/servo/html5ever ">markup5ever 0.11.0</a></li>
<li><a href=" https://github.com/hyperium/mime ">mime 0.3.17</a></li>
<li><a href=" https://github.com/tov/min-max-heap-rs ">min-max-heap 1.3.0</a></li>
+ <li><a href=" https://github.com/rust-num/num-bigint ">num-bigint 0.4.4</a></li>
<li><a href=" https://github.com/rust-num/num-complex ">num-complex 0.4.5</a></li>
+ <li><a href=" https://github.com/rust-num/num-integer ">num-integer 0.1.46</a></li>
+ <li><a href=" https://github.com/rust-num/num-rational ">num-rational 0.4.1</a></li>
<li><a href=" https://github.com/rust-num/num-traits ">num-traits 0.2.18</a></li>
<li><a href=" https://github.com/seanmonstar/num_cpus ">num_cpus 1.16.0</a></li>
<li><a href=" https://github.com/gimli-rs/object ">object 0.32.2</a></li>
diff --git a/configs/search_server.toml b/configs/search_server.toml
index 51cfa902..0453c739 100644
--- a/configs/search_server.toml
+++ b/configs/search_server.toml
@@ -1,14 +1,17 @@
cluster_id = "dev_search"
+dual_encoder_model_path = "data/summarizer/dual_encoder"
entity_index_path = "data/entity"
gossip_addr = "0.0.0.0:3006"
gossip_seed_nodes = ["0.0.0.0:3005"]
host = "0.0.0.0:3002"
index_path = "data/index"
shard = 0
-dual_encoder_model_path = "data/summarizer/dual_encoder"
# linear_model_path = "data/linear_model.json"
# lambda_model_path = "data/lambdamart.txt"
[snippet]
max_considered_words = 10_000
num_words_for_lang_detection = 1_000
+
+[collector]
+max_docs_considered = 1000
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 215c2bae..2e6e07bf 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -20,113 +20,115 @@ name = "stract"
path = "src/main.rs"
[dependencies]
-ahash = { workspace = true }
-aes-gcm = { workspace = true }
-anyhow = { workspace = true }
-async-channel = { workspace = true }
-async-stream = { workspace = true }
-axum = { workspace = true }
-axum-extra = { workspace = true }
-axum-macros = { workspace = true }
-base64 = { workspace = true }
-bincode = { workspace = true }
-bitvec = { workspace = true }
-bloom = { path = "../bloom" }
-bytecount = { workspace = true }
-bytemuck = { workspace = true }
-byteorder = { workspace = true }
-bzip2 = { workspace = true }
-candle-core = { workspace = true }
-candle-nn = { workspace = true }
-candle-transformers = { workspace = true }
-chardetng = { workspace = true }
-chitchat = { workspace = true }
-chrono = { workspace = true }
-clap = { workspace = true }
-crossbeam-channel = { workspace = true }
-csv = { workspace = true }
-dashmap = { workspace = true }
-encoding_rs = { workspace = true }
-enum_dispatch = { workspace = true }
-eventsource-stream = { workspace = true }
-fend-core = { workspace = true }
-file_store = { path = "../file-store" }
-flate2 = { workspace = true }
-fnv = { workspace = true }
-fst = { workspace = true }
-futures = { workspace = true }
-half = { workspace = true }
-hashbrown = { workspace = true }
-http = { workspace = true }
-image = { workspace = true }
-indicatif = { workspace = true }
-itertools = { workspace = true }
-kuchiki = { path = "../kuchiki" }
-libc = { workspace = true }
-log = { workspace = true }
-logos = { workspace = true }
-lz-str = { workspace = true }
-lz4_flex = { workspace = true }
-md5 = { workspace = true }
-memmap2 = { workspace = true }
-mime = { workspace = true }
-min-max-heap = { workspace = true }
-nom = { workspace = true }
-num_cpus = { workspace = true }
-once_cell = { workspace = true }
-openraft = { workspace = true }
-optics = { path = "../optics" }
-publicsuffix = { workspace = true }
-postcard = { workspace = true }
-quick-xml = { workspace = true }
-rand = { workspace = true }
-rayon = { workspace = true }
-redb = { workspace = true }
-regex = { workspace = true }
-reqwest = { workspace = true }
-rkyv = { workspace = true }
-ring = { workspace = true }
-rio_api = { workspace = true }
-rio_turtle = { workspace = true }
-robotstxt-with-cache = { workspace = true }
-rust-s3 = { workspace = true }
-rust-stemmers = { workspace = true }
-safetensors = { workspace = true }
-scylla = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
-serde_urlencoded = { workspace = true }
-speedy_kv = { path = "../speedy-kv" }
-strum = { workspace = true }
-tantivy = { workspace = true }
-thiserror = { workspace = true }
-tokenizers = { workspace = true }
-tokio = { workspace = true }
-tokio-stream = { workspace = true }
-toml = { workspace = true }
-tower = { workspace = true }
-tower-http = { workspace = true }
-tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
-url = { workspace = true }
-urlencoding = { workspace = true }
-utoipa = { workspace = true }
-utoipa-swagger-ui = { workspace = true }
-uuid = { workspace = true }
-whatlang = { workspace = true }
-zimba = { path = "../zimba" }
-xxhash-rust = { workspace = true }
+aes-gcm = {workspace = true}
+ahash = {workspace = true}
+anyhow = {workspace = true}
+async-channel = {workspace = true}
+async-stream = {workspace = true}
+axum = {workspace = true}
+axum-extra = {workspace = true}
+axum-macros = {workspace = true}
+base64 = {workspace = true}
+bincode = {workspace = true}
+bitvec = {workspace = true}
+bloom = {path = "../bloom"}
+bytecount = {workspace = true}
+bytemuck = {workspace = true}
+byteorder = {workspace = true}
+bzip2 = {workspace = true}
+candle-core = {workspace = true}
+candle-nn = {workspace = true}
+candle-transformers = {workspace = true}
+chardetng = {workspace = true}
+chitchat = {workspace = true}
+chrono = {workspace = true}
+clap = {workspace = true}
+crossbeam-channel = {workspace = true}
+csv = {workspace = true}
+dashmap = {workspace = true}
+encoding_rs = {workspace = true}
+enum_dispatch = {workspace = true}
+eventsource-stream = {workspace = true}
+fend-core = {workspace = true}
+file_store = {path = "../file-store"}
+flate2 = {workspace = true}
+fnv = {workspace = true}
+fst = {workspace = true}
+futures = {workspace = true}
+half = {workspace = true}
+hashbrown = {workspace = true}
+http = {workspace = true}
+image = {workspace = true}
+indicatif = {workspace = true}
+itertools = {workspace = true}
+kuchiki = {path = "../kuchiki"}
+libc = {workspace = true}
+log = {workspace = true}
+logos = {workspace = true}
+lz-str = {workspace = true}
+lz4_flex = {workspace = true}
+md5 = {workspace = true}
+memmap2 = {workspace = true}
+mime = {workspace = true}
+min-max-heap = {workspace = true}
+nom = {workspace = true}
+num-bigint = {workspace = true}
+num-rational = {workspace = true}
+num_cpus = {workspace = true}
+once_cell = {workspace = true}
+openraft = {workspace = true}
+optics = {path = "../optics"}
+postcard = {workspace = true}
+publicsuffix = {workspace = true}
+quick-xml = {workspace = true}
+rand = {workspace = true}
+rayon = {workspace = true}
+redb = {workspace = true}
+regex = {workspace = true}
+reqwest = {workspace = true}
+ring = {workspace = true}
+rio_api = {workspace = true}
+rio_turtle = {workspace = true}
+rkyv = {workspace = true}
+robotstxt-with-cache = {workspace = true}
+rust-s3 = {workspace = true}
+rust-stemmers = {workspace = true}
+safetensors = {workspace = true}
+scylla = {workspace = true}
+serde = {workspace = true}
+serde_json = {workspace = true}
+serde_urlencoded = {workspace = true}
+speedy_kv = {path = "../speedy-kv"}
+strum = {workspace = true}
+tantivy = {workspace = true}
+thiserror = {workspace = true}
+tokenizers = {workspace = true}
+tokio = {workspace = true}
+tokio-stream = {workspace = true}
+toml = {workspace = true}
+tower = {workspace = true}
+tower-http = {workspace = true}
+tracing = {workspace = true}
+tracing-subscriber = {workspace = true}
+url = {workspace = true}
+urlencoding = {workspace = true}
+utoipa = {workspace = true}
+utoipa-swagger-ui = {workspace = true}
+uuid = {workspace = true}
+whatlang = {workspace = true}
+xxhash-rust = {workspace = true}
+zimba = {path = "../zimba"}
[target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = { workspace = true }
+tikv-jemallocator = {workspace = true}
[dev-dependencies]
-criterion = { workspace = true }
-insta = { workspace = true }
-maplit = { workspace = true }
-proptest = { workspace = true }
-proptest-derive = { workspace = true }
-tracing-test = { workspace = true }
+criterion = {workspace = true}
+insta = {workspace = true}
+maplit = {workspace = true}
+proptest = {workspace = true}
+proptest-derive = {workspace = true}
+tracing-test = {workspace = true}
[[bench]]
harness = false
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index f8ebd65a..f2dc87d5 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -98,6 +98,8 @@ use utoipa_swagger_ui::SwaggerUi;
crate::search_prettifier::OneOrManyString,
crate::search_prettifier::OneOrManyProperty,
crate::search_prettifier::Property,
+
+ crate::collector::approx_count::Count,
),
),
modifiers(&ApiModifier),
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rs
index 8a2b1180..1a4d20b1 100644
--- a/crates/core/src/api/search.rs
+++ b/crates/core/src/api/search.rs
@@ -69,8 +69,8 @@ pub struct ApiSearchQuery {
#[serde(default = "defaults::SearchQuery::flatten_response")]
pub flatten_response: bool,
- #[serde(default = "defaults::SearchQuery::count_results")]
- pub count_results: bool,
+ #[serde(default = "defaults::SearchQuery::count_results_exact")]
+ pub count_results_exact: bool,
#[serde(default = "defaults::SearchQuery::return_structured_data")]
pub return_structured_data: bool,
@@ -99,7 +99,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
host_rankings: api.host_rankings,
return_ranking_signals: api.return_ranking_signals,
safe_search: api.safe_search.unwrap_or(default.safe_search),
- count_results: api.count_results,
+ count_results_exact: api.count_results_exact,
return_body: api.return_body,
return_structured_data: api.return_structured_data,
})
diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rs
new file mode 100644
index 00000000..cb67f11e
--- /dev/null
+++ b/crates/core/src/collector/approx_count.rs
@@ -0,0 +1,218 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::ops::Add;
+
+use itertools::Itertools;
+use tantivy::{
+ collector::{Collector, SegmentCollector},
+ tokenizer::Tokenizer,
+};
+
+use crate::schema::text_field::{self, TextField};
+
+#[derive(
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ utoipa::ToSchema,
+)]
+#[serde(rename_all = "camelCase", tag = "_type", content = "value")]
+pub enum Count {
+ Exact(u64),
+ Approximate(u64),
+}
+
+impl Count {
+ pub fn is_exact(&self) -> bool {
+ match self {
+ Count::Exact(_) => true,
+ Count::Approximate(_) => false,
+ }
+ }
+
+ pub fn is_approximate(&self) -> bool {
+ match self {
+ Count::Exact(_) => false,
+ Count::Approximate(_) => true,
+ }
+ }
+
+ pub fn compose(&self, other: &Self) -> Self {
+ match (self, other) {
+ (Count::Exact(a), Count::Exact(b)) => Count::Exact(a + b),
+ (Count::Exact(a), Count::Approximate(b))
+ | (Count::Approximate(a), Count::Exact(b))
+ | (Count::Approximate(a), Count::Approximate(b)) => Count::Approximate(a + b),
+ }
+ }
+}
+
+impl Add for Count {
+ type Output = Self;
+
+ fn add(self, other: Self) -> Self {
+ self.compose(&other)
+ }
+}
+
+pub struct ApproxCount {
+ max_docs_per_segment: u64,
+ terms: Vec<String>,
+}
+
+impl ApproxCount {
+ pub fn new(max_docs_per_segment: u64, terms: Vec<String>) -> Self {
+ Self {
+ max_docs_per_segment,
+ terms: terms
+ .into_iter()
+ .map(|term| term.to_lowercase())
+ .sorted()
+ .dedup()
+ .collect(),
+ }
+ }
+}
+
+impl Collector for ApproxCount {
+ type Fruit = Count;
+ type Child = SegmentApproxCount;
+
+ fn for_segment(
+ &self,
+ _: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> tantivy::Result<Self::Child> {
+ let field = text_field::AllBody.tantivy_field(segment.schema()).unwrap();
+ let mut tokenizer = text_field::AllBody.query_tokenizer();
+
+ let inverted_index = segment.inverted_index(field)?;
+ let num_docs = segment.max_doc() as u64;
+ let mut term_freqs = Vec::with_capacity(self.terms.len());
+
+ for term in &self.terms {
+ let mut term_freq = 0;
+ let mut stream = tokenizer.token_stream(term.as_str());
+
+ while let Some(term) = stream.next() {
+ term_freq += inverted_index
+ .doc_freq(&tantivy::Term::from_field_text(field, term.text.as_str()))?;
+ }
+
+ term_freqs.push(term_freq as u64);
+ }
+
+ let approx_count = approx_results_assuming_term_independence(&term_freqs, num_docs);
+
+ Ok(SegmentApproxCount {
+ max_docs_per_segment: self.max_docs_per_segment,
+ approx_count,
+ exact_count: 0,
+ })
+ }
+
+ fn requires_scoring(&self) -> bool {
+ false
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> tantivy::Result<Self::Fruit> {
+ Ok(segment_fruits
+ .into_iter()
+ .fold(Count::Exact(0), |acc, fruit| acc.compose(&fruit)))
+ }
+}
+
+pub struct SegmentApproxCount {
+ max_docs_per_segment: u64,
+ approx_count: u64,
+ exact_count: u64,
+}
+
+impl SegmentCollector for SegmentApproxCount {
+ type Fruit = Count;
+
+ fn collect(&mut self, _doc: u32, _score: f32) {
+ self.exact_count += 1;
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ if self.exact_count >= self.max_docs_per_segment {
+ if self.exact_count > self.approx_count {
+ Count::Approximate(self.exact_count)
+ } else {
+ Count::Approximate(self.approx_count)
+ }
+ } else {
+ Count::Exact(self.exact_count)
+ }
+ }
+}
+
+/// Approximate the number of results for a query assuming each term is independent.
+/// The idea is to estiate the probability that a document contains all terms in the query
+/// as the product of the probabilities that it contains each term.
+///
+/// P(A and B) = P(A) * P(B) if A and B are independent.
+///
+/// returns 0 if term_freqs is empty or num_docs is 0.
+fn approx_results_assuming_term_independence(term_freqs: &[u64], num_docs: u64) -> u64 {
+ if term_freqs.is_empty() || num_docs == 0 {
+ return 0;
+ }
+
+ let mut res = None;
+
+ for term_freq in term_freqs {
+ let prob = num_rational::BigRational::new((*term_freq).into(), num_docs.into());
+
+ res = Some(res.map_or(prob.clone(), move |res| res * prob));
+ }
+
+ let approx_results = res.unwrap() * num_rational::BigRational::new(num_docs.into(), 1.into());
+
+ let (sign, digits) = approx_results.to_integer().to_u64_digits();
+
+ if matches!(sign, num_bigint::Sign::Minus | num_bigint::Sign::NoSign) || digits.is_empty() {
+ 0
+ } else {
+ digits[0]
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_approx_results_assuming_term_independence() {
+ let term_freqs = vec![40, 25];
+ let num_docs = 100;
+
+ let approx_results = approx_results_assuming_term_independence(&term_freqs, num_docs);
+
+ assert_eq!(approx_results, 10);
+ }
+}
diff --git a/crates/core/src/collector/mod.rs b/crates/core/src/collector/mod.rs
new file mode 100644
index 00000000..31412558
--- /dev/null
+++ b/crates/core/src/collector/mod.rs
@@ -0,0 +1,52 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{prehashed::Prehashed, ranking::initial::InitialScoreTweaker, simhash};
+
+pub mod approx_count;
+mod top_docs;
+
+pub use top_docs::{BucketCollector, TopDocs};
+pub type MainCollector = top_docs::TweakedScoreTopCollector<InitialScoreTweaker>;
+
+#[derive(Clone, Debug)]
+pub struct MaxDocsConsidered {
+ pub total_docs: usize,
+ pub segments: usize,
+}
+
+#[derive(
+ Clone,
+ Copy,
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ PartialEq,
+)]
+pub struct Hashes {
+ pub site: Prehashed,
+ pub title: Prehashed,
+ pub url: Prehashed,
+ pub url_without_tld: Prehashed,
+ pub simhash: simhash::HashType,
+}
+
+pub trait Doc: Clone {
+ fn score(&self) -> f64;
+ fn hashes(&self) -> Hashes;
+}
diff --git a/crates/core/src/collector.rs b/crates/core/src/collector/top_docs.rs
similarity index 96%
rename from crates/core/src/collector.rs
rename to crates/core/src/collector/top_docs.rs
index 9e87d440..0bbe6595 100644
--- a/crates/core/src/collector.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -12,7 +12,7 @@
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
+// along with this program. If not, see <https://www.gnu.org/licenses/>
use std::{collections::HashMap, sync::Arc};
@@ -34,36 +34,7 @@ use crate::{
simhash,
};
-pub type MainCollector = TweakedScoreTopCollector<InitialScoreTweaker>;
-
-#[derive(Clone, Debug)]
-pub struct MaxDocsConsidered {
- pub total_docs: usize,
- pub segments: usize,
-}
-
-#[derive(
- Clone,
- Copy,
- Debug,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialEq,
-)]
-pub struct Hashes {
- pub site: Prehashed,
- pub title: Prehashed,
- pub url: Prehashed,
- pub url_without_tld: Prehashed,
- pub simhash: simhash::HashType,
-}
-
-pub trait Doc: Clone {
- fn score(&self) -> f64;
- fn hashes(&self) -> Hashes;
-}
+use super::{Doc, Hashes, MainCollector, MaxDocsConsidered};
pub struct TopDocs {
top_n: usize,
@@ -111,7 +82,7 @@ impl TopDocs {
}
pub fn main_collector(self, score_tweaker: InitialScoreTweaker) -> MainCollector {
- TweakedScoreTopCollector::new(score_tweaker, self)
+ MainCollector::new(score_tweaker, self)
}
}
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index ed4d9824..e690be5e 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -141,7 +141,7 @@ impl SearchQuery {
false
}
- pub fn count_results() -> bool {
+ pub fn count_results_exact() -> bool {
false
}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index c4c39a97..0ebb93de 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -39,7 +39,7 @@ use tantivy::schema::{Schema, Value};
use tantivy::tokenizer::TokenizerManager;
use tantivy::{IndexReader, IndexWriter, TantivyDocument};
-use crate::collector::Hashes;
+use crate::collector::{approx_count, Hashes};
use crate::config::SnippetConfig;
use crate::fastfield_reader::FastFieldReader;
@@ -62,7 +62,7 @@ use std::sync::Arc;
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct InitialSearchResult {
- pub num_websites: Option<usize>,
+ pub num_websites: approx_count::Count,
pub top_websites: Vec<WebpagePointer>,
}
@@ -214,7 +214,7 @@ impl InvertedIndex {
#[derive(Debug, serde::Serialize, bincode::Encode)]
pub struct SearchResult {
- pub num_docs: Option<usize>,
+ pub num_docs: approx_count::Count,
pub documents: Vec<RetrievedWebpage>,
}
@@ -1264,4 +1264,62 @@ mod tests {
assert!(ranking_websites[0].title_embedding().is_some());
assert!(ranking_websites[1].title_embedding().is_none());
}
+
+ #[test]
+ fn test_approximate_count() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ let webpage = Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} test
+ </body>
+ </html>
+ "#,
+ CONTENT = crate::rand_words(100)
+ ),
+ "https://www.a.com",
+ )
+ .unwrap();
+
+ for _ in 0..1_000 {
+ index.insert(&webpage).unwrap();
+ }
+
+ index.commit().expect("failed to commit index");
+
+ let ctx = index.local_search_ctx();
+
+ let query = Query::parse(
+ &ctx,
+ &SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ },
+ &index,
+ )
+ .expect("Failed to parse query");
+
+ let collector_config = CollectorConfig {
+ max_docs_considered: 100,
+ ..Default::default()
+ };
+
+ let ranker = Ranker::new(
+ SignalComputer::new(Some(&query)),
+ ctx.fastfield_reader.clone(),
+ collector_config,
+ );
+
+ let res = index
+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))
+ .unwrap();
+
+ assert_eq!(res.num_websites, approx_count::Count::Approximate(1_000));
+ }
}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 5d3cbb62..5d5cc1b8 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -23,7 +23,8 @@ use tantivy::schema::Value;
use tantivy::TantivyDocument;
use url::Url;
-use crate::collector::MainCollector;
+use crate::collector::approx_count::ApproxCount;
+use crate::collector::{approx_count, MainCollector};
use crate::fastfield_reader::FastFieldReader;
use crate::highlighted::HighlightedFragment;
@@ -47,29 +48,40 @@ impl InvertedIndex {
ctx: &Ctx,
collector: MainCollector,
) -> Result<InitialSearchResult> {
- if !query.count_results() {
- let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());
-
- if let Some(limit) = collector.top_docs().max_docs() {
- let docs_per_segment = limit.total_docs / limit.segments;
- query = Box::new(ShortCircuitQuery::new(query, docs_per_segment as u64));
- }
-
- let pointers = ctx.tv_searcher.search(&query, &collector)?;
+ if query.count_results_exact() {
+ let collector = (Count, collector);
+ let (count, pointers) = ctx.tv_searcher.search(query, &collector)?;
return Ok(InitialSearchResult {
- num_websites: None,
+ num_websites: approx_count::Count::Exact(count as u64),
top_websites: pointers,
});
}
- let collector = (Count, collector);
- let (count, pointers) = ctx.tv_searcher.search(query, &collector)?;
+ let simple_terms = query.simple_terms().to_vec();
+ let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());
+
+ if let Some(limit) = collector.top_docs().max_docs().cloned() {
+ let docs_per_segment = (limit.total_docs / limit.segments) as u64;
+ query = Box::new(ShortCircuitQuery::new(query, docs_per_segment));
+
+ let (count, pointers) = ctx.tv_searcher.search(
+ &query,
+ &(ApproxCount::new(docs_per_segment, simple_terms), collector),
+ )?;
- Ok(InitialSearchResult {
- num_websites: Some(count),
- top_websites: pointers,
- })
+ Ok(InitialSearchResult {
+ num_websites: count,
+ top_websites: pointers,
+ })
+ } else {
+ let (count, pointers) = ctx.tv_searcher.search(&query, &(Count, collector))?;
+
+ Ok(InitialSearchResult {
+ num_websites: approx_count::Count::Approximate(count as u64),
+ top_websites: pointers,
+ })
+ }
}
pub fn local_search_ctx(&self) -> Ctx {
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index 992f40b7..ce2c8abb 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -52,7 +52,7 @@ pub struct Query {
region: Option<Region>,
optics: Vec<Optic>,
top_n: usize,
- count_results: bool,
+ count_results_exact: bool,
}
impl Clone for Query {
@@ -65,7 +65,7 @@ impl Clone for Query {
region: self.region,
optics: self.optics.clone(),
top_n: self.top_n,
- count_results: self.count_results,
+ count_results_exact: self.count_results_exact,
}
}
}
@@ -142,12 +142,12 @@ impl Query {
offset: query.num_results * query.page,
region: query.selected_region,
top_n: query.num_results,
- count_results: query.count_results,
+ count_results_exact: query.count_results_exact,
})
}
- pub fn count_results(&self) -> bool {
- self.count_results
+ pub fn count_results_exact(&self) -> bool {
+ self.count_results_exact
}
pub fn simple_terms(&self) -> &[String] {
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 77a6c000..a0c37032 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -27,7 +27,7 @@ use url::Url;
use ahash::AHashMap as HashMap;
use crate::bangs::{Bang, BangHit};
-use crate::collector::{self, Doc};
+use crate::collector::{self, approx_count, Doc};
use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};
use crate::enum_map::EnumMap;
use crate::image_store::Image;
@@ -599,7 +599,7 @@ where
let num_docs = initial_results
.iter()
.map(|result| result.local_result.num_websites)
- .sum();
+ .fold(approx_count::Count::Exact(0), |acc, count| acc + count);
let (top_websites, has_more_results) = self
.combine_results(
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index 99934c28..e3ce10f8 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -20,6 +20,7 @@ use std::sync::{Arc, RwLockReadGuard};
use itertools::Itertools;
use url::Url;
+use crate::collector::approx_count;
use crate::config::{CollectorConfig, SnippetConfig};
use crate::index::Index;
use crate::inverted_index::{InvertedIndex, RetrievedWebpage};
@@ -119,7 +120,7 @@ where
struct InvertedIndexResult {
webpages: Vec<LocalRecallRankingWebpage>,
- num_hits: Option<usize>,
+ num_hits: approx_count::Count,
has_more: bool,
}
diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rs
index 47b96c51..76622f01 100644
--- a/crates/core/src/searcher/mod.rs
+++ b/crates/core/src/searcher/mod.rs
@@ -28,6 +28,7 @@ use utoipa::ToSchema;
use crate::{
api::search::ReturnBody,
bangs::BangHit,
+ collector::approx_count::Count,
config::defaults,
ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficient},
search_prettifier::DisplayedWebpage,
@@ -59,7 +60,7 @@ impl SearchResult {
#[serde(rename_all = "camelCase")]
pub struct WebsitesResult {
pub webpages: Vec<DisplayedWebpage>,
- pub num_hits: Option<usize>,
+ pub num_hits: Count,
pub search_duration_ms: u128,
pub has_more_results: bool,
}
@@ -74,14 +75,14 @@ pub struct SearchQuery {
pub host_rankings: Option<HostRankings>,
pub return_ranking_signals: bool,
pub safe_search: bool,
- pub count_results: bool,
+ pub count_results_exact: bool,
pub return_body: Option<ReturnBody>,
pub return_structured_data: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct InitialWebsiteResult {
- pub num_websites: Option<usize>,
+ pub num_websites: Count,
pub websites: Vec<LocalRecallRankingWebpage>,
pub has_more: bool,
}
@@ -100,7 +101,7 @@ impl Default for SearchQuery {
host_rankings: Default::default(),
return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),
safe_search: defaults::SearchQuery::safe_search(),
- count_results: defaults::SearchQuery::count_results(),
+ count_results_exact: defaults::SearchQuery::count_results_exact(),
return_body: None,
return_structured_data: defaults::SearchQuery::return_structured_data(),
}
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index c37e22fe..f227ef46 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -189,7 +189,7 @@ export const api = {
};
export type ApiSearchQuery = {
- countResults?: boolean;
+ countResultsExact?: boolean;
flattenResponse?: boolean;
hostRankings?: HostRankings;
numResults?: number;
@@ -235,6 +235,15 @@ export type CodeOrText =
_type: 'text';
value: string;
};
+export type Count =
+ | {
+ _type: 'exact';
+ value: number;
+ }
+ | {
+ _type: 'approximate';
+ value: number;
+ };
export type Definition = string;
export type DisplayedAnswer = {
answer: string;
@@ -411,7 +420,7 @@ export type ThesaurusWidget = {
export type UrlWrapper = string;
export type WebsitesResult = {
hasMoreResults: boolean;
- numHits?: number;
+ numHits: Count;
searchDurationMs: number;
webpages: DisplayedWebpage[];
};
diff --git a/frontend/src/lib/search.ts b/frontend/src/lib/search.ts
index 4ad2ba79..4519c0f2 100644
--- a/frontend/src/lib/search.ts
+++ b/frontend/src/lib/search.ts
@@ -106,7 +106,6 @@ export const search = async (params: SearchParams, options: ApiOptions) => {
optic: params.optic && (await fetchRemoteOptic({ opticUrl: params.optic, fetch })),
selectedRegion: params.selectedRegion,
hostRankings: params.host_rankings,
- countResults: true,
},
options,
);
@@ -141,7 +140,6 @@ export const search = async (params: SearchParams, options: ApiOptions) => {
safeSearch: params.safeSearch,
selectedRegion: params.selectedRegion,
hostRankings: params.host_rankings,
- countResults: false,
},
options,
)
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index 828feb03..6b5f438c 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -12,6 +12,8 @@
import { search } from '$lib/search';
import { Keybind, searchCb, type Refs } from '$lib/keybind';
import SpellCorrection from './SpellCorrection.svelte';
+ import type { Count } from '$lib/api';
+ import { match } from 'ts-pattern';
export let data: PageData;
$: results = data.results;
@@ -101,6 +103,17 @@
keybind.onKeyDown(event, $useKeyboardShortcuts, context);
}
};
+
+ const prettyprintCount = (count: Count): string => {
+ return match(count)
+ .with({ _type: 'exact' }, () => {
+ return count.value.toLocaleString();
+ })
+ .with({ _type: 'approximate' }, () => {
+ return `~${count.value.toLocaleString()}`;
+ })
+ .exhaustive();
+ };
</script>
<svelte:window on:keydown={onKeyDown} />
@@ -127,13 +140,9 @@
<div class="mx-auto flex w-full justify-end sm:justify-between">
<div class="hidden h-full flex-col space-x-2 text-xs text-neutral sm:flex">
<p class="h-fit">
- {#if results && results.numHits != null}
- Found <span class="font-medium">{results.numHits.toLocaleString()}</span> results in
+ {#if results}
+ Found <span class="font-medium">{prettyprintCount(results.numHits)}</span> results in
<span class="font-medium">{((results.searchDurationMs ?? 0) / 1000).toFixed(2)}s</span>
- {:else if results}
- Search took <span class="font-medium"
- >{((results.searchDurationMs ?? 0) / 1000).toFixed(2)}s</span
- >
{/if}
</p>
</div>