Skip to content

web search done right

Ftr/customized ranking (#30)

Mikkel Denkerd064895d0bbb
diff --git a/Cargo.lock b/Cargo.lock
index 46b5f07d..11db888f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -55,6 +55,15 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"
+[[package]]
+name = "ascii-canvas"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"
+dependencies = [
+ "term",
+]
+
[[package]]
name = "askama"
version = "0.11.1"
@@ -103,6 +112,19 @@ dependencies = [
"toml",
]
+[[package]]
+name = "async-compression"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
+dependencies = [
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "async-trait"
version = "0.1.56"
@@ -234,6 +256,21 @@ dependencies = [
"shlex",
]
+[[package]]
+name = "bit-set"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+
[[package]]
name = "bit_field"
version = "0.10.1"
@@ -598,6 +635,8 @@ dependencies = [
"indicatif",
"itertools",
"kuchiki",
+ "lalrpop",
+ "lalrpop-util",
"logos",
"lru",
"maplit",
@@ -612,10 +651,12 @@ dependencies = [
"rocksdb",
"serde",
"serde_json",
+ "strum",
"tantivy",
"thiserror",
"tokio",
"toml",
+ "tower-http",
"tracing",
"tracing-subscriber",
"uuid",
@@ -650,6 +691,27 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
+[[package]]
+name = "dirs-next"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+dependencies = [
+ "cfg-if",
+ "dirs-sys-next",
+]
+
+[[package]]
+name = "dirs-sys-next"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
[[package]]
name = "downcast-rs"
version = "1.2.0"
@@ -677,6 +739,15 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+[[package]]
+name = "ena"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
+dependencies = [
+ "log",
+]
+
[[package]]
name = "encode_unicode"
version = "0.3.6"
@@ -744,6 +815,12 @@ dependencies = [
"instant",
]
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
[[package]]
name = "flate2"
version = "1.0.24"
@@ -1288,6 +1365,38 @@ dependencies = [
"selectors",
]
+[[package]]
+name = "lalrpop"
+version = "0.19.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"
+dependencies = [
+ "ascii-canvas",
+ "atty",
+ "bit-set",
+ "diff",
+ "ena",
+ "itertools",
+ "lalrpop-util",
+ "petgraph",
+ "pico-args",
+ "regex",
+ "regex-syntax 0.6.27",
+ "string_cache",
+ "term",
+ "tiny-keccak",
+ "unicode-xid",
+]
+
+[[package]]
+name = "lalrpop-util"
+version = "0.19.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"
+dependencies = [
+ "regex",
+]
+
[[package]]
name = "lazy_static"
version = "1.4.0"
@@ -1801,6 +1910,16 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+[[package]]
+name = "petgraph"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
[[package]]
name = "phf"
version = "0.8.0"
@@ -1874,6 +1993,12 @@ dependencies = [
"siphasher",
]
+[[package]]
+name = "pico-args"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"
+
[[package]]
name = "pin-project"
version = "1.0.10"
@@ -2119,6 +2244,17 @@ dependencies = [
"bitflags",
]
+[[package]]
+name = "redox_users"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+dependencies = [
+ "getrandom 0.2.6",
+ "redox_syscall",
+ "thiserror",
+]
+
[[package]]
name = "regex"
version = "1.6.0"
@@ -2473,6 +2609,28 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+[[package]]
+name = "strum"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
+dependencies = [
+ "strum_macros",
+]
+
+[[package]]
+name = "strum_macros"
+version = "0.24.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
[[package]]
name = "syn"
version = "1.0.96"
@@ -2601,6 +2759,17 @@ dependencies = [
"utf-8",
]
+[[package]]
+name = "term"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
+dependencies = [
+ "dirs-next",
+ "rustversion",
+ "winapi",
+]
+
[[package]]
name = "termcolor"
version = "1.1.3"
@@ -2704,6 +2873,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
[[package]]
name = "tinyvec"
version = "1.6.0"
@@ -2819,6 +2997,7 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba"
dependencies = [
+ "async-compression",
"bitflags",
"bytes",
"futures-core",
@@ -2949,6 +3128,12 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "unicode-xid"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
+
[[package]]
name = "url"
version = "2.2.2"
diff --git a/Cargo.toml b/Cargo.toml
index d64aa883..6e42ac01 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -51,9 +51,15 @@ once_cell = "1.13.1"
regex = "1.6.0"
http = "0.2.8"
kuchiki = "0.8.1"
+strum = { version = "0.24.1", features = ["derive", "strum_macros"] }
+tower-http = { version = "0.3.4", features = ["compression-gzip"] }
+lalrpop-util = { version = "0.19.8", features = ["lexer"] }
[dev-dependencies]
maplit = "1.0.2"
+[build-dependencies]
+lalrpop = "0.19.8"
+
[profile.release]
debug = true
diff --git a/README.md b/README.md
index ac90fca1..cf8b6b0f 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ Cuely is an open source web search engine hosted at [cuely.io](https://cuely.io/
* De-rank websites with third-party trackers
* Prioritize fresh content
* Regional search
-* (Coming soon) Customize how signals are combined during search for the final search result
+* Customize how signals are combined during search for the final search result
* (Coming soon) Use [goggles](https://brave.com/static-assets/files/goggles.pdf) to almost endlessly customize your search results.
* (Coming soon) Prioritize links (centrality) from the sites you trust.
diff --git a/build.rs b/build.rs
index 9f383a77..98dcc47d 100644
--- a/build.rs
+++ b/build.rs
@@ -10,4 +10,6 @@ fn main() {
println!("cargo:rerun-if-changed=yarn.lock");
println!("cargo:rerun-if-changed=tailwind.config.js");
println!("cargo:rerun-if-changed=postcss.config.js");
+
+ lalrpop::process_root().unwrap();
}
diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.toml
new file mode 100644
index 00000000..dec9e255
--- /dev/null
+++ b/configs/indexer/profile.toml
@@ -0,0 +1,9 @@
+warc_paths_file = "warc.paths"
+centrality_store_path = "./data/centrality"
+limit_warc_files = 1
+download_images = false
+
+[warc_source]
+type = "Local"
+folder = "./data/warc_files"
+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
diff --git a/src/bangs.rs b/src/bangs.rs
index 0e7955d5..c212adb6 100644
--- a/src/bangs.rs
+++ b/src/bangs.rs
@@ -125,7 +125,7 @@ mod tests {
use tantivy::tokenizer::TokenizerManager;
- use crate::schema::create_schema;
+ use crate::{ranking::signal_aggregator::SignalAggregator, schema::create_schema};
use super::*;
@@ -146,13 +146,31 @@ mod tests {
}]"#,
);
- let query = Query::parse("no bangs", schema.clone(), &tokenizer_manager).unwrap();
+ let query = Query::parse(
+ "no bangs",
+ schema.clone(),
+ &tokenizer_manager,
+ &SignalAggregator::default(),
+ )
+ .unwrap();
assert_eq!(bangs.get(&query), None);
- let query = Query::parse("!no bangs", schema.clone(), &tokenizer_manager).unwrap();
+ let query = Query::parse(
+ "!no bangs",
+ schema.clone(),
+ &tokenizer_manager,
+ &SignalAggregator::default(),
+ )
+ .unwrap();
assert_eq!(bangs.get(&query), None);
- let query = Query::parse("!ty bangs", schema, &tokenizer_manager).unwrap();
+ let query = Query::parse(
+ "!ty bangs",
+ schema,
+ &tokenizer_manager,
+ &SignalAggregator::default(),
+ )
+ .unwrap();
assert_eq!(
bangs.get(&query),
Some(BangHit {
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index bf55af17..458b4bb7 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use axum::{body::Body, Extension, Router};
+use tower_http::compression::CompressionLayer;
use crate::{
autosuggest::Autosuggest, bangs::Bangs, entity_index::EntityIndex, index::Index,
@@ -101,5 +102,6 @@ pub fn router(
.route("/about", get(about::route))
.route("/privacy-and-happy-lawyers", get(privacy::route))
.merge(SpaRouter::new("/static", "static"))
- .layer(Extension(state)))
+ .layer(Extension(state))
+ .layer(CompressionLayer::new()))
}
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 14598f0c..14cf46c3 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -269,7 +269,7 @@ pub async fn route(
}
});
- match state.searcher.search(query.as_str(), selected_region) {
+ match state.searcher.search(query.as_str(), selected_region, None) {
Ok(result) => match result {
SearchResult::Websites(result) => {
let search_result = result
diff --git a/src/index.rs b/src/index.rs
index 41e37c84..998b6aaf 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -292,7 +292,7 @@ impl From<Index> for FrozenIndex {
#[cfg(test)]
mod tests {
- use crate::ranking::Ranker;
+ use crate::ranking::{signal_aggregator::SignalAggregator, Ranker};
use super::*;
@@ -333,9 +333,14 @@ mod tests {
let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();
let index: Index = deserialized_frozen.into();
- let query = Query::parse("website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index f7330b88..cf12fd68 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -333,7 +333,7 @@ mod tests {
use maplit::hashset;
use crate::{
- ranking::Ranker,
+ ranking::{signal_aggregator::SignalAggregator, Ranker},
webpage::{region::RegionCount, Link},
};
@@ -344,9 +344,14 @@ mod tests {
#[test]
fn simple_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
@@ -391,9 +396,10 @@ mod tests {
"this query should not match",
index.schema(),
index.tokenizers(),
+ &SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -427,9 +433,14 @@ mod tests {
#[test]
fn english_stemming() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("runner", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "runner",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -463,9 +474,14 @@ mod tests {
#[test]
fn stemmed_query_english() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("runners", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "runners",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -499,9 +515,14 @@ mod tests {
#[test]
fn not_searchable_backlinks() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("great site", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "great site",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -564,9 +585,14 @@ mod tests {
#[test]
fn limited_top_docs() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("runner", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "runner",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
for _ in 0..100 {
index
@@ -602,9 +628,14 @@ mod tests {
#[test]
fn host_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query =
- Query::parse("dr", index.schema(), index.tokenizers()).expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "dr",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -686,9 +717,14 @@ mod tests {
let mut index = index1.merge(index2);
index.commit().unwrap();
- let query = Query::parse("website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
@@ -702,9 +738,14 @@ mod tests {
#[test]
fn match_across_fields() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("example test", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "example test",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
@@ -745,9 +786,14 @@ mod tests {
#[test]
fn fetch_time_ranking() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("test", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "test",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -803,7 +849,7 @@ mod tests {
#[test]
fn only_show_primary_images_when_relevant() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let ranker = Ranker::new(RegionCount::default());
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let mut webpage = Webpage::new(
&format!(
@@ -832,8 +878,13 @@ mod tests {
index.insert(webpage).expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let query = Query::parse("website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
+ let query = Query::parse(
+ "website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
let result = index
.search(&query, ranker.collector())
@@ -851,8 +902,13 @@ mod tests {
})
);
- let query = Query::parse("best website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
+ let query = Query::parse(
+ "best website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
let result = index
.search(&query, ranker.collector())
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 3ab4d47d..4f8b2839 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::Result;
+use crate::{ranking::signal_aggregator::SignalAggregator, Result};
use std::{collections::HashMap, sync::Arc};
use tantivy::{
query::{AllQuery, BooleanQuery, Occur},
@@ -39,6 +39,7 @@ impl Query {
query: &str,
schema: Arc<Schema>,
tokenizer_manager: &TokenizerManager,
+ aggregator: &SignalAggregator,
) -> Result<Query> {
let parsed_terms = parser::parse(query);
@@ -58,9 +59,11 @@ impl Query {
let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =
schema.fields().collect();
+ let field_boost = aggregator.field_boosts();
+
let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
- .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
+ .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager, field_boost))
.collect();
queries.push((Occur::Should, Box::new(AllQuery)));
@@ -128,6 +131,7 @@ mod tests {
"this is a simple query the the the the the the the the the the the the the",
Arc::clone(&schema),
&TokenizerManager::new(),
+ &SignalAggregator::default(),
)
.expect("Failed to parse query");
@@ -161,6 +165,7 @@ mod tests {
" this is a simple query ",
Arc::clone(&schema),
&TokenizerManager::new(),
+ &SignalAggregator::default(),
)
.expect("Failed to parse query");
@@ -180,20 +185,31 @@ mod tests {
fn parse_weird_characters() {
let schema = Arc::new(create_schema());
- let terms = Query::parse("123", Arc::clone(&schema), &TokenizerManager::new())
- .expect("Failed to parse query")
- .simple_terms();
+ let terms = Query::parse(
+ "123",
+ Arc::clone(&schema),
+ &TokenizerManager::new(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query")
+ .simple_terms();
assert_eq!(terms, vec!["123".to_string()]);
- let terms = Query::parse("123 33", Arc::clone(&schema), &TokenizerManager::new())
- .expect("Failed to parse query")
- .simple_terms();
+ let terms = Query::parse(
+ "123 33",
+ Arc::clone(&schema),
+ &TokenizerManager::new(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query")
+ .simple_terms();
assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
let terms = Query::parse(
"term! term# $",
Arc::clone(&schema),
&TokenizerManager::new(),
+ &SignalAggregator::default(),
)
.expect("Failed to parse query")
.simple_terms();
@@ -206,9 +222,14 @@ mod tests {
#[test]
fn not_query() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("test -website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "test -website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
index
.insert(Webpage::new(
@@ -298,9 +319,14 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let query = Query::parse("test site:first.com", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "test site:first.com",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -312,9 +338,10 @@ mod tests {
"test site:www.first.com",
index.schema(),
index.tokenizers(),
+ &SignalAggregator::default(),
)
.expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -322,9 +349,14 @@ mod tests {
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.first.com");
- let query = Query::parse("test -site:first.com", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "test -site:first.com",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -375,9 +407,14 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let query = Query::parse("intitle:website", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "intitle:website",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -428,9 +465,14 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let query = Query::parse("test inurl:forum", index.schema(), index.tokenizers())
- .expect("Failed to parse query");
- let ranker = Ranker::new(RegionCount::default());
+ let query = Query::parse(
+ "test inurl:forum",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());
let result = index
.search(&query, ranker.collector())
.expect("Search failed");
@@ -443,8 +485,13 @@ mod tests {
fn empty_query() {
let schema = Arc::new(create_schema());
- let query = Query::parse("", Arc::clone(&schema), &TokenizerManager::new())
- .expect("Failed to parse query");
+ let query = Query::parse(
+ "",
+ Arc::clone(&schema),
+ &TokenizerManager::new(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
assert!(query.is_empty())
}
diff --git a/src/query/parser.rs b/src/query/parser.rs
index dbae4f62..1ef5417a 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -22,6 +22,7 @@ use tantivy::{
use crate::{
bangs::BANG_PREFIX,
+ ranking::signal_aggregator::FieldBoost,
schema::{Field, ALL_FIELDS},
};
@@ -54,6 +55,7 @@ fn simple_into_tantivy(
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
+ field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
let (backlink_field, backlink_field_entry) = fields
.iter()
@@ -67,6 +69,7 @@ fn simple_into_tantivy(
term,
fields,
tokenizer_manager,
+ field_boost,
))),
),
(
@@ -75,6 +78,7 @@ fn simple_into_tantivy(
backlink_field,
backlink_field_entry,
tokenizer_manager,
+ field_boost,
term,
)),
),
@@ -86,14 +90,17 @@ impl Term {
&self,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
+ field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
match self {
- Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),
+ Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager, field_boost),
Term::Not(subterm) => vec![(
Occur::MustNot,
- Box::new(BooleanQuery::new(
- subterm.as_tantivy_query(fields, tokenizer_manager),
- )),
+ Box::new(BooleanQuery::new(subterm.as_tantivy_query(
+ fields,
+ tokenizer_manager,
+ field_boost,
+ ))),
)],
Term::Site(site) => vec![(
Occur::Must,
@@ -101,6 +108,7 @@ impl Term {
site,
fields,
tokenizer_manager,
+ field_boost,
))),
)],
Term::Title(title) => {
@@ -112,7 +120,7 @@ impl Term {
.unwrap();
vec![(
Occur::Must,
- Term::tantivy_term_query(field, entry, tokenizer_manager, title),
+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, title),
)]
}
Term::Body(body) => {
@@ -124,7 +132,7 @@ impl Term {
.unwrap();
vec![(
Occur::Must,
- Term::tantivy_term_query(field, entry, tokenizer_manager, body),
+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, body),
)]
}
Term::Url(url) => {
@@ -134,7 +142,7 @@ impl Term {
.unwrap();
vec![(
Occur::Must,
- Term::tantivy_term_query(field, entry, tokenizer_manager, url),
+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, url),
)]
}
Term::PossibleBang(text) => {
@@ -142,7 +150,7 @@ impl Term {
term.push(BANG_PREFIX);
term.push_str(text);
- simple_into_tantivy(&term, fields, tokenizer_manager)
+ simple_into_tantivy(&term, fields, tokenizer_manager, field_boost)
}
}
}
@@ -151,6 +159,7 @@ impl Term {
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
+ field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
@@ -159,7 +168,7 @@ impl Term {
.map(|(field, entry)| {
(
Occur::Should,
- Term::tantivy_term_query(field, entry, tokenizer_manager, term),
+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, term),
)
})
.collect()
@@ -169,6 +178,7 @@ impl Term {
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
+ field_boost: &FieldBoost,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
@@ -182,7 +192,7 @@ impl Term {
.map(|(field, entry)| {
(
Occur::Should,
- Term::tantivy_term_query(field, entry, tokenizer_manager, term),
+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, term),
)
})
.collect()
@@ -192,6 +202,7 @@ impl Term {
field: &tantivy::schema::Field,
entry: &tantivy::schema::FieldEntry,
tokenizer_manager: &TokenizerManager,
+ field_boost: &FieldBoost,
term: &str,
) -> Box<dyn tantivy::query::Query + 'static> {
let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);
@@ -206,7 +217,8 @@ impl Term {
IndexRecordOption::WithFreqsAndPositions,
))
};
- let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);
+
+ let boost = field_boost.get(&ALL_FIELDS[field.field_id() as usize]) as f32;
Box::new(BoostQuery::new(processed_query, boost))
}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index fdeb88ed..24660e6a 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -16,34 +16,35 @@
use std::sync::Arc;
-use crate::schema::{Field, CENTRALITY_SCALING};
use crate::webpage::region::{Region, RegionCount};
use chrono::Utc;
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
-use tantivy::fastfield::{Column, DynamicFastFieldReader};
use tantivy::{DocId, Score, SegmentReader};
+use super::signal_aggregator::SignalAggregator;
+
pub(crate) struct InitialScoreTweaker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
+ aggregator: SignalAggregator,
}
impl InitialScoreTweaker {
- pub fn new(region_count: Arc<RegionCount>, selected_region: Option<Region>) -> Self {
+ pub fn new(
+ region_count: Arc<RegionCount>,
+ selected_region: Option<Region>,
+ aggregator: SignalAggregator,
+ ) -> Self {
Self {
region_count,
selected_region,
+ aggregator,
}
}
}
pub(crate) struct InitialSegmentScoreTweaker {
- centrality_reader: DynamicFastFieldReader<u64>,
- is_homepage_reader: DynamicFastFieldReader<u64>,
- fetch_time_ms_reader: DynamicFastFieldReader<u64>,
- update_timestamp_reader: DynamicFastFieldReader<u64>,
- num_trackers_reader: DynamicFastFieldReader<u64>,
- region_reader: DynamicFastFieldReader<u64>,
+ aggregator: SignalAggregator,
region_count: Arc<RegionCount>,
current_timestamp: f64,
selected_region: Option<Region>,
@@ -53,111 +54,28 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
type Child = InitialSegmentScoreTweaker;
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
- let centrality_field = segment_reader
- .schema()
- .get_field(Field::Centrality.as_str())
- .expect("Faild to load centrality field");
- let centrality_reader = segment_reader
- .fast_fields()
- .u64(centrality_field)
- .expect("Failed to get centrality fast-field reader");
-
- let is_homepage_field = segment_reader
- .schema()
- .get_field(Field::IsHomepage.as_str())
- .expect("Faild to load is_homepage field");
- let is_homepage_reader = segment_reader
- .fast_fields()
- .u64(is_homepage_field)
- .expect("Failed to get is_homepage fast-field reader");
-
- let fetch_time_ms_field = segment_reader
- .schema()
- .get_field(Field::FetchTimeMs.as_str())
- .expect("Faild to load fetch_time_ms field");
- let fetch_time_ms_reader = segment_reader
- .fast_fields()
- .u64(fetch_time_ms_field)
- .expect("Failed to get fetch_time_ms fast-field reader");
-
- let update_timestamp_field = segment_reader
- .schema()
- .get_field(Field::LastUpdated.as_str())
- .expect("Faild to load last_updated field");
- let update_timestamp_reader = segment_reader
- .fast_fields()
- .u64(update_timestamp_field)
- .expect("Failed to get last_updated fast-field reader");
-
- let num_trackers_field = segment_reader
- .schema()
- .get_field(Field::NumTrackers.as_str())
- .expect("Faild to load num_trackers field");
- let num_trackers_reader = segment_reader
- .fast_fields()
- .u64(num_trackers_field)
- .expect("Failed to get num_trackers fast-field reader");
-
- let region_field = segment_reader
- .schema()
- .get_field(Field::Region.as_str())
- .expect("Faild to load region field");
- let region_reader = segment_reader
- .fast_fields()
- .u64(region_field)
- .expect("Failed to get region fast-field reader");
+ let mut aggregator = SignalAggregator::new_like(&self.aggregator);
+ aggregator.register_readers(segment_reader);
let current_timestamp = Utc::now().timestamp() as f64;
Ok(InitialSegmentScoreTweaker {
- centrality_reader,
- is_homepage_reader,
- fetch_time_ms_reader,
- update_timestamp_reader,
- num_trackers_reader,
+ aggregator,
current_timestamp,
- region_reader,
selected_region: self.selected_region,
region_count: Arc::clone(&self.region_count),
})
}
}
-fn time_to_score(time: f64) -> f64 {
- 1.0 / ((time + 1.0).log2())
-}
-
-fn region_score(
- region_count: &RegionCount,
- selected_region: Option<Region>,
- webpage_region: Region,
-) -> f64 {
- let boost = selected_region.map_or(
- 0.0,
- |region| if region == webpage_region { 50.0 } else { 0.0 },
- );
-
- boost + region_count.score(&webpage_region)
-}
-
impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
fn score(&mut self, doc: DocId, score: Score) -> f64 {
- let score = score as f64;
- let centrality: f64 =
- self.centrality_reader.get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64;
- let is_homepage = self.is_homepage_reader.get_val(doc as u64) as f64;
- let fetch_time_ms = self.fetch_time_ms_reader.get_val(doc as u64) as f64;
- let update_timestamp = self.update_timestamp_reader.get_val(doc as u64) as f64;
- let hours_since_update = (self.current_timestamp - update_timestamp).max(0.000001) / 3600.0;
- let num_trackers = self.num_trackers_reader.get_val(doc as u64) as f64;
- let region = Region::from_id(self.region_reader.get_val(doc as u64));
-
- (3.0 * score)
- + (3200.0 * centrality)
- + (1.0 * is_homepage)
- + (1.0 / (fetch_time_ms + 1.0))
- + (1500.0 * time_to_score(hours_since_update))
- + (200.0 * (1.0 / (num_trackers + 1.0)))
- + (25.0 * region_score(&self.region_count, self.selected_region, region))
+ self.aggregator.score(
+ doc,
+ score,
+ &self.region_count,
+ self.current_timestamp,
+ self.selected_region,
+ )
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 7a28349a..ca3cc98c 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -16,6 +16,7 @@
pub mod centrality_store;
mod initial;
+pub mod signal_aggregator;
use std::sync::Arc;
@@ -24,16 +25,20 @@ use tantivy::collector::{Collector, TopDocs};
use crate::webpage::region::{Region, RegionCount};
+use self::signal_aggregator::SignalAggregator;
+
pub struct Ranker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
+ aggregator: SignalAggregator,
}
impl Ranker {
- pub fn new(region_count: RegionCount) -> Self {
+ pub fn new(region_count: RegionCount, aggregator: SignalAggregator) -> Self {
Ranker {
region_count: Arc::new(region_count),
selected_region: None,
+ aggregator,
}
}
@@ -43,8 +48,11 @@ impl Ranker {
}
pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
- let score_tweaker =
- InitialScoreTweaker::new(Arc::clone(&self.region_count), self.selected_region);
+ let score_tweaker = InitialScoreTweaker::new(
+ Arc::clone(&self.region_count),
+ self.selected_region,
+ SignalAggregator::new_like(&self.aggregator),
+ );
TopDocs::with_limit(20).tweak_score(score_tweaker)
}
}
@@ -108,7 +116,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
let result = searcher
- .search("example", None)
+ .search("example", None, None)
.expect("Search failed")
.into_websites()
.unwrap();
@@ -183,7 +191,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
let result = searcher
- .search("dr dk", None)
+ .search("dr dk", None, None)
.expect("Search failed")
.into_websites()
.unwrap();
@@ -242,7 +250,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
let result = searcher
- .search("title", None)
+ .search("title", None, None)
.expect("Search failed")
.into_websites()
.unwrap();
@@ -308,7 +316,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
let result = searcher
- .search("test", None)
+ .search("test", None, None)
.expect("Search failed")
.into_websites()
.unwrap();
@@ -365,7 +373,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
let result = searcher
- .search("test", None)
+ .search("test", None, None)
.expect("Search failed")
.into_websites()
.unwrap();
diff --git a/src/ranking/signal_aggregator/ast.rs b/src/ranking/signal_aggregator/ast.rs
new file mode 100644
index 00000000..cf757c38
--- /dev/null
+++ b/src/ranking/signal_aggregator/ast.rs
@@ -0,0 +1,70 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use lalrpop_util::lalrpop_mod;
+
+lalrpop_mod!(pub parser, "/ranking/signal_aggregator/parser.rs");
+
+pub static PARSER: once_cell::sync::Lazy<parser::AlterationsParser> =
+ once_cell::sync::Lazy::new(parser::AlterationsParser::new);
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum Target {
+ Signal(String),
+ Field(String),
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Alteration {
+ pub target: Target,
+ pub score: f64,
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let res: Vec<Alteration> = PARSER
+ .parse(
+ r#"
+ @host_centrality = 3
+ @bm25 = 100
+ @field_url = 2
+ "#,
+ )
+ .unwrap();
+
+ assert_eq!(
+ res,
+ vec![
+ Alteration {
+ target: Target::Signal("host_centrality".to_string()),
+ score: 3.0,
+ },
+ Alteration {
+ target: Target::Signal("bm25".to_string()),
+ score: 100.0
+ },
+ Alteration {
+ target: Target::Field("url".to_string()),
+ score: 2.0
+ }
+ ]
+ )
+ }
+}
diff --git a/src/ranking/signal_aggregator/mod.rs b/src/ranking/signal_aggregator/mod.rs
new file mode 100644
index 00000000..fba1c132
--- /dev/null
+++ b/src/ranking/signal_aggregator/mod.rs
@@ -0,0 +1,285 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod ast;
+
+use crate::{Error, Result};
+use std::{collections::HashMap, sync::Arc};
+
+use strum::{EnumIter, IntoEnumIterator};
+use tantivy::{
+ fastfield::{Column, DynamicFastFieldReader},
+ DocId, Score, SegmentReader,
+};
+
+use crate::{
+ schema::{Field, ALL_FIELDS, CENTRALITY_SCALING},
+ webpage::region::{Region, RegionCount},
+};
+
+use self::ast::{Alteration, Target, PARSER};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, EnumIter)]
+pub enum Signal {
+ Bm25,
+ HostCentrality,
+ IsHomepage,
+ FetchTimeMs,
+ UpdateTimestamp,
+ NumTrackers,
+ Region,
+}
+
+impl Signal {
+ fn from_field(field: &Field) -> Option<Self> {
+ match field {
+ Field::IsHomepage => Some(Signal::IsHomepage),
+ Field::Centrality => Some(Signal::HostCentrality),
+ Field::FetchTimeMs => Some(Signal::FetchTimeMs),
+ Field::LastUpdated => Some(Signal::UpdateTimestamp),
+ Field::NumTrackers => Some(Signal::NumTrackers),
+ Field::Region => Some(Signal::Region),
+ _ => None,
+ }
+ }
+
+ fn has_fast_reader(&self) -> bool {
+ !matches!(self, Signal::Bm25)
+ }
+
+ fn value(
+ &self,
+ doc: DocId,
+ bm25: Score,
+ fastfield: Option<&DynamicFastFieldReader<u64>>,
+ region_count: &Arc<RegionCount>,
+ current_timestamp: f64,
+ selected_region: Option<Region>,
+ ) -> f64 {
+ match self {
+ Signal::Bm25 => bm25 as f64,
+ Signal::HostCentrality => {
+ fastfield.unwrap().get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64
+ }
+ Signal::IsHomepage => fastfield.unwrap().get_val(doc as u64) as f64,
+ Signal::FetchTimeMs => {
+ let fetch_time_ms = fastfield.unwrap().get_val(doc as u64) as f64;
+ 1.0 / (fetch_time_ms + 1.0)
+ }
+ Signal::UpdateTimestamp => {
+ let update_timestamp = fastfield.unwrap().get_val(doc as u64) as f64;
+ let hours_since_update =
+ (current_timestamp - update_timestamp).max(0.000001) / 3600.0;
+ 1.0 / ((hours_since_update + 1.0).log2())
+ }
+ Signal::NumTrackers => {
+ let num_trackers = fastfield.unwrap().get_val(doc as u64) as f64;
+ 1.0 / (num_trackers + 1.0)
+ }
+ Signal::Region => {
+ let webpage_region = Region::from_id(fastfield.unwrap().get_val(doc as u64));
+
+ let boost =
+ selected_region.map_or(
+ 0.0,
+ |region| if region == webpage_region { 50.0 } else { 0.0 },
+ );
+
+ boost + region_count.score(&webpage_region)
+ }
+ }
+ }
+
+ fn default_coefficient(&self) -> f64 {
+ match self {
+ Signal::Bm25 => 3.0,
+ Signal::HostCentrality => 3200.0,
+ Signal::IsHomepage => 1.0,
+ Signal::FetchTimeMs => 1.0,
+ Signal::UpdateTimestamp => 1500.0,
+ Signal::NumTrackers => 200.0,
+ Signal::Region => 25.0,
+ }
+ }
+
+ fn from_string(name: String) -> Option<Signal> {
+ match name.as_str() {
+ "bm25" => Some(Signal::Bm25),
+ "host_centrality" => Some(Signal::HostCentrality),
+ _ => None,
+ }
+ }
+}
+
+fn fastfield_reader(segment_reader: &SegmentReader, field: &Field) -> DynamicFastFieldReader<u64> {
+ let tv_field = segment_reader
+ .schema()
+ .get_field(field.as_str())
+ .unwrap_or_else(|| panic!("Faild to load {} field", field.as_str()));
+
+ segment_reader
+ .fast_fields()
+ .u64(tv_field)
+ .unwrap_or_else(|_| panic!("Failed to get {} fast-field reader", field.as_str()))
+}
+
+#[derive(Debug, Clone)]
+pub struct FieldBoost(HashMap<Field, f64>);
+
+#[derive(Debug, Clone)]
+pub struct SignalCoefficient(HashMap<Signal, f64>);
+
+impl SignalCoefficient {
+ pub fn get(&self, signal: &Signal) -> f64 {
+ self.0
+ .get(signal)
+ .copied()
+ .unwrap_or_else(|| signal.default_coefficient())
+ }
+}
+
+impl FieldBoost {
+ pub fn get(&self, field: &Field) -> f64 {
+ self.0
+ .get(field)
+ .copied()
+ .or_else(|| field.boost().map(|s| s as f64))
+ .unwrap_or(1.0)
+ }
+}
+
+pub struct SignalAggregator {
+ readers: HashMap<Signal, DynamicFastFieldReader<u64>>,
+ signal_coefficients: SignalCoefficient,
+ field_boost: FieldBoost,
+}
+
+impl std::fmt::Debug for SignalAggregator {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SignalAggregator")
+ .field("signal_coefficients", &self.signal_coefficients)
+ .field("field_boost", &self.field_boost)
+ .finish()
+ }
+}
+
+impl Default for SignalAggregator {
+ fn default() -> Self {
+ Self::new(HashMap::new(), HashMap::new())
+ }
+}
+
+impl SignalAggregator {
+ pub fn new(coefficients: HashMap<Signal, f64>, boosts: HashMap<Field, f64>) -> Self {
+ let signal_coefficients = SignalCoefficient(coefficients);
+ let field_boost = FieldBoost(boosts);
+
+ Self {
+ readers: HashMap::new(),
+ signal_coefficients,
+ field_boost,
+ }
+ }
+
+ pub fn new_like(other: &SignalAggregator) -> Self {
+ Self {
+ readers: HashMap::new(),
+ signal_coefficients: other.signal_coefficients.clone(),
+ field_boost: other.field_boost.clone(),
+ }
+ }
+
+ pub fn register_readers(&mut self, segment_reader: &SegmentReader) {
+ for field in &ALL_FIELDS {
+ if let Some(signal) = Signal::from_field(field) {
+ if signal.has_fast_reader() {
+ self.mut_readers()
+ .insert(signal, fastfield_reader(segment_reader, field));
+ }
+ }
+ }
+ }
+
+ pub fn score(
+ &self,
+ doc: DocId,
+ bm25: Score,
+ region_count: &Arc<RegionCount>,
+ current_timestamp: f64,
+ selected_region: Option<Region>,
+ ) -> f64 {
+ Signal::iter()
+ .map(|signal| {
+ self.coefficients().get(&signal)
+ * signal.value(
+ doc,
+ bm25,
+ self.readers().get(&signal),
+ region_count,
+ current_timestamp,
+ selected_region,
+ )
+ })
+ .sum()
+ }
+
+ pub fn coefficients(&self) -> &SignalCoefficient {
+ &self.signal_coefficients
+ }
+
+ pub fn field_boosts(&self) -> &FieldBoost {
+ &self.field_boost
+ }
+
+ pub fn mut_readers(&mut self) -> &mut HashMap<Signal, DynamicFastFieldReader<u64>> {
+ &mut self.readers
+ }
+
+ pub fn readers(&self) -> &HashMap<Signal, DynamicFastFieldReader<u64>> {
+ &self.readers
+ }
+}
+
+impl From<Vec<Alteration>> for SignalAggregator {
+ fn from(alterations: Vec<Alteration>) -> Self {
+ let mut coefficients = HashMap::new();
+ let mut boosts = HashMap::new();
+
+ for alteration in alterations {
+ match alteration.target {
+ Target::Signal(name) => {
+ if let Some(signal) = Signal::from_string(name) {
+ coefficients.insert(signal, alteration.score);
+ }
+ }
+ Target::Field(name) => {
+ if let Some(field) = Field::from_string(name) {
+ boosts.insert(field, alteration.score);
+ }
+ }
+ }
+ }
+
+ Self::new(coefficients, boosts)
+ }
+}
+
+pub fn parse(program: &str) -> Result<SignalAggregator> {
+ match PARSER.parse(program) {
+ Ok(alterations) => Ok(SignalAggregator::from(alterations)),
+ Err(_) => Err(Error::Parse),
+ }
+}
diff --git a/src/ranking/signal_aggregator/parser.lalrpop b/src/ranking/signal_aggregator/parser.lalrpop
new file mode 100644
index 00000000..ebdca900
--- /dev/null
+++ b/src/ranking/signal_aggregator/parser.lalrpop
@@ -0,0 +1,25 @@
+use std::str::FromStr;
+use crate::ranking::signal_aggregator::ast::*;
+
+grammar;
+
+pub Alterations: Vec<Alteration> = <Alteration*> => <>;
+
+Alteration: Alteration = {
+ "@" <target:Target> "=" <score:Num> => Alteration {<>}
+}
+
+Target: Target = {
+ <Name> => {
+ if let Some(name) = <>.strip_prefix("field_") {
+ Target::Field(name.to_string())
+ } else {
+ Target::Signal(<>)
+ }
+ },
+}
+
+Name: String = r"[a-zA-Z_]+[0-9]*" => <>.to_string();
+
+
+Num: f64 = r"[0-9]+" => f64::from_str(<>).unwrap();
\ No newline at end of file
diff --git a/src/schema.rs b/src/schema.rs
index 764a609e..bb0fa4d7 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -22,7 +22,7 @@ use crate::tokenizer::{Normal, Stemmed};
pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
-#[derive(Clone)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Field {
Title,
CleanBody,
@@ -44,6 +44,7 @@ pub enum Field {
NumTrackers,
Region,
}
+
pub static ALL_FIELDS: [Field; 19] = [
Field::Title,
Field::CleanBody,
@@ -189,6 +190,31 @@ impl Field {
pub fn is_searchable(&self) -> bool {
!matches!(self, Field::PrimaryImage | Field::BacklinkText)
}
+
+ pub fn from_string(name: String) -> Option<Field> {
+ match name.as_str() {
+ "title" => Some(Field::Title),
+ "body" => Some(Field::CleanBody),
+ "url" => Some(Field::Url),
+ "host" => Some(Field::Host),
+ "backlink_text" => Some(Field::BacklinkText),
+ "centrality" => Some(Field::Centrality),
+ "stemmed_title" => Some(Field::StemmedTitle),
+ "stemmed_body" => Some(Field::StemmedCleanBody),
+ "domain" => Some(Field::Domain),
+ "domain_if_homepage" => Some(Field::DomainIfHomepage),
+ "is_homepage" => Some(Field::IsHomepage),
+ "fetch_time_ms" => Some(Field::FetchTimeMs),
+ "primary_image_uuid" => Some(Field::PrimaryImage),
+ "last_updated" => Some(Field::LastUpdated),
+ "description" => Some(Field::Description),
+ "all_body" => Some(Field::AllBody),
+ "stemmed_all_body" => Some(Field::StemmedAllBody),
+ "num_trackers" => Some(Field::NumTrackers),
+ "region" => Some(Field::Region),
+ _ => None,
+ }
+ }
}
pub fn create_schema() -> tantivy::schema::Schema {
diff --git a/src/searcher.rs b/src/searcher.rs
index 05f9d34a..dd25c03b 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -25,6 +25,7 @@ use crate::image_store::Image;
use crate::index::Index;
use crate::inverted_index::InvertedIndexSearchResult;
use crate::query::Query;
+use crate::ranking::signal_aggregator;
use crate::ranking::Ranker;
use crate::webpage::region::Region;
use crate::webpage::Url;
@@ -67,11 +68,27 @@ impl Searcher {
}
impl Searcher {
- pub fn search(&self, query: &str, selected_region: Option<Region>) -> Result<SearchResult> {
+ pub fn search(
+ &self,
+ query: &str,
+ selected_region: Option<Region>,
+ goggle_program: Option<&str>,
+ ) -> Result<SearchResult> {
let start = Instant::now();
let raw_query = query.to_string();
- let query = Query::parse(query, self.index.schema(), self.index.tokenizers())?;
+ let aggregator = goggle_program
+ .and_then(|program| signal_aggregator::parse(program).ok())
+ .unwrap_or_default();
+
+ dbg!(&aggregator);
+
+ let query = Query::parse(
+ query,
+ self.index.schema(),
+ self.index.tokenizers(),
+ &aggregator,
+ )?;
if query.is_empty() {
return Err(Error::EmptyQuery);
@@ -83,7 +100,7 @@ impl Searcher {
}
}
- let mut ranker = Ranker::new(self.index.region_count.clone());
+ let mut ranker = Ranker::new(self.index.region_count.clone(), aggregator);
if let Some(region) = selected_region {
if region != Region::All {
@@ -142,3 +159,121 @@ impl SearchResult {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use crate::webpage::Webpage;
+
+ use super::*;
+
+ #[test]
+ fn custom_signal_aggregation() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ example
+ </body>
+ </html>
+ "#,
+ "https://www.body.com",
+ vec![],
+ 1.0,
+ 20,
+ ))
+ .expect("failed to parse webpage");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Example website</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "https://www.title.com",
+ vec![],
+ 1.0,
+ 20,
+ ))
+ .expect("failed to parse webpage");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Example website</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "https://www.centrality.com",
+ vec![],
+ 1.0002,
+ 500,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().unwrap();
+
+ let searcher = Searcher::new(index, None, None);
+
+ let res = searcher
+ .search(
+ "example",
+ None,
+ Some(
+ r#"
+ @field_title = 20000000
+ @host_centrality = 0
+ "#,
+ ),
+ )
+ .unwrap()
+ .into_websites()
+ .unwrap();
+
+ assert_eq!(res.webpages.num_docs, 3);
+ assert_eq!(&res.webpages.documents[0].url, "https://www.title.com");
+
+ let res = searcher
+ .search(
+ "example",
+ None,
+ Some(
+ r#"
+ @field_all_body= 20000000
+ @host_centrality = 0
+ "#,
+ ),
+ )
+ .unwrap()
+ .into_websites()
+ .unwrap();
+
+ assert_eq!(res.webpages.num_docs, 3);
+ assert_eq!(&res.webpages.documents[0].url, "https://www.body.com");
+
+ let res = searcher
+ .search("example", None, Some("@host_centrality= 2000000"))
+ .unwrap()
+ .into_websites()
+ .unwrap();
+
+ assert_eq!(res.webpages.num_docs, 3);
+ assert_eq!(&res.webpages.documents[0].url, "https://www.centrality.com");
+ }
+}
diff --git a/src/snippet.rs b/src/snippet.rs
index 55896eef..769a1311 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -347,7 +347,7 @@ Survey in 2016, 2017, and 2018."#;
let searcher = Searcher::from(index);
let result = searcher
- .search("rust language", None)
+ .search("rust language", None, None)
.expect("Search failed")
.into_websites()
.unwrap();
@@ -387,7 +387,7 @@ Survey in 2016, 2017, and 2018."#;
let searcher = Searcher::from(index);
let result = searcher
- .search("describe", None)
+ .search("describe", None, None)
.expect("Search failed")
.into_websites()
.unwrap();