web search done right
Ftr/customized ranking (#30)
Mikkel Denkerd064895d0bbb
diff --git a/Cargo.lock b/Cargo.lockindex 46b5f07d..11db888f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -55,6 +55,15 @@ version = "1.5.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"+[[package]]+name = "ascii-canvas"+version = "3.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"+dependencies = [+ "term",+]+[[package]]name = "askama"version = "0.11.1"@@ -103,6 +112,19 @@ dependencies = ["toml",]+[[package]]+name = "async-compression"+version = "0.3.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"+dependencies = [+ "flate2",+ "futures-core",+ "memchr",+ "pin-project-lite",+ "tokio",+]+[[package]]name = "async-trait"version = "0.1.56"@@ -234,6 +256,21 @@ dependencies = ["shlex",]+[[package]]+name = "bit-set"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"+dependencies = [+ "bit-vec",+]++[[package]]+name = "bit-vec"+version = "0.6.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"+[[package]]name = "bit_field"version = "0.10.1"@@ -598,6 +635,8 @@ dependencies = ["indicatif","itertools","kuchiki",+ "lalrpop",+ "lalrpop-util","logos","lru","maplit",@@ -612,10 +651,12 @@ dependencies = ["rocksdb","serde","serde_json",+ "strum","tantivy","thiserror","tokio","toml",+ "tower-http","tracing","tracing-subscriber","uuid",@@ -650,6 +691,27 @@ version = "0.1.12"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"+[[package]]+name = "dirs-next"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"+dependencies = [+ "cfg-if",+ "dirs-sys-next",+]++[[package]]+name = "dirs-sys-next"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"+dependencies = [+ "libc",+ "redox_users",+ "winapi",+]+[[package]]name = "downcast-rs"version = "1.2.0"@@ -677,6 +739,15 @@ version = "1.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"+[[package]]+name = "ena"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"+dependencies = [+ "log",+]+[[package]]name = "encode_unicode"version = "0.3.6"@@ -744,6 +815,12 @@ dependencies = ["instant",]+[[package]]+name = "fixedbitset"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"+[[package]]name = "flate2"version = "1.0.24"@@ -1288,6 +1365,38 @@ dependencies = ["selectors",]+[[package]]+name = "lalrpop"+version = "0.19.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"+dependencies = [+ "ascii-canvas",+ "atty",+ "bit-set",+ "diff",+ "ena",+ "itertools",+ "lalrpop-util",+ "petgraph",+ "pico-args",+ "regex",+ "regex-syntax 0.6.27",+ "string_cache",+ "term",+ "tiny-keccak",+ "unicode-xid",+]++[[package]]+name = "lalrpop-util"+version = "0.19.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"+dependencies = [+ "regex",+]+[[package]]name = "lazy_static"version = "1.4.0"@@ -1801,6 +1910,16 @@ version = "2.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"+[[package]]+name = "petgraph"+version = "0.6.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"+dependencies = [+ "fixedbitset",+ "indexmap",+]+[[package]]name = "phf"version = "0.8.0"@@ -1874,6 +1993,12 @@ dependencies = ["siphasher",]+[[package]]+name = "pico-args"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"+[[package]]name = "pin-project"version = "1.0.10"@@ -2119,6 +2244,17 @@ dependencies = ["bitflags",]+[[package]]+name = "redox_users"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"+dependencies = [+ "getrandom 0.2.6",+ "redox_syscall",+ "thiserror",+]+[[package]]name = "regex"version = "1.6.0"@@ -2473,6 +2609,28 @@ version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"+[[package]]+name = "strum"+version = "0.24.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"+dependencies = [+ "strum_macros",+]++[[package]]+name = "strum_macros"+version = "0.24.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"+dependencies = [+ "heck",+ "proc-macro2",+ "quote",+ "rustversion",+ "syn",+]+[[package]]name = "syn"version = "1.0.96"@@ -2601,6 +2759,17 @@ dependencies = ["utf-8",]+[[package]]+name = "term"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"+dependencies = [+ "dirs-next",+ "rustversion",+ "winapi",+]+[[package]]name = "termcolor"version = "1.1.3"@@ -2704,6 +2873,15 @@ dependencies = ["serde",]+[[package]]+name = "tiny-keccak"+version = "2.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"+dependencies = [+ "crunchy",+]+[[package]]name = "tinyvec"version = "1.6.0"@@ -2819,6 +2997,7 @@ version = "0.3.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba"dependencies = [+ "async-compression","bitflags","bytes","futures-core",@@ -2949,6 +3128,12 @@ dependencies = ["tinyvec",]+[[package]]+name = "unicode-xid"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"+[[package]]name = "url"version = "2.2.2"diff --git a/Cargo.toml b/Cargo.tomlindex d64aa883..6e42ac01 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -51,9 +51,15 @@ once_cell = "1.13.1"regex = "1.6.0"http = "0.2.8"kuchiki = "0.8.1"+strum = { version = "0.24.1", features = ["derive", "strum_macros"] }+tower-http = { version = "0.3.4", features = ["compression-gzip"] }+lalrpop-util = { version = "0.19.8", features = ["lexer"] }[dev-dependencies]maplit = "1.0.2"+[build-dependencies]+lalrpop = "0.19.8"+[profile.release]debug = truediff --git a/README.md b/README.mdindex ac90fca1..cf8b6b0f 100644--- a/README.md+++ b/README.md@@ -21,7 +21,7 @@ Cuely is an open source web search engine hosted at [cuely.io](https://cuely.io/* De-rank websites with third-party trackers* Prioritize fresh content* Regional search-* (Coming soon) Customize how signals are combined during search for the final search result+* Customize how signals are combined during search for the final search result* (Coming soon) Use [goggles](https://brave.com/static-assets/files/goggles.pdf) to almost endlessly customize your search results.* (Coming soon) Prioritize links (centrality) from the sites you trust.diff --git a/build.rs b/build.rsindex 9f383a77..98dcc47d 100644--- a/build.rs+++ b/build.rs@@ -10,4 +10,6 @@ fn main() {println!("cargo:rerun-if-changed=yarn.lock");println!("cargo:rerun-if-changed=tailwind.config.js");println!("cargo:rerun-if-changed=postcss.config.js");++ lalrpop::process_root().unwrap();}diff --git a/configs/indexer/profile.toml b/configs/indexer/profile.tomlnew file mode 100644index 00000000..dec9e255--- /dev/null+++ b/configs/indexer/profile.toml@@ -0,0 +1,9 @@+warc_paths_file = "warc.paths"+centrality_store_path = "./data/centrality"+limit_warc_files = 1+download_images = false++[warc_source]+type = "Local"+folder = "./data/warc_files"+names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]diff --git a/src/bangs.rs b/src/bangs.rsindex 0e7955d5..c212adb6 100644--- a/src/bangs.rs+++ b/src/bangs.rs@@ -125,7 +125,7 @@ mod tests {use tantivy::tokenizer::TokenizerManager;- use crate::schema::create_schema;+ use crate::{ranking::signal_aggregator::SignalAggregator, schema::create_schema};use super::*;@@ -146,13 +146,31 @@ mod tests {}]"#,);- let query = Query::parse("no bangs", schema.clone(), &tokenizer_manager).unwrap();+ let query = Query::parse(+ "no bangs",+ schema.clone(),+ &tokenizer_manager,+ &SignalAggregator::default(),+ )+ .unwrap();assert_eq!(bangs.get(&query), None);- let query = Query::parse("!no bangs", schema.clone(), &tokenizer_manager).unwrap();+ let query = Query::parse(+ "!no bangs",+ schema.clone(),+ &tokenizer_manager,+ &SignalAggregator::default(),+ )+ .unwrap();assert_eq!(bangs.get(&query), None);- let query = Query::parse("!ty bangs", schema, &tokenizer_manager).unwrap();+ let query = Query::parse(+ "!ty bangs",+ schema,+ &tokenizer_manager,+ &SignalAggregator::default(),+ )+ .unwrap();assert_eq!(bangs.get(&query),Some(BangHit {diff --git a/src/frontend/mod.rs b/src/frontend/mod.rsindex bf55af17..458b4bb7 100644--- a/src/frontend/mod.rs+++ b/src/frontend/mod.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use axum::{body::Body, Extension, Router};+use tower_http::compression::CompressionLayer;use crate::{autosuggest::Autosuggest, bangs::Bangs, entity_index::EntityIndex, index::Index,@@ -101,5 +102,6 @@ pub fn router(.route("/about", get(about::route)).route("/privacy-and-happy-lawyers", get(privacy::route)).merge(SpaRouter::new("/static", "static"))- .layer(Extension(state)))+ .layer(Extension(state))+ .layer(CompressionLayer::new()))}diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 14598f0c..14cf46c3 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -269,7 +269,7 @@ pub async fn route(}});- match state.searcher.search(query.as_str(), selected_region) {+ match state.searcher.search(query.as_str(), selected_region, None) {Ok(result) => match result {SearchResult::Websites(result) => {let search_result = resultdiff --git a/src/index.rs b/src/index.rsindex 41e37c84..998b6aaf 100644--- a/src/index.rs+++ b/src/index.rs@@ -292,7 +292,7 @@ impl From<Index> for FrozenIndex {#[cfg(test)]mod tests {- use crate::ranking::Ranker;+ use crate::ranking::{signal_aggregator::SignalAggregator, Ranker};use super::*;@@ -333,9 +333,14 @@ mod tests {let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();let index: Index = deserialized_frozen.into();- let query = Query::parse("website", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector())diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex f7330b88..cf12fd68 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -333,7 +333,7 @@ mod tests {use maplit::hashset;use crate::{- ranking::Ranker,+ ranking::{signal_aggregator::SignalAggregator, Ranker},webpage::{region::RegionCount, Link},};@@ -344,9 +344,14 @@ mod tests {#[test]fn simple_search() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("website", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector())@@ -391,9 +396,10 @@ mod tests {"this query should not match",index.schema(),index.tokenizers(),+ &SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -427,9 +433,14 @@ mod tests {#[test]fn english_stemming() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("runner", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "runner",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -463,9 +474,14 @@ mod tests {#[test]fn stemmed_query_english() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("runners", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "runners",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -499,9 +515,14 @@ mod tests {#[test]fn not_searchable_backlinks() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("great site", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "great site",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -564,9 +585,14 @@ mod tests {#[test]fn limited_top_docs() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("runner", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "runner",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());for _ in 0..100 {index@@ -602,9 +628,14 @@ mod tests {#[test]fn host_search() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query =- Query::parse("dr", index.schema(), index.tokenizers()).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "dr",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -686,9 +717,14 @@ mod tests {let mut index = index1.merge(index2);index.commit().unwrap();- let query = Query::parse("website", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector())@@ -702,9 +738,14 @@ mod tests {#[test]fn match_across_fields() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("example test", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "example test",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector())@@ -745,9 +786,14 @@ mod tests {#[test]fn fetch_time_ranking() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("test", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "test",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -803,7 +849,7 @@ mod tests {#[test]fn only_show_primary_images_when_relevant() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let ranker = Ranker::new(RegionCount::default());+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let mut webpage = Webpage::new(&format!(@@ -832,8 +878,13 @@ mod tests {index.insert(webpage).expect("failed to parse webpage");index.commit().expect("failed to commit index");- let query = Query::parse("website", index.schema(), index.tokenizers())- .expect("Failed to parse query");+ let query = Query::parse(+ "website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");let result = index.search(&query, ranker.collector())@@ -851,8 +902,13 @@ mod tests {}));- let query = Query::parse("best website", index.schema(), index.tokenizers())- .expect("Failed to parse query");+ let query = Query::parse(+ "best website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");let result = index.search(&query, ranker.collector())diff --git a/src/query/mod.rs b/src/query/mod.rsindex 3ab4d47d..4f8b2839 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::Result;+use crate::{ranking::signal_aggregator::SignalAggregator, Result};use std::{collections::HashMap, sync::Arc};use tantivy::{query::{AllQuery, BooleanQuery, Occur},@@ -39,6 +39,7 @@ impl Query {query: &str,schema: Arc<Schema>,tokenizer_manager: &TokenizerManager,+ aggregator: &SignalAggregator,) -> Result<Query> {let parsed_terms = parser::parse(query);@@ -58,9 +59,11 @@ impl Query {let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =schema.fields().collect();+ let field_boost = aggregator.field_boosts();+let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms.iter()- .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager))+ .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager, field_boost)).collect();queries.push((Occur::Should, Box::new(AllQuery)));@@ -128,6 +131,7 @@ mod tests {"this is a simple query the the the the the the the the the the the the the",Arc::clone(&schema),&TokenizerManager::new(),+ &SignalAggregator::default(),).expect("Failed to parse query");@@ -161,6 +165,7 @@ mod tests {" this is a simple query ",Arc::clone(&schema),&TokenizerManager::new(),+ &SignalAggregator::default(),).expect("Failed to parse query");@@ -180,20 +185,31 @@ mod tests {fn parse_weird_characters() {let schema = Arc::new(create_schema());- let terms = Query::parse("123", Arc::clone(&schema), &TokenizerManager::new())- .expect("Failed to parse query")- .simple_terms();+ let terms = Query::parse(+ "123",+ Arc::clone(&schema),+ &TokenizerManager::new(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query")+ .simple_terms();assert_eq!(terms, vec!["123".to_string()]);- let terms = Query::parse("123 33", Arc::clone(&schema), &TokenizerManager::new())- .expect("Failed to parse query")- .simple_terms();+ let terms = Query::parse(+ "123 33",+ Arc::clone(&schema),+ &TokenizerManager::new(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query")+ .simple_terms();assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);let terms = Query::parse("term! term# $",Arc::clone(&schema),&TokenizerManager::new(),+ &SignalAggregator::default(),).expect("Failed to parse query").simple_terms();@@ -206,9 +222,14 @@ mod tests {#[test]fn not_query() {let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("test -website", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "test -website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());index.insert(Webpage::new(@@ -298,9 +319,14 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let query = Query::parse("test site:first.com", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "test site:first.com",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -312,9 +338,10 @@ mod tests {"test site:www.first.com",index.schema(),index.tokenizers(),+ &SignalAggregator::default(),).expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -322,9 +349,14 @@ mod tests {assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.first.com");- let query = Query::parse("test -site:first.com", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "test -site:first.com",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -375,9 +407,14 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let query = Query::parse("intitle:website", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "intitle:website",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -428,9 +465,14 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let query = Query::parse("test inurl:forum", index.schema(), index.tokenizers())- .expect("Failed to parse query");- let ranker = Ranker::new(RegionCount::default());+ let query = Query::parse(+ "test inurl:forum",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(RegionCount::default(), SignalAggregator::default());let result = index.search(&query, ranker.collector()).expect("Search failed");@@ -443,8 +485,13 @@ mod tests {fn empty_query() {let schema = Arc::new(create_schema());- let query = Query::parse("", Arc::clone(&schema), &TokenizerManager::new())- .expect("Failed to parse query");+ let query = Query::parse(+ "",+ Arc::clone(&schema),+ &TokenizerManager::new(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");assert!(query.is_empty())}diff --git a/src/query/parser.rs b/src/query/parser.rsindex dbae4f62..1ef5417a 100644--- a/src/query/parser.rs+++ b/src/query/parser.rs@@ -22,6 +22,7 @@ use tantivy::{use crate::{bangs::BANG_PREFIX,+ ranking::signal_aggregator::FieldBoost,schema::{Field, ALL_FIELDS},};@@ -54,6 +55,7 @@ fn simple_into_tantivy(term: &str,fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],tokenizer_manager: &TokenizerManager,+ field_boost: &FieldBoost,) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {let (backlink_field, backlink_field_entry) = fields.iter()@@ -67,6 +69,7 @@ fn simple_into_tantivy(term,fields,tokenizer_manager,+ field_boost,))),),(@@ -75,6 +78,7 @@ fn simple_into_tantivy(backlink_field,backlink_field_entry,tokenizer_manager,+ field_boost,term,)),),@@ -86,14 +90,17 @@ impl Term {&self,fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],tokenizer_manager: &TokenizerManager,+ field_boost: &FieldBoost,) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {match self {- Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),+ Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager, field_boost),Term::Not(subterm) => vec![(Occur::MustNot,- Box::new(BooleanQuery::new(- subterm.as_tantivy_query(fields, tokenizer_manager),- )),+ Box::new(BooleanQuery::new(subterm.as_tantivy_query(+ fields,+ tokenizer_manager,+ field_boost,+ ))),)],Term::Site(site) => vec![(Occur::Must,@@ -101,6 +108,7 @@ impl Term {site,fields,tokenizer_manager,+ field_boost,))),)],Term::Title(title) => {@@ -112,7 +120,7 @@ impl Term {.unwrap();vec![(Occur::Must,- Term::tantivy_term_query(field, entry, tokenizer_manager, title),+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, title),)]}Term::Body(body) => {@@ -124,7 +132,7 @@ impl Term {.unwrap();vec![(Occur::Must,- Term::tantivy_term_query(field, entry, tokenizer_manager, body),+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, body),)]}Term::Url(url) => {@@ -134,7 +142,7 @@ impl Term {.unwrap();vec![(Occur::Must,- Term::tantivy_term_query(field, entry, tokenizer_manager, url),+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, url),)]}Term::PossibleBang(text) => {@@ -142,7 +150,7 @@ impl Term {term.push(BANG_PREFIX);term.push_str(text);- simple_into_tantivy(&term, fields, tokenizer_manager)+ simple_into_tantivy(&term, fields, tokenizer_manager, field_boost)}}}@@ -151,6 +159,7 @@ impl Term {term: &str,fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],tokenizer_manager: &TokenizerManager,+ field_boost: &FieldBoost,) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {fields.iter()@@ -159,7 +168,7 @@ impl Term {.map(|(field, entry)| {(Occur::Should,- Term::tantivy_term_query(field, entry, tokenizer_manager, term),+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, term),)}).collect()@@ -169,6 +178,7 @@ impl Term {term: &str,fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],tokenizer_manager: &TokenizerManager,+ field_boost: &FieldBoost,) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {fields.iter()@@ -182,7 +192,7 @@ impl Term {.map(|(field, entry)| {(Occur::Should,- Term::tantivy_term_query(field, entry, tokenizer_manager, term),+ Term::tantivy_term_query(field, entry, tokenizer_manager, field_boost, term),)}).collect()@@ -192,6 +202,7 @@ impl Term {field: &tantivy::schema::Field,entry: &tantivy::schema::FieldEntry,tokenizer_manager: &TokenizerManager,+ field_boost: &FieldBoost,term: &str,) -> Box<dyn tantivy::query::Query + 'static> {let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);@@ -206,7 +217,8 @@ impl Term {IndexRecordOption::WithFreqsAndPositions,))};- let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);++ let boost = field_boost.get(&ALL_FIELDS[field.field_id() as usize]) as f32;Box::new(BoostQuery::new(processed_query, boost))}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex fdeb88ed..24660e6a 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -16,34 +16,35 @@use std::sync::Arc;-use crate::schema::{Field, CENTRALITY_SCALING};use crate::webpage::region::{Region, RegionCount};use chrono::Utc;use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};-use tantivy::fastfield::{Column, DynamicFastFieldReader};use tantivy::{DocId, Score, SegmentReader};+use super::signal_aggregator::SignalAggregator;+pub(crate) struct InitialScoreTweaker {region_count: Arc<RegionCount>,selected_region: Option<Region>,+ aggregator: SignalAggregator,}impl InitialScoreTweaker {- pub fn new(region_count: Arc<RegionCount>, selected_region: Option<Region>) -> Self {+ pub fn new(+ region_count: Arc<RegionCount>,+ selected_region: Option<Region>,+ aggregator: SignalAggregator,+ ) -> Self {Self {region_count,selected_region,+ aggregator,}}}pub(crate) struct InitialSegmentScoreTweaker {- centrality_reader: DynamicFastFieldReader<u64>,- is_homepage_reader: DynamicFastFieldReader<u64>,- fetch_time_ms_reader: DynamicFastFieldReader<u64>,- update_timestamp_reader: DynamicFastFieldReader<u64>,- num_trackers_reader: DynamicFastFieldReader<u64>,- region_reader: DynamicFastFieldReader<u64>,+ aggregator: SignalAggregator,region_count: Arc<RegionCount>,current_timestamp: f64,selected_region: Option<Region>,@@ -53,111 +54,28 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {type Child = InitialSegmentScoreTweaker;fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {- let centrality_field = segment_reader- .schema()- .get_field(Field::Centrality.as_str())- .expect("Faild to load centrality field");- let centrality_reader = segment_reader- .fast_fields()- .u64(centrality_field)- .expect("Failed to get centrality fast-field reader");-- let is_homepage_field = segment_reader- .schema()- .get_field(Field::IsHomepage.as_str())- .expect("Faild to load is_homepage field");- let is_homepage_reader = segment_reader- .fast_fields()- .u64(is_homepage_field)- .expect("Failed to get is_homepage fast-field reader");-- let fetch_time_ms_field = segment_reader- .schema()- .get_field(Field::FetchTimeMs.as_str())- .expect("Faild to load fetch_time_ms field");- let fetch_time_ms_reader = segment_reader- .fast_fields()- .u64(fetch_time_ms_field)- .expect("Failed to get fetch_time_ms fast-field reader");-- let update_timestamp_field = segment_reader- .schema()- .get_field(Field::LastUpdated.as_str())- .expect("Faild to load last_updated field");- let update_timestamp_reader = segment_reader- .fast_fields()- .u64(update_timestamp_field)- .expect("Failed to get last_updated fast-field reader");-- let num_trackers_field = segment_reader- .schema()- .get_field(Field::NumTrackers.as_str())- .expect("Faild to load num_trackers field");- let num_trackers_reader = segment_reader- .fast_fields()- .u64(num_trackers_field)- .expect("Failed to get num_trackers fast-field reader");-- let region_field = segment_reader- .schema()- .get_field(Field::Region.as_str())- .expect("Faild to load region field");- let region_reader = segment_reader- .fast_fields()- .u64(region_field)- .expect("Failed to get region fast-field reader");+ let mut aggregator = SignalAggregator::new_like(&self.aggregator);+ aggregator.register_readers(segment_reader);let current_timestamp = Utc::now().timestamp() as f64;Ok(InitialSegmentScoreTweaker {- centrality_reader,- is_homepage_reader,- fetch_time_ms_reader,- update_timestamp_reader,- num_trackers_reader,+ aggregator,current_timestamp,- region_reader,selected_region: self.selected_region,region_count: Arc::clone(&self.region_count),})}}-fn time_to_score(time: f64) -> f64 {- 1.0 / ((time + 1.0).log2())-}--fn region_score(- region_count: &RegionCount,- selected_region: Option<Region>,- webpage_region: Region,-) -> f64 {- let boost = selected_region.map_or(- 0.0,- |region| if region == webpage_region { 50.0 } else { 0.0 },- );-- boost + region_count.score(&webpage_region)-}-impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {fn score(&mut self, doc: DocId, score: Score) -> f64 {- let score = score as f64;- let centrality: f64 =- self.centrality_reader.get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64;- let is_homepage = self.is_homepage_reader.get_val(doc as u64) as f64;- let fetch_time_ms = self.fetch_time_ms_reader.get_val(doc as u64) as f64;- let update_timestamp = self.update_timestamp_reader.get_val(doc as u64) as f64;- let hours_since_update = (self.current_timestamp - update_timestamp).max(0.000001) / 3600.0;- let num_trackers = self.num_trackers_reader.get_val(doc as u64) as f64;- let region = Region::from_id(self.region_reader.get_val(doc as u64));-- (3.0 * score)- + (3200.0 * centrality)- + (1.0 * is_homepage)- + (1.0 / (fetch_time_ms + 1.0))- + (1500.0 * time_to_score(hours_since_update))- + (200.0 * (1.0 / (num_trackers + 1.0)))- + (25.0 * region_score(&self.region_count, self.selected_region, region))+ self.aggregator.score(+ doc,+ score,+ &self.region_count,+ self.current_timestamp,+ self.selected_region,+ )}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 7a28349a..ca3cc98c 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -16,6 +16,7 @@pub mod centrality_store;mod initial;+pub mod signal_aggregator;use std::sync::Arc;@@ -24,16 +25,20 @@ use tantivy::collector::{Collector, TopDocs};use crate::webpage::region::{Region, RegionCount};+use self::signal_aggregator::SignalAggregator;+pub struct Ranker {region_count: Arc<RegionCount>,selected_region: Option<Region>,+ aggregator: SignalAggregator,}impl Ranker {- pub fn new(region_count: RegionCount) -> Self {+ pub fn new(region_count: RegionCount, aggregator: SignalAggregator) -> Self {Ranker {region_count: Arc::new(region_count),selected_region: None,+ aggregator,}}@@ -43,8 +48,11 @@ impl Ranker {}pub fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {- let score_tweaker =- InitialScoreTweaker::new(Arc::clone(&self.region_count), self.selected_region);+ let score_tweaker = InitialScoreTweaker::new(+ Arc::clone(&self.region_count),+ self.selected_region,+ SignalAggregator::new_like(&self.aggregator),+ );TopDocs::with_limit(20).tweak_score(score_tweaker)}}@@ -108,7 +116,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);let result = searcher- .search("example", None)+ .search("example", None, None).expect("Search failed").into_websites().unwrap();@@ -183,7 +191,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);let result = searcher- .search("dr dk", None)+ .search("dr dk", None, None).expect("Search failed").into_websites().unwrap();@@ -242,7 +250,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);let result = searcher- .search("title", None)+ .search("title", None, None).expect("Search failed").into_websites().unwrap();@@ -308,7 +316,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);let result = searcher- .search("test", None)+ .search("test", None, None).expect("Search failed").into_websites().unwrap();@@ -365,7 +373,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);let result = searcher- .search("test", None)+ .search("test", None, None).expect("Search failed").into_websites().unwrap();diff --git a/src/ranking/signal_aggregator/ast.rs b/src/ranking/signal_aggregator/ast.rsnew file mode 100644index 00000000..cf757c38--- /dev/null+++ b/src/ranking/signal_aggregator/ast.rs@@ -0,0 +1,70 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use lalrpop_util::lalrpop_mod;++lalrpop_mod!(pub parser, "/ranking/signal_aggregator/parser.rs");++pub static PARSER: once_cell::sync::Lazy<parser::AlterationsParser> =+ once_cell::sync::Lazy::new(parser::AlterationsParser::new);++#[derive(Debug, PartialEq, Eq)]+pub enum Target {+ Signal(String),+ Field(String),+}++#[derive(Debug, PartialEq)]+pub struct Alteration {+ pub target: Target,+ pub score: f64,+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let res: Vec<Alteration> = PARSER+ .parse(+ r#"+ @host_centrality = 3+ @bm25 = 100+ @field_url = 2+ "#,+ )+ .unwrap();++ assert_eq!(+ res,+ vec![+ Alteration {+ target: Target::Signal("host_centrality".to_string()),+ score: 3.0,+ },+ Alteration {+ target: Target::Signal("bm25".to_string()),+ score: 100.0+ },+ Alteration {+ target: Target::Field("url".to_string()),+ score: 2.0+ }+ ]+ )+ }+}diff --git a/src/ranking/signal_aggregator/mod.rs b/src/ranking/signal_aggregator/mod.rsnew file mode 100644index 00000000..fba1c132--- /dev/null+++ b/src/ranking/signal_aggregator/mod.rs@@ -0,0 +1,285 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod ast;++use crate::{Error, Result};+use std::{collections::HashMap, sync::Arc};++use strum::{EnumIter, IntoEnumIterator};+use tantivy::{+ fastfield::{Column, DynamicFastFieldReader},+ DocId, Score, SegmentReader,+};++use crate::{+ schema::{Field, ALL_FIELDS, CENTRALITY_SCALING},+ webpage::region::{Region, RegionCount},+};++use self::ast::{Alteration, Target, PARSER};++#[derive(Debug, Clone, PartialEq, Eq, Hash, EnumIter)]+pub enum Signal {+ Bm25,+ HostCentrality,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ NumTrackers,+ Region,+}++impl Signal {+ fn from_field(field: &Field) -> Option<Self> {+ match field {+ Field::IsHomepage => Some(Signal::IsHomepage),+ Field::Centrality => Some(Signal::HostCentrality),+ Field::FetchTimeMs => Some(Signal::FetchTimeMs),+ Field::LastUpdated => Some(Signal::UpdateTimestamp),+ Field::NumTrackers => Some(Signal::NumTrackers),+ Field::Region => Some(Signal::Region),+ _ => None,+ }+ }++ fn has_fast_reader(&self) -> bool {+ !matches!(self, Signal::Bm25)+ }++ fn value(+ &self,+ doc: DocId,+ bm25: Score,+ fastfield: Option<&DynamicFastFieldReader<u64>>,+ region_count: &Arc<RegionCount>,+ current_timestamp: f64,+ selected_region: Option<Region>,+ ) -> f64 {+ match self {+ Signal::Bm25 => bm25 as f64,+ Signal::HostCentrality => {+ fastfield.unwrap().get_val(doc as u64) as f64 / CENTRALITY_SCALING as f64+ }+ Signal::IsHomepage => fastfield.unwrap().get_val(doc as u64) as f64,+ Signal::FetchTimeMs => {+ let fetch_time_ms = fastfield.unwrap().get_val(doc as u64) as f64;+ 1.0 / (fetch_time_ms + 1.0)+ }+ Signal::UpdateTimestamp => {+ let update_timestamp = fastfield.unwrap().get_val(doc as u64) as f64;+ let hours_since_update =+ (current_timestamp - update_timestamp).max(0.000001) / 3600.0;+ 1.0 / ((hours_since_update + 1.0).log2())+ }+ Signal::NumTrackers => {+ let num_trackers = fastfield.unwrap().get_val(doc as u64) as f64;+ 1.0 / (num_trackers + 1.0)+ }+ Signal::Region => {+ let webpage_region = Region::from_id(fastfield.unwrap().get_val(doc as u64));++ let boost =+ selected_region.map_or(+ 0.0,+ |region| if region == webpage_region { 50.0 } else { 0.0 },+ );++ boost + region_count.score(&webpage_region)+ }+ }+ }++ fn default_coefficient(&self) -> f64 {+ match self {+ Signal::Bm25 => 3.0,+ Signal::HostCentrality => 3200.0,+ Signal::IsHomepage => 1.0,+ Signal::FetchTimeMs => 1.0,+ Signal::UpdateTimestamp => 1500.0,+ Signal::NumTrackers => 200.0,+ Signal::Region => 25.0,+ }+ }++ fn from_string(name: String) -> Option<Signal> {+ match name.as_str() {+ "bm25" => Some(Signal::Bm25),+ "host_centrality" => Some(Signal::HostCentrality),+ _ => None,+ }+ }+}++fn fastfield_reader(segment_reader: &SegmentReader, field: &Field) -> DynamicFastFieldReader<u64> {+ let tv_field = segment_reader+ .schema()+ .get_field(field.as_str())+ .unwrap_or_else(|| panic!("Faild to load {} field", field.as_str()));++ segment_reader+ .fast_fields()+ .u64(tv_field)+ .unwrap_or_else(|_| panic!("Failed to get {} fast-field reader", field.as_str()))+}++#[derive(Debug, Clone)]+pub struct FieldBoost(HashMap<Field, f64>);++#[derive(Debug, Clone)]+pub struct SignalCoefficient(HashMap<Signal, f64>);++impl SignalCoefficient {+ pub fn get(&self, signal: &Signal) -> f64 {+ self.0+ .get(signal)+ .copied()+ .unwrap_or_else(|| signal.default_coefficient())+ }+}++impl FieldBoost {+ pub fn get(&self, field: &Field) -> f64 {+ self.0+ .get(field)+ .copied()+ .or_else(|| field.boost().map(|s| s as f64))+ .unwrap_or(1.0)+ }+}++pub struct SignalAggregator {+ readers: HashMap<Signal, DynamicFastFieldReader<u64>>,+ signal_coefficients: SignalCoefficient,+ field_boost: FieldBoost,+}++impl std::fmt::Debug for SignalAggregator {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("SignalAggregator")+ .field("signal_coefficients", &self.signal_coefficients)+ .field("field_boost", &self.field_boost)+ .finish()+ }+}++impl Default for SignalAggregator {+ fn default() -> Self {+ Self::new(HashMap::new(), HashMap::new())+ }+}++impl SignalAggregator {+ pub fn new(coefficients: HashMap<Signal, f64>, boosts: HashMap<Field, f64>) -> Self {+ let signal_coefficients = SignalCoefficient(coefficients);+ let field_boost = FieldBoost(boosts);++ Self {+ readers: HashMap::new(),+ signal_coefficients,+ field_boost,+ }+ }++ pub fn new_like(other: &SignalAggregator) -> Self {+ Self {+ readers: HashMap::new(),+ signal_coefficients: other.signal_coefficients.clone(),+ field_boost: other.field_boost.clone(),+ }+ }++ pub fn register_readers(&mut self, segment_reader: &SegmentReader) {+ for field in &ALL_FIELDS {+ if let Some(signal) = Signal::from_field(field) {+ if signal.has_fast_reader() {+ self.mut_readers()+ .insert(signal, fastfield_reader(segment_reader, field));+ }+ }+ }+ }++ pub fn score(+ &self,+ doc: DocId,+ bm25: Score,+ region_count: &Arc<RegionCount>,+ current_timestamp: f64,+ selected_region: Option<Region>,+ ) -> f64 {+ Signal::iter()+ .map(|signal| {+ self.coefficients().get(&signal)+ * signal.value(+ doc,+ bm25,+ self.readers().get(&signal),+ region_count,+ current_timestamp,+ selected_region,+ )+ })+ .sum()+ }++ pub fn coefficients(&self) -> &SignalCoefficient {+ &self.signal_coefficients+ }++ pub fn field_boosts(&self) -> &FieldBoost {+ &self.field_boost+ }++ pub fn mut_readers(&mut self) -> &mut HashMap<Signal, DynamicFastFieldReader<u64>> {+ &mut self.readers+ }++ pub fn readers(&self) -> &HashMap<Signal, DynamicFastFieldReader<u64>> {+ &self.readers+ }+}++impl From<Vec<Alteration>> for SignalAggregator {+ fn from(alterations: Vec<Alteration>) -> Self {+ let mut coefficients = HashMap::new();+ let mut boosts = HashMap::new();++ for alteration in alterations {+ match alteration.target {+ Target::Signal(name) => {+ if let Some(signal) = Signal::from_string(name) {+ coefficients.insert(signal, alteration.score);+ }+ }+ Target::Field(name) => {+ if let Some(field) = Field::from_string(name) {+ boosts.insert(field, alteration.score);+ }+ }+ }+ }++ Self::new(coefficients, boosts)+ }+}++pub fn parse(program: &str) -> Result<SignalAggregator> {+ match PARSER.parse(program) {+ Ok(alterations) => Ok(SignalAggregator::from(alterations)),+ Err(_) => Err(Error::Parse),+ }+}diff --git a/src/ranking/signal_aggregator/parser.lalrpop b/src/ranking/signal_aggregator/parser.lalrpopnew file mode 100644index 00000000..ebdca900--- /dev/null+++ b/src/ranking/signal_aggregator/parser.lalrpop@@ -0,0 +1,25 @@+use std::str::FromStr;+use crate::ranking::signal_aggregator::ast::*;++grammar;++pub Alterations: Vec<Alteration> = <Alteration*> => <>;++Alteration: Alteration = {+ "@" <target:Target> "=" <score:Num> => Alteration {<>}+}++Target: Target = {+ <Name> => {+ if let Some(name) = <>.strip_prefix("field_") {+ Target::Field(name.to_string())+ } else {+ Target::Signal(<>)+ }+ },+}++Name: String = r"[a-zA-Z_]+[0-9]*" => <>.to_string();+++Num: f64 = r"[0-9]+" => f64::from_str(<>).unwrap();\ No newline at end of filediff --git a/src/schema.rs b/src/schema.rsindex 764a609e..bb0fa4d7 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -22,7 +22,7 @@ use crate::tokenizer::{Normal, Stemmed};pub const CENTRALITY_SCALING: u64 = 1_000_000_000;-#[derive(Clone)]+#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub enum Field {Title,CleanBody,@@ -44,6 +44,7 @@ pub enum Field {NumTrackers,Region,}+pub static ALL_FIELDS: [Field; 19] = [Field::Title,Field::CleanBody,@@ -189,6 +190,31 @@ impl Field {pub fn is_searchable(&self) -> bool {!matches!(self, Field::PrimaryImage | Field::BacklinkText)}++ pub fn from_string(name: String) -> Option<Field> {+ match name.as_str() {+ "title" => Some(Field::Title),+ "body" => Some(Field::CleanBody),+ "url" => Some(Field::Url),+ "host" => Some(Field::Host),+ "backlink_text" => Some(Field::BacklinkText),+ "centrality" => Some(Field::Centrality),+ "stemmed_title" => Some(Field::StemmedTitle),+ "stemmed_body" => Some(Field::StemmedCleanBody),+ "domain" => Some(Field::Domain),+ "domain_if_homepage" => Some(Field::DomainIfHomepage),+ "is_homepage" => Some(Field::IsHomepage),+ "fetch_time_ms" => Some(Field::FetchTimeMs),+ "primary_image_uuid" => Some(Field::PrimaryImage),+ "last_updated" => Some(Field::LastUpdated),+ "description" => Some(Field::Description),+ "all_body" => Some(Field::AllBody),+ "stemmed_all_body" => Some(Field::StemmedAllBody),+ "num_trackers" => Some(Field::NumTrackers),+ "region" => Some(Field::Region),+ _ => None,+ }+ }}pub fn create_schema() -> tantivy::schema::Schema {diff --git a/src/searcher.rs b/src/searcher.rsindex 05f9d34a..dd25c03b 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -25,6 +25,7 @@ use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::InvertedIndexSearchResult;use crate::query::Query;+use crate::ranking::signal_aggregator;use crate::ranking::Ranker;use crate::webpage::region::Region;use crate::webpage::Url;@@ -67,11 +68,27 @@ impl Searcher {}impl Searcher {- pub fn search(&self, query: &str, selected_region: Option<Region>) -> Result<SearchResult> {+ pub fn search(+ &self,+ query: &str,+ selected_region: Option<Region>,+ goggle_program: Option<&str>,+ ) -> Result<SearchResult> {let start = Instant::now();let raw_query = query.to_string();- let query = Query::parse(query, self.index.schema(), self.index.tokenizers())?;+ let aggregator = goggle_program+ .and_then(|program| signal_aggregator::parse(program).ok())+ .unwrap_or_default();++ dbg!(&aggregator);++ let query = Query::parse(+ query,+ self.index.schema(),+ self.index.tokenizers(),+ &aggregator,+ )?;if query.is_empty() {return Err(Error::EmptyQuery);@@ -83,7 +100,7 @@ impl Searcher {}}- let mut ranker = Ranker::new(self.index.region_count.clone());+ let mut ranker = Ranker::new(self.index.region_count.clone(), aggregator);if let Some(region) = selected_region {if region != Region::All {@@ -142,3 +159,121 @@ impl SearchResult {}}}++#[cfg(test)]+mod tests {+ use crate::webpage::Webpage;++ use super::*;++ #[test]+ fn custom_signal_aggregation() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ example+ </body>+ </html>+ "#,+ "https://www.body.com",+ vec![],+ 1.0,+ 20,+ ))+ .expect("failed to parse webpage");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Example website</title>+ </head>+ <body>+ test+ </body>+ </html>+ "#,+ "https://www.title.com",+ vec![],+ 1.0,+ 20,+ ))+ .expect("failed to parse webpage");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Example website</title>+ </head>+ <body>+ test+ </body>+ </html>+ "#,+ "https://www.centrality.com",+ vec![],+ 1.0002,+ 500,+ ))+ .expect("failed to parse webpage");++ index.commit().unwrap();++ let searcher = Searcher::new(index, None, None);++ let res = searcher+ .search(+ "example",+ None,+ Some(+ r#"+ @field_title = 20000000+ @host_centrality = 0+ "#,+ ),+ )+ .unwrap()+ .into_websites()+ .unwrap();++ assert_eq!(res.webpages.num_docs, 3);+ assert_eq!(&res.webpages.documents[0].url, "https://www.title.com");++ let res = searcher+ .search(+ "example",+ None,+ Some(+ r#"+ @field_all_body= 20000000+ @host_centrality = 0+ "#,+ ),+ )+ .unwrap()+ .into_websites()+ .unwrap();++ assert_eq!(res.webpages.num_docs, 3);+ assert_eq!(&res.webpages.documents[0].url, "https://www.body.com");++ let res = searcher+ .search("example", None, Some("@host_centrality= 2000000"))+ .unwrap()+ .into_websites()+ .unwrap();++ assert_eq!(res.webpages.num_docs, 3);+ assert_eq!(&res.webpages.documents[0].url, "https://www.centrality.com");+ }+}diff --git a/src/snippet.rs b/src/snippet.rsindex 55896eef..769a1311 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -347,7 +347,7 @@ Survey in 2016, 2017, and 2018."#;let searcher = Searcher::from(index);let result = searcher- .search("rust language", None)+ .search("rust language", None, None).expect("Search failed").into_websites().unwrap();@@ -387,7 +387,7 @@ Survey in 2016, 2017, and 2018."#;let searcher = Searcher::from(index);let result = searcher- .search("describe", None)+ .search("describe", None, None).expect("Search failed").into_websites().unwrap();