web search done right
Remove online/personal centrality. It simply wasn't good enough and was too computationally expensive. It is better to have a faster search experience and only rely on inbound similarity for the personalization part, as this tends to work quite well while also being way simpler. Kill your darlings and all that.
Mikkel Denker498e78b430ae
diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 7717bd82..476842df 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -113,10 +113,6 @@ harness = falsename = "harmonic-centrality"harness = false-[[bench]]-name = "online-harmonic"-harness = false-[[bench]]name = "abstractive-summarizer"harness = falsediff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rsdeleted file mode 100644index d812ce6f..00000000--- a/core/benches/online-harmonic.rs+++ /dev/null@@ -1,34 +0,0 @@-use criterion::{criterion_group, criterion_main, Criterion};-use stract::{- ranking::centrality_store::HarmonicCentralityStore,- webgraph::{- centrality::{harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality},- WebgraphBuilder,- },-};--const WEBGRAPH_PATH: &str = "data/webgraph";--pub fn criterion_benchmark(c: &mut Criterion) {- let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).open();- let centrality = HarmonicCentrality::calculate(&webgraph);-- let harmonic_centrality_store = HarmonicCentralityStore::open(stract::gen_temp_path());- for (node, centrality) in centrality.host {- harmonic_centrality_store- .host- .insert(*webgraph.node2id(&node).unwrap(), centrality);- }- harmonic_centrality_store.host.flush();-- c.bench_function("Online harmonic centrality calculation", |b| {- b.iter(|| {- for _ in 0..10 {- OnlineHarmonicCentrality::new(&webgraph, &harmonic_centrality_store);- }- })- });-}--criterion_group!(benches, criterion_benchmark);-criterion_main!(benches);diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 4a2ecda9..0c56634d 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -27,12 +27,6 @@ impl Centrality {CentralityStore::build_harmonic(&graph, output_path);}- pub fn build_online<P: AsRef<Path>>(webgraph_path: P, output_path: P) {- tracing::info!("Building online harmonic centrality");- let graph = WebgraphBuilder::new(webgraph_path).open();- CentralityStore::build_online(&graph, output_path);- }-pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {tracing::info!("Building inbound similarity");let graph = WebgraphBuilder::new(webgraph_path).open();diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 6135bb61..0f06ed52 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -95,7 +95,6 @@ fn calculate_centrality() {let out_path = Path::new(DATA_PATH).join("centrality");Centrality::build_harmonic(&webgraph_path, &out_path);- Centrality::build_online(&webgraph_path, &out_path);Centrality::build_similarity(&webgraph_path, &out_path);}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 5b0bc6c3..62c761c1 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -192,12 +192,10 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {.get(&Node::from_url(html.url()).into_host()).cloned();- let mut host_topic = None;let mut dmoz_description = None;if let Some(mapper) = worker.topics.as_ref() {if let Some(info) = mapper.get(&html.url().site().to_string()) {- host_topic = Some(info.topic.clone());dmoz_description = Some(info.description.clone())}}@@ -210,7 +208,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {fetch_time_ms,pre_computed_score: 0.0,node_id,- host_topic,dmoz_description,};diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 120a9c8c..0056c487 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -26,7 +26,6 @@ mod entity;pub mod frontend;pub mod indexer;pub mod search_server;-pub mod topic_centrality;mod webgraph;pub mod webgraph_server;diff --git a/core/src/entrypoint/topic_centrality.rs b/core/src/entrypoint/topic_centrality.rsdeleted file mode 100644index 79658ab8..00000000--- a/core/src/entrypoint/topic_centrality.rs+++ /dev/null@@ -1,41 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use crate::{- human_website_annotations,- index::Index,- webgraph::{- centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},- WebgraphBuilder,- },-};--pub fn run(- index_path: String,- topics_path: String,- webgraph_path: String,- online_harmonic_path: String,- output_path: String,-) {- let index = Index::open(index_path).unwrap();- let topics = human_website_annotations::Mapper::open(topics_path).unwrap();- let webgraph = WebgraphBuilder::new(webgraph_path).open();- let harmonic = OnlineHarmonicCentrality::open(online_harmonic_path).unwrap();-- let centrality: TopicCentrality = TopicCentrality::build(&index, topics, webgraph, harmonic);-- centrality.save(output_path).unwrap();-}diff --git a/core/src/human_website_annotations.rs b/core/src/human_website_annotations.rsindex 6d20c784..6d326421 100644--- a/core/src/human_website_annotations.rs+++ b/core/src/human_website_annotations.rs@@ -55,10 +55,6 @@ impl<const N: usize> Topic<N> {.collect::<Vec<_>>(),}}-- pub(crate) fn as_facet(&self) -> Facet {- Facet::from_path(self.detailed_topics.clone().into_iter())- }}#[derive(Debug, Clone, Serialize, Deserialize)]diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex b00253c0..4c36c797 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -36,7 +36,6 @@ use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};use crate::collector::Hashes;use crate::fastfield_reader::FastFieldReader;-use crate::human_website_annotations::Topic;use crate::query::Query;use crate::ranking::initial::Score;use crate::ranking::pipeline::RankingWebsite;@@ -489,7 +488,6 @@ pub struct RetrievedWebpage {pub description: Option<String>,pub dmoz_description: Option<String>,pub updated_time: Option<NaiveDateTime>,- pub host_topic: Option<Topic>,pub schema_org: Vec<schema_org::Item>,pub region: Region,}@@ -558,13 +556,6 @@ impl From<Document> for RetrievedWebpage {Region::from_id(id)}}- Field::Text(TextField::HostTopic) => {- let facet = value.value.as_facet().unwrap();-- if !facet.is_root() {- webpage.host_topic = Some(facet.clone().into());- }- }Field::Text(TextField::DmozDescription) => {let desc = value.value@@ -862,7 +853,6 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");diff --git a/core/src/main.rs b/core/src/main.rsindex e119720b..c79c6575 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -89,19 +89,6 @@ enum Commands {output_dir: String,},- /// Calculate the topic centrality for websites. We use the DMOZ dataset to- /// determine set of representative websites for each topic. Harmonic centrality approximations- /// are then calculated for each website in the webgraph.- ///- /// This is currently not used in the search engine, so everything related to this might be buggy.- TopicCentrality {- index_path: String,- topics_path: String,- webgraph_path: String,- online_harmonic_path: String,- output_path: String,- },-/// Deploy the crawl coordinator. The crawl coordinator is responsible for/// distributing crawl jobs to the crawles and deciding which urls to crawl next.CrawlCoordinator { config_path: String },@@ -134,7 +121,6 @@ enum AliceOptions {enum CentralityType {All,Harmonic,- OnlineHarmonic,Similarity,}@@ -236,15 +222,11 @@ fn main() -> Result<()> {CentralityType::Harmonic => {entrypoint::Centrality::build_harmonic(webgraph_path, output_path)}- CentralityType::OnlineHarmonic => {- entrypoint::Centrality::build_online(webgraph_path, output_path)- }CentralityType::Similarity => {entrypoint::Centrality::build_similarity(webgraph_path, output_path)}CentralityType::All => {entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);- entrypoint::Centrality::build_online(&webgraph_path, &output_path);entrypoint::Centrality::build_similarity(&webgraph_path, &output_path);}}@@ -326,19 +308,6 @@ fn main() -> Result<()> {dmoz_file,output_path,} => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),- Commands::TopicCentrality {- index_path,- topics_path,- webgraph_path,- online_harmonic_path,- output_path,- } => entrypoint::topic_centrality::run(- index_path,- topics_path,- webgraph_path,- online_harmonic_path,- output_path,- ),Commands::Alice { options } => match options {AliceOptions::Serve { config_path } => {let config: AliceLocalConfig = load_toml_config(config_path);diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex eab037b4..9caf1374 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -267,7 +267,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -296,7 +295,6 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -395,7 +393,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -423,7 +420,6 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -500,7 +496,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -530,7 +525,6 @@ mod tests {fetch_time_ms: 500,node_id: None,dmoz_description: None,- host_topic: None,}).expect("failed to insert webpage");index@@ -558,7 +552,6 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -661,7 +654,6 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,- host_topic: None,dmoz_description: None,node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),})@@ -691,7 +683,6 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,dmoz_description: None,- host_topic: None,node_id: Some(*graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),}).expect("failed to insert webpage");@@ -719,7 +710,6 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,- host_topic: None,dmoz_description: None,node_id: Some(*graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),})@@ -796,7 +786,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -841,7 +830,6 @@ mod tests {fetch_time_ms: 500,node_id: None,dmoz_description: None,- host_topic: None,}).expect("failed to insert webpage");@@ -975,7 +963,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1050,7 +1037,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1080,7 +1066,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1135,7 +1120,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1194,7 +1178,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1356,7 +1339,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1430,7 +1412,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1491,7 +1472,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1521,7 +1501,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");@@ -1620,7 +1599,6 @@ mod tests {pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,};diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 6070d093..7cc995a9 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -23,10 +23,7 @@ use tracing::log::trace;use crate::Result;use crate::{kv::{rocksdb_store::RocksDbStore, Kv},- webgraph::{- centrality::{harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality},- Node, NodeID, Webgraph,- },+ webgraph::{centrality::harmonic::HarmonicCentrality, Node, NodeID, Webgraph},};use super::inbound_similarity::InboundSimilarity;@@ -74,7 +71,6 @@ impl From<CentralityStore> for IndexerCentralityStore {}pub struct SearchCentralityStore {- pub online_harmonic: OnlineHarmonicCentrality,pub inbound_similarity: InboundSimilarity,pub node2id: BTreeMap<Node, NodeID>,}@@ -82,8 +78,6 @@ pub struct SearchCentralityStore {impl SearchCentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))- .unwrap(),inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).unwrap(),node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),@@ -94,7 +88,6 @@ impl SearchCentralityStore {impl From<CentralityStore> for SearchCentralityStore {fn from(store: CentralityStore) -> Self {Self {- online_harmonic: store.online_harmonic,inbound_similarity: store.inbound_similarity,node2id: store.node2id,}@@ -103,7 +96,6 @@ impl From<CentralityStore> for SearchCentralityStore {pub struct CentralityStore {pub harmonic: HarmonicCentralityStore,- pub online_harmonic: OnlineHarmonicCentrality,pub inbound_similarity: InboundSimilarity,pub node2id: BTreeMap<Node, NodeID>,pub base_path: String,@@ -123,9 +115,6 @@ impl CentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),- online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))- .ok()- .unwrap_or_default(),inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).ok().unwrap_or_default(),@@ -163,7 +152,6 @@ impl CentralityStore {pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {Self::build_harmonic(graph, &output_path);- Self::build_online(graph, &output_path);Self::build_similarity(graph, &output_path)}@@ -181,16 +169,6 @@ impl CentralityStore {store}- pub fn build_online<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {- let mut store = CentralityStore::open(output_path.as_ref());-- debug!("Begin online harmonic");- store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);-- store.flush();- store- }-pub fn build_similarity<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {let mut store = CentralityStore::open(output_path.as_ref());@@ -205,11 +183,6 @@ impl CentralityStore {trace!("flushing");self.harmonic.flush();- trace!("saving online harmonic");- self.online_harmonic- .save(Path::new(&self.base_path).join("online_harmonic"))- .unwrap();-trace!("saving inbound similarity");self.inbound_similarity.save(Path::new(&self.base_path).join("inbound_similarity"))diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex f58c9e59..ee23d63b 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -12,7 +12,7 @@// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.+// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::{cmp::{Ordering, Reverse},@@ -61,7 +61,7 @@ impl NodeScorer {impl Scorer {fn calculate_score(&self, node: &NodeID) -> f64 {match self.vectors.get(&node.0) {- Some(vec) => ((self.disliked.len() as f64+ Some(vec) => ((self.disliked.len() as f64)+ (self.liked.iter()@@ -71,8 +71,8 @@ impl Scorer {.disliked.iter().map(|disliked| disliked.sim(node, vec))- .sum::<f64>()))- / self.liked.len() as f64)+ .sum::<f64>())+ / (self.liked.len() + 1) as f64).max(0.0),None => 0.0,}@@ -127,6 +127,7 @@ impl InboundSimilarity {DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES,)}+fn build_with_threshold(graph: &Webgraph,harmonic: &HarmonicCentralityStore,@@ -333,7 +334,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),dmoz_description: None,})@@ -362,7 +362,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),dmoz_description: None,})diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex ebb125cb..2071858c 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -25,9 +25,6 @@ pub mod pipeline;pub mod query_centrality;pub mod signal;-use std::collections::BTreeMap;--use ::optics::SiteRankings;use initial::InitialScoreTweaker;use tantivy::collector::Collector;@@ -37,14 +34,7 @@ use crate::{inverted_index,search_ctx::Ctx,searcher::NUM_RESULTS_PER_PAGE,- webgraph::{- centrality::{- online_harmonic::{OnlineHarmonicCentrality, Scorer},- topic,- },- Node, NodeID,- },- webpage::{region::Region, Url},+ webpage::region::Region,};pub use self::signal::*;@@ -131,37 +121,11 @@ impl Ranker {collector.tweak_score(score_tweaker)}- pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {- self.aggregator.set_topic_scorer(topic_scorer);- }-pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {self.aggregator.set_query_centrality(query_centrality);}}-pub fn online_centrality_scorer(- site_rankings: &SiteRankings,- harmonic: &OnlineHarmonicCentrality,- node2id: &BTreeMap<Node, NodeID>,-) -> Scorer {- let mut liked = Vec::new();- let mut disliked = Vec::new();-- for site in &site_rankings.liked {- if let Some(nodeid) = node2id.get(&Node::from_url(&Url::from(site.clone())).into_host()) {- liked.push(*nodeid);- }- }-- for site in &site_rankings.disliked {- if let Some(nodeid) = node2id.get(&Node::from_url(&Url::from(site.clone())).into_host()) {- disliked.push(*nodeid);- }- }-- harmonic.scorer(&liked, &disliked)-}#[cfg(test)]mod tests {@@ -206,7 +170,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -235,7 +198,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -283,7 +245,6 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -312,7 +273,6 @@ mod tests {// page_centrality: 5.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -359,7 +319,6 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -388,7 +347,6 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -415,7 +373,6 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -462,7 +419,6 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -491,7 +447,6 @@ mod tests {// pre_computed_score: 0.0,// page_centrality: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -534,7 +489,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -572,7 +526,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -620,7 +573,6 @@ mod tests {// page_centrality: 0.0,// pre_computed_score: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -646,7 +598,6 @@ mod tests {// pre_computed_score: 0.0,// page_centrality: 0.0,//- // host_topic: None,// node_id: None,// dmoz_description: None,// })@@ -690,7 +641,6 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -717,7 +667,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -745,7 +694,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -819,7 +767,6 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,- host_topic: None,node_id: None,dmoz_description: None,};@@ -852,7 +799,6 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,- host_topic: None,node_id: None,dmoz_description: None,};@@ -885,7 +831,6 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,- host_topic: None,node_id: None,dmoz_description: None,};@@ -940,7 +885,6 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -969,7 +913,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,node_id: None,dmoz_description: None,})@@ -989,188 +932,4 @@ mod tests {assert_eq!(result.webpages[0].url, "https://www.first.com");assert_eq!(result.webpages[1].url, "https://www.second.com");}-- // #[test]- // fn topic_score() {- // let mut index = Index::temporary().expect("Unable to open index");- // let topic_a = Topic::from_string("/root/topic_a".to_string());- // let topic_b = Topic::from_string("/root/topic_b".to_string());-- // index- // .insert(Webpage {- // html: Html::parse(- // &format!(- // r#"- // <html>- // <head>- // <title>Test website</title>- // </head>- // <body>- // <div>- // {CONTENT} topic_a- // </div>- // <div>- // {}- // </div>- // </body>- // </html>- // "#,- // crate::rand_words(100)- // ),- // "https://www.a.com",- // ),- // backlinks: vec![],- // host_centrality: 1.0,- // fetch_time_ms: 1000,- // pre_computed_score: 0.0,- // page_centrality: 0.0,- //- // host_topic: Some(topic_a.clone()),- // node_id: Some(1.into()),- // dmoz_description: None,- // })- // .expect("failed to insert webpage");- // index- // .insert(Webpage {- // html: Html::parse(- // &format!(- // r#"- // <html>- // <head>- // <title>Test website</title>- // </head>- // <body>- // <div>- // {CONTENT} topic_b- // </div>- // <div>- // {}- // </div>- // </body>- // </html>- // "#,- // crate::rand_words(100)- // ),- // "https://www.b.com",- // ),- // backlinks: vec![],- // host_centrality: 1.0,- // fetch_time_ms: 1000,- // page_centrality: 0.0,- // pre_computed_score: 0.0,- //- // host_topic: Some(topic_b.clone()),- // node_id: Some(2.into()),- // dmoz_description: None,- // })- // .expect("failed to insert webpage");- // index- // .insert(Webpage {- // html: Html::parse(- // &format!(- // r#"- // <html>- // <head>- // <title>Test website</title>- // </head>- // <body>- // <div>- // {CONTENT_2} topic_a topic_b- // </div>- // <div>- // {}- // </div>- // </body>- // </html>- // "#,- // crate::rand_words(100)- // ),- // "https://www.wrong.com",- // ),- // backlinks: vec![],- // host_centrality: 1.01,- // fetch_time_ms: 0,- // page_centrality: 0.0,- // pre_computed_score: 0.0,- //- // host_topic: None,- // node_id: Some(0.into()),- // dmoz_description: None,- // })- // .expect("failed to insert webpage");- // index.commit().expect("failed to commit index");-- // let mut mapper = HashMap::new();- // mapper.insert(- // "www.a.com".to_string(),- // Info {- // description: String::new(),- // topic: topic_a,- // },- // );- // mapper.insert(- // "www.b.com".to_string(),- // Info {- // description: String::new(),- // topic: topic_b,- // },- // );- // let topics = human_website_annotations::Mapper::from(mapper);-- // let mut webgraph = WebgraphBuilder::new(crate::gen_temp_path()).open();-- // webgraph.insert(- // Node::from("www.wrong.com"),- // Node::from("www.a.com"),- // String::new(),- // );- // webgraph.insert(- // Node::from("www.wrong.com"),- // Node::from("www.b.com"),- // String::new(),- // );-- // webgraph.commit();-- // let harmonic = HarmonicCentrality::calculate(&webgraph);-- // let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());- // for (node, centrality) in harmonic.host {- // harmonic_centrality_store- // .host- // .insert(*webgraph.node2id(&node).unwrap(), centrality);- // }- // harmonic_centrality_store.host.flush();-- // let harmonic = OnlineHarmonicCentrality::new(&webgraph, &harmonic_centrality_store);-- // let topic_centrality = TopicCentrality::build(&index, topics, webgraph, harmonic);-- // let mut searcher = LocalSearcher::new(index);- // searcher.set_topic_centrality(topic_centrality);-- // let result = searcher- // .search(&SearchQuery {- // query: "topic_a".to_string(),- // ..Default::default()- // })- // .expect("Search failed");-- // assert_eq!(result.num_hits, 2);- // assert_eq!(result.webpages.len(), 2);- // assert_eq!(result.webpages[0].url, "https://www.a.com");- // assert_eq!(result.webpages[1].url, "https://www.wrong.com");-- // let result = searcher- // .search(&SearchQuery {- // query: "topic_b".to_string(),- // ..Default::default()- // })- // .expect("Search failed");-- // assert_eq!(result.num_hits, 2);- // assert_eq!(result.webpages.len(), 2);- // assert_eq!(result.webpages[0].url, "https://www.b.com");- // assert_eq!(result.webpages[1].url, "https://www.wrong.com");- // }}diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 30fd3451..1976ca3f 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -59,6 +59,11 @@ mod tests {Node::from("https://www.nan.com").into_host(),String::new(),);+ graph.insert(+ Node::from("https://www.second.com").into_host(),+ Node::from("https://www.third.com").into_host(),+ String::new(),+ );graph.commit();@@ -87,7 +92,6 @@ mod tests {page_centrality: 0.0,dmoz_description: None,- host_topic: None,node_id: Some(graph.node2id(&Node::from("https://www.first.com").into_host())@@ -120,7 +124,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- host_topic: None,dmoz_description: None,node_id: Some(graph@@ -155,7 +158,6 @@ mod tests {pre_computed_score: 0.0,dmoz_description: None,- host_topic: None,node_id: Some(*graph.node2id(&Node::from("https://www.third.com").into_host())@@ -212,6 +214,7 @@ mod tests {disliked: vec!["www.second.com".to_string()],blocked: vec!["www.first.com".to_string()],}),+ return_ranking_signals: true,..Default::default()}).expect("Search failed");diff --git a/core/src/ranking/query_centrality.rs b/core/src/ranking/query_centrality.rsindex 615fd920..ad6a2fb3 100644--- a/core/src/ranking/query_centrality.rs+++ b/core/src/ranking/query_centrality.rs@@ -12,29 +12,22 @@// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.+// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::webgraph::{centrality::online_harmonic, NodeID};+use crate::webgraph::NodeID;use super::inbound_similarity;pub struct Scorer {- online_harmonic: online_harmonic::Scorer,inbound_centrality: inbound_similarity::Scorer,}impl Scorer {- pub fn new(- online_harmonic: online_harmonic::Scorer,- inbound_centrality: inbound_similarity::Scorer,- ) -> Self {- Self {- online_harmonic,- inbound_centrality,- }+ pub fn new(inbound_centrality: inbound_similarity::Scorer) -> Self {+ Self { inbound_centrality }}pub fn score(&self, node: NodeID) -> f64 {- self.online_harmonic.score(node) + self.inbound_centrality.score(&node)+ self.inbound_centrality.score(&node)}}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex b78fa584..52fd98a0 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -22,10 +22,7 @@ use crate::{enum_map::EnumMap,fastfield_reader,schema::{FastField, TextField},- webgraph::{- centrality::{online_harmonic, topic},- NodeID,- },+ webgraph::NodeID,webpage::Webpage,};use optics::ast::RankingTarget;@@ -124,10 +121,6 @@ pub enum Signal {TrackerScore,#[serde(rename = "region")]Region,- #[serde(rename = "personal_centrality")]- PersonalCentrality,- #[serde(rename = "topic_centrality")]- TopicCentrality,#[serde(rename = "query_centrality")]QueryCentrality,#[serde(rename = "inbound_similarity")]@@ -144,7 +137,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 38] = [+pub const ALL_SIGNALS: [Signal; 36] = [Signal::Bm25,Signal::Bm25Title,Signal::Bm25TitleBigrams,@@ -178,8 +171,6 @@ pub const ALL_SIGNALS: [Signal; 38] = [Signal::UpdateTimestamp,Signal::TrackerScore,Signal::Region,- Signal::PersonalCentrality,- Signal::TopicCentrality,Signal::QueryCentrality,Signal::InboundSimilarity,Signal::LambdaMART,@@ -255,14 +246,12 @@ impl Signal {Signal::CrossEncoder => 0.16801589371906178,Signal::HostCentrality => 15.233546236366237,Signal::PageCentrality => 0.0,- Signal::TopicCentrality => 0.0,Signal::QueryCentrality => 0.0,Signal::IsHomepage => 0.0,Signal::FetchTimeMs => 0.0,Signal::UpdateTimestamp => 0.0,Signal::TrackerScore => 0.019542120533715634,Signal::Region => 0.06437975586036043,- Signal::PersonalCentrality => 10.0,Signal::InboundSimilarity => 10.0,Signal::LambdaMART => 10.0,Signal::LinearRegression => 1.0,@@ -355,14 +344,6 @@ impl Signal {.map(Region::from_id).map(|region| score_region(region, signal_aggregator))}- Signal::PersonalCentrality => {- let host_id = self.host_id(signal_aggregator, doc);- host_id.map(|host_id| signal_aggregator.personal_centrality(host_id))- }- Signal::TopicCentrality => {- let host_id = self.host_id(signal_aggregator, doc);- host_id.and_then(|host_id| signal_aggregator.topic_centrality(host_id))- }Signal::QueryCentrality => {let host_id = self.host_id(signal_aggregator, doc);host_id.and_then(|host_id| signal_aggregator.query_centrality(host_id))@@ -501,8 +482,6 @@ impl Signal {| Signal::ProximitySlop4| Signal::ProximitySlop8| Signal::CrossEncoder- | Signal::PersonalCentrality- | Signal::TopicCentrality| Signal::InboundSimilarity| Signal::LambdaMART| Signal::LinearRegression@@ -679,11 +658,9 @@ pub struct SignalAggregator {query: Option<QueryData>,query_signal_coefficients: Option<SignalCoefficient>,segment_reader: Option<SegmentReader>,- personal_centrality: Option<Arc<online_harmonic::Scorer>>,inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,- topic_scorer: Option<topic::Scorer>,query_centrality: Option<Arc<query_centrality::Scorer>>,region_count: Option<Arc<RegionCount>>,selected_region: Option<Region>,@@ -697,11 +674,9 @@ impl Clone for SignalAggregator {query: self.query.clone(),query_signal_coefficients: self.query_signal_coefficients.clone(),segment_reader: None,- personal_centrality: self.personal_centrality.clone(),inbound_similariy: self.inbound_similariy.clone(),fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),update_time_cache: self.update_time_cache.clone(),- topic_scorer: self.topic_scorer.clone(),query_centrality: self.query_centrality.clone(),region_count: self.region_count.clone(),selected_region: self.selected_region,@@ -754,12 +729,10 @@ impl SignalAggregator {Self {segment_reader: None,- personal_centrality: None,inbound_similariy: None,query_signal_coefficients,fetch_time_ms_cache,update_time_cache,- topic_scorer: None,query_centrality: None,region_count: None,selected_region: None,@@ -935,18 +908,10 @@ impl SignalAggregator {Ok(())}- pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {- self.topic_scorer = Some(topic_scorer);- }-pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {self.query_centrality = Some(Arc::new(query_centrality));}- pub fn set_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {- self.personal_centrality = Some(Arc::new(personal_centrality));- }-pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {self.inbound_similariy = Some(Arc::new(scorer));}@@ -967,25 +932,12 @@ impl SignalAggregator {self.linear_regression = Some(linear_model);}- pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {- self.topic_scorer- .as_ref()- .map(|scorer| scorer.score(host_id))- }-pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {self.query_centrality.as_ref().map(|scorer| scorer.score(host_id))}- pub fn personal_centrality(&self, host_id: NodeID) -> f64 {- self.personal_centrality- .as_ref()- .map(|scorer| scorer.score(host_id))- .unwrap_or_default()- }-pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {self.inbound_similariy.as_ref()diff --git a/core/src/schema.rs b/core/src/schema.rsindex 27edc423..671a15ef 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -44,7 +44,6 @@ pub enum TextField {TitleIfHomepage,BacklinkText,Description,- HostTopic,DmozDescription,SchemaOrgJson,FlattenedSchemaOrgJson,@@ -80,7 +79,6 @@ impl TextField {TextField::TitleIfHomepage => Tokenizer::default(),TextField::BacklinkText => Tokenizer::default(),TextField::Description => Tokenizer::default(),- TextField::HostTopic => Tokenizer::default(),TextField::DmozDescription => Tokenizer::default(),TextField::SchemaOrgJson => Tokenizer::Identity(Identity {}),TextField::FlattenedSchemaOrgJson => Tokenizer::Json(JsonField),@@ -118,7 +116,6 @@ impl TextField {TextField::TitleIfHomepage => false,TextField::BacklinkText => false,TextField::Description => true,- TextField::HostTopic => false,TextField::DmozDescription => true,TextField::SchemaOrgJson => false,TextField::FlattenedSchemaOrgJson => true,@@ -148,7 +145,6 @@ impl TextField {TextField::Description => "description",TextField::TitleIfHomepage => "title_if_homepage",TextField::AllBody => "all_body",- TextField::HostTopic => "host_topic",TextField::DmozDescription => "dmoz_description",TextField::SchemaOrgJson => "schema_org_json",TextField::FlattenedSchemaOrgJson => "flattened_schema_org_json",@@ -237,7 +233,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 52] = [+pub static ALL_FIELDS: [Field; 51] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -255,7 +251,6 @@ pub static ALL_FIELDS: [Field; 52] = [Field::Text(TextField::TitleIfHomepage),Field::Text(TextField::BacklinkText),Field::Text(TextField::Description),- Field::Text(TextField::HostTopic),Field::Text(TextField::DmozDescription),Field::Text(TextField::SchemaOrgJson),Field::Text(TextField::FlattenedSchemaOrgJson),@@ -357,9 +352,6 @@ impl Field {Field::Text(TextField::Description) => {IndexingOption::Text(self.default_text_options().set_stored())}- Field::Text(TextField::HostTopic) => {- IndexingOption::Facet(tantivy::schema::FacetOptions::default())- }Field::Text(TextField::DmozDescription) => {IndexingOption::Text(self.default_text_options().set_stored())}@@ -515,7 +507,6 @@ impl Field {!matches!(self,Field::Text(TextField::BacklinkText)- | Field::Text(TextField::HostTopic)| Field::Text(TextField::SchemaOrgJson)| Field::Text(TextField::CleanBodyBigrams)| Field::Text(TextField::TitleBigrams)@@ -551,7 +542,6 @@ pub fn create_schema() -> tantivy::schema::Schema {IndexingOption::Text(options) => builder.add_text_field(field.name(), options),IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),IndexingOption::Float(options) => builder.add_f64_field(field.name(), options),- IndexingOption::Facet(options) => builder.add_facet_field(field.name(), options),};}@@ -562,7 +552,6 @@ pub enum IndexingOption {Text(tantivy::schema::TextOptions),Integer(tantivy::schema::NumericOptions),Float(tantivy::schema::NumericOptions),- Facet(tantivy::schema::FacetOptions),}pub enum DataType {diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex e96c2918..40608036 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -26,14 +26,11 @@ use crate::ranking::centrality_store::SearchCentralityStore;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};-use crate::ranking::{- online_centrality_scorer, query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS,-};+use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};use crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};use crate::spell::Spell;-use crate::webgraph::centrality::topic::TopicCentrality;use crate::webgraph::Node;use crate::webpage::region::Region;use crate::{inverted_index, Error, Result};@@ -46,7 +43,6 @@ pub struct LocalSearcher {spell: Spell,entity_index: Option<EntityIndex>,centrality_store: Option<SearchCentralityStore>,- topic_centrality: Option<TopicCentrality>,linear_regression: Option<Arc<LinearRegression>>,lambda_model: Option<Arc<LambdaMART>>,}@@ -71,7 +67,6 @@ impl LocalSearcher {spell,entity_index: None,centrality_store: None,- topic_centrality: None,linear_regression: None,lambda_model: None,}@@ -85,10 +80,6 @@ impl LocalSearcher {self.centrality_store = Some(centrality_store);}- pub fn set_topic_centrality(&mut self, topic_centrality: TopicCentrality) {- self.topic_centrality = Some(topic_centrality);- }-pub fn set_linear_model(&mut self, model: LinearRegression) {self.linear_regression = Some(Arc::new(model));}@@ -127,11 +118,6 @@ impl LocalSearcher {}}- if let Some(topic_centrality) = self.topic_centrality.as_ref() {- let topic_scorer = topic_centrality.scorer(query);- ranker.set_topic_scorer(topic_scorer);- }-ranker.de_rank_similar(de_rank_similar);if query_centrality_coeff > 0.0 {@@ -143,16 +129,13 @@ impl LocalSearcher {let top_host_nodes =self.index.top_nodes(query, ctx, ranker.collector(ctx.clone()))?;- if !top_host_nodes.is_empty() {- let harmonic = centrality_store- .online_harmonic- .scorer(&top_host_nodes, &[]);+ if !top_host_nodes.is_empty() {let inbound = centrality_store.inbound_similarity.scorer(&top_host_nodes, &[]);- let query_centrality = query_centrality::Scorer::new(harmonic, inbound);+ let query_centrality = query_centrality::Scorer::new(inbound);ranker.set_query_centrality(query_centrality);}@@ -179,12 +162,6 @@ impl LocalSearcher {let mut aggregator = SignalAggregator::new(Some(&parsed_query));if let Some(store) = &self.centrality_store {- aggregator.set_personal_harmonic(online_centrality_scorer(- parsed_query.site_rankings(),- &store.online_harmonic,- &store.node2id,- ));-let liked_sites: Vec<_> = parsed_query.site_rankings().liked@@ -435,7 +412,6 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}).expect("failed to insert webpage");diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rsindex 459d0eba..df9aa96a 100644--- a/core/src/webgraph/centrality/mod.rs+++ b/core/src/webgraph/centrality/mod.rs@@ -16,5 +16,3 @@pub mod betweenness;pub mod harmonic;-pub mod online_harmonic;-pub mod topic;diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rsdeleted file mode 100644index a4d8b407..00000000--- a/core/src/webgraph/centrality/topic.rs+++ /dev/null@@ -1,242 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{- collections::{HashMap, HashSet},- fs::File,- io::{BufReader, Read, Write},- path::Path,- sync::Arc,-};--use indicatif::ProgressIterator;-use itertools::Itertools;-use rayon::prelude::*;-use serde::{Deserialize, Serialize};-use tantivy::{- query::{BooleanQuery, Occur, QueryClone, TermQuery},- schema::IndexRecordOption,-};--use crate::{- collector::TopDocs,- human_website_annotations::Mapper,- index::Index,- ranking::{initial::InitialScoreTweaker, SignalAggregator},- schema::{FastField, Field, TextField},- webgraph::{NodeID, Webgraph},-};-use crate::{query::Query, Result};--use super::online_harmonic::OnlineHarmonicCentrality;--const TOP_TERMS: usize = 1_000_000;-pub const NUM_TOPICS: usize = 50;--#[derive(Clone)]-pub struct Scorer {- term_weights: Vec<f64>,- host_centrality: Arc<Vec<Vec<f64>>>,-}--impl Scorer {- pub fn score(&self, host: NodeID) -> f64 {- match self.host_centrality.get(host.0 as usize) {- Some(host_score) => host_score- .iter()- .zip_eq(self.term_weights.iter())- .map(|(h, t)| h * t)- .sum(),- None => 0.0,- }- }-}--#[derive(Serialize, Deserialize)]-pub struct TopicCentrality {- terms: HashMap<String, Vec<f64>>,- host_centrality: Arc<Vec<Vec<f64>>>, // host_idx => [f64; N]-}--impl TopicCentrality {- pub fn build(- index: &Index,- topics: Mapper,- webgraph: Webgraph,- harmonic: OnlineHarmonicCentrality,- ) -> Self {- let body_field = index- .schema()- .get_field(Field::Text(TextField::CleanBody).name())- .unwrap();-- let mut term_count: HashMap<String, u64> = HashMap::new();-- let tv_searcher = index.inverted_index.tv_searcher();- for segment in tv_searcher.segment_readers() {- let reader = segment.inverted_index(body_field).unwrap();-- let mut stream = reader.terms().stream().unwrap();-- while let Some((term, info)) = stream.next() {- let term = std::str::from_utf8(term)- .expect("Bytes are not unicode")- .to_string();- *term_count.entry(term).or_default() += info.doc_freq as u64;- }- }-- let top_topics = topics.top_topics(NUM_TOPICS);- let topic_field = tv_searcher- .schema()- .get_field(Field::Text(TextField::HostTopic).name())- .unwrap();- let collector = tantivy::collector::Count;-- let term_scores: HashMap<String, Vec<f64>> = term_count- .into_iter()- .sorted_by_key(|(_, count)| *count)- .rev()- .take(TOP_TERMS)- .progress()- .par_bridge()- .map(|(term, total_doc_freq)| {- let mut scores = vec![0.0; NUM_TOPICS];- for (i, topic) in top_topics.iter().enumerate() {- let facet = topic.as_facet();- let topic = tantivy::Term::from_facet(topic_field, &facet);- let topic = TermQuery::new(topic, IndexRecordOption::Basic).box_clone();-- let term = tantivy::Term::from_field_text(body_field, &term);- let term = TermQuery::new(term, IndexRecordOption::Basic).box_clone();-- let query = BooleanQuery::new(vec![(Occur::Must, topic), (Occur::Must, term)]);-- let topic_freq = tv_searcher.search(&query, &collector).unwrap();- scores[i] = topic_freq as f64 / total_doc_freq as f64;- }-- (term, scores)- })- .collect();-- let mut aggregator = SignalAggregator::new(None);- aggregator.set_region_count(index.region_count.clone());-- let score_tweaker = InitialScoreTweaker::new(- tv_searcher.clone(),- aggregator,- index.inverted_index.fastfield_reader(&tv_searcher),- );-- let graph_node_id = index- .schema()- .get_field(Field::Fast(FastField::HostNodeID).name())- .unwrap();-- let collector =- TopDocs::with_limit(1000, index.inverted_index.fastfield_reader(&tv_searcher))- .tweak_score(score_tweaker);-- let mut nodes: Vec<_> = webgraph.nodes().copied().collect();- nodes.sort();-- let mut node_scores = vec![vec![0.0; NUM_TOPICS]; nodes.len()];-- for (i, topic) in top_topics.iter().enumerate() {- let facet = topic.as_facet();- let topic = tantivy::Term::from_facet(topic_field, &facet);- let query = TermQuery::new(topic, IndexRecordOption::Basic).box_clone();-- let top_sites: HashSet<_> = tv_searcher- .search(&query, &collector)- .unwrap()- .into_iter()- .filter_map(|pointer| {- let doc = tv_searcher.doc(pointer.address.into()).unwrap();- let id = doc.get_first(graph_node_id).unwrap().as_u64().unwrap();- if id == u64::MAX {- None- } else {- let id = id.into();- Some(webgraph.id2node(&id).unwrap())- }- })- .collect();-- let top_sites: Vec<_> = top_sites- .into_iter()- .filter_map(|node| webgraph.node2id(node).copied())- .collect();-- if top_sites.is_empty() {- continue;- }-- let scorer = harmonic.scorer(&top_sites, &[]);-- for node in &nodes {- node_scores[node.0 as usize][i] = scorer.score(*node);- }- }-- Self {- terms: term_scores,- host_centrality: Arc::new(node_scores),- }- }-- pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {- let mut file = File::options()- .create(true)- .truncate(true)- .write(true)- .open(path)?;-- let bytes = bincode::serialize(&self)?;- file.write_all(&bytes)?;-- Ok(())- }-- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut reader = BufReader::new(File::open(path)?);-- let mut bytes = Vec::new();- reader.read_to_end(&mut bytes)?;-- Ok(bincode::deserialize(&bytes)?)- }-- pub fn scorer(&self, query: &Query) -> Scorer {- let terms = query.simple_terms().to_vec();- let mut term_weights = [0.0; NUM_TOPICS];-- for term in terms {- if let Some(score) = self.terms.get(&term) {- term_weights- .iter_mut()- .zip_eq(score.iter())- .for_each(|(acc, score)| *acc += score)- }- }-- Scorer {- term_weights: term_weights.to_vec(),- host_centrality: Arc::clone(&self.host_centrality),- }- }-}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 3d6f5d31..c31de376 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- human_website_annotations::Topic,prehashed::{hash, split_u128},schema::{FastField, TextField},simhash, tokenizer,@@ -118,7 +117,6 @@ pub struct Webpage {pub fetch_time_ms: u64,pub pre_computed_score: f64,pub node_id: Option<NodeID>,- pub host_topic: Option<Topic>,pub dmoz_description: Option<String>,}@@ -135,7 +133,6 @@ impl Webpage {fetch_time_ms: 0,pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: None,}}@@ -227,22 +224,10 @@ impl Webpage {self.node_id.map(|n| n.0).unwrap_or(u64::MAX),);- let facet = self- .host_topic- .map(|topic| topic.as_facet())- .unwrap_or_default();-- doc.add_facet(- schema- .get_field(Field::Text(TextField::HostTopic).name())- .expect("failed to get host_topic field"),- facet,- );-doc.add_text(schema.get_field(Field::Text(TextField::DmozDescription).name())- .expect("failed to get host_topic field"),+ .expect("failed to get dmoz_description field"),dmoz_description.unwrap_or_default(),);@@ -929,7 +914,6 @@ impl Html {doc.add_u64(tantivy_field, hash);}Field::Text(TextField::BacklinkText)- | Field::Text(TextField::HostTopic)| Field::Fast(FastField::HostCentrality)| Field::Fast(FastField::PageCentrality)| Field::Fast(FastField::FetchTimeMs)@@ -1822,7 +1806,6 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: Some("dmoz description".to_string()),};@@ -1856,7 +1839,6 @@ mod tests {fetch_time_ms: 500,pre_computed_score: 0.0,node_id: None,- host_topic: None,dmoz_description: Some("dmoz description".to_string()),};diff --git a/core/testcases/lambdamart.txt b/core/testcases/lambdamart.txtindex 4c214ed7..04bd5a33 100644--- a/core/testcases/lambdamart.txt+++ b/core/testcases/lambdamart.txt@@ -3,16 +3,16 @@ version=v3num_class=1num_tree_per_iteration=1label_index=0-max_feature_idx=37+max_feature_idx=35objective=lambdarank-feature_names=tracker_score personal_centrality query_centrality bm25_domain_no_tokenizer update_timestamp inbound_similarity bm25_stemmed_clean_body proximity_slop_8 bm25_title_if_homepage bm25_title bm25_clean_body_trigrams bm25_all_body bm25_clean_body lambda_mart bm25_domain_if_homepage proximity_slop_0 bm25_title_trigrams proximity_slop_4 cross_encoder bm25_description fetch_time_ms bm25_domain bm25_stemmed_title bm25_site_no_tokenizer region bm25 topic_centrality bm25_title_bigrams bm25_clean_body_bigrams bm25_site bm25_domain_name_if_homepage_no_tokenizer bm25_backlink_text bm25_url proximity_slop_2 proximity_slop_1 host_centrality page_centrality is_homepage-feature_infos=[0.012500000000000001:1] none none none none none [0:70.5282745361] [0:68.376205444299998] none [0:73.018150329600005] [0:100.9539489746] [0:55.664962768599999] [0:60.530643463099999] none none [0:68.376205444299998] [0:70.553833007799994] [0:68.376205444299998] [9.6108999999999997e-06:0.99996737879999997] [0:78.642982482899995] [0:0.20000000000000001] [0:53.968997955299997] [0:71.416839599599996] none [0:0.72049163900000002] [0:723.68379974369998] none [0:90.945877075200002] [0:111.3220901489] [0:52.891826629599997] none none [0:92.092498779300001] [0:68.376205444299998] [0:68.376205444299998] [0:0.034201912000000001] none none-tree_sizes=5469 5458 5457 5464+feature_names=tracker_score query_centrality bm25_domain_no_tokenizer update_timestamp inbound_similarity bm25_stemmed_clean_body proximity_slop_8 bm25_title_if_homepage bm25_title bm25_clean_body_trigrams bm25_all_body bm25_clean_body lambda_mart bm25_domain_if_homepage proximity_slop_0 bm25_title_trigrams proximity_slop_4 cross_encoder bm25_description fetch_time_ms bm25_domain bm25_stemmed_title bm25_site_no_tokenizer region bm25 bm25_title_bigrams bm25_clean_body_bigrams bm25_site bm25_domain_name_if_homepage_no_tokenizer bm25_backlink_text bm25_url proximity_slop_2 proximity_slop_1 host_centrality page_centrality is_homepage+feature_infos=[0.012500000000000001:1] none none none none [0:70.5282745361] [0:68.376205444299998] none [0:73.018150329600005] [0:100.9539489746] [0:55.664962768599999] [0:60.530643463099999] none none [0:68.376205444299998] [0:70.553833007799994] [0:68.376205444299998] [9.6108999999999997e-06:0.99996737879999997] [0:78.642982482899995] [0:0.20000000000000001] [0:53.968997955299997] [0:71.416839599599996] none [0:0.72049163900000002] [0:723.68379974369998] [0:90.945877075200002] [0:111.3220901489] [0:52.891826629599997] none none [0:92.092498779300001] [0:68.376205444299998] [0:68.376205444299998] [0:0.034201912000000001] none none+tree_sizes=5468 5457 5457 5464Tree=0num_leaves=50num_cat=0-split_feature=35 28 35 28 18 12 25 20 22 18 20 0 6 21 0 19 11 27 9 11 11 18 10 28 35 35 0 32 32 19 35 35 35 9 24 11 20 0 28 25 19 12 35 6 20 20 0 0 18+split_feature=33 26 33 26 17 11 24 19 21 17 19 0 5 20 0 18 10 25 8 10 10 17 9 26 33 33 0 30 30 18 33 33 33 8 23 10 19 0 26 24 18 11 33 5 19 19 0 0 17split_gain=44.405 15.4883 10.4047 14.0192 8.21314 6.10102 6.05149 5.02747 4.90787 4.25664 3.71093 3.70335 3.67161 3.47685 7.62133 3.24401 3.54211 3.01026 4.34126 2.68312 2.68164 3.86668 2.62272 5.87262 2.48591 2.84619 4.35285 2.45874 2.39354 1.95176 1.63381 2.53615 1.45497 4.12841 1.34471 1.3104 0.960074 0.782643 2.10217 1.83263 0.708825 0.552426 0.770031 0.592349 1.17153 0.375368 0.365438 0.617484 0.277314threshold=0.019958450000000006 28.131596565250003 0.010784120500000003 5.0413153171500005 0.0066878085500000012 26.930888175950006 116.52628231050001 0.0021141743500000007 32.370494842550009 0.039981298100000007 0.0056026365500000007 0.025320512800000004 42.425361633300007 25.618347167950002 0.18333333335000004 21.921377182000004 24.011768341100005 27.740101814250004 42.862270355200003 23.333234787000006 13.504986286150002 1.4367750000000001e-05 5.912924051300001 60.386631011950008 0.022674138000000003 0.021341419500000004 0.095454545450000025 25.740841865550003 1.0000000180025095e-35 23.481465339700005 0.023614218000000003 0.021798707500000004 0.016466209000000006 32.878534317050004 0.41275599895000009 23.831994056700001 0.012270400500000002 0.11805555555000001 60.787229537950004 579.16949462890011 24.733672142050004 30.347360610950002 0.013626538000000002 36.47881507875001 0.0029282639500000005 0.0011933599000000003 0.057189542500000003 0.040833333350000008 5.2856600000000005e-05decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2@@ -31,7 +31,7 @@ shrinkage=0.005Tree=1num_leaves=50num_cat=0-split_feature=35 9 35 28 6 22 19 25 18 20 28 18 20 35 0 6 12 27 9 10 28 11 12 25 18 11 25 0 6 35 9 18 9 0 12 20 6 9 20 12 24 35 35 18 35 0 0 9 19+split_feature=33 8 33 26 5 21 18 24 17 19 26 17 19 33 0 5 11 25 8 9 26 10 11 24 17 10 24 0 5 33 8 17 8 0 11 19 5 8 19 11 23 33 33 17 33 0 0 8 18split_gain=152.25 59.9795 44.9635 69.9036 34.8326 25.4545 25.4404 23.1446 16.9731 17.7931 16.0731 24.166 15.7759 15.4379 13.7787 14.686 13.1813 13.1641 14.2814 9.50549 22.6834 8.11811 14.5284 9.68489 7.75809 11.1195 7.47554 6.31758 10.3168 6.20151 16.6614 5.36193 4.99983 12.7044 4.33421 3.4442 3.9843 2.99276 2.83137 2.73824 4.71467 2.31028 2.77357 2.23162 1.89009 1.08313 2.676 1.07044 0.941669threshold=0.019958450000000006 28.918019294700006 0.010153455000000002 1.5334161222000002 24.617109298700004 28.223515510550005 24.378225326500004 199.75513839720003 0.0066878085500000012 0.0021141743500000007 9.4430942535500026 0.0062548280500000003 0.0056026365500000007 0.015223103000000002 0.025320512800000004 42.425361633300007 26.930888175950006 27.740101814250004 42.862270355200003 5.912924051300001 60.386631011950008 24.193528175350004 20.459006309500001 78.93672296405002 0.00091614705000000009 24.557957649250003 124.34751701355002 0.22500000000000003 40.209836959850008 0.016466209000000006 32.878534317050004 5.7137000000000005e-05 24.108223915100002 0.087121212100000026 31.839186668400004 0.0021276981000000004 35.284496307350004 34.230171203600008 0.0014903262000000002 9.4340581894000017 0.41275599895000009 0.018645591500000006 0.014970756000000002 1.8400600000000005e-05 0.019852502000000005 0.057189542500000003 0.040833333350000008 25.625818252600002 44.153718948350004decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2@@ -50,7 +50,7 @@ shrinkage=0.005Tree=2num_leaves=50num_cat=0-split_feature=35 9 18 12 0 28 28 18 22 18 11 20 22 35 32 6 25 20 12 6 28 16 20 35 6 9 32 22 25 11 0 28 0 11 28 11 18 9 18 11 9 18 12 15 19 9 28 11 19+split_feature=33 8 17 11 0 26 26 17 21 17 10 19 21 33 30 5 24 19 11 5 26 15 19 33 5 8 30 21 24 10 0 26 0 10 26 10 17 8 17 10 8 17 11 14 18 8 26 10 18split_gain=123.56 48.7975 30.9068 24.6786 23.9449 34.8431 19.553 21.9259 27.3465 15.7557 14.4067 13.9334 15.1573 13.9901 26.016 13.4799 14.4816 12.2247 15.5713 12.1234 10.1412 9.37119 8.88257 8.59522 7.47114 7.23078 6.15149 9.43505 6.60996 5.10747 4.91394 4.89723 4.89465 4.70571 4.69164 4.48818 3.50872 2.9719 2.76934 5.14391 2.34614 2.08664 2.45537 1.89102 1.88417 1.59661 5.2212 1.90012 1.57823threshold=0.019958450000000006 27.158587455750006 0.00033038530000000004 27.547188758850002 0.057189542500000003 66.052501678450014 7.1899642944500011 0.0062548280500000003 26.600150108350004 3.4804450000000005e-05 24.557957649250003 0.0056026365500000007 24.183298110950002 0.015295206000000002 34.031578064000009 30.177063941950006 180.53593158720003 0.0029542162000000005 20.176387786900005 19.430234909050004 5.5020630360000009 1.0000000180025095e-35 0.0010005005000000003 0.012058684500000002 20.663570404050002 39.69972801210001 28.999861717200002 44.565795898400005 95.25605201720002 1.0000000180025095e-35 0.15476190480000004 60.787229537950004 0.10555555555000003 35.239673614500006 8.0346117019500021 35.187662124650011 1.0899200000000001e-05 24.846534729000002 0.0076535505000000009 28.534720420800003 35.033393859850008 1.7003300000000004e-05 45.418594360350006 61.749555587750002 45.236038208000004 17.976120948750005 1.0000000180025095e-35 36.921825408950006 21.802299499550006decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2@@ -69,7 +69,7 @@ shrinkage=0.005Tree=3num_leaves=50num_cat=0-split_feature=35 9 18 0 28 18 12 28 18 22 12 12 20 9 35 32 28 11 0 6 20 6 11 32 22 25 11 0 28 6 21 12 24 25 19 6 18 18 28 35 35 18 18 20 32 32 11 35 6+split_feature=33 8 17 0 26 17 11 26 17 21 11 11 19 8 33 30 26 10 0 5 19 5 10 30 21 24 10 0 26 5 20 11 23 24 18 5 17 17 26 33 33 17 17 19 30 30 10 33 5split_gain=122.03 47.1967 30.9872 19.548 30.0652 20.2562 18.4669 17.3936 19.2893 24.2543 15.4895 15.1185 19.8703 13.5899 22.0719 26.5849 15.3893 13.1669 14.5908 11.5188 9.76489 9.07555 6.71338 5.67539 7.46293 6.0919 4.57457 4.42589 4.52685 3.66486 3.38861 3.30437 4.90931 2.79385 2.05371 1.83061 1.58896 1.35776 1.25748 1.23288 7.54702 1.84498 5.5099 1.42987 1.17893 4.5426 1.0637 2.91643 0.999161threshold=0.019958450000000006 27.158587455750006 0.00033038530000000004 0.057189542500000003 66.052501678450014 3.4804450000000005e-05 27.547188758850002 7.1899642944500011 0.0062548280500000003 26.600150108350004 27.467996597300004 20.176387786900005 0.0029542162000000005 27.462724685700007 0.015295206000000002 32.923410415650004 5.5020630360000009 27.969018936150004 0.10555555555000003 19.430234909050004 0.0010005005000000003 54.445077896150003 24.193528175350004 28.999861717200002 44.565795898400005 95.25605201720002 1.0000000180025095e-35 0.15476190480000004 60.787229537950004 40.209836959850008 43.487449645950008 9.4340581894000017 0.41275599895000009 182.36229705810004 20.416842460650006 25.280503273000004 1.0837500000000001e-05 2.3751900000000002e-05 27.136941909800004 0.010422712000000002 0.012167945000000001 1.9047900000000001e-05 2.5816900000000003e-05 0.012270400500000002 34.031578064000009 28.621761322050002 31.730455398550003 0.023952123000000002 28.991178512600005decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2diff --git a/optics/testcases b/optics/testcasesindex ac0cdce3..bbacced9 160000--- a/optics/testcases+++ b/optics/testcases@@ -1 +1 @@-Subproject commit ac0cdce3cac8922eac3528ca4b8c05d87141b69e+Subproject commit bbacced9c9200e2be96e9b481f48e54a466f1de6