web search done right
Higher threshold for inbound similarity nodes
Mikkel Denker79ac702fdcf2
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex b6ca7365..c1413b19 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -14,13 +14,15 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{fs::File, path::Path};+use std::io::{BufReader, BufWriter, Read};+use std::{collections::BTreeMap, fs::File, path::Path};+use crate::Result;use crate::{kv::{rocksdb_store::RocksDbStore, Kv},webgraph::{centrality::{harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality},- Webgraph,+ Node, NodeID, Webgraph,},};@@ -49,9 +51,20 @@ pub struct CentralityStore {pub harmonic: HarmonicCentralityStore,pub online_harmonic: OnlineHarmonicCentrality,pub inbound_similarity: InboundSimilarity,+ pub node2id: BTreeMap<Node, NodeID>,pub base_path: String,}+fn open_node2id<P: AsRef<Path>>(path: P) -> Result<BTreeMap<Node, NodeID>> {+ let file = File::open(path)?;+ let mut reader = BufReader::new(file);++ let mut buf = Vec::new();+ reader.read_to_end(&mut buf)?;++ Ok(bincode::deserialize(&buf)?)+}+impl CentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {@@ -62,6 +75,9 @@ impl CentralityStore {inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).ok().unwrap_or_default(),+ node2id: open_node2id(path.as_ref().join("node2id"))+ .ok()+ .unwrap_or_default(),base_path: path.as_ref().to_str().unwrap().to_string(),}}@@ -98,6 +114,8 @@ impl CentralityStore {store.online_harmonic = OnlineHarmonicCentrality::new(graph, &harmonic_centrality);store.inbound_similarity = InboundSimilarity::build(graph, &harmonic_centrality);+ store.node2id = graph.node_ids().collect();+store.flush();store@@ -113,5 +131,15 @@ impl CentralityStore {self.inbound_similarity.save(Path::new(&self.base_path).join("inbound_similarity")).unwrap();++ let mut file = BufWriter::new(+ File::options()+ .create(true)+ .truncate(true)+ .write(true)+ .open(Path::new(&self.base_path).join("node2id"))+ .unwrap(),+ );+ bincode::serialize_into(&mut file, &self.node2id).unwrap();}}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex ea6ca6f7..8bd65a52 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -20,16 +20,22 @@ use std::{path::Path,};+use optics::SiteRankings;use serde::{Deserialize, Serialize};use crate::{intmap::IntMap,- webgraph::{centrality::harmonic::HarmonicCentrality, Webgraph},+ webgraph::{centrality::harmonic::HarmonicCentrality, NodeID, Webgraph},Result,};use super::bitvec_similarity;-const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.025;+const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.038;++pub struct Scorer<'a> {+ liked: Vec<&'a bitvec_similarity::BitVec>,+ disliked: Vec<&'a bitvec_similarity::BitVec>,+}#[derive(Serialize, Deserialize, Default)]pub struct InboundSimilarity {@@ -66,6 +72,20 @@ impl InboundSimilarity {Self { vectors }}+ pub fn scorer<'a>(&'a self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer<'a> {+ let liked: Vec<_> = liked_sites+ .iter()+ .filter_map(|id| self.vectors.get(&id.0))+ .collect();++ let disliked: Vec<_> = disliked_sites+ .iter()+ .filter_map(|id| self.vectors.get(&id.0))+ .collect();++ Scorer { liked, disliked }+ }+pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {let mut file = BufWriter::new(File::options()diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex aa36b111..0b33c64a 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -358,22 +358,20 @@ impl OnlineHarmonicCentrality {// as an estimatelet mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();- for id in graph.nodes() {- if let Some(node) = graph.id2node(&id) {- node2id.insert(node.clone(), id);+ for (node, id) in graph.node_ids() {+ node2id.insert(node.clone(), id);- let score = *centrality.host.get(&node).unwrap_or(&0.0);- let candidate = ProxyNodeCandidate { node, score };+ let score = *centrality.host.get(&node).unwrap_or(&0.0);+ let candidate = ProxyNodeCandidate { node, score };- if nodes.len() >= num_proxy_nodes {- if let Some(mut existing_node) = nodes.peek_mut() {- if candidate.score > existing_node.score {- *existing_node = candidate;- }+ if nodes.len() >= num_proxy_nodes {+ if let Some(mut existing_node) = nodes.peek_mut() {+ if candidate.score > existing_node.score {+ *existing_node = candidate;}- } else {- nodes.push(candidate);}+ } else {+ nodes.push(candidate);}}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex f3773002..94f2441a 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -520,6 +520,10 @@ impl<S: Store> Webgraph<S> {self.node2id.iter().map(|(_, id)| id)}+ pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {+ self.node2id.iter()+ }+pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {self.segments.iter().flat_map(|segment| segment.edges())}