Skip to content

web search done right

Higher threshold for inbound similarity nodes

Mikkel Denker79ac702fdcf2
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index b6ca7365..c1413b19 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -14,13 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{fs::File, path::Path};
+use std::io::{BufReader, BufWriter, Read};
+use std::{collections::BTreeMap, fs::File, path::Path};
+use crate::Result;
use crate::{
kv::{rocksdb_store::RocksDbStore, Kv},
webgraph::{
centrality::{harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality},
- Webgraph,
+ Node, NodeID, Webgraph,
},
};
@@ -49,9 +51,20 @@ pub struct CentralityStore {
pub harmonic: HarmonicCentralityStore,
pub online_harmonic: OnlineHarmonicCentrality,
pub inbound_similarity: InboundSimilarity,
+ pub node2id: BTreeMap<Node, NodeID>,
pub base_path: String,
}
+fn open_node2id<P: AsRef<Path>>(path: P) -> Result<BTreeMap<Node, NodeID>> {
+ let file = File::open(path)?;
+ let mut reader = BufReader::new(file);
+
+ let mut buf = Vec::new();
+ reader.read_to_end(&mut buf)?;
+
+ Ok(bincode::deserialize(&buf)?)
+}
+
impl CentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
@@ -62,6 +75,9 @@ impl CentralityStore {
inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
.ok()
.unwrap_or_default(),
+ node2id: open_node2id(path.as_ref().join("node2id"))
+ .ok()
+ .unwrap_or_default(),
base_path: path.as_ref().to_str().unwrap().to_string(),
}
}
@@ -98,6 +114,8 @@ impl CentralityStore {
store.online_harmonic = OnlineHarmonicCentrality::new(graph, &harmonic_centrality);
store.inbound_similarity = InboundSimilarity::build(graph, &harmonic_centrality);
+ store.node2id = graph.node_ids().collect();
+
store.flush();
store
@@ -113,5 +131,15 @@ impl CentralityStore {
self.inbound_similarity
.save(Path::new(&self.base_path).join("inbound_similarity"))
.unwrap();
+
+ let mut file = BufWriter::new(
+ File::options()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(Path::new(&self.base_path).join("node2id"))
+ .unwrap(),
+ );
+ bincode::serialize_into(&mut file, &self.node2id).unwrap();
}
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index ea6ca6f7..8bd65a52 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -20,16 +20,22 @@ use std::{
path::Path,
};
+use optics::SiteRankings;
use serde::{Deserialize, Serialize};
use crate::{
intmap::IntMap,
- webgraph::{centrality::harmonic::HarmonicCentrality, Webgraph},
+ webgraph::{centrality::harmonic::HarmonicCentrality, NodeID, Webgraph},
Result,
};
use super::bitvec_similarity;
-const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.025;
+const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.038;
+
+pub struct Scorer<'a> {
+ liked: Vec<&'a bitvec_similarity::BitVec>,
+ disliked: Vec<&'a bitvec_similarity::BitVec>,
+}
#[derive(Serialize, Deserialize, Default)]
pub struct InboundSimilarity {
@@ -66,6 +72,20 @@ impl InboundSimilarity {
Self { vectors }
}
+ pub fn scorer<'a>(&'a self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer<'a> {
+ let liked: Vec<_> = liked_sites
+ .iter()
+ .filter_map(|id| self.vectors.get(&id.0))
+ .collect();
+
+ let disliked: Vec<_> = disliked_sites
+ .iter()
+ .filter_map(|id| self.vectors.get(&id.0))
+ .collect();
+
+ Scorer { liked, disliked }
+ }
+
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
let mut file = BufWriter::new(
File::options()
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index aa36b111..0b33c64a 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -358,22 +358,20 @@ impl OnlineHarmonicCentrality {
// as an estimate
let mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();
- for id in graph.nodes() {
- if let Some(node) = graph.id2node(&id) {
- node2id.insert(node.clone(), id);
+ for (node, id) in graph.node_ids() {
+ node2id.insert(node.clone(), id);
- let score = *centrality.host.get(&node).unwrap_or(&0.0);
- let candidate = ProxyNodeCandidate { node, score };
+ let score = *centrality.host.get(&node).unwrap_or(&0.0);
+ let candidate = ProxyNodeCandidate { node, score };
- if nodes.len() >= num_proxy_nodes {
- if let Some(mut existing_node) = nodes.peek_mut() {
- if candidate.score > existing_node.score {
- *existing_node = candidate;
- }
+ if nodes.len() >= num_proxy_nodes {
+ if let Some(mut existing_node) = nodes.peek_mut() {
+ if candidate.score > existing_node.score {
+ *existing_node = candidate;
}
- } else {
- nodes.push(candidate);
}
+ } else {
+ nodes.push(candidate);
}
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index f3773002..94f2441a 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -520,6 +520,10 @@ impl<S: Store> Webgraph<S> {
self.node2id.iter().map(|(_, id)| id)
}
+ pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {
+ self.node2id.iter()
+ }
+
pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
self.segments.iter().flat_map(|segment| segment.edges())
}