Skip to content

web search done right

find similar sites in webgraph

Mikkel Denkerf0129d724f9a
diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rs
deleted file mode 100644
index 0278282c..00000000
--- a/core/examples/print_approx_harmonic_centrality.rs
+++ /dev/null
@@ -1,164 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{
- cmp,
- collections::{BinaryHeap, HashMap},
-};
-
-use stract::{
- ranking::centrality_store::CentralityStore,
- webgraph::{centrality::online_harmonic::Scorer, Node, NodeID, WebgraphBuilder},
- webpage::Url,
-};
-
-struct ScoredNode {
- score: f64,
- id: u64,
-}
-
-impl PartialOrd for ScoredNode {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- self.score.partial_cmp(&other.score)
- }
-}
-
-impl PartialEq for ScoredNode {
- fn eq(&self, other: &Self) -> bool {
- self.score == other.score
- }
-}
-
-impl Ord for ScoredNode {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
- }
-}
-
-impl Eq for ScoredNode {}
-
-fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[NodeID]) -> Vec<ScoredNode> {
- let mut top_nodes = BinaryHeap::with_capacity(top_n);
-
- for node in nodes {
- top_nodes.push(cmp::Reverse(ScoredNode {
- score: scorer.score(*node),
- id: node.0,
- }));
-
- while top_nodes.len() > top_n {
- top_nodes.pop();
- }
- }
-
- let mut nodes = Vec::new();
- while let Some(node) = top_nodes.pop() {
- nodes.push(node.0);
- }
-
- nodes
-}
-
-fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<NodeID, Node>) {
- for (i, node) in nodes.iter().rev().enumerate() {
- println!(
- "{i} \t {:.3} \t {} ",
- node.score,
- id2node.get(&(node.id.into())).unwrap().name
- );
- }
-}
-
-fn print_top_nodes(
- liked_sites: &[&str],
- top_n: usize,
- store: &CentralityStore,
- id2node: &HashMap<NodeID, Node>,
- nodes: &[NodeID],
-) {
- let liked_nodes: Vec<_> = liked_sites
- .iter()
- .filter_map(|host| {
- store
- .node2id
- .get(&Node::from_url(&Url::from(host.to_string())))
- .copied()
- })
- .collect();
-
- let scorer = store.online_harmonic.scorer(&liked_nodes, &[]);
- let top_nodes = get_top_nodes(scorer, top_n, nodes);
- println!("top {top_n} sites for these liked sites {liked_sites:?}");
- print_nodes(&top_nodes, id2node);
-
- println!();
- println!();
-}
-
-pub fn main() {
- const TOP_N: usize = 50;
- let graph = WebgraphBuilder::new("data/webgraph").open();
- let nodes: Vec<_> = graph.nodes().cloned().collect();
- let id2node: HashMap<_, _> = nodes
- .iter()
- .map(|id| (*id, graph.id2node(id).unwrap().clone()))
- .collect();
-
- let store = CentralityStore::open("data/centrality");
-
- let mut proxy_nodes = Vec::new();
- for node in &store.online_harmonic.proxy_nodes {
- proxy_nodes.push(id2node.get(&node.id).unwrap().name.clone());
- }
-
- print_top_nodes(
- &[
- "dr.dk",
- "berlingske.dk",
- "ekstrabladet.dk",
- "politikken.dk",
- "tv2.dk",
- "seoghoer.dk",
- "zetland.dk",
- ],
- TOP_N,
- &store,
- &id2node,
- &nodes,
- );
-
- print_top_nodes(
- &[
- "stackoverflow.com",
- "github.com",
- "arxiv.org",
- "news.ycombinator.com",
- ],
- TOP_N,
- &store,
- &id2node,
- &nodes,
- );
-
- print_top_nodes(&["nature.com", "who.int"], TOP_N, &store, &id2node, &nodes);
- print_top_nodes(
- &["webmd.com", "medlineplus.gov"],
- TOP_N,
- &store,
- &id2node,
- &nodes,
- );
-}
diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rs
new file mode 100644
index 00000000..bf020e18
--- /dev/null
+++ b/core/examples/print_similar_sites.rs
@@ -0,0 +1,75 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use stract::{
+ ranking::centrality_store::CentralityStore,
+ similar_sites::{ScoredNode, SimilarSitesFinder},
+ webgraph::WebgraphBuilder,
+};
+
+fn print_nodes(nodes: &[ScoredNode]) {
+ for (i, node) in nodes.iter().enumerate() {
+ println!("{i} \t {:.3} \t {} ", node.score, node.node.name);
+ }
+}
+
+fn print_top_nodes(liked_sites: &[&str], top_n: usize, similarity_finder: &SimilarSitesFinder) {
+ let sites: Vec<_> = liked_sites.iter().map(|s| s.to_string()).collect();
+ let top_nodes = similarity_finder.find_similar_sites(&sites, top_n);
+
+ println!("top {top_n} sites for these liked sites {liked_sites:?}");
+ print_nodes(&top_nodes);
+
+ println!();
+ println!();
+}
+
+pub fn main() {
+ const TOP_N: usize = 50;
+ let graph = WebgraphBuilder::new("data/webgraph").open();
+ let store = CentralityStore::open("data/centrality");
+
+ let similarity_finder = SimilarSitesFinder::new(graph, store.inbound_similarity);
+
+ print_top_nodes(
+ &[
+ "dr.dk",
+ "berlingske.dk",
+ "ekstrabladet.dk",
+ "politikken.dk",
+ "tv2.dk",
+ "seoghoer.dk",
+ "zetland.dk",
+ ],
+ TOP_N,
+ &similarity_finder,
+ );
+
+ print_top_nodes(
+ &[
+ "stackoverflow.com",
+ "github.com",
+ "arxiv.org",
+ "news.ycombinator.com",
+ ],
+ TOP_N,
+ &similarity_finder,
+ );
+
+ print_top_nodes(&["nature.com", "who.int"], TOP_N, &similarity_finder);
+
+ print_top_nodes(&["webmd.com", "medlineplus.gov"], TOP_N, &similarity_finder);
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index aebe0b56..53427959 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -64,6 +64,7 @@ mod search_ctx;
mod search_prettifier;
pub mod searcher;
mod simhash;
+pub mod similar_sites;
mod snippet;
mod sonic;
pub mod spell;
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 757cbd5e..4e354d27 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -61,7 +61,7 @@ impl NodeScorer {
impl Scorer {
fn calculate_score(&self, node: &NodeID) -> f64 {
match self.vectors.get(&node.0) {
- Some(vec) => (self.disliked.len() as f64
+ Some(vec) => ((self.disliked.len() as f64
+ (self
.liked
.iter()
@@ -72,7 +72,8 @@ impl Scorer {
.iter()
.map(|disliked| disliked.sim(node, vec))
.sum::<f64>()))
- .max(0.0),
+ / self.liked.len() as f64)
+ .max(0.0),
None => 0.0,
}
}
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
new file mode 100644
index 00000000..6a51f1fe
--- /dev/null
+++ b/core/src/similar_sites.rs
@@ -0,0 +1,135 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ cmp::Reverse,
+ collections::{BinaryHeap, HashMap, HashSet},
+};
+
+use crate::{
+ ranking::inbound_similarity::InboundSimilarity,
+ webgraph::{Node, NodeID, Webgraph},
+};
+
+pub struct ScoredNode {
+ pub node: Node,
+ pub score: f64,
+}
+
+struct ScoredNodeID {
+ node_id: NodeID,
+ score: f64,
+}
+
+impl PartialOrd for ScoredNodeID {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.score.partial_cmp(&other.score)
+ }
+}
+
+impl PartialEq for ScoredNodeID {
+ fn eq(&self, other: &Self) -> bool {
+ self.score == other.score
+ }
+}
+
+impl Ord for ScoredNodeID {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.score
+ .partial_cmp(&other.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ }
+}
+
+impl Eq for ScoredNodeID {}
+
+pub struct SimilarSitesFinder {
+ webgraph: Webgraph,
+ inbound_similarity: InboundSimilarity,
+}
+
+impl SimilarSitesFinder {
+ pub fn new(webgraph: Webgraph, inbound_similarity: InboundSimilarity) -> Self {
+ Self {
+ webgraph,
+ inbound_similarity,
+ }
+ }
+
+ pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {
+ let nodes: Vec<_> = nodes
+ .iter()
+ .map(|url| Node::from(url.to_string()).into_host())
+ .filter_map(|node| self.webgraph.node2id(&node).copied())
+ .collect();
+
+ let scorer = self.inbound_similarity.scorer(&nodes, &[]);
+
+ let mut backlink_count: HashMap<NodeID, usize> = HashMap::new();
+
+ for node in &nodes {
+ for edge in self.webgraph.raw_ingoing_edges(node) {
+ *backlink_count.entry(edge.from).or_default() += 1;
+ }
+ }
+
+ let mut top_backlink_nodes: Vec<_> = backlink_count
+ .into_iter()
+ .map(|(node, count)| (node, count))
+ .collect();
+ top_backlink_nodes.sort_by_key(|(_, count)| *count);
+ top_backlink_nodes.reverse();
+
+ let mut potential_nodes = HashSet::new();
+ for (backlink_node, _) in top_backlink_nodes.into_iter().take(2 * limit) {
+ for edge in self.webgraph.raw_outgoing_edges(&backlink_node) {
+ potential_nodes.insert(edge.to);
+ }
+ }
+
+ let mut scored_nodes = BinaryHeap::with_capacity(limit);
+
+ for potential_node in potential_nodes {
+ let score = scorer.score(&potential_node);
+ let scored_node_id = ScoredNodeID {
+ node_id: potential_node,
+ score,
+ };
+
+ if scored_nodes.len() < limit {
+ scored_nodes.push(Reverse(scored_node_id));
+ } else {
+ let mut min_scored_node = scored_nodes.peek_mut().unwrap();
+
+ if scored_node_id > min_scored_node.0 {
+ *min_scored_node = Reverse(scored_node_id);
+ }
+ }
+ }
+
+ let mut scored_nodes: Vec<_> = scored_nodes.into_iter().take(limit).map(|n| n.0).collect();
+ scored_nodes.sort();
+ scored_nodes.reverse();
+
+ scored_nodes
+ .into_iter()
+ .map(|ScoredNodeID { node_id, score }| {
+ let node = self.webgraph.id2node(&node_id).unwrap().clone();
+ ScoredNode { node, score }
+ })
+ .collect()
+ }
+}
diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_model
index 339c87bc..d353e497 100755
--- a/scripts/export_abstractive_summary_model
+++ b/scripts/export_abstractive_summary_model
@@ -9,7 +9,6 @@ os.makedirs("data/summarizer/abstractive", exist_ok=True)
MODEL = "lidiya/bart-large-xsum-samsum"
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL, torchscript=True)
-# model.half()
model.eval()
tokenizer = AutoTokenizer.from_pretrained(MODEL)
tokenizer.save_pretrained("data/summarizer/abstractive")
@@ -20,16 +19,19 @@ for p in model.parameters():
def run_decoder_wp(input_ids, encoder_hidden_states, past_key_values):
- (logits, past) = model.model.decoder(input_ids=input_ids,
- encoder_hidden_states=encoder_hidden_states,
- past_key_values=past_key_values)
+ (logits, past) = model.model.decoder(
+ input_ids=input_ids,
+ encoder_hidden_states=encoder_hidden_states,
+ past_key_values=past_key_values,
+ )
return (model.lm_head(logits), past)
def run_decoder(input_ids, encoder_hidden_states):
- (logits, past) = model.model.decoder(input_ids=input_ids,
- encoder_hidden_states=encoder_hidden_states)
+ (logits, past) = model.model.decoder(
+ input_ids=input_ids, encoder_hidden_states=encoder_hidden_states
+ )
out = model.lm_head(logits)
return (out, past)
@@ -41,28 +43,34 @@ def run_encoder(input_ids):
embed_size_per_head = model.config.d_model // model.config.decoder_attention_heads
-input_ids = torch.tensor([[19]*1], dtype=torch.long)
-keys = torch.ones(model.config.decoder_layers, 1,
- model.config.decoder_attention_heads, 1, embed_size_per_head)
+input_ids = torch.tensor([[19] * 1], dtype=torch.long)
+keys = torch.ones(
+ model.config.decoder_layers,
+ 1,
+ model.config.decoder_attention_heads,
+ 1,
+ embed_size_per_head,
+)
past_key_values = tuple((key, key) for key in keys)
encoder_outputs = torch.rand(1, 1024, 1024, dtype=torch.float32)
encoder_hidden_states = torch.rand(1, 1024, 1024, dtype=torch.float32)
# trace models
traced_decoder_wp = torch.jit.trace(
- run_decoder_wp, (input_ids, encoder_hidden_states, past_key_values))
+ run_decoder_wp, (input_ids, encoder_hidden_states, past_key_values)
+)
-input_ids = torch.tensor([[19]*1024], dtype=torch.long)
+input_ids = torch.tensor([[19] * 1024], dtype=torch.long)
-traced_decoder = torch.jit.trace(
- run_decoder, (input_ids, encoder_hidden_states))
+traced_decoder = torch.jit.trace(run_decoder, (input_ids, encoder_hidden_states))
-traced_encoder = torch.jit.trace(
- run_encoder, input_ids)
+traced_encoder = torch.jit.trace(run_encoder, input_ids)
traced_decoder_wp.save("data/summarizer/abstractive/traced_decoder_wp.pt")
traced_decoder.save("data/summarizer/abstractive/traced_decoder.pt")
traced_encoder.save("data/summarizer/abstractive/traced_encoder.pt")
-urllib.request.urlretrieve("https://s3.trystract.com/public/truncated_word2vec.bin.gz",
- "data/summarizer/word2vec.bin.gz")
+urllib.request.urlretrieve(
+ "https://s3.trystract.com/public/truncated_word2vec.bin.gz",
+ "data/summarizer/word2vec.bin.gz",
+)