web search done right
find similar sites in webgraph
Mikkel Denkerf0129d724f9a
diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rsdeleted file mode 100644index 0278282c..00000000--- a/core/examples/print_approx_harmonic_centrality.rs+++ /dev/null@@ -1,164 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{- cmp,- collections::{BinaryHeap, HashMap},-};--use stract::{- ranking::centrality_store::CentralityStore,- webgraph::{centrality::online_harmonic::Scorer, Node, NodeID, WebgraphBuilder},- webpage::Url,-};--struct ScoredNode {- score: f64,- id: u64,-}--impl PartialOrd for ScoredNode {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- self.score.partial_cmp(&other.score)- }-}--impl PartialEq for ScoredNode {- fn eq(&self, other: &Self) -> bool {- self.score == other.score- }-}--impl Ord for ScoredNode {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)- }-}--impl Eq for ScoredNode {}--fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[NodeID]) -> Vec<ScoredNode> {- let mut top_nodes = BinaryHeap::with_capacity(top_n);-- for node in nodes {- top_nodes.push(cmp::Reverse(ScoredNode {- score: scorer.score(*node),- id: node.0,- }));-- while top_nodes.len() > top_n {- top_nodes.pop();- }- }-- let mut nodes = Vec::new();- while let Some(node) = top_nodes.pop() {- nodes.push(node.0);- }-- nodes-}--fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<NodeID, Node>) {- for (i, node) in nodes.iter().rev().enumerate() {- println!(- "{i} \t {:.3} \t {} ",- node.score,- id2node.get(&(node.id.into())).unwrap().name- );- }-}--fn print_top_nodes(- liked_sites: &[&str],- top_n: usize,- store: &CentralityStore,- id2node: &HashMap<NodeID, Node>,- nodes: &[NodeID],-) {- let liked_nodes: Vec<_> = liked_sites- .iter()- .filter_map(|host| {- store- .node2id- .get(&Node::from_url(&Url::from(host.to_string())))- .copied()- })- .collect();-- let scorer = store.online_harmonic.scorer(&liked_nodes, &[]);- let top_nodes = get_top_nodes(scorer, top_n, nodes);- println!("top {top_n} sites for these liked sites {liked_sites:?}");- print_nodes(&top_nodes, id2node);-- println!();- println!();-}--pub fn main() {- const TOP_N: usize = 50;- let graph = WebgraphBuilder::new("data/webgraph").open();- let nodes: Vec<_> = graph.nodes().cloned().collect();- let id2node: HashMap<_, _> = nodes- .iter()- .map(|id| (*id, graph.id2node(id).unwrap().clone()))- .collect();-- let store = CentralityStore::open("data/centrality");-- let mut proxy_nodes = Vec::new();- for node in &store.online_harmonic.proxy_nodes {- proxy_nodes.push(id2node.get(&node.id).unwrap().name.clone());- }-- print_top_nodes(- &[- "dr.dk",- "berlingske.dk",- "ekstrabladet.dk",- "politikken.dk",- "tv2.dk",- "seoghoer.dk",- "zetland.dk",- ],- TOP_N,- &store,- &id2node,- &nodes,- );-- print_top_nodes(- &[- "stackoverflow.com",- "github.com",- "arxiv.org",- "news.ycombinator.com",- ],- TOP_N,- &store,- &id2node,- &nodes,- );-- print_top_nodes(&["nature.com", "who.int"], TOP_N, &store, &id2node, &nodes);- print_top_nodes(- &["webmd.com", "medlineplus.gov"],- TOP_N,- &store,- &id2node,- &nodes,- );-}diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rsnew file mode 100644index 00000000..bf020e18--- /dev/null+++ b/core/examples/print_similar_sites.rs@@ -0,0 +1,75 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use stract::{+ ranking::centrality_store::CentralityStore,+ similar_sites::{ScoredNode, SimilarSitesFinder},+ webgraph::WebgraphBuilder,+};++fn print_nodes(nodes: &[ScoredNode]) {+ for (i, node) in nodes.iter().enumerate() {+ println!("{i} \t {:.3} \t {} ", node.score, node.node.name);+ }+}++fn print_top_nodes(liked_sites: &[&str], top_n: usize, similarity_finder: &SimilarSitesFinder) {+ let sites: Vec<_> = liked_sites.iter().map(|s| s.to_string()).collect();+ let top_nodes = similarity_finder.find_similar_sites(&sites, top_n);++ println!("top {top_n} sites for these liked sites {liked_sites:?}");+ print_nodes(&top_nodes);++ println!();+ println!();+}++pub fn main() {+ const TOP_N: usize = 50;+ let graph = WebgraphBuilder::new("data/webgraph").open();+ let store = CentralityStore::open("data/centrality");++ let similarity_finder = SimilarSitesFinder::new(graph, store.inbound_similarity);++ print_top_nodes(+ &[+ "dr.dk",+ "berlingske.dk",+ "ekstrabladet.dk",+ "politikken.dk",+ "tv2.dk",+ "seoghoer.dk",+ "zetland.dk",+ ],+ TOP_N,+ &similarity_finder,+ );++ print_top_nodes(+ &[+ "stackoverflow.com",+ "github.com",+ "arxiv.org",+ "news.ycombinator.com",+ ],+ TOP_N,+ &similarity_finder,+ );++ print_top_nodes(&["nature.com", "who.int"], TOP_N, &similarity_finder);++ print_top_nodes(&["webmd.com", "medlineplus.gov"], TOP_N, &similarity_finder);+}diff --git a/core/src/lib.rs b/core/src/lib.rsindex aebe0b56..53427959 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -64,6 +64,7 @@ mod search_ctx;mod search_prettifier;pub mod searcher;mod simhash;+pub mod similar_sites;mod snippet;mod sonic;pub mod spell;diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 757cbd5e..4e354d27 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -61,7 +61,7 @@ impl NodeScorer {impl Scorer {fn calculate_score(&self, node: &NodeID) -> f64 {match self.vectors.get(&node.0) {- Some(vec) => (self.disliked.len() as f64+ Some(vec) => ((self.disliked.len() as f64+ (self.liked.iter()@@ -72,7 +72,8 @@ impl Scorer {.iter().map(|disliked| disliked.sim(node, vec)).sum::<f64>()))- .max(0.0),+ / self.liked.len() as f64)+ .max(0.0),None => 0.0,}}diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsnew file mode 100644index 00000000..6a51f1fe--- /dev/null+++ b/core/src/similar_sites.rs@@ -0,0 +1,135 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ cmp::Reverse,+ collections::{BinaryHeap, HashMap, HashSet},+};++use crate::{+ ranking::inbound_similarity::InboundSimilarity,+ webgraph::{Node, NodeID, Webgraph},+};++pub struct ScoredNode {+ pub node: Node,+ pub score: f64,+}++struct ScoredNodeID {+ node_id: NodeID,+ score: f64,+}++impl PartialOrd for ScoredNodeID {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.score.partial_cmp(&other.score)+ }+}++impl PartialEq for ScoredNodeID {+ fn eq(&self, other: &Self) -> bool {+ self.score == other.score+ }+}++impl Ord for ScoredNodeID {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.score+ .partial_cmp(&other.score)+ .unwrap_or(std::cmp::Ordering::Equal)+ }+}++impl Eq for ScoredNodeID {}++pub struct SimilarSitesFinder {+ webgraph: Webgraph,+ inbound_similarity: InboundSimilarity,+}++impl SimilarSitesFinder {+ pub fn new(webgraph: Webgraph, inbound_similarity: InboundSimilarity) -> Self {+ Self {+ webgraph,+ inbound_similarity,+ }+ }++ pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {+ let nodes: Vec<_> = nodes+ .iter()+ .map(|url| Node::from(url.to_string()).into_host())+ .filter_map(|node| self.webgraph.node2id(&node).copied())+ .collect();++ let scorer = self.inbound_similarity.scorer(&nodes, &[]);++ let mut backlink_count: HashMap<NodeID, usize> = HashMap::new();++ for node in &nodes {+ for edge in self.webgraph.raw_ingoing_edges(node) {+ *backlink_count.entry(edge.from).or_default() += 1;+ }+ }++ let mut top_backlink_nodes: Vec<_> = backlink_count+ .into_iter()+ .map(|(node, count)| (node, count))+ .collect();+ top_backlink_nodes.sort_by_key(|(_, count)| *count);+ top_backlink_nodes.reverse();++ let mut potential_nodes = HashSet::new();+ for (backlink_node, _) in top_backlink_nodes.into_iter().take(2 * limit) {+ for edge in self.webgraph.raw_outgoing_edges(&backlink_node) {+ potential_nodes.insert(edge.to);+ }+ }++ let mut scored_nodes = BinaryHeap::with_capacity(limit);++ for potential_node in potential_nodes {+ let score = scorer.score(&potential_node);+ let scored_node_id = ScoredNodeID {+ node_id: potential_node,+ score,+ };++ if scored_nodes.len() < limit {+ scored_nodes.push(Reverse(scored_node_id));+ } else {+ let mut min_scored_node = scored_nodes.peek_mut().unwrap();++ if scored_node_id > min_scored_node.0 {+ *min_scored_node = Reverse(scored_node_id);+ }+ }+ }++ let mut scored_nodes: Vec<_> = scored_nodes.into_iter().take(limit).map(|n| n.0).collect();+ scored_nodes.sort();+ scored_nodes.reverse();++ scored_nodes+ .into_iter()+ .map(|ScoredNodeID { node_id, score }| {+ let node = self.webgraph.id2node(&node_id).unwrap().clone();+ ScoredNode { node, score }+ })+ .collect()+ }+}diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_modelindex 339c87bc..d353e497 100755--- a/scripts/export_abstractive_summary_model+++ b/scripts/export_abstractive_summary_model@@ -9,7 +9,6 @@ os.makedirs("data/summarizer/abstractive", exist_ok=True)MODEL = "lidiya/bart-large-xsum-samsum"model = AutoModelForSeq2SeqLM.from_pretrained(MODEL, torchscript=True)-# model.half()model.eval()tokenizer = AutoTokenizer.from_pretrained(MODEL)tokenizer.save_pretrained("data/summarizer/abstractive")@@ -20,16 +19,19 @@ for p in model.parameters():def run_decoder_wp(input_ids, encoder_hidden_states, past_key_values):- (logits, past) = model.model.decoder(input_ids=input_ids,- encoder_hidden_states=encoder_hidden_states,- past_key_values=past_key_values)+ (logits, past) = model.model.decoder(+ input_ids=input_ids,+ encoder_hidden_states=encoder_hidden_states,+ past_key_values=past_key_values,+ )return (model.lm_head(logits), past)def run_decoder(input_ids, encoder_hidden_states):- (logits, past) = model.model.decoder(input_ids=input_ids,- encoder_hidden_states=encoder_hidden_states)+ (logits, past) = model.model.decoder(+ input_ids=input_ids, encoder_hidden_states=encoder_hidden_states+ )out = model.lm_head(logits)return (out, past)@@ -41,28 +43,34 @@ def run_encoder(input_ids):embed_size_per_head = model.config.d_model // model.config.decoder_attention_heads-input_ids = torch.tensor([[19]*1], dtype=torch.long)-keys = torch.ones(model.config.decoder_layers, 1,- model.config.decoder_attention_heads, 1, embed_size_per_head)+input_ids = torch.tensor([[19] * 1], dtype=torch.long)+keys = torch.ones(+ model.config.decoder_layers,+ 1,+ model.config.decoder_attention_heads,+ 1,+ embed_size_per_head,+)past_key_values = tuple((key, key) for key in keys)encoder_outputs = torch.rand(1, 1024, 1024, dtype=torch.float32)encoder_hidden_states = torch.rand(1, 1024, 1024, dtype=torch.float32)# trace modelstraced_decoder_wp = torch.jit.trace(- run_decoder_wp, (input_ids, encoder_hidden_states, past_key_values))+ run_decoder_wp, (input_ids, encoder_hidden_states, past_key_values)+)-input_ids = torch.tensor([[19]*1024], dtype=torch.long)+input_ids = torch.tensor([[19] * 1024], dtype=torch.long)-traced_decoder = torch.jit.trace(- run_decoder, (input_ids, encoder_hidden_states))+traced_decoder = torch.jit.trace(run_decoder, (input_ids, encoder_hidden_states))-traced_encoder = torch.jit.trace(- run_encoder, input_ids)+traced_encoder = torch.jit.trace(run_encoder, input_ids)traced_decoder_wp.save("data/summarizer/abstractive/traced_decoder_wp.pt")traced_decoder.save("data/summarizer/abstractive/traced_decoder.pt")traced_encoder.save("data/summarizer/abstractive/traced_encoder.pt")-urllib.request.urlretrieve("https://s3.trystract.com/public/truncated_word2vec.bin.gz",- "data/summarizer/word2vec.bin.gz")+urllib.request.urlretrieve(+ "https://s3.trystract.com/public/truncated_word2vec.bin.gz",+ "data/summarizer/word2vec.bin.gz",+)