Skip to content

web search done right

Fix crawl plan. Too many spam urls were being prioritized

Mikkel Denker3054fe5c6aa5
diff --git a/core/Cargo.toml b/core/Cargo.toml
index f1ec40ab..1f93fb5d 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -117,10 +117,6 @@ maplit = {workspace = true}
proptest = {workspace = true}
proptest-derive = {workspace = true}
-[[example]]
-name = "alice"
-required-features = ["with_alice"]
-
[[bench]]
harness = false
name = "search-preindexed"
diff --git a/core/examples/alice.rs b/core/examples/alice.rs
deleted file mode 100644
index cb1babbe..00000000
--- a/core/examples/alice.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-use std::io::Write;
-
-use base64::Engine;
-use stract::alice::{
- ExecutionState, BASE64_ENGINE, {AcceleratorConfig, Alice},
-};
-
-#[tokio::main]
-async fn main() {
- let subscriber = tracing_subscriber::FmtSubscriber::builder()
- .with_max_level(tracing::Level::DEBUG)
- .finish();
- tracing::subscriber::set_global_default(subscriber).unwrap();
-
- // dont use this key in production
- let key = BASE64_ENGINE
- .decode("URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc=")
- .unwrap();
-
- let model = Alice::open(
- "data/alice",
- Some(AcceleratorConfig {
- layer_fraction: 1.0,
- quantize_fraction: 0.0,
- device: tch::Device::Mps,
- kind: tch::Kind::Float,
- }),
- &key,
- )
- .unwrap();
-
- tracing::debug!("model loaded");
-
- let mut last_state = None;
-
- loop {
- println!();
- print!("> ");
- std::io::stdout().flush().unwrap();
- let mut input = String::new();
- std::io::stdin().read_line(&mut input).unwrap();
- let input = input.trim();
-
- let gen = model
- .new_executor(
- input,
- last_state.clone(),
- "http://localhost:3000/beta/api/search".to_string(),
- None,
- )
- .unwrap();
-
- for n in gen {
- match n {
- ExecutionState::BeginSearch { query: _ } => {}
- ExecutionState::SearchResult {
- query: _,
- result: _,
- } => {}
- ExecutionState::Speaking { text } => {
- print!("{}", text);
- std::io::stdout().flush().unwrap();
- }
- ExecutionState::Done { state } => {
- last_state = Some(state.decode().unwrap());
- }
- }
- }
- }
-}
diff --git a/core/examples/mapreduce_manager.rs b/core/examples/mapreduce_manager.rs
deleted file mode 100644
index f2fc1d8f..00000000
--- a/core/examples/mapreduce_manager.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-use std::net::SocketAddr;
-
-use serde::{Deserialize, Serialize};
-use stract::mapreduce::{Manager, Map, Reduce, StatelessWorker};
-use tracing::Level;
-use tracing_subscriber::FmtSubscriber;
-
-#[derive(Clone, Serialize, Deserialize, Debug)]
-struct Job {
- id: usize,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-struct Count(usize);
-
-impl Map<StatelessWorker, Count> for Job {
- fn map(&self, _worker: &StatelessWorker) -> Count {
- std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
- Count(1)
- }
-}
-
-impl Reduce<Count> for Count {
- fn reduce(self, element: Self) -> Self {
- Count(self.0 + element.0)
- }
-}
-
-#[tokio::main]
-async fn main() {
- let subscriber = FmtSubscriber::builder()
- .with_max_level(Level::DEBUG)
- .finish();
- tracing::subscriber::set_global_default(subscriber).unwrap();
-
- let jobs = vec![
- Job { id: 0 },
- Job { id: 1 },
- Job { id: 2 },
- Job { id: 3 },
- Job { id: 4 },
- Job { id: 5 },
- ];
-
- let manager = Manager::new(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()]);
- let res: Count = manager.run(jobs.into_iter()).await.unwrap();
-
- println!("{:?}", res.0);
-}
diff --git a/core/examples/mapreduce_worker.rs b/core/examples/mapreduce_worker.rs
deleted file mode 100644
index ccf2dc00..00000000
--- a/core/examples/mapreduce_worker.rs
+++ /dev/null
@@ -1,42 +0,0 @@
-use std::net::SocketAddr;
-
-use serde::{Deserialize, Serialize};
-use stract::mapreduce::{Map, Reduce, StatelessWorker, Worker};
-use tracing::Level;
-use tracing_subscriber::FmtSubscriber;
-
-#[derive(Serialize, Deserialize, Debug)]
-struct Job {
- id: usize,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-struct Count(usize);
-
-impl Map<StatelessWorker, Count> for Job {
- fn map(&self, _worker: &StatelessWorker) -> Count {
- std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
- Count(1)
- }
-}
-
-impl Reduce<Count> for Count {
- fn reduce(self, element: Self) -> Self {
- Count(self.0 + element.0)
- }
-}
-
-#[tokio::main]
-async fn main() {
- let subscriber = FmtSubscriber::builder()
- .with_max_level(Level::DEBUG)
- .finish();
- tracing::subscriber::set_global_default(subscriber).unwrap();
-
- let args: Vec<_> = std::env::args().collect();
-
- StatelessWorker::default()
- .run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())
- .await
- .expect("failed to run worker");
-}
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
deleted file mode 100644
index dccdddd2..00000000
--- a/core/examples/print_inbound_links.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use stract::{
- ranking::inbound_similarity::InboundSimilarity,
- webgraph::{Node, WebgraphBuilder},
-};
-
-pub fn main() {
- let graph = WebgraphBuilder::new("data/webgraph").open();
- let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();
-
- for host in ["www.homedepot.com"] {
- println!("{host}:");
-
- let node = Node::from(host).into_host();
- let inbound_vec = inbound.get(&node.id()).unwrap();
- println!("{:?}", inbound_vec);
-
- for edge in graph.ingoing_edges(node) {
- println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);
- }
-
- println!();
- }
-}
diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rs
deleted file mode 100644
index fac809c6..00000000
--- a/core/examples/print_similar_sites.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use stract::{
- ranking::inbound_similarity::InboundSimilarity,
- similar_sites::{ScoredNode, SimilarSitesFinder},
- webgraph::WebgraphBuilder,
-};
-
-fn print_nodes(nodes: &[ScoredNode]) {
- for (i, node) in nodes.iter().enumerate() {
- println!("{i} \t {:.3} \t {} ", node.score, node.node.name);
- }
-}
-
-fn print_top_nodes(liked_sites: &[&str], top_n: usize, similarity_finder: &SimilarSitesFinder) {
- let sites: Vec<_> = liked_sites.iter().map(|s| s.to_string()).collect();
- let top_nodes = similarity_finder.find_similar_sites(&sites, top_n);
-
- println!("top {top_n} sites for these liked sites {liked_sites:?}");
- print_nodes(&top_nodes);
-
- println!();
- println!();
-}
-
-pub fn main() {
- const TOP_N: usize = 50;
- let graph = WebgraphBuilder::new("data/webgraph").open();
- let inbound_similarity = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();
-
- let similarity_finder = SimilarSitesFinder::new(
- graph.into(),
- inbound_similarity,
- stract::config::defaults::WebgraphServer::max_similar_sites(),
- );
-
- print_top_nodes(
- &[
- "dr.dk",
- "berlingske.dk",
- "ekstrabladet.dk",
- "politikken.dk",
- "tv2.dk",
- "seoghoer.dk",
- "zetland.dk",
- ],
- TOP_N,
- &similarity_finder,
- );
-
- print_top_nodes(
- &[
- "stackoverflow.com",
- "github.com",
- "arxiv.org",
- "news.ycombinator.com",
- ],
- TOP_N,
- &similarity_finder,
- );
-
- print_top_nodes(&["nature.com", "who.int"], TOP_N, &similarity_finder);
-
- print_top_nodes(&["webmd.com", "medlineplus.gov"], TOP_N, &similarity_finder);
-}
diff --git a/core/src/collector.rs b/core/src/collector.rs
index f3737b7e..b3ec02c1 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -47,6 +47,7 @@ pub struct Hashes {
pub site: Prehashed,
pub title: Prehashed,
pub url: Prehashed,
+ pub url_without_tld: Prehashed,
pub simhash: simhash::HashType,
}
@@ -185,6 +186,11 @@ impl TopSegmentCollector {
site: self.get_hash(&doc, &FastField::SiteHash1, &FastField::SiteHash2),
title: self.get_hash(&doc, &FastField::TitleHash1, &FastField::TitleHash2),
url: self.get_hash(&doc, &FastField::UrlHash1, &FastField::UrlHash2),
+ url_without_tld: self.get_hash(
+ &doc,
+ &FastField::UrlWithoutTldHash1,
+ &FastField::UrlWithoutTldHash2,
+ ),
simhash: simhash.unwrap(),
},
id: doc,
@@ -250,12 +256,14 @@ impl BucketCount {
let taken_sites = *self.buckets.get(&hashes.site).unwrap_or(&0);
let taken_urls = *self.buckets.get(&hashes.url).unwrap_or(&0);
+ let taken_urls_without_tld = *self.buckets.get(&hashes.url_without_tld).unwrap_or(&0);
let taken_titles = *self.buckets.get(&hashes.title).unwrap_or(&0);
let adjuster = 1.0
/ (1.0
+ taken_sites as f64 * self.config.site_penalty
+ taken_urls as f64 * self.config.url_penalty
+ + taken_urls_without_tld as f64 * self.config.url_without_tld_penalty
+ taken_titles as f64 * self.config.title_penalty);
doc.adjusted_score = doc.doc.score() * adjuster;
@@ -266,6 +274,7 @@ impl BucketCount {
*self.buckets.entry(hashes.site).or_default() += 1;
*self.buckets.entry(hashes.url).or_default() += 1;
+ *self.buckets.entry(hashes.url_without_tld).or_default() += 1;
*self.buckets.entry(hashes.title).or_default() += 1;
}
}
@@ -310,13 +319,15 @@ impl<T: Doc> BucketCollector<T> {
pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {
let mut res = Vec::new();
+ let mut simhash_dups = Vec::new();
let mut simhash = simhash::Table::default();
while let Some(best_doc) = self.documents.pop_max() {
let hashes = best_doc.doc.hashes();
- if hashes.simhash != 0 {
+ if hashes.simhash != 0 && de_rank_similar {
if simhash.contains(&hashes.simhash) {
+ simhash_dups.push(best_doc.doc);
continue;
}
simhash.insert(hashes.simhash);
@@ -334,6 +345,8 @@ impl<T: Doc> BucketCollector<T> {
}
}
+ res.extend(simhash_dups);
+
res
}
}
@@ -505,6 +518,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ url_without_tld: 1.into(),
simhash: 12,
},
123,
@@ -515,6 +529,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ url_without_tld: 2.into(),
simhash: 123,
},
124,
@@ -525,6 +540,7 @@ mod tests {
site: 3.into(),
title: 3.into(),
url: 3.into(),
+ url_without_tld: 3.into(),
simhash: 1234,
},
125,
@@ -535,6 +551,7 @@ mod tests {
site: 4.into(),
title: 4.into(),
url: 4.into(),
+ url_without_tld: 4.into(),
simhash: 12345,
},
126,
@@ -545,6 +562,7 @@ mod tests {
site: 5.into(),
title: 5.into(),
url: 5.into(),
+ url_without_tld: 5.into(),
simhash: 123456,
},
127,
@@ -565,6 +583,7 @@ mod tests {
site: 3.into(),
title: 3.into(),
url: 3.into(),
+ url_without_tld: 3.into(),
simhash: 12,
},
125,
@@ -575,6 +594,7 @@ mod tests {
site: 4.into(),
title: 4.into(),
url: 4.into(),
+ url_without_tld: 4.into(),
simhash: 123,
},
126,
@@ -585,6 +605,7 @@ mod tests {
site: 5.into(),
title: 5.into(),
url: 5.into(),
+ url_without_tld: 5.into(),
simhash: 1234,
},
127,
@@ -605,6 +626,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ url_without_tld: 1.into(),
simhash: 12,
},
125,
@@ -615,6 +637,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ url_without_tld: 2.into(),
simhash: 123,
},
126,
@@ -625,6 +648,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ url_without_tld: 2.into(),
simhash: 1234,
},
127,
@@ -642,6 +666,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ url_without_tld: 1.into(),
simhash: 12,
},
125,
@@ -652,6 +677,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ url_without_tld: 2.into(),
simhash: 123,
},
126,
@@ -662,6 +688,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ url_without_tld: 2.into(),
simhash: 1234,
},
127,
@@ -682,6 +709,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ url_without_tld: 1.into(),
simhash: 1234,
},
125,
@@ -692,6 +720,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ url_without_tld: 2.into(),
simhash: 1234,
},
126,
@@ -702,13 +731,14 @@ mod tests {
site: 3.into(),
title: 3.into(),
url: 3.into(),
+ url_without_tld: 3.into(),
simhash: 1,
},
127,
5.0,
),
],
- &[(5.0, 127), (3.1, 126)],
+ &[(5.0, 127), (3.1, 126), (3.0, 125)],
);
}
}
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index bf574a22..35722849 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -29,6 +29,10 @@ impl Collector {
20.0
}
+ pub fn url_without_tld_penalty() -> f64 {
+ 1.0
+ }
+
pub fn max_docs_considered() -> usize {
250_000
}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 9684a51c..447c9a00 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -148,6 +148,9 @@ pub struct CollectorConfig {
#[serde(default = "defaults::Collector::url_penalty")]
pub url_penalty: f64,
+ #[serde(default = "defaults::Collector::url_without_tld_penalty")]
+ pub url_without_tld_penalty: f64,
+
#[serde(default = "defaults::Collector::max_docs_considered")]
pub max_docs_considered: usize,
}
@@ -158,6 +161,7 @@ impl Default for CollectorConfig {
site_penalty: defaults::Collector::site_penalty(),
title_penalty: defaults::Collector::title_penalty(),
url_penalty: defaults::Collector::url_penalty(),
+ url_without_tld_penalty: defaults::Collector::url_without_tld_penalty(),
max_docs_considered: defaults::Collector::max_docs_considered(),
}
}
@@ -380,6 +384,7 @@ pub struct CrawlPlannerConfig {
pub crawl_budget: usize,
pub top_host_fraction: f64,
pub wander_fraction: f64,
+ pub top_n_hosts_surplus: usize,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rs
index 16daaa2a..6a6d22f5 100644
--- a/core/src/crawler/planner.rs
+++ b/core/src/crawler/planner.rs
@@ -15,8 +15,10 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::{anyhow, Result};
use indicatif::ParallelProgressIterator;
+use indicatif::ProgressIterator;
use rayon::prelude::*;
use rayon::ThreadPoolBuilder;
+use std::collections::BTreeMap;
use std::collections::HashMap;
use std::{
collections::VecDeque,
@@ -35,6 +37,8 @@ use crate::{
use super::Domain;
+const MAX_SURPLUS_BUDGET_ITERATIONS: usize = 100;
+
fn all_pages(
page_centrality: &RocksDbStore<NodeID, f64>,
page_graph: &Webgraph,
@@ -47,14 +51,14 @@ fn all_pages(
.collect::<Vec<_>>()
}
-fn group_domain(hosts: Vec<NodeID>, host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {
+fn group_domain(hosts: &[NodeID], host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {
let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();
for host in hosts {
- let node = host_graph.id2node(&host).unwrap();
+ let node = host_graph.id2node(host).unwrap();
if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {
let domain = Domain::from(url);
- domains.entry(domain).or_default().push(host);
+ domains.entry(domain).or_default().push(*host);
}
}
@@ -101,8 +105,11 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
let num_hosts = hosts.len();
tracing::info!("found {} hosts", num_hosts);
- let total_host_centrality = host_centrality.iter().map(|(_, v)| v).sum::<f64>();
- let grouped = group_domain(hosts, &host_graph);
+ let mut total_host_centrality = hosts
+ .iter()
+ .filter_map(|v| host_centrality.get(v))
+ .sum::<f64>();
+ let grouped = group_domain(&hosts, &host_graph);
let num_groups = grouped.len();
let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)
@@ -126,6 +133,85 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
.build()?;
pool.install(|| {
+ let host_pages: BTreeMap<_, _> = hosts
+ .par_iter()
+ .progress_count(num_hosts as u64)
+ .map(|host| {
+ let num_pages = page_graph.pages_by_host(host).len() as u64;
+ (*host, num_pages)
+ })
+ .collect();
+
+ let mut host_budgets: BTreeMap<_, _> = hosts
+ .par_iter()
+ .progress_count(num_hosts as u64)
+ .map(|host| {
+ let host_centrality = host_centrality.get(host).unwrap_or_default();
+
+ let host_budget = ((config.crawl_budget as f64 * host_centrality)
+ / total_host_centrality)
+ .round()
+ .max(0.0) as u64;
+
+ let num_pages = host_pages.get(host).copied().unwrap_or_default();
+
+ (*host, host_budget.min(num_pages as u64))
+ })
+ .collect();
+
+ let mut surplus_budget = config.crawl_budget as u64
+ - host_budgets
+ .values()
+ .sum::<u64>()
+ .min(config.crawl_budget as u64);
+
+ let mut has_updated = true;
+ let mut i = 0;
+ while surplus_budget > 0 && has_updated && i < MAX_SURPLUS_BUDGET_ITERATIONS {
+ tracing::info!("trying to schedule surplus budget: {}", surplus_budget);
+ i += 1;
+
+ has_updated = false;
+ let mut new_surplus_budget = surplus_budget;
+ let mut new_total_host_centrality = 0.0;
+
+ for host in hosts.iter().take(config.top_n_hosts_surplus).progress() {
+ if new_surplus_budget == 0 {
+ break;
+ }
+
+ let num_pages = host_pages.get(host).copied().unwrap_or_default();
+ let host_budget = host_budgets.get_mut(host).unwrap();
+
+ if num_pages > *host_budget {
+ let centrality = host_centrality.get(host).unwrap_or_default();
+ new_total_host_centrality += centrality;
+
+ let part = (centrality * surplus_budget as f64 / total_host_centrality)
+ .round()
+ .max(0.0) as u64;
+
+ let diff = num_pages - *host_budget;
+ let extra = part.min(diff);
+
+ if extra > 0 {
+ has_updated = true;
+ }
+
+ let extra = ((extra as f64 / (1.0 - config.wander_fraction)) as u64)
+ .min(new_surplus_budget);
+
+ *host_budget += extra;
+ new_surplus_budget -= extra;
+ }
+ }
+
+ total_host_centrality = new_total_host_centrality;
+ surplus_budget = new_surplus_budget;
+ }
+
+ tracing::info!("surplus done (remaining={})", surplus_budget);
+
grouped
.into_par_iter()
.progress_count(num_groups as u64)
@@ -137,10 +223,12 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
let mut total_wander_budget = 0;
let mut total_schedule_budget = 0;
let mut total_scheduled_urls = 0;
+ let mut total_known_urls = 0;
let mut urls = VecDeque::new();
for host in &hosts {
let mut pages = all_pages(&page_centrality, &page_graph, *host);
+ total_known_urls += pages.len();
if pages.is_empty() {
continue;
@@ -148,16 +236,13 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));
tracing::debug!("num pages: {}", pages.len());
- let host_centrality = host_centrality.get(host).unwrap_or_default();
+ let host_budget = host_budgets.get(host).copied().unwrap_or_default();
- let host_budget = ((config.crawl_budget as f64 * host_centrality)
- / total_host_centrality)
- .max(0.0)
- .round() as u64;
tracing::debug!("host_budget: {host_budget}");
let schedule_budget = (host_budget as f64 * (1.0 - config.wander_fraction))
- .max(0.0)
- .round() as u64;
+ .round()
+ .max(0.0) as u64;
+
tracing::debug!("schedule_budget: {schedule_budget}");
let wander_budget = (host_budget as f64 * config.wander_fraction)
.max(0.0)
@@ -200,6 +285,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
schedule_budget: total_schedule_budget,
wander_budget: total_wander_budget,
scheduled_urls: total_scheduled_urls,
+ known_urls: total_known_urls,
};
stats
.lock()
@@ -227,6 +313,16 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),
};
+ tracing::info!(
+ "total scheduled urls: {}",
+ metadata.stats.iter().map(|d| d.scheduled_urls).sum::<u64>()
+ );
+
+ tracing::info!(
+ "total wander budget: {}",
+ metadata.stats.iter().map(|d| d.wander_budget).sum::<u64>()
+ );
+
metadata
.stats
.sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));
@@ -242,6 +338,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(
#[derive(serde::Serialize)]
struct DomainStats {
domain: Domain,
+ known_urls: usize,
num_hosts: usize,
schedule_budget: u64,
scheduled_urls: u64,
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 9e30e40b..da502782 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -49,7 +49,6 @@ const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
struct ProcessedUrl {
new_urls: Vec<Url>,
response: UrlResponse,
- fetch_time: Duration,
}
pub struct WorkerThread {
@@ -238,18 +237,6 @@ impl<S: DatumStream> JobExecutor<S> {
let res = self.process_url(retryable_url.url.clone()).await;
- let mut delay = res.fetch_time.mul_f32(self.politeness_factor);
-
- if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {
- delay = Duration::from_millis(self.config.min_crawl_delay_ms);
- }
-
- if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {
- delay = Duration::from_millis(self.config.max_crawl_delay_ms);
- }
-
- tokio::time::sleep(delay).await;
-
match res.response {
UrlResponse::Success { url } => {
self.crawled_urls.insert(url.clone());
@@ -337,7 +324,6 @@ impl<S: DatumStream> JobExecutor<S> {
ProcessedUrl {
new_urls,
response: url_res,
- fetch_time: Duration::from_millis(datum.fetch_time_ms),
}
}
Err(_) => ProcessedUrl {
@@ -346,7 +332,6 @@ impl<S: DatumStream> JobExecutor<S> {
url,
status_code: None,
},
- fetch_time: Duration::from_millis(datum.fetch_time_ms),
},
}
} else {
@@ -358,7 +343,6 @@ impl<S: DatumStream> JobExecutor<S> {
ProcessedUrl {
new_urls: Vec::new(),
response: url_res,
- fetch_time: Duration::from_millis(datum.fetch_time_ms),
}
}
} else {
@@ -383,7 +367,6 @@ impl<S: DatumStream> JobExecutor<S> {
url,
status_code: Some(datum.status_code),
},
- fetch_time: Duration::from_millis(datum.fetch_time_ms),
}
}
}
@@ -396,7 +379,6 @@ impl<S: DatumStream> JobExecutor<S> {
url,
status_code: None,
},
- fetch_time: Duration::from_millis(0),
}
}
}
@@ -438,6 +420,18 @@ impl<S: DatumStream> JobExecutor<S> {
let res = self.fetch(url.clone()).await?;
let fetch_time = start.elapsed();
+ let mut delay = fetch_time.mul_f32(self.politeness_factor);
+
+ if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {
+ delay = Duration::from_millis(self.config.min_crawl_delay_ms);
+ }
+
+ if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {
+ delay = Duration::from_millis(self.config.max_crawl_delay_ms);
+ }
+
+ tokio::time::sleep(delay).await;
+
let headers: HashMap<_, _> = res
.headers()
.iter()
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 323714ec..2f9d0f2e 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -1280,76 +1280,6 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com/");
}
- #[test]
- fn remove_duplicates() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
-
- index
- .insert(
- Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>News website</title>
- </head>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- ),
- "https://www.dr.xyz",
- )
- .unwrap(),
- )
- .expect("failed to insert webpage");
-
- index
- .insert(
- Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>News website</title>
- </head>
- <body>
- {CONTENT} dr
- </body>
- </html>
- "#
- ),
- "https://www.dr.dk",
- )
- .unwrap(),
- )
- .expect("failed to insert webpage");
-
- index.commit().expect("failed to commit index");
-
- let ctx = index.local_search_ctx();
- let query = Query::parse(
- &ctx,
- &SearchQuery {
- query: "dr".to_string(),
- ..Default::default()
- },
- &index,
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- SignalAggregator::new(Some(&query)),
- ctx.fastfield_reader.clone(),
- Default::default(),
- );
-
- let result =
- search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.dr.dk/");
- }
-
#[test]
fn schema_org_stored() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 2c60ec63..8e6b5e5f 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -291,10 +291,11 @@ mod tests {
<meta property="og:updated_time" content="1999-06-22T19:37:34+00:00" />
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100),
),
"https://www.old.com",
)
@@ -312,13 +313,14 @@ mod tests {
<html>
<head>
<title>Title</title>
- <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
+ <meta property="og:updated_time" content="2023-06-22T19:37:34+00:00" />
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.new.com",
)
@@ -334,6 +336,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
query: "title".to_string(),
+ return_ranking_signals: true,
..Default::default()
})
.expect("Search failed");
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index e952bb1b..0ca2709f 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -421,6 +421,7 @@ mod tests {
site: Prehashed(0),
title: Prehashed(0),
url: Prehashed(0),
+ url_without_tld: Prehashed(0),
simhash: 0,
},
address: DocAddress {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 047e5b44..85f49f80 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -747,7 +747,7 @@ impl SignalAggregator {
.collect(),
});
- Self {
+ let mut s = Self {
segment_reader: None,
inbound_similarity: None,
query_signal_coefficients,
@@ -759,7 +759,11 @@ impl SignalAggregator {
current_timestamp: None,
linear_regression: None,
query,
- }
+ };
+
+ s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);
+
+ s
}
fn prepare_textfields(
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 88f236a2..b0d250ba 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -219,6 +219,8 @@ pub enum FastField {
UrlHash2,
DomainHash1,
DomainHash2,
+ UrlWithoutTldHash1,
+ UrlWithoutTldHash2,
PreComputedScore,
HostNodeID1,
HostNodeID2,
@@ -256,6 +258,8 @@ impl FastField {
FastField::UrlHash2 => "url_hash2",
FastField::DomainHash1 => "domain_hash1",
FastField::DomainHash2 => "domain_hash2",
+ FastField::UrlWithoutTldHash1 => "url_without_tld_hash1",
+ FastField::UrlWithoutTldHash2 => "url_without_tld_hash2",
FastField::PreComputedScore => "pre_computed_score",
FastField::HostNodeID1 => "host_node_id1",
FastField::HostNodeID2 => "host_node_id2",
@@ -278,7 +282,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 61] = [
+pub static ALL_FIELDS: [Field; 63] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -335,6 +339,8 @@ pub static ALL_FIELDS: [Field; 61] = [
Field::Fast(FastField::UrlHash2),
Field::Fast(FastField::DomainHash1),
Field::Fast(FastField::DomainHash2),
+ Field::Fast(FastField::UrlWithoutTldHash1),
+ Field::Fast(FastField::UrlWithoutTldHash2),
Field::Fast(FastField::PreComputedScore),
Field::Fast(FastField::HostNodeID1),
Field::Fast(FastField::HostNodeID2),
@@ -517,6 +523,12 @@ impl Field {
Field::Fast(FastField::UrlHash2) => {
IndexingOption::Integer(NumericOptions::default().set_fast())
}
+ Field::Fast(FastField::UrlWithoutTldHash1) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
+ Field::Fast(FastField::UrlWithoutTldHash2) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast())
+ }
Field::Fast(FastField::DomainHash1) => {
IndexingOption::Integer(NumericOptions::default().set_fast())
}
@@ -673,6 +685,8 @@ impl FastField {
FastField::UrlHash2 => DataType::U64,
FastField::DomainHash1 => DataType::U64,
FastField::DomainHash2 => DataType::U64,
+ FastField::UrlWithoutTldHash1 => DataType::U64,
+ FastField::UrlWithoutTldHash2 => DataType::U64,
FastField::PreComputedScore => DataType::U64,
FastField::HostNodeID1 => DataType::U64,
FastField::HostNodeID2 => DataType::U64,
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index 5c13cafa..a94576d4 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -233,11 +233,11 @@ impl ApiSearcher {
Ok(None)
}
- async fn sidebar(
+ fn sidebar(
&self,
initial_results: &[distributed::InitialSearchResultShard],
- query: &SearchQuery,
- ) -> Result<Option<DisplayedSidebar>> {
+ stackoverflow: Option<DisplayedSidebar>,
+ ) -> Option<DisplayedSidebar> {
let entity = initial_results
.iter()
.filter_map(|res| res.local_result.entity_sidebar.clone())
@@ -250,8 +250,8 @@ impl ApiSearcher {
});
match entity {
- Some(entity) => Ok(Some(DisplayedSidebar::Entity(entity))),
- None => Ok(self.stackoverflow_sidebar(query).await?),
+ Some(entity) => Some(DisplayedSidebar::Entity(entity)),
+ None => stackoverflow,
}
}
@@ -434,15 +434,17 @@ impl ApiSearcher {
self.collector_config.clone(),
)?;
- let (initial_results, live_results, widget, discussions) = tokio::join!(
+ let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(
self.distributed_searcher.search_initial(&search_query),
self.live_searcher.search_initial(&search_query),
self.widget(query),
- self.discussions_widget(query)
+ self.discussions_widget(query),
+ self.stackoverflow_sidebar(query),
);
let discussions = discussions?;
- let sidebar = self.sidebar(&initial_results, query).await?;
+ let stackoverflow = stackoverflow?;
+ let sidebar = self.sidebar(&initial_results, stackoverflow);
let spell_corrected_query = if widget.is_none() {
initial_results
@@ -486,15 +488,11 @@ impl ApiSearcher {
})
.collect();
- let retrieved_normal: Vec<_> = self
- .distributed_searcher
- .retrieve_webpages(&normal, &query.query)
- .await;
-
- let retrieved_live: Vec<_> = self
- .live_searcher
- .retrieve_webpages(&live, &query.query)
- .await;
+ let (retrieved_normal, retrieved_live) = tokio::join!(
+ self.distributed_searcher
+ .retrieve_webpages(&normal, &query.query),
+ self.live_searcher.retrieve_webpages(&live, &query.query),
+ );
let mut retrieved_webpages: Vec<_> =
retrieved_normal.into_iter().chain(retrieved_live).collect();
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 65253c5a..b3081d88 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -137,7 +137,7 @@ impl Segment {
}
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
- self.full_adjacency.nodes_by_prefix(host_node)
+ self.full_reversed_adjacency.nodes_by_prefix(host_node)
}
pub fn id(&self) -> String {
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index a0690f3a..5f167346 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -1032,6 +1032,20 @@ impl Html {
let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
let url_hash = split_u128(hash(self.url().as_str()).0);
+ let tld = self.url().tld().unwrap_or_default();
+ let url_without_tld = self
+ .url()
+ .host_str()
+ .unwrap_or_default()
+ .trim_end_matches(&tld)
+ .to_string()
+ + "/"
+ + self.url().path()
+ + "?"
+ + self.url().query().unwrap_or_default();
+
+ let url_without_tld_hash = split_u128(hash(url_without_tld).0);
+
let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);
let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
@@ -1286,6 +1300,12 @@ impl Html {
Field::Fast(FastField::UrlHash2) => {
doc.add_u64(tantivy_field, url_hash[1]);
}
+ Field::Fast(FastField::UrlWithoutTldHash1) => {
+ doc.add_u64(tantivy_field, url_without_tld_hash[0]);
+ }
+ Field::Fast(FastField::UrlWithoutTldHash2) => {
+ doc.add_u64(tantivy_field, url_without_tld_hash[1]);
+ }
Field::Fast(FastField::DomainHash1) => {
doc.add_u64(tantivy_field, domain_hash[0]);
}
diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rs
index f166b289..40311545 100644
--- a/core/src/webpage/url_ext.rs
+++ b/core/src/webpage/url_ext.rs
@@ -31,6 +31,7 @@ pub trait UrlExt {
fn icann_domain(&self) -> Option<&str>;
fn root_domain(&self) -> Option<&str>;
fn subdomain(&self) -> Option<&str>;
+ fn tld(&self) -> Option<&str>;
}
impl UrlExt for url::Url {
@@ -58,6 +59,12 @@ impl UrlExt for url::Url {
Some(subdomain)
}
+
+ fn tld(&self) -> Option<&str> {
+ let host = self.host_str()?;
+ let suffix = std::str::from_utf8(ICANN_LIST.suffix(host.as_bytes())?.as_bytes()).ok()?;
+ Some(suffix)
+ }
}
#[cfg(test)]
@@ -85,4 +92,13 @@ mod tests {
assert_eq!(url.domain().unwrap(), "example.blogspot.com");
assert_eq!(url.icann_domain().unwrap(), "blogspot.com");
}
+
+ #[test]
+ fn suffix() {
+ let url: Url = Url::parse("http://example.blogspot.com").unwrap();
+ assert_eq!(url.tld().unwrap(), "com");
+
+ let url: Url = Url::parse("http://example.com").unwrap();
+ assert_eq!(url.tld().unwrap(), "com");
+ }
}
diff --git a/frontend/src/lib/improvements.ts b/frontend/src/lib/improvements.ts
index 68db224c..f692d3a3 100644
--- a/frontend/src/lib/improvements.ts
+++ b/frontend/src/lib/improvements.ts
@@ -1,9 +1,6 @@
import type { Action } from 'svelte/action';
-import { writable } from 'svelte/store';
import { getApiBase, type ApiOptions, type DisplayedWebpage, requestPlain } from './api';
-import { allowStatsStore } from './stores';
-
-const queryIdStore = writable<string | undefined>();
+import { allowStatsStore, queryIdStore } from './stores';
export const updateQueryId = async ({
query,
diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts
index b718ecae..25425f9c 100644
--- a/frontend/src/lib/stores.ts
+++ b/frontend/src/lib/stores.ts
@@ -67,6 +67,10 @@ export const siteRankingsStore = writableLocalStorage<SiteRakings>(SITE_RANKINGS
const SEARCH_QUERY_KEY = 'searchQuery';
export const searchQueryStore = writableLocalStorage<string | undefined>(SEARCH_QUERY_KEY, void 0);
+const QUERY_ID_KEY = 'queryId';
+export const queryIdStore = writableLocalStorage<string | undefined>(QUERY_ID_KEY, void 0);
+
+
const THEME_KEY = 'theme';
export const themeStore = writableLocalStorage<string | void>(THEME_KEY, void 0);
if (browser)
diff --git a/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
deleted file mode 100644
index 85a1d3f0..00000000
--- a/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
+++ /dev/null
@@ -1,28 +0,0 @@
-<script lang="ts">
- import type { PageData } from "./$types";
- export let data: PageData;
-</script>
-
-<div class="flex flex-col w-full items-center">
- <div class="max-w-xl flex flex-col space-y-5">
- <div>
- <b class="font-bold">Query:</b> {data.query.query}
- </div>
-
- <div class="flex flex-col space-y-3">
- {#each data.searchResults as res}
- <div class="py-2 px-2 border shadow-sm rounded-lg">
- <div>
- <a href="{res.webpage.url}">{res.webpage.url}</a>
- </div>
- <div>
- <a href="{res.webpage.url}">{res.webpage.title}</a>
- </div>
- <div class="line-clamp-3 text-sm">
- {res.webpage.snippet}
- </div>
- </div>
- {/each}
- </div>
- </div>
-</div>
\ No newline at end of file
diff --git a/ranking-annotation/vite.config.ts b/ranking-annotation/vite.config.ts
deleted file mode 100644
index 80864b9d..00000000
--- a/ranking-annotation/vite.config.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import { sveltekit } from "@sveltejs/kit/vite";
-import { defineConfig } from "vite";
-
-export default defineConfig({
- plugins: [sveltekit()],
-});
diff --git a/ranking-annotation/.gitignore b/tools/ranking-annotation/.gitignore
similarity index 100%
rename from ranking-annotation/.gitignore
rename to tools/ranking-annotation/.gitignore
diff --git a/ranking-annotation/.npmrc b/tools/ranking-annotation/.npmrc
similarity index 100%
rename from ranking-annotation/.npmrc
rename to tools/ranking-annotation/.npmrc
diff --git a/ranking-annotation/README.md b/tools/ranking-annotation/README.md
similarity index 100%
rename from ranking-annotation/README.md
rename to tools/ranking-annotation/README.md
diff --git a/tools/ranking-annotation/global.d.ts b/tools/ranking-annotation/global.d.ts
new file mode 100644
index 00000000..13d619aa
--- /dev/null
+++ b/tools/ranking-annotation/global.d.ts
@@ -0,0 +1,8 @@
+declare type Item = import("svelte-dnd-action").Item;
+declare type DndEvent<ItemType = Item> = import("svelte-dnd-action").DndEvent<ItemType>;
+declare namespace svelteHTML {
+ interface HTMLAttributes<T> {
+ "on:consider"?: (event: CustomEvent<DndEvent<ItemType>> & {target: EventTarget & T}) => void;
+ "on:finalize"?: (event: CustomEvent<DndEvent<ItemType>> & {target: EventTarget & T}) => void;
+ }
+}
\ No newline at end of file
diff --git a/ranking-annotation/package.json b/tools/ranking-annotation/package.json
similarity index 86%
rename from ranking-annotation/package.json
rename to tools/ranking-annotation/package.json
index 1154e510..66e6ef62 100644
--- a/ranking-annotation/package.json
+++ b/tools/ranking-annotation/package.json
@@ -12,6 +12,9 @@
"format": "prettier --plugin prettier-plugin-svelte --plugin prettier-plugin-tailwindcss --write ."
},
"devDependencies": {
+ "@iconify-json/heroicons": "^1.1.12",
+ "@iconify-json/simple-icons": "^1.1.69",
+ "@iconify/svelte": "^3.1.4",
"@sveltejs/adapter-auto": "^2.0.0",
"@sveltejs/kit": "^1.20.4",
"@types/better-sqlite3": "^7.6.6",
@@ -30,12 +33,14 @@
"tailwindcss": "^3.3.5",
"tslib": "^2.4.1",
"typescript": "^5.0.0",
+ "unplugin-icons": "^0.17.0",
"vite": "^4.4.2"
},
"dependencies": {
"better-sqlite3": "^9.0.0",
"cassandra-driver": "^4.7.2",
- "papaparse": "^5.4.1"
+ "papaparse": "^5.4.1",
+ "svelte-dnd-action": "^0.9.31"
},
"type": "module"
}
diff --git a/ranking-annotation/postcss.config.js b/tools/ranking-annotation/postcss.config.js
similarity index 100%
rename from ranking-annotation/postcss.config.js
rename to tools/ranking-annotation/postcss.config.js
diff --git a/ranking-annotation/src/app.css b/tools/ranking-annotation/src/app.css
similarity index 100%
rename from ranking-annotation/src/app.css
rename to tools/ranking-annotation/src/app.css
diff --git a/ranking-annotation/src/app.d.ts b/tools/ranking-annotation/src/app.d.ts
similarity index 87%
rename from ranking-annotation/src/app.d.ts
rename to tools/ranking-annotation/src/app.d.ts
index 899c7e8f..843ae573 100644
--- a/ranking-annotation/src/app.d.ts
+++ b/tools/ranking-annotation/src/app.d.ts
@@ -1,3 +1,4 @@
+import 'unplugin-icons/types/svelte'
// See https://kit.svelte.dev/docs/types#app
// for information about these interfaces
declare global {
diff --git a/ranking-annotation/src/app.html b/tools/ranking-annotation/src/app.html
similarity index 100%
rename from ranking-annotation/src/app.html
rename to tools/ranking-annotation/src/app.html
diff --git a/ranking-annotation/src/lib/dataset.ts b/tools/ranking-annotation/src/lib/dataset.ts
similarity index 89%
rename from ranking-annotation/src/lib/dataset.ts
rename to tools/ranking-annotation/src/lib/dataset.ts
index 9881d2c4..131f1f03 100644
--- a/ranking-annotation/src/lib/dataset.ts
+++ b/tools/ranking-annotation/src/lib/dataset.ts
@@ -11,7 +11,7 @@ export function getDataset(): Data | undefined {
return dataset;
}
- const csvFilePath = path.resolve("../data/queries_us.csv");
+ const csvFilePath = path.resolve("../../data/queries_us.csv");
const csvFile = readFileSync(csvFilePath, "utf-8");
Papa.parse(csvFile, {
diff --git a/ranking-annotation/src/lib/db.ts b/tools/ranking-annotation/src/lib/db.ts
similarity index 54%
rename from ranking-annotation/src/lib/db.ts
rename to tools/ranking-annotation/src/lib/db.ts
index bc7d04ad..446df6e4 100644
--- a/ranking-annotation/src/lib/db.ts
+++ b/tools/ranking-annotation/src/lib/db.ts
@@ -2,7 +2,7 @@ import Database from "better-sqlite3";
import { getDataset } from "./dataset";
import type { SimpleWebpage } from "./webpage";
-const db = new Database("../data/ranking-annotation.sqlite", {
+const db = new Database("../../data/ranking-annotation.sqlite", {
fileMustExist: false,
});
@@ -22,7 +22,7 @@ function setupDB() {
url TEXT NOT NULL,
orig_rank INTEGER NOT NULL,
webpage_json TEXT NOT NULL,
- anotated_rank INTEGER,
+ annotated_rank INTEGER,
PRIMARY KEY (qid, url)
);
`);
@@ -50,15 +50,48 @@ export type Query = {
export function getQueries(): Query[] {
const query = db.prepare(`
SELECT qid, query, EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.anotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
) AS annotated
- FROM queries
+ FROM queries ORDER BY qid
`);
return query.all() as Query[];
}
+export function getNextQuery(qid: string): Query | undefined {
+ const query = db.prepare(`
+ SELECT qid, query, EXISTS (
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ ) AS annotated
+ FROM queries
+ WHERE qid > @qid
+ ORDER BY qid
+ LIMIT 1
+ `);
+
+ const res = query.get({ qid });
+
+ return res as Query | undefined;
+}
+
+export function getPreviousQuery(qid: string): Query | undefined {
+ const query = db.prepare(`
+ SELECT qid, query, EXISTS (
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
+ ) AS annotated
+ FROM queries
+ WHERE qid < @qid
+ ORDER BY qid DESC
+ LIMIT 1
+ `);
+
+ const res = query.get({ qid });
+
+ return res as Query | undefined;
+}
+
export type SearchResult = {
+ id: string;
origRank: number;
webpage: SimpleWebpage;
annotatedRank: number | null;
@@ -66,7 +99,7 @@ export type SearchResult = {
export function getSearchResults(qid: string): SearchResult[] {
const queryResults = db.prepare(`
- SELECT url, orig_rank as origRank, webpage_json, anotated_rank AS annotatedRank
+ SELECT url, orig_rank as origRank, webpage_json, annotated_rank AS annotatedRank
FROM search_results
WHERE qid = @qid
`);
@@ -74,6 +107,7 @@ export function getSearchResults(qid: string): SearchResult[] {
const res = queryResults.all({ qid });
return res.map((r: any) => ({
+ id: `${qid}-${r.url}`,
origRank: r.origRank,
annotatedRank: r.annotatedRank,
webpage: JSON.parse(r.webpage_json),
@@ -81,29 +115,33 @@ export function getSearchResults(qid: string): SearchResult[] {
}
export const saveSearchResults = (qid: string, searchResults: SearchResult[]) => {
- const insertSearchResults = db.prepare(`
- INSERT INTO search_results (qid, url, orig_rank, webpage_json)
- VALUES (@qid, @url, @origRank, @webpageJson)
- `);
+ const upsertSearchResults = db.prepare(`
+ INSERT INTO search_results (qid, url, orig_rank, webpage_json, annotated_rank)
+ VALUES (@qid, @url, @origRank, @webpageJson, @annotatedRank)
+ ON CONFLICT(qid, url) DO UPDATE SET
+ annotated_rank = excluded.annotated_rank
+ `);
for (const searchResult of searchResults) {
- insertSearchResults.run({
+ upsertSearchResults.run({
qid,
url: searchResult.webpage.url,
origRank: searchResult.origRank,
webpageJson: JSON.stringify(searchResult.webpage),
+ annotatedRank: searchResult.annotatedRank,
});
}
}
-export function getQuery(qid: String): Query {
+export function getQuery(qid: String): Query | undefined {
const query = db.prepare(`
SELECT qid, query, EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.anotated_rank IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL
) AS annotated
FROM queries
WHERE qid = @qid
`);
+ const res = query.get({ qid });
- return query.get({ qid }) as Query;
+ return res as Query | undefined;
}
diff --git a/ranking-annotation/src/lib/index.ts b/tools/ranking-annotation/src/lib/index.ts
similarity index 100%
rename from ranking-annotation/src/lib/index.ts
rename to tools/ranking-annotation/src/lib/index.ts
diff --git a/ranking-annotation/src/lib/webpage.ts b/tools/ranking-annotation/src/lib/webpage.ts
similarity index 100%
rename from ranking-annotation/src/lib/webpage.ts
rename to tools/ranking-annotation/src/lib/webpage.ts
diff --git a/ranking-annotation/src/routes/+layout.svelte b/tools/ranking-annotation/src/routes/+layout.svelte
similarity index 100%
rename from ranking-annotation/src/routes/+layout.svelte
rename to tools/ranking-annotation/src/routes/+layout.svelte
diff --git a/ranking-annotation/src/routes/+page.server.ts b/tools/ranking-annotation/src/routes/+page.server.ts
similarity index 100%
rename from ranking-annotation/src/routes/+page.server.ts
rename to tools/ranking-annotation/src/routes/+page.server.ts
diff --git a/ranking-annotation/src/routes/+page.svelte b/tools/ranking-annotation/src/routes/+page.svelte
similarity index 100%
rename from ranking-annotation/src/routes/+page.svelte
rename to tools/ranking-annotation/src/routes/+page.svelte
diff --git a/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
similarity index 76%
rename from ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
rename to tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
index 06837f7f..f7fc2492 100644
--- a/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts
@@ -1,6 +1,7 @@
-import { getQuery, getSearchResults, saveSearchResults } from "$lib/db";
+import { getNextQuery, getPreviousQuery, getQuery, getSearchResults, saveSearchResults } from "$lib/db";
import type { Webpage } from "$lib/webpage";
import { asSimpleWebpage } from "$lib/webpage";
+import { redirect } from "@sveltejs/kit";
const search = async (query: string) => {
return await fetch(`https://trystract.com/beta/api/search`, {
@@ -25,13 +26,22 @@ export const load = async ({ params }) => {
const { slug } = params;
const qid = slug;
- const query = getQuery(qid);
+
+ if (!getQuery(qid)) {
+ throw redirect(301, "/");
+ }
+ const query = getQuery(qid)!;
+
+ const previousQuery = getPreviousQuery(qid);
+ const nextQuery = getNextQuery(qid);
+
let searchResults = getSearchResults(query.qid);
if (searchResults.length === 0) {
const webpages = (await search(query.query)).map((w) => asSimpleWebpage(w));
searchResults = webpages.map((w, i) => ({
+ id: `${query.qid}-${w.url}`,
origRank: i,
annotatedRank: null,
webpage: w,
@@ -53,5 +63,5 @@ export const load = async ({ params }) => {
}
});
- return { query, searchResults };
+ return { query, searchResults, previousQuery, nextQuery };
};
\ No newline at end of file
diff --git a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
new file mode 100644
index 00000000..1a103788
--- /dev/null
+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte
@@ -0,0 +1,128 @@
+<script lang="ts">
+ import Check from '~icons/heroicons/check-20-solid';
+ import Home from '~icons/heroicons/home-20-solid';
+ import ArrowLeft from '~icons/heroicons/arrow-left-20-solid';
+ import ArrowRight from '~icons/heroicons/arrow-right-20-solid';
+ import { dndzone } from 'svelte-dnd-action';
+ import type { PageData } from "./$types";
+ import type { SearchResult } from '$lib/db';
+ import { browser } from '$app/environment';
+
+ $: shownSignals = -1;
+ $: searchResults = data.searchResults;
+
+ function updateRanks() {
+ searchResults.forEach((res, i) => {
+ if (res.annotatedRank != null) {
+ res.annotatedRank = i;
+ }
+ });
+ }
+
+ function handleConsider(event: CustomEvent<{ items: SearchResult[] }>) {
+ searchResults = event.detail.items;
+ }
+
+ async function handleDrop(event: CustomEvent<{ items: SearchResult[] }>) {
+ searchResults = event.detail.items;
+ updateRanks();
+ await save();
+ }
+
+ async function save() {
+ if (!browser) {
+ return;
+ }
+
+ await fetch(`/api/annotate/${data.query.qid}`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ qid: data.query.qid,
+ results: searchResults,
+ }),
+ })
+ }
+
+ $: searchResults && save();
+
+ export let data: PageData;
+</script>
+
+<div class="flex flex-col w-full items-center mt-5">
+ <div class="max-w-xl flex flex-col space-y-5">
+ <div class="flex w-full justify-around">
+ <div class="w-8 h-8">
+ <a href="/" class="w-full h-full text-sky-300 hover:text-sky-500">
+ <Home class="w-full h-full"/>
+ </a>
+ </div>
+ <div class="text-xl">
+ <div>
+ <b class="font-bold">Query:</b> {data.query.query}
+ </div>
+ </div>
+ <div class="flex">
+ <a href="/annotate/{data.previousQuery?.qid}" class="text-orange-300 hover:text-orange-500">
+ <ArrowLeft class="w-6 h-6"/>
+ </a>
+ <a href="/annotate/{data.nextQuery?.qid}" class="text-orange-300 hover:text-orange-500">
+ <ArrowRight class="w-6 h-6"/>
+ </a>
+ </div>
+ </div>
+ <section use:dndzone={{ items: searchResults, dropTargetStyle: {} }} on:consider={handleConsider} on:finalize={handleDrop} class="flex flex-col space-y-3">
+ {#each searchResults as res, i (res.id)}
+ <div>
+ <div class="relative">
+ <button class="absolute left-[-2em] top-3" on:click={() => (res.annotatedRank == null) ? res.annotatedRank = i : res.annotatedRank = null}>
+ {#if res.annotatedRank != null}
+ <div class="text-green-500">
+ <Check class="w-6 h-6" />
+ </div>
+ {:else}
+ <div class="text-slate-200">
+ <Check class="w-6 h-6" />
+ </div>
+ {/if}
+ </button>
+ <div class="py-2 px-2 bg-slate-100 border shadow-sm rounded-lg">
+ <div>
+ <a href="{res.webpage.url}" target="_blank">{res.webpage.url}</a>
+ </div>
+ <div class="font-bold">
+ <a href="{res.webpage.url}" target="_blank">{res.webpage.title}</a>
+ </div>
+ <div class="line-clamp-3 text-sm">
+ {res.webpage.snippet}
+ </div>
+ <div class="text-sm mt-2 text-slate-500">
+ Host centrality: {res.webpage.rankingSignals["host_centrality"].toFixed(3)}
+ Tracker score: {res.webpage.rankingSignals["tracker_score"].toFixed(3)}
+ </div>
+ <div>
+ <button class="text-sm text-slate-500" on:click={() => shownSignals = (shownSignals == i) ? -1 : i}>
+ {shownSignals === i ? "Hide" : "Show"} signals
+ </button>
+ </div>
+ {#if shownSignals === i}
+ <div class="flex flex-col">
+ {#each Object.entries(res.webpage.rankingSignals) as [name, value]}
+ <div class="flex flex-col text-sm">
+ <div class="flex flex-row space-x-2">
+ <div class="font-bold">{name}:</div>
+ <div>{value}</div>
+ </div>
+ </div>
+ {/each}
+ </div>
+ {/if}
+ </div>
+ </div>
+ </div>
+ {/each}
+ </section>
+ </div>
+</div>
\ No newline at end of file
diff --git a/tools/ranking-annotation/src/routes/api/annotate/[slug]/+server.ts b/tools/ranking-annotation/src/routes/api/annotate/[slug]/+server.ts
new file mode 100644
index 00000000..c1730632
--- /dev/null
+++ b/tools/ranking-annotation/src/routes/api/annotate/[slug]/+server.ts
@@ -0,0 +1,14 @@
+import type { SearchResult } from "$lib/db";
+import { saveSearchResults } from "$lib/db";
+
+export async function POST({request}): Promise<Response> {
+ const {qid, results}: {
+ qid: string;
+ results: SearchResult[];
+ } = await request.json();
+
+ saveSearchResults(qid, results);
+
+ return new Response("OK");
+}
+
\ No newline at end of file
diff --git a/ranking-annotation/src/routes/api/data/+server.ts b/tools/ranking-annotation/src/routes/api/data/+server.ts
similarity index 100%
rename from ranking-annotation/src/routes/api/data/+server.ts
rename to tools/ranking-annotation/src/routes/api/data/+server.ts
diff --git a/ranking-annotation/static/favicon.png b/tools/ranking-annotation/static/favicon.png
similarity index 100%
rename from ranking-annotation/static/favicon.png
rename to tools/ranking-annotation/static/favicon.png
diff --git a/ranking-annotation/svelte.config.js b/tools/ranking-annotation/svelte.config.js
similarity index 100%
rename from ranking-annotation/svelte.config.js
rename to tools/ranking-annotation/svelte.config.js
diff --git a/ranking-annotation/tailwind.config.ts b/tools/ranking-annotation/tailwind.config.ts
similarity index 100%
rename from ranking-annotation/tailwind.config.ts
rename to tools/ranking-annotation/tailwind.config.ts
diff --git a/ranking-annotation/tsconfig.json b/tools/ranking-annotation/tsconfig.json
similarity index 90%
rename from ranking-annotation/tsconfig.json
rename to tools/ranking-annotation/tsconfig.json
index 794b95b6..f237f2d8 100644
--- a/ranking-annotation/tsconfig.json
+++ b/tools/ranking-annotation/tsconfig.json
@@ -1,5 +1,6 @@
{
"extends": "./.svelte-kit/tsconfig.json",
+ "include": ["src/**/*", "global.d.ts", "vite.config.ts"],
"compilerOptions": {
"allowJs": true,
"checkJs": true,
diff --git a/tools/ranking-annotation/vite.config.ts b/tools/ranking-annotation/vite.config.ts
new file mode 100644
index 00000000..83cf07a0
--- /dev/null
+++ b/tools/ranking-annotation/vite.config.ts
@@ -0,0 +1,12 @@
+import { sveltekit } from "@sveltejs/kit/vite";
+import { defineConfig } from "vite";
+import Icons from 'unplugin-icons/vite';
+
+export default defineConfig({
+ plugins: [
+ sveltekit(),
+ Icons({
+ compiler: 'svelte',
+ })
+ ],
+});