web search done right
Fix crawl plan. Too many spam urls were being prioritized
Mikkel Denker3054fe5c6aa5
diff --git a/core/Cargo.toml b/core/Cargo.tomlindex f1ec40ab..1f93fb5d 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -117,10 +117,6 @@ maplit = {workspace = true}proptest = {workspace = true}proptest-derive = {workspace = true}-[[example]]-name = "alice"-required-features = ["with_alice"]-[[bench]]harness = falsename = "search-preindexed"diff --git a/core/examples/alice.rs b/core/examples/alice.rsdeleted file mode 100644index cb1babbe..00000000--- a/core/examples/alice.rs+++ /dev/null@@ -1,70 +0,0 @@-use std::io::Write;--use base64::Engine;-use stract::alice::{- ExecutionState, BASE64_ENGINE, {AcceleratorConfig, Alice},-};--#[tokio::main]-async fn main() {- let subscriber = tracing_subscriber::FmtSubscriber::builder()- .with_max_level(tracing::Level::DEBUG)- .finish();- tracing::subscriber::set_global_default(subscriber).unwrap();-- // dont use this key in production- let key = BASE64_ENGINE- .decode("URyJQTjwUjTq6FSRoZGdbUdTIvqs/QxkPacQio8Lhxc=")- .unwrap();-- let model = Alice::open(- "data/alice",- Some(AcceleratorConfig {- layer_fraction: 1.0,- quantize_fraction: 0.0,- device: tch::Device::Mps,- kind: tch::Kind::Float,- }),- &key,- )- .unwrap();-- tracing::debug!("model loaded");-- let mut last_state = None;-- loop {- println!();- print!("> ");- std::io::stdout().flush().unwrap();- let mut input = String::new();- std::io::stdin().read_line(&mut input).unwrap();- let input = input.trim();-- let gen = model- .new_executor(- input,- last_state.clone(),- "http://localhost:3000/beta/api/search".to_string(),- None,- )- .unwrap();-- for n in gen {- match n {- ExecutionState::BeginSearch { query: _ } => {}- ExecutionState::SearchResult {- query: _,- result: _,- } => {}- ExecutionState::Speaking { text } => {- print!("{}", text);- std::io::stdout().flush().unwrap();- }- ExecutionState::Done { state } => {- last_state = Some(state.decode().unwrap());- }- }- }- }-}diff --git a/core/examples/mapreduce_manager.rs b/core/examples/mapreduce_manager.rsdeleted file mode 100644index f2fc1d8f..00000000--- a/core/examples/mapreduce_manager.rs+++ /dev/null@@ -1,49 +0,0 @@-use std::net::SocketAddr;--use serde::{Deserialize, Serialize};-use stract::mapreduce::{Manager, Map, Reduce, StatelessWorker};-use tracing::Level;-use tracing_subscriber::FmtSubscriber;--#[derive(Clone, Serialize, Deserialize, Debug)]-struct Job {- id: usize,-}--#[derive(Serialize, Deserialize, Debug)]-struct Count(usize);--impl Map<StatelessWorker, Count> for Job {- fn map(&self, _worker: &StatelessWorker) -> Count {- std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task- Count(1)- }-}--impl Reduce<Count> for Count {- fn reduce(self, element: Self) -> Self {- Count(self.0 + element.0)- }-}--#[tokio::main]-async fn main() {- let subscriber = FmtSubscriber::builder()- .with_max_level(Level::DEBUG)- .finish();- tracing::subscriber::set_global_default(subscriber).unwrap();-- let jobs = vec![- Job { id: 0 },- Job { id: 1 },- Job { id: 2 },- Job { id: 3 },- Job { id: 4 },- Job { id: 5 },- ];-- let manager = Manager::new(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()]);- let res: Count = manager.run(jobs.into_iter()).await.unwrap();-- println!("{:?}", res.0);-}diff --git a/core/examples/mapreduce_worker.rs b/core/examples/mapreduce_worker.rsdeleted file mode 100644index ccf2dc00..00000000--- a/core/examples/mapreduce_worker.rs+++ /dev/null@@ -1,42 +0,0 @@-use std::net::SocketAddr;--use serde::{Deserialize, Serialize};-use stract::mapreduce::{Map, Reduce, StatelessWorker, Worker};-use tracing::Level;-use tracing_subscriber::FmtSubscriber;--#[derive(Serialize, Deserialize, Debug)]-struct Job {- id: usize,-}--#[derive(Serialize, Deserialize, Debug)]-struct Count(usize);--impl Map<StatelessWorker, Count> for Job {- fn map(&self, _worker: &StatelessWorker) -> Count {- std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task- Count(1)- }-}--impl Reduce<Count> for Count {- fn reduce(self, element: Self) -> Self {- Count(self.0 + element.0)- }-}--#[tokio::main]-async fn main() {- let subscriber = FmtSubscriber::builder()- .with_max_level(Level::DEBUG)- .finish();- tracing::subscriber::set_global_default(subscriber).unwrap();-- let args: Vec<_> = std::env::args().collect();-- StatelessWorker::default()- .run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())- .await- .expect("failed to run worker");-}diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsdeleted file mode 100644index dccdddd2..00000000--- a/core/examples/print_inbound_links.rs+++ /dev/null@@ -1,39 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use stract::{- ranking::inbound_similarity::InboundSimilarity,- webgraph::{Node, WebgraphBuilder},-};--pub fn main() {- let graph = WebgraphBuilder::new("data/webgraph").open();- let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();-- for host in ["www.homedepot.com"] {- println!("{host}:");-- let node = Node::from(host).into_host();- let inbound_vec = inbound.get(&node.id()).unwrap();- println!("{:?}", inbound_vec);-- for edge in graph.ingoing_edges(node) {- println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);- }-- println!();- }-}diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rsdeleted file mode 100644index fac809c6..00000000--- a/core/examples/print_similar_sites.rs+++ /dev/null@@ -1,79 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use stract::{- ranking::inbound_similarity::InboundSimilarity,- similar_sites::{ScoredNode, SimilarSitesFinder},- webgraph::WebgraphBuilder,-};--fn print_nodes(nodes: &[ScoredNode]) {- for (i, node) in nodes.iter().enumerate() {- println!("{i} \t {:.3} \t {} ", node.score, node.node.name);- }-}--fn print_top_nodes(liked_sites: &[&str], top_n: usize, similarity_finder: &SimilarSitesFinder) {- let sites: Vec<_> = liked_sites.iter().map(|s| s.to_string()).collect();- let top_nodes = similarity_finder.find_similar_sites(&sites, top_n);-- println!("top {top_n} sites for these liked sites {liked_sites:?}");- print_nodes(&top_nodes);-- println!();- println!();-}--pub fn main() {- const TOP_N: usize = 50;- let graph = WebgraphBuilder::new("data/webgraph").open();- let inbound_similarity = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();-- let similarity_finder = SimilarSitesFinder::new(- graph.into(),- inbound_similarity,- stract::config::defaults::WebgraphServer::max_similar_sites(),- );-- print_top_nodes(- &[- "dr.dk",- "berlingske.dk",- "ekstrabladet.dk",- "politikken.dk",- "tv2.dk",- "seoghoer.dk",- "zetland.dk",- ],- TOP_N,- &similarity_finder,- );-- print_top_nodes(- &[- "stackoverflow.com",- "github.com",- "arxiv.org",- "news.ycombinator.com",- ],- TOP_N,- &similarity_finder,- );-- print_top_nodes(&["nature.com", "who.int"], TOP_N, &similarity_finder);-- print_top_nodes(&["webmd.com", "medlineplus.gov"], TOP_N, &similarity_finder);-}diff --git a/core/src/collector.rs b/core/src/collector.rsindex f3737b7e..b3ec02c1 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -47,6 +47,7 @@ pub struct Hashes {pub site: Prehashed,pub title: Prehashed,pub url: Prehashed,+ pub url_without_tld: Prehashed,pub simhash: simhash::HashType,}@@ -185,6 +186,11 @@ impl TopSegmentCollector {site: self.get_hash(&doc, &FastField::SiteHash1, &FastField::SiteHash2),title: self.get_hash(&doc, &FastField::TitleHash1, &FastField::TitleHash2),url: self.get_hash(&doc, &FastField::UrlHash1, &FastField::UrlHash2),+ url_without_tld: self.get_hash(+ &doc,+ &FastField::UrlWithoutTldHash1,+ &FastField::UrlWithoutTldHash2,+ ),simhash: simhash.unwrap(),},id: doc,@@ -250,12 +256,14 @@ impl BucketCount {let taken_sites = *self.buckets.get(&hashes.site).unwrap_or(&0);let taken_urls = *self.buckets.get(&hashes.url).unwrap_or(&0);+ let taken_urls_without_tld = *self.buckets.get(&hashes.url_without_tld).unwrap_or(&0);let taken_titles = *self.buckets.get(&hashes.title).unwrap_or(&0);let adjuster = 1.0/ (1.0+ taken_sites as f64 * self.config.site_penalty+ taken_urls as f64 * self.config.url_penalty+ + taken_urls_without_tld as f64 * self.config.url_without_tld_penalty+ taken_titles as f64 * self.config.title_penalty);doc.adjusted_score = doc.doc.score() * adjuster;@@ -266,6 +274,7 @@ impl BucketCount {*self.buckets.entry(hashes.site).or_default() += 1;*self.buckets.entry(hashes.url).or_default() += 1;+ *self.buckets.entry(hashes.url_without_tld).or_default() += 1;*self.buckets.entry(hashes.title).or_default() += 1;}}@@ -310,13 +319,15 @@ impl<T: Doc> BucketCollector<T> {pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {let mut res = Vec::new();+ let mut simhash_dups = Vec::new();let mut simhash = simhash::Table::default();while let Some(best_doc) = self.documents.pop_max() {let hashes = best_doc.doc.hashes();- if hashes.simhash != 0 {+ if hashes.simhash != 0 && de_rank_similar {if simhash.contains(&hashes.simhash) {+ simhash_dups.push(best_doc.doc);continue;}simhash.insert(hashes.simhash);@@ -334,6 +345,8 @@ impl<T: Doc> BucketCollector<T> {}}+ res.extend(simhash_dups);+res}}@@ -505,6 +518,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ url_without_tld: 1.into(),simhash: 12,},123,@@ -515,6 +529,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ url_without_tld: 2.into(),simhash: 123,},124,@@ -525,6 +540,7 @@ mod tests {site: 3.into(),title: 3.into(),url: 3.into(),+ url_without_tld: 3.into(),simhash: 1234,},125,@@ -535,6 +551,7 @@ mod tests {site: 4.into(),title: 4.into(),url: 4.into(),+ url_without_tld: 4.into(),simhash: 12345,},126,@@ -545,6 +562,7 @@ mod tests {site: 5.into(),title: 5.into(),url: 5.into(),+ url_without_tld: 5.into(),simhash: 123456,},127,@@ -565,6 +583,7 @@ mod tests {site: 3.into(),title: 3.into(),url: 3.into(),+ url_without_tld: 3.into(),simhash: 12,},125,@@ -575,6 +594,7 @@ mod tests {site: 4.into(),title: 4.into(),url: 4.into(),+ url_without_tld: 4.into(),simhash: 123,},126,@@ -585,6 +605,7 @@ mod tests {site: 5.into(),title: 5.into(),url: 5.into(),+ url_without_tld: 5.into(),simhash: 1234,},127,@@ -605,6 +626,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ url_without_tld: 1.into(),simhash: 12,},125,@@ -615,6 +637,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ url_without_tld: 2.into(),simhash: 123,},126,@@ -625,6 +648,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ url_without_tld: 2.into(),simhash: 1234,},127,@@ -642,6 +666,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ url_without_tld: 1.into(),simhash: 12,},125,@@ -652,6 +677,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ url_without_tld: 2.into(),simhash: 123,},126,@@ -662,6 +688,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ url_without_tld: 2.into(),simhash: 1234,},127,@@ -682,6 +709,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ url_without_tld: 1.into(),simhash: 1234,},125,@@ -692,6 +720,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ url_without_tld: 2.into(),simhash: 1234,},126,@@ -702,13 +731,14 @@ mod tests {site: 3.into(),title: 3.into(),url: 3.into(),+ url_without_tld: 3.into(),simhash: 1,},127,5.0,),],- &[(5.0, 127), (3.1, 126)],+ &[(5.0, 127), (3.1, 126), (3.0, 125)],);}}diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex bf574a22..35722849 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -29,6 +29,10 @@ impl Collector {20.0}+ pub fn url_without_tld_penalty() -> f64 {+ 1.0+ }+pub fn max_docs_considered() -> usize {250_000}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 9684a51c..447c9a00 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -148,6 +148,9 @@ pub struct CollectorConfig {#[serde(default = "defaults::Collector::url_penalty")]pub url_penalty: f64,+ #[serde(default = "defaults::Collector::url_without_tld_penalty")]+ pub url_without_tld_penalty: f64,+#[serde(default = "defaults::Collector::max_docs_considered")]pub max_docs_considered: usize,}@@ -158,6 +161,7 @@ impl Default for CollectorConfig {site_penalty: defaults::Collector::site_penalty(),title_penalty: defaults::Collector::title_penalty(),url_penalty: defaults::Collector::url_penalty(),+ url_without_tld_penalty: defaults::Collector::url_without_tld_penalty(),max_docs_considered: defaults::Collector::max_docs_considered(),}}@@ -380,6 +384,7 @@ pub struct CrawlPlannerConfig {pub crawl_budget: usize,pub top_host_fraction: f64,pub wander_fraction: f64,+ pub top_n_hosts_surplus: usize,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/crawler/planner.rs b/core/src/crawler/planner.rsindex 16daaa2a..6a6d22f5 100644--- a/core/src/crawler/planner.rs+++ b/core/src/crawler/planner.rs@@ -15,8 +15,10 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::{anyhow, Result};use indicatif::ParallelProgressIterator;+use indicatif::ProgressIterator;use rayon::prelude::*;use rayon::ThreadPoolBuilder;+use std::collections::BTreeMap;use std::collections::HashMap;use std::{collections::VecDeque,@@ -35,6 +37,8 @@ use crate::{use super::Domain;+const MAX_SURPLUS_BUDGET_ITERATIONS: usize = 100;+fn all_pages(page_centrality: &RocksDbStore<NodeID, f64>,page_graph: &Webgraph,@@ -47,14 +51,14 @@ fn all_pages(.collect::<Vec<_>>()}-fn group_domain(hosts: Vec<NodeID>, host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {+fn group_domain(hosts: &[NodeID], host_graph: &Webgraph) -> HashMap<Domain, Vec<NodeID>> {let mut domains: HashMap<Domain, Vec<NodeID>> = HashMap::new();for host in hosts {- let node = host_graph.id2node(&host).unwrap();+ let node = host_graph.id2node(host).unwrap();if let Ok(url) = Url::parse(&format!("http://{}", node.name)) {let domain = Domain::from(url);- domains.entry(domain).or_default().push(host);+ domains.entry(domain).or_default().push(*host);}}@@ -101,8 +105,11 @@ pub fn make_crawl_plan<P: AsRef<Path>>(let num_hosts = hosts.len();tracing::info!("found {} hosts", num_hosts);- let total_host_centrality = host_centrality.iter().map(|(_, v)| v).sum::<f64>();- let grouped = group_domain(hosts, &host_graph);+ let mut total_host_centrality = hosts+ .iter()+ .filter_map(|v| host_centrality.get(v))+ .sum::<f64>();+ let grouped = group_domain(&hosts, &host_graph);let num_groups = grouped.len();let job_queues: Vec<Mutex<FileQueueWriter<Job>>> = (0..config.num_job_queues)@@ -126,6 +133,85 @@ pub fn make_crawl_plan<P: AsRef<Path>>(.build()?;pool.install(|| {+ let host_pages: BTreeMap<_, _> = hosts+ .par_iter()+ .progress_count(num_hosts as u64)+ .map(|host| {+ let num_pages = page_graph.pages_by_host(host).len() as u64;+ (*host, num_pages)+ })+ .collect();++ let mut host_budgets: BTreeMap<_, _> = hosts+ .par_iter()+ .progress_count(num_hosts as u64)+ .map(|host| {+ let host_centrality = host_centrality.get(host).unwrap_or_default();++ let host_budget = ((config.crawl_budget as f64 * host_centrality)+ / total_host_centrality)+ .round()+ .max(0.0) as u64;++ let num_pages = host_pages.get(host).copied().unwrap_or_default();++ (*host, host_budget.min(num_pages as u64))+ })+ .collect();++ let mut surplus_budget = config.crawl_budget as u64+ - host_budgets+ .values()+ .sum::<u64>()+ .min(config.crawl_budget as u64);++ let mut has_updated = true;+ let mut i = 0;+ while surplus_budget > 0 && has_updated && i < MAX_SURPLUS_BUDGET_ITERATIONS {+ tracing::info!("trying to schedule surplus budget: {}", surplus_budget);+ i += 1;++ has_updated = false;+ let mut new_surplus_budget = surplus_budget;+ let mut new_total_host_centrality = 0.0;++ for host in hosts.iter().take(config.top_n_hosts_surplus).progress() {+ if new_surplus_budget == 0 {+ break;+ }++ let num_pages = host_pages.get(host).copied().unwrap_or_default();+ let host_budget = host_budgets.get_mut(host).unwrap();++ if num_pages > *host_budget {+ let centrality = host_centrality.get(host).unwrap_or_default();+ new_total_host_centrality += centrality;++ let part = (centrality * surplus_budget as f64 / total_host_centrality)+ .round()+ .max(0.0) as u64;++ let diff = num_pages - *host_budget;+ let extra = part.min(diff);++ if extra > 0 {+ has_updated = true;+ }++ let extra = ((extra as f64 / (1.0 - config.wander_fraction)) as u64)+ .min(new_surplus_budget);++ *host_budget += extra;+ new_surplus_budget -= extra;+ }+ }++ total_host_centrality = new_total_host_centrality;+ surplus_budget = new_surplus_budget;+ }++ tracing::info!("surplus done (remaining={})", surplus_budget);+grouped.into_par_iter().progress_count(num_groups as u64)@@ -137,10 +223,12 @@ pub fn make_crawl_plan<P: AsRef<Path>>(let mut total_wander_budget = 0;let mut total_schedule_budget = 0;let mut total_scheduled_urls = 0;+ let mut total_known_urls = 0;let mut urls = VecDeque::new();for host in &hosts {let mut pages = all_pages(&page_centrality, &page_graph, *host);+ total_known_urls += pages.len();if pages.is_empty() {continue;@@ -148,16 +236,13 @@ pub fn make_crawl_plan<P: AsRef<Path>>(pages.sort_by(|(_, a), (_, b)| b.total_cmp(a));tracing::debug!("num pages: {}", pages.len());- let host_centrality = host_centrality.get(host).unwrap_or_default();+ let host_budget = host_budgets.get(host).copied().unwrap_or_default();- let host_budget = ((config.crawl_budget as f64 * host_centrality)- / total_host_centrality)- .max(0.0)- .round() as u64;tracing::debug!("host_budget: {host_budget}");let schedule_budget = (host_budget as f64 * (1.0 - config.wander_fraction))- .max(0.0)- .round() as u64;+ .round()+ .max(0.0) as u64;+tracing::debug!("schedule_budget: {schedule_budget}");let wander_budget = (host_budget as f64 * config.wander_fraction).max(0.0)@@ -200,6 +285,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(schedule_budget: total_schedule_budget,wander_budget: total_wander_budget,scheduled_urls: total_scheduled_urls,+ known_urls: total_known_urls,};stats.lock()@@ -227,6 +313,16 @@ pub fn make_crawl_plan<P: AsRef<Path>>(stats: stats.into_inner().unwrap_or_else(|e| e.into_inner()),};+ tracing::info!(+ "total scheduled urls: {}",+ metadata.stats.iter().map(|d| d.scheduled_urls).sum::<u64>()+ );++ tracing::info!(+ "total wander budget: {}",+ metadata.stats.iter().map(|d| d.wander_budget).sum::<u64>()+ );+metadata.stats.sort_by(|a, b| b.schedule_budget.cmp(&a.schedule_budget));@@ -242,6 +338,7 @@ pub fn make_crawl_plan<P: AsRef<Path>>(#[derive(serde::Serialize)]struct DomainStats {domain: Domain,+ known_urls: usize,num_hosts: usize,schedule_budget: u64,scheduled_urls: u64,diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 9e30e40b..da502782 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -49,7 +49,6 @@ const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MBstruct ProcessedUrl {new_urls: Vec<Url>,response: UrlResponse,- fetch_time: Duration,}pub struct WorkerThread {@@ -238,18 +237,6 @@ impl<S: DatumStream> JobExecutor<S> {let res = self.process_url(retryable_url.url.clone()).await;- let mut delay = res.fetch_time.mul_f32(self.politeness_factor);-- if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {- delay = Duration::from_millis(self.config.min_crawl_delay_ms);- }-- if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {- delay = Duration::from_millis(self.config.max_crawl_delay_ms);- }-- tokio::time::sleep(delay).await;-match res.response {UrlResponse::Success { url } => {self.crawled_urls.insert(url.clone());@@ -337,7 +324,6 @@ impl<S: DatumStream> JobExecutor<S> {ProcessedUrl {new_urls,response: url_res,- fetch_time: Duration::from_millis(datum.fetch_time_ms),}}Err(_) => ProcessedUrl {@@ -346,7 +332,6 @@ impl<S: DatumStream> JobExecutor<S> {url,status_code: None,},- fetch_time: Duration::from_millis(datum.fetch_time_ms),},}} else {@@ -358,7 +343,6 @@ impl<S: DatumStream> JobExecutor<S> {ProcessedUrl {new_urls: Vec::new(),response: url_res,- fetch_time: Duration::from_millis(datum.fetch_time_ms),}}} else {@@ -383,7 +367,6 @@ impl<S: DatumStream> JobExecutor<S> {url,status_code: Some(datum.status_code),},- fetch_time: Duration::from_millis(datum.fetch_time_ms),}}}@@ -396,7 +379,6 @@ impl<S: DatumStream> JobExecutor<S> {url,status_code: None,},- fetch_time: Duration::from_millis(0),}}}@@ -438,6 +420,18 @@ impl<S: DatumStream> JobExecutor<S> {let res = self.fetch(url.clone()).await?;let fetch_time = start.elapsed();+ let mut delay = fetch_time.mul_f32(self.politeness_factor);++ if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {+ delay = Duration::from_millis(self.config.min_crawl_delay_ms);+ }++ if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {+ delay = Duration::from_millis(self.config.max_crawl_delay_ms);+ }++ tokio::time::sleep(delay).await;+let headers: HashMap<_, _> = res.headers().iter()diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 323714ec..2f9d0f2e 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -1280,76 +1280,6 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com/");}- #[test]- fn remove_duplicates() {- let mut index = InvertedIndex::temporary().expect("Unable to open index");-- index- .insert(- Webpage::new(- &format!(- r#"- <html>- <head>- <title>News website</title>- </head>- <body>- {CONTENT}- </body>- </html>- "#- ),- "https://www.dr.xyz",- )- .unwrap(),- )- .expect("failed to insert webpage");-- index- .insert(- Webpage::new(- &format!(- r#"- <html>- <head>- <title>News website</title>- </head>- <body>- {CONTENT} dr- </body>- </html>- "#- ),- "https://www.dr.dk",- )- .unwrap(),- )- .expect("failed to insert webpage");-- index.commit().expect("failed to commit index");-- let ctx = index.local_search_ctx();- let query = Query::parse(- &ctx,- &SearchQuery {- query: "dr".to_string(),- ..Default::default()- },- &index,- )- .expect("Failed to parse query");- let ranker = Ranker::new(- SignalAggregator::new(Some(&query)),- ctx.fastfield_reader.clone(),- Default::default(),- );-- let result =- search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.dr.dk/");- }-#[test]fn schema_org_stored() {let mut index = InvertedIndex::temporary().expect("Unable to open index");diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 2c60ec63..8e6b5e5f 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -291,10 +291,11 @@ mod tests {<meta property="og:updated_time" content="1999-06-22T19:37:34+00:00" /></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100),),"https://www.old.com",)@@ -312,13 +313,14 @@ mod tests {<html><head><title>Title</title>- <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />+ <meta property="og:updated_time" content="2023-06-22T19:37:34+00:00" /></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.new.com",)@@ -334,6 +336,7 @@ mod tests {let result = searcher.search(&SearchQuery {query: "title".to_string(),+ return_ranking_signals: true,..Default::default()}).expect("Search failed");diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex e952bb1b..0ca2709f 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -421,6 +421,7 @@ mod tests {site: Prehashed(0),title: Prehashed(0),url: Prehashed(0),+ url_without_tld: Prehashed(0),simhash: 0,},address: DocAddress {diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 047e5b44..85f49f80 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -747,7 +747,7 @@ impl SignalAggregator {.collect(),});- Self {+ let mut s = Self {segment_reader: None,inbound_similarity: None,query_signal_coefficients,@@ -759,7 +759,11 @@ impl SignalAggregator {current_timestamp: None,linear_regression: None,query,- }+ };++ s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);++ s}fn prepare_textfields(diff --git a/core/src/schema.rs b/core/src/schema.rsindex 88f236a2..b0d250ba 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -219,6 +219,8 @@ pub enum FastField {UrlHash2,DomainHash1,DomainHash2,+ UrlWithoutTldHash1,+ UrlWithoutTldHash2,PreComputedScore,HostNodeID1,HostNodeID2,@@ -256,6 +258,8 @@ impl FastField {FastField::UrlHash2 => "url_hash2",FastField::DomainHash1 => "domain_hash1",FastField::DomainHash2 => "domain_hash2",+ FastField::UrlWithoutTldHash1 => "url_without_tld_hash1",+ FastField::UrlWithoutTldHash2 => "url_without_tld_hash2",FastField::PreComputedScore => "pre_computed_score",FastField::HostNodeID1 => "host_node_id1",FastField::HostNodeID2 => "host_node_id2",@@ -278,7 +282,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 61] = [+pub static ALL_FIELDS: [Field; 63] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -335,6 +339,8 @@ pub static ALL_FIELDS: [Field; 61] = [Field::Fast(FastField::UrlHash2),Field::Fast(FastField::DomainHash1),Field::Fast(FastField::DomainHash2),+ Field::Fast(FastField::UrlWithoutTldHash1),+ Field::Fast(FastField::UrlWithoutTldHash2),Field::Fast(FastField::PreComputedScore),Field::Fast(FastField::HostNodeID1),Field::Fast(FastField::HostNodeID2),@@ -517,6 +523,12 @@ impl Field {Field::Fast(FastField::UrlHash2) => {IndexingOption::Integer(NumericOptions::default().set_fast())}+ Field::Fast(FastField::UrlWithoutTldHash1) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }+ Field::Fast(FastField::UrlWithoutTldHash2) => {+ IndexingOption::Integer(NumericOptions::default().set_fast())+ }Field::Fast(FastField::DomainHash1) => {IndexingOption::Integer(NumericOptions::default().set_fast())}@@ -673,6 +685,8 @@ impl FastField {FastField::UrlHash2 => DataType::U64,FastField::DomainHash1 => DataType::U64,FastField::DomainHash2 => DataType::U64,+ FastField::UrlWithoutTldHash1 => DataType::U64,+ FastField::UrlWithoutTldHash2 => DataType::U64,FastField::PreComputedScore => DataType::U64,FastField::HostNodeID1 => DataType::U64,FastField::HostNodeID2 => DataType::U64,diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 5c13cafa..a94576d4 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -233,11 +233,11 @@ impl ApiSearcher {Ok(None)}- async fn sidebar(+ fn sidebar(&self,initial_results: &[distributed::InitialSearchResultShard],- query: &SearchQuery,- ) -> Result<Option<DisplayedSidebar>> {+ stackoverflow: Option<DisplayedSidebar>,+ ) -> Option<DisplayedSidebar> {let entity = initial_results.iter().filter_map(|res| res.local_result.entity_sidebar.clone())@@ -250,8 +250,8 @@ impl ApiSearcher {});match entity {- Some(entity) => Ok(Some(DisplayedSidebar::Entity(entity))),- None => Ok(self.stackoverflow_sidebar(query).await?),+ Some(entity) => Some(DisplayedSidebar::Entity(entity)),+ None => stackoverflow,}}@@ -434,15 +434,17 @@ impl ApiSearcher {self.collector_config.clone(),)?;- let (initial_results, live_results, widget, discussions) = tokio::join!(+ let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(self.distributed_searcher.search_initial(&search_query),self.live_searcher.search_initial(&search_query),self.widget(query),- self.discussions_widget(query)+ self.discussions_widget(query),+ self.stackoverflow_sidebar(query),);let discussions = discussions?;- let sidebar = self.sidebar(&initial_results, query).await?;+ let stackoverflow = stackoverflow?;+ let sidebar = self.sidebar(&initial_results, stackoverflow);let spell_corrected_query = if widget.is_none() {initial_results@@ -486,15 +488,11 @@ impl ApiSearcher {}).collect();- let retrieved_normal: Vec<_> = self- .distributed_searcher- .retrieve_webpages(&normal, &query.query)- .await;-- let retrieved_live: Vec<_> = self- .live_searcher- .retrieve_webpages(&live, &query.query)- .await;+ let (retrieved_normal, retrieved_live) = tokio::join!(+ self.distributed_searcher+ .retrieve_webpages(&normal, &query.query),+ self.live_searcher.retrieve_webpages(&live, &query.query),+ );let mut retrieved_webpages: Vec<_> =retrieved_normal.into_iter().chain(retrieved_live).collect();diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 65253c5a..b3081d88 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -137,7 +137,7 @@ impl Segment {}pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {- self.full_adjacency.nodes_by_prefix(host_node)+ self.full_reversed_adjacency.nodes_by_prefix(host_node)}pub fn id(&self) -> String {diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex a0690f3a..5f167346 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -1032,6 +1032,20 @@ impl Html {let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);let url_hash = split_u128(hash(self.url().as_str()).0);+ let tld = self.url().tld().unwrap_or_default();+ let url_without_tld = self+ .url()+ .host_str()+ .unwrap_or_default()+ .trim_end_matches(&tld)+ .to_string()+ + "/"+ + self.url().path()+ + "?"+ + self.url().query().unwrap_or_default();++ let url_without_tld_hash = split_u128(hash(url_without_tld).0);+let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);@@ -1286,6 +1300,12 @@ impl Html {Field::Fast(FastField::UrlHash2) => {doc.add_u64(tantivy_field, url_hash[1]);}+ Field::Fast(FastField::UrlWithoutTldHash1) => {+ doc.add_u64(tantivy_field, url_without_tld_hash[0]);+ }+ Field::Fast(FastField::UrlWithoutTldHash2) => {+ doc.add_u64(tantivy_field, url_without_tld_hash[1]);+ }Field::Fast(FastField::DomainHash1) => {doc.add_u64(tantivy_field, domain_hash[0]);}diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rsindex f166b289..40311545 100644--- a/core/src/webpage/url_ext.rs+++ b/core/src/webpage/url_ext.rs@@ -31,6 +31,7 @@ pub trait UrlExt {fn icann_domain(&self) -> Option<&str>;fn root_domain(&self) -> Option<&str>;fn subdomain(&self) -> Option<&str>;+ fn tld(&self) -> Option<&str>;}impl UrlExt for url::Url {@@ -58,6 +59,12 @@ impl UrlExt for url::Url {Some(subdomain)}++ fn tld(&self) -> Option<&str> {+ let host = self.host_str()?;+ let suffix = std::str::from_utf8(ICANN_LIST.suffix(host.as_bytes())?.as_bytes()).ok()?;+ Some(suffix)+ }}#[cfg(test)]@@ -85,4 +92,13 @@ mod tests {assert_eq!(url.domain().unwrap(), "example.blogspot.com");assert_eq!(url.icann_domain().unwrap(), "blogspot.com");}++ #[test]+ fn suffix() {+ let url: Url = Url::parse("http://example.blogspot.com").unwrap();+ assert_eq!(url.tld().unwrap(), "com");++ let url: Url = Url::parse("http://example.com").unwrap();+ assert_eq!(url.tld().unwrap(), "com");+ }}diff --git a/frontend/src/lib/improvements.ts b/frontend/src/lib/improvements.tsindex 68db224c..f692d3a3 100644--- a/frontend/src/lib/improvements.ts+++ b/frontend/src/lib/improvements.ts@@ -1,9 +1,6 @@import type { Action } from 'svelte/action';-import { writable } from 'svelte/store';import { getApiBase, type ApiOptions, type DisplayedWebpage, requestPlain } from './api';-import { allowStatsStore } from './stores';--const queryIdStore = writable<string | undefined>();+import { allowStatsStore, queryIdStore } from './stores';export const updateQueryId = async ({query,diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.tsindex b718ecae..25425f9c 100644--- a/frontend/src/lib/stores.ts+++ b/frontend/src/lib/stores.ts@@ -67,6 +67,10 @@ export const siteRankingsStore = writableLocalStorage<SiteRakings>(SITE_RANKINGSconst SEARCH_QUERY_KEY = 'searchQuery';export const searchQueryStore = writableLocalStorage<string | undefined>(SEARCH_QUERY_KEY, void 0);+const QUERY_ID_KEY = 'queryId';+export const queryIdStore = writableLocalStorage<string | undefined>(QUERY_ID_KEY, void 0);++const THEME_KEY = 'theme';export const themeStore = writableLocalStorage<string | void>(THEME_KEY, void 0);if (browser)diff --git a/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/ranking-annotation/src/routes/annotate/[slug]/+page.sveltedeleted file mode 100644index 85a1d3f0..00000000--- a/ranking-annotation/src/routes/annotate/[slug]/+page.svelte+++ /dev/null@@ -1,28 +0,0 @@-<script lang="ts">- import type { PageData } from "./$types";- export let data: PageData;-</script>--<div class="flex flex-col w-full items-center">- <div class="max-w-xl flex flex-col space-y-5">- <div>- <b class="font-bold">Query:</b> {data.query.query}- </div>-- <div class="flex flex-col space-y-3">- {#each data.searchResults as res}- <div class="py-2 px-2 border shadow-sm rounded-lg">- <div>- <a href="{res.webpage.url}">{res.webpage.url}</a>- </div>- <div>- <a href="{res.webpage.url}">{res.webpage.title}</a>- </div>- <div class="line-clamp-3 text-sm">- {res.webpage.snippet}- </div>- </div>- {/each}- </div>- </div>-</div>\ No newline at end of filediff --git a/ranking-annotation/vite.config.ts b/ranking-annotation/vite.config.tsdeleted file mode 100644index 80864b9d..00000000--- a/ranking-annotation/vite.config.ts+++ /dev/null@@ -1,6 +0,0 @@-import { sveltekit } from "@sveltejs/kit/vite";-import { defineConfig } from "vite";--export default defineConfig({- plugins: [sveltekit()],-});diff --git a/ranking-annotation/.gitignore b/tools/ranking-annotation/.gitignoresimilarity index 100%rename from ranking-annotation/.gitignorerename to tools/ranking-annotation/.gitignorediff --git a/ranking-annotation/.npmrc b/tools/ranking-annotation/.npmrcsimilarity index 100%rename from ranking-annotation/.npmrcrename to tools/ranking-annotation/.npmrcdiff --git a/ranking-annotation/README.md b/tools/ranking-annotation/README.mdsimilarity index 100%rename from ranking-annotation/README.mdrename to tools/ranking-annotation/README.mddiff --git a/tools/ranking-annotation/global.d.ts b/tools/ranking-annotation/global.d.tsnew file mode 100644index 00000000..13d619aa--- /dev/null+++ b/tools/ranking-annotation/global.d.ts@@ -0,0 +1,8 @@+declare type Item = import("svelte-dnd-action").Item;+declare type DndEvent<ItemType = Item> = import("svelte-dnd-action").DndEvent<ItemType>;+declare namespace svelteHTML {+ interface HTMLAttributes<T> {+ "on:consider"?: (event: CustomEvent<DndEvent<ItemType>> & {target: EventTarget & T}) => void;+ "on:finalize"?: (event: CustomEvent<DndEvent<ItemType>> & {target: EventTarget & T}) => void;+ }+}\ No newline at end of filediff --git a/ranking-annotation/package.json b/tools/ranking-annotation/package.jsonsimilarity index 86%rename from ranking-annotation/package.jsonrename to tools/ranking-annotation/package.jsonindex 1154e510..66e6ef62 100644--- a/ranking-annotation/package.json+++ b/tools/ranking-annotation/package.json@@ -12,6 +12,9 @@"format": "prettier --plugin prettier-plugin-svelte --plugin prettier-plugin-tailwindcss --write ."},"devDependencies": {+ "@iconify-json/heroicons": "^1.1.12",+ "@iconify-json/simple-icons": "^1.1.69",+ "@iconify/svelte": "^3.1.4","@sveltejs/adapter-auto": "^2.0.0","@sveltejs/kit": "^1.20.4","@types/better-sqlite3": "^7.6.6",@@ -30,12 +33,14 @@"tailwindcss": "^3.3.5","tslib": "^2.4.1","typescript": "^5.0.0",+ "unplugin-icons": "^0.17.0","vite": "^4.4.2"},"dependencies": {"better-sqlite3": "^9.0.0","cassandra-driver": "^4.7.2",- "papaparse": "^5.4.1"+ "papaparse": "^5.4.1",+ "svelte-dnd-action": "^0.9.31"},"type": "module"}diff --git a/ranking-annotation/postcss.config.js b/tools/ranking-annotation/postcss.config.jssimilarity index 100%rename from ranking-annotation/postcss.config.jsrename to tools/ranking-annotation/postcss.config.jsdiff --git a/ranking-annotation/src/app.css b/tools/ranking-annotation/src/app.csssimilarity index 100%rename from ranking-annotation/src/app.cssrename to tools/ranking-annotation/src/app.cssdiff --git a/ranking-annotation/src/app.d.ts b/tools/ranking-annotation/src/app.d.tssimilarity index 87%rename from ranking-annotation/src/app.d.tsrename to tools/ranking-annotation/src/app.d.tsindex 899c7e8f..843ae573 100644--- a/ranking-annotation/src/app.d.ts+++ b/tools/ranking-annotation/src/app.d.ts@@ -1,3 +1,4 @@+import 'unplugin-icons/types/svelte'// See https://kit.svelte.dev/docs/types#app// for information about these interfacesdeclare global {diff --git a/ranking-annotation/src/app.html b/tools/ranking-annotation/src/app.htmlsimilarity index 100%rename from ranking-annotation/src/app.htmlrename to tools/ranking-annotation/src/app.htmldiff --git a/ranking-annotation/src/lib/dataset.ts b/tools/ranking-annotation/src/lib/dataset.tssimilarity index 89%rename from ranking-annotation/src/lib/dataset.tsrename to tools/ranking-annotation/src/lib/dataset.tsindex 9881d2c4..131f1f03 100644--- a/ranking-annotation/src/lib/dataset.ts+++ b/tools/ranking-annotation/src/lib/dataset.ts@@ -11,7 +11,7 @@ export function getDataset(): Data | undefined {return dataset;}- const csvFilePath = path.resolve("../data/queries_us.csv");+ const csvFilePath = path.resolve("../../data/queries_us.csv");const csvFile = readFileSync(csvFilePath, "utf-8");Papa.parse(csvFile, {diff --git a/ranking-annotation/src/lib/db.ts b/tools/ranking-annotation/src/lib/db.tssimilarity index 54%rename from ranking-annotation/src/lib/db.tsrename to tools/ranking-annotation/src/lib/db.tsindex bc7d04ad..446df6e4 100644--- a/ranking-annotation/src/lib/db.ts+++ b/tools/ranking-annotation/src/lib/db.ts@@ -2,7 +2,7 @@ import Database from "better-sqlite3";import { getDataset } from "./dataset";import type { SimpleWebpage } from "./webpage";-const db = new Database("../data/ranking-annotation.sqlite", {+const db = new Database("../../data/ranking-annotation.sqlite", {fileMustExist: false,});@@ -22,7 +22,7 @@ function setupDB() {url TEXT NOT NULL,orig_rank INTEGER NOT NULL,webpage_json TEXT NOT NULL,- anotated_rank INTEGER,+ annotated_rank INTEGER,PRIMARY KEY (qid, url));`);@@ -50,15 +50,48 @@ export type Query = {export function getQueries(): Query[] {const query = db.prepare(`SELECT qid, query, EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.anotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL) AS annotated- FROM queries+ FROM queries ORDER BY qid`);return query.all() as Query[];}+export function getNextQuery(qid: string): Query | undefined {+ const query = db.prepare(`+ SELECT qid, query, EXISTS (+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ ) AS annotated+ FROM queries+ WHERE qid > @qid+ ORDER BY qid+ LIMIT 1+ `);++ const res = query.get({ qid });++ return res as Query | undefined;+}++export function getPreviousQuery(qid: string): Query | undefined {+ const query = db.prepare(`+ SELECT qid, query, EXISTS (+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL+ ) AS annotated+ FROM queries+ WHERE qid < @qid+ ORDER BY qid DESC+ LIMIT 1+ `);++ const res = query.get({ qid });++ return res as Query | undefined;+}+export type SearchResult = {+ id: string;origRank: number;webpage: SimpleWebpage;annotatedRank: number | null;@@ -66,7 +99,7 @@ export type SearchResult = {export function getSearchResults(qid: string): SearchResult[] {const queryResults = db.prepare(`- SELECT url, orig_rank as origRank, webpage_json, anotated_rank AS annotatedRank+ SELECT url, orig_rank as origRank, webpage_json, annotated_rank AS annotatedRankFROM search_resultsWHERE qid = @qid`);@@ -74,6 +107,7 @@ export function getSearchResults(qid: string): SearchResult[] {const res = queryResults.all({ qid });return res.map((r: any) => ({+ id: `${qid}-${r.url}`,origRank: r.origRank,annotatedRank: r.annotatedRank,webpage: JSON.parse(r.webpage_json),@@ -81,29 +115,33 @@ export function getSearchResults(qid: string): SearchResult[] {}export const saveSearchResults = (qid: string, searchResults: SearchResult[]) => {- const insertSearchResults = db.prepare(`- INSERT INTO search_results (qid, url, orig_rank, webpage_json)- VALUES (@qid, @url, @origRank, @webpageJson)- `);+ const upsertSearchResults = db.prepare(`+ INSERT INTO search_results (qid, url, orig_rank, webpage_json, annotated_rank)+ VALUES (@qid, @url, @origRank, @webpageJson, @annotatedRank)+ ON CONFLICT(qid, url) DO UPDATE SET+ annotated_rank = excluded.annotated_rank+ `);for (const searchResult of searchResults) {- insertSearchResults.run({+ upsertSearchResults.run({qid,url: searchResult.webpage.url,origRank: searchResult.origRank,webpageJson: JSON.stringify(searchResult.webpage),+ annotatedRank: searchResult.annotatedRank,});}}-export function getQuery(qid: String): Query {+export function getQuery(qid: String): Query | undefined {const query = db.prepare(`SELECT qid, query, EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.anotated_rank IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotated_rank IS NOT NULL) AS annotatedFROM queriesWHERE qid = @qid`);+ const res = query.get({ qid });- return query.get({ qid }) as Query;+ return res as Query | undefined;}diff --git a/ranking-annotation/src/lib/index.ts b/tools/ranking-annotation/src/lib/index.tssimilarity index 100%rename from ranking-annotation/src/lib/index.tsrename to tools/ranking-annotation/src/lib/index.tsdiff --git a/ranking-annotation/src/lib/webpage.ts b/tools/ranking-annotation/src/lib/webpage.tssimilarity index 100%rename from ranking-annotation/src/lib/webpage.tsrename to tools/ranking-annotation/src/lib/webpage.tsdiff --git a/ranking-annotation/src/routes/+layout.svelte b/tools/ranking-annotation/src/routes/+layout.sveltesimilarity index 100%rename from ranking-annotation/src/routes/+layout.svelterename to tools/ranking-annotation/src/routes/+layout.sveltediff --git a/ranking-annotation/src/routes/+page.server.ts b/tools/ranking-annotation/src/routes/+page.server.tssimilarity index 100%rename from ranking-annotation/src/routes/+page.server.tsrename to tools/ranking-annotation/src/routes/+page.server.tsdiff --git a/ranking-annotation/src/routes/+page.svelte b/tools/ranking-annotation/src/routes/+page.sveltesimilarity index 100%rename from ranking-annotation/src/routes/+page.svelterename to tools/ranking-annotation/src/routes/+page.sveltediff --git a/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.tssimilarity index 76%rename from ranking-annotation/src/routes/annotate/[slug]/+page.server.tsrename to tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.tsindex 06837f7f..f7fc2492 100644--- a/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.server.ts@@ -1,6 +1,7 @@-import { getQuery, getSearchResults, saveSearchResults } from "$lib/db";+import { getNextQuery, getPreviousQuery, getQuery, getSearchResults, saveSearchResults } from "$lib/db";import type { Webpage } from "$lib/webpage";import { asSimpleWebpage } from "$lib/webpage";+import { redirect } from "@sveltejs/kit";const search = async (query: string) => {return await fetch(`https://trystract.com/beta/api/search`, {@@ -25,13 +26,22 @@ export const load = async ({ params }) => {const { slug } = params;const qid = slug;- const query = getQuery(qid);++ if (!getQuery(qid)) {+ throw redirect(301, "/");+ }+ const query = getQuery(qid)!;++ const previousQuery = getPreviousQuery(qid);+ const nextQuery = getNextQuery(qid);+let searchResults = getSearchResults(query.qid);if (searchResults.length === 0) {const webpages = (await search(query.query)).map((w) => asSimpleWebpage(w));searchResults = webpages.map((w, i) => ({+ id: `${query.qid}-${w.url}`,origRank: i,annotatedRank: null,webpage: w,@@ -53,5 +63,5 @@ export const load = async ({ params }) => {}});- return { query, searchResults };+ return { query, searchResults, previousQuery, nextQuery };};\ No newline at end of filediff --git a/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.sveltenew file mode 100644index 00000000..1a103788--- /dev/null+++ b/tools/ranking-annotation/src/routes/annotate/[slug]/+page.svelte@@ -0,0 +1,128 @@+<script lang="ts">+ import Check from '~icons/heroicons/check-20-solid';+ import Home from '~icons/heroicons/home-20-solid';+ import ArrowLeft from '~icons/heroicons/arrow-left-20-solid';+ import ArrowRight from '~icons/heroicons/arrow-right-20-solid';+ import { dndzone } from 'svelte-dnd-action';+ import type { PageData } from "./$types";+ import type { SearchResult } from '$lib/db';+ import { browser } from '$app/environment';++ $: shownSignals = -1;+ $: searchResults = data.searchResults;++ function updateRanks() {+ searchResults.forEach((res, i) => {+ if (res.annotatedRank != null) {+ res.annotatedRank = i;+ }+ });+ }++ function handleConsider(event: CustomEvent<{ items: SearchResult[] }>) {+ searchResults = event.detail.items;+ }++ async function handleDrop(event: CustomEvent<{ items: SearchResult[] }>) {+ searchResults = event.detail.items;+ updateRanks();+ await save();+ }++ async function save() {+ if (!browser) {+ return;+ }++ await fetch(`/api/annotate/${data.query.qid}`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ qid: data.query.qid,+ results: searchResults,+ }),+ })+ }++ $: searchResults && save();++ export let data: PageData;+</script>++<div class="flex flex-col w-full items-center mt-5">+ <div class="max-w-xl flex flex-col space-y-5">+ <div class="flex w-full justify-around">+ <div class="w-8 h-8">+ <a href="/" class="w-full h-full text-sky-300 hover:text-sky-500">+ <Home class="w-full h-full"/>+ </a>+ </div>+ <div class="text-xl">+ <div>+ <b class="font-bold">Query:</b> {data.query.query}+ </div>+ </div>+ <div class="flex">+ <a href="/annotate/{data.previousQuery?.qid}" class="text-orange-300 hover:text-orange-500">+ <ArrowLeft class="w-6 h-6"/>+ </a>+ <a href="/annotate/{data.nextQuery?.qid}" class="text-orange-300 hover:text-orange-500">+ <ArrowRight class="w-6 h-6"/>+ </a>+ </div>+ </div>+ <section use:dndzone={{ items: searchResults, dropTargetStyle: {} }} on:consider={handleConsider} on:finalize={handleDrop} class="flex flex-col space-y-3">+ {#each searchResults as res, i (res.id)}+ <div>+ <div class="relative">+ <button class="absolute left-[-2em] top-3" on:click={() => (res.annotatedRank == null) ? res.annotatedRank = i : res.annotatedRank = null}>+ {#if res.annotatedRank != null}+ <div class="text-green-500">+ <Check class="w-6 h-6" />+ </div>+ {:else}+ <div class="text-slate-200">+ <Check class="w-6 h-6" />+ </div>+ {/if}+ </button>+ <div class="py-2 px-2 bg-slate-100 border shadow-sm rounded-lg">+ <div>+ <a href="{res.webpage.url}" target="_blank">{res.webpage.url}</a>+ </div>+ <div class="font-bold">+ <a href="{res.webpage.url}" target="_blank">{res.webpage.title}</a>+ </div>+ <div class="line-clamp-3 text-sm">+ {res.webpage.snippet}+ </div>+ <div class="text-sm mt-2 text-slate-500">+ Host centrality: {res.webpage.rankingSignals["host_centrality"].toFixed(3)}+ Tracker score: {res.webpage.rankingSignals["tracker_score"].toFixed(3)}+ </div>+ <div>+ <button class="text-sm text-slate-500" on:click={() => shownSignals = (shownSignals == i) ? -1 : i}>+ {shownSignals === i ? "Hide" : "Show"} signals+ </button>+ </div>+ {#if shownSignals === i}+ <div class="flex flex-col">+ {#each Object.entries(res.webpage.rankingSignals) as [name, value]}+ <div class="flex flex-col text-sm">+ <div class="flex flex-row space-x-2">+ <div class="font-bold">{name}:</div>+ <div>{value}</div>+ </div>+ </div>+ {/each}+ </div>+ {/if}+ </div>+ </div>+ </div>+ {/each}+ </section>+ </div>+</div>\ No newline at end of filediff --git a/tools/ranking-annotation/src/routes/api/annotate/[slug]/+server.ts b/tools/ranking-annotation/src/routes/api/annotate/[slug]/+server.tsnew file mode 100644index 00000000..c1730632--- /dev/null+++ b/tools/ranking-annotation/src/routes/api/annotate/[slug]/+server.ts@@ -0,0 +1,14 @@+import type { SearchResult } from "$lib/db";+import { saveSearchResults } from "$lib/db";++export async function POST({request}): Promise<Response> {+ const {qid, results}: {+ qid: string;+ results: SearchResult[];+ } = await request.json();++ saveSearchResults(qid, results);++ return new Response("OK");+}+\ No newline at end of filediff --git a/ranking-annotation/src/routes/api/data/+server.ts b/tools/ranking-annotation/src/routes/api/data/+server.tssimilarity index 100%rename from ranking-annotation/src/routes/api/data/+server.tsrename to tools/ranking-annotation/src/routes/api/data/+server.tsdiff --git a/ranking-annotation/static/favicon.png b/tools/ranking-annotation/static/favicon.pngsimilarity index 100%rename from ranking-annotation/static/favicon.pngrename to tools/ranking-annotation/static/favicon.pngdiff --git a/ranking-annotation/svelte.config.js b/tools/ranking-annotation/svelte.config.jssimilarity index 100%rename from ranking-annotation/svelte.config.jsrename to tools/ranking-annotation/svelte.config.jsdiff --git a/ranking-annotation/tailwind.config.ts b/tools/ranking-annotation/tailwind.config.tssimilarity index 100%rename from ranking-annotation/tailwind.config.tsrename to tools/ranking-annotation/tailwind.config.tsdiff --git a/ranking-annotation/tsconfig.json b/tools/ranking-annotation/tsconfig.jsonsimilarity index 90%rename from ranking-annotation/tsconfig.jsonrename to tools/ranking-annotation/tsconfig.jsonindex 794b95b6..f237f2d8 100644--- a/ranking-annotation/tsconfig.json+++ b/tools/ranking-annotation/tsconfig.json@@ -1,5 +1,6 @@{"extends": "./.svelte-kit/tsconfig.json",+ "include": ["src/**/*", "global.d.ts", "vite.config.ts"],"compilerOptions": {"allowJs": true,"checkJs": true,diff --git a/tools/ranking-annotation/vite.config.ts b/tools/ranking-annotation/vite.config.tsnew file mode 100644index 00000000..83cf07a0--- /dev/null+++ b/tools/ranking-annotation/vite.config.ts@@ -0,0 +1,12 @@+import { sveltekit } from "@sveltejs/kit/vite";+import { defineConfig } from "vite";+import Icons from 'unplugin-icons/vite';++export default defineConfig({+ plugins: [+ sveltekit(),+ Icons({+ compiler: 'svelte',+ })+ ],+});