web search done right
Fixed bug where liked sites would show up in discardall optics, even though they matched none of the rules
Mikkel Denkerf1ad0067994d
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex e0b05eae..2e2334d2 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -201,6 +201,8 @@ impl AsTantivyQuery for Matching {#[cfg(test)]mod tests {+ use optics::SiteRankings;+use crate::{gen_temp_path,index::Index,@@ -986,4 +988,102 @@ mod tests {assert_eq!(res.len(), 0);}++ #[test]+ fn discard_all_discard_like() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ <body>+ {CONTENT} {}+ example example example+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://a.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ <body>+ {CONTENT} {}+ example example example+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://b.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");++ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic_program: Some(+ r#"+ DiscardNonMatching;+ Rule {+ Matches {+ Site("b.com")+ }+ };+ "#+ .to_string(),+ ),+ site_rankings: Some(SiteRankings {+ liked: vec!["a.com".to_string()],+ disliked: vec![],+ blocked: vec![],+ }),+ ..Default::default()+ })+ .unwrap()+ .webpages;++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].url, "https://b.com");+ }}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 38ca29ad..eed33d83 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -340,11 +340,9 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {- dbg!(websites.len());self.last_stage.populate(websites);- self.last_stage- .apply(self.top_n, dbg!(self.pipeline_offset()))+ self.last_stage.apply(self.top_n, self.pipeline_offset())}pub fn collector_top_n(&self) -> usize {@@ -360,7 +358,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn collector_offset(&self) -> usize {- dbg!((self.offset / self.initial_top_n()) * self.initial_top_n())+ (self.offset / self.initial_top_n()) * self.initial_top_n()}}@@ -418,7 +416,7 @@ mod tests {Arc::new(DummyCrossEncoder {}),).unwrap();- assert_eq!(pipeline.collector_top_n(), 200);+ assert_eq!(pipeline.collector_top_n(), 80);let sample = sample_websites(pipeline.collector_top_n());let res: Vec<_> = pipelinediff --git a/frontend/src/components/Discussions.astro b/frontend/src/components/Discussions.astroindex 8b7f2976..67b97ff7 100644--- a/frontend/src/components/Discussions.astro+++ b/frontend/src/components/Discussions.astro@@ -28,7 +28,7 @@ askama.if_("let Some(discussions) = discussions", () => ())}</div>- <div id="discussion-more-btn" class="hover:cursor-pointer hover:bg-neutral-100 w-fit px-2 py-1 mt-3 border rounded-full">More discussions</div>+ <div id="discussion-more-btn" class="hover:cursor-pointer hover:bg-neutral-100 w-fit px-2 py-1 mt-3 border rounded-full">Show more</div></div></div>))@@ -44,10 +44,10 @@ askama.if_("let Some(discussions) = discussions", () => (if (moreIsShown) {discussionMore.classList.remove("hidden");- discussionMoreBtn.innerHTML = "Less discussions";+ discussionMoreBtn.innerHTML = "Show less";} else {discussionMore.classList.add("hidden");- discussionMoreBtn.innerHTML = "More discussions";+ discussionMoreBtn.innerHTML = "Show more";}})diff --git a/justfile b/justfileindex 619f920d..742129dc 100644--- a/justfile+++ b/justfile@@ -6,13 +6,13 @@rm -rf data/webgraphcargo run --release -- webgraph local configs/webgraph/local.toml-@frontend-rerun:+@frontend-rerun *ARGS:cd frontend; npm run build- bash scripts/run_frontend.sh+ bash scripts/run_frontend.sh {{ARGS}}-@frontend:+@frontend *ARGS:cd frontend; npm install- cargo watch -s 'just frontend-rerun'+ cargo watch -s 'just frontend-rerun {{ARGS}}'@astro:cd frontend; npm run devdiff --git a/optics/src/lib.rs b/optics/src/lib.rsindex 05594128..ff960b42 100644--- a/optics/src/lib.rs+++ b/optics/src/lib.rs@@ -275,39 +275,16 @@ pub struct SiteRankings {impl SiteRankings {pub fn rules(&self) -> Vec<Rule> {- let mut instructions = Vec::new();-- for site in &self.liked {- instructions.push(Rule {- matches: vec![Matching {- pattern: vec![PatternPart::Raw(site.clone())],- location: MatchLocation::Site,- }],- action: Action::Boost(5),- });- }-- for site in &self.disliked {- instructions.push(Rule {- matches: vec![Matching {- pattern: vec![PatternPart::Raw(site.clone())],- location: MatchLocation::Site,- }],- action: Action::Downrank(5),- });- }-- for site in &self.blocked {- instructions.push(Rule {+ self.blocked+ .iter()+ .map(|site| Rule {matches: vec![Matching {pattern: vec![PatternPart::Raw(site.clone())],location: MatchLocation::Site,}],action: Action::Discard,- });- }-- instructions+ })+ .collect()}pub fn into_optic(self) -> Optic {diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.shindex 0a8564b1..985d844f 100644--- a/scripts/run_frontend.sh+++ b/scripts/run_frontend.sh@@ -1,4 +1,4 @@#!/bin/bashtrap 'kill $(jobs -p)' EXIT-cargo run -- search-server configs/search_server.toml &-cargo run -- frontend configs/frontend.toml+cargo run $1 -- search-server configs/search_server.toml &+cargo run $1 -- frontend configs/frontend.toml