Skip to content

web search done right

Fixed bug where liked sites would show up in discardall optics, even though they matched none of the rules

Mikkel Denkerf1ad0067994d
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index e0b05eae..2e2334d2 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -201,6 +201,8 @@ impl AsTantivyQuery for Matching {
#[cfg(test)]
mod tests {
+ use optics::SiteRankings;
+
use crate::{
gen_temp_path,
index::Index,
@@ -986,4 +988,102 @@ mod tests {
assert_eq!(res.len(), 0);
}
+
+ #[test]
+ fn discard_all_discard_like() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ example example example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://a.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ example example example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://b.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic_program: Some(
+ r#"
+ DiscardNonMatching;
+ Rule {
+ Matches {
+ Site("b.com")
+ }
+ };
+ "#
+ .to_string(),
+ ),
+ site_rankings: Some(SiteRankings {
+ liked: vec!["a.com".to_string()],
+ disliked: vec![],
+ blocked: vec![],
+ }),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].url, "https://b.com");
+ }
}
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 38ca29ad..eed33d83 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -340,11 +340,9 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {
- dbg!(websites.len());
self.last_stage.populate(websites);
- self.last_stage
- .apply(self.top_n, dbg!(self.pipeline_offset()))
+ self.last_stage.apply(self.top_n, self.pipeline_offset())
}
pub fn collector_top_n(&self) -> usize {
@@ -360,7 +358,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
pub fn collector_offset(&self) -> usize {
- dbg!((self.offset / self.initial_top_n()) * self.initial_top_n())
+ (self.offset / self.initial_top_n()) * self.initial_top_n()
}
}
@@ -418,7 +416,7 @@ mod tests {
Arc::new(DummyCrossEncoder {}),
)
.unwrap();
- assert_eq!(pipeline.collector_top_n(), 200);
+ assert_eq!(pipeline.collector_top_n(), 80);
let sample = sample_websites(pipeline.collector_top_n());
let res: Vec<_> = pipeline
diff --git a/frontend/src/components/Discussions.astro b/frontend/src/components/Discussions.astro
index 8b7f2976..67b97ff7 100644
--- a/frontend/src/components/Discussions.astro
+++ b/frontend/src/components/Discussions.astro
@@ -28,7 +28,7 @@ askama.if_("let Some(discussions) = discussions", () => (
))
}
</div>
- <div id="discussion-more-btn" class="hover:cursor-pointer hover:bg-neutral-100 w-fit px-2 py-1 mt-3 border rounded-full">More discussions</div>
+ <div id="discussion-more-btn" class="hover:cursor-pointer hover:bg-neutral-100 w-fit px-2 py-1 mt-3 border rounded-full">Show more</div>
</div>
</div>
))
@@ -44,10 +44,10 @@ askama.if_("let Some(discussions) = discussions", () => (
if (moreIsShown) {
discussionMore.classList.remove("hidden");
- discussionMoreBtn.innerHTML = "Less discussions";
+ discussionMoreBtn.innerHTML = "Show less";
} else {
discussionMore.classList.add("hidden");
- discussionMoreBtn.innerHTML = "More discussions";
+ discussionMoreBtn.innerHTML = "Show more";
}
})
diff --git a/justfile b/justfile
index 619f920d..742129dc 100644
--- a/justfile
+++ b/justfile
@@ -6,13 +6,13 @@
rm -rf data/webgraph
cargo run --release -- webgraph local configs/webgraph/local.toml
-@frontend-rerun:
+@frontend-rerun *ARGS:
cd frontend; npm run build
- bash scripts/run_frontend.sh
+ bash scripts/run_frontend.sh {{ARGS}}
-@frontend:
+@frontend *ARGS:
cd frontend; npm install
- cargo watch -s 'just frontend-rerun'
+ cargo watch -s 'just frontend-rerun {{ARGS}}'
@astro:
cd frontend; npm run dev
diff --git a/optics/src/lib.rs b/optics/src/lib.rs
index 05594128..ff960b42 100644
--- a/optics/src/lib.rs
+++ b/optics/src/lib.rs
@@ -275,39 +275,16 @@ pub struct SiteRankings {
impl SiteRankings {
pub fn rules(&self) -> Vec<Rule> {
- let mut instructions = Vec::new();
-
- for site in &self.liked {
- instructions.push(Rule {
- matches: vec![Matching {
- pattern: vec![PatternPart::Raw(site.clone())],
- location: MatchLocation::Site,
- }],
- action: Action::Boost(5),
- });
- }
-
- for site in &self.disliked {
- instructions.push(Rule {
- matches: vec![Matching {
- pattern: vec![PatternPart::Raw(site.clone())],
- location: MatchLocation::Site,
- }],
- action: Action::Downrank(5),
- });
- }
-
- for site in &self.blocked {
- instructions.push(Rule {
+ self.blocked
+ .iter()
+ .map(|site| Rule {
matches: vec![Matching {
pattern: vec![PatternPart::Raw(site.clone())],
location: MatchLocation::Site,
}],
action: Action::Discard,
- });
- }
-
- instructions
+ })
+ .collect()
}
pub fn into_optic(self) -> Optic {
diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.sh
index 0a8564b1..985d844f 100644
--- a/scripts/run_frontend.sh
+++ b/scripts/run_frontend.sh
@@ -1,4 +1,4 @@
#!/bin/bash
trap 'kill $(jobs -p)' EXIT
-cargo run -- search-server configs/search_server.toml &
-cargo run -- frontend configs/frontend.toml
+cargo run $1 -- search-server configs/search_server.toml &
+cargo run $1 -- frontend configs/frontend.toml