web search done right
make sure urls discovered in crawler are deduplicated (extra precaution)
Mikkel Denker8df44a56ae1e
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex f50525a8..4360c4e7 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::anyhow;use hashbrown::HashSet;+use itertools::Itertools;use rand::seq::SliceRandom;use std::{@@ -222,11 +223,9 @@ impl<S: DatumStream> JobExecutor<S> {.filter(|(url, _)| !self.crawled_urls.contains(url)).filter(|(url, _)| self.job.domain == Domain::from(url)).filter(|(_, score)| score.is_finite())+ .unique_by(|(url, _)| url.clone()).collect();- urls.sort_by(|(a, _), (b, _)| a.cmp(b));- urls.dedup_by(|(a, _), (b, _)| a == b);-urls.sort_by(|(_, a), (_, b)| b.total_cmp(a));let urls: VecDeque<_> = urls@@ -433,6 +432,7 @@ impl<S: DatumStream> JobExecutor<S> {.all(|ext| !url.as_str().ends_with(ext))}).filter(|url| !self.crawled_urls.contains(url))+ .unique().collect()}