Skip to content

web search done right

Lower domain weight after the urls have been sampled

Mikkel Denkere858c004f1fc
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 9656e2ee..b197312e 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
+ cmp::Ordering,
collections::{BTreeMap, BinaryHeap, HashMap, VecDeque},
hash::Hash,
num::NonZeroUsize,
@@ -580,6 +581,20 @@ impl CrawlDb {
state.status = UrlStatus::Crawling;
}
+ let domain_state = self.domain_state.get_mut(domain_id).unwrap();
+
+ domain_state.weight = urls
+ .iter()
+ .filter_map(|(_, state)| {
+ if state.status == UrlStatus::Pending {
+ Some(state.weight)
+ } else {
+ None
+ }
+ })
+ .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))
+ .unwrap_or(0.0);
+
let mut job = Job {
domain: self.domain_ids.value(domain_id.0)?.unwrap(),
fetch_sitemap: false, // todo: fetch for new sites