web search done right
Lower domain weight after the urls have been sampled
Mikkel Denkere858c004f1fc
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 9656e2ee..b197312e 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::{+ cmp::Ordering,collections::{BTreeMap, BinaryHeap, HashMap, VecDeque},hash::Hash,num::NonZeroUsize,@@ -580,6 +581,20 @@ impl CrawlDb {state.status = UrlStatus::Crawling;}+ let domain_state = self.domain_state.get_mut(domain_id).unwrap();++ domain_state.weight = urls+ .iter()+ .filter_map(|(_, state)| {+ if state.status == UrlStatus::Pending {+ Some(state.weight)+ } else {+ None+ }+ })+ .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))+ .unwrap_or(0.0);+let mut job = Job {domain: self.domain_ids.value(domain_id.0)?.unwrap(),fetch_sitemap: false, // todo: fetch for new sites