web search done right
increase crawldb link limits
Mikkel Denker241223b2b9c4
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 6d705a5d..c4bf392e 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -38,8 +38,8 @@ mod worker;pub use coordinator::CrawlCoordinator;pub const MAX_URL_LEN_BYTES: usize = 8192;-pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 200;-pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 100;+pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 500;+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 200;#[derive(thiserror::Error, Debug)]pub enum Error {