Skip to content

web search done right

increase crawldb link limits

Mikkel Denker241223b2b9c4
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 6d705a5d..c4bf392e 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -38,8 +38,8 @@ mod worker;
pub use coordinator::CrawlCoordinator;
pub const MAX_URL_LEN_BYTES: usize = 8192;
-pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 200;
-pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 100;
+pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 500;
+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 200;
#[derive(thiserror::Error, Debug)]
pub enum Error {