web search done right
more crawl_db indexes to increase sample performance
Mikkel Denker86ebdaa1926f
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex f5831429..afb0cdad 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -124,6 +124,16 @@ impl CrawlDb {[],)?;+ conn.execute(+ "CREATE INDEX IF NOT EXISTS url_domain_status ON url (domain, status);",+ [],+ )?;++ conn.execute(+ "CREATE INDEX IF NOT EXISTS url_incoming_links ON url (incoming_links);",+ [],+ )?;+// there should be one table that contains all known domains// and whether a crawl is in progress for that domain.// It should also contain a copy of the maximum count of incoming links