Skip to content

web search done right

more crawl_db indexes to increase sample performance

Mikkel Denker86ebdaa1926f
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index f5831429..afb0cdad 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -124,6 +124,16 @@ impl CrawlDb {
[],
)?;
+ conn.execute(
+ "CREATE INDEX IF NOT EXISTS url_domain_status ON url (domain, status);",
+ [],
+ )?;
+
+ conn.execute(
+ "CREATE INDEX IF NOT EXISTS url_incoming_links ON url (incoming_links);",
+ [],
+ )?;
+
// there should be one table that contains all known domains
// and whether a crawl is in progress for that domain.
// It should also contain a copy of the maximum count of incoming links