web search done right
for some reason, the needed array feature broke pragme settings in sqlite. Hopefully fixed now
Mikkel Denker5c670d615386
diff --git a/Cargo.lock b/Cargo.lockindex d0081d47..57da3d0a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2262,6 +2262,7 @@ version = "0.26.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"dependencies = [+ "bindgen","cc","pkg-config","vcpkg",diff --git a/Cargo.toml b/Cargo.tomlindex e045d1b5..d65d146b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -88,4 +88,4 @@ safetensors = "0.3.1"publicsuffix = "2"robotstxt-with-cache = "0.4.0"rust-s3 = "0.33.0"-rusqlite = { version = "0.29.0", features = ["bundled", "modern-full"] }\ No newline at end of file+rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }\ No newline at end of filediff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex f26e13ba..f5831429 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -156,13 +156,16 @@ impl CrawlDb {// update performance stuff// WAL mode- conn.pragma_update(None, "journal_mode", "'WAL'")?;+ let _ = conn+ .prepare("PRAGMA journal_mode = 'WAL';")?+ .query([])?+ .next();// sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.- conn.pragma_update(None, "synchronous", 0)?;+ let _ = conn.prepare("PRAGMA synchronous = OFF;")?.query([])?.next();// store temp tables in memory- conn.pragma_update(None, "temp_store", 2)?;+ let _ = conn.prepare("PRAGMA temp_store = 2;")?.query([])?.next();// set cache size to 64 MBconn.pragma_update(None, "cache_size", -64_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)@@ -246,7 +249,7 @@ impl Transaction<'_> {"CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain ASSELECT domain, MAX(incoming_links) as max_incoming_linksFROM url- WHERE domain IN ?1 AND status = ?2+ WHERE domain IN (?1) AND status = ?2GROUP BY domain;",(domains.clone(), UrlStatus::Pending),)?;@@ -257,7 +260,7 @@ impl Transaction<'_> {SET max_incoming_links = IFNULL((SELECT max_incoming_links FROM temp_domain WHERE temp_domain.domain = domain.domain), 0)- WHERE domain IN ?1;",+ WHERE domain IN (?1);",(domains, ),)?;