Skip to content

web search done right

for some reason, the needed array feature broke pragme settings in sqlite. Hopefully fixed now

Mikkel Denker5c670d615386
diff --git a/Cargo.lock b/Cargo.lock
index d0081d47..57da3d0a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2262,6 +2262,7 @@ version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
dependencies = [
+ "bindgen",
"cc",
"pkg-config",
"vcpkg",
diff --git a/Cargo.toml b/Cargo.toml
index e045d1b5..d65d146b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -88,4 +88,4 @@ safetensors = "0.3.1"
publicsuffix = "2"
robotstxt-with-cache = "0.4.0"
rust-s3 = "0.33.0"
-rusqlite = { version = "0.29.0", features = ["bundled", "modern-full"] }
\ No newline at end of file
+rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
\ No newline at end of file
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index f26e13ba..f5831429 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -156,13 +156,16 @@ impl CrawlDb {
// update performance stuff
// WAL mode
- conn.pragma_update(None, "journal_mode", "'WAL'")?;
+ let _ = conn
+ .prepare("PRAGMA journal_mode = 'WAL';")?
+ .query([])?
+ .next();
// sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.
- conn.pragma_update(None, "synchronous", 0)?;
+ let _ = conn.prepare("PRAGMA synchronous = OFF;")?.query([])?.next();
// store temp tables in memory
- conn.pragma_update(None, "temp_store", 2)?;
+ let _ = conn.prepare("PRAGMA temp_store = 2;")?.query([])?.next();
// set cache size to 64 MB
conn.pragma_update(None, "cache_size", -64_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)
@@ -246,7 +249,7 @@ impl Transaction<'_> {
"CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain AS
SELECT domain, MAX(incoming_links) as max_incoming_links
FROM url
- WHERE domain IN ?1 AND status = ?2
+ WHERE domain IN (?1) AND status = ?2
GROUP BY domain;",
(domains.clone(), UrlStatus::Pending),
)?;
@@ -257,7 +260,7 @@ impl Transaction<'_> {
SET max_incoming_links = IFNULL(
(SELECT max_incoming_links FROM temp_domain WHERE temp_domain.domain = domain.domain), 0
)
- WHERE domain IN ?1;",
+ WHERE domain IN (?1);",
(domains, ),
)?;