Skip to content

web search done right

hopefully increase coordinator performance

Mikkel Denker3bf75621b373
diff --git a/.cargo/config.toml b/.cargo/config.toml
index b389d05c..a70b3118 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -5,4 +5,5 @@ rustflags = ["-C", "target-cpu=native"]
LIBTORCH = {value = "libtorch", force=true, relative=true}
DYLD_LIBRARY_PATH = {value = "libtorch/lib", force=true, relative=true}
LD_LIBRARY_PATH = {value = "libtorch/lib", force=true, relative=true}
-LIBSQLITE3_FLAGS = { value = "-DSQLITE_ENABLE_MATH_FUNCTIONS", force=true }
\ No newline at end of file
+LIBSQLITE3_FLAGS = { value = "-DSQLITE_ENABLE_MATH_FUNCTIONS", force=true }
+SQLITE_MAX_VARIABLE_NUMBER = { value = "32784", force=true }
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index a1190990..d0081d47 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3589,11 +3589,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
dependencies = [
"bitflags 2.3.2",
+ "chrono",
+ "csv",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
+ "serde_json",
"smallvec",
+ "time 0.3.20",
+ "url",
+ "uuid",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 5f2286d0..e045d1b5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -88,4 +88,4 @@ safetensors = "0.3.1"
publicsuffix = "2"
robotstxt-with-cache = "0.4.0"
rust-s3 = "0.33.0"
-rusqlite = { version = "0.29.0", features = ["bundled"] }
\ No newline at end of file
+rusqlite = { version = "0.29.0", features = ["bundled", "modern-full"] }
\ No newline at end of file
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 9c755ba5..a307ba73 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -14,10 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{path::Path, sync::Arc};
+use std::{collections::HashSet, path::Path, rc::Rc, sync::Arc};
+use itertools::Itertools;
use rusqlite::{
- types::{FromSql, FromSqlError, FromSqlResult, ValueRef},
+ types::{FromSql, FromSqlError, FromSqlResult, Value, ValueRef},
ToSql,
};
@@ -163,8 +164,7 @@ impl CrawlDb {
// store temp tables in memory
conn.pragma_update(None, "temp_store", 2)?;
- // set cache size to 512 MB
- conn.pragma_update(None, "cache_size", -512_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)
+ rusqlite::vtab::array::load_module(&conn)?;
Ok(Self { conn })
}
@@ -217,22 +217,49 @@ impl Transaction<'_> {
}
pub fn update_max_inlinks_domains<'a, I: Iterator<Item = &'a Domain>>(
+ &self,
+ mut domains: I,
+ ) -> Result<()> {
+ for chunk in domains.by_ref().chunks(32_784).into_iter() {
+ self.update_max_inlinks_domains_chunk(chunk)?;
+ }
+
+ Ok(())
+ }
+
+ fn update_max_inlinks_domains_chunk<'a, I: Iterator<Item = &'a Domain>>(
&self,
domains: I,
) -> Result<()> {
- // update max_incoming_links for domain based on
- // max_incoming_links for URLs in domain.
- // if no pending URLs in domain, set max_incoming_links to 0
- let mut prepared = self.tx().prepare(
- "UPDATE domain SET max_incoming_links = IFNULL((
- SELECT MAX(incoming_links) FROM url
- WHERE domain = ?1 AND status = ?2
- ), 0) WHERE domain = ?1;",
+ // Collect domains into a Vec to use them later in the SQL
+ let domains: Rc<Vec<Value>> =
+ Rc::new(domains.map(|d| d.0.to_string()).map(Value::from).collect());
+
+ // Start transaction
+ let tx = self.tx();
+
+ // Create a temporary table to hold the max incoming links for each domain
+ tx.execute(
+ "CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain AS
+ SELECT domain, MAX(incoming_links) as max_incoming_links
+ FROM url
+ WHERE domain IN ?1 AND status = ?2
+ GROUP BY domain;",
+ (domains.clone(), UrlStatus::Pending),
)?;
- for domain in domains {
- prepared.execute((&domain.0, UrlStatus::Pending))?;
- }
+ // Update the domain table
+ tx.execute(
+ "UPDATE domain
+ SET max_incoming_links = IFNULL(
+ (SELECT max_incoming_links FROM temp_domain WHERE temp_domain.domain = domain.domain), 0
+ )
+ WHERE domain IN ?1;",
+ (domains, ),
+ )?;
+
+ // Drop the temporary table
+ tx.execute("DROP TABLE temp_domain;", [])?;
Ok(())
}
@@ -258,6 +285,8 @@ impl Transaction<'_> {
ON CONFLICT (domain) DO NOTHING;",
)?;
+ let mut unique_domains = HashSet::new();
+
for url in urls {
if domain.domain() != url.domain() {
prepared_diff_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;
@@ -265,8 +294,13 @@ impl Transaction<'_> {
prepared_same_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;
}
- prepared_insert_domain.execute((url.domain(), DomainStatus::Pending))?;
+ unique_domains.insert(url.domain().to_string());
}
+
+ for domain in unique_domains {
+ prepared_insert_domain.execute((domain, DomainStatus::Pending))?;
+ }
+
Ok(())
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 4326d9f6..084ddd30 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -83,8 +83,7 @@ impl Worker {
}
fn coordinator_conn(&self) -> sonic::ResilientConnection<ExponentialBackoff> {
- let retry =
- ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_millis(20_000));
+ let retry = ExponentialBackoff::from_millis(1_000).with_limit(Duration::from_secs(10));
sonic::ResilientConnection::create(self.coordinator_host, retry)
}