Skip to content

web search done right

Re-wrote crawldb to be backed by rocksdb This should significantly increase throughput and at the very least allow for more performance tuning. Hopefully the crawl coordinator is no longer the bottleneck

Mikkel Denkera36a826c3f1e
diff --git a/Cargo.lock b/Cargo.lock
index c279fdc7..c39c741f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1782,6 +1782,15 @@ dependencies = [
"ahash 0.7.6",
]
+[[package]]
+name = "hashbrown"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
+dependencies = [
+ "ahash 0.8.3",
+]
+
[[package]]
name = "hashbrown"
version = "0.14.0"
@@ -2386,6 +2395,15 @@ dependencies = [
"hashbrown 0.12.3",
]
+[[package]]
+name = "lru"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e"
+dependencies = [
+ "hashbrown 0.13.2",
+]
+
[[package]]
name = "lz-str"
version = "0.2.1"
@@ -4176,6 +4194,7 @@ dependencies = [
"lalrpop-util",
"libc",
"logos",
+ "lru 0.10.0",
"lz-str",
"maplit",
"md5",
@@ -4329,7 +4348,7 @@ dependencies = [
"itertools 0.10.5",
"levenshtein_automata",
"log",
- "lru",
+ "lru 0.7.8",
"lz4_flex",
"measure_time",
"memmap2",
diff --git a/Cargo.toml b/Cargo.toml
index dcad068a..322db5be 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -90,3 +90,4 @@ robotstxt-with-cache = "0.4.0"
rust-s3 = "0.33.0"
rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
async-channel = "1.8.0"
+lru = "0.10.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 2c4e7103..7717bd82 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -92,6 +92,7 @@ robotstxt-with-cache = { workspace = true }
rust-s3 = { workspace = true }
rusqlite = { workspace = true }
async-channel = { workspace = true }
+lru = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index bbe34b35..5df87e7b 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -2,7 +2,7 @@
use criterion::{criterion_group, criterion_main, Criterion};
use rand::seq::SliceRandom;
use stract::{
- crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse},
+ crawler::{CrawlCoordinator, JobResponse, UrlResponse},
webpage::Url,
};
@@ -1743,6 +1743,9 @@ Url::from("https://www.apple.com/apple-music".to_string()),
Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".to_string()),
];
+ // pre-initialize domain prefix list
+ letsencrypt_urls[0].domain();
+
let letsencrypt_url_responses: Vec<UrlResponse> = letsencrypt_urls
.clone()
.into_iter()
@@ -1757,25 +1760,31 @@ Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".t
let responses = vec![
JobResponse {
- domain: Domain("letsencrypt.org".to_string()),
+ domain: "letsencrypt.org".to_string().into(),
url_responses: letsencrypt_url_responses,
discovered_urls: apple_urls,
},
JobResponse {
- domain: Domain("apple.com".to_string()),
+ domain: "apple.com".to_string().into(),
url_responses: apple_url_responses,
discovered_urls: letsencrypt_urls,
},
];
let mut rng = rand::thread_rng();
- for _ in 0..1000 {
+ for _ in 0..10 {
c.bench_function("Add response to coordinator", |b| {
b.iter(|| {
let response = responses.choose(&mut rng).unwrap();
coordinator.add_response(response).unwrap();
})
});
+
+ c.bench_function("Sample jobs", |b| {
+ b.iter(|| {
+ coordinator.sample_jobs(256).unwrap();
+ })
+ });
}
}
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 792cd489..9bb813c6 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -18,10 +18,9 @@ use crate::{call_counter::CallCounter, webpage::Url};
use super::{
crawl_db::{CrawlDb, DomainStatus},
- Domain, Job, JobResponse, Result,
+ Job, JobResponse, Result,
};
use std::{
- collections::HashSet,
path::Path,
sync::{
atomic::{AtomicU64, Ordering},
@@ -33,7 +32,7 @@ use std::{
const DEFAULT_JOB_URLS: usize = 200;
pub struct CrawlCoordinator {
- db: CrawlDb,
+ db: Mutex<CrawlDb>,
num_crawled_urls: AtomicU64,
num_urls_to_crawl: u64,
call_counter: Mutex<CallCounter>,
@@ -45,21 +44,14 @@ impl CrawlCoordinator {
num_urls_to_crawl: u64,
seed_urls: Vec<String>,
) -> Result<Self> {
- let db = CrawlDb::open(crawldb_folder)?;
+ let mut db = CrawlDb::open(crawldb_folder)?;
let seed_urls = seed_urls.into_iter().map(Url::from).collect::<Vec<_>>();
- let domain_seeds = seed_urls
- .iter()
- .map(|url| Domain(url.domain().to_string()))
- .collect::<HashSet<_>>();
-
- db.transaction()?.insert_seed_urls(&seed_urls)?;
- db.transaction()?
- .update_max_inlinks_domains(domain_seeds.iter())?;
+ db.insert_seed_urls(&seed_urls)?;
Ok(Self {
- db,
+ db: Mutex::new(db),
num_urls_to_crawl,
num_crawled_urls: AtomicU64::new(0),
call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),
@@ -79,29 +71,19 @@ impl CrawlCoordinator {
pub fn add_response(&self, response: &JobResponse) -> Result<()> {
let start = Instant::now();
+ let mut db = self.db.lock().unwrap();
self.log_crawls_per_second(response.url_responses.len());
self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);
- let tx = self.db.transaction().unwrap();
- let mut domains = HashSet::new();
- tx.insert_urls(&response.domain, &response.discovered_urls)
+ db.insert_urls(&response.domain, &response.discovered_urls)
.unwrap();
- tx.update_url_status(&response.url_responses).unwrap();
+ db.update_url_status(&response.url_responses).unwrap();
- tx.set_domain_status(&response.domain, DomainStatus::Pending)
+ db.set_domain_status(&response.domain, DomainStatus::Pending)
.unwrap();
- domains.insert(response.domain.clone());
- domains.extend(
- response
- .discovered_urls
- .iter()
- .map(|url| Domain(url.domain().to_string())),
- );
-
- tx.update_max_inlinks_domains(domains.iter()).unwrap();
tracing::debug!("inserted responses in {:?}", start.elapsed());
@@ -114,11 +96,11 @@ impl CrawlCoordinator {
pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
let start = Instant::now();
- let tx = self.db.transaction()?;
+ let mut db = self.db.lock().unwrap();
- let domains = tx.sample_domains(num_jobs)?;
+ let domains = db.sample_domains(num_jobs)?;
tracing::debug!("sampled domains: {:?}", domains);
- let jobs = tx.prepare_jobs(&domains, DEFAULT_JOB_URLS)?;
+ let jobs = db.prepare_jobs(&domains, DEFAULT_JOB_URLS)?;
tracing::debug!("sampled jobs: {:?}", jobs);
tracing::info!("sampled {} jobs in {:?}", jobs.len(), start.elapsed());
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index ad8d67ee..da8011e6 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -14,367 +14,374 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::HashSet, path::Path, rc::Rc, sync::Arc};
-
-use itertools::Itertools;
-use rusqlite::{
- types::{FromSql, FromSqlError, FromSqlResult, Value, ValueRef},
- ToSql,
+use std::{
+ cmp::Reverse,
+ collections::{BinaryHeap, HashMap, VecDeque},
+ hash::Hash,
+ num::NonZeroUsize,
+ path::Path,
};
+use lru::LruCache;
+use rand::Rng;
+
use crate::webpage::Url;
use super::{Domain, Job, Result, UrlResponse};
-#[derive(Clone)]
+#[derive(Clone, PartialEq, Eq)]
pub enum UrlStatus {
Pending,
Crawling,
- Failed,
+ Failed { status_code: Option<u16> },
Done,
}
-impl ToSql for UrlStatus {
- fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
- match self {
- UrlStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(
- rusqlite::types::Value::Integer(0),
- )),
- UrlStatus::Crawling => Ok(rusqlite::types::ToSqlOutput::Owned(
- rusqlite::types::Value::Integer(1),
- )),
- UrlStatus::Failed => Ok(rusqlite::types::ToSqlOutput::Owned(
- rusqlite::types::Value::Integer(2),
- )),
- UrlStatus::Done => Ok(rusqlite::types::ToSqlOutput::Owned(
- rusqlite::types::Value::Integer(3),
- )),
- }
- }
-}
-
-impl FromSql for UrlStatus {
- fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
- match value {
- ValueRef::Integer(0) => Ok(UrlStatus::Pending),
- ValueRef::Integer(1) => Ok(UrlStatus::Crawling),
- ValueRef::Integer(2) => Ok(UrlStatus::Failed),
- ValueRef::Integer(3) => Ok(UrlStatus::Done),
- _ => Err(FromSqlError::InvalidType),
- }
- }
-}
-
+#[derive(Clone, PartialEq, Eq)]
pub enum DomainStatus {
Pending,
CrawlInProgress,
}
-impl ToSql for DomainStatus {
- fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
- match self {
- DomainStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(
- rusqlite::types::Value::Integer(0),
- )),
- DomainStatus::CrawlInProgress => Ok(rusqlite::types::ToSqlOutput::Owned(
- rusqlite::types::Value::Integer(1),
- )),
- }
- }
-}
+struct IdTable<T> {
+ t2id: rocksdb::DB,
+ id2t: rocksdb::DB,
+ next_id: u64,
-impl FromSql for DomainStatus {
- fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
- match value {
- ValueRef::Integer(0) => Ok(DomainStatus::Pending),
- ValueRef::Integer(1) => Ok(DomainStatus::CrawlInProgress),
- _ => Err(FromSqlError::InvalidType),
- }
- }
-}
+ t2id_cache: LruCache<T, u64>,
+ id2t_cache: LruCache<u64, T>,
-pub struct CrawlDb {
- conn: Arc<rusqlite::Connection>,
+ _marker: std::marker::PhantomData<T>,
}
-impl CrawlDb {
+impl<T> IdTable<T>
+where
+ T: serde::Serialize + serde::de::DeserializeOwned + Hash + Eq + Clone,
+{
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let conn = Arc::new(rusqlite::Connection::open(path)?);
- // create tables if not exists
- // there should be one table that contains all known URLs
- // when a URL is added to the table, it should be marked as `UrlStatus::Pending`
- // when a URL is being fetched, it should be marked as `UrlStatus::Crawling`
- // when a URL has been fetched, it should be marked as `UrlStatus::Done`.
- // Each URL should have a counter that counts the number of incoming links
- // from URLs on other domains.
- conn.execute(
- "CREATE TABLE IF NOT EXISTS url(
- url TEXT NOT NULL UNIQUE,
- domain TEXT NOT NULL,
- status INTEGER NOT NULL,
- error_code INTEGER,
- incoming_links INTEGER NOT NULL,
- PRIMARY KEY (url)
- );",
- [],
- )?;
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+ options.increase_parallelism(3);
- // // index by url and status
- // conn.execute(
- // "CREATE INDEX IF NOT EXISTS url_url_status ON url (url, status);",
- // [],
- // )?;
-
- conn.execute("CREATE INDEX IF NOT EXISTS url_domain ON url (domain);", [])?;
-
- // conn.execute(
- // "CREATE INDEX IF NOT EXISTS url_incoming_links ON url (incoming_links);",
- // [],
- // )?;
-
- // there should be one table that contains all known domains
- // and whether a crawl is in progress for that domain.
- // It should also contain a copy of the maximum count of incoming links
- // for any URL in that domain.
- conn.execute(
- "CREATE TABLE IF NOT EXISTS domain (
- domain TEXT NOT NULL UNIQUE,
- max_incoming_links INTEGER NOT NULL,
- status INTEGER NOT NULL,
- PRIMARY KEY (domain)
- );",
- [],
- )?;
+ // create dir if not exists
+ std::fs::create_dir_all(path.as_ref())?;
- // index by domain and status
- conn.execute(
- "CREATE INDEX IF NOT EXISTS domain_domain_status ON domain (domain, status);",
- [],
- )?;
+ let _ = rocksdb::DB::destroy(&options, path.as_ref().join("t2id"));
+ let _ = rocksdb::DB::destroy(&options, path.as_ref().join("id2t"));
- // store redirects
- conn.execute(
- "CREATE TABLE IF NOT EXISTS redirect (
- from_url INTEGER NOT NULL,
- to_url INTEGER NOT NULL,
- PRIMARY KEY (from_url, to_url)
- );",
- [],
- )?;
+ let t2id = rocksdb::DB::open_default(path.as_ref().join("t2id"))?;
+ let id2t = rocksdb::DB::open_default(path.as_ref().join("id2t"))?;
+
+ Ok(Self {
+ t2id,
+ id2t,
+ next_id: 0,
+
+ t2id_cache: LruCache::new(NonZeroUsize::new(1_000_000).unwrap()),
+ id2t_cache: LruCache::new(NonZeroUsize::new(1_000_000).unwrap()),
+
+ _marker: std::marker::PhantomData,
+ })
+ }
+
+ pub fn bulk_ids(&mut self, items: impl Iterator<Item = T>) -> Result<Vec<u64>> {
+ let mut ids = Vec::new();
+
+ let mut batch_id2t = rocksdb::WriteBatch::default();
+ let mut batch_t2id = rocksdb::WriteBatch::default();
+
+ for item in items {
+ // check cache
+ if let Some(id) = self.t2id_cache.get(&item) {
+ ids.push(*id);
+ continue;
+ }
+
+ // check if item exists
+ let item_bytes = bincode::serialize(&item)?;
+ let id = self.t2id.get(&item_bytes)?;
+ if let Some(id) = id {
+ let id = bincode::deserialize(&id)?;
+
+ // update cache
+ self.t2id_cache.put(item, id);
- // update performance stuff
- // WAL mode
- let _ = conn
- .prepare("PRAGMA journal_mode = WAL;")?
- .query([])?
- .next();
+ ids.push(id);
+ continue;
+ }
- // sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.
- let _ = conn.prepare("PRAGMA synchronous = OFF;")?.query([])?.next();
+ // insert item
+ let id = self.next_id;
+ self.next_id += 1;
+ let id_bytes = bincode::serialize(&id)?;
+ batch_t2id.put(&item_bytes, &id_bytes);
+ batch_id2t.put(&id_bytes, &item_bytes);
- // lock held by the SQLite connection is never released in EXCLUSIVE mode
- let _ = conn
- .prepare("PRAGMA locking_mode = EXCLUSIVE;")?
- .query([])?
- .next();
+ // update cache
+ self.t2id_cache.put(item, id);
- // store temp tables in memory
- let _ = conn.prepare("PRAGMA temp_store = 2;")?.query([])?.next();
+ ids.push(id);
+ }
- // set cache size to 128 MB
- conn.pragma_update(None, "cache_size", -128_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)
+ self.id2t.write(batch_id2t)?;
+ self.t2id.write(batch_t2id)?;
- conn.pragma_update(None, "page_size", 32_768)?;
+ Ok(ids)
+ }
+
+ pub fn id(&mut self, item: T) -> Result<u64> {
+ // check cache
+ if let Some(id) = self.t2id_cache.get(&item) {
+ return Ok(*id);
+ }
+
+ // check if item exists
+ let item_bytes = bincode::serialize(&item)?;
+ let id = self.t2id.get(&item_bytes)?;
+ if let Some(id) = id {
+ let id = bincode::deserialize(&id)?;
+
+ // update cache
+ self.t2id_cache.put(item.clone(), id);
+ self.id2t_cache.put(id, item);
+
+ return Ok(id);
+ }
- rusqlite::vtab::array::load_module(&conn)?;
+ // insert item
+ let id = self.next_id;
+ self.next_id += 1;
+ let id_bytes = bincode::serialize(&id)?;
+ self.t2id.put(&item_bytes, &id_bytes)?;
+ self.id2t.put(&id_bytes, &item_bytes)?;
- Ok(Self { conn })
+ // update cache
+ self.t2id_cache.put(item.clone(), id);
+ self.id2t_cache.put(id, item);
+
+ Ok(id)
}
- pub fn transaction(&self) -> Result<Transaction<'_>> {
- Ok(Transaction {
- tx: Some(self.conn.unchecked_transaction()?),
- })
+ pub fn value(&mut self, id: u64) -> Result<Option<T>> {
+ // check cache
+ if let Some(value) = self.id2t_cache.get(&id) {
+ return Ok(Some(value.clone()));
+ }
+
+ let id_bytes = bincode::serialize(&id)?;
+ let value_bytes = self.id2t.get(id_bytes)?;
+ if let Some(value_bytes) = value_bytes {
+ let value: T = bincode::deserialize(&value_bytes)?;
+
+ // update cache
+ self.t2id_cache.put(value.clone(), id);
+ self.id2t_cache.put(id, value.clone());
+
+ return Ok(Some(value));
+ }
+
+ Ok(None)
}
}
-pub struct Transaction<'a> {
- tx: Option<rusqlite::Transaction<'a>>,
+struct SampledItem<'a, T> {
+ item: &'a T,
+ priority: f64,
}
-impl Drop for Transaction<'_> {
- fn drop(&mut self) {
- self.tx.take().unwrap().commit().unwrap();
+impl<'a, T> PartialEq for SampledItem<'a, T> {
+ fn eq(&self, other: &Self) -> bool {
+ self.priority == other.priority
}
}
-impl Transaction<'_> {
- fn tx(&self) -> &rusqlite::Transaction<'_> {
- self.tx.as_ref().unwrap()
+impl<'a, T> Eq for SampledItem<'a, T> {}
+
+impl<'a, T> PartialOrd for SampledItem<'a, T> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.priority.partial_cmp(&other.priority)
}
+}
- pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {
- let mut prepared_insert_url = self.tx().prepare(
- "INSERT INTO url (url, domain, status, incoming_links)
- VALUES (?1, ?2, ?3, ?4)
- ON CONFLICT (url) DO NOTHING;",
- )?;
- let mut prepared_insert_domain = self.tx().prepare(
- "INSERT INTO domain (domain, max_incoming_links, status)
- VALUES (?1, 0, ?2)
- ON CONFLICT (domain) DO NOTHING;",
- )?;
+impl<'a, T> Ord for SampledItem<'a, T> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.priority
+ .partial_cmp(&other.priority)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ }
+}
- for url in urls {
- prepared_insert_url.execute((
- url.full(),
- url.domain().to_string(),
- UrlStatus::Pending,
- 0,
- ))?;
- prepared_insert_domain.execute((url.domain().to_string(), DomainStatus::Pending))?;
+fn weighted_sample<'a, T: 'a>(
+ items: impl Iterator<Item = (&'a T, f64)>,
+ num_items: usize,
+) -> Vec<&'a T> {
+ let mut sampled_items: BinaryHeap<Reverse<SampledItem<T>>> =
+ BinaryHeap::with_capacity(num_items);
+
+ let mut rng = rand::thread_rng();
+
+ for (item, weight) in items {
+ // see https://www.kaggle.com/code/kotamori/random-sample-with-weights-on-sql/notebook for details on math
+ // let priority = -log((abs(random()) % 1000000 + 0.5) / 1000000.0) / (max_incoming_links + 1)
+ let priority = -(rng.gen::<f64>().abs() + 0.5).ln() / (weight + 1.0);
+ if sampled_items.len() < num_items {
+ sampled_items.push(Reverse(SampledItem { item, priority }));
+ } else if let Some(mut min) = sampled_items.peek_mut() {
+ if min.0.priority < priority {
+ min.0.item = item;
+ min.0.priority = priority;
+ }
}
+ }
- Ok(())
+ sampled_items.into_iter().map(|s| s.0.item).collect()
+}
+
+struct UrlState {
+ weight: f64,
+ status: UrlStatus,
+}
+struct DomainState {
+ weight: f64,
+ status: DomainStatus,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct DomainId(u64);
+
+impl From<u64> for DomainId {
+ fn from(id: u64) -> Self {
+ Self(id)
}
+}
- pub fn update_max_inlinks_domains<'a, I: Iterator<Item = &'a Domain>>(
- &self,
- mut domains: I,
- ) -> Result<()> {
- // 32_784 is the maximum number of parameters that SQLite supports.
- // This is configured in the SQLITE_MAX_VARIABLE_NUMBER environment variable in .cargo/config.toml.
- for chunk in domains.by_ref().chunks(32_784).into_iter() {
- self.update_max_inlinks_domains_chunk(chunk)?;
- }
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+struct UrlId(u64);
- Ok(())
+impl From<u64> for UrlId {
+ fn from(id: u64) -> Self {
+ Self(id)
}
+}
- fn update_max_inlinks_domains_chunk<'a, I: Iterator<Item = &'a Domain>>(
- &self,
- domains: I,
- ) -> Result<()> {
- // Collect domains into a Vec to use them later in the SQL
- let domains: Rc<Vec<Value>> =
- Rc::new(domains.map(|d| d.0.to_string()).map(Value::from).collect());
-
- // Start transaction
- let tx = self.tx();
-
- tx.execute("DROP TABLE IF EXISTS temp_domain;", [])?;
- // Create a temporary table to hold the max incoming links for each domain
- tx.execute(
- "CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain AS
- SELECT domain, MAX(incoming_links) as max_incoming_links
- FROM url
- WHERE domain IN rarray(?1) AND status = ?2
- GROUP BY domain;",
- (domains.clone(), UrlStatus::Pending),
- )?;
+struct RedirectDb {
+ inner: rocksdb::DB,
+}
- // Update the domain table
- tx.execute(
- "UPDATE domain
- SET max_incoming_links = IFNULL(
- (SELECT max_incoming_links FROM temp_domain WHERE temp_domain.domain = domain.domain), 0
- )
- WHERE domain in rarray(?1);",
- (domains,),
- )?;
+impl RedirectDb {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let inner = rocksdb::DB::open_default(path.as_ref())?;
+
+ Ok(Self { inner })
+ }
- // Drop the temporary table
- tx.execute("DROP TABLE temp_domain;", [])?;
+ pub fn put(&self, from: &Url, to: &Url) -> Result<()> {
+ let url_bytes = bincode::serialize(from)?;
+ let redirect_bytes = bincode::serialize(to)?;
+ self.inner.put(url_bytes, redirect_bytes)?;
Ok(())
}
+}
- pub fn insert_urls(&self, crawled_domain: &Domain, urls: &[Url]) -> Result<()> {
- // insert discovered URLs if not already in database
- // update incoming link counts for discovered URLs
- // if count is above prev max for domain, update max
- let domain = Url::from(crawled_domain.0.clone());
- let mut prepared_diff_domain = self.tx().prepare(
- "INSERT INTO url (url, domain, status, incoming_links)
- VALUES (?1, ?2, ?3, 1)
- ON CONFLICT (url) DO UPDATE SET incoming_links = incoming_links + 1;",
- )?;
- let mut prepared_same_domain = self.tx().prepare(
- "INSERT INTO url (url, domain, status, incoming_links)
- VALUES (?1, ?2, ?3, 0)
- ON CONFLICT (url) DO NOTHING;",
- )?;
- let mut prepared_insert_domain = self.tx().prepare(
- "INSERT INTO domain (domain, max_incoming_links, status)
- VALUES (?1, 0, ?2)
- ON CONFLICT (domain) DO NOTHING;",
- )?;
+pub struct CrawlDb {
+ url_ids: IdTable<Url>,
+ domain_ids: IdTable<Domain>,
- let mut unique_domains = HashSet::new();
+ redirects: RedirectDb,
- for url in urls {
- if domain.domain() != url.domain() {
- prepared_diff_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;
- } else {
- prepared_same_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;
- }
+ domain_state: HashMap<DomainId, DomainState>,
- unique_domains.insert(url.domain().to_string());
- }
+ urls: HashMap<DomainId, HashMap<UrlId, UrlState>>,
+}
+
+impl CrawlDb {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let url_ids = IdTable::open(path.as_ref().join("urls"))?;
+ let domain_ids = IdTable::open(path.as_ref().join("domains"))?;
+ let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;
+
+ Ok(Self {
+ url_ids,
+ domain_ids,
+ redirects,
+ domain_state: HashMap::new(),
+ urls: HashMap::new(),
+ })
+ }
- for domain in unique_domains {
- prepared_insert_domain.execute((domain, DomainStatus::Pending))?;
+ pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {
+ for url in urls {
+ let domain_id = self.domain_ids.id(url.into())?.into();
+ let url_id = self.url_ids.id(url.clone())?.into();
+
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+
+ self.urls.entry(domain_id).or_default().insert(
+ url_id,
+ UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ },
+ );
}
Ok(())
}
- pub fn update_url_status(&self, url_responses: &[UrlResponse]) -> Result<()> {
- let mut prepared_update_status_url = self
- .tx()
- .prepare("UPDATE url SET status = ?1, error_code = ?2 WHERE url = ?3;")?;
+ pub fn insert_urls(&mut self, crawled_domain: &Domain, urls: &[Url]) -> Result<()> {
+ let mut domains: HashMap<Domain, Vec<Url>> = HashMap::new();
- let mut prepared_insert_redirect = self.tx().prepare(
- "INSERT INTO redirect (from_url, to_url)
- VALUES (?1, ?2)
- ON CONFLICT (from_url, to_url) DO NOTHING;",
- )?;
+ for url in urls {
+ let domain: Domain = url.domain().to_string().into();
- let mut prepared_insert_url = self.tx().prepare(
- "INSERT INTO url (url, domain, status, incoming_links)
- VALUES (?1, ?2, ?3, 0)
- ON CONFLICT (url) DO NOTHING;",
- )?;
+ domains.entry(domain).or_default().push(url.clone());
+ }
- // update status of URLs
- for url_res in url_responses {
- match url_res {
- UrlResponse::Success { url } => {
- prepared_update_status_url.execute((
- UrlStatus::Done,
- None::<u16>,
- url.full(),
- ))?;
+ let domain_ids: Vec<DomainId> = self
+ .domain_ids
+ .bulk_ids(domains.keys().cloned())?
+ .into_iter()
+ .map(DomainId::from)
+ .collect();
+
+ self.url_ids.bulk_ids(domains.values().flatten().cloned())?;
+
+ for (domain_id, (domain, urls)) in domain_ids.into_iter().zip(domains.into_iter()) {
+ for url in urls {
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let domain_state =
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+
+ domain_state.status = DomainStatus::Pending;
+
+ let url_state = self
+ .urls
+ .entry(domain_id)
+ .or_default()
+ .entry(url_id)
+ .or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ if &domain != crawled_domain {
+ url_state.weight += 1.0;
}
- UrlResponse::Failed { url, status_code } => {
- prepared_update_status_url.execute((
- UrlStatus::Failed,
- status_code,
- url.full(),
- ))?;
- }
- UrlResponse::Redirected { url, new_url } => {
- prepared_update_status_url.execute((
- UrlStatus::Done,
- None::<u16>,
- url.full(),
- ))?;
- prepared_insert_redirect.execute((url.full(), new_url.full()))?;
- prepared_insert_url.execute((
- new_url.full(),
- new_url.domain(),
- UrlStatus::Done,
- ))?;
+
+ if url_state.weight > domain_state.weight {
+ domain_state.weight = url_state.weight;
}
}
}
@@ -382,85 +389,169 @@ impl Transaction<'_> {
Ok(())
}
- pub fn set_domain_status(&self, domain: &Domain, status: DomainStatus) -> Result<()> {
- self.tx().execute(
- "UPDATE domain SET status = ?1 WHERE domain = ?2;",
- (status, &domain.0),
+ pub fn update_url_status(&mut self, url_responses: &[UrlResponse]) -> Result<()> {
+ // bulk register urls
+ self.url_ids.bulk_ids(
+ url_responses
+ .iter()
+ .flat_map(|res| match res {
+ UrlResponse::Success { url } => vec![url].into_iter(),
+ UrlResponse::Failed {
+ url,
+ status_code: _,
+ } => vec![url].into_iter(),
+ UrlResponse::Redirected { url, new_url } => vec![url, new_url].into_iter(),
+ })
+ .cloned(),
)?;
+ for response in url_responses {
+ match response {
+ UrlResponse::Success { url } => {
+ let domain: Domain = url.domain().to_string().into();
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let domain_state =
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+ domain_state.status = DomainStatus::Pending;
+
+ let url_state = self
+ .urls
+ .entry(domain_id)
+ .or_default()
+ .entry(url_id)
+ .or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ url_state.status = UrlStatus::Done;
+ }
+ UrlResponse::Failed { url, status_code } => {
+ let domain: Domain = url.domain().to_string().into();
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let domain_state =
+ self.domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+ domain_state.status = DomainStatus::Pending;
+
+ let url_state = self
+ .urls
+ .entry(domain_id)
+ .or_default()
+ .entry(url_id)
+ .or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ url_state.status = UrlStatus::Failed {
+ status_code: *status_code,
+ };
+ }
+ UrlResponse::Redirected { url, new_url } => {
+ self.redirects.put(url, new_url)?;
+ }
+ }
+ }
Ok(())
}
- pub fn sample_domains(&self, num_jobs: usize) -> Result<Vec<Domain>> {
- // weighted sample from domains that are not currently being crawled
- // see https://www.kaggle.com/code/kotamori/random-sample-with-weights-on-sql/notebook for details on math
- let mut stmt = self.tx().prepare(
- "SELECT domain, -log((abs(random()) % 1000000 + 0.5) / 1000000.0) / (max_incoming_links + 1) as priority FROM domain
- WHERE status = ?1
- ORDER BY priority DESC
- LIMIT ?2;",
- )?;
-
- // dfs sample
- // let mut stmt = self.tx().prepare(
- // "SELECT domain FROM domain
- // WHERE status = ?1
- // LIMIT ?2;",
- // )?;
+ pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
- let rows = stmt.query_map((DomainStatus::Pending, num_jobs), |row| {
- row.get::<_, String>(0)
- })?;
+ let domain_state = self
+ .domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
- let mut domains = Vec::new();
- for domain in rows {
- domains.push(Domain(domain?));
- }
+ domain_state.status = status;
- Ok(domains)
+ Ok(())
}
- pub fn prepare_jobs(&self, domains: &[Domain], urls_per_job: usize) -> Result<Vec<Job>> {
- let mut jobs = Vec::new();
- // get URLs from sampled domains
- let mut stmt = self.tx().prepare(
- "SELECT url FROM url
- WHERE domain = ?1 AND status = ?2
- ORDER BY incoming_links DESC
- LIMIT ?3;",
- )?;
-
- let mut prepared_update_status_url = self
- .tx()
- .prepare("UPDATE url SET status = ?1 WHERE url = ?2;")?;
- let mut prepared_update_status_domain = self
- .tx()
- .prepare("UPDATE domain SET status = ?1 WHERE domain = ?2;")?;
+ pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<DomainId>> {
+ let sampled = weighted_sample(
+ self.domain_state.iter_mut().filter_map(|(id, state)| {
+ if state.status == DomainStatus::Pending {
+ state.status = DomainStatus::CrawlInProgress;
+ Some((id, state.weight))
+ } else {
+ None
+ }
+ }),
+ num_jobs,
+ );
- for domain in domains {
- let mut urls = Vec::new();
+ Ok(sampled.into_iter().copied().collect())
+ }
- let rows = stmt.query_map((&domain.0, UrlStatus::Pending, urls_per_job), |row| {
- row.get::<_, String>(0)
- })?;
+ pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {
+ let mut jobs = Vec::with_capacity(domains.len());
+ for domain_id in domains {
+ let urls = self.urls.entry(*domain_id).or_default();
+
+ let sampled = weighted_sample(
+ urls.iter_mut().filter_map(|(id, state)| {
+ if state.status == UrlStatus::Pending {
+ state.status = UrlStatus::Crawling;
+ Some((id, state.weight))
+ } else {
+ None
+ }
+ }),
+ urls_per_job,
+ );
+
+ let mut job = Job {
+ domain: self.domain_ids.value(domain_id.0)?.unwrap(),
+ fetch_sitemap: true, // todo: make this configureable
+ urls: VecDeque::with_capacity(urls_per_job),
+ };
+
+ for url_id in sampled {
+ let url = self.url_ids.value(url_id.0)?.unwrap();
+ job.urls.push_back(url);
+ }
- prepared_update_status_domain.execute((DomainStatus::CrawlInProgress, &domain.0))?;
+ jobs.push(job);
+ }
- for url in rows {
- let url = url?;
+ Ok(jobs)
+ }
+}
- prepared_update_status_url.execute((UrlStatus::Crawling, url.clone()))?;
+#[cfg(test)]
+mod tests {
+ use super::*;
- urls.push(url.into());
- }
+ #[test]
+ fn sampling() {
+ let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];
+ let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 10);
+ assert_eq!(sampled.len(), items.len());
- jobs.push(Job {
- domain: domain.clone(),
- fetch_sitemap: true, // TODO: only fetch sitemap if we haven't already
- urls: urls.into(),
- });
- }
+ let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];
+ let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 1);
+ assert_eq!(sampled.len(), 1);
- Ok(jobs)
+ let items: Vec<(usize, f64)> = vec![(0, 1.0), (1, 2.0), (2, 3.0), (3, 4.0)];
+ let sampled = weighted_sample(items.iter().map(|(i, w)| (i, *w)), 0);
+ assert_eq!(sampled.len(), 0);
}
}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index fb76d2aa..7b87301b 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -68,6 +68,15 @@ pub enum Error {
#[error("async channel: {0}")]
SendError(#[from] async_channel::SendError<WarcWriterMessage>),
+
+ #[error("bincode: {0}")]
+ Bincode(#[from] bincode::Error),
+
+ #[error("rocksdb: {0}")]
+ Rocksdb(#[from] rocksdb::Error),
+
+ #[error("io: {0}")]
+ Io(#[from] std::io::Error),
}
type Result<T> = std::result::Result<T, Error>;
@@ -76,7 +85,19 @@ type Result<T> = std::result::Result<T, Error>;
struct Site(String);
#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
-pub struct Domain(pub String);
+pub struct Domain(String);
+
+impl From<&Url> for Domain {
+ fn from(url: &Url) -> Self {
+ Self(url.domain().to_string())
+ }
+}
+
+impl From<String> for Domain {
+ fn from(s: String) -> Self {
+ Self(s)
+ }
+}
/// All urls in a job must be from the same domain and only one job per domain.
/// at a time. This ensures that we stay polite when crawling.