web search done right
refactor crawl_db out from crawl coordinator
Mikkel Denkerb2937661fbed
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 667bbfc4..23ed943c 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -14,94 +14,23 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use rusqlite::{- types::{FromSql, FromSqlError, FromSqlResult, ValueRef},- ToSql,-};-use crate::webpage::Url;-use super::{Domain, Job, JobResponse, Result, UrlResponse};+use super::{+ crawl_db::{CrawlDb, DomainStatus},+ Domain, Job, JobResponse, Result,+};use std::{collections::HashSet,path::Path,- sync::{- atomic::{AtomicU64, Ordering},- Arc,- },+ sync::atomic::{AtomicU64, Ordering},time::Instant,};const DEFAULT_JOB_URLS: usize = 1000;-enum UrlStatus {- Pending,- Crawling,- Failed,- Done,-}--impl ToSql for UrlStatus {- fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {- match self {- UrlStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(- rusqlite::types::Value::Integer(0),- )),- UrlStatus::Crawling => Ok(rusqlite::types::ToSqlOutput::Owned(- rusqlite::types::Value::Integer(1),- )),- UrlStatus::Failed => Ok(rusqlite::types::ToSqlOutput::Owned(- rusqlite::types::Value::Integer(2),- )),- UrlStatus::Done => Ok(rusqlite::types::ToSqlOutput::Owned(- rusqlite::types::Value::Integer(3),- )),- }- }-}--impl FromSql for UrlStatus {- fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {- match value {- ValueRef::Integer(0) => Ok(UrlStatus::Pending),- ValueRef::Integer(1) => Ok(UrlStatus::Crawling),- ValueRef::Integer(2) => Ok(UrlStatus::Failed),- ValueRef::Integer(3) => Ok(UrlStatus::Done),- _ => Err(FromSqlError::InvalidType),- }- }-}--enum DomainStatus {- Pending,- CrawlInProgress,-}--impl ToSql for DomainStatus {- fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {- match self {- DomainStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(- rusqlite::types::Value::Integer(0),- )),- DomainStatus::CrawlInProgress => Ok(rusqlite::types::ToSqlOutput::Owned(- rusqlite::types::Value::Integer(1),- )),- }- }-}--impl FromSql for DomainStatus {- fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {- match value {- ValueRef::Integer(0) => Ok(DomainStatus::Pending),- ValueRef::Integer(1) => Ok(DomainStatus::CrawlInProgress),- _ => Err(FromSqlError::InvalidType),- }- }-}-pub struct CrawlCoordinator {- conn: Arc<rusqlite::Connection>,+ db: CrawlDb,num_crawled_urls: AtomicU64,num_urls_to_crawl: u64,}@@ -112,203 +41,34 @@ impl CrawlCoordinator {num_urls_to_crawl: u64,seed_urls: Vec<String>,) -> Result<Self> {- let conn = Arc::new(rusqlite::Connection::open(crawldb_folder)?);-- // create tables if not exists- // there should be one table that contains all known URLs- // when a URL is added to the table, it should be marked as `UrlStatus::Pending`- // when a URL is being fetched, it should be marked as `UrlStatus::Crawling`- // when a URL has been fetched, it should be marked as `UrlStatus::Done`.- // Each URL should have a counter that counts the number of incoming links- // from URLs on other domains.- conn.execute(- "CREATE TABLE IF NOT EXISTS url(- url TEXT NOT NULL UNIQUE,- domain TEXT NOT NULL,- status INTEGER NOT NULL,- error_code INTEGER,- incoming_links INTEGER NOT NULL,- PRIMARY KEY (url)- );",- [],- )?;-- // index by url and status- conn.execute(- "CREATE INDEX IF NOT EXISTS url_url_status ON url (url, status);",- [],- )?;+ let db = CrawlDb::open(crawldb_folder)?;- // there should be one table that contains all known domains- // and whether a crawl is in progress for that domain.- // It should also contain a copy of the maximum count of incoming links- // for any URL in that domain.- conn.execute(- "CREATE TABLE IF NOT EXISTS domain (- domain TEXT NOT NULL UNIQUE,- max_incoming_links INTEGER NOT NULL,- status INTEGER NOT NULL,- PRIMARY KEY (domain)- );",- [],- )?;-- // index by domain and status- conn.execute(- "CREATE INDEX IF NOT EXISTS domain_domain_status ON domain (domain, status);",- [],- )?;-- // store redirects- conn.execute(- "CREATE TABLE IF NOT EXISTS redirect (- from_url INTEGER NOT NULL,- to_url INTEGER NOT NULL,- PRIMARY KEY (from_url, to_url)- );",- [],- )?;-- // update performance stuff- // WAL mode- conn.pragma_update(None, "journal_mode", "'WAL'")?;-- // sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.- conn.pragma_update(None, "synchronous", 0)?;-- // store temp tables in memory- conn.pragma_update(None, "temp_store", 2)?;-- // set cache size to 512 MB- conn.pragma_update(None, "cache_size", -512_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)+ let seed_urls = seed_urls.into_iter().map(Url::from).collect::<Vec<_>>();let domain_seeds = seed_urls.iter()- .map(|url| Domain(Url::from(url.clone()).domain().to_string()))+ .map(|url| Domain(url.domain().to_string())).collect::<HashSet<_>>();- {- let mut prepared_insert_url = conn.prepare(- "INSERT INTO url (url, domain, status, incoming_links)- VALUES (?1, ?2, ?3, ?4)- ON CONFLICT (url) DO NOTHING;",- )?;- let mut prepared_insert_domain = conn.prepare(- "INSERT INTO domain (domain, max_incoming_links, status)- VALUES (?1, 0, ?2)- ON CONFLICT (domain) DO NOTHING;",- )?;-- // insert seed URLs- for url in seed_urls {- let url = Url::from(url);- prepared_insert_url.execute((- url.full(),- url.domain().to_string(),- UrlStatus::Pending,- 0,- ))?;- prepared_insert_domain- .execute((url.domain().to_string(), DomainStatus::Pending))?;- }- }+ db.transaction()?.insert_seed_urls(&seed_urls)?;+ db.transaction()?+ .update_max_inlinks_domains(domain_seeds.iter())?;- let coord = Self {- conn,+ Ok(Self {+ db,num_urls_to_crawl,num_crawled_urls: AtomicU64::new(0),- };-- coord.update_max_inlinks_domains(domain_seeds.iter())?;-- Ok(coord)+ })}pub fn add_response(&self, response: &JobResponse) -> Result<()> {self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);- // begin transaction- let tx = self.conn.unchecked_transaction()?;-- {- // insert discovered URLs if not already in database- // update incoming link counts for discovered URLs- // if count is above prev max for domain, update max- let domain = Url::from(response.domain.0.clone());- let mut prepared_diff_domain = tx.prepare(- "INSERT INTO url (url, domain, status, incoming_links)- VALUES (?1, ?2, ?3, 1)- ON CONFLICT (url) DO UPDATE SET incoming_links = incoming_links + 1;",- )?;- let mut prepared_same_domain = tx.prepare(- "INSERT INTO url (url, domain, status, incoming_links)- VALUES (?1, ?2, ?3, 0)- ON CONFLICT (url) DO NOTHING;",- )?;- let mut prepared_insert_domain = tx.prepare(- "INSERT INTO domain (domain, max_incoming_links, status)- VALUES (?1, 0, ?2)- ON CONFLICT (domain) DO NOTHING;",- )?;-- for url in &response.discovered_urls {- if domain.domain() != url.domain() {- prepared_diff_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;- } else {- prepared_same_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;- }-- prepared_insert_domain.execute((url.domain(), DomainStatus::Pending))?;- }-- let mut prepared_update_status_url =- tx.prepare("UPDATE url SET status = ?1, error_code = ?2 WHERE url = ?3;")?;- let mut prepared_insert_redirect = tx.prepare(- "INSERT INTO redirect (from_url, to_url)- VALUES (?1, ?2)- ON CONFLICT (from_url, to_url) DO NOTHING;",- )?;-- // update status of URLs- for url_res in &response.url_responses {- match url_res {- UrlResponse::Success { url } => {- prepared_update_status_url.execute((- UrlStatus::Done,- None::<u16>,- url.full(),- ))?;- }- UrlResponse::Failed { url, status_code } => {- prepared_update_status_url.execute((- UrlStatus::Failed,- status_code,- url.full(),- ))?;- }- UrlResponse::Redirected { url, new_url } => {- prepared_update_status_url.execute((- UrlStatus::Done,- None::<u16>,- url.full(),- ))?;- prepared_insert_redirect.execute((url.full(), new_url.full()))?;- prepared_same_domain.execute((- new_url.full(),- new_url.domain(),- UrlStatus::Done,- 0,- ))?;- }- }- }+ let tx = self.db.transaction()?;- tx.execute(- "UPDATE domain SET status = ?1 WHERE domain = ?2;",- (DomainStatus::Pending, &response.domain.0),- )?;- }+ tx.insert_urls(&response.domain, &response.discovered_urls)?;+ tx.update_url_status(&response.url_responses)?;- tx.commit()?;+ tx.set_domain_status(&response.domain, DomainStatus::Pending)?;let mut domains = HashSet::new();domains.insert(response.domain.clone());@@ -319,34 +79,7 @@ impl CrawlCoordinator {.map(|url| Domain(url.domain().to_string())),);- self.update_max_inlinks_domains(domains.iter())?;-- Ok(())- }-- fn update_max_inlinks_domains<'a, I: Iterator<Item = &'a Domain>>(- &self,- domains: I,- ) -> Result<()> {- let tx = self.conn.unchecked_transaction()?;-- {- // update max_incoming_links for domain based on- // max_incoming_links for URLs in domain.- // if no pending URLs in domain, set max_incoming_links to 0- let mut prepared = tx.prepare(- "UPDATE domain SET max_incoming_links = IFNULL((- SELECT MAX(incoming_links) FROM url- WHERE domain = ?1 AND status = ?2- ), 0) WHERE domain = ?1;",- )?;-- for domain in domains {- prepared.execute((&domain.0, UrlStatus::Pending))?;- }- }-- tx.commit()?;+ tx.update_max_inlinks_domains(domains.iter())?;Ok(())}@@ -357,78 +90,11 @@ impl CrawlCoordinator {pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {let start = Instant::now();- let tx = self.conn.unchecked_transaction()?;- let mut jobs = Vec::new();-- {- // weighted sample from domains that are not currently being crawled- // see https://www.kaggle.com/code/kotamori/random-sample-with-weights-on-sql/notebook for details on math- let mut stmt = tx.prepare(- "SELECT domain, -log((abs(random()) % 1000000 + 0.5) / 1000000.0) / (max_incoming_links + 1) as priority FROM domain- WHERE status = ?1- ORDER BY priority DESC- LIMIT ?2;",- )?;-- // dfs sample- // let mut stmt = tx.prepare(- // "SELECT domain FROM domain- // WHERE status = ?1- // LIMIT ?2;",- // )?;-- let rows = stmt.query_map((DomainStatus::Pending, num_jobs), |row| {- row.get::<_, String>(0)- })?;-- let mut domains = Vec::new();- for domain in rows {- domains.push(Domain(domain?));- }-- tracing::debug!("sampled domains: {:?}", domains);-- // get URLs from sampled domains- let mut stmt = tx.prepare(- "SELECT url FROM url- WHERE domain = ?1 AND status = ?2- ORDER BY incoming_links DESC- LIMIT ?3;",- )?;-- let mut prepared_update_status_url =- tx.prepare("UPDATE url SET status = ?1 WHERE url = ?2;")?;- let mut prepared_update_status_domain =- tx.prepare("UPDATE domain SET status = ?1 WHERE domain = ?2;")?;-- for domain in domains {- let mut urls = Vec::new();-- let rows = stmt- .query_map((&domain.0, UrlStatus::Pending, DEFAULT_JOB_URLS), |row| {- row.get::<_, String>(0)- })?;-- prepared_update_status_domain- .execute((DomainStatus::CrawlInProgress, &domain.0))?;-- for url in rows {- let url = url?;-- prepared_update_status_url.execute((UrlStatus::Crawling, url.clone()))?;-- urls.push(url.into());- }-- jobs.push(Job {- domain,- fetch_sitemap: true, // TODO: only fetch sitemap if we haven't already- urls: urls.into(),- });- }- }- tx.commit()?;+ let tx = self.db.transaction()?;+ let domains = tx.sample_domains(num_jobs)?;+ tracing::debug!("sampled domains: {:?}", domains);+ let jobs = tx.prepare_jobs(&domains, DEFAULT_JOB_URLS)?;tracing::debug!("sampled jobs: {:?}", jobs);tracing::info!("sampled {} jobs in {:?}", jobs.len(), start.elapsed());diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsnew file mode 100644index 00000000..9c755ba5--- /dev/null+++ b/core/src/crawler/crawl_db.rs@@ -0,0 +1,407 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{path::Path, sync::Arc};++use rusqlite::{+ types::{FromSql, FromSqlError, FromSqlResult, ValueRef},+ ToSql,+};++use crate::webpage::Url;++use super::{Domain, Job, Result, UrlResponse};++pub enum UrlStatus {+ Pending,+ Crawling,+ Failed,+ Done,+}++impl ToSql for UrlStatus {+ fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {+ match self {+ UrlStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(0),+ )),+ UrlStatus::Crawling => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(1),+ )),+ UrlStatus::Failed => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(2),+ )),+ UrlStatus::Done => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(3),+ )),+ }+ }+}++impl FromSql for UrlStatus {+ fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {+ match value {+ ValueRef::Integer(0) => Ok(UrlStatus::Pending),+ ValueRef::Integer(1) => Ok(UrlStatus::Crawling),+ ValueRef::Integer(2) => Ok(UrlStatus::Failed),+ ValueRef::Integer(3) => Ok(UrlStatus::Done),+ _ => Err(FromSqlError::InvalidType),+ }+ }+}++pub enum DomainStatus {+ Pending,+ CrawlInProgress,+}++impl ToSql for DomainStatus {+ fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {+ match self {+ DomainStatus::Pending => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(0),+ )),+ DomainStatus::CrawlInProgress => Ok(rusqlite::types::ToSqlOutput::Owned(+ rusqlite::types::Value::Integer(1),+ )),+ }+ }+}++impl FromSql for DomainStatus {+ fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {+ match value {+ ValueRef::Integer(0) => Ok(DomainStatus::Pending),+ ValueRef::Integer(1) => Ok(DomainStatus::CrawlInProgress),+ _ => Err(FromSqlError::InvalidType),+ }+ }+}++pub struct CrawlDb {+ conn: Arc<rusqlite::Connection>,+}++impl CrawlDb {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let conn = Arc::new(rusqlite::Connection::open(path)?);+ // create tables if not exists+ // there should be one table that contains all known URLs+ // when a URL is added to the table, it should be marked as `UrlStatus::Pending`+ // when a URL is being fetched, it should be marked as `UrlStatus::Crawling`+ // when a URL has been fetched, it should be marked as `UrlStatus::Done`.+ // Each URL should have a counter that counts the number of incoming links+ // from URLs on other domains.+ conn.execute(+ "CREATE TABLE IF NOT EXISTS url(+ url TEXT NOT NULL UNIQUE,+ domain TEXT NOT NULL,+ status INTEGER NOT NULL,+ error_code INTEGER,+ incoming_links INTEGER NOT NULL,+ PRIMARY KEY (url)+ );",+ [],+ )?;++ // index by url and status+ conn.execute(+ "CREATE INDEX IF NOT EXISTS url_url_status ON url (url, status);",+ [],+ )?;++ // there should be one table that contains all known domains+ // and whether a crawl is in progress for that domain.+ // It should also contain a copy of the maximum count of incoming links+ // for any URL in that domain.+ conn.execute(+ "CREATE TABLE IF NOT EXISTS domain (+ domain TEXT NOT NULL UNIQUE,+ max_incoming_links INTEGER NOT NULL,+ status INTEGER NOT NULL,+ PRIMARY KEY (domain)+ );",+ [],+ )?;++ // index by domain and status+ conn.execute(+ "CREATE INDEX IF NOT EXISTS domain_domain_status ON domain (domain, status);",+ [],+ )?;++ // store redirects+ conn.execute(+ "CREATE TABLE IF NOT EXISTS redirect (+ from_url INTEGER NOT NULL,+ to_url INTEGER NOT NULL,+ PRIMARY KEY (from_url, to_url)+ );",+ [],+ )?;++ // update performance stuff+ // WAL mode+ conn.pragma_update(None, "journal_mode", "'WAL'")?;++ // sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.+ conn.pragma_update(None, "synchronous", 0)?;++ // store temp tables in memory+ conn.pragma_update(None, "temp_store", 2)?;++ // set cache size to 512 MB+ conn.pragma_update(None, "cache_size", -512_000)?; // negative value means kilobytes (https://www.sqlite.org/pragma.html#pragma_cache_size)++ Ok(Self { conn })+ }++ pub fn transaction(&self) -> Result<Transaction<'_>> {+ Ok(Transaction {+ tx: Some(self.conn.unchecked_transaction()?),+ })+ }+}++pub struct Transaction<'a> {+ tx: Option<rusqlite::Transaction<'a>>,+}++impl Drop for Transaction<'_> {+ fn drop(&mut self) {+ self.tx.take().unwrap().commit().unwrap();+ }+}++impl Transaction<'_> {+ fn tx(&self) -> &rusqlite::Transaction<'_> {+ self.tx.as_ref().unwrap()+ }++ pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {+ let mut prepared_insert_url = self.tx().prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, ?4)+ ON CONFLICT (url) DO NOTHING;",+ )?;+ let mut prepared_insert_domain = self.tx().prepare(+ "INSERT INTO domain (domain, max_incoming_links, status)+ VALUES (?1, 0, ?2)+ ON CONFLICT (domain) DO NOTHING;",+ )?;++ for url in urls {+ prepared_insert_url.execute((+ url.full(),+ url.domain().to_string(),+ UrlStatus::Pending,+ 0,+ ))?;+ prepared_insert_domain.execute((url.domain().to_string(), DomainStatus::Pending))?;+ }++ Ok(())+ }++ pub fn update_max_inlinks_domains<'a, I: Iterator<Item = &'a Domain>>(+ &self,+ domains: I,+ ) -> Result<()> {+ // update max_incoming_links for domain based on+ // max_incoming_links for URLs in domain.+ // if no pending URLs in domain, set max_incoming_links to 0+ let mut prepared = self.tx().prepare(+ "UPDATE domain SET max_incoming_links = IFNULL((+ SELECT MAX(incoming_links) FROM url+ WHERE domain = ?1 AND status = ?2+ ), 0) WHERE domain = ?1;",+ )?;++ for domain in domains {+ prepared.execute((&domain.0, UrlStatus::Pending))?;+ }++ Ok(())+ }++ pub fn insert_urls(&self, crawled_domain: &Domain, urls: &[Url]) -> Result<()> {+ // insert discovered URLs if not already in database+ // update incoming link counts for discovered URLs+ // if count is above prev max for domain, update max+ let domain = Url::from(crawled_domain.0.clone());+ let mut prepared_diff_domain = self.tx().prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, 1)+ ON CONFLICT (url) DO UPDATE SET incoming_links = incoming_links + 1;",+ )?;+ let mut prepared_same_domain = self.tx().prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, 0)+ ON CONFLICT (url) DO NOTHING;",+ )?;+ let mut prepared_insert_domain = self.tx().prepare(+ "INSERT INTO domain (domain, max_incoming_links, status)+ VALUES (?1, 0, ?2)+ ON CONFLICT (domain) DO NOTHING;",+ )?;++ for url in urls {+ if domain.domain() != url.domain() {+ prepared_diff_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;+ } else {+ prepared_same_domain.execute((url.full(), url.domain(), UrlStatus::Pending))?;+ }++ prepared_insert_domain.execute((url.domain(), DomainStatus::Pending))?;+ }+ Ok(())+ }++ pub fn update_url_status(&self, url_responses: &[UrlResponse]) -> Result<()> {+ let mut prepared_update_status_url = self+ .tx()+ .prepare("UPDATE url SET status = ?1, error_code = ?2 WHERE url = ?3;")?;++ let mut prepared_insert_redirect = self.tx().prepare(+ "INSERT INTO redirect (from_url, to_url)+ VALUES (?1, ?2)+ ON CONFLICT (from_url, to_url) DO NOTHING;",+ )?;++ let mut prepared_insert_url = self.tx().prepare(+ "INSERT INTO url (url, domain, status, incoming_links)+ VALUES (?1, ?2, ?3, 0)+ ON CONFLICT (url) DO NOTHING;",+ )?;++ // update status of URLs+ for url_res in url_responses {+ match url_res {+ UrlResponse::Success { url } => {+ prepared_update_status_url.execute((+ UrlStatus::Done,+ None::<u16>,+ url.full(),+ ))?;+ }+ UrlResponse::Failed { url, status_code } => {+ prepared_update_status_url.execute((+ UrlStatus::Failed,+ status_code,+ url.full(),+ ))?;+ }+ UrlResponse::Redirected { url, new_url } => {+ prepared_update_status_url.execute((+ UrlStatus::Done,+ None::<u16>,+ url.full(),+ ))?;+ prepared_insert_redirect.execute((url.full(), new_url.full()))?;+ prepared_insert_url.execute((+ new_url.full(),+ new_url.domain(),+ UrlStatus::Done,+ ))?;+ }+ }+ }++ Ok(())+ }++ pub fn set_domain_status(&self, domain: &Domain, status: DomainStatus) -> Result<()> {+ self.tx().execute(+ "UPDATE domain SET status = ?1 WHERE domain = ?2;",+ (status, &domain.0),+ )?;++ Ok(())+ }++ pub fn sample_domains(&self, num_jobs: usize) -> Result<Vec<Domain>> {+ // weighted sample from domains that are not currently being crawled+ // see https://www.kaggle.com/code/kotamori/random-sample-with-weights-on-sql/notebook for details on math+ let mut stmt = self.tx().prepare(+ "SELECT domain, -log((abs(random()) % 1000000 + 0.5) / 1000000.0) / (max_incoming_links + 1) as priority FROM domain+ WHERE status = ?1+ ORDER BY priority DESC+ LIMIT ?2;",+ )?;++ // dfs sample+ // let mut stmt = self.tx().prepare(+ // "SELECT domain FROM domain+ // WHERE status = ?1+ // LIMIT ?2;",+ // )?;++ let rows = stmt.query_map((DomainStatus::Pending, num_jobs), |row| {+ row.get::<_, String>(0)+ })?;++ let mut domains = Vec::new();+ for domain in rows {+ domains.push(Domain(domain?));+ }++ Ok(domains)+ }++ pub fn prepare_jobs(&self, domains: &[Domain], urls_per_job: usize) -> Result<Vec<Job>> {+ let mut jobs = Vec::new();+ // get URLs from sampled domains+ let mut stmt = self.tx().prepare(+ "SELECT url FROM url+ WHERE domain = ?1 AND status = ?2+ ORDER BY incoming_links DESC+ LIMIT ?3;",+ )?;++ let mut prepared_update_status_url = self+ .tx()+ .prepare("UPDATE url SET status = ?1 WHERE url = ?2;")?;+ let mut prepared_update_status_domain = self+ .tx()+ .prepare("UPDATE domain SET status = ?1 WHERE domain = ?2;")?;++ for domain in domains {+ let mut urls = Vec::new();++ let rows = stmt.query_map((&domain.0, UrlStatus::Pending, urls_per_job), |row| {+ row.get::<_, String>(0)+ })?;++ prepared_update_status_domain.execute((DomainStatus::CrawlInProgress, &domain.0))?;++ for url in rows {+ let url = url?;++ prepared_update_status_url.execute((UrlStatus::Crawling, url.clone()))?;++ urls.push(url.into());+ }++ jobs.push(Job {+ domain: domain.clone(),+ fetch_sitemap: true, // TODO: only fetch sitemap if we haven't already+ urls: urls.into(),+ });+ }++ Ok(jobs)+ }+}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex b7fb44b1..84319cf4 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -31,6 +31,7 @@ use self::{};mod coordinator;+mod crawl_db;mod robots_txt;mod warc_writer;mod worker;@@ -87,7 +88,7 @@ pub struct Job {}#[derive(Debug, serde::Serialize, serde::Deserialize)]-enum UrlResponse {+pub enum UrlResponse {Success { url: Url },Failed { url: Url, status_code: Option<u16> },Redirected { url: Url, new_url: Url },