web search done right
Number of domain inlinks not updated in crawldb.
Mikkel Denkerf040cca1abd3
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex cb30e38b..ecf47d17 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -78,6 +78,8 @@ impl CrawlCoordinator {}pub fn add_response(&self, response: &JobResponse) -> Result<()> {+ let start = Instant::now();+self.log_crawls_per_second(response.url_responses.len());self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);let tx = self.db.transaction()?;@@ -98,6 +100,8 @@ impl CrawlCoordinator {tx.update_max_inlinks_domains(domains.iter())?;+ tracing::debug!("inserted response in {:?}", start.elapsed());+Ok(())}diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex afb0cdad..f7855ef5 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -174,6 +174,12 @@ impl CrawlDb {// sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.let _ = conn.prepare("PRAGMA synchronous = OFF;")?.query([])?.next();+ // lock held by the SQLite connection is never released in EXCLUSIVE mode+ let _ = conn+ .prepare("PRAGMA locking_mode = EXCLUSIVE;")?+ .query([])?+ .next();+// store temp tables in memorylet _ = conn.prepare("PRAGMA temp_store = 2;")?.query([])?.next();@@ -236,6 +242,8 @@ impl Transaction<'_> {&self,mut domains: I,) -> Result<()> {+ // 32_784 is the maximum number of parameters that SQLite supports.+ // This is configured in the SQLITE_MAX_VARIABLE_NUMBER environment variable in .cargo/config.toml.for chunk in domains.by_ref().chunks(32_784).into_iter() {self.update_max_inlinks_domains_chunk(chunk)?;}@@ -254,24 +262,25 @@ impl Transaction<'_> {// Start transactionlet tx = self.tx();+ tx.execute("DROP TABLE IF EXISTS temp_domain;", [])?;// Create a temporary table to hold the max incoming links for each domaintx.execute(- "CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain AS+ "CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain ASSELECT domain, MAX(incoming_links) as max_incoming_links- FROM url- WHERE domain IN (?1) AND status = ?2+ FROM url+ WHERE domain IN rarray(?1) AND status = ?2GROUP BY domain;",(domains.clone(), UrlStatus::Pending),)?;// Update the domain tabletx.execute(- "UPDATE domain+ "UPDATE domainSET max_incoming_links = IFNULL((SELECT max_incoming_links FROM temp_domain WHERE temp_domain.domain = domain.domain), 0- )- WHERE domain IN (?1);",- (domains, ),+ )+ WHERE domain in rarray(?1);",+ (domains,),)?;// Drop the temporary tablediff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex 75bc6320..2b4a290d 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::net::SocketAddr;+use std::{net::SocketAddr, sync::Arc};use crate::{crawler::{CrawlCoordinator, Crawler},@@ -31,11 +31,11 @@ pub async fn worker(config: CrawlerConfig) -> Result<()> {}pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {- let coordinator = CrawlCoordinator::new(+ let coordinator = Arc::new(CrawlCoordinator::new(config.crawldb_folder,config.num_urls_to_crawl,config.seed_urls,- )?;+ )?);let addr: SocketAddr = config.host;let server = sonic::Server::bind(addr).await.unwrap();