Skip to content

web search done right

Number of domain inlinks not updated in crawldb.

Mikkel Denkerf040cca1abd3
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index cb30e38b..ecf47d17 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -78,6 +78,8 @@ impl CrawlCoordinator {
}
pub fn add_response(&self, response: &JobResponse) -> Result<()> {
+ let start = Instant::now();
+
self.log_crawls_per_second(response.url_responses.len());
self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);
let tx = self.db.transaction()?;
@@ -98,6 +100,8 @@ impl CrawlCoordinator {
tx.update_max_inlinks_domains(domains.iter())?;
+ tracing::debug!("inserted response in {:?}", start.elapsed());
+
Ok(())
}
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index afb0cdad..f7855ef5 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -174,6 +174,12 @@ impl CrawlDb {
// sync OFF - if coordinator crashes, we are SOL anyway and will have to restart the crawl.
let _ = conn.prepare("PRAGMA synchronous = OFF;")?.query([])?.next();
+ // lock held by the SQLite connection is never released in EXCLUSIVE mode
+ let _ = conn
+ .prepare("PRAGMA locking_mode = EXCLUSIVE;")?
+ .query([])?
+ .next();
+
// store temp tables in memory
let _ = conn.prepare("PRAGMA temp_store = 2;")?.query([])?.next();
@@ -236,6 +242,8 @@ impl Transaction<'_> {
&self,
mut domains: I,
) -> Result<()> {
+ // 32_784 is the maximum number of parameters that SQLite supports.
+ // This is configured in the SQLITE_MAX_VARIABLE_NUMBER environment variable in .cargo/config.toml.
for chunk in domains.by_ref().chunks(32_784).into_iter() {
self.update_max_inlinks_domains_chunk(chunk)?;
}
@@ -254,24 +262,25 @@ impl Transaction<'_> {
// Start transaction
let tx = self.tx();
+ tx.execute("DROP TABLE IF EXISTS temp_domain;", [])?;
// Create a temporary table to hold the max incoming links for each domain
tx.execute(
- "CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain AS
+ "CREATE TEMPORARY TABLE IF NOT EXISTS temp_domain AS
SELECT domain, MAX(incoming_links) as max_incoming_links
- FROM url
- WHERE domain IN (?1) AND status = ?2
+ FROM url
+ WHERE domain IN rarray(?1) AND status = ?2
GROUP BY domain;",
(domains.clone(), UrlStatus::Pending),
)?;
// Update the domain table
tx.execute(
- "UPDATE domain
+ "UPDATE domain
SET max_incoming_links = IFNULL(
(SELECT max_incoming_links FROM temp_domain WHERE temp_domain.domain = domain.domain), 0
- )
- WHERE domain IN (?1);",
- (domains, ),
+ )
+ WHERE domain in rarray(?1);",
+ (domains,),
)?;
// Drop the temporary table
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index 75bc6320..2b4a290d 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::net::SocketAddr;
+use std::{net::SocketAddr, sync::Arc};
use crate::{
crawler::{CrawlCoordinator, Crawler},
@@ -31,11 +31,11 @@ pub async fn worker(config: CrawlerConfig) -> Result<()> {
}
pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {
- let coordinator = CrawlCoordinator::new(
+ let coordinator = Arc::new(CrawlCoordinator::new(
config.crawldb_folder,
config.num_urls_to_crawl,
config.seed_urls,
- )?;
+ )?);
let addr: SocketAddr = config.host;
let server = sonic::Server::bind(addr).await.unwrap();