web search done right
live_index construction
Mikkel Denker869382528366
diff --git a/Cargo.toml b/Cargo.tomlindex 7563f9fb..ecc9f3ec 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -85,7 +85,7 @@ rusqlite = {version = "0.29.0", features = [rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}rust-stemmers = "1.2.0"safetensors = "0.3.1"-scylla = { git = "https://github.com/scylladb/scylla-rust-driver", rev = "82c1c99f0ff86509f9dd1e649ecdaddc5a3660cf" }+scylla = {git = "https://github.com/scylladb/scylla-rust-driver", rev = "82c1c99f0ff86509f9dd1e649ecdaddc5a3660cf"}serde = {version = "1.0.137", features = ["rc", "derive"]}serde_json = "1.0.81"serde_urlencoded = "0.7.1"diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 4beb5b63..88366ec2 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -17,6 +17,7 @@pub mod defaults;use super::Result;+use crate::feed::scheduler::SplitId;use crate::searcher::ShardId;use serde::{Deserialize, Serialize};use std::fs::File;@@ -380,3 +381,58 @@ pub struct CrawlPlannerConfig {pub top_host_fraction: f64,pub wander_fraction: f64,}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct LiveIndexConfig {+ pub split_path: String,+ pub downloaded_db_path: String,++ // crawler+ pub user_agent: UserAgent,+ #[serde(default = "defaults::Crawler::robots_txt_cache_sec")]+ pub robots_txt_cache_sec: u64,+ #[serde(default = "defaults::Crawler::politeness_factor")]+ pub politeness_factor: f32,+ #[serde(default = "defaults::Crawler::min_crawl_delay_ms")]+ pub min_crawl_delay_ms: u64,+ #[serde(default = "defaults::Crawler::max_crawl_delay_ms")]+ pub max_crawl_delay_ms: u64,+ #[serde(default = "defaults::Crawler::max_politeness_factor")]+ pub max_politeness_factor: f32,+ #[serde(default = "defaults::Crawler::max_url_slowdown_retry")]+ pub max_url_slowdown_retry: u8,+ #[serde(default = "defaults::Crawler::max_redirects")]+ pub max_redirects: usize,+ pub timeout_seconds: u64,++ // indexer+ pub host_centrality_store_path: String,+ pub page_webgraph_path: Option<String>,+ pub page_centrality_store_path: Option<String>,+ pub safety_classifier_path: Option<String>,+ pub host_centrality_threshold: Option<f64>,+ pub minimum_clean_words: Option<usize>,++ // search+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+ pub split_id: SplitId,+ pub index_path: String,+ pub linear_model_path: Option<String>,+ pub lambda_model_path: Option<String>,+ pub host: SocketAddr,+ #[serde(default)]+ pub collector: CollectorConfig,+ #[serde(default)]+ pub snippet: SnippetConfig,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct LiveIndexSchedulerConfig {+ pub schedule_path: String,+ pub feed_index_path: String,+ pub host_centrality_store_path: String,+ pub host_graph_path: String,+ pub num_splits: u64,+}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex aae8416b..00e74954 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -23,6 +23,7 @@ use url::Url;use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};use self::{warc_writer::WarcWriter, worker::WorkerThread};+pub use worker::JobExecutor;pub mod coordinator;mod robots_txt;@@ -175,7 +176,7 @@ pub struct DomainCrawled {pub budget_used: f64,}-struct RetrieableUrl {+pub struct RetrieableUrl {url: Url,retries: u8,}@@ -186,10 +187,10 @@ impl From<Url> for RetrieableUrl {}}-struct WorkerJob {+pub struct WorkerJob {pub domain: Domain,pub urls: VecDeque<RetrieableUrl>,- pub wanrdering_urls: u64,+ pub wandering_urls: u64,}impl From<Job> for WorkerJob {@@ -197,18 +198,18 @@ impl From<Job> for WorkerJob {Self {domain: value.domain,urls: value.urls.into_iter().map(RetrieableUrl::from).collect(),- wanrdering_urls: value.wandering_urls,+ wandering_urls: value.wandering_urls,}}}#[derive(Debug, Clone)]pub struct CrawlDatum {- url: Url,- status_code: u16,- payload_type: warc::PayloadType,- body: String,- fetch_time_ms: u64,+ pub url: Url,+ pub status_code: u16,+ pub payload_type: warc::PayloadType,+ pub body: String,+ pub fetch_time_ms: u64,}pub struct Crawler {@@ -219,7 +220,6 @@ pub struct Crawler {impl Crawler {pub async fn new(config: CrawlerConfig) -> Result<Self> {let writer = Arc::new(WarcWriter::new(config.s3.clone()));- let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();let mut router_hosts = Vec::new();@@ -228,12 +228,8 @@ impl Crawler {}for _ in 0..config.num_worker_threads {- let worker = WorkerThread::new(- Arc::clone(&writer),- config.clone(),- timeout,- router_hosts.clone(),- )?;+ let worker =+ WorkerThread::new(Arc::clone(&writer), config.clone(), router_hosts.clone())?;handles.push(tokio::spawn(async move {worker.run().await;@@ -251,3 +247,32 @@ impl Crawler {self.writer.finish().await.unwrap();}}++#[async_trait::async_trait]+pub trait DatumStream: Send + Sync {+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()>;+ async fn finish(&self) -> Result<()>;+}++pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {+ let timeout = Duration::from_secs(config.timeout_seconds);++ let mut headers = reqwest::header::HeaderMap::default();+ headers.insert(+ reqwest::header::ACCEPT,+ reqwest::header::HeaderValue::from_static("text/html"),+ );+ headers.insert(+ reqwest::header::ACCEPT_LANGUAGE,+ reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),+ );++ Ok(reqwest::Client::builder()+ .timeout(timeout)+ .connect_timeout(timeout)+ .http2_keep_alive_interval(None)+ .default_headers(headers)+ .redirect(reqwest::redirect::Policy::limited(config.max_redirects))+ .user_agent(&config.user_agent.full)+ .build()?)+}diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex a2b06e54..078d41d0 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -21,7 +21,7 @@ use crate::{warc,};-use super::{CrawlDatum, Result};+use super::{CrawlDatum, DatumStream, Result};/// The WarcWriter is responsible for storing the crawl datums/// as WARC files on S3.@@ -29,6 +29,23 @@ pub struct WarcWriter {tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,}+#[async_trait::async_trait]+impl DatumStream for WarcWriter {+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {+ self.tx+ .blocking_send(WarcWriterMessage::Crawl(crawl_datum))?;++ Ok(())+ }++ async fn finish(&self) -> Result<()> {+ self.tx.blocking_send(WarcWriterMessage::Finish)?;+ self.tx.closed().await;++ Ok(())+ }+}+#[derive(Debug, Clone)]pub enum WarcWriterMessage {Crawl(CrawlDatum),@@ -97,7 +114,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:payload_type: Some(datum.payload_type),},metadata: warc::Metadata {- fetch_time_ms: datum.fetch_time_ms as usize,+ fetch_time_ms: datum.fetch_time_ms,},};@@ -131,17 +148,4 @@ impl WarcWriter {Self { tx }}-- pub async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {- self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;-- Ok(())- }-- pub async fn finish(&self) -> Result<()> {- self.tx.send(WarcWriterMessage::Finish).await?;- self.tx.closed().await;-- Ok(())- }}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 0e67ad9c..9e30e40b 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -40,8 +40,8 @@ use crate::{};use super::{- robots_txt::RobotsTxtManager, site_graph::SiteGraph, CrawlDatum, Domain, Error, Result,- RetrieableUrl, Site, UrlResponse, WarcWriter, WorkerJob,+ reqwest_client, robots_txt::RobotsTxtManager, site_graph::SiteGraph, CrawlDatum, DatumStream,+ Domain, Error, Result, RetrieableUrl, Site, UrlResponse, WarcWriter, WorkerJob,};const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB@@ -63,27 +63,9 @@ impl WorkerThread {pub fn new(writer: Arc<WarcWriter>,config: CrawlerConfig,- timeout: Duration,router_hosts: Vec<SocketAddr>,) -> Result<Self> {- let mut headers = reqwest::header::HeaderMap::default();- headers.insert(- reqwest::header::ACCEPT,- reqwest::header::HeaderValue::from_static("text/html"),- );- headers.insert(- reqwest::header::ACCEPT_LANGUAGE,- reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),- );-- let client = reqwest::Client::builder()- .timeout(timeout)- .connect_timeout(timeout)- .http2_keep_alive_interval(None)- .default_headers(headers)- .redirect(reqwest::redirect::Policy::limited(config.max_redirects))- .user_agent(&config.user_agent.full)- .build()?;+ let client = reqwest_client(&config)?;Ok(Self {writer,@@ -135,8 +117,8 @@ impl WorkerThread {}}-struct JobExecutor {- writer: Arc<WarcWriter>,+pub struct JobExecutor<S: DatumStream> {+ writer: Arc<S>,client: reqwest::Client,politeness_factor: f32,robotstxt: RobotsTxtManager,@@ -148,12 +130,12 @@ struct JobExecutor {job: WorkerJob,}-impl JobExecutor {- fn new(+impl<S: DatumStream> JobExecutor<S> {+ pub fn new(job: WorkerJob,client: reqwest::Client,config: Arc<CrawlerConfig>,- writer: Arc<WarcWriter>,+ writer: Arc<S>,) -> Self {Self {writer,@@ -171,11 +153,15 @@ impl JobExecutor {job,}}- async fn run(mut self) {++ pub async fn run(mut self) {tracing::info!("Processing job: {:?}", self.job.domain);self.scheduled_urls().await;- self.wander().await;++ if self.job.wandering_urls > 0 {+ self.wander().await;+ }}async fn scheduled_urls(&mut self) {@@ -207,7 +193,7 @@ impl JobExecutor {let urls = urls.into_iter()- .take(self.job.wanrdering_urls as usize)+ .take(self.job.wandering_urls as usize).map(|(url, _)| url).map(RetrieableUrl::from).collect();diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 66593336..5f2313db 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -19,6 +19,7 @@ use tokio::io;use tokio_stream::StreamExt;use tracing::{debug, info};+use crate::entrypoint::indexer::JobSettings;use crate::entrypoint::{dmoz_parser, indexer};use crate::Result;use std::fs::{self};@@ -142,8 +143,10 @@ fn create_inverted_index() -> Result<()> {}),warc_paths: vec![warc_path.to_str().unwrap().to_string()],base_path: out_path_tmp.to_str().unwrap().to_string(),- host_centrality_threshold: None,- minimum_clean_words: None,+ settings: JobSettings {+ host_centrality_threshold: None,+ minimum_clean_words: None,+ },};let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex feb7b224..b1bfeda8 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -13,6 +13,7 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use anyhow::anyhow;use chrono::Utc;use rayon::prelude::*;use std::path::Path;@@ -69,6 +70,11 @@ pub struct Job {pub source_config: JobConfig,pub warc_paths: Vec<String>,pub base_path: String,+ pub settings: JobSettings,+}++#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+pub struct JobSettings {pub host_centrality_threshold: Option<f64>,pub minimum_clean_words: Option<usize>,}@@ -76,16 +82,17 @@ pub struct Job {pub struct IndexingWorker {host_centrality_store: RocksDbStore<NodeID, f64>,page_centrality_store: Option<RocksDbStore<NodeID, f64>>,- webgraph: Option<Webgraph>,+ page_webgraph: Option<Webgraph>,topics: Option<human_website_annotations::Mapper>,safety_classifier: Option<safety_classifier::Model>,+ job_settings: Option<JobSettings>,}impl IndexingWorker {pub fn new(host_centrality_store_path: String,page_centrality_store_path: Option<String>,- webgraph_path: Option<String>,+ page_webgraph_path: Option<String>,topics_path: Option<String>,safety_classifier_path: Option<String>,) -> Self {@@ -95,173 +102,182 @@ impl IndexingWorker {),page_centrality_store: page_centrality_store_path.map(|p| RocksDbStore::open(Path::new(&p).join("derived_harmonic"))),- webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),+ page_webgraph: page_webgraph_path+ .map(|path| WebgraphBuilder::new(path).single_threaded().open()),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),safety_classifier: safety_classifier_path.map(|path| safety_classifier::Model::open(path).unwrap()),+ job_settings: None,}}-}--pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {- let name = job.warc_paths.first().unwrap().split('/').last().unwrap();- let mut has_host_centrality = false;- let mut has_page_centrality = false;- let mut has_backlinks = false;+ pub fn set_job_settings(&mut self, job_settings: JobSettings) {+ self.job_settings = Some(job_settings);+ }- info!("processing {}", name);+ pub fn prepare_webpage(&self, body: &str, url: &str, fetch_time_ms: u64) -> Result<Webpage> {+ let mut html = match Html::parse_without_text(body, url) {+ Ok(html) => html,+ Err(err) => {+ debug!("error parsing html: {:?}", err);+ return Err(anyhow!("error parsing html: {:?}", err));+ }+ };- let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();+ if html.is_no_index() {+ return Err(anyhow!("noindex"));+ }- let source: config::WarcSource = job.source_config.clone().into();+ let title = html.title().unwrap_or_default();+ if title.is_empty() || title.chars().all(|c| c.is_whitespace()) {+ return Err(anyhow!("empty title"));+ }- let warc_files = download_all_warc_files(&job.warc_paths, &source);- pin!(warc_files);+ let node = Node::from(html.url());+ let host_node_id = node.clone().into_host().id();- let current_timestamp = Utc::now().timestamp().max(0) as usize;+ let mut host_centrality = self+ .host_centrality_store+ .get(&host_node_id)+ .unwrap_or_default();- for file in warc_files.by_ref() {- for record in- file.records()- .flatten()- .filter(|record| match &record.response.payload_type {- Some(payload_type) => matches!(payload_type, PayloadType::Html),- None => false,- })+ if let Some(host_centrality_threshold) =+ self.job_settings.and_then(|s| s.host_centrality_threshold){- let mut html =- match Html::parse_without_text(&record.response.body, &record.request.url) {- Ok(html) => html,- Err(err) => {- debug!("error parsing html: {:?}", err);- continue;- }- };-- if html.is_no_index() {- continue;- }-- let title = html.title().unwrap_or_default();- if title.is_empty() || title.chars().all(|c| c.is_whitespace()) {- continue;+ if host_centrality < host_centrality_threshold {+ debug!("skipping due to low host_centrality value");+ return Err(anyhow!("low host_centrality value"));}+ }- let node = Node::from(html.url());- let host_node_id = node.clone().into_host().id();+ html.parse_text();- let mut host_centrality = worker- .host_centrality_store- .get(&host_node_id)- .unwrap_or_default();+ if html.empty_all_text() {+ return Err(anyhow!("empty all text"));+ }- if let Some(host_centrality_threshold) = job.host_centrality_threshold {- if host_centrality < host_centrality_threshold {- debug!("skipping due to low host_centrality value");- continue;+ if let Some(minimum_clean_words) = self.job_settings.and_then(|s| s.minimum_clean_words) {+ match html.clean_text() {+ Some(clean_text) => {+ if clean_text.split_whitespace().count() < minimum_clean_words {+ return Err(anyhow!("too few clean words"));+ }}+ None => return Err(anyhow!("no clean text")),}+ }- html.parse_text();+ let backlink_labels: Vec<String> = self+ .page_webgraph+ .as_ref()+ .map(|webgraph| {+ webgraph+ .raw_ingoing_edges_with_labels(&Node::from(html.url()).id())+ .into_iter()+ .map(|edge| edge.label)+ .filter(|label| !label.is_empty())+ .collect()+ })+ .unwrap_or_default();- if html.empty_all_text() {- continue;- }+ let mut page_centrality = 0.0;- if let Some(minimum_clean_words) = job.minimum_clean_words {- match html.clean_text() {- Some(clean_text) => {- if clean_text.split_whitespace().count() < minimum_clean_words {- continue;- }- }- None => continue,- }- }+ if let Some(store) = self.page_centrality_store.as_ref() {+ let node_id = node.id();- let backlink_labels: Vec<String> = worker- .webgraph- .as_ref()- .map(|webgraph| {- webgraph- .raw_ingoing_edges_with_labels(&Node::from(html.url()).id())- .into_iter()- .map(|edge| edge.label)- .filter(|label| !label.is_empty())- .collect()- })- .unwrap_or_default();+ page_centrality = store.get(&node_id).unwrap_or_default();+ }- let mut page_centrality = 0.0;+ if !page_centrality.is_finite() {+ page_centrality = 0.0;+ }- if let Some(store) = worker.page_centrality_store.as_ref() {- let node_id = node.id();+ if !host_centrality.is_finite() {+ host_centrality = 0.0;+ }- page_centrality = store.get(&node_id).unwrap_or_default();- }+ let mut dmoz_description = None;- if host_centrality > 0.0 {- has_host_centrality = true;+ if let Some(mapper) = self.topics.as_ref() {+ if let Some(info) = mapper.get(&html.url().host_str().unwrap_or_default().to_string()) {+ dmoz_description = Some(info.description.clone())}+ }- if page_centrality > 0.0 {- has_page_centrality = true;- }+ let mut webpage = Webpage {+ html,+ backlink_labels,+ page_centrality,+ host_centrality,+ fetch_time_ms,+ pre_computed_score: 0.0,+ node_id: Some(host_node_id),+ dmoz_description,+ safety_classification: None,+ };++ if let Some(model) = self.safety_classifier.as_ref() {+ webpage.safety_classification = Some(model.predict(&webpage).label);+ }- if !backlink_labels.is_empty() {- has_backlinks = true;- }+ let mut signal_aggregator = SignalAggregator::new(None);+ signal_aggregator.set_current_timestamp(Utc::now().timestamp().max(0) as usize);- if !page_centrality.is_finite() {- page_centrality = 0.0;- }+ webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);- if !host_centrality.is_finite() {- host_centrality = 0.0;- }+ Ok(webpage)+ }+}- let fetch_time_ms = record.metadata.fetch_time_ms as u64;+pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {+ let name = job.warc_paths.first().unwrap().split('/').last().unwrap();- trace!("inserting webpage: {:?}", html.url());+ let mut has_host_centrality = false;+ let mut has_page_centrality = false;+ let mut has_backlinks = false;- trace!("title = {:?}", html.title());- trace!("text = {:?}", html.clean_text());+ info!("processing {}", name);- let mut dmoz_description = None;+ let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();- if let Some(mapper) = worker.topics.as_ref() {- if let Some(info) =- mapper.get(&html.url().host_str().unwrap_or_default().to_string())- {- dmoz_description = Some(info.description.clone())- }- }+ let source: config::WarcSource = job.source_config.clone().into();- let mut webpage = Webpage {- html,- backlink_labels,- page_centrality,- host_centrality,- fetch_time_ms,- pre_computed_score: 0.0,- node_id: Some(host_node_id),- dmoz_description,- safety_classification: None,- };-- if let Some(model) = worker.safety_classifier.as_ref() {- webpage.safety_classification = Some(model.predict(&webpage).label);- }+ let warc_files = download_all_warc_files(&job.warc_paths, &source);+ pin!(warc_files);- let mut signal_aggregator = SignalAggregator::new(None);- signal_aggregator.set_current_timestamp(current_timestamp);+ for file in warc_files.by_ref() {+ for record in+ file.records()+ .flatten()+ .filter(|record| match &record.response.payload_type {+ Some(payload_type) => matches!(payload_type, PayloadType::Html),+ None => false,+ })+ {+ if let Ok(webpage) = worker.prepare_webpage(+ &record.response.body,+ &record.request.url,+ record.metadata.fetch_time_ms,+ ) {+ if webpage.host_centrality > 0.0 {+ has_host_centrality = true;+ }- webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);+ if webpage.page_centrality > 0.0 {+ has_page_centrality = true;+ }++ if !webpage.backlink_labels.is_empty() {+ has_backlinks = true;+ }+ trace!("inserting webpage: {:?}", webpage.html.url());+ trace!("title = {:?}", webpage.html.title());+ trace!("text = {:?}", webpage.html.clean_text());- if let Err(err) = index.insert(webpage) {- warn!("{:?}", err);- panic!();+ if let Err(err) = index.insert(webpage) {+ warn!("{:?}", err);+ panic!();+ }}}@@ -276,7 +292,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {warn!("no page centrality values found in {}", name);}- if !has_backlinks && worker.webgraph.is_some() {+ if !has_backlinks && worker.page_webgraph.is_some() {warn!("no backlinks found in {}", name);}@@ -365,12 +381,14 @@ impl Indexer {.map(|warc_paths| Job {source_config: job_config.clone(),warc_paths,- host_centrality_threshold: config.host_centrality_threshold,base_path: config.output_path.clone().unwrap_or_else(|| "data/index".to_string()),- minimum_clean_words: config.minimum_clean_words,+ settings: JobSettings {+ host_centrality_threshold: config.host_centrality_threshold,+ minimum_clean_words: config.minimum_clean_words,+ },}).map(|job| {let pointer: IndexPointer = job.map(&worker);diff --git a/core/src/entrypoint/live_index.rs b/core/src/entrypoint/live_index.rsnew file mode 100644index 00000000..3e19bf26--- /dev/null+++ b/core/src/entrypoint/live_index.rs@@ -0,0 +1,46 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;++use crate::{+ config::{LiveIndexConfig, LiveIndexSchedulerConfig},+ feed::{self, index::FeedIndex},+ kv::rocksdb_store::RocksDbStore,+ live_index::IndexManager,+ webgraph::WebgraphBuilder,+};+use anyhow::Result;++pub async fn serve(config: LiveIndexConfig) -> Result<()> {+ let manager = IndexManager::new(config)?;+ tokio::task::spawn(manager.run());++ todo!("accept search requests");+}++pub fn schedule(config: LiveIndexSchedulerConfig) -> Result<()> {+ let feed_index = FeedIndex::open(config.feed_index_path)?;+ let host_harmonic =+ RocksDbStore::open(Path::new(&config.host_centrality_store_path).join("harmonic"));+ let host_graph = WebgraphBuilder::new(config.host_graph_path).open();++ let schedule =+ feed::scheduler::schedule(&feed_index, &host_harmonic, &host_graph, config.num_splits);+ schedule.save(config.schedule_path)?;++ Ok(())+}diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 230da23a..a307e6ff 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -36,6 +36,7 @@ pub use entity::EntityIndexer;pub use indexer::Indexer;use tracing::{debug, log::error};pub use webgraph::Webgraph;+pub mod live_index;use crate::{config, warc::WarcFile};diff --git a/core/src/feed/mod.rs b/core/src/feed/mod.rsindex 284c2451..1f9ebf9b 100644--- a/core/src/feed/mod.rs+++ b/core/src/feed/mod.rs@@ -30,8 +30,8 @@ pub enum FeedKind {#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub struct Feed {- url: Url,- kind: FeedKind,+ pub url: Url,+ pub kind: FeedKind,}pub struct ParsedFeed {diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rsindex 1d0795f3..0f876704 100644--- a/core/src/feed/scheduler.rs+++ b/core/src/feed/scheduler.rs@@ -14,6 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::path::Path;++use anyhow::Result;use hashbrown::{HashMap, HashSet};use url::Url;@@ -28,21 +31,71 @@ use crate::{use super::{index::FeedIndex, Feed};-#[derive(Debug, Clone, PartialEq, Eq, Hash)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub struct Domain(String);+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct DomainFeeds {- domain: Domain,- feeds: Vec<Feed>,+ pub domain: Domain,+ pub feeds: Vec<Feed>,}+impl From<&Url> for Domain {+ fn from(url: &Url) -> Self {+ Domain(url.icann_domain().unwrap_or_default().to_string())+ }+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct SplitId(uuid::Uuid);++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct Split {- feeds: Vec<DomainFeeds>,+ pub id: SplitId,+ pub feeds: Vec<DomainFeeds>,+}++impl Split {+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {+ let file = std::fs::OpenOptions::new()+ .write(true)+ .create(true)+ .truncate(true)+ .open(path.as_ref())?;+ let writer = std::io::BufWriter::new(file);++ serde_json::to_writer_pretty(writer, &self)+ .map_err(|e| anyhow::anyhow!("Failed to serialize split: {}", e))?;++ Ok(())+ }++ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let file = std::fs::OpenOptions::new().read(true).open(path.as_ref())?;+ let reader = std::io::BufReader::new(file);++ serde_json::from_reader(reader)+ .map_err(|e| anyhow::anyhow!("Failed to deserialize split: {}", e))+ }}pub struct Schedule {splits: Vec<Split>,}+impl Schedule {+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {+ if !path.as_ref().exists() {+ std::fs::create_dir_all(&path)?;+ }++ for split in self.splits {+ let name = split.id.0.to_string() + ".json";+ split.save(path.as_ref().join(name))?;+ }++ Ok(())+ }+}pub fn schedule(index: &FeedIndex,@@ -63,7 +116,7 @@ pub fn schedule(}let url = url.unwrap();- let domain = Domain(url.icann_domain().unwrap().to_string());+ let domain = Domain::from(&url);all_feeds.entry(domain)@@ -74,7 +127,10 @@ pub fn schedule(let mut splits = Vec::new();for _ in 0..num_splits {- splits.push(Split { feeds: Vec::new() });+ splits.push(Split {+ id: SplitId(uuid::Uuid::new_v4()),+ feeds: Vec::new(),+ });}for (i, (domain, feeds)) in all_feeds.into_iter().enumerate() {diff --git a/core/src/index.rs b/core/src/index.rsindex 8da18312..51572977 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -17,7 +17,8 @@use std::collections::HashSet;use std::fs;use std::path::Path;-use std::sync::Arc;+use std::sync::{Arc, Mutex};+use std::time::SystemTime;use serde::{Deserialize, Serialize};use tantivy::schema::Schema;@@ -29,7 +30,6 @@ use crate::directory::{self, DirEntry};use crate::inverted_index::{self, InvertedIndex};use crate::query::Query;use crate::search_ctx::Ctx;-use crate::subdomain_count::SubdomainCounter;use crate::webgraph::NodeID;use crate::webpage::region::{Region, RegionCount};use crate::webpage::Webpage;@@ -37,12 +37,10 @@ use crate::Result;const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";const REGION_COUNT_FILE_NAME: &str = "region_count.json";-const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";pub struct Index {pub inverted_index: InvertedIndex,- pub region_count: RegionCount,- pub subdomain_counter: SubdomainCounter,+ pub region_count: Mutex<RegionCount>,pub path: String,}@@ -59,10 +57,7 @@ impl Index {Ok(Self {inverted_index,- region_count,- subdomain_counter: SubdomainCounter::open(- path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),- ),+ region_count: Mutex::new(region_count),path: path.as_ref().to_str().unwrap().to_string(),})}@@ -73,6 +68,10 @@ impl Index {Ok(())}+ pub fn set_auto_merge_policy(&mut self) {+ self.inverted_index.set_auto_merge_policy();+ }+pub fn tokenizers(&self) -> &TokenizerManager {self.inverted_index.tokenizers()}@@ -83,20 +82,26 @@ impl Index {Self::open(path)}- pub fn insert(&mut self, webpage: Webpage) -> Result<()> {- self.subdomain_counter.increment(webpage.html.url().clone());-+ pub fn insert(&self, webpage: Webpage) -> Result<()> {if let Ok(region) = Region::guess_from(&webpage) {- self.region_count.increment(®ion);+ let mut reg = self.region_count.lock().unwrap_or_else(|e| e.into_inner());+ reg.increment(®ion);}self.inverted_index.insert(webpage)}+ pub fn delete_all_before(&self, timestamp: SystemTime) -> Result<()> {+ self.inverted_index+ .delete_all_before(tantivy::DateTime::from_utc(timestamp.into()))+ }+pub fn commit(&mut self) -> Result<()> {self.inverted_index.commit()?;- self.region_count.commit();- self.subdomain_counter.commit();++ let mut reg = self.region_count.lock().unwrap_or_else(|e| e.into_inner());+ reg.commit();+Ok(())}@@ -128,13 +133,19 @@ impl Index {self.inverted_index.retrieve_websites(websites, query)}- pub fn merge(mut self, other: Self) -> Self {+ pub fn merge(self, other: Self) -> Self {self.inverted_index.merge(other.inverted_index);- self.region_count.merge(other.region_count);+ let mut self_region_count = self+ .region_count+ .into_inner()+ .unwrap_or_else(|e| e.into_inner());+ let other_region_count = other+ .region_count+ .into_inner()+ .unwrap_or_else(|e| e.into_inner());- self.subdomain_counter.merge(other.subdomain_counter);- drop(self.subdomain_counter);+ self_region_count.merge(other_region_count);Self::open(&self.path).expect("failed to open index")}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex f0e9aa33..e765b287 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -188,6 +188,11 @@ impl InvertedIndex {self.snippet_config = config;}+ pub fn set_auto_merge_policy(&mut self) {+ let merge_policy = tantivy::merge_policy::LogMergePolicy::default();+ self.writer.set_merge_policy(Box::new(merge_policy));+ }+pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {FastFieldReader::new(tv_searcher)}@@ -215,6 +220,24 @@ impl InvertedIndex {Ok(())}+ fn delete(&self, query: Box<dyn tantivy::query::Query>) -> Result<()> {+ self.writer.delete_query(query)?;++ Ok(())+ }++ pub fn delete_all_before(&self, timestamp: tantivy::DateTime) -> Result<()> {+ let query = tantivy::query::RangeQuery::new_date_bounds(+ Field::Text(TextField::InsertionTimestamp)+ .name()+ .to_string(),+ std::ops::Bound::Unbounded,+ std::ops::Bound::Excluded(timestamp),+ );++ self.delete(Box::new(query))+ }+pub fn search_initial(&self,query: &Query,diff --git a/core/src/lib.rs b/core/src/lib.rsindex 7069c4ca..395e6ad8 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -57,6 +57,7 @@ mod intmap;mod kahan_sum;mod kv;mod leaky_queue;+mod live_index;#[cfg(feature = "libtorch")]mod llm_utils;mod metrics;@@ -74,7 +75,6 @@ mod simhash;pub mod similar_sites;mod snippet;pub mod spell;-mod subdomain_count;#[cfg(feature = "libtorch")]pub mod summarizer;mod tokenizer;diff --git a/core/src/live_index.rs b/core/src/live_index.rsnew file mode 100644index 00000000..f66169e2--- /dev/null+++ b/core/src/live_index.rs@@ -0,0 +1,367 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::Result;+use std::{+ path::Path,+ sync::{Arc, RwLock},+ time::{Duration, SystemTime},+};++use chrono::{DateTime, Utc};+use url::Url;++use crate::{+ config::{CrawlerConfig, LiveIndexConfig},+ crawler::{reqwest_client, CrawlDatum, DatumStream, JobExecutor, RetrieableUrl, WorkerJob},+ entrypoint::indexer::IndexingWorker,+ feed::{+ self,+ scheduler::{Domain, DomainFeeds, Split},+ },+};++const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 days+const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour+const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes+const AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutes+const EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);++#[derive(Debug, Clone)]+struct Feeds {+ last_checked: DateTime<Utc>,+ feed: DomainFeeds,+}++impl From<Split> for Vec<Feeds> {+ fn from(split: Split) -> Self {+ split+ .feeds+ .into_iter()+ .map(|feed| Feeds {+ last_checked: Utc::now(),+ feed,+ })+ .collect()+ }+}++struct DownloadedDb {+ db: rocksdb::DB,+}++impl DownloadedDb {+ fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = rocksdb::BlockBasedOptions::default();+ block_options.set_block_size(16 * 1024);+ block_options.set_format_version(5);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);++ block_options.set_ribbon_filter(10.0);++ options.set_block_based_table_factory(&block_options);+ options.optimize_for_point_lookup(512); // 512 mb++ let db = rocksdb::DB::open_with_ttl(&options, path, TTL)?;+ Ok(Self { db })+ }++ fn has_downloaded(&self, url: &Url) -> Result<bool> {+ let key = url.as_str().as_bytes();++ if self.db.key_may_exist(key) {+ self.db.get(key).map(|v| v.is_some()).map_err(Into::into)+ } else {+ Ok(false)+ }+ }++ fn insert(&self, url: &Url) -> Result<()> {+ let key = url.as_str().as_bytes();+ self.db.put(key, b"")?;+ Ok(())+ }+}++struct Indexer {+ search_index: Arc<RwLock<crate::index::Index>>,+ worker: IndexingWorker,+}++enum CrawlResults {+ None,+ HasInserts,+}++struct Crawler {+ feeds: Vec<Feeds>,+ indexer: Arc<Indexer>,+ downloaded_db: DownloadedDb,+ config: Arc<CrawlerConfig>,+ client: reqwest::Client,+}++impl Crawler {+ fn new(+ split: Split,+ indexer: Arc<Indexer>,+ downloaded_db: DownloadedDb,+ config: Arc<CrawlerConfig>,+ ) -> Result<Self> {+ let client = reqwest_client(&config)?;++ Ok(Self {+ feeds: split.into(),+ indexer,+ downloaded_db,+ config,+ client,+ })+ }++ async fn process_urls(&self, urls: Vec<Url>) -> Result<bool> {+ if urls.is_empty() {+ return Ok(false);+ }++ let domain = urls.first().unwrap().into();+ let job = WorkerJob {+ domain,+ urls: urls.clone().into_iter().map(RetrieableUrl::from).collect(),+ wandering_urls: 0,+ };++ let executor = JobExecutor::new(+ job,+ self.client.clone(),+ self.config.clone(),+ self.indexer.clone(),+ );+ executor.run().await;++ for url in &urls {+ self.downloaded_db.insert(url)?;+ }++ Ok(true)+ }++ async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {+ let mut urls = Vec::new();++ for feed in domain_feeds.feeds.iter() {+ let content = self+ .client+ .get(feed.url.as_str())+ .send()+ .await?+ .text()+ .await?;++ let feed = feed::parse(&content, feed.kind)?;++ for url in feed.links {+ if !self.downloaded_db.has_downloaded(&url)?+ && Domain::from(&url) == domain_feeds.domain+ {+ urls.push(url);+ }+ }+ }++ self.process_urls(urls).await+ }++ async fn check_feeds(&mut self) -> CrawlResults {+ let mut feeds = self.feeds.clone();++ let mut futures = Vec::new();+ for feeds in feeds.iter_mut() {+ if feeds.last_checked + FEED_CHECK_INTERVAL < Utc::now() {+ futures.push(self.process_feed(&feeds.feed));+ feeds.last_checked = Utc::now();+ }+ }++ let res = futures::future::join_all(futures).await;++ self.feeds = feeds;++ if res.iter().filter_map(|r| r.as_ref().ok()).any(|r| *r) {+ CrawlResults::HasInserts+ } else {+ CrawlResults::None+ }+ }+}++struct Index {+ search_index: Arc<RwLock<crate::index::Index>>,+}++impl Index {+ pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {+ let mut search_index = crate::index::Index::open(path.as_ref().join("index"))?;+ search_index.set_auto_merge_policy();++ let search_index = Arc::new(RwLock::new(search_index));++ Ok(Self { search_index })+ }++ fn commit(&self) {+ self.search_index+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .commit()+ .ok();+ }++ fn prune(&self) {+ self.search_index+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .delete_all_before(SystemTime::now() - TTL)+ .ok();+ }++ fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {+ self.search_index.clone()+ }+}++#[async_trait::async_trait]+impl DatumStream for Indexer {+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {+ let webpage = self.worker.prepare_webpage(+ &crawl_datum.body,+ crawl_datum.url.as_str(),+ crawl_datum.fetch_time_ms,+ )?;++ self.search_index+ .read()+ .unwrap_or_else(|e| e.into_inner())+ .insert(webpage)?;++ Ok(())+ }++ async fn finish(&self) -> Result<()> {+ self.search_index+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .commit()?;++ Ok(())+ }+}++impl From<&LiveIndexConfig> for CrawlerConfig {+ fn from(live: &LiveIndexConfig) -> Self {+ Self {+ num_worker_threads: 1, // no impact+ user_agent: live.user_agent.clone(),+ robots_txt_cache_sec: live.robots_txt_cache_sec,+ politeness_factor: live.politeness_factor,+ min_crawl_delay_ms: live.min_crawl_delay_ms,+ max_crawl_delay_ms: live.max_crawl_delay_ms,+ max_politeness_factor: live.max_politeness_factor,+ max_url_slowdown_retry: live.max_url_slowdown_retry,+ max_redirects: live.max_redirects,+ dry_run: false,+ timeout_seconds: live.timeout_seconds,+ // no impact+ s3: crate::config::S3Config {+ bucket: String::new(),+ folder: String::new(),+ access_key: String::new(),+ secret_key: String::new(),+ endpoint: String::new(),+ },+ router_hosts: Vec::new(),+ }+ }+}++pub struct IndexManager {+ index: Arc<Index>,+ crawler: Crawler,+}++impl IndexManager {+ pub fn new(config: LiveIndexConfig) -> Result<Self> {+ let index = Index::new(&config.index_path)?;+ let indexer = Arc::new(Indexer {+ search_index: index.clone_inner_index(),+ worker: IndexingWorker::new(+ config.host_centrality_store_path.clone(),+ config.page_centrality_store_path.clone(),+ config.page_webgraph_path.clone(),+ None,+ config.safety_classifier_path.clone(),+ ),+ });++ let crawler_config = CrawlerConfig::from(&config);++ let crawler = Crawler::new(+ Split::open(&config.split_path)?,+ indexer,+ DownloadedDb::open(&config.downloaded_db_path)?,+ Arc::new(crawler_config),+ )?;++ Ok(Self {+ index: Arc::new(index),+ crawler,+ })+ }++ pub async fn run(mut self) {+ let mut has_inserts = false;+ let mut last_commit = Utc::now();+ let mut last_prune = Utc::now();++ loop {+ if let CrawlResults::HasInserts = self.crawler.check_feeds().await {+ has_inserts = true;+ }++ if last_prune + PRUNE_INTERVAL < Utc::now() {+ self.index.prune();++ last_prune = Utc::now();+ }++ if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && has_inserts {+ self.index.commit();++ last_commit = Utc::now();+ has_inserts = false;+ }++ tokio::time::sleep(EVENT_LOOP_INTERVAL).await;+ }+ }+}diff --git a/core/src/main.rs b/core/src/main.rsindex 9876fcdd..5d0dc3c8 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -107,6 +107,17 @@ enum Commands {#[clap(long)]alice: bool,},++ LiveIndex {+ #[clap(subcommand)]+ options: LiveIndex,+ },+}++#[derive(Subcommand)]+enum LiveIndex {+ Schedule { config_path: String },+ Serve { config_path: String },}#[derive(Subcommand)]@@ -344,6 +355,20 @@ fn main() -> Result<()> {safety_classifier::predict(model_path, &text)?}},+ Commands::LiveIndex { options } => match options {+ LiveIndex::Schedule { config_path } => {+ let config = load_toml_config(config_path);+ entrypoint::live_index::schedule(config)?;+ }+ LiveIndex::Serve { config_path } => {+ let config = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::live_index::serve(config))?+ }+ },}Ok(())diff --git a/core/src/schema.rs b/core/src/schema.rsindex 7aefebdd..b9b653e2 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -59,6 +59,7 @@ pub enum TextField {MicroformatTags,/// can either be NSFW or SFW (see safety classifier)SafetyClassification,+ InsertionTimestamp,}impl From<TextField> for usize {@@ -109,6 +110,7 @@ impl TextField {TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),TextField::MicroformatTags => Tokenizer::default(),TextField::SafetyClassification => Tokenizer::Identity(Identity {}),+ TextField::InsertionTimestamp => Tokenizer::Identity(Identity {}),}}@@ -151,6 +153,7 @@ impl TextField {TextField::TitleTrigrams => false,TextField::MicroformatTags => true,TextField::SafetyClassification => false,+ TextField::InsertionTimestamp => false,}}@@ -185,6 +188,7 @@ impl TextField {TextField::TitleTrigrams => "title_trigrams",TextField::MicroformatTags => "microformat_tags",TextField::SafetyClassification => "safety_classification",+ TextField::InsertionTimestamp => "insertion_timestamp",}}}@@ -274,7 +278,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 60] = [+pub static ALL_FIELDS: [Field; 61] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -304,6 +308,7 @@ pub static ALL_FIELDS: [Field; 60] = [Field::Text(TextField::TitleTrigrams),Field::Text(TextField::MicroformatTags),Field::Text(TextField::SafetyClassification),+ Field::Text(TextField::InsertionTimestamp),// FAST FIELDSField::Fast(FastField::IsHomepage),Field::Fast(FastField::HostCentrality),@@ -440,6 +445,9 @@ impl Field {Field::Text(TextField::SafetyClassification) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::InsertionTimestamp) => {+ IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())+ }Field::Fast(FastField::IsHomepage) => {IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())}@@ -619,6 +627,7 @@ pub fn create_schema() -> tantivy::schema::Schema {match field.options() {IndexingOption::Text(options) => builder.add_text_field(field.name(), options),IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),+ IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),};}@@ -628,6 +637,7 @@ pub fn create_schema() -> tantivy::schema::Schema {pub enum IndexingOption {Text(tantivy::schema::TextOptions),Integer(tantivy::schema::NumericOptions),+ DateTime(tantivy::schema::DateOptions),}pub enum DataType {diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 481d603f..3b3eb244 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -211,7 +211,13 @@ impl LocalSearcher {aggregator.set_inbound_similarity(scorer);}- aggregator.set_region_count(self.index.region_count.clone());+ aggregator.set_region_count(+ self.index+ .region_count+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .clone(),+ );if let Some(model) = self.linear_regression.as_ref() {aggregator.set_linear_model(model.clone());diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rsdeleted file mode 100644index 5ab10e53..00000000--- a/core/src/subdomain_count.rs+++ /dev/null@@ -1,61 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use crate::{- kv::{rocksdb_store::RocksDbStore, Kv},- prehashed::{hash, Prehashed},- webpage::url_ext::UrlExt,-};-use std::{collections::HashSet, path::Path};-use url::Url;--pub struct SubdomainCounter {- inner: Box<dyn Kv<Prehashed, HashSet<String>>>,-}--impl SubdomainCounter {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- Self {- inner: Box::new(RocksDbStore::open(path)),- }- }-- pub fn increment(&mut self, url: Url) {- let domain = url.root_domain().unwrap_or_default();-- if let Some(subdomain) = url.subdomain() {- let domain = hash(domain);- let subdomain = subdomain.to_string();-- let mut set = self.inner.get(&domain).unwrap_or_default();- set.insert(subdomain);-- self.inner.insert(domain, set);- }- }-- pub fn commit(&self) {- self.inner.flush();- }-- pub fn merge(&mut self, other: Self) {- for (key, val) in other.inner.iter() {- let mut current = self.inner.get(&key).unwrap_or_default();- current.extend(val);- self.inner.insert(key, current);- }- }-}diff --git a/core/src/warc.rs b/core/src/warc.rsindex 200429a0..9e0094d6 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -299,7 +299,7 @@ impl Response {#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]pub struct Metadata {// fetchTimeMs- pub fetch_time_ms: usize,+ pub fetch_time_ms: u64,}impl Metadata {@@ -313,7 +313,7 @@ impl Metadata {line.pop(); // remove colonlet key = line;if key == "fetchTimeMs" {- let fetch_time_ms = value.parse::<usize>()?;+ let fetch_time_ms = value.parse::<u64>()?;return Ok(Self { fetch_time_ms });}}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 9487df6b..d0cdd145 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -28,6 +28,7 @@ use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};use regex::Regex;use std::{collections::HashMap, panic, str::FromStr};use tantivy::{+ time::OffsetDateTime,tokenizer::{PreTokenizedString, Tokenizer},TantivyDocument,};@@ -1196,6 +1197,12 @@ impl Html {Field::Text(TextField::MicroformatTags) => {doc.add_pre_tokenized_text(tantivy_field, microformats.clone());}+ Field::Text(TextField::InsertionTimestamp) => {+ doc.add_date(+ tantivy_field,+ tantivy::DateTime::from_utc(OffsetDateTime::now_utc()),+ );+ }Field::Fast(FastField::IsHomepage) => {doc.add_u64(tantivy_field, (self.is_homepage()).into());}