Skip to content

web search done right

live_index construction

Mikkel Denker869382528366
diff --git a/Cargo.toml b/Cargo.toml
index 7563f9fb..ecc9f3ec 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -85,7 +85,7 @@ rusqlite = {version = "0.29.0", features = [
rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}
rust-stemmers = "1.2.0"
safetensors = "0.3.1"
-scylla = { git = "https://github.com/scylladb/scylla-rust-driver", rev = "82c1c99f0ff86509f9dd1e649ecdaddc5a3660cf" }
+scylla = {git = "https://github.com/scylladb/scylla-rust-driver", rev = "82c1c99f0ff86509f9dd1e649ecdaddc5a3660cf"}
serde = {version = "1.0.137", features = ["rc", "derive"]}
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 4beb5b63..88366ec2 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -17,6 +17,7 @@
pub mod defaults;
use super::Result;
+use crate::feed::scheduler::SplitId;
use crate::searcher::ShardId;
use serde::{Deserialize, Serialize};
use std::fs::File;
@@ -380,3 +381,58 @@ pub struct CrawlPlannerConfig {
pub top_host_fraction: f64,
pub wander_fraction: f64,
}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct LiveIndexConfig {
+ pub split_path: String,
+ pub downloaded_db_path: String,
+
+ // crawler
+ pub user_agent: UserAgent,
+ #[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
+ pub robots_txt_cache_sec: u64,
+ #[serde(default = "defaults::Crawler::politeness_factor")]
+ pub politeness_factor: f32,
+ #[serde(default = "defaults::Crawler::min_crawl_delay_ms")]
+ pub min_crawl_delay_ms: u64,
+ #[serde(default = "defaults::Crawler::max_crawl_delay_ms")]
+ pub max_crawl_delay_ms: u64,
+ #[serde(default = "defaults::Crawler::max_politeness_factor")]
+ pub max_politeness_factor: f32,
+ #[serde(default = "defaults::Crawler::max_url_slowdown_retry")]
+ pub max_url_slowdown_retry: u8,
+ #[serde(default = "defaults::Crawler::max_redirects")]
+ pub max_redirects: usize,
+ pub timeout_seconds: u64,
+
+ // indexer
+ pub host_centrality_store_path: String,
+ pub page_webgraph_path: Option<String>,
+ pub page_centrality_store_path: Option<String>,
+ pub safety_classifier_path: Option<String>,
+ pub host_centrality_threshold: Option<f64>,
+ pub minimum_clean_words: Option<usize>,
+
+ // search
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+ pub split_id: SplitId,
+ pub index_path: String,
+ pub linear_model_path: Option<String>,
+ pub lambda_model_path: Option<String>,
+ pub host: SocketAddr,
+ #[serde(default)]
+ pub collector: CollectorConfig,
+ #[serde(default)]
+ pub snippet: SnippetConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct LiveIndexSchedulerConfig {
+ pub schedule_path: String,
+ pub feed_index_path: String,
+ pub host_centrality_store_path: String,
+ pub host_graph_path: String,
+ pub num_splits: u64,
+}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index aae8416b..00e74954 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -23,6 +23,7 @@ use url::Url;
use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};
use self::{warc_writer::WarcWriter, worker::WorkerThread};
+pub use worker::JobExecutor;
pub mod coordinator;
mod robots_txt;
@@ -175,7 +176,7 @@ pub struct DomainCrawled {
pub budget_used: f64,
}
-struct RetrieableUrl {
+pub struct RetrieableUrl {
url: Url,
retries: u8,
}
@@ -186,10 +187,10 @@ impl From<Url> for RetrieableUrl {
}
}
-struct WorkerJob {
+pub struct WorkerJob {
pub domain: Domain,
pub urls: VecDeque<RetrieableUrl>,
- pub wanrdering_urls: u64,
+ pub wandering_urls: u64,
}
impl From<Job> for WorkerJob {
@@ -197,18 +198,18 @@ impl From<Job> for WorkerJob {
Self {
domain: value.domain,
urls: value.urls.into_iter().map(RetrieableUrl::from).collect(),
- wanrdering_urls: value.wandering_urls,
+ wandering_urls: value.wandering_urls,
}
}
}
#[derive(Debug, Clone)]
pub struct CrawlDatum {
- url: Url,
- status_code: u16,
- payload_type: warc::PayloadType,
- body: String,
- fetch_time_ms: u64,
+ pub url: Url,
+ pub status_code: u16,
+ pub payload_type: warc::PayloadType,
+ pub body: String,
+ pub fetch_time_ms: u64,
}
pub struct Crawler {
@@ -219,7 +220,6 @@ pub struct Crawler {
impl Crawler {
pub async fn new(config: CrawlerConfig) -> Result<Self> {
let writer = Arc::new(WarcWriter::new(config.s3.clone()));
- let timeout = Duration::from_secs(config.timeout_seconds);
let mut handles = Vec::new();
let mut router_hosts = Vec::new();
@@ -228,12 +228,8 @@ impl Crawler {
}
for _ in 0..config.num_worker_threads {
- let worker = WorkerThread::new(
- Arc::clone(&writer),
- config.clone(),
- timeout,
- router_hosts.clone(),
- )?;
+ let worker =
+ WorkerThread::new(Arc::clone(&writer), config.clone(), router_hosts.clone())?;
handles.push(tokio::spawn(async move {
worker.run().await;
@@ -251,3 +247,32 @@ impl Crawler {
self.writer.finish().await.unwrap();
}
}
+
+#[async_trait::async_trait]
+pub trait DatumStream: Send + Sync {
+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()>;
+ async fn finish(&self) -> Result<()>;
+}
+
+pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
+ let timeout = Duration::from_secs(config.timeout_seconds);
+
+ let mut headers = reqwest::header::HeaderMap::default();
+ headers.insert(
+ reqwest::header::ACCEPT,
+ reqwest::header::HeaderValue::from_static("text/html"),
+ );
+ headers.insert(
+ reqwest::header::ACCEPT_LANGUAGE,
+ reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
+ );
+
+ Ok(reqwest::Client::builder()
+ .timeout(timeout)
+ .connect_timeout(timeout)
+ .http2_keep_alive_interval(None)
+ .default_headers(headers)
+ .redirect(reqwest::redirect::Policy::limited(config.max_redirects))
+ .user_agent(&config.user_agent.full)
+ .build()?)
+}
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index a2b06e54..078d41d0 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -21,7 +21,7 @@ use crate::{
warc,
};
-use super::{CrawlDatum, Result};
+use super::{CrawlDatum, DatumStream, Result};
/// The WarcWriter is responsible for storing the crawl datums
/// as WARC files on S3.
@@ -29,6 +29,23 @@ pub struct WarcWriter {
tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,
}
+#[async_trait::async_trait]
+impl DatumStream for WarcWriter {
+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
+ self.tx
+ .blocking_send(WarcWriterMessage::Crawl(crawl_datum))?;
+
+ Ok(())
+ }
+
+ async fn finish(&self) -> Result<()> {
+ self.tx.blocking_send(WarcWriterMessage::Finish)?;
+ self.tx.closed().await;
+
+ Ok(())
+ }
+}
+
#[derive(Debug, Clone)]
pub enum WarcWriterMessage {
Crawl(CrawlDatum),
@@ -97,7 +114,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:
payload_type: Some(datum.payload_type),
},
metadata: warc::Metadata {
- fetch_time_ms: datum.fetch_time_ms as usize,
+ fetch_time_ms: datum.fetch_time_ms,
},
};
@@ -131,17 +148,4 @@ impl WarcWriter {
Self { tx }
}
-
- pub async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
- self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;
-
- Ok(())
- }
-
- pub async fn finish(&self) -> Result<()> {
- self.tx.send(WarcWriterMessage::Finish).await?;
- self.tx.closed().await;
-
- Ok(())
- }
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 0e67ad9c..9e30e40b 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -40,8 +40,8 @@ use crate::{
};
use super::{
- robots_txt::RobotsTxtManager, site_graph::SiteGraph, CrawlDatum, Domain, Error, Result,
- RetrieableUrl, Site, UrlResponse, WarcWriter, WorkerJob,
+ reqwest_client, robots_txt::RobotsTxtManager, site_graph::SiteGraph, CrawlDatum, DatumStream,
+ Domain, Error, Result, RetrieableUrl, Site, UrlResponse, WarcWriter, WorkerJob,
};
const MAX_CONTENT_LENGTH: usize = 32 * 1024 * 1024; // 32 MB
@@ -63,27 +63,9 @@ impl WorkerThread {
pub fn new(
writer: Arc<WarcWriter>,
config: CrawlerConfig,
- timeout: Duration,
router_hosts: Vec<SocketAddr>,
) -> Result<Self> {
- let mut headers = reqwest::header::HeaderMap::default();
- headers.insert(
- reqwest::header::ACCEPT,
- reqwest::header::HeaderValue::from_static("text/html"),
- );
- headers.insert(
- reqwest::header::ACCEPT_LANGUAGE,
- reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
- );
-
- let client = reqwest::Client::builder()
- .timeout(timeout)
- .connect_timeout(timeout)
- .http2_keep_alive_interval(None)
- .default_headers(headers)
- .redirect(reqwest::redirect::Policy::limited(config.max_redirects))
- .user_agent(&config.user_agent.full)
- .build()?;
+ let client = reqwest_client(&config)?;
Ok(Self {
writer,
@@ -135,8 +117,8 @@ impl WorkerThread {
}
}
-struct JobExecutor {
- writer: Arc<WarcWriter>,
+pub struct JobExecutor<S: DatumStream> {
+ writer: Arc<S>,
client: reqwest::Client,
politeness_factor: f32,
robotstxt: RobotsTxtManager,
@@ -148,12 +130,12 @@ struct JobExecutor {
job: WorkerJob,
}
-impl JobExecutor {
- fn new(
+impl<S: DatumStream> JobExecutor<S> {
+ pub fn new(
job: WorkerJob,
client: reqwest::Client,
config: Arc<CrawlerConfig>,
- writer: Arc<WarcWriter>,
+ writer: Arc<S>,
) -> Self {
Self {
writer,
@@ -171,11 +153,15 @@ impl JobExecutor {
job,
}
}
- async fn run(mut self) {
+
+ pub async fn run(mut self) {
tracing::info!("Processing job: {:?}", self.job.domain);
self.scheduled_urls().await;
- self.wander().await;
+
+ if self.job.wandering_urls > 0 {
+ self.wander().await;
+ }
}
async fn scheduled_urls(&mut self) {
@@ -207,7 +193,7 @@ impl JobExecutor {
let urls = urls
.into_iter()
- .take(self.job.wanrdering_urls as usize)
+ .take(self.job.wandering_urls as usize)
.map(|(url, _)| url)
.map(RetrieableUrl::from)
.collect();
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 66593336..5f2313db 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -19,6 +19,7 @@ use tokio::io;
use tokio_stream::StreamExt;
use tracing::{debug, info};
+use crate::entrypoint::indexer::JobSettings;
use crate::entrypoint::{dmoz_parser, indexer};
use crate::Result;
use std::fs::{self};
@@ -142,8 +143,10 @@ fn create_inverted_index() -> Result<()> {
}),
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
base_path: out_path_tmp.to_str().unwrap().to_string(),
- host_centrality_threshold: None,
- minimum_clean_words: None,
+ settings: JobSettings {
+ host_centrality_threshold: None,
+ minimum_clean_words: None,
+ },
};
let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index feb7b224..b1bfeda8 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -13,6 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use anyhow::anyhow;
use chrono::Utc;
use rayon::prelude::*;
use std::path::Path;
@@ -69,6 +70,11 @@ pub struct Job {
pub source_config: JobConfig,
pub warc_paths: Vec<String>,
pub base_path: String,
+ pub settings: JobSettings,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct JobSettings {
pub host_centrality_threshold: Option<f64>,
pub minimum_clean_words: Option<usize>,
}
@@ -76,16 +82,17 @@ pub struct Job {
pub struct IndexingWorker {
host_centrality_store: RocksDbStore<NodeID, f64>,
page_centrality_store: Option<RocksDbStore<NodeID, f64>>,
- webgraph: Option<Webgraph>,
+ page_webgraph: Option<Webgraph>,
topics: Option<human_website_annotations::Mapper>,
safety_classifier: Option<safety_classifier::Model>,
+ job_settings: Option<JobSettings>,
}
impl IndexingWorker {
pub fn new(
host_centrality_store_path: String,
page_centrality_store_path: Option<String>,
- webgraph_path: Option<String>,
+ page_webgraph_path: Option<String>,
topics_path: Option<String>,
safety_classifier_path: Option<String>,
) -> Self {
@@ -95,173 +102,182 @@ impl IndexingWorker {
),
page_centrality_store: page_centrality_store_path
.map(|p| RocksDbStore::open(Path::new(&p).join("derived_harmonic"))),
- webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),
+ page_webgraph: page_webgraph_path
+ .map(|path| WebgraphBuilder::new(path).single_threaded().open()),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
safety_classifier: safety_classifier_path
.map(|path| safety_classifier::Model::open(path).unwrap()),
+ job_settings: None,
}
}
-}
-
-pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
- let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
- let mut has_host_centrality = false;
- let mut has_page_centrality = false;
- let mut has_backlinks = false;
+ pub fn set_job_settings(&mut self, job_settings: JobSettings) {
+ self.job_settings = Some(job_settings);
+ }
- info!("processing {}", name);
+ pub fn prepare_webpage(&self, body: &str, url: &str, fetch_time_ms: u64) -> Result<Webpage> {
+ let mut html = match Html::parse_without_text(body, url) {
+ Ok(html) => html,
+ Err(err) => {
+ debug!("error parsing html: {:?}", err);
+ return Err(anyhow!("error parsing html: {:?}", err));
+ }
+ };
- let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
+ if html.is_no_index() {
+ return Err(anyhow!("noindex"));
+ }
- let source: config::WarcSource = job.source_config.clone().into();
+ let title = html.title().unwrap_or_default();
+ if title.is_empty() || title.chars().all(|c| c.is_whitespace()) {
+ return Err(anyhow!("empty title"));
+ }
- let warc_files = download_all_warc_files(&job.warc_paths, &source);
- pin!(warc_files);
+ let node = Node::from(html.url());
+ let host_node_id = node.clone().into_host().id();
- let current_timestamp = Utc::now().timestamp().max(0) as usize;
+ let mut host_centrality = self
+ .host_centrality_store
+ .get(&host_node_id)
+ .unwrap_or_default();
- for file in warc_files.by_ref() {
- for record in
- file.records()
- .flatten()
- .filter(|record| match &record.response.payload_type {
- Some(payload_type) => matches!(payload_type, PayloadType::Html),
- None => false,
- })
+ if let Some(host_centrality_threshold) =
+ self.job_settings.and_then(|s| s.host_centrality_threshold)
{
- let mut html =
- match Html::parse_without_text(&record.response.body, &record.request.url) {
- Ok(html) => html,
- Err(err) => {
- debug!("error parsing html: {:?}", err);
- continue;
- }
- };
-
- if html.is_no_index() {
- continue;
- }
-
- let title = html.title().unwrap_or_default();
- if title.is_empty() || title.chars().all(|c| c.is_whitespace()) {
- continue;
+ if host_centrality < host_centrality_threshold {
+ debug!("skipping due to low host_centrality value");
+ return Err(anyhow!("low host_centrality value"));
}
+ }
- let node = Node::from(html.url());
- let host_node_id = node.clone().into_host().id();
+ html.parse_text();
- let mut host_centrality = worker
- .host_centrality_store
- .get(&host_node_id)
- .unwrap_or_default();
+ if html.empty_all_text() {
+ return Err(anyhow!("empty all text"));
+ }
- if let Some(host_centrality_threshold) = job.host_centrality_threshold {
- if host_centrality < host_centrality_threshold {
- debug!("skipping due to low host_centrality value");
- continue;
+ if let Some(minimum_clean_words) = self.job_settings.and_then(|s| s.minimum_clean_words) {
+ match html.clean_text() {
+ Some(clean_text) => {
+ if clean_text.split_whitespace().count() < minimum_clean_words {
+ return Err(anyhow!("too few clean words"));
+ }
}
+ None => return Err(anyhow!("no clean text")),
}
+ }
- html.parse_text();
+ let backlink_labels: Vec<String> = self
+ .page_webgraph
+ .as_ref()
+ .map(|webgraph| {
+ webgraph
+ .raw_ingoing_edges_with_labels(&Node::from(html.url()).id())
+ .into_iter()
+ .map(|edge| edge.label)
+ .filter(|label| !label.is_empty())
+ .collect()
+ })
+ .unwrap_or_default();
- if html.empty_all_text() {
- continue;
- }
+ let mut page_centrality = 0.0;
- if let Some(minimum_clean_words) = job.minimum_clean_words {
- match html.clean_text() {
- Some(clean_text) => {
- if clean_text.split_whitespace().count() < minimum_clean_words {
- continue;
- }
- }
- None => continue,
- }
- }
+ if let Some(store) = self.page_centrality_store.as_ref() {
+ let node_id = node.id();
- let backlink_labels: Vec<String> = worker
- .webgraph
- .as_ref()
- .map(|webgraph| {
- webgraph
- .raw_ingoing_edges_with_labels(&Node::from(html.url()).id())
- .into_iter()
- .map(|edge| edge.label)
- .filter(|label| !label.is_empty())
- .collect()
- })
- .unwrap_or_default();
+ page_centrality = store.get(&node_id).unwrap_or_default();
+ }
- let mut page_centrality = 0.0;
+ if !page_centrality.is_finite() {
+ page_centrality = 0.0;
+ }
- if let Some(store) = worker.page_centrality_store.as_ref() {
- let node_id = node.id();
+ if !host_centrality.is_finite() {
+ host_centrality = 0.0;
+ }
- page_centrality = store.get(&node_id).unwrap_or_default();
- }
+ let mut dmoz_description = None;
- if host_centrality > 0.0 {
- has_host_centrality = true;
+ if let Some(mapper) = self.topics.as_ref() {
+ if let Some(info) = mapper.get(&html.url().host_str().unwrap_or_default().to_string()) {
+ dmoz_description = Some(info.description.clone())
}
+ }
- if page_centrality > 0.0 {
- has_page_centrality = true;
- }
+ let mut webpage = Webpage {
+ html,
+ backlink_labels,
+ page_centrality,
+ host_centrality,
+ fetch_time_ms,
+ pre_computed_score: 0.0,
+ node_id: Some(host_node_id),
+ dmoz_description,
+ safety_classification: None,
+ };
+
+ if let Some(model) = self.safety_classifier.as_ref() {
+ webpage.safety_classification = Some(model.predict(&webpage).label);
+ }
- if !backlink_labels.is_empty() {
- has_backlinks = true;
- }
+ let mut signal_aggregator = SignalAggregator::new(None);
+ signal_aggregator.set_current_timestamp(Utc::now().timestamp().max(0) as usize);
- if !page_centrality.is_finite() {
- page_centrality = 0.0;
- }
+ webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);
- if !host_centrality.is_finite() {
- host_centrality = 0.0;
- }
+ Ok(webpage)
+ }
+}
- let fetch_time_ms = record.metadata.fetch_time_ms as u64;
+pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
+ let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
- trace!("inserting webpage: {:?}", html.url());
+ let mut has_host_centrality = false;
+ let mut has_page_centrality = false;
+ let mut has_backlinks = false;
- trace!("title = {:?}", html.title());
- trace!("text = {:?}", html.clean_text());
+ info!("processing {}", name);
- let mut dmoz_description = None;
+ let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
- if let Some(mapper) = worker.topics.as_ref() {
- if let Some(info) =
- mapper.get(&html.url().host_str().unwrap_or_default().to_string())
- {
- dmoz_description = Some(info.description.clone())
- }
- }
+ let source: config::WarcSource = job.source_config.clone().into();
- let mut webpage = Webpage {
- html,
- backlink_labels,
- page_centrality,
- host_centrality,
- fetch_time_ms,
- pre_computed_score: 0.0,
- node_id: Some(host_node_id),
- dmoz_description,
- safety_classification: None,
- };
-
- if let Some(model) = worker.safety_classifier.as_ref() {
- webpage.safety_classification = Some(model.predict(&webpage).label);
- }
+ let warc_files = download_all_warc_files(&job.warc_paths, &source);
+ pin!(warc_files);
- let mut signal_aggregator = SignalAggregator::new(None);
- signal_aggregator.set_current_timestamp(current_timestamp);
+ for file in warc_files.by_ref() {
+ for record in
+ file.records()
+ .flatten()
+ .filter(|record| match &record.response.payload_type {
+ Some(payload_type) => matches!(payload_type, PayloadType::Html),
+ None => false,
+ })
+ {
+ if let Ok(webpage) = worker.prepare_webpage(
+ &record.response.body,
+ &record.request.url,
+ record.metadata.fetch_time_ms,
+ ) {
+ if webpage.host_centrality > 0.0 {
+ has_host_centrality = true;
+ }
- webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);
+ if webpage.page_centrality > 0.0 {
+ has_page_centrality = true;
+ }
+
+ if !webpage.backlink_labels.is_empty() {
+ has_backlinks = true;
+ }
+ trace!("inserting webpage: {:?}", webpage.html.url());
+ trace!("title = {:?}", webpage.html.title());
+ trace!("text = {:?}", webpage.html.clean_text());
- if let Err(err) = index.insert(webpage) {
- warn!("{:?}", err);
- panic!();
+ if let Err(err) = index.insert(webpage) {
+ warn!("{:?}", err);
+ panic!();
+ }
}
}
@@ -276,7 +292,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
warn!("no page centrality values found in {}", name);
}
- if !has_backlinks && worker.webgraph.is_some() {
+ if !has_backlinks && worker.page_webgraph.is_some() {
warn!("no backlinks found in {}", name);
}
@@ -365,12 +381,14 @@ impl Indexer {
.map(|warc_paths| Job {
source_config: job_config.clone(),
warc_paths,
- host_centrality_threshold: config.host_centrality_threshold,
base_path: config
.output_path
.clone()
.unwrap_or_else(|| "data/index".to_string()),
- minimum_clean_words: config.minimum_clean_words,
+ settings: JobSettings {
+ host_centrality_threshold: config.host_centrality_threshold,
+ minimum_clean_words: config.minimum_clean_words,
+ },
})
.map(|job| {
let pointer: IndexPointer = job.map(&worker);
diff --git a/core/src/entrypoint/live_index.rs b/core/src/entrypoint/live_index.rs
new file mode 100644
index 00000000..3e19bf26
--- /dev/null
+++ b/core/src/entrypoint/live_index.rs
@@ -0,0 +1,46 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::path::Path;
+
+use crate::{
+ config::{LiveIndexConfig, LiveIndexSchedulerConfig},
+ feed::{self, index::FeedIndex},
+ kv::rocksdb_store::RocksDbStore,
+ live_index::IndexManager,
+ webgraph::WebgraphBuilder,
+};
+use anyhow::Result;
+
+pub async fn serve(config: LiveIndexConfig) -> Result<()> {
+ let manager = IndexManager::new(config)?;
+ tokio::task::spawn(manager.run());
+
+ todo!("accept search requests");
+}
+
+pub fn schedule(config: LiveIndexSchedulerConfig) -> Result<()> {
+ let feed_index = FeedIndex::open(config.feed_index_path)?;
+ let host_harmonic =
+ RocksDbStore::open(Path::new(&config.host_centrality_store_path).join("harmonic"));
+ let host_graph = WebgraphBuilder::new(config.host_graph_path).open();
+
+ let schedule =
+ feed::scheduler::schedule(&feed_index, &host_harmonic, &host_graph, config.num_splits);
+ schedule.save(config.schedule_path)?;
+
+ Ok(())
+}
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 230da23a..a307e6ff 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -36,6 +36,7 @@ pub use entity::EntityIndexer;
pub use indexer::Indexer;
use tracing::{debug, log::error};
pub use webgraph::Webgraph;
+pub mod live_index;
use crate::{config, warc::WarcFile};
diff --git a/core/src/feed/mod.rs b/core/src/feed/mod.rs
index 284c2451..1f9ebf9b 100644
--- a/core/src/feed/mod.rs
+++ b/core/src/feed/mod.rs
@@ -30,8 +30,8 @@ pub enum FeedKind {
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
pub struct Feed {
- url: Url,
- kind: FeedKind,
+ pub url: Url,
+ pub kind: FeedKind,
}
pub struct ParsedFeed {
diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rs
index 1d0795f3..0f876704 100644
--- a/core/src/feed/scheduler.rs
+++ b/core/src/feed/scheduler.rs
@@ -14,6 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::path::Path;
+
+use anyhow::Result;
use hashbrown::{HashMap, HashSet};
use url::Url;
@@ -28,21 +31,71 @@ use crate::{
use super::{index::FeedIndex, Feed};
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
pub struct Domain(String);
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct DomainFeeds {
- domain: Domain,
- feeds: Vec<Feed>,
+ pub domain: Domain,
+ pub feeds: Vec<Feed>,
}
+impl From<&Url> for Domain {
+ fn from(url: &Url) -> Self {
+ Domain(url.icann_domain().unwrap_or_default().to_string())
+ }
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct SplitId(uuid::Uuid);
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Split {
- feeds: Vec<DomainFeeds>,
+ pub id: SplitId,
+ pub feeds: Vec<DomainFeeds>,
+}
+
+impl Split {
+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
+ let file = std::fs::OpenOptions::new()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(path.as_ref())?;
+ let writer = std::io::BufWriter::new(file);
+
+ serde_json::to_writer_pretty(writer, &self)
+ .map_err(|e| anyhow::anyhow!("Failed to serialize split: {}", e))?;
+
+ Ok(())
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let file = std::fs::OpenOptions::new().read(true).open(path.as_ref())?;
+ let reader = std::io::BufReader::new(file);
+
+ serde_json::from_reader(reader)
+ .map_err(|e| anyhow::anyhow!("Failed to deserialize split: {}", e))
+ }
}
pub struct Schedule {
splits: Vec<Split>,
}
+impl Schedule {
+ pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(&path)?;
+ }
+
+ for split in self.splits {
+ let name = split.id.0.to_string() + ".json";
+ split.save(path.as_ref().join(name))?;
+ }
+
+ Ok(())
+ }
+}
pub fn schedule(
index: &FeedIndex,
@@ -63,7 +116,7 @@ pub fn schedule(
}
let url = url.unwrap();
- let domain = Domain(url.icann_domain().unwrap().to_string());
+ let domain = Domain::from(&url);
all_feeds
.entry(domain)
@@ -74,7 +127,10 @@ pub fn schedule(
let mut splits = Vec::new();
for _ in 0..num_splits {
- splits.push(Split { feeds: Vec::new() });
+ splits.push(Split {
+ id: SplitId(uuid::Uuid::new_v4()),
+ feeds: Vec::new(),
+ });
}
for (i, (domain, feeds)) in all_feeds.into_iter().enumerate() {
diff --git a/core/src/index.rs b/core/src/index.rs
index 8da18312..51572977 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -17,7 +17,8 @@
use std::collections::HashSet;
use std::fs;
use std::path::Path;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
+use std::time::SystemTime;
use serde::{Deserialize, Serialize};
use tantivy::schema::Schema;
@@ -29,7 +30,6 @@ use crate::directory::{self, DirEntry};
use crate::inverted_index::{self, InvertedIndex};
use crate::query::Query;
use crate::search_ctx::Ctx;
-use crate::subdomain_count::SubdomainCounter;
use crate::webgraph::NodeID;
use crate::webpage::region::{Region, RegionCount};
use crate::webpage::Webpage;
@@ -37,12 +37,10 @@ use crate::Result;
const INVERTED_INDEX_SUBFOLDER_NAME: &str = "inverted_index";
const REGION_COUNT_FILE_NAME: &str = "region_count.json";
-const SUBDOMAIN_COUNT_SUBFOLDER_NAME: &str = "subdomain_count";
pub struct Index {
pub inverted_index: InvertedIndex,
- pub region_count: RegionCount,
- pub subdomain_counter: SubdomainCounter,
+ pub region_count: Mutex<RegionCount>,
pub path: String,
}
@@ -59,10 +57,7 @@ impl Index {
Ok(Self {
inverted_index,
- region_count,
- subdomain_counter: SubdomainCounter::open(
- path.as_ref().join(SUBDOMAIN_COUNT_SUBFOLDER_NAME),
- ),
+ region_count: Mutex::new(region_count),
path: path.as_ref().to_str().unwrap().to_string(),
})
}
@@ -73,6 +68,10 @@ impl Index {
Ok(())
}
+ pub fn set_auto_merge_policy(&mut self) {
+ self.inverted_index.set_auto_merge_policy();
+ }
+
pub fn tokenizers(&self) -> &TokenizerManager {
self.inverted_index.tokenizers()
}
@@ -83,20 +82,26 @@ impl Index {
Self::open(path)
}
- pub fn insert(&mut self, webpage: Webpage) -> Result<()> {
- self.subdomain_counter.increment(webpage.html.url().clone());
-
+ pub fn insert(&self, webpage: Webpage) -> Result<()> {
if let Ok(region) = Region::guess_from(&webpage) {
- self.region_count.increment(&region);
+ let mut reg = self.region_count.lock().unwrap_or_else(|e| e.into_inner());
+ reg.increment(&region);
}
self.inverted_index.insert(webpage)
}
+ pub fn delete_all_before(&self, timestamp: SystemTime) -> Result<()> {
+ self.inverted_index
+ .delete_all_before(tantivy::DateTime::from_utc(timestamp.into()))
+ }
+
pub fn commit(&mut self) -> Result<()> {
self.inverted_index.commit()?;
- self.region_count.commit();
- self.subdomain_counter.commit();
+
+ let mut reg = self.region_count.lock().unwrap_or_else(|e| e.into_inner());
+ reg.commit();
+
Ok(())
}
@@ -128,13 +133,19 @@ impl Index {
self.inverted_index.retrieve_websites(websites, query)
}
- pub fn merge(mut self, other: Self) -> Self {
+ pub fn merge(self, other: Self) -> Self {
self.inverted_index.merge(other.inverted_index);
- self.region_count.merge(other.region_count);
+ let mut self_region_count = self
+ .region_count
+ .into_inner()
+ .unwrap_or_else(|e| e.into_inner());
+ let other_region_count = other
+ .region_count
+ .into_inner()
+ .unwrap_or_else(|e| e.into_inner());
- self.subdomain_counter.merge(other.subdomain_counter);
- drop(self.subdomain_counter);
+ self_region_count.merge(other_region_count);
Self::open(&self.path).expect("failed to open index")
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index f0e9aa33..e765b287 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -188,6 +188,11 @@ impl InvertedIndex {
self.snippet_config = config;
}
+ pub fn set_auto_merge_policy(&mut self) {
+ let merge_policy = tantivy::merge_policy::LogMergePolicy::default();
+ self.writer.set_merge_policy(Box::new(merge_policy));
+ }
+
pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {
FastFieldReader::new(tv_searcher)
}
@@ -215,6 +220,24 @@ impl InvertedIndex {
Ok(())
}
+ fn delete(&self, query: Box<dyn tantivy::query::Query>) -> Result<()> {
+ self.writer.delete_query(query)?;
+
+ Ok(())
+ }
+
+ pub fn delete_all_before(&self, timestamp: tantivy::DateTime) -> Result<()> {
+ let query = tantivy::query::RangeQuery::new_date_bounds(
+ Field::Text(TextField::InsertionTimestamp)
+ .name()
+ .to_string(),
+ std::ops::Bound::Unbounded,
+ std::ops::Bound::Excluded(timestamp),
+ );
+
+ self.delete(Box::new(query))
+ }
+
pub fn search_initial(
&self,
query: &Query,
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 7069c4ca..395e6ad8 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -57,6 +57,7 @@ mod intmap;
mod kahan_sum;
mod kv;
mod leaky_queue;
+mod live_index;
#[cfg(feature = "libtorch")]
mod llm_utils;
mod metrics;
@@ -74,7 +75,6 @@ mod simhash;
pub mod similar_sites;
mod snippet;
pub mod spell;
-mod subdomain_count;
#[cfg(feature = "libtorch")]
pub mod summarizer;
mod tokenizer;
diff --git a/core/src/live_index.rs b/core/src/live_index.rs
new file mode 100644
index 00000000..f66169e2
--- /dev/null
+++ b/core/src/live_index.rs
@@ -0,0 +1,367 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::Result;
+use std::{
+ path::Path,
+ sync::{Arc, RwLock},
+ time::{Duration, SystemTime},
+};
+
+use chrono::{DateTime, Utc};
+use url::Url;
+
+use crate::{
+ config::{CrawlerConfig, LiveIndexConfig},
+ crawler::{reqwest_client, CrawlDatum, DatumStream, JobExecutor, RetrieableUrl, WorkerJob},
+ entrypoint::indexer::IndexingWorker,
+ feed::{
+ self,
+ scheduler::{Domain, DomainFeeds, Split},
+ },
+};
+
+const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 days
+const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
+const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes
+const AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutes
+const EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);
+
+#[derive(Debug, Clone)]
+struct Feeds {
+ last_checked: DateTime<Utc>,
+ feed: DomainFeeds,
+}
+
+impl From<Split> for Vec<Feeds> {
+ fn from(split: Split) -> Self {
+ split
+ .feeds
+ .into_iter()
+ .map(|feed| Feeds {
+ last_checked: Utc::now(),
+ feed,
+ })
+ .collect()
+ }
+}
+
+struct DownloadedDb {
+ db: rocksdb::DB,
+}
+
+impl DownloadedDb {
+ fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+
+ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
+ options.set_level_compaction_dynamic_level_bytes(true);
+ options.set_bytes_per_sync(1048576);
+ let mut block_options = rocksdb::BlockBasedOptions::default();
+ block_options.set_block_size(16 * 1024);
+ block_options.set_format_version(5);
+ block_options.set_cache_index_and_filter_blocks(true);
+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
+
+ block_options.set_ribbon_filter(10.0);
+
+ options.set_block_based_table_factory(&block_options);
+ options.optimize_for_point_lookup(512); // 512 mb
+
+ let db = rocksdb::DB::open_with_ttl(&options, path, TTL)?;
+ Ok(Self { db })
+ }
+
+ fn has_downloaded(&self, url: &Url) -> Result<bool> {
+ let key = url.as_str().as_bytes();
+
+ if self.db.key_may_exist(key) {
+ self.db.get(key).map(|v| v.is_some()).map_err(Into::into)
+ } else {
+ Ok(false)
+ }
+ }
+
+ fn insert(&self, url: &Url) -> Result<()> {
+ let key = url.as_str().as_bytes();
+ self.db.put(key, b"")?;
+ Ok(())
+ }
+}
+
+struct Indexer {
+ search_index: Arc<RwLock<crate::index::Index>>,
+ worker: IndexingWorker,
+}
+
+enum CrawlResults {
+ None,
+ HasInserts,
+}
+
+struct Crawler {
+ feeds: Vec<Feeds>,
+ indexer: Arc<Indexer>,
+ downloaded_db: DownloadedDb,
+ config: Arc<CrawlerConfig>,
+ client: reqwest::Client,
+}
+
+impl Crawler {
+ fn new(
+ split: Split,
+ indexer: Arc<Indexer>,
+ downloaded_db: DownloadedDb,
+ config: Arc<CrawlerConfig>,
+ ) -> Result<Self> {
+ let client = reqwest_client(&config)?;
+
+ Ok(Self {
+ feeds: split.into(),
+ indexer,
+ downloaded_db,
+ config,
+ client,
+ })
+ }
+
+ async fn process_urls(&self, urls: Vec<Url>) -> Result<bool> {
+ if urls.is_empty() {
+ return Ok(false);
+ }
+
+ let domain = urls.first().unwrap().into();
+ let job = WorkerJob {
+ domain,
+ urls: urls.clone().into_iter().map(RetrieableUrl::from).collect(),
+ wandering_urls: 0,
+ };
+
+ let executor = JobExecutor::new(
+ job,
+ self.client.clone(),
+ self.config.clone(),
+ self.indexer.clone(),
+ );
+ executor.run().await;
+
+ for url in &urls {
+ self.downloaded_db.insert(url)?;
+ }
+
+ Ok(true)
+ }
+
+ async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {
+ let mut urls = Vec::new();
+
+ for feed in domain_feeds.feeds.iter() {
+ let content = self
+ .client
+ .get(feed.url.as_str())
+ .send()
+ .await?
+ .text()
+ .await?;
+
+ let feed = feed::parse(&content, feed.kind)?;
+
+ for url in feed.links {
+ if !self.downloaded_db.has_downloaded(&url)?
+ && Domain::from(&url) == domain_feeds.domain
+ {
+ urls.push(url);
+ }
+ }
+ }
+
+ self.process_urls(urls).await
+ }
+
+ async fn check_feeds(&mut self) -> CrawlResults {
+ let mut feeds = self.feeds.clone();
+
+ let mut futures = Vec::new();
+ for feeds in feeds.iter_mut() {
+ if feeds.last_checked + FEED_CHECK_INTERVAL < Utc::now() {
+ futures.push(self.process_feed(&feeds.feed));
+ feeds.last_checked = Utc::now();
+ }
+ }
+
+ let res = futures::future::join_all(futures).await;
+
+ self.feeds = feeds;
+
+ if res.iter().filter_map(|r| r.as_ref().ok()).any(|r| *r) {
+ CrawlResults::HasInserts
+ } else {
+ CrawlResults::None
+ }
+ }
+}
+
+struct Index {
+ search_index: Arc<RwLock<crate::index::Index>>,
+}
+
+impl Index {
+ pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let mut search_index = crate::index::Index::open(path.as_ref().join("index"))?;
+ search_index.set_auto_merge_policy();
+
+ let search_index = Arc::new(RwLock::new(search_index));
+
+ Ok(Self { search_index })
+ }
+
+ fn commit(&self) {
+ self.search_index
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .commit()
+ .ok();
+ }
+
+ fn prune(&self) {
+ self.search_index
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .delete_all_before(SystemTime::now() - TTL)
+ .ok();
+ }
+
+ fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {
+ self.search_index.clone()
+ }
+}
+
+#[async_trait::async_trait]
+impl DatumStream for Indexer {
+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
+ let webpage = self.worker.prepare_webpage(
+ &crawl_datum.body,
+ crawl_datum.url.as_str(),
+ crawl_datum.fetch_time_ms,
+ )?;
+
+ self.search_index
+ .read()
+ .unwrap_or_else(|e| e.into_inner())
+ .insert(webpage)?;
+
+ Ok(())
+ }
+
+ async fn finish(&self) -> Result<()> {
+ self.search_index
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .commit()?;
+
+ Ok(())
+ }
+}
+
+impl From<&LiveIndexConfig> for CrawlerConfig {
+ fn from(live: &LiveIndexConfig) -> Self {
+ Self {
+ num_worker_threads: 1, // no impact
+ user_agent: live.user_agent.clone(),
+ robots_txt_cache_sec: live.robots_txt_cache_sec,
+ politeness_factor: live.politeness_factor,
+ min_crawl_delay_ms: live.min_crawl_delay_ms,
+ max_crawl_delay_ms: live.max_crawl_delay_ms,
+ max_politeness_factor: live.max_politeness_factor,
+ max_url_slowdown_retry: live.max_url_slowdown_retry,
+ max_redirects: live.max_redirects,
+ dry_run: false,
+ timeout_seconds: live.timeout_seconds,
+ // no impact
+ s3: crate::config::S3Config {
+ bucket: String::new(),
+ folder: String::new(),
+ access_key: String::new(),
+ secret_key: String::new(),
+ endpoint: String::new(),
+ },
+ router_hosts: Vec::new(),
+ }
+ }
+}
+
+pub struct IndexManager {
+ index: Arc<Index>,
+ crawler: Crawler,
+}
+
+impl IndexManager {
+ pub fn new(config: LiveIndexConfig) -> Result<Self> {
+ let index = Index::new(&config.index_path)?;
+ let indexer = Arc::new(Indexer {
+ search_index: index.clone_inner_index(),
+ worker: IndexingWorker::new(
+ config.host_centrality_store_path.clone(),
+ config.page_centrality_store_path.clone(),
+ config.page_webgraph_path.clone(),
+ None,
+ config.safety_classifier_path.clone(),
+ ),
+ });
+
+ let crawler_config = CrawlerConfig::from(&config);
+
+ let crawler = Crawler::new(
+ Split::open(&config.split_path)?,
+ indexer,
+ DownloadedDb::open(&config.downloaded_db_path)?,
+ Arc::new(crawler_config),
+ )?;
+
+ Ok(Self {
+ index: Arc::new(index),
+ crawler,
+ })
+ }
+
+ pub async fn run(mut self) {
+ let mut has_inserts = false;
+ let mut last_commit = Utc::now();
+ let mut last_prune = Utc::now();
+
+ loop {
+ if let CrawlResults::HasInserts = self.crawler.check_feeds().await {
+ has_inserts = true;
+ }
+
+ if last_prune + PRUNE_INTERVAL < Utc::now() {
+ self.index.prune();
+
+ last_prune = Utc::now();
+ }
+
+ if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && has_inserts {
+ self.index.commit();
+
+ last_commit = Utc::now();
+ has_inserts = false;
+ }
+
+ tokio::time::sleep(EVENT_LOOP_INTERVAL).await;
+ }
+ }
+}
diff --git a/core/src/main.rs b/core/src/main.rs
index 9876fcdd..5d0dc3c8 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -107,6 +107,17 @@ enum Commands {
#[clap(long)]
alice: bool,
},
+
+ LiveIndex {
+ #[clap(subcommand)]
+ options: LiveIndex,
+ },
+}
+
+#[derive(Subcommand)]
+enum LiveIndex {
+ Schedule { config_path: String },
+ Serve { config_path: String },
}
#[derive(Subcommand)]
@@ -344,6 +355,20 @@ fn main() -> Result<()> {
safety_classifier::predict(model_path, &text)?
}
},
+ Commands::LiveIndex { options } => match options {
+ LiveIndex::Schedule { config_path } => {
+ let config = load_toml_config(config_path);
+ entrypoint::live_index::schedule(config)?;
+ }
+ LiveIndex::Serve { config_path } => {
+ let config = load_toml_config(config_path);
+
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()?
+ .block_on(entrypoint::live_index::serve(config))?
+ }
+ },
}
Ok(())
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 7aefebdd..b9b653e2 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -59,6 +59,7 @@ pub enum TextField {
MicroformatTags,
/// can either be NSFW or SFW (see safety classifier)
SafetyClassification,
+ InsertionTimestamp,
}
impl From<TextField> for usize {
@@ -109,6 +110,7 @@ impl TextField {
TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
TextField::MicroformatTags => Tokenizer::default(),
TextField::SafetyClassification => Tokenizer::Identity(Identity {}),
+ TextField::InsertionTimestamp => Tokenizer::Identity(Identity {}),
}
}
@@ -151,6 +153,7 @@ impl TextField {
TextField::TitleTrigrams => false,
TextField::MicroformatTags => true,
TextField::SafetyClassification => false,
+ TextField::InsertionTimestamp => false,
}
}
@@ -185,6 +188,7 @@ impl TextField {
TextField::TitleTrigrams => "title_trigrams",
TextField::MicroformatTags => "microformat_tags",
TextField::SafetyClassification => "safety_classification",
+ TextField::InsertionTimestamp => "insertion_timestamp",
}
}
}
@@ -274,7 +278,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 60] = [
+pub static ALL_FIELDS: [Field; 61] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -304,6 +308,7 @@ pub static ALL_FIELDS: [Field; 60] = [
Field::Text(TextField::TitleTrigrams),
Field::Text(TextField::MicroformatTags),
Field::Text(TextField::SafetyClassification),
+ Field::Text(TextField::InsertionTimestamp),
// FAST FIELDS
Field::Fast(FastField::IsHomepage),
Field::Fast(FastField::HostCentrality),
@@ -440,6 +445,9 @@ impl Field {
Field::Text(TextField::SafetyClassification) => {
IndexingOption::Text(self.default_text_options())
}
+ Field::Text(TextField::InsertionTimestamp) => {
+ IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())
+ }
Field::Fast(FastField::IsHomepage) => {
IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
}
@@ -619,6 +627,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
match field.options() {
IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
+ IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),
};
}
@@ -628,6 +637,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
pub enum IndexingOption {
Text(tantivy::schema::TextOptions),
Integer(tantivy::schema::NumericOptions),
+ DateTime(tantivy::schema::DateOptions),
}
pub enum DataType {
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 481d603f..3b3eb244 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -211,7 +211,13 @@ impl LocalSearcher {
aggregator.set_inbound_similarity(scorer);
}
- aggregator.set_region_count(self.index.region_count.clone());
+ aggregator.set_region_count(
+ self.index
+ .region_count
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .clone(),
+ );
if let Some(model) = self.linear_regression.as_ref() {
aggregator.set_linear_model(model.clone());
diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rs
deleted file mode 100644
index 5ab10e53..00000000
--- a/core/src/subdomain_count.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use crate::{
- kv::{rocksdb_store::RocksDbStore, Kv},
- prehashed::{hash, Prehashed},
- webpage::url_ext::UrlExt,
-};
-use std::{collections::HashSet, path::Path};
-use url::Url;
-
-pub struct SubdomainCounter {
- inner: Box<dyn Kv<Prehashed, HashSet<String>>>,
-}
-
-impl SubdomainCounter {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- Self {
- inner: Box::new(RocksDbStore::open(path)),
- }
- }
-
- pub fn increment(&mut self, url: Url) {
- let domain = url.root_domain().unwrap_or_default();
-
- if let Some(subdomain) = url.subdomain() {
- let domain = hash(domain);
- let subdomain = subdomain.to_string();
-
- let mut set = self.inner.get(&domain).unwrap_or_default();
- set.insert(subdomain);
-
- self.inner.insert(domain, set);
- }
- }
-
- pub fn commit(&self) {
- self.inner.flush();
- }
-
- pub fn merge(&mut self, other: Self) {
- for (key, val) in other.inner.iter() {
- let mut current = self.inner.get(&key).unwrap_or_default();
- current.extend(val);
- self.inner.insert(key, current);
- }
- }
-}
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 200429a0..9e0094d6 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -299,7 +299,7 @@ impl Response {
#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]
pub struct Metadata {
// fetchTimeMs
- pub fetch_time_ms: usize,
+ pub fetch_time_ms: u64,
}
impl Metadata {
@@ -313,7 +313,7 @@ impl Metadata {
line.pop(); // remove colon
let key = line;
if key == "fetchTimeMs" {
- let fetch_time_ms = value.parse::<usize>()?;
+ let fetch_time_ms = value.parse::<u64>()?;
return Ok(Self { fetch_time_ms });
}
}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 9487df6b..d0cdd145 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -28,6 +28,7 @@ use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
use std::{collections::HashMap, panic, str::FromStr};
use tantivy::{
+ time::OffsetDateTime,
tokenizer::{PreTokenizedString, Tokenizer},
TantivyDocument,
};
@@ -1196,6 +1197,12 @@ impl Html {
Field::Text(TextField::MicroformatTags) => {
doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
}
+ Field::Text(TextField::InsertionTimestamp) => {
+ doc.add_date(
+ tantivy_field,
+ tantivy::DateTime::from_utc(OffsetDateTime::now_utc()),
+ );
+ }
Field::Fast(FastField::IsHomepage) => {
doc.add_u64(tantivy_field, (self.is_homepage()).into());
}