Skip to content

web search done right

RSS and Atom feed indexer

Mikkel Denker0f688c5f4b9c
diff --git a/configs/indexer/feed.toml b/configs/indexer/feed.toml
new file mode 100644
index 00000000..d1bfc28f
--- /dev/null
+++ b/configs/indexer/feed.toml
@@ -0,0 +1,7 @@
+output_path = "./data/feed_index"
+# minimum_clean_words = 40
+
+[warc_source]
+folder = "./data"
+names = ["sample.warc.gz"]
+type = "Local"
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 2700a2e2..8cc9656d 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -3,9 +3,6 @@ limit_warc_files = 20
# minimum_clean_words = 40
[warc_source]
-base_url = "https://data.commoncrawl.org/"
folder = "./data"
names = ["sample.warc.gz"]
type = "Local"
-type = "HTTP"
-warc_paths_file = "data/warc.paths"
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 88366ec2..9684a51c 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -436,3 +436,10 @@ pub struct LiveIndexSchedulerConfig {
pub host_graph_path: String,
pub num_splits: u64,
}
+#[derive(Debug, Deserialize, Clone)]
+pub struct FeedIndexingConfig {
+ pub output_path: String,
+ pub warc_source: WarcSource,
+ pub limit_warc_files: Option<usize>,
+ pub skip_warc_files: Option<usize>,
+}
diff --git a/core/src/directory.rs b/core/src/directory.rs
deleted file mode 100644
index 9d33ee33..00000000
--- a/core/src/directory.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use crate::Result;
-use serde::{Deserialize, Serialize};
-use std::fs;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub enum DirEntry {
- Folder {
- name: String,
- entries: Vec<DirEntry>,
- },
- File {
- name: String,
- content: Vec<u8>,
- },
-}
-
-fn iterate_children(path: &str) -> Result<Vec<DirEntry>> {
- let mut res = Vec::new();
-
- for f in fs::read_dir(path)? {
- let entry = f?;
- let metadata = entry.metadata()?;
-
- if metadata.is_dir() {
- res.push(DirEntry::Folder {
- name: entry.path().as_os_str().to_str().unwrap().to_string(),
- entries: iterate_children(entry.path().as_os_str().to_str().unwrap())?,
- });
- } else if metadata.is_file() {
- res.push(DirEntry::File {
- name: entry.path().as_os_str().to_str().unwrap().to_string(),
- content: fs::read(entry.path())?,
- });
- }
- }
-
- Ok(res)
-}
-
-pub fn recreate_folder(entry: &DirEntry) -> Result<()> {
- match entry {
- DirEntry::Folder { name, entries } => {
- fs::create_dir(name)?;
-
- for entry in entries {
- recreate_folder(entry)?;
- }
-
- Ok(())
- }
- DirEntry::File { name, content } => Ok(fs::write(name, content)?),
- }
-}
-
-pub fn scan_folder(path: String) -> Result<DirEntry> {
- Ok(DirEntry::Folder {
- entries: iterate_children(&path)?,
- name: path,
- })
-}
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index 138ff815..c0941383 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -218,6 +218,7 @@ impl EntityIndex {
});
info.into_iter()
+ .filter(|(_, value)| !value.text.is_empty())
.map(|(key, mut value)| {
if let Some(start) = value.text.find(|c: char| !(c.is_whitespace() || c == '*')) {
value.text.replace_range(0..start, "");
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 5f2313db..1216f2b2 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -137,7 +137,7 @@ fn create_inverted_index() -> Result<()> {
let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
let job = indexer::Job {
- source_config: indexer::JobConfig::Local(crate::config::LocalConfig {
+ source_config: crate::config::WarcSource::Local(crate::config::LocalConfig {
folder: ".".to_string(),
names: vec![warc_path.to_str().unwrap().to_string()],
}),
diff --git a/core/src/entrypoint/feed_indexer.rs b/core/src/entrypoint/feed_indexer.rs
new file mode 100644
index 00000000..50254027
--- /dev/null
+++ b/core/src/entrypoint/feed_indexer.rs
@@ -0,0 +1,194 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ path::{Path, PathBuf},
+ thread,
+};
+
+use anyhow::Result;
+use itertools::Itertools;
+use rayon::prelude::*;
+use serde::{Deserialize, Serialize};
+use tracing::info;
+use url::Url;
+
+use crate::{
+ config::{FeedIndexingConfig, WarcSource},
+ entrypoint::download_all_warc_files,
+ feed::{index::FeedIndex, Feed, FeedKind},
+ mapreduce::{Map, Worker},
+ warc::PayloadType,
+};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Job {
+ pub source_config: WarcSource,
+ pub warc_path: String,
+ pub base_path: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct IndexPointer(PathBuf);
+
+impl From<String> for IndexPointer {
+ fn from(path: String) -> Self {
+ IndexPointer(Path::new(&path).to_path_buf())
+ }
+}
+
+impl Worker for IndexingWorker {}
+
+impl Map<IndexingWorker, IndexPointer> for Job {
+ fn map(&self, worker: &IndexingWorker) -> IndexPointer {
+ let index = worker.process_job(self);
+ IndexPointer(index.path)
+ }
+}
+
+pub struct IndexingWorker {}
+impl IndexingWorker {
+ fn process_job(&self, job: &Job) -> FeedIndex {
+ let name = job.warc_path.split('/').last().unwrap().to_string();
+ info!("processing {}", name);
+
+ let mut index = FeedIndex::open(Path::new(&job.base_path).join(&name)).unwrap();
+
+ let w = &[job.warc_path.clone()];
+ let warc_files = download_all_warc_files(w, &job.source_config);
+ tokio::pin!(warc_files);
+
+ for file in warc_files.by_ref() {
+ for record in
+ file.records()
+ .flatten()
+ .filter(|record| match &record.response.payload_type {
+ Some(payload_type) => {
+ matches!(payload_type, PayloadType::Rss | PayloadType::Atom)
+ }
+ None => false,
+ })
+ {
+ let kind = match &record.response.payload_type {
+ Some(payload_type) => match payload_type {
+ PayloadType::Rss => FeedKind::Rss,
+ PayloadType::Atom => FeedKind::Atom,
+ _ => unreachable!(),
+ },
+ None => unreachable!(),
+ };
+
+ let url = Url::parse(&record.request.url);
+
+ if url.is_err() {
+ continue;
+ }
+
+ let feed = Feed {
+ url: url.unwrap(),
+ kind,
+ };
+
+ index.insert(&feed).unwrap();
+ }
+
+ index.commit().unwrap();
+ }
+ index.merge_into_max_segments(1).unwrap();
+
+ info!("{} done", name);
+
+ index
+ }
+}
+
+pub fn build(config: FeedIndexingConfig) -> Result<()> {
+ let warc_paths = config.warc_source.paths()?;
+
+ let job_config: WarcSource = config.warc_source.clone();
+
+ let worker = IndexingWorker {};
+
+ let indexes = warc_paths
+ .into_par_iter()
+ .skip(config.skip_warc_files.unwrap_or(0))
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
+ .map(|warc_path| Job {
+ source_config: job_config.clone(),
+ warc_path,
+ base_path: config.output_path.clone(),
+ })
+ .map(|job| {
+ let pointer: IndexPointer = job.map(&worker);
+ pointer
+ })
+ .collect();
+
+ merge(indexes)?;
+ Ok(())
+}
+
+pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
+ let num_indexes = indexes.len();
+ let mut it = indexes.into_iter();
+ let num_cores = num_cpus::get();
+
+ let mut threads = Vec::new();
+
+ for _ in 0..(num_cores + 1) {
+ let indexes = it
+ .by_ref()
+ .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)
+ .collect_vec();
+
+ if indexes.is_empty() {
+ break;
+ }
+
+ threads.push(thread::spawn(move || {
+ let mut it = indexes.into_iter();
+ let mut index = FeedIndex::open(it.next().unwrap().0).unwrap();
+
+ for other in it {
+ let other_path = other.0;
+ let other = FeedIndex::open(&other_path).unwrap();
+ index = index.merge(other);
+
+ std::fs::remove_dir_all(other_path).unwrap();
+ }
+
+ index.merge_into_max_segments(1).unwrap();
+
+ index
+ }));
+ }
+
+ let mut indexes = Vec::new();
+ for thread in threads {
+ indexes.push(thread.join().unwrap());
+ }
+
+ let mut it = indexes.into_iter();
+ let mut index = it.next().unwrap();
+
+ for other in it {
+ let other_path = other.path.clone();
+ index = index.merge(other);
+ std::fs::remove_dir_all(other_path).unwrap();
+ }
+
+ Ok(())
+}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index b1bfeda8..0f1f2840 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize};
use tokio::pin;
use tracing::{debug, info, trace, warn};
-use crate::config;
+use crate::config::{self, WarcSource};
use crate::entrypoint::download_all_warc_files;
-use crate::index::{FrozenIndex, Index};
+use crate::index::Index;
use crate::kv::rocksdb_store::RocksDbStore;
use crate::kv::Kv;
use crate::mapreduce::{Map, Reduce, Worker};
@@ -36,38 +36,9 @@ use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};
use crate::webpage::{safety_classifier, Html, Webpage};
use crate::{human_website_annotations, Result};
-pub struct Indexer {}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum JobConfig {
- Http(config::HttpConfig),
- Local(config::LocalConfig),
- S3(config::S3Config),
-}
-
-impl From<config::WarcSource> for JobConfig {
- fn from(value: config::WarcSource) -> Self {
- match value {
- config::WarcSource::HTTP(config) => JobConfig::Http(config),
- config::WarcSource::Local(config) => JobConfig::Local(config),
- config::WarcSource::S3(config) => JobConfig::S3(config),
- }
- }
-}
-
-impl From<JobConfig> for config::WarcSource {
- fn from(value: JobConfig) -> Self {
- match value {
- JobConfig::Http(config) => config::WarcSource::HTTP(config),
- JobConfig::Local(config) => config::WarcSource::Local(config),
- JobConfig::S3(config) => config::WarcSource::S3(config),
- }
- }
-}
-
#[derive(Debug, Serialize, Deserialize)]
pub struct Job {
- pub source_config: JobConfig,
+ pub source_config: config::WarcSource,
pub warc_paths: Vec<String>,
pub base_path: String,
pub settings: JobSettings,
@@ -214,6 +185,7 @@ impl IndexingWorker {
node_id: Some(host_node_id),
dmoz_description,
safety_classification: None,
+ inserted_at: Utc::now(),
};
if let Some(model) = self.safety_classifier.as_ref() {
@@ -240,9 +212,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
- let source: config::WarcSource = job.source_config.clone().into();
-
- let warc_files = download_all_warc_files(&job.warc_paths, &source);
+ let warc_files = download_all_warc_files(&job.warc_paths, &job.source_config);
pin!(warc_files);
for file in warc_files.by_ref() {
@@ -314,13 +284,6 @@ impl From<String> for IndexPointer {
impl Worker for IndexingWorker {}
-impl Map<IndexingWorker, FrozenIndex> for Job {
- fn map(&self, worker: &IndexingWorker) -> FrozenIndex {
- let index = process_job(self, worker);
- index.into()
- }
-}
-
impl Map<IndexingWorker, IndexPointer> for Job {
fn map(&self, worker: &IndexingWorker) -> IndexPointer {
let index = process_job(self, worker);
@@ -328,20 +291,6 @@ impl Map<IndexingWorker, IndexPointer> for Job {
}
}
-impl Reduce<FrozenIndex> for Index {
- fn reduce(self, element: FrozenIndex) -> Self {
- let other: Index = element.into();
-
- let other_path = other.path.clone();
-
- let res = self.merge(other);
-
- std::fs::remove_dir_all(other_path).unwrap();
-
- res
- }
-}
-
impl Reduce<Index> for Index {
fn reduce(self, element: Index) -> Self {
let other = element;
@@ -355,11 +304,12 @@ impl Reduce<Index> for Index {
}
}
+pub struct Indexer {}
impl Indexer {
pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
- let job_config: JobConfig = config.warc_source.clone().into();
+ let job_config: WarcSource = config.warc_source.clone();
let worker = IndexingWorker::new(
config.host_centrality_store_path.clone(),
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index a307e6ff..3db29cf7 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -25,6 +25,7 @@ pub mod configure;
pub mod crawler;
pub mod dmoz_parser;
mod entity;
+pub mod feed_indexer;
pub mod indexer;
pub mod safety_classifier;
pub mod search_server;
diff --git a/core/src/feed/index.rs b/core/src/feed/index.rs
index 4547f493..cdee6603 100644
--- a/core/src/feed/index.rs
+++ b/core/src/feed/index.rs
@@ -14,11 +14,19 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::Path;
+use std::{
+ fs,
+ path::{Path, PathBuf},
+};
-use crate::tokenizer::{SiteOperatorUrlTokenizer, Tokenizer};
+use crate::{
+ inverted_index::merge_tantivy_segments,
+ tokenizer::{SiteOperatorUrlTokenizer, Tokenizer},
+};
use anyhow::Result;
+use hashbrown::HashSet;
use tantivy::{
+ merge_policy::NoMergePolicy,
query::{PhraseQuery, TermQuery},
schema::{IndexRecordOption, TextFieldIndexing, TextOptions, Value},
tokenizer::Tokenizer as TantivyTokenizer,
@@ -28,17 +36,15 @@ use url::Url;
use super::{Feed, FeedKind};
pub struct FeedIndex {
+ tantivy_index: tantivy::Index,
writer: tantivy::IndexWriter,
reader: tantivy::IndexReader,
schema: tantivy::schema::Schema,
+ pub path: PathBuf,
}
impl FeedIndex {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- if !path.as_ref().exists() {
- std::fs::create_dir_all(path.as_ref())?;
- }
-
let url_tokenizer = Tokenizer::SiteOperator(SiteOperatorUrlTokenizer);
let kind_tokenizer = Tokenizer::default();
@@ -67,7 +73,13 @@ impl FeedIndex {
let schema = builder.build();
- let tv_index = tantivy::Index::create_in_dir(path, schema.clone())?;
+ let tv_index = if !path.as_ref().exists() {
+ fs::create_dir_all(path.as_ref())?;
+ tantivy::Index::create_in_dir(path.as_ref(), schema.clone())?
+ } else {
+ tantivy::Index::open_in_dir(path.as_ref())?
+ };
+
tv_index
.tokenizers()
.register(url_tokenizer.as_str(), url_tokenizer);
@@ -78,12 +90,17 @@ impl FeedIndex {
let writer = tv_index.writer(50_000_000)?;
+ let merge_policy = NoMergePolicy;
+ writer.set_merge_policy(Box::new(merge_policy));
+
let reader = tv_index.reader()?;
Ok(Self {
+ tantivy_index: tv_index,
writer,
reader,
schema,
+ path: path.as_ref().to_path_buf(),
})
}
@@ -170,6 +187,78 @@ impl FeedIndex {
Ok(res)
}
+
+ pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {
+ let segments: Vec<_> = self
+ .tantivy_index
+ .load_metas()?
+ .segments
+ .into_iter()
+ .collect();
+
+ merge_tantivy_segments(&mut self.writer, segments, &self.path, max_num_segments)?;
+
+ Ok(())
+ }
+
+ pub fn merge(mut self, mut other: Self) -> Self {
+ let path = self.path.clone();
+
+ {
+ other.commit().expect("failed to commit index");
+ self.commit().expect("failed to commit index");
+
+ let other_meta = other
+ .tantivy_index
+ .load_metas()
+ .expect("failed to load tantivy metadata for index");
+
+ let mut meta = self
+ .tantivy_index
+ .load_metas()
+ .expect("failed to load tantivy metadata for index");
+
+ let other_path = other.path.clone();
+ let other_path = other_path.as_path();
+ other.writer.wait_merging_threads().unwrap();
+
+ let path = self.path.clone();
+ let self_path = path.as_path();
+ self.writer.wait_merging_threads().unwrap();
+
+ let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();
+
+ for segment in other_meta.segments {
+ if ids.contains(&segment.id()) {
+ continue;
+ }
+
+ // TODO: handle case where current index has segment with same name
+ for file in segment.list_files() {
+ let p = other_path.join(&file);
+ if p.exists() {
+ fs::rename(p, self_path.join(&file)).unwrap();
+ }
+ }
+ meta.segments.push(segment);
+ }
+
+ meta.segments
+ .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));
+
+ fs::remove_dir_all(other_path).ok();
+
+ let self_path = Path::new(&path);
+
+ std::fs::write(
+ self_path.join("meta.json"),
+ serde_json::to_string_pretty(&meta).unwrap(),
+ )
+ .unwrap();
+ }
+
+ Self::open(path).expect("failed to open index")
+ }
}
#[cfg(test)]
diff --git a/core/src/index.rs b/core/src/index.rs
index 51572977..4e9ff281 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -20,13 +20,11 @@ use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::SystemTime;
-use serde::{Deserialize, Serialize};
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use url::Url;
use crate::collector::MainCollector;
-use crate::directory::{self, DirEntry};
use crate::inverted_index::{self, InvertedIndex};
use crate::query::Query;
use crate::search_ctx::Ctx;
@@ -167,43 +165,6 @@ impl Index {
}
}
-#[derive(Debug, Serialize, Deserialize)]
-pub struct FrozenIndex {
- pub root: DirEntry,
-}
-
-impl From<FrozenIndex> for Index {
- fn from(frozen: FrozenIndex) -> Self {
- let path = match &frozen.root {
- DirEntry::Folder { name, entries: _ } => name.clone(),
- DirEntry::File {
- name: _,
- content: _,
- } => {
- panic!("Cannot open index from a file - must be directory.")
- }
- };
-
- if Path::new(&path).exists() {
- fs::remove_dir_all(&path).unwrap();
- }
-
- directory::recreate_folder(&frozen.root).unwrap();
- Index::open(path).expect("failed to open index")
- }
-}
-
-impl From<Index> for FrozenIndex {
- fn from(mut index: Index) -> Self {
- index.commit().expect("failed to commit index");
- let path = index.path.clone();
- index.inverted_index.stop();
- let root = directory::scan_folder(path).unwrap();
-
- Self { root }
- }
-}
-
#[cfg(test)]
mod tests {
use crate::searcher::{LocalSearcher, SearchQuery};
@@ -212,54 +173,6 @@ mod tests {
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
- #[test]
- fn serialize_deserialize_bincode() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(
- Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- ),
- "https://www.example.com",
- )
- .unwrap(),
- )
- .expect("failed to insert webpage");
-
- index.commit().unwrap();
-
- let path = index.path.clone();
- let frozen: FrozenIndex = index.into();
- let bytes = bincode::serialize(&frozen).unwrap();
-
- std::fs::remove_dir_all(path).unwrap();
-
- let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();
- let index: Index = deserialized_frozen.into();
- let searcher = LocalSearcher::from(index);
-
- let result = searcher
- .search(&SearchQuery {
- query: "website".to_string(),
- ..Default::default()
- })
- .expect("Search failed");
-
- assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.example.com/");
- }
-
#[test]
fn bm25_all_docs() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index e765b287..323714ec 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -97,6 +97,58 @@ impl From<DocAddress> for tantivy::DocAddress {
}
}
+pub fn merge_tantivy_segments<P: AsRef<Path>>(
+ writer: &mut IndexWriter,
+ mut segments: Vec<SegmentMeta>,
+ base_path: P,
+ max_num_segments: u64,
+) -> Result<()> {
+ assert!(max_num_segments > 0);
+
+ if segments.len() <= max_num_segments as usize {
+ return Ok(());
+ }
+
+ let num_segments = (max_num_segments + 1) / 2; // ceil(num_segments/2)
+
+ let mut merge_segments = Vec::new();
+
+ for _ in 0..num_segments {
+ merge_segments.push(SegmentMergeCandidate {
+ num_docs: 0,
+ segments: Vec::new(),
+ });
+ }
+
+ segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
+
+ for segment in segments {
+ let best_candidate = merge_segments
+ .iter_mut()
+ .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
+ .unwrap();
+
+ best_candidate.num_docs += segment.num_docs();
+ best_candidate.segments.push(segment);
+ }
+
+ for merge in merge_segments
+ .into_iter()
+ .filter(|merge| !merge.segments.is_empty())
+ {
+ let segment_ids: Vec<_> = merge.segments.iter().map(|segment| segment.id()).collect();
+ writer.merge(&segment_ids[..]).wait()?;
+
+ for segment in merge.segments {
+ for file in segment.list_files() {
+ std::fs::remove_file(base_path.as_ref().join(file)).ok();
+ }
+ }
+ }
+
+ Ok(())
+}
+
struct SegmentMergeCandidate {
num_docs: u32,
segments: Vec<SegmentMeta>,
@@ -377,57 +429,16 @@ impl InvertedIndex {
Ok(webpages)
}
- pub fn merge_into_max_segments(&mut self, max_num_segments: u32) -> Result<()> {
- assert!(max_num_segments > 0);
-
- let mut segments: Vec<_> = self
+ pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {
+ let base_path = Path::new(&self.path);
+ let segments: Vec<_> = self
.tantivy_index
.load_metas()?
.segments
.into_iter()
.collect();
- if segments.len() <= max_num_segments as usize {
- return Ok(());
- }
-
- let num_segments = (max_num_segments + 1) / 2; // ceil(num_segments/2)
-
- let mut merge_segments = Vec::new();
-
- for _ in 0..num_segments {
- merge_segments.push(SegmentMergeCandidate {
- num_docs: 0,
- segments: Vec::new(),
- });
- }
-
- segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
-
- for segment in segments {
- let best_candidate = merge_segments
- .iter_mut()
- .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
- .unwrap();
-
- best_candidate.num_docs += segment.num_docs();
- best_candidate.segments.push(segment);
- }
-
- for merge in merge_segments
- .into_iter()
- .filter(|merge| !merge.segments.is_empty())
- {
- let segment_ids: Vec<_> = merge.segments.iter().map(|segment| segment.id()).collect();
- self.writer.merge(&segment_ids[..]).wait()?;
-
- let path = Path::new(&self.path);
- for segment in merge.segments {
- for file in segment.list_files() {
- std::fs::remove_file(path.join(file)).ok();
- }
- }
- }
+ merge_tantivy_segments(&mut self.writer, segments, base_path, max_num_segments)?;
Ok(())
}
@@ -458,8 +469,8 @@ impl InvertedIndex {
.load_metas()
.expect("failed to load tantivy metadata for index");
- let x = other.path.clone();
- let other_path = Path::new(x.as_str());
+ let other_path = other.path.clone();
+ let other_path = Path::new(other_path.as_str());
other.writer.wait_merging_threads().unwrap();
let path = self.path.clone();
@@ -953,12 +964,8 @@ mod tests {
.unwrap(),
backlink_labels: vec!["B site is great".to_string()],
host_centrality: 1.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -1492,4 +1499,102 @@ mod tests {
assert_eq!(webpage.title, "News website".to_string());
assert_eq!(webpage.url, "https://www.example.com/".to_string());
}
+
+ #[test]
+ fn insertion_time_deletion() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ let inserted_at = chrono::Utc::now() - chrono::Duration::days(4);
+ let cutoff = inserted_at + chrono::Duration::hours(1);
+
+ let a = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>News website</title>
+ </head>
+ <body>
+ {CONTENT} test
+ </body>
+ </html>
+ "#,
+ CONTENT = crate::rand_words(100)
+ ),
+ "https://www.a.com",
+ )
+ .unwrap(),
+ inserted_at,
+ ..Default::default()
+ };
+ let b = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>News website</title>
+ </head>
+ <body>
+ {CONTENT} test
+ </body>
+ </html>
+ "#,
+ CONTENT = crate::rand_words(100)
+ ),
+ "https://www.b.com",
+ )
+ .unwrap(),
+ inserted_at: chrono::Utc::now(),
+ ..Default::default()
+ };
+
+ index.insert(a).unwrap();
+ index.insert(b).unwrap();
+
+ index.commit().expect("failed to commit index");
+
+ let ctx = index.local_search_ctx();
+ let query = Query::parse(
+ &ctx,
+ &SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ },
+ &index,
+ )
+ .expect("Failed to parse query");
+
+ let ranker = Ranker::new(
+ SignalAggregator::new(Some(&query)),
+ ctx.fastfield_reader.clone(),
+ Default::default(),
+ );
+
+ let mut result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
+
+ assert_eq!(result.documents.len(), 2);
+
+ index
+ .delete_all_before(tantivy::DateTime::from_utc(
+ tantivy::time::OffsetDateTime::from_unix_timestamp(cutoff.timestamp()).unwrap(),
+ ))
+ .unwrap();
+ index.commit().expect("failed to commit index");
+
+ let ctx = index.local_search_ctx();
+
+ let ranker = Ranker::new(
+ SignalAggregator::new(Some(&query)),
+ ctx.fastfield_reader.clone(),
+ Default::default(),
+ );
+
+ result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
+
+ assert_eq!(result.documents.len(), 1);
+ }
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 395e6ad8..e7761ca0 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -40,7 +40,6 @@ mod bloom;
mod collector;
pub mod config;
pub mod crawler;
-mod directory;
mod distributed;
mod entity_index;
mod enum_map;
diff --git a/core/src/main.rs b/core/src/main.rs
index 5d0dc3c8..86a044b8 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -24,7 +24,6 @@ use stract::entrypoint::autosuggest_scrape::{self, Gl};
#[cfg(feature = "dev")]
use stract::entrypoint::configure;
-use stract::entrypoint::indexer::IndexPointer;
use stract::entrypoint::{self, api, safety_classifier, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
use tracing_subscriber::prelude::*;
@@ -116,7 +115,10 @@ enum Commands {
#[derive(Subcommand)]
enum LiveIndex {
+ /// Create a schedule of which feeds should go to which index.
Schedule { config_path: String },
+
+ /// Serve the live index.
Serve { config_path: String },
}
@@ -193,8 +195,8 @@ enum IndexingOptions {
output_path: String,
},
- /// Merge multiple search indexes into a single index.
- Merge { indexes: Vec<String> },
+ /// Create the feed index. Used to find feeds to put into the live index.
+ Feed { config_path: String },
}
fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
@@ -228,8 +230,9 @@ fn main() -> Result<()> {
wikipedia_dump_path,
output_path,
} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,
- IndexingOptions::Merge { indexes } => {
- entrypoint::Indexer::merge(indexes.into_iter().map(IndexPointer::from).collect())?
+ IndexingOptions::Feed { config_path } => {
+ let config = load_toml_config(config_path);
+ entrypoint::feed_indexer::build(config)?;
}
},
Commands::Centrality { mode } => {
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 4433ebae..55738d40 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -270,15 +270,8 @@ mod tests {
"https://www.a.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -300,15 +293,9 @@ mod tests {
"https://www.b.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.01,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -400,15 +387,8 @@ mod tests {
"https://www.a.com/this/is/a/pattern",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -429,15 +409,9 @@ mod tests {
"https://www.b.com/this/is/b/pattern",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.0001,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -511,15 +485,8 @@ mod tests {
"https://www.a.com/this/is/a/pattern",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -541,15 +508,9 @@ mod tests {
"https://www.b.com/this/is/b/pattern",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.0001,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -571,15 +532,9 @@ mod tests {
"https://www.c.com/this/is/c/pattern",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.0001,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -678,15 +633,9 @@ mod tests {
"https://www.a.com/this/is/a/pattern",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- dmoz_description: None,
- safety_classification: None,
node_id: Some(Node::from("www.a.com").into_host().id()),
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -708,15 +657,10 @@ mod tests {
"https://www.b.com/this/is/b/pattern",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.0001,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- dmoz_description: None,
- safety_classification: None,
node_id: Some(Node::from("www.b.com").into_host().id()),
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -738,15 +682,10 @@ mod tests {
"https://www.c.com/this/is/c/pattern",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.0002,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- dmoz_description: None,
- safety_classification: None,
node_id: Some(Node::from("www.c.com").into_host().id()),
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -815,15 +754,8 @@ mod tests {
"https://www.a.com/",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -859,15 +791,9 @@ mod tests {
),
"https://www.b.com/",
).unwrap(),
- backlink_labels: vec![],
host_centrality: 0.0001,
- page_centrality: 0.0,
-
- pre_computed_score: 0.0,
fetch_time_ms: 500,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -995,15 +921,8 @@ mod tests {
"https://chat.stackoverflow.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -1071,15 +990,8 @@ mod tests {
"https://a.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -1102,15 +1014,8 @@ mod tests {
"https://b.com/",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -1158,15 +1063,8 @@ mod tests {
"https://a.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1218,15 +1116,8 @@ mod tests {
"https://example.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1381,15 +1272,8 @@ mod tests {
"https://example.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1456,15 +1340,8 @@ mod tests {
"https://example.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1518,15 +1395,8 @@ mod tests {
"https://example.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -1549,15 +1419,8 @@ mod tests {
"https://example.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1649,15 +1512,8 @@ mod tests {
"https://example.com/",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
};
page.html.set_clean_text("".to_string());
@@ -1729,15 +1585,8 @@ mod tests {
"#,
"https://example.com/",
).unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
};
page.html.set_clean_text("".to_string());
@@ -1759,15 +1608,8 @@ mod tests {
"https://non-indie-example.com/",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
};
page.html.set_clean_text("".to_string());
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index dd49abea..13571032 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -294,15 +294,9 @@ mod tests {
"https://e.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
node_id: Some(Node::from("e.com").id()),
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -324,15 +318,10 @@ mod tests {
"https://d.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.01,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
node_id: Some(Node::from("d.com").id()),
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 01a9a5df..2c60ec63 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -167,15 +167,8 @@ mod tests {
"https://www.a.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -197,15 +190,9 @@ mod tests {
"https://www.b.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 5.0,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -246,15 +233,8 @@ mod tests {
"https://www.a.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -277,14 +257,9 @@ mod tests {
)
.unwrap(),
backlink_labels: vec![],
- host_centrality: 0.0,
fetch_time_ms: 500,
page_centrality: 5.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -324,15 +299,9 @@ mod tests {
"https://www.old.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 499,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -354,15 +323,9 @@ mod tests {
"https://www.new.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -398,15 +361,8 @@ mod tests {
"https://www.first.com",
)
.unwrap(),
- backlink_labels: vec![],
- host_centrality: 0.0,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -436,15 +392,9 @@ mod tests {
</html>
"#,
"https://www.second.com").unwrap(),
- backlink_labels: vec![],
host_centrality: 0.00003,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -482,14 +432,8 @@ mod tests {
)
.unwrap(),
backlink_labels: vec!["test this is the best test site".to_string()],
- host_centrality: 0.0,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -508,15 +452,9 @@ mod tests {
"https://www.second.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 0.00003,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -553,15 +491,9 @@ mod tests {
"https://www.body.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -581,15 +513,9 @@ mod tests {
"https://www.title.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -610,15 +536,9 @@ mod tests {
"https://www.centrality.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.02,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
@@ -687,15 +607,8 @@ mod tests {
"https://www.first.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
- fetch_time_ms: 0,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -717,15 +630,9 @@ mod tests {
"https://www.second.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 5000,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -766,15 +673,9 @@ mod tests {
"https://www.first.com/one",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 2,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -796,15 +697,9 @@ mod tests {
"https://www.second.com/one/two",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 1,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -826,15 +721,8 @@ mod tests {
"https://www.third.com/one/two123",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
- fetch_time_ms: 0,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index e9f18f76..ba051aeb 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -100,15 +100,10 @@ mod tests {
"https://www.first.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- dmoz_description: None,
node_id: Some(Node::from("https://www.first.com").into_host().id()),
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -130,15 +125,10 @@ mod tests {
"https://www.second.com",
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: 1.0,
fetch_time_ms: 499,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- dmoz_description: None,
node_id: Some(Node::from("https://www.second.com").into_host().id()),
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index
@@ -162,13 +152,8 @@ mod tests {
.unwrap(),
backlink_labels: vec![],
host_centrality: 1.0,
- fetch_time_ms: 0,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
-
- dmoz_description: None,
node_id: Some(Node::from("https://www.third.com").into_host().id()),
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/core/src/schema.rs b/core/src/schema.rs
index b9b653e2..88f236a2 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -598,6 +598,7 @@ impl Field {
| Field::Text(TextField::TitleIfHomepage)
| Field::Text(TextField::SiteWithout) // will match url
| Field::Text(TextField::Domain) // will match url
+ | Field::Text(TextField::InsertionTimestamp)
) && !self.is_fast()
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 3b3eb244..38d41d9a 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -452,14 +452,9 @@ mod tests {
&format!("https://www.{i}.com"),
)
.unwrap(),
- backlink_labels: vec![],
host_centrality: (NUM_WEBSITES - i) as f64,
fetch_time_ms: 500,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
+ ..Default::default()
})
.expect("failed to insert webpage");
}
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 9e0094d6..4c1b6af0 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -253,7 +253,9 @@ impl FromStr for PayloadType {
"text/html" => Ok(Self::Html),
"application/pdf" => Ok(Self::Pdf),
"application/rss" => Ok(Self::Rss),
+ "application/rss+xml" => Ok(Self::Rss),
"application/atom" => Ok(Self::Atom),
+ "application/atom+xml" => Ok(Self::Atom),
_ => Err(Error::WarcParse("Unknown payload type")),
}
}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index d0cdd145..a0690f3a 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -128,6 +128,32 @@ pub struct Webpage {
pub node_id: Option<NodeID>,
pub dmoz_description: Option<String>,
pub safety_classification: Option<safety_classifier::Label>,
+ pub inserted_at: DateTime<Utc>,
+}
+
+#[cfg(test)]
+impl Default for Webpage {
+ fn default() -> Self {
+ Self {
+ html: Html {
+ url: Url::parse("https://example.com").expect("Failed to parse url"),
+ root: kuchiki::parse_html().one("<html></html>"),
+ all_text: None,
+ clean_text: None,
+ lang: None,
+ robots: None,
+ },
+ backlink_labels: Default::default(),
+ host_centrality: Default::default(),
+ page_centrality: Default::default(),
+ fetch_time_ms: Default::default(),
+ pre_computed_score: Default::default(),
+ node_id: Default::default(),
+ dmoz_description: Default::default(),
+ safety_classification: Default::default(),
+ inserted_at: Utc::now(),
+ }
+ }
}
impl Webpage {
@@ -145,6 +171,7 @@ impl Webpage {
node_id: None,
dmoz_description: None,
safety_classification: None,
+ inserted_at: Utc::now(),
})
}
@@ -197,6 +224,15 @@ impl Webpage {
backlink_text,
);
+ doc.add_date(
+ schema
+ .get_field(Field::Text(TextField::InsertionTimestamp).name())
+ .expect("Failed to get insertion-timestamp field"),
+ tantivy::DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
+ self.inserted_at.timestamp(),
+ )?),
+ );
+
let safety = self
.safety_classification
.map(|label| label.to_string())
@@ -1197,12 +1233,6 @@ impl Html {
Field::Text(TextField::MicroformatTags) => {
doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
}
- Field::Text(TextField::InsertionTimestamp) => {
- doc.add_date(
- tantivy_field,
- tantivy::DateTime::from_utc(OffsetDateTime::now_utc()),
- );
- }
Field::Fast(FastField::IsHomepage) => {
doc.add_u64(tantivy_field, (self.is_homepage()).into());
}
@@ -1304,6 +1334,7 @@ impl Html {
}
Field::Text(TextField::BacklinkText)
| Field::Text(TextField::SafetyClassification)
+ | Field::Text(TextField::InsertionTimestamp)
| Field::Fast(FastField::HostCentrality)
| Field::Fast(FastField::PageCentrality)
| Field::Fast(FastField::FetchTimeMs)
@@ -2199,14 +2230,9 @@ mod tests {
let webpage = Webpage {
html,
- backlink_labels: Vec::new(),
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
- node_id: None,
dmoz_description: Some("dmoz description".to_string()),
- safety_classification: None,
+ ..Default::default()
};
assert_eq!(
@@ -2234,14 +2260,9 @@ mod tests {
.unwrap();
let webpage = Webpage {
html,
- backlink_labels: Vec::new(),
- host_centrality: 0.0,
- page_centrality: 0.0,
fetch_time_ms: 500,
- pre_computed_score: 0.0,
- node_id: None,
dmoz_description: Some("dmoz description".to_string()),
- safety_classification: None,
+ ..Default::default()
};
assert_eq!(webpage.dmoz_description(), None)