web search done right
use redb in live-index downloaded db. to truncate the database, we would have to implement deletes and possibly also some kind of auto merging strategy in speedy_kv. to keep things simple, we use redb for this db instead.
Mikkel Denkerc43283cc5d0b
diff --git a/Cargo.lock b/Cargo.lockindex 12dc0a03..73e51ea3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3824,6 +3824,15 @@ version = "0.5.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"+[[package]]+name = "redb"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1100a056c5dcdd4e5513d5333385223b26ef1bf92f31eb38f407e8c20549256"+dependencies = [+ "libc",+]+[[package]]name = "redox_syscall"version = "0.4.1"@@ -4683,6 +4692,7 @@ dependencies = ["quick-xml 0.30.0","rand 0.8.5","rayon",+ "redb","regex","reqwest","ring",diff --git a/Cargo.toml b/Cargo.tomlindex 7f158123..3262452b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -91,6 +91,7 @@ publicsuffix = "2.2.3"quick-xml = "0.30.0"rand = "0.8.5"rayon = "1.5.3"+redb = "2.0.0"regex = "1.6.0"reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }rkyv = { version = "0.7.44", features = ["validation", "bytecheck"] }diff --git a/assets/licenses.html b/assets/licenses.htmlindex 39e6c2ae..2970eec3 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,7 +44,7 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (396)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (397)</li><li><a href="#MIT">MIT License</a> (192)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (7)</li>@@ -2788,6 +2788,7 @@<li><a href=" https://github.com/clap-rs/clap/tree/master/clap_derive ">clap_derive 4.5.0</a></li><li><a href=" https://github.com/clap-rs/clap/tree/master/clap_lex ">clap_lex 0.7.0</a></li><li><a href=" https://github.com/Narsil/esaxx-rs ">esaxx-rs 0.1.10</a></li>+ <li><a href=" https://github.com/cberner/redb ">redb 2.0.0</a></li><li><a href=" https://github.com/Folyd/robotstxt ">robotstxt-with-cache 0.4.0</a></li><li><a href=" https://github.com/huggingface/safetensors ">safetensors 0.4.2</a></li><li><a href=" https://github.com/huggingface/spm_precompiled ">spm_precompiled 0.1.4</a></li>diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex c31ce589..e6de92c9 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -80,6 +80,7 @@ postcard = { workspace = true }quick-xml = { workspace = true }rand = { workspace = true }rayon = { workspace = true }+redb = { workspace = true }regex = { workspace = true }reqwest = { workspace = true }rkyv = { workspace = true }diff --git a/crates/core/src/live_index/crawler.rs b/crates/core/src/live_index/crawler.rsindex 379108ae..b389d7f9 100644--- a/crates/core/src/live_index/crawler.rs+++ b/crates/core/src/live_index/crawler.rs@@ -108,7 +108,7 @@ impl Crawler {let feed = feed::parse(&content, feed.kind)?;for url in feed.links {- if !self.downloaded_db.has_downloaded(&url)+ if !self.downloaded_db.has_downloaded(&url)?&& Domain::from(&url) == domain_feeds.domain{urls.push(url);diff --git a/crates/core/src/live_index/downloaded_db.rs b/crates/core/src/live_index/downloaded_db.rsindex cabb84c0..69a51d37 100644--- a/crates/core/src/live_index/downloaded_db.rs+++ b/crates/core/src/live_index/downloaded_db.rs@@ -14,41 +14,301 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/+use redb::ReadableTable;use url::Url;-use crate::{speedy_kv, Result};-use std::path::Path;+use crate::Result;+use std::{path::Path, time::Duration};++use super::TTL;++#[derive(Debug, Clone)]+struct InsertionTime {+ time: chrono::DateTime<chrono::Utc>,+}++#[derive(Debug, Clone)]+struct TruncatedUrl(String);++impl TruncatedUrl {+ fn new(url: &Url) -> Self {+ let mut url = url.clone().to_string();++ // redb has a hard limit of 4gb per value+ // and is better suited for smaller values.+ // we therefore truncate to ~8kb+ if url.len() > 8192 {+ url = url.chars().take(8192).collect();+ }++ Self(url)+ }+}++impl redb::Key for TruncatedUrl {+ fn compare(a: &[u8], b: &[u8]) -> std::cmp::Ordering {+ a.cmp(b)+ }+}++impl redb::Value for TruncatedUrl {+ type SelfType<'a> = TruncatedUrl+ where+ Self: 'a;++ type AsBytes<'a> = &'a [u8]+ where+ Self: 'a;++ fn fixed_width() -> Option<usize> {+ None+ }++ fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>+ where+ Self: 'a,+ {+ let s = std::str::from_utf8(data).unwrap();+ Self(s.to_string())+ }++ fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>+ where+ Self: 'a,+ Self: 'b,+ {+ value.0.as_bytes()+ }++ fn type_name() -> redb::TypeName {+ redb::TypeName::new("TruncatedUrl")+ }+}++impl redb::Key for InsertionTime {+ fn compare(a: &[u8], b: &[u8]) -> std::cmp::Ordering {+ a.cmp(b)+ }+}++impl redb::Value for InsertionTime {+ type SelfType<'a> = InsertionTime+ where+ Self: 'a;++ type AsBytes<'a> = Vec<u8>+ where+ Self: 'a;++ fn fixed_width() -> Option<usize> {+ None+ }++ fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>+ where+ Self: 'a,+ {+ let time = chrono::DateTime::parse_from_rfc3339(std::str::from_utf8(data).unwrap())+ .unwrap()+ .with_timezone(&chrono::Utc);++ Self { time }+ }++ fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>+ where+ Self: 'a,+ Self: 'b,+ {+ let s = value.time.to_rfc3339();+ s.as_bytes().to_vec()+ }++ fn type_name() -> redb::TypeName {+ redb::TypeName::new("InsertionTime")+ }+}++struct InnerDb {+ db: redb::Database,+ last_truncate: chrono::DateTime<chrono::Utc>,+}++impl InnerDb {+ fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let path = path.as_ref().join("downloaded.db");++ let db = if !path.exists() {+ if let Some(parent) = path.parent() {+ if !parent.exists() {+ std::fs::create_dir_all(parent)?;+ }+ }++ redb::Database::create(&path)?+ } else {+ redb::Database::open(&path)?+ };++ // make sure the tables exist+ let txn = db.begin_write()?;++ txn.open_table(Self::urls_table_definition())?;+ txn.open_table(Self::times_table_definition())?;++ txn.commit()?;++ Ok(Self {+ db,+ last_truncate: chrono::Utc::now(),+ })+ }++ fn urls_table_definition() -> redb::TableDefinition<'static, TruncatedUrl, ()> {+ redb::TableDefinition::new("urls")+ }++ fn times_table_definition() -> redb::TableDefinition<'static, InsertionTime, TruncatedUrl> {+ redb::TableDefinition::new("times")+ }++ fn has_downloaded(&self, url: &Url) -> Result<bool> {+ let url = TruncatedUrl::new(url);++ Ok(self+ .db+ .begin_read()?+ .open_table(Self::urls_table_definition())?+ .get(&url)?+ .is_some())+ }++ fn truncate(&mut self, ttl: Duration) -> Result<()> {+ let now = chrono::Utc::now();+ let thresh = now - ttl;++ let thresh = InsertionTime { time: thresh };+ let txn = self.db.begin_write()?;++ let to_remove = {+ let times = txn.open_table(Self::times_table_definition())?;++ let to_remove: Vec<_> = times+ .range(..thresh)?+ .map(|r| {+ let (time, url) = r.unwrap();++ (time.value().clone(), url.value().clone())+ })+ .collect();++ to_remove+ };++ {+ let mut times = txn.open_table(Self::times_table_definition())?;+ let mut urls = txn.open_table(Self::urls_table_definition())?;++ for (time, url) in to_remove {+ times.remove(time)?;+ urls.remove(url)?;+ }+ }++ txn.commit()?;++ self.last_truncate = chrono::Utc::now();++ Ok(())+ }++ fn maybe_truncate(&mut self) -> Result<()> {+ let now = chrono::Utc::now();++ if self.last_truncate.signed_duration_since(now).num_seconds() < 60 {+ return Ok(());+ }++ self.truncate(TTL)+ }++ pub fn insert(&mut self, url: &Url) -> Result<()> {+ {+ let key = TruncatedUrl::new(url);+ let txn = self.db.begin_write()?;+ {+ let mut table = txn.open_table(Self::urls_table_definition())?;+ table.insert(&key, ())?;+ }++ let time = InsertionTime {+ time: chrono::Utc::now(),+ };+ {+ let mut table = txn.open_table(Self::times_table_definition())?;+ table.insert(time, key)?;+ }++ txn.commit()?;+ }++ self.maybe_truncate()?;++ Ok(())+ }+}pub struct DownloadedDb {- db: std::sync::Mutex<speedy_kv::Db<Url, ()>>,+ inner: std::sync::Mutex<InnerDb>,}impl DownloadedDb {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let db = speedy_kv::Db::open_or_create(path)?;+ let db = InnerDb::open(path)?;Ok(Self {- db: std::sync::Mutex::new(db),+ inner: std::sync::Mutex::new(db),})}- pub fn has_downloaded(&self, url: &Url) -> bool {- let key = url.as_str().as_bytes();- self.db.lock().unwrap().get_raw_with_live(key).is_some()+ pub fn has_downloaded(&self, url: &Url) -> Result<bool> {+ self.inner.lock().unwrap().has_downloaded(url)}pub fn insert(&self, url: &Url) -> Result<()> {- let key = url.as_str().as_bytes().to_vec();- let mut db = self.db.lock().unwrap();+ self.inner.lock().unwrap().insert(url)+ }+}- db.insert_raw(key, vec![]);+#[cfg(test)]+mod tests {+ use super::*;- if db.uncommitted_inserts() > 1_000_000 {- db.commit()?;+ #[test]+ fn test_downloaded_db() {+ let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();- // TODO: Truncate the database using a ttl- }+ let url = Url::parse("https://example.com").unwrap();+ assert!(!db.has_downloaded(&url).unwrap());- Ok(())+ db.insert(&url).unwrap();+ assert!(db.has_downloaded(&url).unwrap());+ }++ #[test]+ fn test_truncate_ttl() {+ let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();++ let url = Url::parse("https://example.com").unwrap();+ db.insert(&url).unwrap();++ assert!(db.has_downloaded(&url).unwrap());++ let ttl = Duration::from_secs(1);+ std::thread::sleep(ttl + Duration::from_secs(1));++ db.inner.lock().unwrap().truncate(ttl).unwrap();++ assert!(!db.has_downloaded(&url).unwrap());}}diff --git a/crates/core/src/speedy_kv/blob_id_index.rs b/crates/core/src/speedy_kv/blob_id_index.rsindex 12ef87fa..2f7e6143 100644--- a/crates/core/src/speedy_kv/blob_id_index.rs+++ b/crates/core/src/speedy_kv/blob_id_index.rs@@ -19,10 +19,11 @@ use fst::{IntoStreamer, Streamer};use crate::Result;use std::{io::{BufWriter, Write},+ ops::RangeBounds,path::{Path, PathBuf},};-use super::{BlobId, Serialized};+use super::{BlobId, Serialized, SerializedRef};pub struct BlobIdIndex<K> {path: PathBuf,@@ -69,6 +70,27 @@ impl<K> BlobIdIndex<K> {BlobIdIndexIter::new(self.fst.search(automaton).into_stream())}+ pub fn range<'a, R>(&'a self, range: R) -> impl Iterator<Item = (Serialized<K>, BlobId)> + 'a+ where+ R: RangeBounds<SerializedRef<'a, K>>,+ {+ let builder = self.fst.range();++ let builder = match range.start_bound() {+ std::ops::Bound::Included(s) => builder.ge(s),+ std::ops::Bound::Excluded(s) => builder.gt(s),+ std::ops::Bound::Unbounded => builder,+ };++ let builder = match range.end_bound() {+ std::ops::Bound::Included(e) => builder.le(e),+ std::ops::Bound::Excluded(e) => builder.lt(e),+ std::ops::Bound::Unbounded => builder,+ };++ BlobIdIndexIter::new(builder.into_stream())+ }+pub fn path(&self) -> &Path {&self.path}diff --git a/crates/core/src/speedy_kv/mod.rs b/crates/core/src/speedy_kv/mod.rsindex e4efcb38..bc8703b3 100644--- a/crates/core/src/speedy_kv/mod.rs+++ b/crates/core/src/speedy_kv/mod.rs@@ -28,7 +28,7 @@use crate::Result;use std::{collections::BTreeMap,- ops::Range,+ ops::{Range, RangeBounds},path::{Path, PathBuf},};@@ -331,7 +331,7 @@ impl<K, V> Db<K, V> {pub fn search_raw<'a, A>(&'a self,query: A,- ) -> impl Iterator<Item = (Serialized<K>, Serialized<V>)> + 'a+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'awhereA: fst::Automaton + Clone + 'a,{@@ -341,6 +341,19 @@ impl<K, V> Db<K, V> {.flat_map(move |segment| segment.search_raw(query.clone()))}+ pub fn range_raw<'a, R>(+ &'a self,+ range: R,+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'a+ where+ R: RangeBounds<SerializedRef<'a, K>> + Clone + 'a,+ {+ self.segments+ .iter()+ .rev()+ .flat_map(move |segment| segment.range_raw(range.clone()))+ }+pub fn commit(&mut self) -> Result<()> {if self.live_segment.db.is_empty() {return Ok(());diff --git a/crates/core/src/speedy_kv/segment.rs b/crates/core/src/speedy_kv/segment.rsindex 94e9d100..25f26b26 100644--- a/crates/core/src/speedy_kv/segment.rs+++ b/crates/core/src/speedy_kv/segment.rs@@ -16,6 +16,7 @@use std::{io::{BufWriter, Write},+ ops::RangeBounds,path::{Path, PathBuf},};@@ -221,16 +222,30 @@ impl<K, V> Segment<K, V> {pub fn search_raw<'a, A>(&'a self,query: A,- ) -> impl Iterator<Item = (Serialized<K>, Serialized<V>)> + 'a+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'awhereA: fst::Automaton + 'a,{- self.id_index.search(query).map(move |(ser_k, id)| {+ self.id_index.search(query).map(move |(_, id)| {let ptr = self.blob_index.get(id);let blob = self.store.get_raw(&ptr).unwrap();- let value = Serialized::from(blob.value.as_bytes().to_vec());- (ser_k, value)+ (blob.key, blob.value)+ })+ }++ pub fn range_raw<'a, R>(+ &'a self,+ range: R,+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'a+ where+ R: RangeBounds<SerializedRef<'a, K>>,+ {+ self.id_index.range(range).map(move |(_, id)| {+ let ptr = self.blob_index.get(id);+ let blob = self.store.get_raw(&ptr).unwrap();++ (blob.key, blob.value)})}