Skip to content

web search done right

use redb in live-index downloaded db. to truncate the database, we would have to implement deletes and possibly also some kind of auto merging strategy in speedy_kv. to keep things simple, we use redb for this db instead.

Mikkel Denkerc43283cc5d0b
diff --git a/Cargo.lock b/Cargo.lock
index 12dc0a03..73e51ea3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3824,6 +3824,15 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
+[[package]]
+name = "redb"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1100a056c5dcdd4e5513d5333385223b26ef1bf92f31eb38f407e8c20549256"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "redox_syscall"
version = "0.4.1"
@@ -4683,6 +4692,7 @@ dependencies = [
"quick-xml 0.30.0",
"rand 0.8.5",
"rayon",
+ "redb",
"regex",
"reqwest",
"ring",
diff --git a/Cargo.toml b/Cargo.toml
index 7f158123..3262452b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -91,6 +91,7 @@ publicsuffix = "2.2.3"
quick-xml = "0.30.0"
rand = "0.8.5"
rayon = "1.5.3"
+redb = "2.0.0"
regex = "1.6.0"
reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
rkyv = { version = "0.7.44", features = ["validation", "bytecheck"] }
diff --git a/assets/licenses.html b/assets/licenses.html
index 39e6c2ae..2970eec3 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,7 +44,7 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (396)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (397)</li>
<li><a href="#MIT">MIT License</a> (192)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (7)</li>
@@ -2788,6 +2788,7 @@
<li><a href=" https://github.com/clap-rs/clap/tree/master/clap_derive ">clap_derive 4.5.0</a></li>
<li><a href=" https://github.com/clap-rs/clap/tree/master/clap_lex ">clap_lex 0.7.0</a></li>
<li><a href=" https://github.com/Narsil/esaxx-rs ">esaxx-rs 0.1.10</a></li>
+ <li><a href=" https://github.com/cberner/redb ">redb 2.0.0</a></li>
<li><a href=" https://github.com/Folyd/robotstxt ">robotstxt-with-cache 0.4.0</a></li>
<li><a href=" https://github.com/huggingface/safetensors ">safetensors 0.4.2</a></li>
<li><a href=" https://github.com/huggingface/spm_precompiled ">spm_precompiled 0.1.4</a></li>
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index c31ce589..e6de92c9 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -80,6 +80,7 @@ postcard = { workspace = true }
quick-xml = { workspace = true }
rand = { workspace = true }
rayon = { workspace = true }
+redb = { workspace = true }
regex = { workspace = true }
reqwest = { workspace = true }
rkyv = { workspace = true }
diff --git a/crates/core/src/live_index/crawler.rs b/crates/core/src/live_index/crawler.rs
index 379108ae..b389d7f9 100644
--- a/crates/core/src/live_index/crawler.rs
+++ b/crates/core/src/live_index/crawler.rs
@@ -108,7 +108,7 @@ impl Crawler {
let feed = feed::parse(&content, feed.kind)?;
for url in feed.links {
- if !self.downloaded_db.has_downloaded(&url)
+ if !self.downloaded_db.has_downloaded(&url)?
&& Domain::from(&url) == domain_feeds.domain
{
urls.push(url);
diff --git a/crates/core/src/live_index/downloaded_db.rs b/crates/core/src/live_index/downloaded_db.rs
index cabb84c0..69a51d37 100644
--- a/crates/core/src/live_index/downloaded_db.rs
+++ b/crates/core/src/live_index/downloaded_db.rs
@@ -14,41 +14,301 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/
+use redb::ReadableTable;
use url::Url;
-use crate::{speedy_kv, Result};
-use std::path::Path;
+use crate::Result;
+use std::{path::Path, time::Duration};
+
+use super::TTL;
+
+#[derive(Debug, Clone)]
+struct InsertionTime {
+ time: chrono::DateTime<chrono::Utc>,
+}
+
+#[derive(Debug, Clone)]
+struct TruncatedUrl(String);
+
+impl TruncatedUrl {
+ fn new(url: &Url) -> Self {
+ let mut url = url.clone().to_string();
+
+ // redb has a hard limit of 4gb per value
+ // and is better suited for smaller values.
+ // we therefore truncate to ~8kb
+ if url.len() > 8192 {
+ url = url.chars().take(8192).collect();
+ }
+
+ Self(url)
+ }
+}
+
+impl redb::Key for TruncatedUrl {
+ fn compare(a: &[u8], b: &[u8]) -> std::cmp::Ordering {
+ a.cmp(b)
+ }
+}
+
+impl redb::Value for TruncatedUrl {
+ type SelfType<'a> = TruncatedUrl
+ where
+ Self: 'a;
+
+ type AsBytes<'a> = &'a [u8]
+ where
+ Self: 'a;
+
+ fn fixed_width() -> Option<usize> {
+ None
+ }
+
+ fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+ where
+ Self: 'a,
+ {
+ let s = std::str::from_utf8(data).unwrap();
+ Self(s.to_string())
+ }
+
+ fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+ where
+ Self: 'a,
+ Self: 'b,
+ {
+ value.0.as_bytes()
+ }
+
+ fn type_name() -> redb::TypeName {
+ redb::TypeName::new("TruncatedUrl")
+ }
+}
+
+impl redb::Key for InsertionTime {
+ fn compare(a: &[u8], b: &[u8]) -> std::cmp::Ordering {
+ a.cmp(b)
+ }
+}
+
+impl redb::Value for InsertionTime {
+ type SelfType<'a> = InsertionTime
+ where
+ Self: 'a;
+
+ type AsBytes<'a> = Vec<u8>
+ where
+ Self: 'a;
+
+ fn fixed_width() -> Option<usize> {
+ None
+ }
+
+ fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+ where
+ Self: 'a,
+ {
+ let time = chrono::DateTime::parse_from_rfc3339(std::str::from_utf8(data).unwrap())
+ .unwrap()
+ .with_timezone(&chrono::Utc);
+
+ Self { time }
+ }
+
+ fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
+ where
+ Self: 'a,
+ Self: 'b,
+ {
+ let s = value.time.to_rfc3339();
+ s.as_bytes().to_vec()
+ }
+
+ fn type_name() -> redb::TypeName {
+ redb::TypeName::new("InsertionTime")
+ }
+}
+
+struct InnerDb {
+ db: redb::Database,
+ last_truncate: chrono::DateTime<chrono::Utc>,
+}
+
+impl InnerDb {
+ fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let path = path.as_ref().join("downloaded.db");
+
+ let db = if !path.exists() {
+ if let Some(parent) = path.parent() {
+ if !parent.exists() {
+ std::fs::create_dir_all(parent)?;
+ }
+ }
+
+ redb::Database::create(&path)?
+ } else {
+ redb::Database::open(&path)?
+ };
+
+ // make sure the tables exist
+ let txn = db.begin_write()?;
+
+ txn.open_table(Self::urls_table_definition())?;
+ txn.open_table(Self::times_table_definition())?;
+
+ txn.commit()?;
+
+ Ok(Self {
+ db,
+ last_truncate: chrono::Utc::now(),
+ })
+ }
+
+ fn urls_table_definition() -> redb::TableDefinition<'static, TruncatedUrl, ()> {
+ redb::TableDefinition::new("urls")
+ }
+
+ fn times_table_definition() -> redb::TableDefinition<'static, InsertionTime, TruncatedUrl> {
+ redb::TableDefinition::new("times")
+ }
+
+ fn has_downloaded(&self, url: &Url) -> Result<bool> {
+ let url = TruncatedUrl::new(url);
+
+ Ok(self
+ .db
+ .begin_read()?
+ .open_table(Self::urls_table_definition())?
+ .get(&url)?
+ .is_some())
+ }
+
+ fn truncate(&mut self, ttl: Duration) -> Result<()> {
+ let now = chrono::Utc::now();
+ let thresh = now - ttl;
+
+ let thresh = InsertionTime { time: thresh };
+ let txn = self.db.begin_write()?;
+
+ let to_remove = {
+ let times = txn.open_table(Self::times_table_definition())?;
+
+ let to_remove: Vec<_> = times
+ .range(..thresh)?
+ .map(|r| {
+ let (time, url) = r.unwrap();
+
+ (time.value().clone(), url.value().clone())
+ })
+ .collect();
+
+ to_remove
+ };
+
+ {
+ let mut times = txn.open_table(Self::times_table_definition())?;
+ let mut urls = txn.open_table(Self::urls_table_definition())?;
+
+ for (time, url) in to_remove {
+ times.remove(time)?;
+ urls.remove(url)?;
+ }
+ }
+
+ txn.commit()?;
+
+ self.last_truncate = chrono::Utc::now();
+
+ Ok(())
+ }
+
+ fn maybe_truncate(&mut self) -> Result<()> {
+ let now = chrono::Utc::now();
+
+ if self.last_truncate.signed_duration_since(now).num_seconds() < 60 {
+ return Ok(());
+ }
+
+ self.truncate(TTL)
+ }
+
+ pub fn insert(&mut self, url: &Url) -> Result<()> {
+ {
+ let key = TruncatedUrl::new(url);
+ let txn = self.db.begin_write()?;
+ {
+ let mut table = txn.open_table(Self::urls_table_definition())?;
+ table.insert(&key, ())?;
+ }
+
+ let time = InsertionTime {
+ time: chrono::Utc::now(),
+ };
+ {
+ let mut table = txn.open_table(Self::times_table_definition())?;
+ table.insert(time, key)?;
+ }
+
+ txn.commit()?;
+ }
+
+ self.maybe_truncate()?;
+
+ Ok(())
+ }
+}
pub struct DownloadedDb {
- db: std::sync::Mutex<speedy_kv::Db<Url, ()>>,
+ inner: std::sync::Mutex<InnerDb>,
}
impl DownloadedDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let db = speedy_kv::Db::open_or_create(path)?;
+ let db = InnerDb::open(path)?;
Ok(Self {
- db: std::sync::Mutex::new(db),
+ inner: std::sync::Mutex::new(db),
})
}
- pub fn has_downloaded(&self, url: &Url) -> bool {
- let key = url.as_str().as_bytes();
- self.db.lock().unwrap().get_raw_with_live(key).is_some()
+ pub fn has_downloaded(&self, url: &Url) -> Result<bool> {
+ self.inner.lock().unwrap().has_downloaded(url)
}
pub fn insert(&self, url: &Url) -> Result<()> {
- let key = url.as_str().as_bytes().to_vec();
- let mut db = self.db.lock().unwrap();
+ self.inner.lock().unwrap().insert(url)
+ }
+}
- db.insert_raw(key, vec![]);
+#[cfg(test)]
+mod tests {
+ use super::*;
- if db.uncommitted_inserts() > 1_000_000 {
- db.commit()?;
+ #[test]
+ fn test_downloaded_db() {
+ let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();
- // TODO: Truncate the database using a ttl
- }
+ let url = Url::parse("https://example.com").unwrap();
+ assert!(!db.has_downloaded(&url).unwrap());
- Ok(())
+ db.insert(&url).unwrap();
+ assert!(db.has_downloaded(&url).unwrap());
+ }
+
+ #[test]
+ fn test_truncate_ttl() {
+ let db = DownloadedDb::open(crate::gen_temp_path()).unwrap();
+
+ let url = Url::parse("https://example.com").unwrap();
+ db.insert(&url).unwrap();
+
+ assert!(db.has_downloaded(&url).unwrap());
+
+ let ttl = Duration::from_secs(1);
+ std::thread::sleep(ttl + Duration::from_secs(1));
+
+ db.inner.lock().unwrap().truncate(ttl).unwrap();
+
+ assert!(!db.has_downloaded(&url).unwrap());
}
}
diff --git a/crates/core/src/speedy_kv/blob_id_index.rs b/crates/core/src/speedy_kv/blob_id_index.rs
index 12ef87fa..2f7e6143 100644
--- a/crates/core/src/speedy_kv/blob_id_index.rs
+++ b/crates/core/src/speedy_kv/blob_id_index.rs
@@ -19,10 +19,11 @@ use fst::{IntoStreamer, Streamer};
use crate::Result;
use std::{
io::{BufWriter, Write},
+ ops::RangeBounds,
path::{Path, PathBuf},
};
-use super::{BlobId, Serialized};
+use super::{BlobId, Serialized, SerializedRef};
pub struct BlobIdIndex<K> {
path: PathBuf,
@@ -69,6 +70,27 @@ impl<K> BlobIdIndex<K> {
BlobIdIndexIter::new(self.fst.search(automaton).into_stream())
}
+ pub fn range<'a, R>(&'a self, range: R) -> impl Iterator<Item = (Serialized<K>, BlobId)> + 'a
+ where
+ R: RangeBounds<SerializedRef<'a, K>>,
+ {
+ let builder = self.fst.range();
+
+ let builder = match range.start_bound() {
+ std::ops::Bound::Included(s) => builder.ge(s),
+ std::ops::Bound::Excluded(s) => builder.gt(s),
+ std::ops::Bound::Unbounded => builder,
+ };
+
+ let builder = match range.end_bound() {
+ std::ops::Bound::Included(e) => builder.le(e),
+ std::ops::Bound::Excluded(e) => builder.lt(e),
+ std::ops::Bound::Unbounded => builder,
+ };
+
+ BlobIdIndexIter::new(builder.into_stream())
+ }
+
pub fn path(&self) -> &Path {
&self.path
}
diff --git a/crates/core/src/speedy_kv/mod.rs b/crates/core/src/speedy_kv/mod.rs
index e4efcb38..bc8703b3 100644
--- a/crates/core/src/speedy_kv/mod.rs
+++ b/crates/core/src/speedy_kv/mod.rs
@@ -28,7 +28,7 @@
use crate::Result;
use std::{
collections::BTreeMap,
- ops::Range,
+ ops::{Range, RangeBounds},
path::{Path, PathBuf},
};
@@ -331,7 +331,7 @@ impl<K, V> Db<K, V> {
pub fn search_raw<'a, A>(
&'a self,
query: A,
- ) -> impl Iterator<Item = (Serialized<K>, Serialized<V>)> + 'a
+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'a
where
A: fst::Automaton + Clone + 'a,
{
@@ -341,6 +341,19 @@ impl<K, V> Db<K, V> {
.flat_map(move |segment| segment.search_raw(query.clone()))
}
+ pub fn range_raw<'a, R>(
+ &'a self,
+ range: R,
+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'a
+ where
+ R: RangeBounds<SerializedRef<'a, K>> + Clone + 'a,
+ {
+ self.segments
+ .iter()
+ .rev()
+ .flat_map(move |segment| segment.range_raw(range.clone()))
+ }
+
pub fn commit(&mut self) -> Result<()> {
if self.live_segment.db.is_empty() {
return Ok(());
diff --git a/crates/core/src/speedy_kv/segment.rs b/crates/core/src/speedy_kv/segment.rs
index 94e9d100..25f26b26 100644
--- a/crates/core/src/speedy_kv/segment.rs
+++ b/crates/core/src/speedy_kv/segment.rs
@@ -16,6 +16,7 @@
use std::{
io::{BufWriter, Write},
+ ops::RangeBounds,
path::{Path, PathBuf},
};
@@ -221,16 +222,30 @@ impl<K, V> Segment<K, V> {
pub fn search_raw<'a, A>(
&'a self,
query: A,
- ) -> impl Iterator<Item = (Serialized<K>, Serialized<V>)> + 'a
+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'a
where
A: fst::Automaton + 'a,
{
- self.id_index.search(query).map(move |(ser_k, id)| {
+ self.id_index.search(query).map(move |(_, id)| {
let ptr = self.blob_index.get(id);
let blob = self.store.get_raw(&ptr).unwrap();
- let value = Serialized::from(blob.value.as_bytes().to_vec());
- (ser_k, value)
+ (blob.key, blob.value)
+ })
+ }
+
+ pub fn range_raw<'a, R>(
+ &'a self,
+ range: R,
+ ) -> impl Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)> + 'a
+ where
+ R: RangeBounds<SerializedRef<'a, K>>,
+ {
+ self.id_index.range(range).map(move |(_, id)| {
+ let ptr = self.blob_index.get(id);
+ let blob = self.store.get_raw(&ptr).unwrap();
+
+ (blob.key, blob.value)
})
}