Skip to content

web search done right

cratify file-store

Mikkel Denkerda4f930b036b
diff --git a/Cargo.lock b/Cargo.lock
index 83c0bc84..465e44ec 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1538,6 +1538,14 @@ version = "1.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a58880ee1c8f961b122908bb1a195e13cd31b81a28ed1136910bf881ce23bbe8"
+[[package]]
+name = "file_store"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "bincode",
+]
+
[[package]]
name = "filetime"
version = "0.2.23"
@@ -4621,6 +4629,7 @@ dependencies = [
"anyhow",
"bincode",
"bloom",
+ "file_store",
"fst",
"memmap",
"serde",
@@ -4696,6 +4705,7 @@ dependencies = [
"enum_dispatch",
"eventsource-stream",
"fend-core",
+ "file_store",
"flate2",
"fnv",
"fst",
diff --git a/Cargo.toml b/Cargo.toml
index 5b343c5e..bd673207 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,7 @@ members = [
"crates/client-wasm",
"crates/client-wasm",
"crates/core",
+ "crates/file-store",
"crates/kuchiki",
"crates/optics",
"crates/speedy-kv",
diff --git a/assets/licenses.html b/assets/licenses.html
index ba2fbb28..517fc6fd 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -49,8 +49,8 @@
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (7)</li>
<li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li>
+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (3)</li>
<li><a href="#ISC">ISC License</a> (3)</li>
- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (2)</li>
<li><a href="#CC0-1.0">Creative Commons Zero v1.0 Universal</a> (2)</li>
<li><a href="#Zlib">zlib License</a> (2)</li>
<li><a href="#BSL-1.0">Boost Software License 1.0</a> (1)</li>
@@ -69,6 +69,7 @@
<h4>Used by:</h4>
<ul class="license-used-by">
<li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>
+ <li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li>
<li><a href=" https://crates.io/crates/speedy_kv ">speedy_kv 0.1.0</a></li>
</ul>
<pre class="license-text">GNU AFFERO GENERAL PUBLIC LICENSE
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 44b0df3d..123fc076 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -50,6 +50,7 @@ encoding_rs = { workspace = true }
enum_dispatch = { workspace = true }
eventsource-stream = { workspace = true }
fend-core = { workspace = true }
+file_store = { path = "../file-store" }
flate2 = { workspace = true }
fnv = { workspace = true }
fst = { workspace = true }
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 4d9d83c6..48fed302 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -51,7 +51,6 @@ mod executor;
mod external_sort;
mod fastfield_reader;
pub mod feed;
-mod file_store;
mod highlighted;
mod human_website_annotations;
pub mod hyperloglog;
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index 5b2f4280..3adcca8a 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -24,11 +24,8 @@ use std::{
use itertools::Itertools;
-use crate::file_store::{
- self,
- iterable::{IterableStoreReader, SortedIterableStoreReader},
-};
use crate::Result;
+use file_store::iterable::{IterableStoreReader, SortedIterableStoreReader};
use super::{store::EdgeStore, Compression, EdgeLabel, InnerEdge, NodeID};
diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.toml
new file mode 100644
index 00000000..7be7177e
--- /dev/null
+++ b/crates/file-store/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "file_store"
+version = "0.1.0"
+edition = "2021"
+license = "AGPL-3.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow.workspace = true
+bincode.workspace = true
diff --git a/crates/core/src/file_store/iterable.rs b/crates/file-store/src/iterable.rs
similarity index 100%
rename from crates/core/src/file_store/iterable.rs
rename to crates/file-store/src/iterable.rs
diff --git a/crates/core/src/file_store/mod.rs b/crates/file-store/src/lib.rs
similarity index 87%
rename from crates/core/src/file_store/mod.rs
rename to crates/file-store/src/lib.rs
index 806478cb..5a1157d1 100644
--- a/crates/core/src/file_store/mod.rs
+++ b/crates/file-store/src/lib.rs
@@ -14,6 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
+//! A collection of simple disk-based data structures.
+
+pub type Result<T> = std::result::Result<T, anyhow::Error>;
+
pub mod iterable;
pub mod peekable;
diff --git a/crates/core/src/file_store/peekable.rs b/crates/file-store/src/peekable.rs
similarity index 95%
rename from crates/core/src/file_store/peekable.rs
rename to crates/file-store/src/peekable.rs
index 00d62225..d0757769 100644
--- a/crates/core/src/file_store/peekable.rs
+++ b/crates/file-store/src/peekable.rs
@@ -14,8 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/
-// TODO: Remove peekable from speedy_kv::segment and replace it with this
-
/// An iterator that allows peeking at the next element.
/// Unlike the standard library's `Peekable`, this implementation
/// *is not* lazy and will always consume the next element when peeking.
diff --git a/crates/speedy-kv/Cargo.toml b/crates/speedy-kv/Cargo.toml
index e9f8e459..35173333 100644
--- a/crates/speedy-kv/Cargo.toml
+++ b/crates/speedy-kv/Cargo.toml
@@ -10,6 +10,7 @@ license = "AGPL-3.0"
anyhow.workspace = true
bincode.workspace = true
bloom = { path = "../bloom" }
+file_store = { path = "../file-store" }
fst.workspace = true
memmap.workspace = true
serde.workspace = true
diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rs
index 1b37816f..47f820fc 100644
--- a/crates/speedy-kv/src/segment.rs
+++ b/crates/speedy-kv/src/segment.rs
@@ -20,6 +20,8 @@ use std::{
path::{Path, PathBuf},
};
+use file_store::Peekable;
+
use super::{
blob_id_index::{BlobIdIndex, BlobIdIndexWriter},
blob_index::{BlobIndex, BlobIndexWriter},
@@ -297,45 +299,6 @@ impl<K, V> Segment<K, V> {
}
}
-struct Peekable<I>
-where
- I: Iterator,
-{
- iter: I,
- peeked: Option<I::Item>,
-}
-
-impl<I> Peekable<I>
-where
- I: Iterator,
-{
- fn new(iter: I) -> Self {
- let mut iter = iter;
- let peeked = iter.next();
- Self { iter, peeked }
- }
-
- fn peek(&self) -> Option<&I::Item> {
- self.peeked.as_ref()
- }
-}
-
-impl<I> Iterator for Peekable<I>
-where
- I: Iterator,
-{
- type Item = I::Item;
-
- fn next(&mut self) -> Option<Self::Item> {
- let peeked = self.peeked.take();
- if peeked.is_some() {
- self.peeked = self.iter.next();
- }
-
- peeked
- }
-}
-
struct SortedSegments<I>
where
I: Iterator,