web search done right
cratify file-store
Mikkel Denkerda4f930b036b
diff --git a/Cargo.lock b/Cargo.lockindex 83c0bc84..465e44ec 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1538,6 +1538,14 @@ version = "1.4.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a58880ee1c8f961b122908bb1a195e13cd31b81a28ed1136910bf881ce23bbe8"+[[package]]+name = "file_store"+version = "0.1.0"+dependencies = [+ "anyhow",+ "bincode",+]+[[package]]name = "filetime"version = "0.2.23"@@ -4621,6 +4629,7 @@ dependencies = ["anyhow","bincode","bloom",+ "file_store","fst","memmap","serde",@@ -4696,6 +4705,7 @@ dependencies = ["enum_dispatch","eventsource-stream","fend-core",+ "file_store","flate2","fnv","fst",diff --git a/Cargo.toml b/Cargo.tomlindex 5b343c5e..bd673207 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -4,6 +4,7 @@ members = ["crates/client-wasm","crates/client-wasm","crates/core",+ "crates/file-store","crates/kuchiki","crates/optics","crates/speedy-kv",diff --git a/assets/licenses.html b/assets/licenses.htmlindex ba2fbb28..517fc6fd 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -49,8 +49,8 @@<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (7)</li><li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li>+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (3)</li><li><a href="#ISC">ISC License</a> (3)</li>- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (2)</li><li><a href="#CC0-1.0">Creative Commons Zero v1.0 Universal</a> (2)</li><li><a href="#Zlib">zlib License</a> (2)</li><li><a href="#BSL-1.0">Boost Software License 1.0</a> (1)</li>@@ -69,6 +69,7 @@<h4>Used by:</h4><ul class="license-used-by"><li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>+ <li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li><li><a href=" https://crates.io/crates/speedy_kv ">speedy_kv 0.1.0</a></li></ul><pre class="license-text">GNU AFFERO GENERAL PUBLIC LICENSEdiff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 44b0df3d..123fc076 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -50,6 +50,7 @@ encoding_rs = { workspace = true }enum_dispatch = { workspace = true }eventsource-stream = { workspace = true }fend-core = { workspace = true }+file_store = { path = "../file-store" }flate2 = { workspace = true }fnv = { workspace = true }fst = { workspace = true }diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 4d9d83c6..48fed302 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -51,7 +51,6 @@ mod executor;mod external_sort;mod fastfield_reader;pub mod feed;-mod file_store;mod highlighted;mod human_website_annotations;pub mod hyperloglog;diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex 5b2f4280..3adcca8a 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -24,11 +24,8 @@ use std::{use itertools::Itertools;-use crate::file_store::{- self,- iterable::{IterableStoreReader, SortedIterableStoreReader},-};use crate::Result;+use file_store::iterable::{IterableStoreReader, SortedIterableStoreReader};use super::{store::EdgeStore, Compression, EdgeLabel, InnerEdge, NodeID};diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.tomlnew file mode 100644index 00000000..7be7177e--- /dev/null+++ b/crates/file-store/Cargo.toml@@ -0,0 +1,11 @@+[package]+name = "file_store"+version = "0.1.0"+edition = "2021"+license = "AGPL-3.0"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html++[dependencies]+anyhow.workspace = true+bincode.workspace = truediff --git a/crates/core/src/file_store/iterable.rs b/crates/file-store/src/iterable.rssimilarity index 100%rename from crates/core/src/file_store/iterable.rsrename to crates/file-store/src/iterable.rsdiff --git a/crates/core/src/file_store/mod.rs b/crates/file-store/src/lib.rssimilarity index 87%rename from crates/core/src/file_store/mod.rsrename to crates/file-store/src/lib.rsindex 806478cb..5a1157d1 100644--- a/crates/core/src/file_store/mod.rs+++ b/crates/file-store/src/lib.rs@@ -14,6 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>+//! A collection of simple disk-based data structures.++pub type Result<T> = std::result::Result<T, anyhow::Error>;+pub mod iterable;pub mod peekable;diff --git a/crates/core/src/file_store/peekable.rs b/crates/file-store/src/peekable.rssimilarity index 95%rename from crates/core/src/file_store/peekable.rsrename to crates/file-store/src/peekable.rsindex 00d62225..d0757769 100644--- a/crates/core/src/file_store/peekable.rs+++ b/crates/file-store/src/peekable.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/-// TODO: Remove peekable from speedy_kv::segment and replace it with this-/// An iterator that allows peeking at the next element./// Unlike the standard library's `Peekable`, this implementation/// *is not* lazy and will always consume the next element when peeking.diff --git a/crates/speedy-kv/Cargo.toml b/crates/speedy-kv/Cargo.tomlindex e9f8e459..35173333 100644--- a/crates/speedy-kv/Cargo.toml+++ b/crates/speedy-kv/Cargo.toml@@ -10,6 +10,7 @@ license = "AGPL-3.0"anyhow.workspace = truebincode.workspace = truebloom = { path = "../bloom" }+file_store = { path = "../file-store" }fst.workspace = truememmap.workspace = trueserde.workspace = truediff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rsindex 1b37816f..47f820fc 100644--- a/crates/speedy-kv/src/segment.rs+++ b/crates/speedy-kv/src/segment.rs@@ -20,6 +20,8 @@ use std::{path::{Path, PathBuf},};+use file_store::Peekable;+use super::{blob_id_index::{BlobIdIndex, BlobIdIndexWriter},blob_index::{BlobIndex, BlobIndexWriter},@@ -297,45 +299,6 @@ impl<K, V> Segment<K, V> {}}-struct Peekable<I>-where- I: Iterator,-{- iter: I,- peeked: Option<I::Item>,-}--impl<I> Peekable<I>-where- I: Iterator,-{- fn new(iter: I) -> Self {- let mut iter = iter;- let peeked = iter.next();- Self { iter, peeked }- }-- fn peek(&self) -> Option<&I::Item> {- self.peeked.as_ref()- }-}--impl<I> Iterator for Peekable<I>-where- I: Iterator,-{- type Item = I::Item;-- fn next(&mut self) -> Option<Self::Item> {- let peeked = self.peeked.take();- if peeked.is_some() {- self.peeked = self.iter.next();- }-- peeked- }-}-struct SortedSegments<I>whereI: Iterator,