Skip to content

web search done right

cleanup temporary directories automatically in tests (#228)

Mikkel Denker4e8c165a1c24
diff --git a/Cargo.lock b/Cargo.lock
index c5ad6596..d447929d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1683,6 +1683,8 @@ dependencies = [
"common",
"memmap2",
"ownedbytes",
+ "tracing",
+ "uuid",
]
[[package]]
diff --git a/crates/core/src/crawler/file_queue.rs b/crates/core/src/crawler/file_queue.rs
index c6a311c2..e098b7b4 100644
--- a/crates/core/src/crawler/file_queue.rs
+++ b/crates/core/src/crawler/file_queue.rs
@@ -193,7 +193,8 @@ mod tests {
#[test]
fn simple() {
- let mut writer = FileQueueWriter::new(crate::gen_temp_path()).unwrap();
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let mut writer = FileQueueWriter::new(&temp_dir).unwrap();
writer.push("Hello".to_string()).unwrap();
writer.push("World".to_string()).unwrap();
@@ -210,7 +211,8 @@ mod tests {
fn prop(data: Vec<String>) {
let expected = data.clone();
- let mut writer = FileQueueWriter::new(crate::gen_temp_path()).unwrap();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut writer = FileQueueWriter::new(&temp_dir).unwrap();
for item in data {
writer.push(item).unwrap();
diff --git a/crates/core/src/distributed/remote_cp.rs b/crates/core/src/distributed/remote_cp.rs
index d0ccfbea..3b925b2c 100644
--- a/crates/core/src/distributed/remote_cp.rs
+++ b/crates/core/src/distributed/remote_cp.rs
@@ -169,7 +169,7 @@ fn files(root: PathBuf) -> Vec<PathBuf> {
mod tests {
use std::path::Path;
- use file_store::gen_temp_path;
+ use file_store::gen_temp_dir;
use super::*;
@@ -183,104 +183,104 @@ mod tests {
#[tokio::test]
async fn test_directory() {
- let a = gen_temp_path();
- std::fs::create_dir_all(a.join("test")).unwrap();
+ let a = gen_temp_dir().unwrap();
+ std::fs::create_dir_all(a.as_ref().join("test")).unwrap();
let contents = "this is a test";
- std::fs::write(a.join("test").join("file.txt"), contents).unwrap();
+ std::fs::write(a.as_ref().join("test").join("file.txt"), contents).unwrap();
- let b = gen_temp_path();
+ let b = gen_temp_dir().unwrap();
download(&a, &b).await;
- assert!(b.join("test").exists());
- assert!(b.join("test").join("file.txt").exists());
+ assert!(b.as_ref().join("test").exists());
+ assert!(b.as_ref().join("test").join("file.txt").exists());
- let res = std::fs::read_to_string(b.join("test").join("file.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("test").join("file.txt")).unwrap();
assert_eq!(contents, res);
}
#[tokio::test]
async fn test_single_file() {
- let a = gen_temp_path();
+ let a = gen_temp_dir().unwrap();
std::fs::create_dir_all(&a).unwrap();
let contents = "this is a test";
- std::fs::write(a.join("file.txt"), contents).unwrap();
+ std::fs::write(a.as_ref().join("file.txt"), contents).unwrap();
- let b = gen_temp_path();
+ let b = gen_temp_dir().unwrap();
download(&a, &b).await;
- assert!(b.join("file.txt").exists());
+ assert!(b.as_ref().join("file.txt").exists());
- let res = std::fs::read_to_string(b.join("file.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("file.txt")).unwrap();
assert_eq!(contents, res);
}
#[tokio::test]
async fn test_overwrite() {
- let a = gen_temp_path();
+ let a = gen_temp_dir().unwrap();
std::fs::create_dir_all(&a).unwrap();
let contents = "this is a test";
- std::fs::write(a.join("file.txt"), contents).unwrap();
+ std::fs::write(a.as_ref().join("file.txt"), contents).unwrap();
- let b = gen_temp_path();
+ let b = gen_temp_dir().unwrap();
std::fs::create_dir_all(&b).unwrap();
- std::fs::write(b.join("file.txt"), "this is another test").unwrap();
+ std::fs::write(b.as_ref().join("file.txt"), "this is another test").unwrap();
download(&a, &b).await;
- assert!(b.join("file.txt").exists());
+ assert!(b.as_ref().join("file.txt").exists());
- let res = std::fs::read_to_string(b.join("file.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("file.txt")).unwrap();
assert_eq!(contents, res);
}
#[tokio::test]
async fn test_keep_non_copied() {
- let a = gen_temp_path();
- std::fs::create_dir_all(a.join("test")).unwrap();
+ let a = gen_temp_dir().unwrap();
+ std::fs::create_dir_all(a.as_ref().join("test")).unwrap();
let contents = "this is a test";
- std::fs::write(a.join("test").join("a.txt"), contents).unwrap();
+ std::fs::write(a.as_ref().join("test").join("a.txt"), contents).unwrap();
- let b = gen_temp_path();
- std::fs::create_dir_all(b.join("test")).unwrap();
- std::fs::write(b.join("test").join("b.txt"), contents).unwrap();
+ let b = gen_temp_dir().unwrap();
+ std::fs::create_dir_all(b.as_ref().join("test")).unwrap();
+ std::fs::write(b.as_ref().join("test").join("b.txt"), contents).unwrap();
download(&a, &b).await;
- assert!(b.join("test").exists());
- assert!(b.join("test").join("a.txt").exists());
- assert!(b.join("test").join("b.txt").exists());
+ assert!(b.as_ref().join("test").exists());
+ assert!(b.as_ref().join("test").join("a.txt").exists());
+ assert!(b.as_ref().join("test").join("b.txt").exists());
- let res = std::fs::read_to_string(b.join("test").join("a.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("test").join("a.txt")).unwrap();
assert_eq!(contents, res);
- let res = std::fs::read_to_string(b.join("test").join("b.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("test").join("b.txt")).unwrap();
assert_eq!(contents, res);
}
#[tokio::test]
async fn test_file_size_edge_case() {
let content = "a".repeat(CHUNK_SIZE_BYTES - 1);
- let a = gen_temp_path();
+ let a = gen_temp_dir().unwrap();
std::fs::create_dir_all(&a).unwrap();
- std::fs::write(a.join("minus_1.txt"), &content).unwrap();
- std::fs::write(a.join("edge.txt"), format!("{}a", &content)).unwrap();
- std::fs::write(a.join("plus_1.txt"), format!("{}aa", &content)).unwrap();
+ std::fs::write(a.as_ref().join("minus_1.txt"), &content).unwrap();
+ std::fs::write(a.as_ref().join("edge.txt"), format!("{}a", &content)).unwrap();
+ std::fs::write(a.as_ref().join("plus_1.txt"), format!("{}aa", &content)).unwrap();
- let b = gen_temp_path();
+ let b = gen_temp_dir().unwrap();
download(&a, &b).await;
- assert!(b.join("minus_1.txt").exists());
- assert!(b.join("edge.txt").exists());
- assert!(b.join("plus_1.txt").exists());
+ assert!(b.as_ref().join("minus_1.txt").exists());
+ assert!(b.as_ref().join("edge.txt").exists());
+ assert!(b.as_ref().join("plus_1.txt").exists());
- let res = std::fs::read_to_string(b.join("minus_1.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("minus_1.txt")).unwrap();
assert_eq!(content, res);
- let res = std::fs::read_to_string(b.join("edge.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("edge.txt")).unwrap();
assert_eq!(format!("{}a", &content), res);
- let res = std::fs::read_to_string(b.join("plus_1.txt")).unwrap();
+ let res = std::fs::read_to_string(b.as_ref().join("plus_1.txt")).unwrap();
assert_eq!(format!("{}aa", &content), res);
std::fs::remove_dir_all(&a).unwrap();
diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rs
index 2b090830..60b24585 100644
--- a/crates/core/src/entity_index/mod.rs
+++ b/crates/core/src/entity_index/mod.rs
@@ -435,7 +435,8 @@ mod tests {
#[test]
fn stopwords_title_ignored() {
- let mut index = EntityIndex::open(crate::gen_temp_path()).unwrap();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut index = EntityIndex::open(&temp_dir).unwrap();
index.prepare_writer();
index.insert(Entity {
@@ -465,7 +466,8 @@ mod tests {
#[test]
fn image() {
- let mut index = EntityIndex::open(crate::gen_temp_path()).unwrap();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut index = EntityIndex::open(&temp_dir).unwrap();
index.prepare_writer();
index.insert(Entity {
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
index 51b6c1cc..ed63b38b 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
@@ -94,20 +94,21 @@ mod tests {
#[test]
#[traced_test]
fn test_simple_graph() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut combined = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("combined"),
Executor::single_thread(),
Compression::default(),
None,
);
let mut a = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("a"),
Executor::single_thread(),
Compression::default(),
None,
);
let mut b = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("b"),
Executor::single_thread(),
Compression::default(),
None,
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 7b0a7587..cec45517 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -496,14 +496,22 @@ impl IndexingWorker {
#[cfg(test)]
mod tests {
+ use file_store::temp::TempDir;
+
use crate::config::WarcSource;
use super::*;
- fn setup_worker(data_path: &Path, threshold: Option<u64>) -> IndexingWorker {
- crate::block_on(IndexingWorker::new(
+ fn setup_worker(data_path: &Path, threshold: Option<u64>) -> (IndexingWorker, TempDir) {
+ let temp_dir = TempDir::new().unwrap();
+ let worker = crate::block_on(IndexingWorker::new(
IndexerConfig {
- host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ host_centrality_store_path: temp_dir
+ .as_ref()
+ .join("host_centrality")
+ .to_str()
+ .unwrap()
+ .to_string(),
page_centrality_store_path: None,
page_webgraph: None,
safety_classifier_path: None,
@@ -511,11 +519,16 @@ mod tests {
model_path: data_path.to_str().unwrap().to_string(),
page_centrality_rank_threshold: threshold,
}),
- output_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ output_path: temp_dir
+ .as_ref()
+ .join("output")
+ .to_str()
+ .unwrap()
+ .to_string(),
limit_warc_files: None,
skip_warc_files: None,
warc_source: WarcSource::Local(crate::config::LocalConfig {
- folder: crate::gen_temp_path().to_str().unwrap().to_string(),
+ folder: temp_dir.as_ref().join("warc").to_str().unwrap().to_string(),
names: vec!["".to_string()],
}),
host_centrality_threshold: None,
@@ -525,7 +538,9 @@ mod tests {
crate::config::defaults::Indexing::autocommit_after_num_inserts(),
}
.into(),
- ))
+ ));
+
+ (worker, temp_dir)
}
#[test]
@@ -535,7 +550,7 @@ mod tests {
// Skip the test if the test data is not available
return;
}
- let worker = setup_worker(data_path, None);
+ let (worker, _temp_dir) = setup_worker(data_path, None);
let webpages = vec![
IndexableWebpage {
@@ -637,7 +652,7 @@ mod tests {
// Skip the test if the test data is not available
return;
}
- let worker = setup_worker(data_path, Some(100_000));
+ let (worker, _temp_dir) = setup_worker(data_path, Some(100_000));
let mut a = Webpage::test_parse("<html><head><title>Homemade Heart Brownie Recipe</title></head><body>Example</body></html>", "https://a.com").unwrap();
a.page_centrality_rank = 1;
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index 26c9afb2..c605dd74 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -24,7 +24,7 @@ use crate::{
};
use std::{net::SocketAddr, path::Path, sync::Arc};
-use file_store::gen_temp_path;
+use file_store::{gen_temp_dir, temp::TempDir};
use crate::{
ampc::dht::ShardId,
@@ -69,15 +69,15 @@ struct RemoteIndex {
gossip_addr: SocketAddr,
underlying_index: Arc<LiveIndex>,
cluster: Arc<Cluster>,
+ _temp_dir: TempDir,
}
impl RemoteIndex {
async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {
- let path = gen_temp_path();
- let mut config = config(&path);
+ let dir = gen_temp_dir()?;
+ let mut config = config(&dir);
config.shard_id = shard;
-
let host = config.host;
let gossip_addr = config.gossip_addr;
@@ -107,6 +107,7 @@ impl RemoteIndex {
gossip_addr,
underlying_index: index,
cluster,
+ _temp_dir: dir,
})
}
@@ -403,7 +404,8 @@ async fn test_replica_recovery() -> Result<()> {
#[tokio::test]
async fn test_meta_segments() -> Result<()> {
- let config = config(gen_temp_path());
+ let dir = gen_temp_dir()?;
+ let config = config(&dir);
let indexer_config = crate::entrypoint::indexer::worker::Config {
host_centrality_store_path: config.host_centrality_store_path.clone(),
page_centrality_store_path: config.page_centrality_store_path.clone(),
@@ -441,7 +443,8 @@ async fn test_meta_segments() -> Result<()> {
#[tokio::test]
async fn test_segment_compaction() -> Result<()> {
- let config = config(gen_temp_path());
+ let dir = gen_temp_dir()?;
+ let config = config(&dir);
let indexer_config = crate::entrypoint::indexer::worker::Config {
host_centrality_store_path: config.host_centrality_store_path.clone(),
page_centrality_store_path: config.page_centrality_store_path.clone(),
diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rs
index 39f706f5..b0e61250 100644
--- a/crates/core/src/external_sort.rs
+++ b/crates/core/src/external_sort.rs
@@ -15,10 +15,11 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
+use file_store::temp::{TempDir, TempFile};
use std::{
cmp::Reverse,
collections::BinaryHeap,
- fs::{File, OpenOptions},
+ fs::File,
io::{BufReader, BufWriter, Read, Seek, Write},
};
@@ -74,79 +75,12 @@ impl<T> Chunk<T> {
T: bincode::Encode + Ord,
{
self.sort();
- self.write_into(&mut file.inner)?;
+ self.write_into(file.inner_mut())?;
StoredChunk::new(file)
}
}
-struct TempDir {
- path: std::path::PathBuf,
-}
-
-impl TempDir {
- fn new() -> Result<Self> {
- let path = std::env::temp_dir().join(uuid::Uuid::new_v4().to_string());
-
- std::fs::create_dir(&path)?;
-
- Ok(Self { path })
- }
-
- fn as_ref(&self) -> &std::path::Path {
- self.path.as_ref()
- }
-}
-
-impl Drop for TempDir {
- fn drop(&mut self) {
- if self.path.exists() {
- std::fs::remove_dir_all(&self.path).unwrap();
- }
- }
-}
-
-struct TempFile {
- inner: File,
- path: std::path::PathBuf,
-}
-
-impl TempFile {
- fn new(dir: &TempDir) -> Result<Self> {
- let path = dir.as_ref().join(uuid::Uuid::new_v4().to_string());
-
- Ok(Self {
- inner: OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .truncate(true)
- .open(&path)?,
- path,
- })
- }
-}
-
-impl Drop for TempFile {
- fn drop(&mut self) {
- if self.path.exists() {
- std::fs::remove_file(&self.path).unwrap();
- }
- }
-}
-
-impl Read for TempFile {
- fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
- self.inner.read(buf)
- }
-}
-
-impl Seek for TempFile {
- fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
- self.inner.seek(pos)
- }
-}
-
struct StoredChunk<T> {
buf: Vec<u8>,
data: BufReader<TempFile>,
diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rs
index 688901a6..74790c59 100644
--- a/crates/core/src/image_store.rs
+++ b/crates/core/src/image_store.rs
@@ -261,7 +261,8 @@ mod tests {
.into(),
);
let key = "test".to_string();
- let mut store = BaseImageStore::open(crate::gen_temp_path());
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let mut store = BaseImageStore::open(temp_dir.as_ref().join("image_store"));
store.prepare_writer();
assert_eq!(store.get(&key), None);
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index 9a2fc4d2..b93cd2ab 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -70,13 +70,13 @@ impl Index {
}
#[cfg(test)]
- pub fn temporary() -> Result<Self> {
- let path = crate::gen_temp_path();
- let mut s = Self::open(path)?;
+ pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {
+ let dir = crate::gen_temp_dir()?;
+ let mut s = Self::open(&dir)?;
s.prepare_writer()?;
- Ok(s)
+ Ok((s, dir))
}
pub fn insert(&self, webpage: &Webpage) -> Result<()> {
@@ -162,7 +162,7 @@ mod tests {
#[test]
fn bm25_all_docs() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index 267db096..5b6ec21b 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -312,7 +312,7 @@ mod test {
#[test]
fn test_delete_segments() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 01da755d..f455f3af 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -222,13 +222,13 @@ impl InvertedIndex {
}
#[cfg(test)]
- pub fn temporary() -> Result<Self> {
- let path = crate::gen_temp_path();
- let mut s = Self::open(path)?;
+ pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {
+ let dir = crate::gen_temp_dir()?;
+ let mut s = Self::open(dir.as_ref().join("index"))?;
s.prepare_writer()?;
- Ok(s)
+ Ok((s, dir))
}
}
@@ -280,7 +280,7 @@ mod tests {
#[test]
fn simple_search() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
let ctx = index.local_search_ctx();
let query = Query::parse(
@@ -339,7 +339,7 @@ mod tests {
#[test]
fn document_not_matching() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
@@ -387,7 +387,7 @@ mod tests {
#[test]
fn english_stemming() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
@@ -435,7 +435,7 @@ mod tests {
#[test]
fn stemmed_query_english() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
@@ -483,7 +483,7 @@ mod tests {
#[test]
fn not_searchable_backlinks() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
@@ -566,7 +566,7 @@ mod tests {
#[test]
fn limited_top_docs() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
for _ in 0..100 {
let dedup_s = crate::rand_words(100);
@@ -618,7 +618,7 @@ mod tests {
#[test]
fn host_search() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
@@ -666,7 +666,7 @@ mod tests {
#[test]
fn merge() {
- let index1 = InvertedIndex::temporary().expect("Unable to open index");
+ let (index1, _dir1) = InvertedIndex::temporary().expect("Unable to open index");
index1
.insert(
@@ -690,7 +690,7 @@ mod tests {
)
.expect("failed to insert webpage");
- let index2 = InvertedIndex::temporary().expect("Unable to open index");
+ let (index2, _dir2) = InvertedIndex::temporary().expect("Unable to open index");
index2
.insert(
@@ -743,7 +743,7 @@ mod tests {
#[test]
fn match_across_fields() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
let ctx = index.local_search_ctx();
let query = Query::parse(
@@ -801,7 +801,7 @@ mod tests {
#[test]
fn id_links_removed_during_indexing() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(
@@ -849,7 +849,7 @@ mod tests {
#[test]
fn schema_org_stored() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(&Webpage::test_parse(
@@ -928,7 +928,7 @@ mod tests {
#[test]
fn get_webpage() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(&Webpage::test_parse(
@@ -962,7 +962,7 @@ mod tests {
#[test]
fn get_homepage() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
index
.insert(&Webpage::test_parse(
@@ -999,7 +999,7 @@ mod tests {
#[test]
fn test_title_embeddings_stored() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
let mut webpage = Webpage::test_parse(
&format!(
@@ -1087,7 +1087,7 @@ mod tests {
#[test]
fn test_approximate_count() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
let webpage = Webpage::test_parse(
&format!(
@@ -1145,7 +1145,7 @@ mod tests {
#[test]
fn test_search_special_characters() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
let webpage = Webpage::test_parse(
&format!(
@@ -1202,7 +1202,7 @@ mod tests {
#[test]
fn test_unicode_normalization() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let (mut index, _dir) = InvertedIndex::temporary().expect("Unable to open index");
let webpage = Webpage::test_parse(
&format!(
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 28979087..d88728a3 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -33,7 +33,7 @@ use distributed::{
cluster::Cluster,
member::{Member, Service},
};
-pub use file_store::gen_temp_path;
+pub use file_store::{gen_temp_dir, gen_temp_path};
use std::{cmp::Reverse, sync::Arc};
use thiserror::Error;
diff --git a/crates/core/src/live_index/crawler/crawled_db.rs b/crates/core/src/live_index/crawler/crawled_db.rs
index 9e90eb06..2fd1106c 100644
--- a/crates/core/src/live_index/crawler/crawled_db.rs
+++ b/crates/core/src/live_index/crawler/crawled_db.rs
@@ -356,7 +356,8 @@ mod tests {
#[test]
fn test_downloaded_db() {
- let db = CrawledDb::open(crate::gen_temp_path()).unwrap();
+ let dir = crate::gen_temp_dir().unwrap();
+ let db = CrawledDb::open(&dir).unwrap();
let url = Url::parse("https://example.com").unwrap();
assert!(!db.has_crawled(&url).unwrap());
@@ -367,7 +368,8 @@ mod tests {
#[test]
fn test_truncate_ttl() {
- let db = CrawledDb::open(crate::gen_temp_path()).unwrap();
+ let dir = crate::gen_temp_dir().unwrap();
+ let db = CrawledDb::open(&dir).unwrap();
let url = Url::parse("https://example.com").unwrap();
db.insert(&url).unwrap();
@@ -384,7 +386,8 @@ mod tests {
#[test]
fn test_sharded_downloaded_db() {
- let db = ShardedCrawledDb::open(crate::gen_temp_path()).unwrap();
+ let dir = crate::gen_temp_dir().unwrap();
+ let db = ShardedCrawledDb::open(&dir).unwrap();
let url = Url::parse("https://example.com").unwrap();
assert!(!db.has_crawled(&url).unwrap());
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index ea81f1e9..6382f52f 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -211,13 +211,13 @@ mod tests {
use super::*;
- fn empty_index() -> InvertedIndex {
+ fn empty_index() -> (InvertedIndex, file_store::temp::TempDir) {
InvertedIndex::temporary().unwrap()
}
#[test]
fn simple_parse() {
- let index = empty_index();
+ let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let query = Query::parse(
@@ -244,7 +244,7 @@ mod tests {
#[test]
fn parse_trailing_leading_whitespace() {
- let index = empty_index();
+ let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let query = Query::parse(
@@ -271,7 +271,7 @@ mod tests {
#[test]
fn parse_weird_characters() {
- let index = empty_index();
+ let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let terms = Query::parse(
@@ -319,7 +319,7 @@ mod tests {
#[test]
fn simple_terms_phrase() {
- let index = empty_index();
+ let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let terms = Query::parse(
@@ -339,7 +339,7 @@ mod tests {
#[test]
fn not_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let query = SearchQuery {
query: "test -website".to_string(),
..Default::default()
@@ -391,7 +391,7 @@ mod tests {
#[test]
fn site_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -481,7 +481,7 @@ mod tests {
#[test]
fn links_to_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -583,7 +583,7 @@ mod tests {
#[test]
fn links_to_uppercase() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -637,7 +637,7 @@ mod tests {
#[test]
fn title_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -689,7 +689,7 @@ mod tests {
#[test]
fn url_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -741,7 +741,7 @@ mod tests {
#[test]
fn empty_query() {
- let index = empty_index();
+ let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let query = Query::parse(
@@ -762,7 +762,7 @@ mod tests {
#[test]
fn query_term_only_special_char() {
- let index = empty_index();
+ let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let _query = Query::parse(
@@ -778,7 +778,7 @@ mod tests {
#[test]
fn site_query_split_domain() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -851,7 +851,7 @@ mod tests {
#[test]
fn phrase_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -916,7 +916,7 @@ mod tests {
#[test]
fn match_compound_words() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -997,7 +997,7 @@ mod tests {
#[test]
fn safe_search() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut webpage = Webpage::test_parse(
&format!(
r#"
@@ -1070,7 +1070,7 @@ mod tests {
#[test]
fn suffix_domain_prefix_path_site_operator() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -1177,7 +1177,7 @@ mod tests {
#[test]
fn exact_url_operator() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -1262,7 +1262,7 @@ mod tests {
#[test]
fn mix_phrase_term_query() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 23e0cb62..5a4df880 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -252,7 +252,7 @@ mod tests {
use crate::{
bangs::Bangs,
- enum_map, gen_temp_path,
+ enum_map,
index::Index,
searcher::{
api::{ApiSearcher, Config},
@@ -268,7 +268,7 @@ mod tests {
#[test]
#[allow(clippy::too_many_lines)]
fn discard_and_boost_hosts() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -391,7 +391,7 @@ mod tests {
#[test]
fn example_optics_dont_crash() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -489,7 +489,7 @@ mod tests {
#[test]
#[allow(clippy::too_many_lines)]
fn empty_discard() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -602,10 +602,11 @@ mod tests {
#[tokio::test]
#[allow(clippy::too_many_lines)]
async fn liked_hosts() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let dir = crate::gen_temp_dir().unwrap();
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut writer = WebgraphWriter::new(
- gen_temp_path(),
+ &dir,
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -761,7 +762,7 @@ mod tests {
#[test]
#[allow(clippy::too_many_lines)]
fn schema_org_search() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -941,7 +942,7 @@ mod tests {
#[test]
fn pattern_same_phrase() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1007,7 +1008,7 @@ mod tests {
#[test]
fn discard_all_discard_like() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1094,7 +1095,7 @@ mod tests {
#[test]
#[allow(clippy::too_many_lines)]
fn special_pattern_syntax() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1250,7 +1251,7 @@ mod tests {
#[test]
fn active_optic_with_blocked_hosts() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1318,7 +1319,7 @@ mod tests {
#[test]
fn empty_optic_noop() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1374,7 +1375,7 @@ mod tests {
#[test]
#[allow(clippy::too_many_lines)]
fn wildcard_edge_cases() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1496,7 +1497,7 @@ mod tests {
#[test]
fn empty_double_anchor() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut page = Webpage {
html: Html::parse(
@@ -1564,7 +1565,7 @@ mod tests {
#[test]
fn indieweb_search() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut page = Webpage {
html: Html::parse(
@@ -1648,7 +1649,7 @@ mod tests {
#[test]
fn site_double_anchor() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut page = Webpage {
html: Html::parse(
@@ -1741,7 +1742,7 @@ mod tests {
#[test]
fn apostrophe_token() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut page = Webpage {
html: Html::parse(
@@ -1830,7 +1831,7 @@ mod tests {
#[test]
fn discard_double_matching() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut page = Webpage {
html: Html::parse(
@@ -1918,7 +1919,7 @@ mod tests {
#[test]
fn test_site_in_domain_rule() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let page = Webpage {
html: Html::parse(
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index dd31e635..b374d367 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -143,7 +143,6 @@ mod tests {
use crate::{
bangs::Bangs,
- gen_temp_path,
index::Index,
rand_words,
searcher::{
@@ -167,8 +166,9 @@ mod tests {
#[tokio::test]
async fn it_favors_liked_hosts() {
+ let dir = crate::gen_temp_dir().unwrap();
let mut wrt = WebgraphWriter::new(
- gen_temp_path(),
+ &dir,
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -236,8 +236,9 @@ mod tests {
#[tokio::test]
#[allow(clippy::too_many_lines)]
async fn it_ranks_search_results() {
+ let dir = crate::gen_temp_dir().unwrap();
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &dir,
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -270,7 +271,7 @@ mod tests {
let graph = wrt.finalize();
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index dc5c6653..de2e0bfe 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -144,7 +144,7 @@ mod tests {
#[test]
fn host_centrality_ranking() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -210,7 +210,7 @@ mod tests {
#[test]
fn page_centrality_ranking() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -276,7 +276,7 @@ mod tests {
#[test]
fn freshness_ranking() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -347,7 +347,7 @@ mod tests {
#[test]
fn derank_trackers() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -417,7 +417,7 @@ mod tests {
#[test]
fn backlink_text() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut webpage = Webpage {
html: Html::parse(
@@ -483,7 +483,7 @@ mod tests {
#[test]
fn custom_signal_aggregation() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -590,7 +590,7 @@ mod tests {
#[test]
fn fetch_time_ranking() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -659,7 +659,7 @@ mod tests {
#[test]
fn num_slashes_and_digits() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -755,10 +755,16 @@ mod tests {
assert_eq!(result.webpages[2].url, "https://www.third.com/one/two123");
}
- fn setup_worker(data_path: &Path) -> IndexingWorker {
- crate::block_on(IndexingWorker::new(
+ fn setup_worker(data_path: &Path) -> (IndexingWorker, file_store::temp::TempDir) {
+ let temp_dir = file_store::temp::TempDir::new().unwrap();
+ let worker = crate::block_on(IndexingWorker::new(
IndexerConfig {
- host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ host_centrality_store_path: temp_dir
+ .as_ref()
+ .join("host_centrality")
+ .to_str()
+ .unwrap()
+ .to_string(),
page_centrality_store_path: None,
page_webgraph: None,
safety_classifier_path: None,
@@ -766,11 +772,16 @@ mod tests {
model_path: data_path.to_str().unwrap().to_string(),
page_centrality_rank_threshold: None,
}),
- output_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ output_path: temp_dir
+ .as_ref()
+ .join("output")
+ .to_str()
+ .unwrap()
+ .to_string(),
limit_warc_files: None,
skip_warc_files: None,
warc_source: WarcSource::Local(crate::config::LocalConfig {
- folder: crate::gen_temp_path().to_str().unwrap().to_string(),
+ folder: temp_dir.as_ref().join("warc").to_str().unwrap().to_string(),
names: vec!["".to_string()],
}),
host_centrality_threshold: None,
@@ -780,7 +791,9 @@ mod tests {
crate::config::defaults::Indexing::autocommit_after_num_inserts(),
}
.into(),
- ))
+ ));
+
+ (worker, temp_dir)
}
#[test]
@@ -791,9 +804,9 @@ mod tests {
return;
}
- let worker = setup_worker(data_path);
+ let (worker, _worker_dir) = setup_worker(data_path);
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut pages = vec![
Webpage::test_parse(
@@ -869,9 +882,9 @@ mod tests {
return;
}
- let worker = setup_worker(data_path);
+ let (worker, _worker_dir) = setup_worker(data_path);
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut a = Webpage::test_parse(
&format!(
@@ -951,7 +964,7 @@ mod tests {
#[test]
fn title_coverage() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(&Webpage {
@@ -1029,7 +1042,7 @@ mod tests {
#[test]
fn clean_body_coverage() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut page = Webpage {
html: Html::parse(
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 148764f0..cc59912d 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -20,7 +20,6 @@ mod tests {
use crate::{
bangs::Bangs,
- gen_temp_path,
index::Index,
searcher::{
api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,
@@ -33,10 +32,11 @@ mod tests {
#[tokio::test]
#[allow(clippy::too_many_lines)]
async fn host_rankings() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let dir = crate::gen_temp_dir().unwrap();
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut wrt = WebgraphWriter::new(
- gen_temp_path(),
+ &dir,
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index e4baf94f..58efa36c 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -366,7 +366,7 @@ mod tests {
const NUM_PAGES: usize = 50;
const NUM_WEBSITES: usize = NUM_PAGES * NUM_RESULTS_PER_PAGE;
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
for i in 0..NUM_WEBSITES {
index
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index 0615f07e..6bc3dfc2 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -389,7 +389,7 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn snippet_during_search() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -428,7 +428,7 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn stemmed_words_snippet_highlight() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
@@ -467,7 +467,7 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn test_stemmed_term() {
- let mut index = Index::temporary().expect("Unable to open index");
+ let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rs
index 4783e92b..0d7ef596 100644
--- a/crates/core/src/web_spell/stupid_backoff.rs
+++ b/crates/core/src/web_spell/stupid_backoff.rs
@@ -424,7 +424,7 @@ impl NextWordsStrategy for IntoMiddle {
#[cfg(test)]
mod tests {
use super::*;
- use crate::gen_temp_path;
+ use crate::gen_temp_dir;
#[test]
fn test_contexts() {
@@ -434,11 +434,11 @@ mod tests {
"a b c d e f g h i j k l m n o p q r s t u v w x y z",
));
- let path = gen_temp_path();
+ let temp_dir = gen_temp_dir().unwrap();
- trainer.build(&path).unwrap();
+ trainer.build(&temp_dir).unwrap();
- let model = StupidBackoff::open(&path).unwrap();
+ let model = StupidBackoff::open(&temp_dir).unwrap();
assert_eq!(
model.contexts("b"),
@@ -456,11 +456,11 @@ mod tests {
"a b c d e f g h i j k l m n o p q r s t u v w x y z",
));
- let path = gen_temp_path();
+ let temp_dir = gen_temp_dir().unwrap();
- a.build(&path).unwrap();
+ a.build(&temp_dir.as_ref().join("a")).unwrap();
- let a = StupidBackoff::open(&path).unwrap();
+ let a = StupidBackoff::open(&temp_dir.as_ref().join("a")).unwrap();
let mut b = StupidBackoffTrainer::new(3);
@@ -468,18 +468,15 @@ mod tests {
"a b c d e f g h i j k l m n o p q r s t u v w x y z",
));
- let path = gen_temp_path();
+ b.build(&temp_dir.as_ref().join("b")).unwrap();
- b.build(&path).unwrap();
+ let b = StupidBackoff::open(&temp_dir.as_ref().join("b")).unwrap();
- let b = StupidBackoff::open(&path).unwrap();
-
- let path = gen_temp_path();
- let model = StupidBackoff::merge(vec![a, b], &path).unwrap();
+ let model = StupidBackoff::merge(vec![a, b], &temp_dir.as_ref().join("merged")).unwrap();
assert_eq!(model.n_counts, vec![48, 48, 48]);
- let model = StupidBackoff::open(path).unwrap();
+ let model = StupidBackoff::open(&temp_dir.as_ref().join("merged")).unwrap();
assert_eq!(model.n_counts, vec![48, 48, 48]);
}
diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rs
index a9762ffc..4fbebd69 100644
--- a/crates/core/src/web_spell/term_freqs.rs
+++ b/crates/core/src/web_spell/term_freqs.rs
@@ -377,12 +377,13 @@ impl TermDict {
#[cfg(test)]
mod tests {
use super::*;
- use crate::gen_temp_path;
use anyhow::Result;
#[test]
fn test_term_dict() -> Result<()> {
- let mut dict = TermDict::open(gen_temp_path())?;
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let path = temp_dir.as_ref().join("dicts");
+ let mut dict = TermDict::open(&path)?;
dict.insert("foo");
dict.insert("bar");
@@ -417,7 +418,8 @@ mod tests {
#[test]
fn reopen() {
- let path = gen_temp_path();
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let path = temp_dir.as_ref().join("dicts");
{
let mut dict = TermDict::open(&path).unwrap();
diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rs
index b4a69d29..8a840c3f 100644
--- a/crates/core/src/webgraph/centrality/betweenness.rs
+++ b/crates/core/src/webgraph/centrality/betweenness.rs
@@ -163,15 +163,17 @@ impl Betweenness {
#[cfg(test)]
mod tests {
+ use file_store::temp::TempDir;
use maplit::hashmap;
use crate::{webgraph::WebgraphWriter, webpage::html::links::RelFlags};
use super::*;
- fn create_path_graph(n: usize) -> Webgraph {
+ fn create_path_graph(n: usize) -> (Webgraph, TempDir) {
+ let temp_dir = file_store::gen_temp_dir().unwrap();
let mut writer = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("test-webgraph"),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -186,12 +188,13 @@ mod tests {
);
}
- writer.finalize()
+ let graph = writer.finalize();
+ (graph, temp_dir)
}
#[test]
fn path() {
- let p = create_path_graph(5);
+ let (p, _temp_dir) = create_path_graph(5);
let centrality = Betweenness::calculate(&p);
assert_eq!(
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index 6eced6bd..192ba90d 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -312,6 +312,8 @@ impl HarmonicCentrality {
#[cfg(test)]
mod tests {
+ use file_store::temp::TempDir;
+
use super::*;
use crate::{
webgraph::{Node, WebgraphWriter},
@@ -338,9 +340,10 @@ mod tests {
]
}
- fn test_graph() -> Webgraph {
+ fn test_graph() -> (Webgraph, TempDir) {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut writer = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("test"),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -350,13 +353,16 @@ mod tests {
writer.insert(from, to, label, RelFlags::default());
}
- writer.finalize()
+ let graph = writer.finalize();
+
+ (graph, temp_dir)
}
#[test]
fn host_harmonic_centrality() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut writer = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("test"),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -459,7 +465,7 @@ mod tests {
#[test]
fn harmonic_centrality() {
- let graph = test_graph();
+ let (graph, _temp_dir) = test_graph();
let centrality = HarmonicCentrality::calculate(&graph);
assert!(
@@ -475,11 +481,12 @@ mod tests {
#[test]
fn additional_edges_ignored() {
- let graph = test_graph();
+ let (graph, _temp_dir) = test_graph();
let centrality = HarmonicCentrality::calculate(&graph);
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut other = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("test"),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -549,8 +556,9 @@ mod tests {
#[test]
fn test_rel_flag_ignored() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut graph = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join("test"),
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
@@ -569,8 +577,9 @@ mod tests {
#[test]
fn test_same_icann_domain_ignored() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut graph = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir,
crate::executor::Executor::single_thread(),
crate::webgraph::Compression::default(),
None,
diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rs
index 97ff9335..54964aaa 100644
--- a/crates/core/src/webgraph/id_node_db.rs
+++ b/crates/core/src/webgraph/id_node_db.rs
@@ -65,11 +65,11 @@ impl Id2NodeDb {
#[cfg(test)]
mod tests {
use super::*;
- use crate::gen_temp_path;
#[test]
fn test_id2node_db() {
- let mut db = Id2NodeDb::open(gen_temp_path());
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut db = Id2NodeDb::open(&temp_dir);
let a_node = Node::from("a".to_string());
let a_id = NodeID::from(0_u64);
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index d66c8315..b56bcb1e 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -438,6 +438,7 @@ pub mod tests {
use crate::webpage::html::links::RelFlags;
use super::*;
+ use file_store::temp::TempDir;
use proptest::prelude::*;
pub fn test_edges() -> Vec<(Node, Node, String)> {
@@ -450,7 +451,7 @@ pub mod tests {
]
}
- pub fn test_graph() -> Webgraph {
+ pub fn test_graph() -> (Webgraph, TempDir) {
// ┌-----┐
// │ │
// ┌───A◄─┐ │
@@ -462,8 +463,9 @@ pub mod tests {
// │
// D
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut graph = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir,
Executor::single_thread(),
Compression::default(),
None,
@@ -475,12 +477,12 @@ pub mod tests {
graph.commit();
- graph.finalize()
+ (graph.finalize(), temp_dir)
}
#[test]
fn distance_calculation() {
- let graph = test_graph();
+ let (graph, _temp_dir) = test_graph();
let distances = graph.distances(Node::from("D"));
@@ -491,14 +493,14 @@ pub mod tests {
#[test]
fn nonexisting_node() {
- let graph = test_graph();
+ let (graph, _temp_dir) = test_graph();
assert_eq!(graph.distances(Node::from("E")).len(), 0);
assert_eq!(graph.reversed_distances(Node::from("E")).len(), 0);
}
#[test]
fn reversed_distance_calculation() {
- let graph = test_graph();
+ let (graph, _temp_dir) = test_graph();
let distances = graph.reversed_distances(Node::from("D"));
@@ -516,7 +518,8 @@ pub mod tests {
#[test]
fn merge_path() {
let mut graphs = Vec::new();
- for (from, to, label) in &[
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip(&[
(Node::from("A"), Node::from("B"), String::new()),
(Node::from("B"), Node::from("C"), String::new()),
(Node::from("C"), Node::from("D"), String::new()),
@@ -524,9 +527,9 @@ pub mod tests {
(Node::from("E"), Node::from("F"), String::new()),
(Node::from("F"), Node::from("G"), String::new()),
(Node::from("G"), Node::from("H"), String::new()),
- ] {
+ ]) {
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir.as_ref().join(format!("test_{}", i)),
Executor::single_thread(),
Compression::default(),
None,
@@ -560,9 +563,10 @@ pub mod tests {
#[test]
fn merge_simple() {
let mut graphs = Vec::new();
- for (from, to, label) in test_edges() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip(&test_edges()) {
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir.as_ref().join(format!("test_{}", i)),
Executor::single_thread(),
Compression::default(),
None,
@@ -629,14 +633,15 @@ pub mod tests {
#[test]
fn merge_cycle() {
let mut graphs = Vec::new();
- for (from, to, label) in &[
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip(&[
(Node::from("A"), Node::from("B"), String::new()),
(Node::from("B"), Node::from("B"), String::new()),
(Node::from("B"), Node::from("C"), String::new()),
(Node::from("C"), Node::from("A"), String::new()),
- ] {
+ ]) {
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir.as_ref().join(format!("test_{}", i)),
Executor::single_thread(),
Compression::default(),
None,
@@ -686,14 +691,15 @@ pub mod tests {
#[test]
fn merge_star() {
let mut graphs = Vec::new();
- for (from, to, label) in &[
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip(&[
(Node::from("A"), Node::from("B"), String::new()),
(Node::from("A"), Node::from("C"), String::new()),
(Node::from("A"), Node::from("D"), String::new()),
(Node::from("A"), Node::from("E"), String::new()),
- ] {
+ ]) {
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir.as_ref().join(format!("test_{}", i)),
Executor::single_thread(),
Compression::default(),
None,
@@ -743,14 +749,15 @@ pub mod tests {
#[test]
fn merge_reverse_star() {
let mut graphs = Vec::new();
- for (from, to, label) in &[
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip(&[
(Node::from("B"), Node::from("A"), String::new()),
(Node::from("C"), Node::from("A"), String::new()),
(Node::from("D"), Node::from("A"), String::new()),
(Node::from("E"), Node::from("A"), String::new()),
- ] {
+ ]) {
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir.as_ref().join(format!("test_{}", i)),
Executor::single_thread(),
Compression::default(),
None,
@@ -806,19 +813,20 @@ pub mod tests {
)
) {
let mut graphs = Vec::new();
- let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut wrt = WebgraphWriter::new(
+ temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
for (from, to) in nodes.clone() {
wrt.insert(Node::new_for_test(from.as_str()), Node::new_for_test(to.as_str()), String::new(), RelFlags::default());
if rand::random::<usize>() % 10 == 0 {
graphs.push(wrt.finalize());
wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
Executor::single_thread(),
Compression::default(),
None,
@@ -860,8 +868,9 @@ pub mod tests {
fn proptest_case(nodes: &[(&str, &str)]) {
let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
Executor::single_thread(),
Compression::default(),
None,
@@ -878,7 +887,7 @@ pub mod tests {
if i % 2 == 0 {
graphs.push(wrt.finalize());
wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
Executor::single_thread(),
Compression::default(),
None,
@@ -953,8 +962,9 @@ pub mod tests {
#[test]
fn cap_label_length() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut writer = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
Executor::single_thread(),
Compression::default(),
None,
@@ -978,7 +988,7 @@ pub mod tests {
#[test]
fn test_edge_limits() {
- let graph = test_graph();
+ let (graph, temp_dir) = test_graph();
assert_eq!(
graph
@@ -1000,7 +1010,7 @@ pub mod tests {
(Node::from("A"), Node::from("C"), String::new()),
] {
let mut wrt = WebgraphWriter::new(
- crate::gen_temp_path(),
+ temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
Executor::single_thread(),
Compression::default(),
None,
@@ -1057,8 +1067,9 @@ pub mod tests {
#[test]
fn test_rel_flags() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut writer = WebgraphWriter::new(
- crate::gen_temp_path(),
+ &temp_dir,
Executor::single_thread(),
Compression::default(),
None,
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index b7799fbb..d65f7ba7 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -218,6 +218,7 @@ mod test {
#[test]
#[allow(clippy::too_many_lines)]
fn simple_triangle_graph() {
+ let temp_dir = file_store::gen_temp_dir().unwrap();
// ┌────┐
// │ │
// ┌───0◄─┐ │
@@ -226,7 +227,7 @@ mod test {
// 1─────►2◄┘
let mut writer = SegmentWriter::open(
- crate::gen_temp_path(),
+ temp_dir.as_ref(),
"test".to_string(),
Compression::default(),
None,
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 0805ca97..21ecd574 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -616,12 +616,9 @@ mod tests {
#[test]
fn test_insert() {
- let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
- crate::gen_temp_path().join("test-segment"),
- Compression::default(),
- false,
- None,
- );
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut kv: EdgeStoreWriter =
+ EdgeStoreWriter::new(&temp_dir, Compression::default(), false, None);
let e = InsertableEdge {
from: FullNodeID {
@@ -656,12 +653,9 @@ mod tests {
#[test]
fn test_reversed() {
- let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
- crate::gen_temp_path().join("test-segment"),
- Compression::default(),
- true,
- None,
- );
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut kv: EdgeStoreWriter =
+ EdgeStoreWriter::new(&temp_dir, Compression::default(), true, None);
let e = InsertableEdge {
from: FullNodeID {
@@ -690,12 +684,9 @@ mod tests {
#[test]
fn test_limit() {
- let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
- crate::gen_temp_path().join("test-segment"),
- Compression::default(),
- true,
- None,
- );
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut kv: EdgeStoreWriter =
+ EdgeStoreWriter::new(&temp_dir, Compression::default(), true, None);
for i in 0..10 {
let e = InsertableEdge {
@@ -722,8 +713,9 @@ mod tests {
#[test]
fn test_edge_ordering() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
let mut rank_store =
- speedy_kv::Db::open_or_create(crate::gen_temp_path().join("test-rank-store")).unwrap();
+ speedy_kv::Db::open_or_create(temp_dir.as_ref().join("test-rank-store")).unwrap();
rank_store.insert(NodeID::from(2_u64), 1).unwrap();
rank_store.insert(NodeID::from(3_u64), 2).unwrap();
@@ -732,7 +724,7 @@ mod tests {
rank_store.commit().unwrap();
let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
- crate::gen_temp_path().join("test-segment"),
+ &temp_dir.as_ref().join("test-segment"),
Compression::default(),
true,
Some(Arc::new(rank_store)),
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index c0eda2c1..e6de556c 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -784,7 +784,7 @@ mod tests {
assert!(webpage.all_text().is_some());
assert!(!webpage.all_text().unwrap().is_empty());
- let index =
+ let (index, _dir) =
crate::inverted_index::InvertedIndex::temporary().expect("Unable to open index");
webpage.as_tantivy(&index).unwrap();
}
diff --git a/crates/core/src/webpage/region.rs b/crates/core/src/webpage/region.rs
index 30722b3f..45614daa 100644
--- a/crates/core/src/webpage/region.rs
+++ b/crates/core/src/webpage/region.rs
@@ -229,19 +229,18 @@ impl RegionCount {
#[cfg(test)]
mod tests {
- use crate::gen_temp_path;
-
use super::*;
#[test]
fn simple() {
- let mut a = RegionCount::open(gen_temp_path().join("region_count.json"));
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let mut a = RegionCount::open(temp_dir.as_ref().join("a").join("region_count.json"));
a.increment(&Region::Denmark);
a.increment(&Region::Denmark);
a.increment(&Region::US);
- let mut b = RegionCount::open(gen_temp_path().join("region_count.json"));
+ let mut b = RegionCount::open(temp_dir.as_ref().join("b").join("region_count.json"));
b.increment(&Region::US);
b.increment(&Region::Germany);
diff --git a/crates/file-store/Cargo.toml b/crates/file-store/Cargo.toml
index c883142b..8e8dd8f5 100644
--- a/crates/file-store/Cargo.toml
+++ b/crates/file-store/Cargo.toml
@@ -12,3 +12,5 @@ bincode.workspace = true
common.workspace = true
memmap2.workspace = true
ownedbytes.workspace = true
+uuid.workspace = true
+tracing.workspace = true
\ No newline at end of file
diff --git a/crates/file-store/src/lib.rs b/crates/file-store/src/lib.rs
index e2bd4a93..a10b44d9 100644
--- a/crates/file-store/src/lib.rs
+++ b/crates/file-store/src/lib.rs
@@ -22,9 +22,11 @@ pub mod const_serializable;
pub mod iterable;
pub mod peekable;
pub mod random_lookup;
+pub mod temp;
pub use const_serializable::ConstSerializable;
pub use peekable::Peekable;
+use temp::TempDir;
// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445
pub fn gen_temp_path() -> std::path::PathBuf {
@@ -52,3 +54,7 @@ pub fn gen_temp_path() -> std::path::PathBuf {
std::env::temp_dir().join(format!("pagecache.tmp.{salt}"))
}
}
+
+pub fn gen_temp_dir() -> Result<TempDir> {
+ TempDir::new()
+}
diff --git a/crates/file-store/src/temp.rs b/crates/file-store/src/temp.rs
new file mode 100644
index 00000000..ceaa170c
--- /dev/null
+++ b/crates/file-store/src/temp.rs
@@ -0,0 +1,109 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use std::{
+ fs::{File, OpenOptions},
+ io::{Read, Seek, Write},
+};
+
+use crate::gen_temp_path;
+
+pub struct TempDir {
+ path: std::path::PathBuf,
+}
+
+impl TempDir {
+ pub fn new() -> anyhow::Result<Self> {
+ let path = gen_temp_path();
+
+ std::fs::create_dir(&path)?;
+
+ Ok(Self { path })
+ }
+}
+
+impl AsRef<std::path::Path> for TempDir {
+ fn as_ref(&self) -> &std::path::Path {
+ self.path.as_ref()
+ }
+}
+
+impl Drop for TempDir {
+ fn drop(&mut self) {
+ if self.path.exists() {
+ if let Err(e) = std::fs::remove_dir_all(&self.path) {
+ tracing::error!("Failed to remove temp dir: {e}");
+ }
+ }
+ }
+}
+
+pub struct TempFile {
+ inner: File,
+ path: std::path::PathBuf,
+}
+
+impl TempFile {
+ pub fn new(dir: &TempDir) -> anyhow::Result<Self> {
+ let path = dir.as_ref().join(uuid::Uuid::new_v4().to_string());
+
+ Ok(Self {
+ inner: OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(&path)?,
+ path,
+ })
+ }
+
+ pub fn inner_mut(&mut self) -> &mut File {
+ &mut self.inner
+ }
+}
+
+impl Drop for TempFile {
+ fn drop(&mut self) {
+ if self.path.exists() {
+ if let Err(e) = std::fs::remove_file(&self.path) {
+ tracing::error!("Failed to remove temp file: {e}");
+ }
+ }
+ }
+}
+
+impl Read for TempFile {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ self.inner.read(buf)
+ }
+}
+
+impl Seek for TempFile {
+ fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+ self.inner.seek(pos)
+ }
+}
+
+impl Write for TempFile {
+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+ self.inner.write(buf)
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ self.inner.flush()
+ }
+}
diff --git a/crates/simple-wal/src/lib.rs b/crates/simple-wal/src/lib.rs
index d23d52c3..8f8932e8 100644
--- a/crates/simple-wal/src/lib.rs
+++ b/crates/simple-wal/src/lib.rs
@@ -109,7 +109,8 @@ mod tests {
#[test]
fn test_write_read() -> Result<()> {
- let mut writer = Wal::open(file_store::gen_temp_path())?;
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let mut writer = Wal::open(temp_dir.as_ref().join("test-wal"))?;
writer.write(&1u64)?;
writer.write(&2u64)?;
@@ -130,7 +131,8 @@ mod tests {
#[test]
fn test_empty_write() -> Result<()> {
- let writer: Wal<u64> = Wal::open(file_store::gen_temp_path())?;
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let writer: Wal<u64> = Wal::open(temp_dir.as_ref().join("test-wal"))?;
let res: Vec<_> = writer.iter()?.collect();
@@ -141,7 +143,8 @@ mod tests {
#[test]
fn test_clear() -> Result<()> {
- let mut writer = Wal::open(file_store::gen_temp_path())?;
+ let temp_dir = file_store::gen_temp_dir().unwrap();
+ let mut writer = Wal::open(temp_dir.as_ref().join("test-wal"))?;
writer.write(&1u64)?;
writer.write(&2u64)?;
diff --git a/crates/speedy-kv/src/lib.rs b/crates/speedy-kv/src/lib.rs
index 08a2c64e..3266bd69 100644
--- a/crates/speedy-kv/src/lib.rs
+++ b/crates/speedy-kv/src/lib.rs
@@ -510,37 +510,12 @@ where
#[cfg(test)]
mod tests {
use super::*;
-
- // taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445
- fn gen_temp_path() -> PathBuf {
- use std::sync::atomic::{AtomicUsize, Ordering};
- use std::time::SystemTime;
-
- static SALT_COUNTER: AtomicUsize = AtomicUsize::new(0);
-
- let seed = SALT_COUNTER.fetch_add(1, Ordering::SeqCst) as u128;
-
- let now = SystemTime::now()
- .duration_since(SystemTime::UNIX_EPOCH)
- .unwrap()
- .as_nanos()
- << 48;
-
- let pid = u128::from(std::process::id());
-
- let salt = (pid << 16) + now + seed;
-
- if cfg!(target_os = "linux") {
- // use shared memory for temporary linux files
- format!("/dev/shm/pagecache.tmp.{salt}").into()
- } else {
- std::env::temp_dir().join(format!("pagecache.tmp.{salt}"))
- }
- }
+ use file_store::gen_temp_dir;
#[test]
fn test_simple() {
- let mut db = Db::open_or_create(gen_temp_path()).unwrap();
+ let temp_dir = gen_temp_dir().unwrap();
+ let mut db = Db::open_or_create(&temp_dir).unwrap();
db.insert(1, 2).unwrap();
db.insert(2, 3).unwrap();
@@ -553,7 +528,8 @@ mod tests {
#[test]
fn test_multiple_segments() {
- let mut db = Db::open_or_create(gen_temp_path()).unwrap();
+ let temp_dir = gen_temp_dir().unwrap();
+ let mut db = Db::open_or_create(&temp_dir).unwrap();
db.insert(1, 2).unwrap();
db.insert(2, 3).unwrap();
@@ -573,7 +549,8 @@ mod tests {
#[test]
fn test_segment_merge() {
- let mut db = Db::open_or_create(gen_temp_path()).unwrap();
+ let temp_dir = gen_temp_dir().unwrap();
+ let mut db = Db::open_or_create(&temp_dir).unwrap();
db.insert(1, 2).unwrap();
db.insert(2, 3).unwrap();
@@ -601,7 +578,8 @@ mod tests {
#[test]
fn test_overwrite_key() {
- let mut db = Db::open_or_create(gen_temp_path()).unwrap();
+ let temp_dir = gen_temp_dir().unwrap();
+ let mut db = Db::open_or_create(&temp_dir).unwrap();
db.insert(1, 2).unwrap();
db.insert(1, 3).unwrap();
@@ -622,7 +600,8 @@ mod tests {
#[test]
fn test_len() {
- let mut db = Db::open_or_create(gen_temp_path()).unwrap();
+ let temp_dir = gen_temp_dir().unwrap();
+ let mut db = Db::open_or_create(&temp_dir).unwrap();
assert_eq!(db.len(), 0);
@@ -638,8 +617,9 @@ mod tests {
#[test]
fn test_merge_db() {
- let mut db1 = Db::open_or_create(gen_temp_path()).unwrap();
- let mut db2 = Db::open_or_create(gen_temp_path()).unwrap();
+ let temp_dir = gen_temp_dir().unwrap();
+ let mut db1 = Db::open_or_create(&temp_dir.as_ref().join("db1")).unwrap();
+ let mut db2 = Db::open_or_create(&temp_dir.as_ref().join("db2")).unwrap();
db1.insert(1, 2).unwrap();
db1.insert(2, 3).unwrap();