Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

volume server: refuse to compact a volume tiered to remote storage (#11545)

Eliah Rusineeec9ec09aba
diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs
index 1aaa417217..b525b5e5b0 100644
--- a/seaweed-volume/src/server/grpc_server.rs
+++ b/seaweed-volume/src/server/grpc_server.rs
@@ -7982,6 +7982,68 @@ mod tests {
assert_eq!(read.unwrap(), b"needle-2");
}
+ // An explicit vacuum of a tiered volume must not copy it out of remote
+ // storage only for the commit to refuse the result.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn test_compaction_refused_before_copying_a_tiered_volume() {
+ let (service, tmp) = make_local_service_with_volume("", None);
+ seed_compactable_volume(&service);
+ let dat_bytes = std::fs::read(tmp.path().join("1.dat")).unwrap();
+ let (endpoint, shutdown_tx, _deletes) = spawn_fake_s3_server(dat_bytes.clone());
+ register_tier_backend("s3.tiered_then_compact", endpoint);
+ {
+ let mut store = service.state.store.write().unwrap();
+ let (_, vol) = store.find_volume_mut(VolumeId(1)).unwrap();
+ vol.update_remote_files(|files| {
+ files.push(volume_server_pb::RemoteFile {
+ backend_type: "s3".to_string(),
+ backend_id: "tiered_then_compact".to_string(),
+ key: "remote-key".to_string(),
+ offset: 0,
+ file_size: dat_bytes.len() as u64,
+ modified_time: 0,
+ extension: ".dat".to_string(),
+ })
+ })
+ .unwrap();
+ vol.save_volume_info().unwrap();
+ vol.load_remote_dat_file().unwrap();
+ }
+ std::fs::remove_file(tmp.path().join("1.dat")).unwrap();
+
+ let mut stream = service
+ .vacuum_volume_compact(Request::new(volume_server_pb::VacuumVolumeCompactRequest {
+ volume_id: 1,
+ preallocate: 0,
+ }))
+ .await
+ .unwrap()
+ .into_inner();
+ let mut error = None;
+ while let Some(message) = stream.next().await {
+ if let Err(e) = message {
+ error = Some(e);
+ }
+ }
+ let compacting = {
+ let store = service.state.store.read().unwrap();
+ store.find_volume(VolumeId(1)).unwrap().1.is_compacting()
+ };
+ let read = read_surviving_needle(&service);
+ global_s3_tier_registry()
+ .write()
+ .unwrap()
+ .remove("s3.tiered_then_compact");
+ let _ = shutdown_tx.send(());
+
+ let err = error.expect("a tiered volume must refuse the compaction");
+ assert!(err.message().contains("tiered"), "{err:?}");
+ assert!(!tmp.path().join("1.cpd").exists());
+ assert!(!tmp.path().join("1.cpx").exists());
+ assert!(!compacting);
+ assert_eq!(read.unwrap(), b"needle-2");
+ }
+
/// Build a local service whose volume has a `.dat` large enough to span
/// several 2MB copy chunks, so the streaming copy paths are exercised
/// across multiple messages rather than a single buffer.
diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs
index 359dbd9836..e07e948984 100644
--- a/seaweed-volume/src/storage/volume.rs
+++ b/seaweed-volume/src/storage/volume.rs
@@ -4408,6 +4408,13 @@ impl Volume {
return Err(e);
}
let idx_size = nm.index_file_size();
+ // The copy would stream the .dat from remote storage for a commit that refuses it.
+ if self.has_remote_file() {
+ return Err(VolumeError::Io(io::Error::other(format!(
+ "volume {} is tiered to remote storage, cannot compact",
+ self.id
+ ))));
+ }
// Fresh opens, not `try_clone`: see `dat_scan_plan`.
let src_dat = if self.dat_file.is_some() {
@@ -8650,9 +8657,17 @@ mod tests {
fn test_compaction_aborts_on_truncated_index() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
- let mut v = reload_as_tiered(dir, "vif_compact_test", 4);
+ {
+ let mut v = make_test_volume(dir);
+ for i in 1..=4u64 {
+ write_test_needle(&mut v, i, format!("needle-{i}").as_bytes());
+ }
+ v.set_read_only_persist(false, true).unwrap();
+ v.sync_to_disk().unwrap();
+ }
+ let mut v = make_test_volume(dir);
let Some(NeedleMap::SortedFile(_)) = v.nm else {
- panic!("tiered volume should search the on-disk .sdx");
+ panic!("read-only volume should search the on-disk .sdx");
};
let idx = OpenOptions::new()
@@ -8669,11 +8684,6 @@ mod tests {
matches!(err, VolumeError::Io(ref e) if e.kind() == io::ErrorKind::UnexpectedEof),
"unexpected error: {err:?}"
);
-
- crate::remote_storage::s3_tier::global_s3_tier_registry()
- .write()
- .unwrap()
- .remove("s3.vif_compact_test");
}
// Building .sdx writes to the index directory, which a read-only volume's