SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
volume.fsck: fix orphan purge against the rust volume server (#10289)
Chris Lu6f816c955dd8
diff --git a/seaweed-volume/src/storage/needle/needle.rs b/seaweed-volume/src/storage/needle/needle.rsindex bbc55c9d00..7df518e9d4 100644--- a/seaweed-volume/src/storage/needle/needle.rs+++ b/seaweed-volume/src/storage/needle/needle.rs@@ -743,19 +743,14 @@ pub fn parse_needle_id_cookie(s: &str) -> Result<(NeedleId, Cookie), String> {let needle_id_hex = &hex_part[..split];let cookie_hex = &hex_part[split..];- let needle_id_bytes = hex::decode(needle_id_hex).map_err(|e| format!("Parse needleId error: {}", e))?;- let cookie_bytes = hex::decode(cookie_hex).map_err(|e| format!("Parse cookie error: {}", e))?;-- // Pad needle id to 8 bytes- let mut nid_buf = [0u8; 8];- if needle_id_bytes.len() > 8 {- return Err(format!("KeyHash is too long."));- }- let start = 8 - needle_id_bytes.len();- nid_buf[start..].copy_from_slice(&needle_id_bytes);-- let mut key = NeedleId::from_bytes(&nid_buf[0..8]);- let cookie = Cookie::from_bytes(&cookie_bytes[0..4]);+ // Go parses both with strconv.ParseUint, which accepts odd-length hex+ let mut key = NeedleId(+ u64::from_str_radix(needle_id_hex, 16)+ .map_err(|e| format!("Parse needleId error: {}", e))?,+ );+ let cookie = Cookie(+ u32::from_str_radix(cookie_hex, 16).map_err(|e| format!("Parse cookie error: {}", e))?,+ );// Apply delta if present (Go: n.Id += Uint64ToNeedleId(d))if let Some(d) = delta {@@ -941,4 +936,18 @@ mod tests {assert_eq!(key, NeedleId(1));assert_eq!(cookie, Cookie(0x12345678));}++ #[test]+ fn test_parse_odd_length_needle_id() {+ // Go's shell formats purge fids with strconv.FormatUint, which does not+ // pad the needle id hex to an even length.+ let (key, cookie) = parse_needle_id_cookie("100000000").unwrap();+ assert_eq!(key, NeedleId(1));+ assert_eq!(cookie, Cookie(0));++ let fid = FileId::parse("3,12300000000").unwrap();+ assert_eq!(fid.volume_id, VolumeId(3));+ assert_eq!(fid.key, NeedleId(0x123));+ assert_eq!(fid.cookie, Cookie(0));+ }}diff --git a/weed/storage/needle/file_id_test.go b/weed/storage/needle/file_id_test.goindex 191c4aa96d..f3f3f226fd 100644--- a/weed/storage/needle/file_id_test.go+++ b/weed/storage/needle/file_id_test.go@@ -54,3 +54,29 @@ func TestParseFileIdFromString(t *testing.T) {t.Errorf("%s : needleId is too long", fidStr1)}}++func TestNeedleIdFileId(t *testing.T) {+ tests := []struct {+ key types.NeedleId+ volumeId uint32+ want string+ }{+ {types.NeedleId(1), 3, "3,0100000000"},+ {types.NeedleId(0x123), 3, "3,012300000000"},+ {types.NeedleId(0x1234), 7, "7,123400000000"},+ }+ for _, tt := range tests {+ fid := tt.key.FileId(tt.volumeId)+ if fid != tt.want {+ t.Errorf("NeedleId(%d).FileId(%d) = %s, want %s", tt.key, tt.volumeId, fid, tt.want)+ }+ fileId, err := ParseFileIdFromString(fid)+ if err != nil {+ t.Errorf("%s : should be OK: %v", fid, err)+ continue+ }+ if fileId.VolumeId != VolumeId(tt.volumeId) || fileId.Key != tt.key || fileId.Cookie != 0 {+ t.Errorf("src : %s, dest : %v", fid, fileId)+ }+ }+}diff --git a/weed/storage/types/needle_id_type.go b/weed/storage/types/needle_id_type.goindex a927b6e3ca..8ac3cc7ff5 100644--- a/weed/storage/types/needle_id_type.go+++ b/weed/storage/types/needle_id_type.go@@ -36,7 +36,12 @@ func (k NeedleId) String() string {}func (k NeedleId) FileId(volumeId uint32) string {- return fmt.Sprintf("%d,%s00000000", volumeId, k.String())+ // pad to even length so strict hex fid parsers accept it+ needleIdHex := k.String()+ if len(needleIdHex)%2 == 1 {+ needleIdHex = "0" + needleIdHex+ }+ return fmt.Sprintf("%d,%s00000000", volumeId, needleIdHex)}func ParseNeedleId(idString string) (NeedleId, error) {