SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package operation
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"github.com/seaweedfs/seaweedfs/weed/pb"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
type DeleteResult struct {
Fid string `json:"fid"`
Size int `json:"size"`
Status int `json:"status"`
Error string `json:"error,omitempty"`
}
func ParseFileId(fid string) (vid string, key_cookie string, err error) {
commaIndex := strings.Index(fid, ",")
if commaIndex <= 0 {
return "", "", errors.New("Wrong fid format.")
}
return fid[:commaIndex], fid[commaIndex+1:], nil
}
// DeleteFileIds batch deletes a list of fileIds
// Returns individual results for each file ID. Check result.Error for per-file failures.
func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) []*volume_server_pb.DeleteResult {
lookupFunc := func(vids []string) (results map[string]*LookupResult, err error) {
results, err = LookupVolumeIds(masterFn, grpcDialOption, vids)
if err == nil && usePublicUrl {
for _, result := range results {
for _, loc := range result.Locations {
loc.Url = loc.PublicUrl
}
}
}
return
}
return DeleteFileIdsWithLookupVolumeId(context.Background(), grpcDialOption, fileIds, lookupFunc)
}
func DeleteFileIdsWithLookupVolumeId(ctx context.Context, grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult {
var ret []*volume_server_pb.DeleteResult
vid_to_fileIds := make(map[string][]string)
var vids []string
for _, fileId := range fileIds {
vid, _, err := ParseFileId(fileId)
if err != nil {
ret = append(ret, &volume_server_pb.DeleteResult{
FileId: fileId,
Status: http.StatusBadRequest,
Error: err.Error()},
)
continue
}
if _, ok := vid_to_fileIds[vid]; !ok {
vid_to_fileIds[vid] = make([]string, 0)
vids = append(vids, vid)
}
vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
}
lookupResults, err := lookupFunc(vids)
if err != nil {
// Lookup failed - return error results for all file IDs that passed parsing
for _, fids := range vid_to_fileIds {
for _, fileId := range fids {
ret = append(ret, &volume_server_pb.DeleteResult{
FileId: fileId,
Status: http.StatusInternalServerError,
Error: fmt.Sprintf("lookup error: %v", err),
})
}
}
return ret
}
server_to_fileIds := make(map[pb.ServerAddress][]string)
for vid, result := range lookupResults {
if result.Error != "" {
// Lookup error for this volume - mark all its files as failed
for _, fileId := range vid_to_fileIds[vid] {
ret = append(ret, &volume_server_pb.DeleteResult{
FileId: fileId,
Status: http.StatusBadRequest,
Error: result.Error},
)
}
continue
}
for _, location := range result.Locations {
serverAddress := location.ServerAddress()
if _, ok := server_to_fileIds[serverAddress]; !ok {
server_to_fileIds[serverAddress] = make([]string, 0)
}
server_to_fileIds[serverAddress] = append(
server_to_fileIds[serverAddress], vid_to_fileIds[vid]...)
}
}
resultChan := make(chan []*volume_server_pb.DeleteResult, len(server_to_fileIds))
var wg sync.WaitGroup
for server, fidList := range server_to_fileIds {
wg.Add(1)
go func(server pb.ServerAddress, fidList []string) {
defer wg.Done()
resultChan <- DeleteFileIdsAtOneVolumeServer(ctx, server, grpcDialOption, fidList, false)
}(server, fidList)
}
wg.Wait()
close(resultChan)
for result := range resultChan {
ret = append(ret, result...)
}
return ret
}
// DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
// Returns individual results for each file ID. Check result.Error for per-file failures.
func DeleteFileIdsAtOneVolumeServer(ctx context.Context, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult {
var ret []*volume_server_pb.DeleteResult
err := WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
req := &volume_server_pb.BatchDeleteRequest{
FileIds: fileIds,
SkipCookieCheck: !includeCookie,
}
resp, err := volumeServerClient.BatchDelete(ctx, req)
// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)
if err != nil {
return err
}
ret = append(ret, resp.Results...)
return nil
})
if err != nil {
// Connection or communication error - return error results for all files
ret = make([]*volume_server_pb.DeleteResult, 0, len(fileIds))
for _, fileId := range fileIds {
ret = append(ret, &volume_server_pb.DeleteResult{
FileId: fileId,
Status: http.StatusInternalServerError,
Error: err.Error(),
})
}
}
return ret
}