SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package shell
import (
"context"
"flag"
"io"
"log"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
)
func init() {
Commands = append(Commands, &commandVolumeDeleteEmpty{})
}
type commandVolumeDeleteEmpty struct {
}
func (c *commandVolumeDeleteEmpty) Name() string {
return "volume.deleteEmpty"
}
func (c *commandVolumeDeleteEmpty) Help() string {
return `delete empty volumes from all volume servers
volume.deleteEmpty -quietFor=24h -apply
volume.deleteEmpty -collectionPattern=important* -quietFor=24h -apply
This command deletes all empty volumes from one volume server.
A volume with no live needles left is empty too, even when its
.dat file is still large: compacting it first would only rewrite
bytes that are all deleted already.
`
}
func (c *commandVolumeDeleteEmpty) HasTag(CommandTag) bool {
return false
}
func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones")
collectionPattern := volDeleteCommand.String("collectionPattern", "", "comma-separated collection names, with '*' and '?' wildcards; empty matches all")
applyBalancing := volDeleteCommand.Bool("apply", false, "apply to delete empty volumes")
// TODO: remove this alias
applyBalancingAlias := volDeleteCommand.Bool("force", false, "apply to delete empty volumes (alias for -apply)")
if err = volDeleteCommand.Parse(args); err != nil {
return err
}
handleDeprecatedForceFlag(writer, volDeleteCommand, applyBalancingAlias, applyBalancing)
infoAboutSimulationMode(writer, *applyBalancing, "-apply")
if err = commandEnv.confirmIsLocked(args); err != nil {
return
}
// collect topology information
topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
if err != nil {
return err
}
collectionMatcher, err := wildcard.CompileCollectionMatcher(*collectionPattern)
if err != nil {
return err
}
quietSeconds := int64(*quietPeriod / time.Second)
nowUnixSeconds := time.Now().Unix()
eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
for _, diskInfo := range dn.DiskInfos {
for _, v := range diskInfo.VolumeInfos {
if isEmptyVolumeDeleteCandidate(v, quietSeconds, nowUnixSeconds, collectionMatcher) {
if *applyBalancing {
log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
onlyGarbage := v.FileCount > 0 && v.FileCount <= v.DeleteCount
// onlyEmpty stays set so a pre-upgrade server checks
// emptiness and refuses instead of deleting unseen.
if deleteErr := deleteVolume(context.Background(), commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id),
pb.NewServerAddressFromDataNode(dn), true, onlyGarbage, false); deleteErr != nil {
err = deleteErr
}
continue
} else {
log.Printf("empty volume %d from %s", v.Id, dn.Id)
}
}
}
}
})
return
}
func isEmptyVolumeDeleteCandidate(v *master_pb.VolumeInformationMessage, quietSeconds, nowUnixSeconds int64, collectionMatcher *wildcard.CollectionMatcher) bool {
return collectionMatcher.Matches(v.Collection) &&
// A remote-backed replica shares its cloud object with the others, so
// deleting one cannot drop it without hurting the survivors.
v.RemoteStorageName == "" &&
(!v.ReadOnly || v.ReadOnlyCanDelete) &&
(v.Size <= super_block.SuperBlockSize || v.FileCount > 0 && v.FileCount <= v.DeleteCount) &&
v.ModifiedAtSecond > 0 &&
v.ModifiedAtSecond+quietSeconds < nowUnixSeconds
}