SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package shell
import (
"context"
"flag"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/ec"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func init() {
Commands = append(Commands, &commandEcDecode{})
}
type commandEcDecode struct {
}
func (c *commandEcDecode) Name() string {
return "ec.decode"
}
func (c *commandEcDecode) Help() string {
return `decode a erasure coded volume into a normal volume
ec.decode [-collection=""] [-volumeId=<volume_id>] [-batchSize=10] [-diskType=<disk_type>] [-checkMinFreeSpace]
The -collection parameter supports regular expressions for pattern matching:
- Use exact match: ec.decode -collection="^mybucket$"
- Match multiple buckets: ec.decode -collection="bucket.*"
- Match all collections: ec.decode -collection=".*"
Options:
-diskType: source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)
-checkMinFreeSpace: check min free space when selecting the decode target (default true)
-batchSize: decode this many volumes per topology refresh (default 10; 0 = one snapshot for all volumes)
Examples:
# Decode EC shards from HDD (default)
ec.decode -collection=mybucket
# Decode EC shards from SSD
ec.decode -collection=mybucket -diskType=ssd
`
}
func (c *commandEcDecode) HasTag(CommandTag) bool {
return false
}
func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
decodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
volumeId := decodeCommand.Int("volumeId", 0, "the volume id")
collection := decodeCommand.String("collection", "", "comma-separated collection names, wildcards, or regex patterns; empty matches the collection with no name")
diskTypeStr := decodeCommand.String("diskType", "", "source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)")
checkMinFreeSpace := decodeCommand.Bool("checkMinFreeSpace", true, "check min free space when selecting the decode target")
batchSize := decodeCommand.Int("batchSize", DefaultEcBatchSize, "decode up to this many volumes per topology refresh (0 = one snapshot for all volumes)")
if err = decodeCommand.Parse(args); err != nil {
return err
}
if *batchSize < 0 {
return fmt.Errorf("-batchSize must be >= 0")
}
if err = commandEnv.confirmIsLocked(args); err != nil {
return
}
vid := needle.VolumeId(*volumeId)
diskType := types.ToDiskType(*diskTypeStr)
env := commandEnv.ecEnv()
// collect topology information
topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
if err != nil {
return err
}
var diskUsageState *ec.DecodeDiskUsageState
if *checkMinFreeSpace {
diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType)
}
// volumeId is provided
if vid != 0 {
return ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState)
}
// apply to all volumes in the collection
collectionMatcher, err := compileCollectionPattern(*collection)
if err != nil {
return fmt.Errorf("invalid collection pattern '%s': %v", *collection, err)
}
volumeIds := ec.CollectEcShardIds(topologyInfo, collectionMatcher, diskType)
fmt.Printf("ec decode volumes: %v\n", volumeIds)
batches := chunkVolumeIds(volumeIds, *batchSize)
for i, batch := range batches {
if i > 0 {
// earlier batches moved shards and created volumes; re-snapshot so
// shard locations and free-space accounting stay accurate
topologyInfo, _, err = collectTopologyInfo(commandEnv, 0)
if err != nil {
return err
}
if *checkMinFreeSpace {
diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType)
}
}
if len(batches) > 1 {
fmt.Printf("ec decode batch %d/%d: %v\n", i+1, len(batches), batch)
}
for _, vid := range batch {
// -collection is a selector, not the volume's collection
volumeCollection, _ := ec.FindEcVolumeCollection(topologyInfo, vid)
if err = ec.DoEcDecode(env, topologyInfo, volumeCollection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil {
return err
}
}
}
return nil
}
func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) {
var resp *master_pb.LookupVolumeResponse
err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
resp, err = client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: volumeIds})
return err
})
if err != nil {
return nil, err
}
return resp.VolumeIdLocations, nil
}