SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package operation
import (
"context"
"io"
"math/rand/v2"
"mime"
"net/url"
"os"
"path"
"strconv"
"strings"
"sync"
"github.com/seaweedfs/seaweedfs/weed/pb"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/security"
)
type FilePart struct {
Reader io.Reader
FileName string
FileSize int64
MimeType string
ModTime int64 //in seconds
Pref StoragePreference
Server string //this comes from assign result
Fid string //this comes from assign result, but customizable
Fsync bool
}
type SubmitResult struct {
FileName string `json:"fileName,omitempty"`
FileUrl string `json:"url,omitempty"`
Fid string `json:"fid,omitempty"`
Size uint32 `json:"size,omitempty"`
Error string `json:"error,omitempty"`
}
type StoragePreference struct {
Replication string
Collection string
DataCenter string
Ttl string
DiskType string
MaxMB int
}
type GetMasterFn func(ctx context.Context) pb.ServerAddress
// submitUploadWorkers bounds how many files one SubmitFiles call uploads at
// once; the batch assign above already fetched all their file ids.
const submitUploadWorkers = 4
func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
results := make([]SubmitResult, len(files))
var totalBytes int64
for index, file := range files {
results[index].FileName = file.FileName
totalBytes += file.FileSize
}
var avgBytes uint64
if n := len(files); n > 0 {
avgBytes = uint64((totalBytes + int64(n) - 1) / int64(n))
}
ar := &VolumeAssignRequest{
Count: uint64(len(files)),
Replication: pref.Replication,
Collection: pref.Collection,
DataCenter: pref.DataCenter,
Ttl: pref.Ttl,
DiskType: pref.DiskType,
ExpectedDataSize: avgBytes,
}
ret, err := Assign(context.Background(), masterFn, grpcDialOption, ar)
if err != nil {
for index, file := range files {
results[index].Error = err.Error()
if closer, ok := file.Reader.(io.Closer); ok {
closer.Close()
}
}
return results, err
}
for index, file := range files {
file.Fid = ret.Fid
if index > 0 {
file.Fid = file.Fid + "_" + strconv.Itoa(index)
}
file.Server = ret.Url
if usePublicUrl {
file.Server = ret.PublicUrl
}
file.Pref = pref
results[index].Fid = file.Fid
results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
}
var wg sync.WaitGroup
var lastErrLock sync.Mutex
jobs := make(chan int)
for w := 0; w < min(submitUploadWorkers, len(files)); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for index := range jobs {
size, uploadErr := files[index].Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
results[index].Size = size
if uploadErr != nil {
results[index].Error = uploadErr.Error()
lastErrLock.Lock()
err = uploadErr
lastErrLock.Unlock()
}
}
}()
}
for index := range files {
jobs <- index
}
close(jobs)
wg.Wait()
return results, err
}
func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
ret = make([]*FilePart, len(fullPathFilenames))
for index, file := range fullPathFilenames {
if ret[index], err = newFilePart(file); err != nil {
return
}
}
return
}
func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
ret = &FilePart{}
fh, openErr := os.Open(fullPathFilename)
if openErr != nil {
glog.V(0).Info("Failed to open file: ", fullPathFilename)
return ret, openErr
}
ret.Reader = fh
fi, fiErr := fh.Stat()
if fiErr != nil {
glog.V(0).Info("Failed to stat file:", fullPathFilename)
fh.Close()
ret.Reader = nil
return ret, fiErr
}
ret.ModTime = fi.ModTime().UTC().Unix()
ret.FileSize = fi.Size()
ext := strings.ToLower(path.Ext(fullPathFilename))
ret.FileName = fi.Name()
if ext != "" {
ret.MimeType = mime.TypeByExtension(ext)
}
return ret, nil
}
func (fi *FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jwt security.EncodedJwt, grpcDialOption grpc.DialOption) (retSize uint32, err error) {
fileUrl := "http://" + fi.Server + "/" + fi.Fid
if fi.ModTime != 0 {
fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
}
if fi.Fsync {
fileUrl += "?fsync=true"
}
if closer, ok := fi.Reader.(io.Closer); ok {
defer closer.Close()
}
baseName := path.Base(fi.FileName)
if maxMB > 0 && fi.FileSize > int64(maxMB*1024*1024) {
chunkSize := int64(maxMB * 1024 * 1024)
chunks := fi.FileSize/chunkSize + 1
cm := ChunkManifest{
Name: baseName,
Size: fi.FileSize,
Mime: fi.MimeType,
Chunks: make([]*ChunkInfo, 0, chunks),
}
var ret *AssignResult
var id string
if fi.Pref.DataCenter != "" {
ar := &VolumeAssignRequest{
Count: uint64(chunks),
Replication: fi.Pref.Replication,
Collection: fi.Pref.Collection,
Ttl: fi.Pref.Ttl,
DiskType: fi.Pref.DiskType,
ExpectedDataSize: uint64(chunkSize),
}
ret, err = Assign(context.Background(), masterFn, grpcDialOption, ar)
if err != nil {
return
}
}
for i := int64(0); i < chunks; i++ {
if fi.Pref.DataCenter == "" {
remaining := fi.FileSize - i*chunkSize
thisChunk := chunkSize
if remaining < thisChunk {
thisChunk = remaining
}
ar := &VolumeAssignRequest{
Count: 1,
Replication: fi.Pref.Replication,
Collection: fi.Pref.Collection,
Ttl: fi.Pref.Ttl,
DiskType: fi.Pref.DiskType,
ExpectedDataSize: uint64(thisChunk),
}
ret, err = Assign(context.Background(), masterFn, grpcDialOption, ar)
if err != nil {
// delete all uploaded chunks
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
return
}
id = ret.Fid
} else {
id = ret.Fid
if i > 0 {
id += "_" + strconv.FormatInt(i, 10)
}
}
fileUrl := genFileUrl(ret, id, usePublicUrl)
count, e := uploadOneChunk(
baseName+"-"+strconv.FormatInt(i+1, 10),
io.LimitReader(fi.Reader, chunkSize),
masterFn, fileUrl,
ret.Auth)
if e != nil {
// delete all uploaded chunks
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
return 0, e
}
cm.Chunks = append(cm.Chunks,
&ChunkInfo{
Offset: i * chunkSize,
Size: int64(count),
Fid: id,
},
)
retSize += count
}
err = uploadChunkedFileManifest(fileUrl, &cm, jwt)
if err != nil {
// delete all uploaded chunks
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
}
} else {
uploadOption := &UploadOption{
UploadUrl: fileUrl,
Filename: baseName,
Cipher: false,
IsInputCompressed: false,
MimeType: fi.MimeType,
PairMap: nil,
Jwt: jwt,
}
uploader, e := NewUploader()
if e != nil {
return 0, e
}
ret, e, _ := uploader.Upload(context.Background(), fi.Reader, uploadOption)
if e != nil {
return 0, e
}
return ret.Size, e
}
return
}
func genFileUrl(ret *AssignResult, id string, usePublicUrl bool) string {
fileUrl := "http://" + ret.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + ret.PublicUrl + "/" + id
}
for _, replica := range ret.Replicas {
if rand.IntN(len(ret.Replicas)+1) == 0 {
fileUrl = "http://" + replica.Url + "/" + id
if usePublicUrl {
fileUrl = "http://" + replica.PublicUrl + "/" + id
}
}
}
return fileUrl
}
func uploadOneChunk(filename string, reader io.Reader, masterFn GetMasterFn,
fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
uploadOption := &UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: false,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: jwt,
}
uploader, uploaderError := NewUploader()
if uploaderError != nil {
return 0, uploaderError
}
uploadResult, uploadError, _ := uploader.Upload(context.Background(), reader, uploadOption)
if uploadError != nil {
return 0, uploadError
}
return uploadResult.Size, nil
}
func uploadChunkedFileManifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
buf, e := manifest.Marshal()
if e != nil {
return e
}
glog.V(4).Info("Uploading chunks manifest ", manifest.Name, " to ", fileUrl, "...")
u, _ := url.Parse(fileUrl)
q := u.Query()
q.Set("cm", "true")
u.RawQuery = q.Encode()
uploadOption := &UploadOption{
UploadUrl: u.String(),
Filename: manifest.Name,
Cipher: false,
IsInputCompressed: false,
MimeType: "application/json",
PairMap: nil,
Jwt: jwt,
}
uploader, e := NewUploader()
if e != nil {
return e
}
_, e = uploader.UploadData(context.Background(), buf, uploadOption)
return e
}