Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

7.5 KB
Raw Blame
package source

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strings"

	"google.golang.org/grpc"

	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/security"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
)

// ErrVolumeNotFound reports that the source cluster has no location for a
// chunk's volume: vacuumed away, deleted, or every replica offline. Callers
// need it apart from a lookup that simply failed, which a later attempt can
// still get past.
var ErrVolumeNotFound = errors.New("volume not found")

// ErrChunkNotFound reports that every located volume replica answered 404 for
// the chunk. A failed request or a filer-proxied 404 cannot establish this.
var ErrChunkNotFound = errors.New("chunk not found on any volume replica")

type FilerSource struct {
	grpcAddress    string
	grpcDialOption grpc.DialOption
	Dir            string
	address        string
	proxyByFiler   bool
	dataCenter     string
	signature      int32
	httpClient     *util_http_client.HTTPClient
	jwtForFiler    security.FilerJwtProvider
}

func (fs *FilerSource) Initialize(configuration util.Configuration, prefix string) error {
	fs.dataCenter = configuration.GetString(prefix + "dataCenter")
	fs.signature = util.RandomInt32()
	return fs.DoInitialize(
		"",
		configuration.GetString(prefix+"grpcAddress"),
		configuration.GetString(prefix+"directory"),
		false,
	)
}

func (fs *FilerSource) DoInitialize(address, grpcAddress string, dir string, readChunkFromFiler bool) (err error) {
	fs.address = address
	if fs.address == "" {
		fs.address = pb.GrpcAddressToServerAddress(grpcAddress)
	}
	fs.grpcAddress = grpcAddress
	fs.Dir = dir
	fs.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
	fs.proxyByFiler = readChunkFromFiler
	return nil
}

func (fs *FilerSource) SetGrpcDialOption(option grpc.DialOption) {
	fs.grpcDialOption = option
}

func (fs *FilerSource) SetHttpClient(client *util_http_client.HTTPClient) {
	fs.httpClient = client
}

func (fs *FilerSource) SetFilerJwtProvider(provider security.FilerJwtProvider) {
	fs.jwtForFiler = provider
}

// FilerJwt returns the side-specific filer API credential, or nil when the
// process-wide jwt.filer_signing configuration applies.
// ProxyByFiler reports whether chunk reads are proxied through the source
// filer. A proxied 404 can also mean the filer found no volume locations, so it
// does not isolate a missing needle the way a direct volume read's 404 does.
func (fs *FilerSource) ProxyByFiler() bool {
	return fs.proxyByFiler
}

func (fs *FilerSource) FilerJwt() security.FilerJwtProvider {
	return fs.jwtForFiler
}

func (fs *FilerSource) LookupFileId(ctx context.Context, part string) (fileUrls []string, err error) {

	vid2Locations := make(map[string]*filer_pb.Locations)

	vid := volumeId(part)

	err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
			VolumeIds: []string{vid},
		})
		if err != nil {
			return err
		}

		vid2Locations = resp.LocationsMap

		return nil
	})

	if err != nil {
		glog.V(1).InfofCtx(ctx, "LookupFileId volume id %s: %v", vid, err)
		return nil, fmt.Errorf("LookupFileId volume id %s: %v", vid, err)
	}

	locations := vid2Locations[vid]

	if locations == nil || len(locations.Locations) == 0 {
		glog.V(1).InfofCtx(ctx, "LookupFileId locate volume id %s: %v", vid, ErrVolumeNotFound)
		return nil, fmt.Errorf("LookupFileId locate volume id %s: %w", vid, ErrVolumeNotFound)
	}

	if !fs.proxyByFiler {
		for _, loc := range locations.Locations {
			fileUrl := fmt.Sprintf("http://%s/%s?readDeleted=true", loc.Url, part)
			// Prefer same data center
			if fs.dataCenter != "" && fs.dataCenter == loc.DataCenter {
				fileUrls = append([]string{fileUrl}, fileUrls...)
			} else {
				fileUrls = append(fileUrls, fileUrl)
			}
		}
	} else {
		fileUrls = append(fileUrls, util_http.ProxyChunkUrl(fs.address, part))
	}

	return
}

func (fs *FilerSource) ReadPart(fileId string, offset int64) (filename string, header http.Header, resp *http.Response, err error) {
	downloadFn := util_http.DownloadFile
	if fs.httpClient != nil {
		downloadFn = func(fileUrl string, jwt string, offset ...int64) (string, http.Header, *http.Response, error) {
			return util_http.DownloadFileWithClient(fs.httpClient, fileUrl, jwt, offset...)
		}
	}

	if fs.proxyByFiler {
		fileUrl := util_http.ProxyChunkUrl(fs.address, fileId)
		jwt := util_http.JwtForFilerServer(false)
		if fs.jwtForFiler != nil {
			jwt = string(fs.jwtForFiler(false))
		}
		filename, header, resp, err = downloadFn(fileUrl, jwt, offset)
		if err == nil {
			err = readPartStatusError(fileUrl, resp)
		}
		if err != nil {
			glog.V(0).Infof("read part %s via filer proxy %s offset %d: %v", fileId, fs.address, offset, err)
			return "", nil, nil, err
		}
		glog.V(4).Infof("read part %s via filer proxy %s offset %d content-length:%s", fileId, fs.address, offset, header.Get("Content-Length"))
		return
	}

	fileUrls, err := fs.LookupFileId(context.Background(), fileId)
	if err != nil {
		return "", nil, nil, err
	}

	var replicaErr error
	for _, fileUrl := range fileUrls {
		filename, header, resp, err = downloadFn(fileUrl, "", offset)
		if err == nil {
			err = readPartStatusError(fileUrl, resp)
		}
		if err != nil {
			resp = nil
			if !errors.Is(err, util_http.ErrNotFound) {
				replicaErr = err
			}
			glog.V(0).Infof("fail to read part %s from %s offset %d: %v", fileId, fileUrl, offset, err)
			continue
		}
		glog.V(4).Infof("read part %s from %s offset %d content-length:%s", fileId, fileUrl, offset, header.Get("Content-Length"))
		break
	}
	if err != nil {
		if replicaErr != nil {
			// An unreachable replica may still hold the chunk. Its failure must
			// not be hidden by another replica answering 404 later in the list.
			err = replicaErr
		} else if errors.Is(err, util_http.ErrNotFound) {
			err = fmt.Errorf("%w: %w", ErrChunkNotFound, err)
		}
	}

	return filename, header, resp, err
}

// readPartStatusError turns a failure status into an error and closes the
// response. Otherwise the caller copies the error page as chunk content and
// reports it as a short read; a needle vacuum has removed answers 404, which
// is the source having lost the data rather than corruption.
func readPartStatusError(fileUrl string, resp *http.Response) error {
	if resp == nil || resp.StatusCode < http.StatusBadRequest {
		return nil
	}
	defer util_http.CloseResponse(resp)
	if resp.StatusCode == http.StatusNotFound {
		return fmt.Errorf("%s: %s: %w", fileUrl, resp.Status, util_http.ErrNotFound)
	}
	return fmt.Errorf("%s: %s", fileUrl, resp.Status)
}

var _ = filer_pb.FilerClient(&FilerSource{})

func (fs *FilerSource) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {

	return pb.WithGrpcClient(context.Background(), streamingMode, fs.signature, func(grpcConnection *grpc.ClientConn) error {
		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
		return fn(client)
	}, fs.grpcAddress, false, fs.grpcDialOption)

}

func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string {
	return location.Url
}

func (fs *FilerSource) GetDataCenter() string {
	return fs.dataCenter
}

func volumeId(fileId string) string {
	lastCommaIndex := strings.LastIndex(fileId, ",")
	if lastCommaIndex > 0 {
		return fileId[:lastCommaIndex]
	}
	return fileId
}