SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
Export `file_read_invalid_needles` metric for REST read requests on invalid file IDs. (#9241)
Lisandro Pin2c404f66bc6e
diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.goindex 42263b9f80..581fb5f9bd 100644--- a/weed/server/volume_server_handlers_read.go+++ b/weed/server/volume_server_handlers_read.go@@ -57,6 +57,7 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.RelookupResult, err := operation.LookupVolumeId(vs.GetMaster, vs.grpcDialOption, volumeId.String())if err != nil || len(lookupResult.Locations) <= 0 {glog.V(0).Infoln("lookup error:", err, r.URL.Path)+ stats.VolumeServerFileReadInvalidNeedles.Inc()NotFound(w)return}@@ -65,16 +66,17 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.RelookupResult.Locations[i], lookupResult.Locations[j] = lookupResult.Locations[j], lookupResult.Locations[i]})}- var tragetUrl *url.URL+ var targetUrl *url.URLlocation := fmt.Sprintf("%s:%d", vs.store.Ip, vs.store.Port)for _, loc := range lookupResult.Locations {if !strings.Contains(loc.Url, location) {rawURL, _ := util_http.NormalizeUrl(loc.Url)- tragetUrl, _ = url.Parse(rawURL)+ targetUrl, _ = url.Parse(rawURL)break}}- if tragetUrl == nil {+ if targetUrl == nil {+ stats.VolumeServerFileReadInvalidNeedles.Inc()stats.VolumeServerHandlerCounter.WithLabelValues(stats.EmptyReadProxyLoc).Inc()glog.Errorf("failed lookup target host is empty locations: %+v, %s", lookupResult.Locations, location)NotFound(w)@@ -83,8 +85,8 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Reif vs.ReadMode == "proxy" {stats.VolumeServerHandlerCounter.WithLabelValues(stats.ReadProxyReq).Inc()// proxy client request to target server- r.URL.Host = tragetUrl.Host- r.URL.Scheme = tragetUrl.Scheme+ r.URL.Host = targetUrl.Host+ r.URL.Scheme = targetUrl.Schemequery := r.URL.Query()query.Set(reqIsProxied, "true")r.URL.RawQuery = query.Encode()@@ -125,14 +127,14 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re} else {// redirectstats.VolumeServerHandlerCounter.WithLabelValues(stats.ReadRedirectReq).Inc()- tragetUrl.Path = fmt.Sprintf("%s/%s,%s", tragetUrl.Path, vid, fid)+ targetUrl.Path = fmt.Sprintf("%s/%s,%s", targetUrl.Path, vid, fid)arg := url.Values{}if c := r.FormValue("collection"); c != "" {arg.Set("collection", c)}arg.Set(reqIsProxied, "true")- tragetUrl.RawQuery = arg.Encode()- http.Redirect(w, r, tragetUrl.String(), http.StatusMovedPermanently)+ targetUrl.RawQuery = arg.Encode()+ http.Redirect(w, r, targetUrl.String(), http.StatusMovedPermanently)return}}@@ -164,6 +166,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)_, hasEcVolume := vs.store.FindEcVolume(volumeId)if !hasVolume && !hasEcVolume {if vs.ReadMode == "local" {+ stats.VolumeServerFileReadInvalidNeedles.Inc()glog.V(0).Infoln("volume is not local:", err, r.URL.Path)NotFound(w)return@@ -207,7 +210,11 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)// glog.V(4).Infoln("read bytes", count, "error", err)if err != nil || count < 0 {glog.V(3).Infof("read %s isNormalVolume %v error: %v", r.URL.Path, hasVolume, err)- if err == storage.ErrorNotFound || err == storage.ErrorDeleted || errors.Is(err, erasure_coding.NotFoundError) {+ invalid_needle := err == storage.ErrorNotFound || errors.Is(err, erasure_coding.NotFoundError)+ if invalid_needle {+ stats.VolumeServerFileReadInvalidNeedles.Inc()+ }+ if invalid_needle || err == storage.ErrorDeleted {NotFound(w)} else {InternalError(w)diff --git a/weed/stats/metrics.go b/weed/stats/metrics.goindex b7468b7f59..e943f5726f 100644--- a/weed/stats/metrics.go+++ b/weed/stats/metrics.go@@ -378,6 +378,14 @@ var (Help: "Counter of overall failed file read requests from clients.",})+ VolumeServerFileReadInvalidNeedles = prometheus.NewCounter(+ prometheus.CounterOpts{+ Namespace: Namespace,+ Subsystem: "volumeServer",+ Name: "file_read_invalid_needles",+ Help: "Counter of failed file read requests due to invalid needle IDs from clients.",+ })+VolumeServerFileWriteFailures = prometheus.NewCounter(prometheus.CounterOpts{Namespace: Namespace,@@ -554,6 +562,7 @@ func init() {Gather.MustRegister(VolumeServerInFlightUploadSize)Gather.MustRegister(VolumeServerMasterDisconnections)Gather.MustRegister(VolumeServerFileReadFailures)+ Gather.MustRegister(VolumeServerFileReadInvalidNeedles)Gather.MustRegister(VolumeServerFileWriteFailures)Gather.MustRegister(S3RequestCounter)