Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3vectors: bound QueryVectors topK at the AWS limit (#12202)

Chris Lufc9be77dbb95
diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go
index 791d9ad983..1a77c21fa5 100644
--- a/weed/s3api/s3vectors/handlers.go
+++ b/weed/s3api/s3vectors/handlers.go
@@ -995,12 +995,13 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) {
})
}
-// Mirrors the engine's page sizing so a segmented listing pages like a plain
-// one does.
+// The page constants mirror the engine's page sizing so a segmented listing
+// pages like a plain one does; vectorsMaxTopK is the AWS QueryVectors limit.
const (
vectorsDefaultPage = 500
vectorsMaxPutBatch = 500
vectorsMaxPage = 1000
+ vectorsMaxTopK = 10000
)
// listSegmentVectors scans the index from offset, keeping the rows that fall
@@ -1360,6 +1361,11 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", "queryVector is required")
return
}
+ if req.TopK <= 0 || req.TopK > vectorsMaxTopK {
+ writeError(w, http.StatusBadRequest, "ValidationException",
+ fmt.Sprintf("topK must be between 1 and %d", vectorsMaxTopK))
+ return
+ }
record, err := s.indexRecord(r.Context(), r, arns.bucket, arns.index)
if err != nil {
catalogErr(w, err)
@@ -1382,10 +1388,6 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "ValidationException", err.Error())
return
}
- if req.TopK <= 0 {
- writeError(w, http.StatusBadRequest, "ValidationException", "topK must be positive")
- return
- }
resp, err := s.vectors.QueryVectors(r.Context(), &vector_pb.QueryVectorsRequest{
Table: tableID(arns.bucket, arns.index),
QueryVector: queryVector,
diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go
index 214c8c9dce..5776251971 100644
--- a/weed/s3api/s3vectors/handlers_test.go
+++ b/weed/s3api/s3vectors/handlers_test.go
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
+ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/vector_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -507,3 +508,42 @@ func TestPutVectorsUsesCatalogDimension(t *testing.T) {
t.Fatalf("engine PutVectors calls = %d, want a second call of %d vectors", len(engine.puts), vectorsMaxPutBatch)
}
}
+
+// errFiler fails any catalog access, so a request that passes request-shape
+// validation dies in the catalog rather than reaching the engine.
+type errFiler struct{}
+
+func (errFiler) WithFilerClient(_ bool, _ func(filer_pb.SeaweedFilerClient) error) error {
+ return fmt.Errorf("no filer")
+}
+
+// Rejected topK values fail before the catalog and the engine; valid ones get
+// past the gate (here: into the catalog, which the errFiler fails).
+func TestQueryVectorsTopKBounds(t *testing.T) {
+ s := &Server{
+ vectors: newFakeVectors(0),
+ tables: s3tables.NewManager(),
+ filerClient: errFiler{},
+ }
+ for _, tc := range []struct {
+ name string
+ topK int
+ rejected bool
+ }{
+ {"zero", 0, true},
+ {"negative", -3, true},
+ {"over the limit", vectorsMaxTopK + 1, true},
+ {"one", 1, false},
+ {"at the limit", vectorsMaxTopK, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ body := fmt.Sprintf(`{"vectorBucketName":"b","indexName":"i","queryVector":{"float32":[0.1]},"topK":%d}`, tc.topK)
+ w := httptest.NewRecorder()
+ s.handleQueryVectors(w, httptest.NewRequest(http.MethodPost, "/QueryVectors", strings.NewReader(body)))
+ rejected := w.Code == http.StatusBadRequest && strings.Contains(w.Body.String(), "topK must be between")
+ if rejected != tc.rejected {
+ t.Fatalf("topK %d: rejected=%v (%d %s), want %v", tc.topK, rejected, w.Code, w.Body.String(), tc.rejected)
+ }
+ })
+ }
+}