SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package volume_server_http_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"mime/multipart"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
func TestStatsEndpoints(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
// /stats/counter — expect 200 with non-empty body
// Note: Go server guards these with WhiteList which may return 400
counterResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/counter"))
counterBody := framework.ReadAllAndClose(t, counterResp)
if counterResp.StatusCode == http.StatusBadRequest {
t.Logf("/stats/counter returned 400 (whitelist guard), skipping stats checks")
return
}
if counterResp.StatusCode != http.StatusOK {
t.Fatalf("/stats/counter expected 200, got %d, body: %s", counterResp.StatusCode, string(counterBody))
}
if len(counterBody) == 0 {
t.Fatalf("/stats/counter returned empty body")
}
// /stats/memory — expect 200, valid JSON with Version and Memory
memoryResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/memory"))
memoryBody := framework.ReadAllAndClose(t, memoryResp)
if memoryResp.StatusCode != http.StatusOK {
t.Fatalf("/stats/memory expected 200, got %d, body: %s", memoryResp.StatusCode, string(memoryBody))
}
var memoryPayload map[string]any
if err := json.Unmarshal(memoryBody, &memoryPayload); err != nil {
t.Fatalf("/stats/memory response is not valid JSON: %v, body: %s", err, string(memoryBody))
}
if _, ok := memoryPayload["Version"]; !ok {
t.Fatalf("/stats/memory missing Version field")
}
if _, ok := memoryPayload["Memory"]; !ok {
t.Fatalf("/stats/memory missing Memory field")
}
// /stats/disk — expect 200, valid JSON with Version and DiskStatuses
diskResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/stats/disk"))
diskBody := framework.ReadAllAndClose(t, diskResp)
if diskResp.StatusCode != http.StatusOK {
t.Fatalf("/stats/disk expected 200, got %d, body: %s", diskResp.StatusCode, string(diskBody))
}
var diskPayload map[string]any
if err := json.Unmarshal(diskBody, &diskPayload); err != nil {
t.Fatalf("/stats/disk response is not valid JSON: %v, body: %s", err, string(diskBody))
}
if _, ok := diskPayload["Version"]; !ok {
t.Fatalf("/stats/disk missing Version field")
}
if _, ok := diskPayload["DiskStatuses"]; !ok {
t.Fatalf("/stats/disk missing DiskStatuses field")
}
}
func TestStatusPrettyJsonAndCallbackIgnored(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
// ?pretty=y — expect indented multi-line JSON
prettyResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?pretty=y"))
prettyBody := framework.ReadAllAndClose(t, prettyResp)
if prettyResp.StatusCode != http.StatusOK {
t.Fatalf("/status?pretty=y expected 200, got %d", prettyResp.StatusCode)
}
lines := strings.Split(strings.TrimSpace(string(prettyBody)), "\n")
if len(lines) < 3 {
t.Fatalf("/status?pretty=y expected multi-line indented JSON, got %d lines: %s", len(lines), string(prettyBody))
}
var prettyPayload map[string]interface{}
if err := json.Unmarshal(prettyBody, &prettyPayload); err != nil {
t.Fatalf("/status?pretty=y is not valid JSON: %v", err)
}
// ?callback=myFunc — must be ignored; response is plain JSON with nosniff.
cbResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?callback=myFunc"))
cbBody := framework.ReadAllAndClose(t, cbResp)
if cbResp.StatusCode != http.StatusOK {
t.Fatalf("/status?callback=myFunc expected 200, got %d", cbResp.StatusCode)
}
if ct := cbResp.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
t.Fatalf("/status?callback=myFunc expected Content-Type application/json, got %q", ct)
}
if nosniff := cbResp.Header.Get("X-Content-Type-Options"); nosniff != "nosniff" {
t.Fatalf("/status?callback=myFunc expected X-Content-Type-Options nosniff, got %q", nosniff)
}
if strings.Contains(string(cbBody), "myFunc(") {
t.Fatalf("/status?callback=myFunc must not wrap response in callback; body: %q", string(cbBody))
}
var cbPayload map[string]interface{}
if err := json.Unmarshal(cbBody, &cbPayload); err != nil {
t.Fatalf("/status?callback=myFunc is not valid JSON: %v", err)
}
}
func TestUploadWithCustomTimestamp(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(91)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 910001, 0xAABBCC01)
client := framework.NewHTTPClient()
data := []byte("custom-timestamp-data")
// Upload with ?ts=1700000000
uploadURL := fmt.Sprintf("%s/%s?ts=1700000000", cluster.VolumeAdminURL(), fid)
req, err := http.NewRequest(http.MethodPost, uploadURL, bytes.NewReader(data))
if err != nil {
t.Fatalf("create upload request: %v", err)
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
uploadResp := framework.DoRequest(t, client, req)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload with ts expected 201, got %d", uploadResp.StatusCode)
}
// Read back and verify Last-Modified
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read expected 200, got %d", getResp.StatusCode)
}
expectedLastModified := time.Unix(1700000000, 0).UTC().Format(http.TimeFormat)
gotLastModified := getResp.Header.Get("Last-Modified")
if gotLastModified != expectedLastModified {
t.Fatalf("Last-Modified mismatch: got %q, want %q", gotLastModified, expectedLastModified)
}
}
func TestMultipartUploadUsesFormFieldsForTimestampAndTTL(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
// Go's r.FormValue() cannot read multipart text fields after r.MultipartReader()
// consumes the body, so ts/ttl sent as multipart fields only work with the Rust server.
if os.Getenv("VOLUME_SERVER_IMPL") != "rust" {
t.Skip("skipping: multipart form field extraction for ts/ttl is Rust-only")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(94)
const needleID = uint64(940001)
const cookie = uint32(0xAABBCC04)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, needleID, cookie)
payload := []byte("multipart-form-fields-data")
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.WriteField("ts", "1700000000"); err != nil {
t.Fatalf("write multipart ts field: %v", err)
}
if err := writer.WriteField("ttl", "7d"); err != nil {
t.Fatalf("write multipart ttl field: %v", err)
}
filePart, err := writer.CreateFormFile("file", "multipart.txt")
if err != nil {
t.Fatalf("create multipart file field: %v", err)
}
if _, err := filePart.Write(payload); err != nil {
t.Fatalf("write multipart file payload: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("close multipart writer: %v", err)
}
req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid, &body)
if err != nil {
t.Fatalf("create multipart upload request: %v", err)
}
req.Header.Set("Content-Type", writer.FormDataContentType())
client := framework.NewHTTPClient()
uploadResp := framework.DoRequest(t, client, req)
uploadBody := framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("multipart upload expected 201, got %d, body: %s", uploadResp.StatusCode, string(uploadBody))
}
readResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusOK {
t.Fatalf("multipart upload read expected 200, got %d", readResp.StatusCode)
}
expectedLastModified := time.Unix(1700000000, 0).UTC().Format(http.TimeFormat)
if got := readResp.Header.Get("Last-Modified"); got != expectedLastModified {
t.Fatalf("multipart upload Last-Modified mismatch: got %q want %q", got, expectedLastModified)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: volumeID,
NeedleId: needleID,
})
if err != nil {
t.Fatalf("VolumeNeedleStatus after multipart upload failed: %v", err)
}
// Go's ReadTTL normalizes via fitTtlCount: 7d → 1w (7 days = 1 week)
if got := statusResp.GetTtl(); got != "1w" {
t.Fatalf("multipart upload TTL mismatch: got %q want %q", got, "1w")
}
}
func TestRequestIdGeneration(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
client := framework.NewHTTPClient()
// GET /status WITHOUT setting x-amz-request-id header
req := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status")
resp := framework.DoRequest(t, client, req)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusOK {
t.Fatalf("/status expected 200, got %d", resp.StatusCode)
}
reqID := resp.Header.Get("x-amz-request-id")
if reqID == "" {
t.Fatalf("expected auto-generated x-amz-request-id header, got empty")
}
// Go format: "%X%08X" (timestamp hex + 8 random hex), typically 20-24 chars, all hex, no hyphens.
if len(reqID) < 16 {
t.Fatalf("x-amz-request-id too short: %q (len=%d)", reqID, len(reqID))
}
}
func TestS3ResponsePassthroughHeaders(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(92)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 920001, 0xAABBCC02)
client := framework.NewHTTPClient()
data := []byte("passthrough-headers-test-data")
uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
// Read back with S3 passthrough query params
// Test response-content-language which both Go and Rust support
readURL := fmt.Sprintf("%s/%s?response-content-language=fr&response-expires=%s",
cluster.VolumeAdminURL(), fid,
"Thu,+01+Jan+2099+00:00:00+GMT",
)
readResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, readURL))
readBody := framework.ReadAllAndClose(t, readResp)
if readResp.StatusCode != http.StatusOK {
t.Fatalf("read with passthrough expected 200, got %d, body: %s", readResp.StatusCode, string(readBody))
}
if got := readResp.Header.Get("Content-Language"); got != "fr" {
t.Fatalf("Content-Language expected 'fr', got %q", got)
}
if got := readResp.Header.Get("Expires"); got != "Thu, 01 Jan 2099 00:00:00 GMT" {
t.Fatalf("Expires expected 'Thu, 01 Jan 2099 00:00:00 GMT', got %q", got)
}
}
func TestLargeFileWriteAndRead(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(93)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 930001, 0xAABBCC03)
client := framework.NewHTTPClient()
data := bytes.Repeat([]byte("A"), 1024*1024) // 1MB
uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload 1MB expected 201, got %d", uploadResp.StatusCode)
}
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
getBody := framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read 1MB expected 200, got %d", getResp.StatusCode)
}
if len(getBody) != len(data) {
t.Fatalf("read 1MB body length mismatch: got %d, want %d", len(getBody), len(data))
}
if !bytes.Equal(getBody, data) {
t.Fatalf("read 1MB body content mismatch")
}
}
func TestUploadWithContentTypePreservation(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(94)
framework.AllocateVolume(t, grpcClient, volumeID, "")
fid := framework.NewFileID(volumeID, 940001, 0xAABBCC04)
client := framework.NewHTTPClient()
data := []byte("fake-png-data-for-content-type-test")
// Upload with Content-Type: image/png
uploadURL := fmt.Sprintf("%s/%s", cluster.VolumeAdminURL(), fid)
req, err := http.NewRequest(http.MethodPost, uploadURL, bytes.NewReader(data))
if err != nil {
t.Fatalf("create upload request: %v", err)
}
req.Header.Set("Content-Type", "image/png")
req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
uploadResp := framework.DoRequest(t, client, req)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload with image/png expected 201, got %d", uploadResp.StatusCode)
}
// Read back and verify Content-Type is preserved
getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
_ = framework.ReadAllAndClose(t, getResp)
if getResp.StatusCode != http.StatusOK {
t.Fatalf("read expected 200, got %d", getResp.StatusCode)
}
if got := getResp.Header.Get("Content-Type"); got != "image/png" {
t.Fatalf("Content-Type expected 'image/png', got %q", got)
}
}