SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package spark
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/testcontainers/testcontainers-go"
)
var (
miniProcessMu sync.Mutex
lastMiniProcess *exec.Cmd
)
func stopPreviousMini() {
miniProcessMu.Lock()
defer miniProcessMu.Unlock()
if lastMiniProcess != nil && lastMiniProcess.Process != nil {
_ = lastMiniProcess.Process.Kill()
_ = lastMiniProcess.Wait()
}
lastMiniProcess = nil
}
func registerMiniProcess(cmd *exec.Cmd) {
miniProcessMu.Lock()
lastMiniProcess = cmd
miniProcessMu.Unlock()
}
func clearMiniProcess(cmd *exec.Cmd) {
miniProcessMu.Lock()
if lastMiniProcess == cmd {
lastMiniProcess = nil
}
miniProcessMu.Unlock()
}
type TestEnvironment struct {
dockerAvailable bool
weedBinary string
seaweedfsDataDir string
weedLogPath string
weedLogFile *os.File
masterPort int
filerPort int
s3Port int
accessKey string
secretKey string
sparkContainer testcontainers.Container
masterProcess *exec.Cmd
}
func NewTestEnvironment() *TestEnvironment {
env := &TestEnvironment{
accessKey: "test",
secretKey: "test",
}
cmd := exec.Command("docker", "version")
env.dockerAvailable = cmd.Run() == nil
if weedPath, err := exec.LookPath("weed"); err == nil {
env.weedBinary = weedPath
}
return env
}
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Helper()
if env.weedBinary == "" {
t.Skip("weed binary not found in PATH, skipping Spark S3 integration test")
}
stopPreviousMini()
var err error
env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-s3-spark-test-")
if err != nil {
t.Fatalf("failed to create temp directory: %v", err)
}
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Admin"})
env.masterPort = ports[0]
env.filerPort = ports[1]
env.s3Port = ports[2]
bindIP := testutil.FindBindIP()
iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
if err != nil {
t.Fatalf("failed to create IAM config: %v", err)
}
miniArgs := append([]string{"mini",
"-ip", bindIP,
"-ip.bind", "0.0.0.0",
"-master.port", fmt.Sprintf("%d", env.masterPort),
"-filer.port", fmt.Sprintf("%d", env.filerPort),
"-s3.port", fmt.Sprintf("%d", env.s3Port),
"-s3.config", iamConfigPath,
"-dir", env.seaweedfsDataDir,
}, testutil.MiniAdminArgs(ports[3])...)
miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0")
env.masterProcess = exec.Command(env.weedBinary, miniArgs...)
weedLogPath := filepath.Join(env.seaweedfsDataDir, "weed-mini.log")
weedLogFile, err := os.Create(weedLogPath)
if err != nil {
t.Fatalf("failed to create weed log file: %v", err)
}
env.weedLogPath = weedLogPath
env.weedLogFile = weedLogFile
env.masterProcess.Stdout = weedLogFile
env.masterProcess.Stderr = weedLogFile
env.masterProcess.Env = append(os.Environ(),
"AWS_ACCESS_KEY_ID="+env.accessKey,
"AWS_SECRET_ACCESS_KEY="+env.secretKey,
)
if err := env.masterProcess.Start(); err != nil {
t.Fatalf("failed to start weed mini: %v", err)
}
registerMiniProcess(env.masterProcess)
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
}
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
}
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
}
}
func (env *TestEnvironment) startSparkContainer(t *testing.T) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
req := testcontainers.ContainerRequest{
Image: "apache/spark:3.5.1",
ExposedPorts: []string{"4040/tcp"},
Env: map[string]string{
"SPARK_LOCAL_IP": "localhost",
},
ExtraHosts: []string{"host.docker.internal:host-gateway"},
Cmd: []string{"/bin/sh", "-c", "sleep 7200"},
}
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
if err != nil {
t.Fatalf("failed to start spark container: %v", err)
}
env.sparkContainer = container
}
func (env *TestEnvironment) Cleanup(t *testing.T) {
if env.masterProcess != nil && env.masterProcess.Process != nil {
_ = env.masterProcess.Process.Kill()
_ = env.masterProcess.Wait()
}
clearMiniProcess(env.masterProcess)
if env.weedLogFile != nil {
_ = env.weedLogFile.Close()
}
if t.Failed() && os.Getenv("CI") != "" && env.weedLogPath != "" {
logData, err := os.ReadFile(env.weedLogPath)
if err != nil {
t.Logf("failed to read weed mini log file %s: %v", env.weedLogPath, err)
} else {
// Print the tail to keep CI output manageable while preserving failure context.
const maxTailBytes = 64 * 1024
start := 0
if len(logData) > maxTailBytes {
start = len(logData) - maxTailBytes
}
t.Logf("weed mini logs (tail, %d bytes):\n%s", len(logData)-start, string(logData[start:]))
}
}
if env.sparkContainer != nil {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = env.sparkContainer.Terminate(ctx)
}
if env.seaweedfsDataDir != "" {
_ = os.RemoveAll(env.seaweedfsDataDir)
}
}
func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
defer cancel()
pythonScript := fmt.Sprintf(`
import glob
import os
import sys
spark_home = os.environ.get("SPARK_HOME", "/opt/spark")
python_path = os.path.join(spark_home, "python")
py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip"))
ivy_dir = "/tmp/ivy"
os.makedirs(ivy_dir, exist_ok=True)
os.environ["AWS_REGION"] = "us-east-1"
os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
os.environ["AWS_ACCESS_KEY_ID"] = "test"
os.environ["AWS_SECRET_ACCESS_KEY"] = "test"
if python_path not in sys.path:
sys.path.insert(0, python_path)
if py4j_glob and py4j_glob[0] not in sys.path:
sys.path.insert(0, py4j_glob[0])
from pyspark.sql import SparkSession
spark = (SparkSession.builder
.master("local[2]")
.appName("SeaweedFS S3 Spark Issue 8234 Repro")
.config("spark.sql.catalogImplementation", "hive")
.config("spark.executor.extraJavaOptions", "-Djdk.tls.client.protocols=TLSv1")
.config("spark.sql.parquet.int96RebaseModeInRead", "CORRECTED")
.config("spark.sql.parquet.int96RebaseModeInWrite", "CORRECTED")
.config("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED")
.config("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED")
.config("spark.sql.parquet.enableVectorizedReader", "false")
.config("spark.sql.parquet.mergeSchema", "false")
.config("spark.sql.parquet.writeLegacyFormat", "true")
.config("spark.sql.broadcastTimeout", "-1")
.config("spark.network.timeout", "600")
.config("spark.hadoop.hive.metastore.schema.verification", "false")
.config("spark.hadoop.hive.metastore.schema.verification.record.version", "false")
.config("spark.jars.ivy", ivy_dir)
.config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.3.2,com.amazonaws:aws-java-sdk-bundle:1.12.262")
.config("spark.hadoop.fs.s3a.access.key", "test")
.config("spark.hadoop.fs.s3a.secret.key", "test")
.config("spark.hadoop.fs.s3a.endpoint", "host.docker.internal:%d")
.config("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version", "1")
.config("spark.hadoop.fs.s3a.path.style.access", "true")
.config("spark.hadoop.fs.s3a.fast.upload", "true")
.config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
.config("spark.hadoop.fs.s3a.multiobjectdelete.enable", "true")
.config("spark.hadoop.fs.s3a.directory.marker.retention", "keep")
.config("spark.hadoop.fs.s3a.change.detection.version.required", "false")
.config("spark.hadoop.fs.s3a.change.detection.mode", "warn")
.config("spark.local.dir", "/tmp/spark-temp")
.getOrCreate())
%s
`, s3Port, script)
code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript})
var output string
if out != nil {
outputBytes, readErr := io.ReadAll(out)
if readErr != nil {
output = fmt.Sprintf("failed to read container output: %v", readErr)
} else {
output = string(outputBytes)
}
}
if err != nil {
output = output + fmt.Sprintf("\ncontainer exec error: %v\n", err)
}
return code, output
}
func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
t.Helper()
cfg := aws.Config{
Region: "us-east-1",
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
})
_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
if err != nil {
t.Fatalf("failed to create object bucket %s: %v", bucketName, err)
}
}