SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package iceberg
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path"
"strconv"
"strings"
"github.com/apache/iceberg-go"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// parseNamespace parses the namespace from path parameter.
// Iceberg uses unit separator (0x1F) for multi-level namespaces.
// Note: mux already decodes URL-encoded path parameters, so we only split by unit separator.
func parseNamespace(encoded string) []string {
if encoded == "" {
return nil
}
parts := strings.Split(encoded, "\x1F")
// Filter empty parts
result := make([]string, 0, len(parts))
for _, p := range parts {
if p != "" {
result = append(result, p)
}
}
return result
}
// encodeNamespace encodes namespace parts using the Iceberg REST protocol's
// unit separator (0x1F) convention. This is only appropriate for protocol-level
// encoding (e.g. URL path parameters), NOT for filesystem/S3 paths.
func encodeNamespace(parts []string) string {
return strings.Join(parts, "\x1F")
}
// flattenNamespacePath joins namespace parts with "." for use in S3 location
// and filer paths, matching the S3 Tables storage layer convention.
func flattenNamespacePath(parts []string) string {
return strings.Join(parts, ".")
}
func parseS3Location(location string) (bucketName, tablePath string, err error) {
if !strings.HasPrefix(location, "s3://") {
return "", "", fmt.Errorf("unsupported location: %s", location)
}
trimmed := strings.TrimPrefix(location, "s3://")
trimmed = strings.TrimSuffix(trimmed, "/")
if trimmed == "" {
return "", "", fmt.Errorf("invalid location: %s", location)
}
parts := strings.SplitN(trimmed, "/", 2)
bucketName = parts[0]
if bucketName == "" {
return "", "", fmt.Errorf("invalid location bucket: %s", location)
}
if len(parts) == 2 {
tablePath = parts[1]
}
return bucketName, tablePath, nil
}
func tableLocationFromMetadataLocation(metadataLocation string) string {
trimmed := strings.TrimSuffix(metadataLocation, "/")
if idx := strings.LastIndex(trimmed, "/metadata/"); idx != -1 {
return trimmed[:idx]
}
return trimmed
}
// tableLocation is where the table GetTable answered for keeps its files: the
// directory of its metadata, else the one assigned when it was created
// without a location, else its name path.
func tableLocation(getResp s3tables.GetTableResponse, bucketName string, namespace []string, tableName string) string {
if location := tableLocationFromMetadataLocation(getResp.MetadataLocation); location != "" {
return location
}
if location := strings.TrimSuffix(getResp.WarehouseLocation, "/"); location != "" {
return location
}
return fmt.Sprintf("s3://%s/%s", bucketName, path.Join(flattenNamespacePath(namespace), tableName))
}
// accessDelegationHeader is how a client asks the catalog to hand back storage
// credentials along with the table metadata.
const accessDelegationHeader = "X-Iceberg-Access-Delegation"
// wantsVendedCredentials reports whether the client asked for vended
// credentials. The header carries a comma-separated list of mechanisms.
func wantsVendedCredentials(r *http.Request) bool {
if r == nil {
return false
}
for _, value := range r.Header.Values(accessDelegationHeader) {
for _, mechanism := range strings.Split(value, ",") {
if strings.EqualFold(strings.TrimSpace(mechanism), "vended-credentials") {
return true
}
}
}
return false
}
// writeLoadResult writes a table or view load response. The FileIO config it
// carries depends on the delegation the client asked for, so a cache between
// us and the client must key on that header and not on the URL alone.
func writeLoadResult(w http.ResponseWriter, status int, v interface{}) {
w.Header().Set("Vary", accessDelegationHeader)
writeJSON(w, status, v)
}
// writeJSON writes a JSON response.
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if v != nil {
data, err := json.Marshal(v)
if err != nil {
glog.Errorf("Iceberg: failed to encode response: %v", err)
return
}
w.Write(data)
}
}
// writeError writes an Iceberg error response.
func writeError(w http.ResponseWriter, status int, errType, message string) {
resp := ErrorResponse{
Error: ErrorModel{
Message: message,
Type: errType,
Code: status,
},
}
writeJSON(w, status, resp)
}
// nameValidationError reports whether err is an S3 Tables namespace or table
// name validation failure. Such names are client input (the S3 Tables charset
// is stricter than the Iceberg REST spec), so the catalog answers 400 rather
// than treating it as a server fault.
func nameValidationError(err error) bool {
if err == nil {
return false
}
// Match the validator's own phrasings rather than a bare "namespace name"/
// "table name" so unrelated faults (e.g. "failed to resolve table name")
// aren't misreported as client errors. Lowercased for resilience to
// capitalization changes.
msg := strings.ToLower(err.Error())
for _, marker := range []string{
"invalid namespace name", "namespace name must", "namespace name cannot",
"invalid table name", "table name must", "table name cannot",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}
// writeManagerError maps a residual error to a response: name validation
// failures and rejected schemas come from the request body, so they are client
// errors (400) and must carry the reason -- a variant field without
// format-version 3 otherwise reads as a bare 500 with "variant is not supported
// until v3" only in the log. Anything else is a server fault (500).
// classifyManagerError maps a storage-layer error to the HTTP status, the
// Iceberg error type and the message the catalog answers with.
func classifyManagerError(err error) (int, string, string) {
switch {
case nameValidationError(err),
errors.Is(err, iceberg.ErrInvalidSchema),
errors.Is(err, iceberg.ErrInvalidPartitionSpec),
errors.Is(err, iceberg.ErrInvalidTypeString),
errors.Is(err, iceberg.ErrInvalidTransform),
errors.Is(err, iceberg.ErrInvalidArgument):
return http.StatusBadRequest, "BadRequestException", err.Error()
}
// Storage-layer failures are mostly the client's, not the server's. Reporting
// a missing namespace or a name conflict as a 500 makes the catalog look
// broken and gives the client nothing to act on.
var tableErr *s3tables.S3TablesError
if errors.As(err, &tableErr) {
switch tableErr.Type {
case s3tables.ErrCodeNoSuchBucket:
// The storage-layer message names the resolved bucket, which for a
// client that sent no warehouse at all is the default one it never
// asked for, so say how to select a real table bucket.
return http.StatusNotFound, "NoSuchNamespaceException",
fmt.Sprintf("%s: each table bucket is a separate catalog, select one with warehouse=s3://<table-bucket>/ or /v1/<table-bucket>/", tableErr.Message)
case s3tables.ErrCodeNoSuchNamespace:
return http.StatusNotFound, "NoSuchNamespaceException", tableErr.Message
case s3tables.ErrCodeNoSuchTable:
return http.StatusNotFound, "NoSuchTableException", tableErr.Message
case s3tables.ErrCodeNoSuchView:
return http.StatusNotFound, "NoSuchViewException", tableErr.Message
case s3tables.ErrCodeNamespaceAlreadyExists, s3tables.ErrCodeBucketAlreadyExists,
s3tables.ErrCodeTableAlreadyExists, s3tables.ErrCodeViewAlreadyExists:
return http.StatusConflict, "AlreadyExistsException", tableErr.Message
case s3tables.ErrCodeNamespaceNotEmpty, s3tables.ErrCodeBucketNotEmpty:
return http.StatusConflict, "AlreadyExistsException", tableErr.Message
case s3tables.ErrCodeConflict:
return http.StatusConflict, "CommitFailedException", tableErr.Message
case s3tables.ErrCodeAccessDenied:
return http.StatusForbidden, "ForbiddenException", tableErr.Message
case s3tables.ErrCodeInvalidRequest, s3tables.ErrCodeInvalidIcebergLayout:
return http.StatusBadRequest, "BadRequestException", tableErr.Message
}
}
return http.StatusInternalServerError, "InternalServerError", err.Error()
}
func writeManagerError(w http.ResponseWriter, err error) {
status, typ, msg := classifyManagerError(err)
writeError(w, status, typ, msg)
}
// A storage-layer error the catalog answers with a 4xx (access denied, not
// found, already exists, not empty, a conflict) is the API refusing a request
// as intended, not a server fault; it is logged at V(1). A 5xx stays at ERROR.
var (
logManagerRefusal = func(format string, args ...interface{}) { glog.V(1).Infof(format, args...) }
logManagerFault = glog.Errorf
)
func logManagerError(op string, err error) {
if status, _, _ := classifyManagerError(err); status >= 400 && status < 500 {
logManagerRefusal("Iceberg: %s refused (%d): %v", op, status, err)
return
}
logManagerFault("Iceberg: %s error: %v", op, err)
}
// resolveWarehouseBucket maps a client-supplied warehouse value to a table
// bucket name. Clients spell the warehouse three ways: the s3://<bucket>/
// location this catalog advertises, the table bucket ARN that AWS S3 Tables
// uses, and the bare bucket name. Accepting only the first sends every other
// spelling to the default bucket, where the request fails naming a bucket the
// client never asked for. Returns "" when the value names no usable bucket so
// the caller keeps its own default.
func resolveWarehouseBucket(warehouse string) string {
warehouse = strings.TrimSpace(warehouse)
var bucket string
switch {
case warehouse == "":
return ""
case strings.HasPrefix(warehouse, "s3://"):
parsed, _, err := parseS3Location(warehouse)
if err != nil {
return ""
}
bucket = parsed
case strings.HasPrefix(warehouse, "arn:"):
parsed, err := s3tables.ParseBucketNameFromARN(warehouse)
if err != nil {
return ""
}
bucket = parsed
default:
// Bare name, possibly with a sub-path that bucket-scoped routing ignores.
bucket, _, _ = strings.Cut(strings.TrimSuffix(warehouse, "/"), "/")
}
if !s3tables.IsValidBucketName(bucket) {
return ""
}
return bucket
}
// getBucketFromPrefix extracts table bucket name from prefix parameter.
// For now, we use the prefix as the table bucket name.
//
// The Iceberg REST spec lets clients identify a catalog either by embedding
// its prefix in the URL (/v1/{prefix}/...) or by passing ?warehouse= as a
// query parameter. Clients that skip the /v1/config handshake (or ignore its
// overrides) still routinely send the warehouse parameter on every request,
// so honor it as a fallback before the env-var default.
func getBucketFromPrefix(r *http.Request) string {
vars := mux.Vars(r)
if prefix := vars["prefix"]; prefix != "" {
return prefix
}
if bucket := resolveWarehouseBucket(r.URL.Query().Get("warehouse")); bucket != "" {
return bucket
}
if bucket := os.Getenv("S3TABLES_DEFAULT_BUCKET"); bucket != "" {
return bucket
}
// Some writers commit to the unprefixed path even though their reads
// honor the /v1/config prefix; the deployment's first table bucket is
// where those commits belong.
for _, name := range strings.Split(os.Getenv("S3_TABLE_BUCKET"), ",") {
if name = strings.TrimSpace(name); name != "" {
return name
}
}
// Default bucket if no prefix - use "warehouse" for Iceberg
return "warehouse"
}
// buildTableBucketARN builds an ARN for a table bucket.
func buildTableBucketARN(bucketName string) string {
arn, _ := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3_constants.AccountAdminId, bucketName)
return arn
}
const (
defaultListPageSize = 1000
maxListPageSize = 1000
)
func getPaginationQueryParam(r *http.Request, primary, fallback string) string {
if v := strings.TrimSpace(r.URL.Query().Get(primary)); v != "" {
return v
}
return strings.TrimSpace(r.URL.Query().Get(fallback))
}
func parsePagination(r *http.Request) (pageToken string, pageSize int, err error) {
pageToken = getPaginationQueryParam(r, "pageToken", "page-token")
pageSize = defaultListPageSize
pageSizeValue := getPaginationQueryParam(r, "pageSize", "page-size")
if pageSizeValue == "" {
return pageToken, pageSize, nil
}
parsedPageSize, parseErr := strconv.Atoi(pageSizeValue)
if parseErr != nil || parsedPageSize <= 0 {
return pageToken, 0, fmt.Errorf("invalid pageSize %q: must be a positive integer", pageSizeValue)
}
if parsedPageSize > maxListPageSize {
return pageToken, 0, fmt.Errorf("invalid pageSize %q: must be <= %d", pageSizeValue, maxListPageSize)
}
return pageToken, parsedPageSize, nil
}
func normalizeNamespaceProperties(properties map[string]string) map[string]string {
if properties == nil {
return map[string]string{}
}
return properties
}
// namespaceLocationProperty is the well-known Iceberg key used to advertise a
// default base path for tables created in a namespace.
const namespaceLocationProperty = "location"
// defaultNamespaceLocation returns the s3 path under which tables in the
// namespace should be stored when the namespace itself does not carry an
// explicit "location" property. The flattened namespace path mirrors the on-
// disk layout used by handleCreateTable so a deferred client-side commit ends
// up at the same place a server-computed one would.
func defaultNamespaceLocation(bucket string, namespace []string) string {
if bucket == "" || len(namespace) == 0 {
return ""
}
return fmt.Sprintf("s3://%s/%s", bucket, flattenNamespacePath(namespace))
}
// withDefaultNamespaceLocation populates the Iceberg "location" property on a
// namespace response when the storage layer does not carry one. Trino's REST
// catalog falls back to an eager createTable code path when the namespace
// has no location, which races our metadata write against Trino's emptiness
// check and surfaces as a "Cannot create a table on a non-empty location"
// error on the very first CREATE TABLE. Advertising a default location lets
// Trino take the deferred-transaction path and pick a unique per-table path.
// See issue #9074.
func withDefaultNamespaceLocation(properties map[string]string, bucket string, namespace []string) map[string]string {
properties = normalizeNamespaceProperties(properties)
if _, ok := properties[namespaceLocationProperty]; ok {
return properties
}
if def := defaultNamespaceLocation(bucket, namespace); def != "" {
properties[namespaceLocationProperty] = def
}
return properties
}