SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package iceberg
import (
"encoding/json"
"errors"
"github.com/apache/iceberg-go/table"
)
type statisticsUpdate struct {
set *table.StatisticsFile
remove *int64
}
var ErrIncompleteSetStatistics = errors.New("set-statistics requires snapshot-id, statistics-path, file-size-in-bytes, and file-footer-size-in-bytes")
type commitAction struct {
Action string `json:"action"`
}
type setStatisticsUpdate struct {
Action string `json:"action"`
SnapshotID *int64 `json:"snapshot-id,omitempty"`
StatisticsPath string `json:"statistics-path,omitempty"`
FileSizeInBytes *int64 `json:"file-size-in-bytes,omitempty"`
FileFooterSizeInBytes *int64 `json:"file-footer-size-in-bytes,omitempty"`
KeyMetadata *string `json:"key-metadata,omitempty"`
BlobMetadata []table.BlobMetadata `json:"blob-metadata,omitempty"`
Statistics *table.StatisticsFile `json:"statistics,omitempty"`
}
func (u *setStatisticsUpdate) asStatisticsFile() (*table.StatisticsFile, error) {
if u.Statistics != nil {
if u.Statistics.BlobMetadata == nil {
u.Statistics.BlobMetadata = []table.BlobMetadata{}
}
return u.Statistics, nil
}
if u.SnapshotID == nil || u.StatisticsPath == "" || u.FileSizeInBytes == nil || u.FileFooterSizeInBytes == nil {
return nil, ErrIncompleteSetStatistics
}
stats := &table.StatisticsFile{
SnapshotID: *u.SnapshotID,
StatisticsPath: u.StatisticsPath,
FileSizeInBytes: *u.FileSizeInBytes,
FileFooterSizeInBytes: *u.FileFooterSizeInBytes,
KeyMetadata: u.KeyMetadata,
BlobMetadata: u.BlobMetadata,
}
if stats.BlobMetadata == nil {
stats.BlobMetadata = []table.BlobMetadata{}
}
return stats, nil
}
type removeStatisticsUpdate struct {
Action string `json:"action"`
SnapshotID int64 `json:"snapshot-id"`
}
func parseCommitUpdates(rawUpdates []json.RawMessage) (table.Updates, []statisticsUpdate, error) {
filtered := make([]json.RawMessage, 0, len(rawUpdates))
statisticsUpdates := make([]statisticsUpdate, 0)
for _, raw := range rawUpdates {
var action commitAction
if err := json.Unmarshal(raw, &action); err != nil {
return nil, nil, err
}
switch action.Action {
case "set-statistics":
var setUpdate setStatisticsUpdate
if err := json.Unmarshal(raw, &setUpdate); err != nil {
return nil, nil, err
}
stats, err := setUpdate.asStatisticsFile()
if err != nil {
return nil, nil, err
}
statisticsUpdates = append(statisticsUpdates, statisticsUpdate{set: stats})
case "remove-statistics":
var removeUpdate removeStatisticsUpdate
if err := json.Unmarshal(raw, &removeUpdate); err != nil {
return nil, nil, err
}
snapshotID := removeUpdate.SnapshotID
statisticsUpdates = append(statisticsUpdates, statisticsUpdate{remove: &snapshotID})
default:
filtered = append(filtered, raw)
}
}
if len(filtered) == 0 {
return nil, statisticsUpdates, nil
}
data, err := json.Marshal(filtered)
if err != nil {
return nil, nil, err
}
var updates table.Updates
if err := json.Unmarshal(data, &updates); err != nil {
return nil, nil, err
}
return updates, statisticsUpdates, nil
}
func applyStatisticsUpdates(metadataBytes []byte, updates []statisticsUpdate) ([]byte, error) {
if len(updates) == 0 {
return metadataBytes, nil
}
var metadata map[string]json.RawMessage
if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
return nil, err
}
var statistics []table.StatisticsFile
if rawStatistics, ok := metadata["statistics"]; ok && len(rawStatistics) > 0 {
if err := json.Unmarshal(rawStatistics, &statistics); err != nil {
return nil, err
}
}
statisticsBySnapshot := make(map[int64]table.StatisticsFile, len(statistics))
orderedSnapshotIDs := make([]int64, 0, len(statistics))
inOrder := make(map[int64]bool, len(statistics))
for _, stat := range statistics {
statisticsBySnapshot[stat.SnapshotID] = stat
if !inOrder[stat.SnapshotID] {
orderedSnapshotIDs = append(orderedSnapshotIDs, stat.SnapshotID)
inOrder[stat.SnapshotID] = true
}
}
for _, update := range updates {
if update.set != nil {
statisticsBySnapshot[update.set.SnapshotID] = *update.set
if !inOrder[update.set.SnapshotID] {
orderedSnapshotIDs = append(orderedSnapshotIDs, update.set.SnapshotID)
inOrder[update.set.SnapshotID] = true
}
continue
}
if update.remove != nil {
delete(statisticsBySnapshot, *update.remove)
}
}
statistics = make([]table.StatisticsFile, 0, len(statisticsBySnapshot))
for _, snapshotID := range orderedSnapshotIDs {
stat, ok := statisticsBySnapshot[snapshotID]
if !ok {
continue
}
statistics = append(statistics, stat)
}
if len(statistics) == 0 {
delete(metadata, "statistics")
} else {
data, err := json.Marshal(statistics)
if err != nil {
return nil, err
}
metadata["statistics"] = data
}
return json.Marshal(metadata)
}