SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package dash
import (
"context"
"fmt"
"io"
"path"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
)
// TopicRetentionPurger handles topic data purging based on retention policies
type TopicRetentionPurger struct {
adminServer *AdminServer
}
// NewTopicRetentionPurger creates a new topic retention purger
func NewTopicRetentionPurger(adminServer *AdminServer) *TopicRetentionPurger {
return &TopicRetentionPurger{
adminServer: adminServer,
}
}
// PurgeExpiredTopicData purges expired topic data based on retention policies
func (p *TopicRetentionPurger) PurgeExpiredTopicData() error {
glog.V(1).Infof("Starting topic data purge based on retention policies")
// Get all topics with retention enabled
topics, err := p.getTopicsWithRetention()
if err != nil {
return fmt.Errorf("failed to get topics with retention: %w", err)
}
glog.V(1).Infof("Found %d topics with retention enabled", len(topics))
// Process each topic
for _, topicRetention := range topics {
err := p.purgeTopicData(topicRetention)
if err != nil {
glog.Errorf("Failed to purge data for topic %s: %v", topicRetention.TopicName, err)
continue
}
}
glog.V(1).Infof("Completed topic data purge")
return nil
}
// TopicRetentionConfig represents a topic with its retention configuration
type TopicRetentionConfig struct {
TopicName string
Namespace string
Name string
RetentionSeconds int64
}
// getTopicsWithRetention retrieves all topics that have retention enabled
func (p *TopicRetentionPurger) getTopicsWithRetention() ([]TopicRetentionConfig, error) {
var topicsWithRetention []TopicRetentionConfig
// Find broker leader to get topics
brokerLeader, err := p.adminServer.findBrokerLeader()
if err != nil {
return nil, fmt.Errorf("failed to find broker leader: %w", err)
}
// Get all topics from the broker
err = p.adminServer.withBrokerClient(brokerLeader, func(client mq_pb.SeaweedMessagingClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.ListTopics(ctx, &mq_pb.ListTopicsRequest{})
if err != nil {
return err
}
// Check each topic for retention configuration
for _, pbTopic := range resp.Topics {
configResp, err := client.GetTopicConfiguration(ctx, &mq_pb.GetTopicConfigurationRequest{
Topic: pbTopic,
})
if err != nil {
glog.Warningf("Failed to get configuration for topic %s.%s: %v", pbTopic.Namespace, pbTopic.Name, err)
continue
}
// Check if retention is enabled
if configResp.Retention != nil && configResp.Retention.Enabled && configResp.Retention.RetentionSeconds > 0 {
topicRetention := TopicRetentionConfig{
TopicName: fmt.Sprintf("%s.%s", pbTopic.Namespace, pbTopic.Name),
Namespace: pbTopic.Namespace,
Name: pbTopic.Name,
RetentionSeconds: configResp.Retention.RetentionSeconds,
}
topicsWithRetention = append(topicsWithRetention, topicRetention)
}
}
return nil
})
if err != nil {
return nil, err
}
return topicsWithRetention, nil
}
// purgeTopicData purges expired data for a specific topic
func (p *TopicRetentionPurger) purgeTopicData(topicRetention TopicRetentionConfig) error {
glog.V(1).Infof("Purging expired data for topic %s with retention %d seconds", topicRetention.TopicName, topicRetention.RetentionSeconds)
// Calculate cutoff time
cutoffTime := time.Now().Add(-time.Duration(topicRetention.RetentionSeconds) * time.Second)
// Get topic directory
topicObj := topic.NewTopic(topicRetention.Namespace, topicRetention.Name)
topicDir := topicObj.Dir()
var purgedDirs []string
err := p.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// List all version directories under the topic directory
versionStream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: topicDir,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return fmt.Errorf("failed to list topic directory %s: %v", topicDir, err)
}
var versionDirs []VersionDirInfo
// Collect all version directories
for {
versionResp, err := versionStream.Recv()
if err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to receive version entries: %w", err)
}
// Only process directories that are versions (start with "v")
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
versionTime, err := p.parseVersionTime(versionResp.Entry.Name)
if err != nil {
glog.Warningf("Failed to parse version time from %s: %v", versionResp.Entry.Name, err)
continue
}
var modTime time.Time
if versionResp.Entry.Attributes != nil {
modTime = time.Unix(versionResp.Entry.Attributes.Mtime, 0)
}
versionDirs = append(versionDirs, VersionDirInfo{
Name: versionResp.Entry.Name,
VersionTime: versionTime,
ModTime: modTime,
})
}
}
// Sort version directories by time (oldest first)
sort.Slice(versionDirs, func(i, j int) bool {
return versionDirs[i].VersionTime.Before(versionDirs[j].VersionTime)
})
// Keep at least the most recent version directory, even if it's expired
if len(versionDirs) <= 1 {
glog.V(1).Infof("Topic %s has %d version directories, keeping all", topicRetention.TopicName, len(versionDirs))
return nil
}
// Purge expired directories (keep the most recent one)
for i := 0; i < len(versionDirs)-1; i++ {
versionDir := versionDirs[i]
// Check if this version directory is expired
if versionDir.VersionTime.Before(cutoffTime) {
dirPath := path.Join(topicDir, versionDir.Name)
// Delete the entire version directory
err := p.deleteDirectoryRecursively(client, dirPath)
if err != nil {
glog.Errorf("Failed to delete expired directory %s: %v", dirPath, err)
} else {
purgedDirs = append(purgedDirs, dirPath)
glog.V(1).Infof("Purged expired directory: %s (created: %s)", dirPath, versionDir.VersionTime.Format("2006-01-02 15:04:05"))
}
}
}
return nil
})
if err != nil {
return err
}
if len(purgedDirs) > 0 {
glog.V(0).Infof("Purged %d expired directories for topic %s", len(purgedDirs), topicRetention.TopicName)
}
return nil
}
// VersionDirInfo represents a version directory with its timestamp
type VersionDirInfo struct {
Name string
VersionTime time.Time
ModTime time.Time
}
// parseVersionTime parses the version directory name to extract the timestamp
// Version format: v2025-01-10-05-44-34
func (p *TopicRetentionPurger) parseVersionTime(versionName string) (time.Time, error) {
// Remove the 'v' prefix
if !strings.HasPrefix(versionName, "v") {
return time.Time{}, fmt.Errorf("invalid version format: %s", versionName)
}
timeStr := versionName[1:] // Remove 'v'
// Parse the time format: 2025-01-10-05-44-34
versionTime, err := time.Parse("2006-01-02-15-04-05", timeStr)
if err != nil {
return time.Time{}, fmt.Errorf("failed to parse version time %s: %v", timeStr, err)
}
return versionTime, nil
}
// deleteDirectoryRecursively deletes a directory and all its contents
func (p *TopicRetentionPurger) deleteDirectoryRecursively(client filer_pb.SeaweedFilerClient, dirPath string) error {
// List all entries in the directory
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: dirPath,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return fmt.Errorf("failed to list directory %s: %v", dirPath, err)
}
// Delete all entries
for {
resp, err := stream.Recv()
if err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to receive entries: %w", err)
}
if resp.Entry == nil {
continue
}
entryPath := path.Join(dirPath, resp.Entry.Name)
if resp.Entry.IsDirectory {
// Recursively delete subdirectory
err = p.deleteDirectoryRecursively(client, entryPath)
if err != nil {
return fmt.Errorf("failed to delete subdirectory %s: %v", entryPath, err)
}
} else {
// Delete file
if err := filer_pb.DoRemove(context.Background(), client, dirPath, resp.Entry.Name, false, false, false, false, nil); err != nil {
return fmt.Errorf("failed to delete file %s: %v", entryPath, err)
}
}
}
// Delete the directory itself
parentDir := path.Dir(dirPath)
dirName := path.Base(dirPath)
if err := filer_pb.DoRemove(context.Background(), client, parentDir, dirName, false, false, false, false, nil); err != nil {
return fmt.Errorf("failed to delete directory %s: %v", dirPath, err)
}
return nil
}