Skip to content

Modern Docker Management, Designed for Everyone

12.4 KB
Raw Blame
// Package upload owns chunked upload sessions: filesystem-backed staging for
// large files that arrive as independently retryable chunks, consumed by the
// image, volume-backup, and build-workspace endpoints.
package upload

import (
	"cmp"
	"context"
	"encoding/json/v2"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"time"

	"github.com/getarcaneapp/arcane/types/v2/upload"
	"go.getarcane.app/acfs"

	"github.com/getarcaneapp/arcane/backend/v2/internal/common"
	"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)

const (
	sessionMetaFilename = "meta.json"
	sessionDataFilename = "data"
)

// uploadIDPattern matches the random hex component acfs.MkdirTemp generates
// for session directories; anything else is rejected before touching disk.
var uploadIDPattern = regexp.MustCompile(`^[0-9a-f]{16}$`)

// UploadService stages chunked upload sessions on local disk. Sessions are
// node-local by design: chunks and the consuming request must reach the same
// instance, which holds for Arcane's single-instance-per-node deployment (for
// remote environments the session lives on that environment's agent).
type UploadService struct {
	settingsService *settings.SettingsService
	root            string
	locks           utils.KeyedMutex
}

// NewUploadService builds the session service rooted in the system temp
// directory; the root is created lazily on first session.
func NewUploadService(settingsService *settings.SettingsService) *UploadService {
	return &UploadService{
		settingsService: settingsService,
		root:            filepath.Join(os.TempDir(), "arcane-uploads"),
	}
}

func validFilenameForKindInternal(kind, filename string) bool {
	lower := strings.ToLower(filename)
	switch kind {
	case upload.KindImage:
		return strings.HasSuffix(lower, ".tar") || strings.HasSuffix(lower, ".tar.gz") ||
			strings.HasSuffix(lower, ".tgz") || strings.HasSuffix(lower, ".tar.xz")
	case upload.KindVolumeBackup:
		return strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz")
	case upload.KindBuildWorkspace:
		return true
	}
	return false
}

// CreateSession validates the request against the kind's filename and size
// rules and stages a new empty session on disk.
func (s *UploadService) CreateSession(ctx context.Context, kind string, request upload.CreateSessionRequest) (*upload.Session, error) {
	filename := path.Base(strings.ReplaceAll(request.Filename, "\\", "/"))
	if filename == "" || filename == "." || filename == "/" {
		return nil, fmt.Errorf("%w: a filename is required", common.ErrUploadSessionInvalid)
	}
	if !validFilenameForKindInternal(kind, filename) {
		return nil, fmt.Errorf("%w: file type not allowed for %s uploads", common.ErrUploadSessionInvalid, kind)
	}
	if request.Size <= 0 {
		return nil, fmt.Errorf("%w: size must be positive", common.ErrUploadSessionInvalid)
	}
	if kind == upload.KindImage {
		maxSizeMB := s.settingsService.GetIntSetting(ctx, "maxImageUploadSize", 500)
		if maxSizeBytes := int64(maxSizeMB) * 1024 * 1024; request.Size > maxSizeBytes {
			return nil, fmt.Errorf("%w: file size exceeds maximum allowed size of %d MB", common.ErrUploadSessionInvalid, maxSizeMB)
		}
	}
	chunkSize := cmp.Or(request.ChunkSize, upload.DefaultChunkSize)
	if chunkSize < upload.MinChunkSize || chunkSize > upload.MaxChunkSize {
		return nil, fmt.Errorf("%w: chunk size must be between %d and %d bytes", common.ErrUploadSessionInvalid, upload.MinChunkSize, upload.MaxChunkSize)
	}

	if err := os.MkdirAll(s.root, 0o700); err != nil {
		return nil, fmt.Errorf("create upload root: %w", err)
	}
	sessionDir, err := acfs.MkdirTemp(ctx, s.root, "/", "*")
	if err != nil {
		return nil, fmt.Errorf("create upload session directory: %w", err)
	}
	session := &upload.Session{
		ID:             path.Base(sessionDir),
		Kind:           kind,
		Filename:       filename,
		Size:           request.Size,
		ChunkSize:      chunkSize,
		TotalChunks:    int((request.Size + chunkSize - 1) / chunkSize),
		ReceivedChunks: []int{},
		CreatedAt:      time.Now().UTC(),
	}
	if writeErr := acfs.Write(ctx, s.root, path.Join(sessionDir, sessionDataFilename), nil, acfs.WriteOptions{Mode: 0o600}); writeErr != nil {
		_ = acfs.RemoveAll(ctx, s.root, sessionDir)
		return nil, fmt.Errorf("create upload session data file: %w", writeErr)
	}
	if writeMetaErr := s.writeMetaInternal(ctx, session); writeMetaErr != nil {
		_ = acfs.RemoveAll(ctx, s.root, sessionDir)
		return nil, writeMetaErr
	}
	return session, nil
}

func (s *UploadService) sessionPathInternal(uploadID, filename string) (string, error) {
	if !uploadIDPattern.MatchString(uploadID) {
		return "", fmt.Errorf("%w: invalid upload ID", common.ErrUploadSessionNotFound)
	}
	return path.Join("/", uploadID, filename), nil
}

func (s *UploadService) loadMetaInternal(ctx context.Context, kind, uploadID string) (*upload.Session, error) {
	metaPath, err := s.sessionPathInternal(uploadID, sessionMetaFilename)
	if err != nil {
		return nil, err
	}
	payload, err := acfs.ReadFile(ctx, s.root, metaPath)
	if err != nil {
		return nil, fmt.Errorf("%w: %s", common.ErrUploadSessionNotFound, uploadID)
	}
	var session upload.Session
	if unmarshalErr := json.Unmarshal(payload, &session); unmarshalErr != nil {
		return nil, fmt.Errorf("%w: corrupt session metadata", common.ErrUploadSessionNotFound)
	}
	if session.Kind != kind {
		return nil, fmt.Errorf("%w: session is for %s uploads", common.ErrUploadKindMismatch, session.Kind)
	}
	session.Complete = len(session.ReceivedChunks) == session.TotalChunks
	return &session, nil
}

func (s *UploadService) writeMetaInternal(ctx context.Context, session *upload.Session) error {
	session.Complete = len(session.ReceivedChunks) == session.TotalChunks
	payload, err := json.Marshal(session)
	if err != nil {
		return fmt.Errorf("encode upload session metadata: %w", err)
	}
	if writeErr := acfs.Write(ctx, s.root, path.Join("/", session.ID, sessionMetaFilename), payload, acfs.WriteOptions{Mode: 0o600}); writeErr != nil {
		return fmt.Errorf("write upload session metadata: %w", writeErr)
	}
	return nil
}

// WriteChunk stores one chunk at its final offset and records it as received.
// Re-sending an already-received chunk is an idempotent overwrite.
func (s *UploadService) WriteChunk(ctx context.Context, kind, uploadID string, index int, data []byte) (*upload.Session, error) {
	defer s.locks.Lock(uploadID)()

	session, err := s.loadMetaInternal(ctx, kind, uploadID)
	if err != nil {
		return nil, err
	}
	if index < 0 || index >= session.TotalChunks {
		return nil, fmt.Errorf("%w: chunk index %d out of range [0,%d)", common.ErrUploadChunkInvalid, index, session.TotalChunks)
	}
	expected := session.ChunkSize
	if index == session.TotalChunks-1 {
		expected = session.Size - int64(index)*session.ChunkSize
	}
	if int64(len(data)) != expected {
		return nil, fmt.Errorf("%w: chunk %d is %d bytes, expected %d", common.ErrUploadChunkInvalid, index, len(data), expected)
	}

	dataPath, err := s.sessionPathInternal(uploadID, sessionDataFilename)
	if err != nil {
		return nil, err
	}
	if writeAtErr := acfs.WriteAt(ctx, s.root, dataPath, int64(index)*session.ChunkSize, data); writeAtErr != nil {
		return nil, fmt.Errorf("write chunk %d: %w", index, writeAtErr)
	}
	if position, found := slices.BinarySearch(session.ReceivedChunks, index); !found {
		session.ReceivedChunks = slices.Insert(session.ReceivedChunks, position, index)
		if writeMetaErr := s.writeMetaInternal(ctx, session); writeMetaErr != nil {
			return nil, writeMetaErr
		}
	}
	return session, nil
}

// GetSession reports the session state, including which chunks have been
// received so clients can resume.
func (s *UploadService) GetSession(ctx context.Context, kind, uploadID string) (*upload.Session, error) {
	defer s.locks.RLock(uploadID)()
	return s.loadMetaInternal(ctx, kind, uploadID)
}

// DeleteSession aborts a session and discards its received chunks.
func (s *UploadService) DeleteSession(ctx context.Context, kind, uploadID string) error {
	defer s.locks.Lock(uploadID)()

	if _, err := s.loadMetaInternal(ctx, kind, uploadID); err != nil {
		return err
	}
	if err := acfs.RemoveAll(ctx, s.root, path.Join("/", uploadID)); err != nil {
		return fmt.Errorf("delete upload session: %w", err)
	}
	return nil
}

// IngestSession creates a session already containing the complete file. It
// backs the deprecated single-request multipart compatibility path, applying
// the same per-kind validation as chunked sessions.
func (s *UploadService) IngestSession(ctx context.Context, kind, filename string, size int64, source io.Reader) (*upload.Session, error) {
	session, err := s.CreateSession(ctx, kind, upload.CreateSessionRequest{Filename: filename, Size: size})
	if err != nil {
		return nil, err
	}
	if _, writeFromErr := acfs.WriteFrom(ctx, s.root, path.Join("/", session.ID, sessionDataFilename), source, size, 0o600); writeFromErr != nil {
		_ = acfs.RemoveAll(ctx, s.root, path.Join("/", session.ID))
		return nil, fmt.Errorf("write uploaded file: %w", writeFromErr)
	}
	for index := range session.TotalChunks {
		session.ReceivedChunks = append(session.ReceivedChunks, index)
	}
	if writeMetaErr := s.writeMetaInternal(ctx, session); writeMetaErr != nil {
		_ = acfs.RemoveAll(ctx, s.root, path.Join("/", session.ID))
		return nil, writeMetaErr
	}
	return session, nil
}

// Consume validates that the session matches kind and is complete, then opens
// the assembled file for reading. The session metadata is removed under the
// session lock before returning, so a session can be consumed at most once
// even by concurrent requests; the returned cleanup closes the file and
// deletes the remaining session data.
func (s *UploadService) Consume(ctx context.Context, kind, uploadID string) (io.ReadSeekCloser, *upload.Session, func(), error) {
	defer s.locks.Lock(uploadID)()

	session, err := s.loadMetaInternal(ctx, kind, uploadID)
	if err != nil {
		return nil, nil, nil, err
	}
	if !session.Complete {
		return nil, nil, nil, fmt.Errorf("%w: %d of %d chunks received", common.ErrUploadSessionIncomplete, len(session.ReceivedChunks), session.TotalChunks)
	}
	dataPath, err := s.sessionPathInternal(uploadID, sessionDataFilename)
	if err != nil {
		return nil, nil, nil, err
	}
	file, size, err := acfs.OpenReadSeek(ctx, s.root, dataPath)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("open upload session data: %w", err)
	}
	if size != session.Size {
		_ = file.Close()
		return nil, nil, nil, fmt.Errorf("%w: assembled size %d does not match declared size %d", common.ErrUploadSessionIncomplete, size, session.Size)
	}
	if removeErr := acfs.Remove(ctx, s.root, path.Join("/", uploadID, sessionMetaFilename)); removeErr != nil {
		_ = file.Close()
		return nil, nil, nil, fmt.Errorf("mark upload session consumed: %w", removeErr)
	}
	cleanup := func() {
		_ = file.Close()
		// context.WithoutCancel: the session must be removed even when the
		// consuming request was cancelled mid-restore.
		_ = acfs.RemoveAll(context.WithoutCancel(ctx), s.root, path.Join("/", uploadID))
	}
	return file, session, cleanup, nil
}

// PurgeExpiredSessions removes sessions that have been idle for longer than
// maxAge. Idleness is judged by the newest modification time among the
// session's files — the data file advances on every chunk write and the
// metadata on every chunk recorded — so a slow but active upload is never
// swept, and the check runs under the session lock so an in-flight chunk
// write cannot race the removal.
func (s *UploadService) PurgeExpiredSessions(ctx context.Context, maxAge time.Duration) (int, error) {
	if _, err := os.Stat(s.root); os.IsNotExist(err) {
		return 0, nil
	}
	entries, err := acfs.List(ctx, s.root, "/")
	if err != nil {
		return 0, fmt.Errorf("list upload sessions: %w", err)
	}
	cutoff := time.Now().Add(-maxAge)
	removed := 0
	for _, entry := range entries {
		if !entry.IsDirectory {
			continue
		}
		unlock := s.locks.Lock(entry.Name)
		lastActivity := entry.ModTime
		for _, filename := range []string{sessionDataFilename, sessionMetaFilename} {
			if info, statErr := acfs.Stat(ctx, s.root, path.Join("/", entry.Name, filename), true); statErr == nil && info.ModTime.After(lastActivity) {
				lastActivity = info.ModTime
			}
		}
		var removeErr error
		if lastActivity.Before(cutoff) {
			removeErr = acfs.RemoveAll(ctx, s.root, path.Join("/", entry.Name))
		}
		unlock()
		if removeErr != nil {
			return removed, fmt.Errorf("remove expired upload session %s: %w", entry.Name, removeErr)
		}
		if lastActivity.Before(cutoff) {
			removed++
		}
	}
	return removed, nil
}