Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

15.4 KB
Raw Blame
package api

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"maps"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/ente/museum/ente"
	filedata "github.com/ente/museum/ente/filedata"
	"github.com/ente/museum/internal/testutil"
	"github.com/ente/museum/pkg/controller"
	"github.com/ente/museum/pkg/controller/access"
	filedatactrl "github.com/ente/museum/pkg/controller/filedata"
	"github.com/ente/museum/pkg/repo"
	"github.com/ente/museum/pkg/utils/s3config"
	"github.com/gin-gonic/gin"
	log "github.com/sirupsen/logrus"
	logtest "github.com/sirupsen/logrus/hooks/test"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/require"
)

const (
	previewUploadPath          = "/files/data/preview-upload-url"
	multipartPreviewUploadPath = "/files/data/multipart-preview-upload-url"
	previewChecksum            = "XUFAKrxLKna5cZ2REBfFkg=="
	previewFinalPartChecksum   = "fXkwN6B2AYZXSwKC8vQ15w=="
)

func TestPreviewUploadRejectsInvalidMetadata(t *testing.T) {
	router := previewUploadRouter(&FileHandler{FileDataCtrl: &filedatactrl.Controller{}})
	for _, multipart := range []bool{false, true} {
		path, valid := previewUploadBody(1, ente.PreviewVideo, multipart)
		for _, tc := range []struct {
			name  string
			field string
			value any
		}{
			{"missing file ID", "fileID", nil},
			{"negative file ID", "fileID", -1},
			{"missing type", "type", nil},
			{"unsupported type", "type", "file"},
			{"missing length", "contentLength", nil},
			{"negative length", "contentLength", -1},
		} {
			t.Run(fmt.Sprintf("multipart=%t/%s", multipart, tc.name), func(t *testing.T) {
				body := maps.Clone(valid)
				body[tc.field] = tc.value
				rec := performPreviewUpload(t, router, http.MethodPost, path, 1, body)
				require.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String())
			})
		}
	}
	for _, tc := range []struct {
		name      string
		multipart bool
		field     string
		value     any
	}{
		{"single too large", false, "contentLength", int64(5<<30) + 1},
		{"missing checksum", false, "contentMD5", nil},
		{"malformed checksum", false, "contentMD5", "not-md5"},
		{"wrong digest size", false, "contentMD5", "aGVsbG8="},
		{"checksum newline", false, "contentMD5", previewChecksum + "\n"},
		{"missing part length", true, "partLength", nil},
		{"negative part length", true, "partLength", -1},
		{"part too small", true, "partLength", (5 << 20) - 1},
		{"part too large", true, "partLength", int64(5<<30) + 1},
		{"multipart too large", true, "contentLength", int64(10<<30) + 1},
		{"missing part checksums", true, "partMd5s", nil},
		{"empty part checksums", true, "partMd5s", []string{}},
		{"too few checksums", true, "partMd5s", []string{previewChecksum}},
		{"too many checksums", true, "partMd5s", []string{previewChecksum, previewChecksum, previewChecksum}},
		{"invalid final checksum", true, "partMd5s", []string{previewChecksum, "invalid"}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			path, body := previewUploadBody(1, ente.PreviewVideo, tc.multipart)
			body[tc.field] = tc.value
			rec := performPreviewUpload(t, router, http.MethodPost, path, 1, body)
			require.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String())
		})
	}
}

func TestPreviewUploadMetadataAndLegacyCompatibility(t *testing.T) {
	router, db, userID, fileID := setupPreviewUploadTest(t)
	for _, tc := range []struct {
		name       string
		objectType ente.ObjectType
		multipart  bool
		legacy     bool
	}{
		{name: "single image", objectType: ente.PreviewImage},
		{name: "single video", objectType: ente.PreviewVideo},
		{name: "multipart image", objectType: ente.PreviewImage, multipart: true},
		{name: "multipart video", objectType: ente.PreviewVideo, multipart: true},
		{name: "legacy single", objectType: ente.PreviewVideo, legacy: true},
		{name: "legacy multipart", objectType: ente.PreviewVideo, multipart: true, legacy: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			path, body := previewUploadBody(fileID, tc.objectType, tc.multipart)
			method := http.MethodPost
			if tc.legacy {
				method = http.MethodGet
				path = fmt.Sprintf("%s?fileID=%d&type=%s", previewUploadPath, fileID, tc.objectType)
				if tc.multipart {
					path += "&isMultiPart=true&count=2"
				}
				body = nil
			}
			rec := performPreviewUpload(t, router, method, path, userID, body)
			require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
			var upload filedata.PreviewUploadUrl
			require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &upload))
			require.NotEmpty(t, upload.ObjectID)
			objectKey := filedata.ObjectKey(fileID, userID, tc.objectType, &upload.ObjectID)
			var length sql.NullInt64
			var checksum, uploadID sql.NullString
			var owner int64
			var bucket, app, purpose string
			var multipart bool
			require.NoError(t, db.QueryRow(`SELECT user_id, bucket_id, app, purpose, content_length,
				    content_md5, is_multipart, upload_id FROM temp_objects WHERE object_key = $1`, objectKey).
				Scan(&owner, &bucket, &app, &purpose, &length, &checksum, &multipart, &uploadID))
			require.Equal(t, userID, owner)
			require.Equal(t, "b5", bucket)
			require.Equal(t, "photos", app)
			require.Equal(t, string(tc.objectType), purpose)
			require.Equal(t, tc.multipart, multipart)
			require.Equal(t, !tc.legacy, length.Valid)
			require.Equal(t, !tc.legacy && !tc.multipart, checksum.Valid)
			if !tc.legacy {
				require.Equal(t, body["contentLength"], length.Int64)
			}
			if tc.multipart {
				require.Nil(t, upload.Url)
				require.NotNil(t, upload.PartURLs)
				require.Len(t, *upload.PartURLs, 2)
				require.NotNil(t, upload.CompleteURL)
				require.Equal(t, "preview-upload", uploadID.String)
				complete, err := url.Parse(*upload.CompleteURL)
				require.NoError(t, err)
				require.Equal(t, "/preview-bucket/"+objectKey, complete.Path)
				require.Equal(t, uploadID.String, complete.Query().Get("uploadId"))
				for i, partURL := range *upload.PartURLs {
					part, err := url.Parse(partURL)
					require.NoError(t, err)
					require.Equal(t, strconv.Itoa(i+1), part.Query().Get("partNumber"))
					require.Equal(t, complete.Path, part.Path)
					if tc.legacy {
						require.Equal(t, "host", part.Query().Get("X-Amz-SignedHeaders"))
					} else {
						lengths := []int64{5 << 20, 3}
						checksums := []string{previewChecksum, previewFinalPartChecksum}
						assertPreviewUploadSignature(t, partURL, lengths[i], checksums[i])
					}
				}
			} else {
				require.NotNil(t, upload.Url)
				require.Nil(t, upload.PartURLs)
				require.Nil(t, upload.CompleteURL)
				require.False(t, uploadID.Valid)
				parsed, err := url.Parse(*upload.Url)
				require.NoError(t, err)
				require.Equal(t, "/preview-bucket/"+objectKey, parsed.Path)
				if tc.legacy {
					require.Equal(t, "host", parsed.Query().Get("X-Amz-SignedHeaders"))
				} else {
					require.Equal(t, previewChecksum, checksum.String)
					assertPreviewUploadSignature(t, *upload.Url, 5, previewChecksum)
				}
			}
		})
	}
}

func TestPreviewUploadRequiresOwnedActiveFile(t *testing.T) {
	router, db, ownerID, fileID := setupPreviewUploadTest(t)
	for _, multipart := range []bool{false, true} {
		path, body := previewUploadBody(fileID, ente.PreviewVideo, multipart)
		rec := performPreviewUpload(t, router, http.MethodPost, path, ownerID+1, body)
		require.Equal(t, http.StatusForbidden, rec.Code, rec.Body.String())
	}
	_, err := db.Exec("UPDATE collection_files SET is_deleted = true WHERE file_id = $1", fileID)
	require.NoError(t, err)
	for _, multipart := range []bool{false, true} {
		path, body := previewUploadBody(fileID, ente.PreviewVideo, multipart)
		rec := performPreviewUpload(t, router, http.MethodPost, path, ownerID, body)
		require.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String())
	}
	var count int
	require.NoError(t, db.QueryRow("SELECT count(*) FROM temp_objects").Scan(&count))
	require.Zero(t, count)
}

func TestPreviewUploadRejectsOversizedVideo(t *testing.T) {
	router, db, userID, fileID := setupPreviewUploadTest(t)
	_, err := db.Exec(`UPDATE files SET info = '{"fileSize":40000000}' WHERE file_id = $1`, fileID)
	require.NoError(t, err)
	oldHooks := log.StandardLogger().ReplaceHooks(make(log.LevelHooks))
	t.Cleanup(func() { log.StandardLogger().ReplaceHooks(oldHooks) })
	hook := logtest.NewGlobal()
	for _, tc := range []struct {
		name       string
		objectType ente.ObjectType
		multipart  bool
		legacy     bool
		size       int64
		oversized  bool
	}{
		{"single oversized video", ente.PreviewVideo, false, false, 160_000_000, true},
		{"multipart oversized video", ente.PreviewVideo, true, false, 160_000_000, true},
		{"single video within limit", ente.PreviewVideo, false, false, 120_000_000, false},
		{"multipart video within limit", ente.PreviewVideo, true, false, 120_000_000, false},
		{"single image", ente.PreviewImage, false, false, 160_000_000, false},
		{"multipart image", ente.PreviewImage, true, false, 160_000_000, false},
		{"legacy single", ente.PreviewVideo, false, true, 0, false},
		{"legacy multipart", ente.PreviewVideo, true, true, 0, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			hook.Reset()
			path, body := previewUploadBody(fileID, tc.objectType, tc.multipart)
			method := http.MethodPost
			if tc.legacy {
				method = http.MethodGet
				path = fmt.Sprintf("%s?fileID=%d&type=%s", previewUploadPath, fileID, tc.objectType)
				if tc.multipart {
					path += "&isMultiPart=true&count=2"
				}
				body = nil
			} else {
				body["contentLength"] = tc.size
				if tc.multipart {
					body["partLength"] = tc.size / 2
				}
			}
			rec := performPreviewUpload(t, router, method, path, userID, body)
			if tc.oversized {
				require.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String())
				require.Contains(t, rec.Body.String(), "video preview exceeds")
			} else {
				require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
			}
			var warnings []*log.Entry
			for _, entry := range hook.AllEntries() {
				if entry.Level == log.WarnLevel && entry.Message == "video preview exceeds size limit" {
					warnings = append(warnings, entry)
				}
			}
			if !tc.oversized {
				require.Empty(t, warnings)
				return
			}
			require.Len(t, warnings, 1)
			entry := warnings[0]
			require.Equal(t, "video preview exceeds size limit", entry.Message)
			require.Equal(t, "upload_url", entry.Data["size_check_stage"])
			require.Equal(t, "declared", entry.Data["preview_size_source"])
			require.Equal(t, userID, entry.Data["user_id"])
			require.Equal(t, fileID, entry.Data["file_id"])
			require.Equal(t, int64(40_000_000), entry.Data["original_size_bytes"])
			require.Equal(t, tc.size, entry.Data["preview_size_bytes"])
			require.Equal(t, float64(4), entry.Data["preview_original_ratio"])
			require.Equal(t, true, entry.Data["size_limit_enforced"])
		})
	}
}

func previewUploadBody(fileID int64, objectType ente.ObjectType, multipart bool) (string, map[string]any) {
	body := map[string]any{"fileID": fileID, "type": objectType, "contentLength": int64(5)}
	if multipart {
		body["contentLength"] = int64(5<<20) + 3
		body["partLength"] = int64(5 << 20)
		body["partMd5s"] = []string{previewChecksum, previewFinalPartChecksum}
		return multipartPreviewUploadPath, body
	}
	body["contentMD5"] = previewChecksum
	return previewUploadPath, body
}

func previewUploadRouter(h *FileHandler) *gin.Engine {
	gin.SetMode(gin.TestMode)
	router := gin.New()
	router.GET(previewUploadPath, h.GetPreviewUploadURL)
	router.POST(previewUploadPath, h.GetPreviewUploadURLV2)
	router.POST(multipartPreviewUploadPath, h.GetMultipartPreviewUploadURL)
	return router
}

func performPreviewUpload(t *testing.T, router *gin.Engine, method, path string, userID int64, body map[string]any) *httptest.ResponseRecorder {
	t.Helper()
	encoded, err := json.Marshal(body)
	require.NoError(t, err)
	req := httptest.NewRequest(method, path, bytes.NewReader(encoded))
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Auth-User-ID", strconv.FormatInt(userID, 10))
	recorder := httptest.NewRecorder()
	router.ServeHTTP(recorder, req)
	return recorder
}

func assertPreviewUploadSignature(t *testing.T, rawURL string, length int64, checksum string) {
	t.Helper()
	req, err := http.NewRequest(http.MethodPut, rawURL, nil)
	require.NoError(t, err)
	query := req.URL.Query()
	require.Equal(t, "content-length;content-md5;host", query.Get("X-Amz-SignedHeaders"))
	signedAt, err := time.Parse("20060102T150405Z", query.Get("X-Amz-Date"))
	require.NoError(t, err)
	expires, err := strconv.Atoi(query.Get("X-Amz-Expires"))
	require.NoError(t, err)
	signature := query.Get("X-Amz-Signature")
	query.Del("X-Amz-Signature")
	req.URL.RawQuery = query.Encode()
	req.ContentLength = length
	req.Header.Set("Content-Length", strconv.FormatInt(length, 10))
	req.Header.Set("Content-MD5", checksum)
	signer := v4.NewSigner(credentials.NewStaticCredentials("test-key", "test-secret", ""))
	signer.DisableURIPathEscaping = true
	_, err = signer.Presign(req, nil, "s3", "us-east-1", time.Duration(expires)*time.Second, signedAt)
	require.NoError(t, err)
	require.Equal(t, signature, req.URL.Query().Get("X-Amz-Signature"), "signature must bind the expected length and checksum")
}

func setupPreviewUploadTest(t *testing.T) (*gin.Engine, *sql.DB, int64, int64) {
	t.Helper()
	db := testutil.RequireTestDB(t)
	testutil.ResetTables(t, db)
	t.Cleanup(func() { testutil.ResetTables(t, db) })
	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost || !r.URL.Query().Has("uploads") {
			t.Errorf("unexpected storage request: %s %s", r.Method, r.URL)
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		_, _ = w.Write([]byte(`<InitiateMultipartUploadResult><UploadId>preview-upload</UploadId></InitiateMultipartUploadResult>`))
	}))
	t.Cleanup(storage.Close)
	viper.Reset()
	t.Cleanup(viper.Reset)
	viper.Set("s3.derived-storage", "b5")
	viper.Set("s3.b5.key", "test-key")
	viper.Set("s3.b5.secret", "test-secret")
	viper.Set("s3.b5.endpoint", storage.URL)
	viper.Set("s3.b5.region", "us-east-1")
	viper.Set("s3.b5.bucket", "preview-bucket")
	viper.Set("s3.b5.disable_ssl", true)
	viper.Set("s3.use_path_style_urls", true)
	s3Config := s3config.NewS3Config()
	files := &repo.FileRepository{DB: db}
	collections := &repo.CollectionRepository{DB: db}
	h := &FileHandler{FileDataCtrl: &filedatactrl.Controller{
		FileRepo: files, CollectionRepo: collections, S3Config: s3Config,
		AccessCtrl:              access.NewAccessController(collections, files),
		ObjectCleanupController: &controller.ObjectCleanupController{Repo: &repo.ObjectCleanupRepository{DB: db}},
	}}
	userID := testutil.InsertUser(t, db, testutil.UserFixture{Email: "[email protected]", CreationTime: 1})
	var fileID, collectionID int64
	require.NoError(t, db.QueryRow(`INSERT INTO files(owner_id, file_decryption_header, thumbnail_decryption_header,
	    metadata_decryption_header, encrypted_metadata, updation_time, info)
	    VALUES($1, 'header', 'header', 'header', 'metadata', 1, '{}') RETURNING file_id`, userID).Scan(&fileID))
	require.NoError(t, db.QueryRow(`INSERT INTO collections(owner_id, encrypted_key, key_decryption_nonce, name, type, attributes, updation_time, app)
	    VALUES($1, 'key', 'nonce', 'Previews', 'album', '{}', 1, 'photos') RETURNING collection_id`, userID).Scan(&collectionID))
	_, err := db.Exec(`INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time, c_owner_id, f_owner_id)
	    VALUES($1, $2, 'key', 'nonce', 1, $3, $3)`, collectionID, fileID, userID)
	require.NoError(t, err)
	return previewUploadRouter(h), db, userID, fileID
}