Open platform to collect and prioritize feedback
enhancement: copy etc files on startup to support ephemeral environments (#971)
Guilherme Oenningb342db679bdf
diff --git a/app/cmd/server.go b/app/cmd/server.goindex 300b504dc..9cca2f7bf 100644--- a/app/cmd/server.go+++ b/app/cmd/server.go@@ -4,12 +4,15 @@ import ("context""os""os/signal"+ "path""syscall""github.com/getfider/fider/app/models/cmd""github.com/getfider/fider/app/models/dto"+ "github.com/getfider/fider/app/models/query""github.com/getfider/fider/app/pkg/bus""github.com/getfider/fider/app/pkg/env"+ "github.com/getfider/fider/app/pkg/errors""github.com/getfider/fider/app/pkg/log""github.com/getfider/fider/app/pkg/web"@@ -38,6 +41,7 @@ func RunServer() int {}bus.Publish(ctx, &cmd.PurgeExpiredNotifications{})+ copyEtcFiles(ctx)e := routes(web.New())@@ -45,6 +49,37 @@ func RunServer() int {return listenSignals(e)}+// on startup, copy all etc/ files from configured blob storage into local etc/ folder+// this can be used to avoid having to mount volumes on ephemeral environments+func copyEtcFiles(ctx context.Context) {+ q := &query.ListBlobs{Prefix: "etc/"}+ if err := bus.Dispatch(ctx, q); err != nil {+ panic(errors.Wrap(err, "failed to list etc/ blobs"))+ }++ if len(q.Result) == 0 {+ log.Debug(ctx, "No etc/ files to copy")+ return+ }++ for _, blobKey := range q.Result {+ getBlob := &query.GetBlobByKey{Key: blobKey}+ if err := bus.Dispatch(ctx, getBlob); err != nil {+ panic(errors.Wrap(err, "failed to get blob by key '%s'", blobKey))+ }+ if err := os.MkdirAll(path.Dir(blobKey), 0774); err != nil {+ panic(errors.Wrap(err, "failed to create dir"))+ }+ if err := os.WriteFile(blobKey, getBlob.Result.Content, 0774); err != nil {+ panic(errors.Wrap(err, "failed to write blob '%s'", blobKey))+ }++ log.Debugf(ctx, "Copied '@{BlobKey}' to etc/ folder.", dto.Props{+ "BlobKey": blobKey,+ })+ }+}+func listenSignals(e *web.Engine) int {signals := make(chan os.Signal, 1)signal.Notify(signals, append([]os.Signal{syscall.SIGTERM, syscall.SIGINT}, extraSignals...)...)diff --git a/app/models/query/blob.go b/app/models/query/blob.goindex 87451c693..976c4d925 100644--- a/app/models/query/blob.go+++ b/app/models/query/blob.go@@ -3,6 +3,8 @@ package queryimport "github.com/getfider/fider/app/models/dto"type ListBlobs struct {+ Prefix string+Result []string}diff --git a/app/services/blob/blob.go b/app/services/blob/blob.goindex 24bff40ee..07548ddf7 100644--- a/app/services/blob/blob.go+++ b/app/services/blob/blob.go@@ -1,10 +1,12 @@package blobimport (+ "context""errors""path/filepath""strings"+ "github.com/getfider/fider/app""github.com/gosimple/slug")@@ -34,3 +36,16 @@ func ValidateKey(key string) error {}return nil}++// EnsureAuthorizedPrefix panics if the path is invalid under the given context+func EnsureAuthorizedPrefix(ctx context.Context, path string) {+ // if it's running under the context of a tenant, any prefix is valid+ if ctx.Value(app.TenantCtxKey) != nil {+ return+ }++ // 'tenants' prefix is not valid when running outside a tenant context+ if strings.HasPrefix(path, "tenants") {+ panic(errors.New("Unauthorized access to 'tenants' path."))+ }+}\ No newline at end of filediff --git a/app/services/blob/blob_test.go b/app/services/blob/blob_test.goindex 7195d1a30..7c4a2da7e 100644--- a/app/services/blob/blob_test.go+++ b/app/services/blob/blob_test.go@@ -69,6 +69,8 @@ var tests = []struct {{"SameKey_DifferentTenant", SameKey_DifferentTenant},{"SameKey_DifferentTenant_Delete", SameKey_DifferentTenant_Delete},{"ListBlobsFromTenant", ListBlobsFromTenant},+ {"ListBlobsOutsideTenant", ListBlobsOutsideTenant},+ {"ListUnauthorizedBlobs", ListUnauthorizedBlobs},}func TestBlobStorage(t *testing.T) {@@ -260,6 +262,48 @@ func ListBlobsFromTenant(ctx context.Context) {Expect(tenant2Files.Result).Equals([]string{"texts/hello.txt"})}+func ListBlobsOutsideTenant(ctx context.Context) {+ err := bus.Dispatch(ctx, &cmd.StoreBlob{+ Key: "texts/hello.txt",+ Content: make([]byte, 0),+ ContentType: "text/plain; charset=utf-8",+ })+ Expect(err).IsNil()++ err = bus.Dispatch(ctx, &cmd.StoreBlob{+ Key: "texts/world.txt",+ Content: make([]byte, 0),+ ContentType: "text/plain; charset=utf-8",+ })+ Expect(err).IsNil()++ textFiles := &query.ListBlobs{Prefix: "texts/"}+ err = bus.Dispatch(ctx, textFiles)+ Expect(err).IsNil()+ Expect(textFiles.Result).HasLen(2)+ Expect(textFiles.Result).Equals([]string{"texts/hello.txt", "texts/world.txt"})++ imageFiles := &query.ListBlobs{Prefix: "images/"}+ err = bus.Dispatch(ctx, imageFiles)+ Expect(err).IsNil()+ Expect(imageFiles.Result).HasLen(0)+ Expect(imageFiles.Result).Equals([]string{})+}++func ListUnauthorizedBlobs(ctx context.Context) {+ Expect(func() {+ _ = bus.Dispatch(ctx, &query.ListBlobs{Prefix: "tenants/"})+ }).Panics()++ Expect(func() {+ _ = bus.Dispatch(ctx, &query.ListBlobs{Prefix: "tenants"})+ }).Panics()++ ctxWithTenant := context.WithValue(ctx, app.TenantCtxKey, tenant1)+ err := bus.Dispatch(ctxWithTenant, &query.ListBlobs{Prefix: "tenants"})+ Expect(err).IsNil()+}+func KeyFormats(ctx context.Context) {testCases := []struct {key stringdiff --git a/app/services/blob/fs/fs.go b/app/services/blob/fs/fs.goindex 22a32fc65..045d86179 100644--- a/app/services/blob/fs/fs.go+++ b/app/services/blob/fs/fs.go@@ -51,7 +51,7 @@ func (s Service) Init() {}func listBlobs(ctx context.Context, q *query.ListBlobs) error {- basePath := basePath(ctx)+ basePath := basePath(ctx, q.Prefix)files := make([]string, 0)err := filepath.Walk(basePath,@@ -60,11 +60,11 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {return err}if !info.IsDir() {- files = append(files, path[len(basePath)+1:])+ files = append(files, q.Prefix+path[len(basePath)+1:])}return nil})- if err != nil {+ if err != nil && !os.IsNotExist(err) {return errors.Wrap(err, "failed to read dir '%s'", basePath)}@@ -126,14 +126,16 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {}func keyFullPath(ctx context.Context, key string) string {- return path.Join(basePath(ctx), key)+ return basePath(ctx, key)}-func basePath(ctx context.Context) string {+func basePath(ctx context.Context, segment string) string {+ blob.EnsureAuthorizedPrefix(ctx, segment)+startPath := env.Config.BlobStorage.FS.Pathtenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)if ok {- return path.Join(startPath, "tenants", strconv.Itoa(tenant.ID))+ return path.Join(startPath, "tenants", strconv.Itoa(tenant.ID), segment)}- return path.Join(startPath)+ return path.Join(startPath, segment)}diff --git a/app/services/blob/s3/s3.go b/app/services/blob/s3/s3.goindex 0a186dc8e..d2d3c9b27 100644--- a/app/services/blob/s3/s3.go+++ b/app/services/blob/s3/s3.go@@ -3,7 +3,6 @@ package s3import ("bytes""context"- "fmt""io/ioutil""path""sort"@@ -74,12 +73,11 @@ func (s Service) Init() {}func listBlobs(ctx context.Context, q *query.ListBlobs) error {- tenant := ctx.Value(app.TenantCtxKey).(*entity.Tenant)- basePath := fmt.Sprintf("tenants/%d/", tenant.ID)+ prefix := basePath(ctx, q.Prefix)response, err := DefaultClient.ListObjectsWithContext(ctx, &s3.ListObjectsInput{Bucket: aws.String(env.Config.BlobStorage.S3.BucketName),MaxKeys: aws.Int64(3000),- Prefix: aws.String(basePath),+ Prefix: aws.String(prefix),})if err != nil {return wrap(err, "failed to list blobs from S3")@@ -88,7 +86,14 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {files := make([]string, 0)for _, item := range response.Contents {key := *item.Key- files = append(files, key[len(basePath):])++ // if it ends with '/' it's not an actual blob+ if strings.HasSuffix(key, "/") {+ continue+ }++ fullKey := q.Prefix + key[len(prefix):]+ files = append(files, strings.TrimLeft(fullKey, "/"))}sort.Strings(files)@@ -153,11 +158,17 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {}func keyFullPathURL(ctx context.Context, key string) string {+ return path.Join(basePath(ctx, ""), key)+}++func basePath(ctx context.Context, segment string) string {+ blob.EnsureAuthorizedPrefix(ctx, segment)+tenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)if ok {- return path.Join("tenants", strconv.Itoa(tenant.ID), key)+ return path.Join("tenants", strconv.Itoa(tenant.ID), segment)}- return key+ return segment}func isNotFound(err error) bool {diff --git a/app/services/blob/sql/sql.go b/app/services/blob/sql/sql.goindex 3b9839592..f85c147d0 100644--- a/app/services/blob/sql/sql.go+++ b/app/services/blob/sql/sql.go@@ -54,7 +54,9 @@ type dbBlob struct {}func listBlobs(ctx context.Context, q *query.ListBlobs) error {- return using(ctx, func(tenant *entity.Tenant) error {+ blob.EnsureAuthorizedPrefix(ctx, q.Prefix)++ return using(ctx, func(tenantID sql.NullInt64) error {trx, err := dbx.BeginTx(ctx)if err != nil {return errors.Wrap(err, "failed to open transaction")@@ -62,7 +64,7 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {defer trx.MustCommit()blobs := []*dbBlob{}- err = trx.Select(&blobs, "SELECT key FROM blobs WHERE tenant_id = $1", tenant.ID)+ err = trx.Select(&blobs, "SELECT key FROM blobs WHERE key LIKE $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", q.Prefix+"%", tenantID)if err != nil {return errors.Wrap(err, "failed list blobs")}@@ -79,15 +81,9 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {}func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {- return using(ctx, func(tenant *entity.Tenant) error {- var tenantID sql.NullInt64- if tenant != nil {- err := tenantID.Scan(tenant.ID)- if err != nil {- return errors.Wrap(err, "failed scan tenant id")- }- }+ blob.EnsureAuthorizedPrefix(ctx, q.Key)+ return using(ctx, func(tenantID sql.NullInt64) error {trx, err := dbx.BeginTx(ctx)if err != nil {return errors.Wrap(err, "failed to open transaction")@@ -113,19 +109,13 @@ func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {}func storeBlob(ctx context.Context, c *cmd.StoreBlob) error {+ blob.EnsureAuthorizedPrefix(ctx, c.Key)+if err := blob.ValidateKey(c.Key); err != nil {return errors.Wrap(err, "failed to validate blob key '%s'", c.Key)}- return using(ctx, func(tenant *entity.Tenant) error {- var tenantID sql.NullInt64- if tenant != nil {- err := tenantID.Scan(tenant.ID)- if err != nil {- return errors.Wrap(err, "failed scan tenant id")- }- }-+ return using(ctx, func(tenantID sql.NullInt64) error {trx, err := dbx.BeginTx(ctx)if err != nil {return errors.Wrap(err, "failed to open transaction")@@ -151,15 +141,9 @@ func storeBlob(ctx context.Context, c *cmd.StoreBlob) error {}func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {- return using(ctx, func(tenant *entity.Tenant) error {- var tenantID sql.NullInt64- if tenant != nil {- err := tenantID.Scan(tenant.ID)- if err != nil {- return errors.Wrap(err, "failed scan tenant id")- }- }-+ blob.EnsureAuthorizedPrefix(ctx, c.Key)++ return using(ctx, func(tenantID sql.NullInt64) error {trx, err := dbx.BeginTx(ctx)if err != nil {return errors.Wrap(err, "failed to open transaction")@@ -179,7 +163,11 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {})}-func using(ctx context.Context, handler func(tenant *entity.Tenant) error) error {- tenant, _ := ctx.Value(app.TenantCtxKey).(*entity.Tenant)- return handler(tenant)-}+func using(ctx context.Context, handler func(tenantId sql.NullInt64) error) error {+ var tenantID sql.NullInt64+ tenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)+ if ok {+ _ = tenantID.Scan(tenant.ID)+ }+ return handler(tenantID)+}\ No newline at end of file