Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

enhancement: copy etc files on startup to support ephemeral environments (#971)

Guilherme Oenningb342db679bdf
diff --git a/app/cmd/server.go b/app/cmd/server.go
index 300b504dc..9cca2f7bf 100644
--- a/app/cmd/server.go
+++ b/app/cmd/server.go
@@ -4,12 +4,15 @@ import (
"context"
"os"
"os/signal"
+ "path"
"syscall"
"github.com/getfider/fider/app/models/cmd"
"github.com/getfider/fider/app/models/dto"
+ "github.com/getfider/fider/app/models/query"
"github.com/getfider/fider/app/pkg/bus"
"github.com/getfider/fider/app/pkg/env"
+ "github.com/getfider/fider/app/pkg/errors"
"github.com/getfider/fider/app/pkg/log"
"github.com/getfider/fider/app/pkg/web"
@@ -38,6 +41,7 @@ func RunServer() int {
}
bus.Publish(ctx, &cmd.PurgeExpiredNotifications{})
+ copyEtcFiles(ctx)
e := routes(web.New())
@@ -45,6 +49,37 @@ func RunServer() int {
return listenSignals(e)
}
+// on startup, copy all etc/ files from configured blob storage into local etc/ folder
+// this can be used to avoid having to mount volumes on ephemeral environments
+func copyEtcFiles(ctx context.Context) {
+ q := &query.ListBlobs{Prefix: "etc/"}
+ if err := bus.Dispatch(ctx, q); err != nil {
+ panic(errors.Wrap(err, "failed to list etc/ blobs"))
+ }
+
+ if len(q.Result) == 0 {
+ log.Debug(ctx, "No etc/ files to copy")
+ return
+ }
+
+ for _, blobKey := range q.Result {
+ getBlob := &query.GetBlobByKey{Key: blobKey}
+ if err := bus.Dispatch(ctx, getBlob); err != nil {
+ panic(errors.Wrap(err, "failed to get blob by key '%s'", blobKey))
+ }
+ if err := os.MkdirAll(path.Dir(blobKey), 0774); err != nil {
+ panic(errors.Wrap(err, "failed to create dir"))
+ }
+ if err := os.WriteFile(blobKey, getBlob.Result.Content, 0774); err != nil {
+ panic(errors.Wrap(err, "failed to write blob '%s'", blobKey))
+ }
+
+ log.Debugf(ctx, "Copied '@{BlobKey}' to etc/ folder.", dto.Props{
+ "BlobKey": blobKey,
+ })
+ }
+}
+
func listenSignals(e *web.Engine) int {
signals := make(chan os.Signal, 1)
signal.Notify(signals, append([]os.Signal{syscall.SIGTERM, syscall.SIGINT}, extraSignals...)...)
diff --git a/app/models/query/blob.go b/app/models/query/blob.go
index 87451c693..976c4d925 100644
--- a/app/models/query/blob.go
+++ b/app/models/query/blob.go
@@ -3,6 +3,8 @@ package query
import "github.com/getfider/fider/app/models/dto"
type ListBlobs struct {
+ Prefix string
+
Result []string
}
diff --git a/app/services/blob/blob.go b/app/services/blob/blob.go
index 24bff40ee..07548ddf7 100644
--- a/app/services/blob/blob.go
+++ b/app/services/blob/blob.go
@@ -1,10 +1,12 @@
package blob
import (
+ "context"
"errors"
"path/filepath"
"strings"
+ "github.com/getfider/fider/app"
"github.com/gosimple/slug"
)
@@ -34,3 +36,16 @@ func ValidateKey(key string) error {
}
return nil
}
+
+// EnsureAuthorizedPrefix panics if the path is invalid under the given context
+func EnsureAuthorizedPrefix(ctx context.Context, path string) {
+ // if it's running under the context of a tenant, any prefix is valid
+ if ctx.Value(app.TenantCtxKey) != nil {
+ return
+ }
+
+ // 'tenants' prefix is not valid when running outside a tenant context
+ if strings.HasPrefix(path, "tenants") {
+ panic(errors.New("Unauthorized access to 'tenants' path."))
+ }
+}
\ No newline at end of file
diff --git a/app/services/blob/blob_test.go b/app/services/blob/blob_test.go
index 7195d1a30..7c4a2da7e 100644
--- a/app/services/blob/blob_test.go
+++ b/app/services/blob/blob_test.go
@@ -69,6 +69,8 @@ var tests = []struct {
{"SameKey_DifferentTenant", SameKey_DifferentTenant},
{"SameKey_DifferentTenant_Delete", SameKey_DifferentTenant_Delete},
{"ListBlobsFromTenant", ListBlobsFromTenant},
+ {"ListBlobsOutsideTenant", ListBlobsOutsideTenant},
+ {"ListUnauthorizedBlobs", ListUnauthorizedBlobs},
}
func TestBlobStorage(t *testing.T) {
@@ -260,6 +262,48 @@ func ListBlobsFromTenant(ctx context.Context) {
Expect(tenant2Files.Result).Equals([]string{"texts/hello.txt"})
}
+func ListBlobsOutsideTenant(ctx context.Context) {
+ err := bus.Dispatch(ctx, &cmd.StoreBlob{
+ Key: "texts/hello.txt",
+ Content: make([]byte, 0),
+ ContentType: "text/plain; charset=utf-8",
+ })
+ Expect(err).IsNil()
+
+ err = bus.Dispatch(ctx, &cmd.StoreBlob{
+ Key: "texts/world.txt",
+ Content: make([]byte, 0),
+ ContentType: "text/plain; charset=utf-8",
+ })
+ Expect(err).IsNil()
+
+ textFiles := &query.ListBlobs{Prefix: "texts/"}
+ err = bus.Dispatch(ctx, textFiles)
+ Expect(err).IsNil()
+ Expect(textFiles.Result).HasLen(2)
+ Expect(textFiles.Result).Equals([]string{"texts/hello.txt", "texts/world.txt"})
+
+ imageFiles := &query.ListBlobs{Prefix: "images/"}
+ err = bus.Dispatch(ctx, imageFiles)
+ Expect(err).IsNil()
+ Expect(imageFiles.Result).HasLen(0)
+ Expect(imageFiles.Result).Equals([]string{})
+}
+
+func ListUnauthorizedBlobs(ctx context.Context) {
+ Expect(func() {
+ _ = bus.Dispatch(ctx, &query.ListBlobs{Prefix: "tenants/"})
+ }).Panics()
+
+ Expect(func() {
+ _ = bus.Dispatch(ctx, &query.ListBlobs{Prefix: "tenants"})
+ }).Panics()
+
+ ctxWithTenant := context.WithValue(ctx, app.TenantCtxKey, tenant1)
+ err := bus.Dispatch(ctxWithTenant, &query.ListBlobs{Prefix: "tenants"})
+ Expect(err).IsNil()
+}
+
func KeyFormats(ctx context.Context) {
testCases := []struct {
key string
diff --git a/app/services/blob/fs/fs.go b/app/services/blob/fs/fs.go
index 22a32fc65..045d86179 100644
--- a/app/services/blob/fs/fs.go
+++ b/app/services/blob/fs/fs.go
@@ -51,7 +51,7 @@ func (s Service) Init() {
}
func listBlobs(ctx context.Context, q *query.ListBlobs) error {
- basePath := basePath(ctx)
+ basePath := basePath(ctx, q.Prefix)
files := make([]string, 0)
err := filepath.Walk(basePath,
@@ -60,11 +60,11 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {
return err
}
if !info.IsDir() {
- files = append(files, path[len(basePath)+1:])
+ files = append(files, q.Prefix+path[len(basePath)+1:])
}
return nil
})
- if err != nil {
+ if err != nil && !os.IsNotExist(err) {
return errors.Wrap(err, "failed to read dir '%s'", basePath)
}
@@ -126,14 +126,16 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
}
func keyFullPath(ctx context.Context, key string) string {
- return path.Join(basePath(ctx), key)
+ return basePath(ctx, key)
}
-func basePath(ctx context.Context) string {
+func basePath(ctx context.Context, segment string) string {
+ blob.EnsureAuthorizedPrefix(ctx, segment)
+
startPath := env.Config.BlobStorage.FS.Path
tenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)
if ok {
- return path.Join(startPath, "tenants", strconv.Itoa(tenant.ID))
+ return path.Join(startPath, "tenants", strconv.Itoa(tenant.ID), segment)
}
- return path.Join(startPath)
+ return path.Join(startPath, segment)
}
diff --git a/app/services/blob/s3/s3.go b/app/services/blob/s3/s3.go
index 0a186dc8e..d2d3c9b27 100644
--- a/app/services/blob/s3/s3.go
+++ b/app/services/blob/s3/s3.go
@@ -3,7 +3,6 @@ package s3
import (
"bytes"
"context"
- "fmt"
"io/ioutil"
"path"
"sort"
@@ -74,12 +73,11 @@ func (s Service) Init() {
}
func listBlobs(ctx context.Context, q *query.ListBlobs) error {
- tenant := ctx.Value(app.TenantCtxKey).(*entity.Tenant)
- basePath := fmt.Sprintf("tenants/%d/", tenant.ID)
+ prefix := basePath(ctx, q.Prefix)
response, err := DefaultClient.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
Bucket: aws.String(env.Config.BlobStorage.S3.BucketName),
MaxKeys: aws.Int64(3000),
- Prefix: aws.String(basePath),
+ Prefix: aws.String(prefix),
})
if err != nil {
return wrap(err, "failed to list blobs from S3")
@@ -88,7 +86,14 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {
files := make([]string, 0)
for _, item := range response.Contents {
key := *item.Key
- files = append(files, key[len(basePath):])
+
+ // if it ends with '/' it's not an actual blob
+ if strings.HasSuffix(key, "/") {
+ continue
+ }
+
+ fullKey := q.Prefix + key[len(prefix):]
+ files = append(files, strings.TrimLeft(fullKey, "/"))
}
sort.Strings(files)
@@ -153,11 +158,17 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
}
func keyFullPathURL(ctx context.Context, key string) string {
+ return path.Join(basePath(ctx, ""), key)
+}
+
+func basePath(ctx context.Context, segment string) string {
+ blob.EnsureAuthorizedPrefix(ctx, segment)
+
tenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)
if ok {
- return path.Join("tenants", strconv.Itoa(tenant.ID), key)
+ return path.Join("tenants", strconv.Itoa(tenant.ID), segment)
}
- return key
+ return segment
}
func isNotFound(err error) bool {
diff --git a/app/services/blob/sql/sql.go b/app/services/blob/sql/sql.go
index 3b9839592..f85c147d0 100644
--- a/app/services/blob/sql/sql.go
+++ b/app/services/blob/sql/sql.go
@@ -54,7 +54,9 @@ type dbBlob struct {
}
func listBlobs(ctx context.Context, q *query.ListBlobs) error {
- return using(ctx, func(tenant *entity.Tenant) error {
+ blob.EnsureAuthorizedPrefix(ctx, q.Prefix)
+
+ return using(ctx, func(tenantID sql.NullInt64) error {
trx, err := dbx.BeginTx(ctx)
if err != nil {
return errors.Wrap(err, "failed to open transaction")
@@ -62,7 +64,7 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {
defer trx.MustCommit()
blobs := []*dbBlob{}
- err = trx.Select(&blobs, "SELECT key FROM blobs WHERE tenant_id = $1", tenant.ID)
+ err = trx.Select(&blobs, "SELECT key FROM blobs WHERE key LIKE $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", q.Prefix+"%", tenantID)
if err != nil {
return errors.Wrap(err, "failed list blobs")
}
@@ -79,15 +81,9 @@ func listBlobs(ctx context.Context, q *query.ListBlobs) error {
}
func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {
- return using(ctx, func(tenant *entity.Tenant) error {
- var tenantID sql.NullInt64
- if tenant != nil {
- err := tenantID.Scan(tenant.ID)
- if err != nil {
- return errors.Wrap(err, "failed scan tenant id")
- }
- }
+ blob.EnsureAuthorizedPrefix(ctx, q.Key)
+ return using(ctx, func(tenantID sql.NullInt64) error {
trx, err := dbx.BeginTx(ctx)
if err != nil {
return errors.Wrap(err, "failed to open transaction")
@@ -113,19 +109,13 @@ func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {
}
func storeBlob(ctx context.Context, c *cmd.StoreBlob) error {
+ blob.EnsureAuthorizedPrefix(ctx, c.Key)
+
if err := blob.ValidateKey(c.Key); err != nil {
return errors.Wrap(err, "failed to validate blob key '%s'", c.Key)
}
- return using(ctx, func(tenant *entity.Tenant) error {
- var tenantID sql.NullInt64
- if tenant != nil {
- err := tenantID.Scan(tenant.ID)
- if err != nil {
- return errors.Wrap(err, "failed scan tenant id")
- }
- }
-
+ return using(ctx, func(tenantID sql.NullInt64) error {
trx, err := dbx.BeginTx(ctx)
if err != nil {
return errors.Wrap(err, "failed to open transaction")
@@ -151,15 +141,9 @@ func storeBlob(ctx context.Context, c *cmd.StoreBlob) error {
}
func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
- return using(ctx, func(tenant *entity.Tenant) error {
- var tenantID sql.NullInt64
- if tenant != nil {
- err := tenantID.Scan(tenant.ID)
- if err != nil {
- return errors.Wrap(err, "failed scan tenant id")
- }
- }
-
+ blob.EnsureAuthorizedPrefix(ctx, c.Key)
+
+ return using(ctx, func(tenantID sql.NullInt64) error {
trx, err := dbx.BeginTx(ctx)
if err != nil {
return errors.Wrap(err, "failed to open transaction")
@@ -179,7 +163,11 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
})
}
-func using(ctx context.Context, handler func(tenant *entity.Tenant) error) error {
- tenant, _ := ctx.Value(app.TenantCtxKey).(*entity.Tenant)
- return handler(tenant)
-}
+func using(ctx context.Context, handler func(tenantId sql.NullInt64) error) error {
+ var tenantID sql.NullInt64
+ tenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)
+ if ok {
+ _ = tenantID.Scan(tenant.ID)
+ }
+ return handler(tenantID)
+}
\ No newline at end of file