Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

re-use global connection - fixes #1042

Kevin Papst54c1266c4b33
diff --git a/app/services/blob/blob_test.go b/app/services/blob/blob_test.go
index 3c3b841bb..6a0587115 100644
--- a/app/services/blob/blob_test.go
+++ b/app/services/blob/blob_test.go
@@ -4,6 +4,7 @@ import (
"context"
"os"
"testing"
+ "time"
"github.com/aws/aws-sdk-go/aws"
awss3 "github.com/aws/aws-sdk-go/service/s3"
@@ -395,6 +396,63 @@ func KeyFormats(ctx context.Context) {
}
}
+// Regression test for https://github.com/getfider/fider/issues/1042
+// SQL blob operations must reuse the transaction from the context instead of
+// requesting a second connection from the pool, otherwise concurrent requests deadlock
+func TestSQLBlobStorage_ReusesContextTransaction(t *testing.T) {
+ setupSQL(t)
+
+ conn := dbx.Connection()
+ conn.SetMaxOpenConns(1)
+ defer conn.SetMaxOpenConns(env.Config.Database.MaxOpenConns)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ trx, err := dbx.BeginTx(ctx)
+ Expect(err).IsNil()
+ defer trx.MustRollback()
+
+ trxCtx := context.WithValue(ctx, app.TenantCtxKey, tenant1)
+ trxCtx = context.WithValue(trxCtx, app.TransactionCtxKey, trx)
+
+ err = bus.Dispatch(trxCtx, &cmd.StoreBlob{Key: "texts/locked.txt", Content: []byte("hello"), ContentType: "text/plain; charset=utf-8"})
+ Expect(err).IsNil()
+
+ q := &query.GetBlobByKey{Key: "texts/locked.txt"}
+ err = bus.Dispatch(trxCtx, q)
+ Expect(err).IsNil()
+ Expect(q.Result.Content).Equals([]byte("hello"))
+
+ list := &query.ListBlobs{Prefix: "texts/"}
+ err = bus.Dispatch(trxCtx, list)
+ Expect(err).IsNil()
+ Expect(list.Result).Equals([]string{"texts/locked.txt"})
+
+ err = bus.Dispatch(trxCtx, &cmd.DeleteBlob{Key: "texts/locked.txt"})
+ Expect(err).IsNil()
+}
+
+// Blobs stored as part of a transaction are discarded when it is rolled back
+func TestSQLBlobStorage_RollbackDiscardsBlob(t *testing.T) {
+ setupSQL(t)
+
+ ctx := context.WithValue(context.Background(), app.TenantCtxKey, tenant1)
+
+ trx, err := dbx.BeginTx(ctx)
+ Expect(err).IsNil()
+
+ trxCtx := context.WithValue(ctx, app.TransactionCtxKey, trx)
+ err = bus.Dispatch(trxCtx, &cmd.StoreBlob{Key: "texts/rollback.txt", Content: []byte("hello"), ContentType: "text/plain; charset=utf-8"})
+ Expect(err).IsNil()
+
+ trx.MustRollback()
+
+ q := &query.GetBlobByKey{Key: "texts/rollback.txt"}
+ err = bus.Dispatch(ctx, q)
+ Expect(errors.Cause(err)).Equals(blob.ErrNotFound)
+}
+
func TestSanitizeFileName(t *testing.T) {
RegisterT(t)
diff --git a/app/services/blob/sql/sql.go b/app/services/blob/sql/sql.go
index 37f6baa70..b02c68754 100644
--- a/app/services/blob/sql/sql.go
+++ b/app/services/blob/sql/sql.go
@@ -56,15 +56,9 @@ type dbBlob struct {
func listBlobs(ctx context.Context, q *query.ListBlobs) error {
blob.EnsureAuthorizedPrefix(ctx, q.Prefix)
- return using(ctx, func(tenantID sql.NullInt64) error {
- trx, err := dbx.BeginTx(ctx)
- if err != nil {
- return errors.Wrap(err, "failed to open transaction")
- }
- defer trx.MustCommit()
-
+ return using(ctx, func(trx *dbx.Trx, tenantID sql.NullInt64) error {
blobs := []*dbBlob{}
- err = trx.Select(&blobs, "SELECT key FROM blobs WHERE key LIKE $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", q.Prefix+"%", tenantID)
+ err := trx.Select(&blobs, "SELECT key FROM blobs WHERE key LIKE $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", q.Prefix+"%", tenantID)
if err != nil {
return errors.Wrap(err, "failed list blobs")
}
@@ -87,15 +81,9 @@ func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {
blob.EnsureAuthorizedPrefix(ctx, q.Key)
- return using(ctx, func(tenantID sql.NullInt64) error {
- trx, err := dbx.BeginTx(ctx)
- if err != nil {
- return errors.Wrap(err, "failed to open transaction")
- }
- defer trx.MustCommit()
-
+ return using(ctx, func(trx *dbx.Trx, tenantID sql.NullInt64) error {
b := dbBlob{}
- err = trx.Get(&b, "SELECT file, content_type, size FROM blobs WHERE key = $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", q.Key, tenantID)
+ err := trx.Get(&b, "SELECT file, content_type, size FROM blobs WHERE key = $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", q.Key, tenantID)
if err != nil {
if err == app.ErrNotFound {
return blob.ErrNotFound
@@ -119,15 +107,9 @@ func storeBlob(ctx context.Context, c *cmd.StoreBlob) error {
return errors.Wrap(err, "failed to validate blob key '%s'", c.Key)
}
- return using(ctx, func(tenantID sql.NullInt64) error {
- trx, err := dbx.BeginTx(ctx)
- if err != nil {
- return errors.Wrap(err, "failed to open transaction")
- }
- defer trx.MustCommit()
-
+ return using(ctx, func(trx *dbx.Trx, tenantID sql.NullInt64) error {
now := time.Now()
- _, err = trx.Execute(`
+ _, err := trx.Execute(`
INSERT INTO blobs (tenant_id, key, size, content_type, file, created_at, modified_at)
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (tenant_id, key)
DO UPDATE SET size = $3, content_type = $4, file = $5, modified_at = $7
@@ -146,15 +128,9 @@ func storeBlob(ctx context.Context, c *cmd.StoreBlob) error {
func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
blob.EnsureAuthorizedPrefix(ctx, c.Key)
-
- return using(ctx, func(tenantID sql.NullInt64) error {
- trx, err := dbx.BeginTx(ctx)
- if err != nil {
- return errors.Wrap(err, "failed to open transaction")
- }
- defer trx.MustCommit()
- _, err = trx.Execute("DELETE FROM blobs WHERE key = $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", c.Key, tenantID)
+ return using(ctx, func(trx *dbx.Trx, tenantID sql.NullInt64) error {
+ _, err := trx.Execute("DELETE FROM blobs WHERE key = $1 AND (tenant_id = $2 OR ($2 IS NULL AND tenant_id IS NULL))", c.Key, tenantID)
if err != nil {
return errors.Wrap(err, "failed to delete blob with key '%s'", c.Key)
}
@@ -167,11 +143,29 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
})
}
-func using(ctx context.Context, handler func(tenantId sql.NullInt64) error) error {
+func using(ctx context.Context, handler func(trx *dbx.Trx, tenantId sql.NullInt64) error) error {
var tenantID sql.NullInt64
tenant, ok := ctx.Value(app.TenantCtxKey).(*entity.Tenant)
if ok {
_ = tenantID.Scan(tenant.ID)
}
- return handler(tenantID)
+
+ // Reuse the transaction of the current request/job if there is one.
+ // Opening a new one would need a second connection while the caller already holds one,
+ // which deadlocks the pool once all connections are held by requests waiting for a second one.
+ if trx, ok := ctx.Value(app.TransactionCtxKey).(*dbx.Trx); ok && trx != nil {
+ return handler(trx, tenantID)
+ }
+
+ trx, err := dbx.BeginTx(ctx)
+ if err != nil {
+ return errors.Wrap(err, "failed to open transaction")
+ }
+
+ if err := handler(trx, tenantID); err != nil {
+ trx.MustRollback()
+ return err
+ }
+
+ return trx.Commit()
}
\ No newline at end of file