Open platform to collect and prioritize feedback
feature: export all data (#758)
Guilherme Oenning345fec2fbe33
diff --git a/app/cmd/routes.go b/app/cmd/routes.goindex 9d3bcc43e..216b75801 100644--- a/app/cmd/routes.go+++ b/app/cmd/routes.go@@ -151,6 +151,7 @@ func routes(r *web.Engine) *web.Engine {ui.Get("/admin/export", handlers.Page("Export · Site Settings", "", "Export.page"))ui.Get("/admin/export/posts.csv", handlers.ExportPostsToCSV())+ ui.Get("/admin/export/backup.zip", handlers.ExportBackupZip())ui.Post("/_api/admin/settings/general", handlers.UpdateSettings())ui.Post("/_api/admin/settings/advanced", handlers.UpdateAdvancedSettings())ui.Post("/_api/admin/settings/privacy", handlers.UpdatePrivacy())diff --git a/app/handlers/backup.go b/app/handlers/backup.gonew file mode 100644index 000000000..5bba216c6--- /dev/null+++ b/app/handlers/backup.go@@ -0,0 +1,19 @@+package handlers++import (+ "github.com/getfider/fider/app/pkg/backup"+ "github.com/getfider/fider/app/pkg/web"+)++// ExportBackupZip returns a Zip file with all content+func ExportBackupZip() web.HandlerFunc {+ return func(c *web.Context) error {++ file, err := backup.Create(c)+ if err != nil {+ return c.Failure(err)+ }++ return c.Attachment("backup.zip", "application/zip", file.Bytes())+ }+}diff --git a/app/models/query/blob.go b/app/models/query/blob.goindex 8d786d629..87451c693 100644--- a/app/models/query/blob.go+++ b/app/models/query/blob.go@@ -2,6 +2,10 @@ package queryimport "github.com/getfider/fider/app/models/dto"+type ListBlobs struct {+ Result []string+}+type GetBlobByKey struct {Key stringdiff --git a/app/pkg/backup/backup.go b/app/pkg/backup/backup.gonew file mode 100644index 000000000..8accd2169--- /dev/null+++ b/app/pkg/backup/backup.go@@ -0,0 +1,97 @@+package backup++import (+ "archive/zip"+ "bytes"+ "context"+ "fmt"++ "github.com/getfider/fider/app/models/query"+ "github.com/getfider/fider/app/pkg/bus"++ "github.com/getfider/fider/app/pkg/errors"+)++func Create(ctx context.Context) (*bytes.Buffer, error) {++ buffer := new(bytes.Buffer)+ zipWriter := zip.NewWriter(buffer)++ for _, tableName := range []string{+ "attachments",+ "comments",+ "email_verifications",+ "notifications",+ "oauth_providers",+ "posts",+ "post_subscribers",+ "post_tags",+ "post_votes",+ "tags",+ "tenants",+ "user_providers",+ "users",+ "user_settings",+ } {+ err := addTableDataToZipFile(ctx, zipWriter, tableName)+ if err != nil {+ return nil, err+ }+ }++ listBlobs := &query.ListBlobs{}+ if err := bus.Dispatch(ctx, listBlobs); err != nil {+ return nil, err+ }++ for _, bkey := range listBlobs.Result {+ err := addBlobToZipFile(ctx, zipWriter, bkey)+ if err != nil {+ return nil, err+ }+ }++ err := zipWriter.Close()+ if err != nil {+ return nil, errors.Wrap(err, "failed to close zip file")+ }++ return buffer, nil+}++func addBlobToZipFile(ctx context.Context, zipWriter *zip.Writer, bkey string) error {+ getBlob := &query.GetBlobByKey{Key: bkey}+ if err := bus.Dispatch(ctx, getBlob); err != nil {+ return err+ }++ fileName := fmt.Sprintf("blobs/%s", bkey)+ fileWriter, err := zipWriter.Create(fileName)+ if err != nil {+ return errors.Wrap(err, "failed to create %s in zip file", fileName)+ }+ _, err = fileWriter.Write(getBlob.Result.Content)+ if err != nil {+ return errors.Wrap(err, "failed to write %s to zip file", fileName)+ }++ return nil+}++func addTableDataToZipFile(ctx context.Context, zipWriter *zip.Writer, tableName string) error {+ tableData, err := exportTable(ctx, tableName)+ if err != nil {+ return errors.Wrap(err, "failed to export %s table", tableName)+ }++ fileWriter, err := zipWriter.Create(fmt.Sprintf("%s.json", tableName))+ if err != nil {+ return errors.Wrap(err, "failed to create %s.json in zip file", tableName)+ }+ _, err = fileWriter.Write(tableData)+ if err != nil {+ return errors.Wrap(err, "failed to write %s.json to zip file", tableName)+ }++ return nil+}diff --git a/app/pkg/backup/export.go b/app/pkg/backup/export.gonew file mode 100644index 000000000..293fe4085--- /dev/null+++ b/app/pkg/backup/export.go@@ -0,0 +1,77 @@+package backup++import (+ "context"+ "database/sql"+ "encoding/json"+ "fmt"+ "strconv"++ "github.com/getfider/fider/app"+ "github.com/getfider/fider/app/models"+ "github.com/getfider/fider/app/pkg/dbx"+)++func exportTable(ctx context.Context, tableName string) ([]byte, error) {+ trx := ctx.Value(app.TransactionCtxKey).(*dbx.Trx)+ tenant, _ := ctx.Value(app.TenantCtxKey).(*models.Tenant)+ columnName := "tenant_id"+ if tableName == "tenants" {+ columnName = "id"+ }++ rows, err := trx.Query(fmt.Sprintf("SELECT * FROM %s WHERE %s = $1", tableName, columnName), tenant.ID)+ if err != nil {+ return nil, err+ }++ return json.Marshal(jsonify(rows))+}++func jsonify(rows *sql.Rows) []map[string]interface{} {+ defer rows.Close()+ columns, err := rows.Columns()+ if err != nil {+ panic(err.Error())+ }++ allResults := make([]map[string]interface{}, 0)++ for rows.Next() {+ results := make(map[string]interface{})+ values := make([]interface{}, len(columns))+ scanArgs := make([]interface{}, len(values))+ for i := range values {+ scanArgs[i] = &values[i]+ }++ err = rows.Scan(scanArgs...)+ if err != nil {+ panic(err.Error())+ }++ for i, value := range values {+ switch value.(type) {+ case nil:+ results[columns[i]] = nil++ case []byte:+ s := string(value.([]byte))+ x, err := strconv.Atoi(s)++ if err != nil {+ results[columns[i]] = s+ } else {+ results[columns[i]] = x+ }++ default:+ results[columns[i]] = value+ }+ }++ allResults = append(allResults, results)+ }++ return allResults+}diff --git a/app/pkg/dbx/dbx.go b/app/pkg/dbx/dbx.goindex abd848288..67e6094a7 100644--- a/app/pkg/dbx/dbx.go+++ b/app/pkg/dbx/dbx.go@@ -250,6 +250,27 @@ func (trx *Trx) Select(data interface{}, command string, args ...interface{}) erreturn nil}+//Query all matched rows and return raw sql.Rows+func (trx *Trx) Query(command string, args ...interface{}) (*sql.Rows, error) {+ if log.IsEnabled(log.DEBUG) {+ command = formatter.Replace(command)+ start := time.Now()+ defer func() {+ log.Debugf(trx.ctx, "@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", dto.Props{+ "Command": command,+ "Args": args,+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),+ })+ }()+ }++ rows, err := trx.tx.QueryContext(trx.ctx, command, args...)+ if err != nil {+ return nil, wrap(err, "failed to execute trx.Select")+ }+ return rows, nil+}+// Commit current transactionfunc (trx *Trx) Commit() error {err := trx.tx.Commit()diff --git a/app/services/blob/blob_test.go b/app/services/blob/blob_test.goindex a5be9cc05..0a0505318 100644--- a/app/services/blob/blob_test.go+++ b/app/services/blob/blob_test.go@@ -36,9 +36,6 @@ func setupS3(t *testing.T) {bus.Init(s3.Service{})bucket := aws.String(env.Config.BlobStorage.S3.BucketName)- s3.DefaultClient.DeleteBucket(&awss3.DeleteBucketInput{- Bucket: bucket,- })s3.DefaultClient.CreateBucket(&awss3.CreateBucketInput{Bucket: bucket,})@@ -72,6 +69,7 @@ var tests = []struct {{"KeyFormats", KeyFormats},{"SameKey_DifferentTenant", SameKey_DifferentTenant},{"SameKey_DifferentTenant_Delete", SameKey_DifferentTenant_Delete},+ {"ListBlobsFromTenant", ListBlobsFromTenant},}func TestBlobStorage(t *testing.T) {@@ -171,6 +169,9 @@ func SameKey_DifferentTenant(ctx context.Context) {err = bus.Dispatch(ctx, q)Expect(err).Equals(blob.ErrNotFound)Expect(q.Result).IsNil()++ err = bus.Dispatch(ctxWithTenant1, &cmd.DeleteBlob{Key: key})+ Expect(err).IsNil()}func SameKey_DifferentTenant_Delete(ctx context.Context) {@@ -217,6 +218,47 @@ func SameKey_DifferentTenant_Delete(ctx context.Context) {err = bus.Dispatch(ctx, q)Expect(err).Equals(blob.ErrNotFound)Expect(q.Result).IsNil()++ err = bus.Dispatch(ctxWithTenant2, &cmd.DeleteBlob{Key: key})+ Expect(err).IsNil()+}++func ListBlobsFromTenant(ctx context.Context) {+ ctxWithTenant1 := context.WithValue(ctx, app.TenantCtxKey, tenant1)+ ctxWithTenant2 := context.WithValue(ctx, app.TenantCtxKey, tenant2)++ err := bus.Dispatch(ctxWithTenant1, &cmd.StoreBlob{+ Key: "texts/hello.txt",+ Content: make([]byte, 0),+ ContentType: "text/plain; charset=utf-8",+ })+ Expect(err).IsNil()++ err = bus.Dispatch(ctxWithTenant1, &cmd.StoreBlob{+ Key: "memos/hello1.txt",+ Content: make([]byte, 0),+ ContentType: "text/plain; charset=utf-8",+ })+ Expect(err).IsNil()++ err = bus.Dispatch(ctxWithTenant2, &cmd.StoreBlob{+ Key: "texts/hello.txt",+ Content: make([]byte, 0),+ ContentType: "text/plain; charset=utf-8",+ })+ Expect(err).IsNil()++ tenant1Files := &query.ListBlobs{}+ err = bus.Dispatch(ctxWithTenant1, tenant1Files)+ Expect(err).IsNil()+ Expect(tenant1Files.Result).HasLen(2)+ Expect(tenant1Files.Result).Equals([]string{"memos/hello1.txt", "texts/hello.txt"})++ tenant2Files := &query.ListBlobs{}+ err = bus.Dispatch(ctxWithTenant2, tenant2Files)+ Expect(err).IsNil()+ Expect(tenant2Files.Result).HasLen(1)+ Expect(tenant2Files.Result).Equals([]string{"texts/hello.txt"})}func KeyFormats(ctx context.Context) {diff --git a/app/services/blob/fs/fs.go b/app/services/blob/fs/fs.goindex e3996fcfc..66ca362ad 100644--- a/app/services/blob/fs/fs.go+++ b/app/services/blob/fs/fs.go@@ -7,6 +7,7 @@ import ("os""path""path/filepath"+ "sort""strconv""github.com/getfider/fider/app"@@ -43,11 +44,35 @@ func (s Service) Enabled() bool {}func (s Service) Init() {+ bus.AddHandler(listBlobs)bus.AddHandler(getBlobByKey)bus.AddHandler(storeBlob)bus.AddHandler(deleteBlob)}+func listBlobs(ctx context.Context, q *query.ListBlobs) error {+ basePath := basePath(ctx)+ files := make([]string, 0)++ err := filepath.Walk(basePath,+ func(path string, info os.FileInfo, err error) error {+ if err != nil {+ return err+ }+ if !info.IsDir() {+ files = append(files, path[len(basePath)+1:])+ }+ return nil+ })+ if err != nil {+ return errors.Wrap(err, "failed to read dir '%s'", basePath)+ }++ sort.Strings(files)+ q.Result = files+ return nil+}+func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {fullPath := keyFullPath(ctx, q.Key)stats, err := os.Stat(fullPath)@@ -101,10 +126,14 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {}func keyFullPath(ctx context.Context, key string) string {- basePath := env.Config.BlobStorage.FS.Path+ return path.Join(basePath(ctx), key)+}++func basePath(ctx context.Context) string {+ startPath := env.Config.BlobStorage.FS.Pathtenant, ok := ctx.Value(app.TenantCtxKey).(*models.Tenant)if ok {- return path.Join(basePath, "tenants", strconv.Itoa(tenant.ID), key)+ return path.Join(startPath, "tenants", strconv.Itoa(tenant.ID))}- return path.Join(basePath, key)+ return path.Join(startPath)}diff --git a/app/services/blob/s3/s3.go b/app/services/blob/s3/s3.goindex 210ee764f..59e3655e1 100644--- a/app/services/blob/s3/s3.go+++ b/app/services/blob/s3/s3.go@@ -3,8 +3,10 @@ package s3import ("bytes""context"+ "fmt""io/ioutil""path"+ "sort""strconv""strings"@@ -60,11 +62,36 @@ func (s Service) Init() {awsSession := session.New(s3Config)DefaultClient = s3.New(awsSession)}++ bus.AddHandler(listBlobs)bus.AddHandler(getBlobByKey)bus.AddHandler(storeBlob)bus.AddHandler(deleteBlob)}+func listBlobs(ctx context.Context, q *query.ListBlobs) error {+ tenant := ctx.Value(app.TenantCtxKey).(*models.Tenant)+ basePath := fmt.Sprintf("tenants/%d/", tenant.ID)+ response, err := DefaultClient.ListObjectsWithContext(ctx, &s3.ListObjectsInput{+ Bucket: aws.String(env.Config.BlobStorage.S3.BucketName),+ MaxKeys: aws.Int64(3000),+ Prefix: aws.String(basePath),+ })+ if err != nil {+ return wrap(err, "failed to list blobs from S3")+ }++ files := make([]string, 0)+ for _, item := range response.Contents {+ key := *item.Key+ files = append(files, key[len(basePath):])+ }++ sort.Strings(files)+ q.Result = files+ return nil+}+func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {resp, err := DefaultClient.GetObjectWithContext(ctx, &s3.GetObjectInput{Bucket: aws.String(env.Config.BlobStorage.S3.BucketName),diff --git a/app/services/blob/sql/sql.go b/app/services/blob/sql/sql.goindex 2706a5528..31e4bb1db 100644--- a/app/services/blob/sql/sql.go+++ b/app/services/blob/sql/sql.go@@ -3,6 +3,7 @@ package sqlimport ("context""database/sql"+ "sort""time""github.com/getfider/fider/app/models/cmd"@@ -39,17 +40,44 @@ func (s Service) Enabled() bool {}func (s Service) Init() {+ bus.AddHandler(listBlobs)bus.AddHandler(getBlobByKey)bus.AddHandler(storeBlob)bus.AddHandler(deleteBlob)}type dbBlob struct {+ Key string `db:"key"`ContentType string `db:"content_type"`Size int64 `db:"size"`Content []byte `db:"file"`}+func listBlobs(ctx context.Context, q *query.ListBlobs) error {+ return using(ctx, func(tenant *models.Tenant) error {+ trx, err := dbx.BeginTx(ctx)+ if err != nil {+ return errors.Wrap(err, "failed to open transaction")+ }+ defer trx.Commit()++ blobs := []*dbBlob{}+ err = trx.Select(&blobs, "SELECT key FROM blobs WHERE tenant_id = $1", tenant.ID)+ if err != nil {+ return errors.Wrap(err, "failed list blobs")+ }++ files := make([]string, len(blobs))+ for i, b := range blobs {+ files[i] = b.Key+ }++ sort.Strings(files)+ q.Result = files+ return nil+ })+}+func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {return using(ctx, func(tenant *models.Tenant) error {var tenantID sql.NullInt64diff --git a/public/pages/Administration/pages/Export.page.tsx b/public/pages/Administration/pages/Export.page.tsxindex c34363e4f..7ec4754a6 100644--- a/public/pages/Administration/pages/Export.page.tsx+++ b/public/pages/Administration/pages/Export.page.tsx@@ -1,6 +1,6 @@import React from "react";-import { Button, Form, Field } from "@fider/components";+import { Button, Form, Field, Segment } from "@fider/components";import { FaRegFileExcel } from "react-icons/fa";import { AdminBasePage } from "../components/AdminBasePage";@@ -14,17 +14,32 @@ export default class ExportPage extends AdminBasePage<{}, {}> {public content() {return (<Form>- <Field label="Export Posts">- <p className="info">- Use this button to download a CSV file with all posts in this site. This can be useful to analyse the data- with an external tool or simply to back it up.- </p>- </Field>- <Field>- <Button color="positive" href="/admin/export/posts.csv">- Download- </Button>- </Field>+ <Segment>+ <Field label="Export Posts">+ <p className="info">+ Use this button to download a CSV file with all posts in this site. This can be useful to analyse the data+ with an external tool or simply to back it up.+ </p>+ </Field>+ <Field>+ <Button color="positive" href="/admin/export/posts.csv">+ posts.csv+ </Button>+ </Field>+ </Segment>+ <Segment>+ <Field label="Backup your data">+ <p className="info">+ Use this button to download a ZIP file with your data in JSON format. This is a full backup and contains+ all of your data.+ </p>+ </Field>+ <Field>+ <Button color="positive" href="/admin/export/backup.zip">+ backup.zip+ </Button>+ </Field>+ </Segment></Form>);}