Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

feature: export all data (#758)

Guilherme Oenning345fec2fbe33
diff --git a/app/cmd/routes.go b/app/cmd/routes.go
index 9d3bcc43e..216b75801 100644
--- a/app/cmd/routes.go
+++ b/app/cmd/routes.go
@@ -151,6 +151,7 @@ func routes(r *web.Engine) *web.Engine {
ui.Get("/admin/export", handlers.Page("Export · Site Settings", "", "Export.page"))
ui.Get("/admin/export/posts.csv", handlers.ExportPostsToCSV())
+ ui.Get("/admin/export/backup.zip", handlers.ExportBackupZip())
ui.Post("/_api/admin/settings/general", handlers.UpdateSettings())
ui.Post("/_api/admin/settings/advanced", handlers.UpdateAdvancedSettings())
ui.Post("/_api/admin/settings/privacy", handlers.UpdatePrivacy())
diff --git a/app/handlers/backup.go b/app/handlers/backup.go
new file mode 100644
index 000000000..5bba216c6
--- /dev/null
+++ b/app/handlers/backup.go
@@ -0,0 +1,19 @@
+package handlers
+
+import (
+ "github.com/getfider/fider/app/pkg/backup"
+ "github.com/getfider/fider/app/pkg/web"
+)
+
+// ExportBackupZip returns a Zip file with all content
+func ExportBackupZip() web.HandlerFunc {
+ return func(c *web.Context) error {
+
+ file, err := backup.Create(c)
+ if err != nil {
+ return c.Failure(err)
+ }
+
+ return c.Attachment("backup.zip", "application/zip", file.Bytes())
+ }
+}
diff --git a/app/models/query/blob.go b/app/models/query/blob.go
index 8d786d629..87451c693 100644
--- a/app/models/query/blob.go
+++ b/app/models/query/blob.go
@@ -2,6 +2,10 @@ package query
import "github.com/getfider/fider/app/models/dto"
+type ListBlobs struct {
+ Result []string
+}
+
type GetBlobByKey struct {
Key string
diff --git a/app/pkg/backup/backup.go b/app/pkg/backup/backup.go
new file mode 100644
index 000000000..8accd2169
--- /dev/null
+++ b/app/pkg/backup/backup.go
@@ -0,0 +1,97 @@
+package backup
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "fmt"
+
+ "github.com/getfider/fider/app/models/query"
+ "github.com/getfider/fider/app/pkg/bus"
+
+ "github.com/getfider/fider/app/pkg/errors"
+)
+
+func Create(ctx context.Context) (*bytes.Buffer, error) {
+
+ buffer := new(bytes.Buffer)
+ zipWriter := zip.NewWriter(buffer)
+
+ for _, tableName := range []string{
+ "attachments",
+ "comments",
+ "email_verifications",
+ "notifications",
+ "oauth_providers",
+ "posts",
+ "post_subscribers",
+ "post_tags",
+ "post_votes",
+ "tags",
+ "tenants",
+ "user_providers",
+ "users",
+ "user_settings",
+ } {
+ err := addTableDataToZipFile(ctx, zipWriter, tableName)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ listBlobs := &query.ListBlobs{}
+ if err := bus.Dispatch(ctx, listBlobs); err != nil {
+ return nil, err
+ }
+
+ for _, bkey := range listBlobs.Result {
+ err := addBlobToZipFile(ctx, zipWriter, bkey)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ err := zipWriter.Close()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to close zip file")
+ }
+
+ return buffer, nil
+}
+
+func addBlobToZipFile(ctx context.Context, zipWriter *zip.Writer, bkey string) error {
+ getBlob := &query.GetBlobByKey{Key: bkey}
+ if err := bus.Dispatch(ctx, getBlob); err != nil {
+ return err
+ }
+
+ fileName := fmt.Sprintf("blobs/%s", bkey)
+ fileWriter, err := zipWriter.Create(fileName)
+ if err != nil {
+ return errors.Wrap(err, "failed to create %s in zip file", fileName)
+ }
+ _, err = fileWriter.Write(getBlob.Result.Content)
+ if err != nil {
+ return errors.Wrap(err, "failed to write %s to zip file", fileName)
+ }
+
+ return nil
+}
+
+func addTableDataToZipFile(ctx context.Context, zipWriter *zip.Writer, tableName string) error {
+ tableData, err := exportTable(ctx, tableName)
+ if err != nil {
+ return errors.Wrap(err, "failed to export %s table", tableName)
+ }
+
+ fileWriter, err := zipWriter.Create(fmt.Sprintf("%s.json", tableName))
+ if err != nil {
+ return errors.Wrap(err, "failed to create %s.json in zip file", tableName)
+ }
+ _, err = fileWriter.Write(tableData)
+ if err != nil {
+ return errors.Wrap(err, "failed to write %s.json to zip file", tableName)
+ }
+
+ return nil
+}
diff --git a/app/pkg/backup/export.go b/app/pkg/backup/export.go
new file mode 100644
index 000000000..293fe4085
--- /dev/null
+++ b/app/pkg/backup/export.go
@@ -0,0 +1,77 @@
+package backup
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "fmt"
+ "strconv"
+
+ "github.com/getfider/fider/app"
+ "github.com/getfider/fider/app/models"
+ "github.com/getfider/fider/app/pkg/dbx"
+)
+
+func exportTable(ctx context.Context, tableName string) ([]byte, error) {
+ trx := ctx.Value(app.TransactionCtxKey).(*dbx.Trx)
+ tenant, _ := ctx.Value(app.TenantCtxKey).(*models.Tenant)
+ columnName := "tenant_id"
+ if tableName == "tenants" {
+ columnName = "id"
+ }
+
+ rows, err := trx.Query(fmt.Sprintf("SELECT * FROM %s WHERE %s = $1", tableName, columnName), tenant.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ return json.Marshal(jsonify(rows))
+}
+
+func jsonify(rows *sql.Rows) []map[string]interface{} {
+ defer rows.Close()
+ columns, err := rows.Columns()
+ if err != nil {
+ panic(err.Error())
+ }
+
+ allResults := make([]map[string]interface{}, 0)
+
+ for rows.Next() {
+ results := make(map[string]interface{})
+ values := make([]interface{}, len(columns))
+ scanArgs := make([]interface{}, len(values))
+ for i := range values {
+ scanArgs[i] = &values[i]
+ }
+
+ err = rows.Scan(scanArgs...)
+ if err != nil {
+ panic(err.Error())
+ }
+
+ for i, value := range values {
+ switch value.(type) {
+ case nil:
+ results[columns[i]] = nil
+
+ case []byte:
+ s := string(value.([]byte))
+ x, err := strconv.Atoi(s)
+
+ if err != nil {
+ results[columns[i]] = s
+ } else {
+ results[columns[i]] = x
+ }
+
+ default:
+ results[columns[i]] = value
+ }
+ }
+
+ allResults = append(allResults, results)
+ }
+
+ return allResults
+}
diff --git a/app/pkg/dbx/dbx.go b/app/pkg/dbx/dbx.go
index abd848288..67e6094a7 100644
--- a/app/pkg/dbx/dbx.go
+++ b/app/pkg/dbx/dbx.go
@@ -250,6 +250,27 @@ func (trx *Trx) Select(data interface{}, command string, args ...interface{}) er
return nil
}
+//Query all matched rows and return raw sql.Rows
+func (trx *Trx) Query(command string, args ...interface{}) (*sql.Rows, error) {
+ if log.IsEnabled(log.DEBUG) {
+ command = formatter.Replace(command)
+ start := time.Now()
+ defer func() {
+ log.Debugf(trx.ctx, "@{Command:yellow} @{Args:blue} executed in @{ElapsedMs:magenta}ms", dto.Props{
+ "Command": command,
+ "Args": args,
+ "ElapsedMs": time.Since(start).Nanoseconds() / int64(time.Millisecond),
+ })
+ }()
+ }
+
+ rows, err := trx.tx.QueryContext(trx.ctx, command, args...)
+ if err != nil {
+ return nil, wrap(err, "failed to execute trx.Select")
+ }
+ return rows, nil
+}
+
// Commit current transaction
func (trx *Trx) Commit() error {
err := trx.tx.Commit()
diff --git a/app/services/blob/blob_test.go b/app/services/blob/blob_test.go
index a5be9cc05..0a0505318 100644
--- a/app/services/blob/blob_test.go
+++ b/app/services/blob/blob_test.go
@@ -36,9 +36,6 @@ func setupS3(t *testing.T) {
bus.Init(s3.Service{})
bucket := aws.String(env.Config.BlobStorage.S3.BucketName)
- s3.DefaultClient.DeleteBucket(&awss3.DeleteBucketInput{
- Bucket: bucket,
- })
s3.DefaultClient.CreateBucket(&awss3.CreateBucketInput{
Bucket: bucket,
})
@@ -72,6 +69,7 @@ var tests = []struct {
{"KeyFormats", KeyFormats},
{"SameKey_DifferentTenant", SameKey_DifferentTenant},
{"SameKey_DifferentTenant_Delete", SameKey_DifferentTenant_Delete},
+ {"ListBlobsFromTenant", ListBlobsFromTenant},
}
func TestBlobStorage(t *testing.T) {
@@ -171,6 +169,9 @@ func SameKey_DifferentTenant(ctx context.Context) {
err = bus.Dispatch(ctx, q)
Expect(err).Equals(blob.ErrNotFound)
Expect(q.Result).IsNil()
+
+ err = bus.Dispatch(ctxWithTenant1, &cmd.DeleteBlob{Key: key})
+ Expect(err).IsNil()
}
func SameKey_DifferentTenant_Delete(ctx context.Context) {
@@ -217,6 +218,47 @@ func SameKey_DifferentTenant_Delete(ctx context.Context) {
err = bus.Dispatch(ctx, q)
Expect(err).Equals(blob.ErrNotFound)
Expect(q.Result).IsNil()
+
+ err = bus.Dispatch(ctxWithTenant2, &cmd.DeleteBlob{Key: key})
+ Expect(err).IsNil()
+}
+
+func ListBlobsFromTenant(ctx context.Context) {
+ ctxWithTenant1 := context.WithValue(ctx, app.TenantCtxKey, tenant1)
+ ctxWithTenant2 := context.WithValue(ctx, app.TenantCtxKey, tenant2)
+
+ err := bus.Dispatch(ctxWithTenant1, &cmd.StoreBlob{
+ Key: "texts/hello.txt",
+ Content: make([]byte, 0),
+ ContentType: "text/plain; charset=utf-8",
+ })
+ Expect(err).IsNil()
+
+ err = bus.Dispatch(ctxWithTenant1, &cmd.StoreBlob{
+ Key: "memos/hello1.txt",
+ Content: make([]byte, 0),
+ ContentType: "text/plain; charset=utf-8",
+ })
+ Expect(err).IsNil()
+
+ err = bus.Dispatch(ctxWithTenant2, &cmd.StoreBlob{
+ Key: "texts/hello.txt",
+ Content: make([]byte, 0),
+ ContentType: "text/plain; charset=utf-8",
+ })
+ Expect(err).IsNil()
+
+ tenant1Files := &query.ListBlobs{}
+ err = bus.Dispatch(ctxWithTenant1, tenant1Files)
+ Expect(err).IsNil()
+ Expect(tenant1Files.Result).HasLen(2)
+ Expect(tenant1Files.Result).Equals([]string{"memos/hello1.txt", "texts/hello.txt"})
+
+ tenant2Files := &query.ListBlobs{}
+ err = bus.Dispatch(ctxWithTenant2, tenant2Files)
+ Expect(err).IsNil()
+ Expect(tenant2Files.Result).HasLen(1)
+ Expect(tenant2Files.Result).Equals([]string{"texts/hello.txt"})
}
func KeyFormats(ctx context.Context) {
diff --git a/app/services/blob/fs/fs.go b/app/services/blob/fs/fs.go
index e3996fcfc..66ca362ad 100644
--- a/app/services/blob/fs/fs.go
+++ b/app/services/blob/fs/fs.go
@@ -7,6 +7,7 @@ import (
"os"
"path"
"path/filepath"
+ "sort"
"strconv"
"github.com/getfider/fider/app"
@@ -43,11 +44,35 @@ func (s Service) Enabled() bool {
}
func (s Service) Init() {
+ bus.AddHandler(listBlobs)
bus.AddHandler(getBlobByKey)
bus.AddHandler(storeBlob)
bus.AddHandler(deleteBlob)
}
+func listBlobs(ctx context.Context, q *query.ListBlobs) error {
+ basePath := basePath(ctx)
+ files := make([]string, 0)
+
+ err := filepath.Walk(basePath,
+ func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() {
+ files = append(files, path[len(basePath)+1:])
+ }
+ return nil
+ })
+ if err != nil {
+ return errors.Wrap(err, "failed to read dir '%s'", basePath)
+ }
+
+ sort.Strings(files)
+ q.Result = files
+ return nil
+}
+
func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {
fullPath := keyFullPath(ctx, q.Key)
stats, err := os.Stat(fullPath)
@@ -101,10 +126,14 @@ func deleteBlob(ctx context.Context, c *cmd.DeleteBlob) error {
}
func keyFullPath(ctx context.Context, key string) string {
- basePath := env.Config.BlobStorage.FS.Path
+ return path.Join(basePath(ctx), key)
+}
+
+func basePath(ctx context.Context) string {
+ startPath := env.Config.BlobStorage.FS.Path
tenant, ok := ctx.Value(app.TenantCtxKey).(*models.Tenant)
if ok {
- return path.Join(basePath, "tenants", strconv.Itoa(tenant.ID), key)
+ return path.Join(startPath, "tenants", strconv.Itoa(tenant.ID))
}
- return path.Join(basePath, key)
+ return path.Join(startPath)
}
diff --git a/app/services/blob/s3/s3.go b/app/services/blob/s3/s3.go
index 210ee764f..59e3655e1 100644
--- a/app/services/blob/s3/s3.go
+++ b/app/services/blob/s3/s3.go
@@ -3,8 +3,10 @@ package s3
import (
"bytes"
"context"
+ "fmt"
"io/ioutil"
"path"
+ "sort"
"strconv"
"strings"
@@ -60,11 +62,36 @@ func (s Service) Init() {
awsSession := session.New(s3Config)
DefaultClient = s3.New(awsSession)
}
+
+ bus.AddHandler(listBlobs)
bus.AddHandler(getBlobByKey)
bus.AddHandler(storeBlob)
bus.AddHandler(deleteBlob)
}
+func listBlobs(ctx context.Context, q *query.ListBlobs) error {
+ tenant := ctx.Value(app.TenantCtxKey).(*models.Tenant)
+ basePath := fmt.Sprintf("tenants/%d/", tenant.ID)
+ response, err := DefaultClient.ListObjectsWithContext(ctx, &s3.ListObjectsInput{
+ Bucket: aws.String(env.Config.BlobStorage.S3.BucketName),
+ MaxKeys: aws.Int64(3000),
+ Prefix: aws.String(basePath),
+ })
+ if err != nil {
+ return wrap(err, "failed to list blobs from S3")
+ }
+
+ files := make([]string, 0)
+ for _, item := range response.Contents {
+ key := *item.Key
+ files = append(files, key[len(basePath):])
+ }
+
+ sort.Strings(files)
+ q.Result = files
+ return nil
+}
+
func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {
resp, err := DefaultClient.GetObjectWithContext(ctx, &s3.GetObjectInput{
Bucket: aws.String(env.Config.BlobStorage.S3.BucketName),
diff --git a/app/services/blob/sql/sql.go b/app/services/blob/sql/sql.go
index 2706a5528..31e4bb1db 100644
--- a/app/services/blob/sql/sql.go
+++ b/app/services/blob/sql/sql.go
@@ -3,6 +3,7 @@ package sql
import (
"context"
"database/sql"
+ "sort"
"time"
"github.com/getfider/fider/app/models/cmd"
@@ -39,17 +40,44 @@ func (s Service) Enabled() bool {
}
func (s Service) Init() {
+ bus.AddHandler(listBlobs)
bus.AddHandler(getBlobByKey)
bus.AddHandler(storeBlob)
bus.AddHandler(deleteBlob)
}
type dbBlob struct {
+ Key string `db:"key"`
ContentType string `db:"content_type"`
Size int64 `db:"size"`
Content []byte `db:"file"`
}
+func listBlobs(ctx context.Context, q *query.ListBlobs) error {
+ return using(ctx, func(tenant *models.Tenant) error {
+ trx, err := dbx.BeginTx(ctx)
+ if err != nil {
+ return errors.Wrap(err, "failed to open transaction")
+ }
+ defer trx.Commit()
+
+ blobs := []*dbBlob{}
+ err = trx.Select(&blobs, "SELECT key FROM blobs WHERE tenant_id = $1", tenant.ID)
+ if err != nil {
+ return errors.Wrap(err, "failed list blobs")
+ }
+
+ files := make([]string, len(blobs))
+ for i, b := range blobs {
+ files[i] = b.Key
+ }
+
+ sort.Strings(files)
+ q.Result = files
+ return nil
+ })
+}
+
func getBlobByKey(ctx context.Context, q *query.GetBlobByKey) error {
return using(ctx, func(tenant *models.Tenant) error {
var tenantID sql.NullInt64
diff --git a/public/pages/Administration/pages/Export.page.tsx b/public/pages/Administration/pages/Export.page.tsx
index c34363e4f..7ec4754a6 100644
--- a/public/pages/Administration/pages/Export.page.tsx
+++ b/public/pages/Administration/pages/Export.page.tsx
@@ -1,6 +1,6 @@
import React from "react";
-import { Button, Form, Field } from "@fider/components";
+import { Button, Form, Field, Segment } from "@fider/components";
import { FaRegFileExcel } from "react-icons/fa";
import { AdminBasePage } from "../components/AdminBasePage";
@@ -14,17 +14,32 @@ export default class ExportPage extends AdminBasePage<{}, {}> {
public content() {
return (
<Form>
- <Field label="Export Posts">
- <p className="info">
- Use this button to download a CSV file with all posts in this site. This can be useful to analyse the data
- with an external tool or simply to back it up.
- </p>
- </Field>
- <Field>
- <Button color="positive" href="/admin/export/posts.csv">
- Download
- </Button>
- </Field>
+ <Segment>
+ <Field label="Export Posts">
+ <p className="info">
+ Use this button to download a CSV file with all posts in this site. This can be useful to analyse the data
+ with an external tool or simply to back it up.
+ </p>
+ </Field>
+ <Field>
+ <Button color="positive" href="/admin/export/posts.csv">
+ posts.csv
+ </Button>
+ </Field>
+ </Segment>
+ <Segment>
+ <Field label="Backup your data">
+ <p className="info">
+ Use this button to download a ZIP file with your data in JSON format. This is a full backup and contains
+ all of your data.
+ </p>
+ </Field>
+ <Field>
+ <Button color="positive" href="/admin/export/backup.zip">
+ backup.zip
+ </Button>
+ </Field>
+ </Segment>
</Form>
);
}