Open platform to collect and prioritize feedback
Refactor pkg/dbx for better performance (#236)
Guilherme Oenningc0785a94feba
diff --git a/app/pkg/dbx/dbx.go b/app/pkg/dbx/dbx.goindex 51c86b3fb..8f1465b3d 100644--- a/app/pkg/dbx/dbx.go+++ b/app/pkg/dbx/dbx.go@@ -2,12 +2,9 @@ package dbximport ("database/sql"- "fmt""io/ioutil""reflect"- "github.com/lib/pq"-"strings""github.com/getfider/fider/app"@@ -35,7 +32,7 @@ func NewWithLogger(logger log.Logger) (*Database, error) {return nil, err}- db := &Database{conn, logger}+ db := &Database{conn, logger, NewRowMapper()}return db, nil}@@ -43,12 +40,13 @@ func NewWithLogger(logger log.Logger) (*Database, error) {type Database struct {conn *sql.DBlogger log.Logger+ mapper RowMapper}// Begin returns a new SQL transactionfunc (db Database) Begin() (*Trx, error) {tx, err := db.conn.Begin()- return &Trx{tx: tx, logger: db.logger}, err+ return &Trx{tx: tx, logger: db.logger, mapper: db.mapper}, err}// Close connection to database@@ -59,56 +57,6 @@ func (db Database) Close() error {return nil}-func scan(prefix string, data interface{}) map[string]interface{} {- fields := make(map[string]interface{})-- s := reflect.ValueOf(data).Elem()-- for i := 0; i < s.NumField(); i++ {- field := s.Field(i)- typeField := s.Type().Field(i)- tag := typeField.Tag.Get("db")-- if tag != "" {- if typeField.Type.Kind() == reflect.Slice {- obj := reflect.New(reflect.MakeSlice(typeField.Type, 0, 0).Type()).Elem()- field.Set(obj)- fields[prefix+tag] = pq.Array(field.Addr().Interface())- } else if typeField.Type.Kind() != reflect.Ptr {- fields[prefix+tag] = field.Addr().Interface()- } else if field.Type().Elem().Kind() != reflect.Struct || field.Type().Elem().String() == "time.Time" {- obj := reflect.New(field.Type().Elem()).Elem()- field.Set(obj.Addr())- fields[prefix+tag] = field.Interface()- } else {- obj := reflect.New(field.Type().Elem()).Elem()- field.Set(obj.Addr())- for name, address := range scan(prefix+tag+"_", field.Interface()) {- fields[name] = address- }- }- }- }-- return fields-}--func fill(rows *sql.Rows, data interface{}) error {- columns, _ := rows.Columns()- fields := scan("", data)-- pointers := make([]interface{}, len(columns))- for i, column := range columns {- if pointer, ok := fields[column]; ok {- pointers[i] = pointer- } else {- panic(fmt.Sprintf("No target for column %s", column))- }- }-- return rows.Scan(pointers...)-}-func (db Database) load(path string) {content, err := ioutil.ReadFile(env.Path(path))if err != nil {@@ -154,6 +102,7 @@ func (db Database) Migrate() {type Trx struct {tx *sql.Txlogger log.Logger+ mapper RowMapper}// QueryRow the database with given SQL command and returns only one row@@ -203,7 +152,8 @@ func (trx Trx) Get(data interface{}, command string, args ...interface{}) errordefer rows.Close()if rows.Next() {- return fill(rows, data)+ columns, _ := rows.Columns()+ return trx.mapper.Map(data, columns, rows.Scan)}return app.ErrNotFound@@ -222,8 +172,7 @@ func (trx Trx) QueryIntArray(command string, args ...interface{}) ([]int, error)if rows != nil {defer rows.Close()for rows.Next() {- err := rows.Scan(&value)- if err != nil {+ if err := rows.Scan(&value); err != nil {return nil, err}values = append(values, value)@@ -265,10 +214,13 @@ func (trx Trx) Select(data interface{}, command string, args ...interface{}) errsliceType := reflect.TypeOf(data).Elem()items := reflect.New(sliceType).Elem()itemType := sliceType.Elem().Elem()+ var columns []stringfor rows.Next() {+ if columns == nil {+ columns, _ = rows.Columns()+ }item := reflect.New(itemType)- err = fill(rows, item.Interface())- if err != nil {+ if err = trx.mapper.Map(item.Interface(), columns, rows.Scan); err != nil {return err}items = reflect.Append(items, item)diff --git a/app/pkg/dbx/dbx_test.go b/app/pkg/dbx/dbx_test.goindex 977aa37c8..07b119b02 100644--- a/app/pkg/dbx/dbx_test.go+++ b/app/pkg/dbx/dbx_test.go@@ -26,8 +26,8 @@ type userProvider struct {}type tenant struct {- ID int `db:"id"`- Name *string `db:"name"`+ ID int `db:"id"`+ Name string `db:"name"`}func TestMain(m *testing.M) {@@ -133,7 +133,7 @@ func TestBind_NestedStruct(t *testing.T) {Expect(u.Name).To(Equal("Jon Snow"))Expect(u.Tenant).NotTo(BeNil())Expect(u.Tenant.ID).To(Equal(1))- Expect(*u.Tenant.Name).To(Equal("Demonstration"))+ Expect(u.Tenant.Name).To(Equal("Demonstration"))}func TestBind_NestedStruct_Multiple(t *testing.T) {@@ -156,10 +156,9 @@ func TestBind_NestedStruct_Multiple(t *testing.T) {Expect(err).To(BeNil())Expect(len(u)).To(Equal(2))Expect(u[0].Name).To(Equal("Jon Snow"))- Expect(*u[0].Tenant.Name).To(Equal("Demonstration"))+ Expect(u[0].Tenant.Name).To(Equal("Demonstration"))Expect(u[1].Name).To(Equal("Arya Stark"))- Expect(*u[1].Tenant.Name).To(Equal("Demonstration"))-+ Expect(u[1].Tenant.Name).To(Equal("Demonstration"))}func TestExists_True(t *testing.T) {diff --git a/app/pkg/dbx/mapping.go b/app/pkg/dbx/mapping.gonew file mode 100644index 000000000..29a4a4272--- /dev/null+++ b/app/pkg/dbx/mapping.go@@ -0,0 +1,114 @@+package dbx++import (+ "fmt"+ "reflect"++ "github.com/lib/pq"+)++//RowMapper is resposible for mapping a sql.Rows into a Struct (model)+type RowMapper struct {+ cache map[reflect.Type]TypeMapper+}++//NewRowMapper creates a new instance of RowMapper+func NewRowMapper() RowMapper {+ return RowMapper{+ cache: make(map[reflect.Type]TypeMapper, 0),+ }+}++//Map values from scanner (usually sql.Rows.Scan) into dest based on columns+func (m *RowMapper) Map(dest interface{}, columns []string, scanner func(dest ...interface{}) error) error {+ t := reflect.TypeOf(dest)+ if t.Kind() == reflect.Ptr {+ t = t.Elem()+ }++ var (+ typeMapper TypeMapper+ ok bool+ )++ if typeMapper, ok = m.cache[t]; !ok {+ typeMapper = NewTypeMapper(t)+ m.cache[t] = typeMapper+ }++ pointers := make([]interface{}, len(columns))+ for i, c := range columns {+ mapping := typeMapper.Fields[c]+ field := reflect.ValueOf(dest).Elem()++ for _, f := range mapping.FieldName {+ if field.Kind() == reflect.Ptr {+ if field.IsNil() {+ field.Set(reflect.New(field.Type().Elem()))+ }+ field = field.Elem()+ }+ field = field.FieldByName(f)+ }++ if !field.CanAddr() {+ panic(fmt.Sprintf("Field not found for column %s", c))+ }++ if field.Kind() == reflect.Slice {+ obj := reflect.New(reflect.MakeSlice(field.Type(), 0, 0).Type()).Elem()+ field.Set(obj)+ pointers[i] = pq.Array(field.Addr().Interface())+ } else {+ pointers[i] = field.Addr().Interface()+ }+ }++ return scanner(pointers...)+}++//TypeMapper holds information about how to map SQL ResultSet to a Struct+type TypeMapper struct {+ Type reflect.Type+ Fields map[string]FieldInfo+}++//NewTypeMapper creates a new instance of TypeMapper for given reflect.Type+func NewTypeMapper(t reflect.Type) TypeMapper {+ all := make(map[string]FieldInfo, 0)+ for i := 0; i < t.NumField(); i++ {+ field := t.Field(i)+ columnName := field.Tag.Get("db")+ if columnName != "" {+ fieldType := field.Type+ fieldKind := fieldType.Kind()++ if fieldKind == reflect.Ptr {+ fieldType = field.Type.Elem()+ fieldKind = fieldType.Kind()+ mapper := NewTypeMapper(fieldType)+ for _, f := range mapper.Fields {+ all[columnName+"_"+f.ColumnName] = FieldInfo{+ ColumnName: columnName + "_" + f.ColumnName,+ FieldName: append([]string{field.Name}, f.FieldName...),+ }+ }+ } else {+ all[columnName] = FieldInfo{+ FieldName: []string{field.Name},+ ColumnName: columnName,+ }+ }+ }+ }+ return TypeMapper{+ Type: t,+ Fields: all,+ }+}++//FieldInfo is a simple struct to map Column -> Field+type FieldInfo struct {+ FieldName []string+ ColumnName string+}diff --git a/app/pkg/dbx/mapping_test.go b/app/pkg/dbx/mapping_test.gonew file mode 100644index 000000000..558680ed5--- /dev/null+++ b/app/pkg/dbx/mapping_test.go@@ -0,0 +1,91 @@+package dbx_test++import (+ "reflect"+ "testing"++ "github.com/getfider/fider/app/pkg/dbx"+ . "github.com/onsi/gomega"+)++func createScanner(values ...interface{}) func(dest ...interface{}) error {+ return func(dest ...interface{}) error {+ for i := 0; i < len(values); i++ {+ reflect.Indirect(reflect.ValueOf(dest[i])).Set(reflect.ValueOf(values[i]))+ }+ return nil+ }+}++func TestTypeMapper_SimpleStruct(t *testing.T) {+ RegisterTestingT(t)+ u := user{}+ mapping := dbx.NewTypeMapper(reflect.TypeOf(u))+ Expect(len(mapping.Fields)).To(Equal(2))+ Expect(mapping.Fields["id"].FieldName).To(Equal([]string{"ID"}))+ Expect(mapping.Fields["name"].FieldName).To(Equal([]string{"Name"}))+}++func TestTypeMapper_StringPointer(t *testing.T) {+ RegisterTestingT(t)+ m := tenant{}+ mapping := dbx.NewTypeMapper(reflect.TypeOf(m))+ Expect(len(mapping.Fields)).To(Equal(2))+ Expect(mapping.Fields["id"].FieldName).To(Equal([]string{"ID"}))+ Expect(mapping.Fields["name"].FieldName).To(Equal([]string{"Name"}))+}++func TestTypeMapper_NestedStruct(t *testing.T) {+ RegisterTestingT(t)+ u := userWithTenant{}+ mapping := dbx.NewTypeMapper(reflect.TypeOf(u))+ Expect(len(mapping.Fields)).To(Equal(4))+ Expect(mapping.Fields["id"].FieldName).To(Equal([]string{"ID"}))+ Expect(mapping.Fields["name"].FieldName).To(Equal([]string{"Name"}))+ Expect(mapping.Fields["tenant_id"].FieldName).To(Equal([]string{"Tenant", "ID"}))+ Expect(mapping.Fields["tenant_name"].FieldName).To(Equal([]string{"Tenant", "Name"}))+}++func TestTypeMapper_DeepNestedStruct(t *testing.T) {+ RegisterTestingT(t)+ u := userProvider{}+ mapping := dbx.NewTypeMapper(reflect.TypeOf(u))+ Expect(len(mapping.Fields)).To(Equal(5))+ Expect(mapping.Fields["provider"].FieldName).To(Equal([]string{"Provider"}))+ Expect(mapping.Fields["user_id"].FieldName).To(Equal([]string{"User", "ID"}))+ Expect(mapping.Fields["user_name"].FieldName).To(Equal([]string{"User", "Name"}))+ Expect(mapping.Fields["user_tenant_id"].FieldName).To(Equal([]string{"User", "Tenant", "ID"}))+ Expect(mapping.Fields["user_tenant_name"].FieldName).To(Equal([]string{"User", "Tenant", "Name"}))+}++func TestRowMapper_SimpleStruct(t *testing.T) {+ RegisterTestingT(t)+ mapper := dbx.NewRowMapper()+ u := user{}+ mapper.Map(&u, []string{"id", "name"}, createScanner(1, "Hello World"))+ Expect(u.ID).To(Equal(1))+ Expect(u.Name).To(Equal("Hello World"))+}++func TestRowMapper_NestedStruct(t *testing.T) {+ RegisterTestingT(t)+ mapper := dbx.NewRowMapper()+ u := userWithTenant{}+ mapper.Map(&u, []string{"id", "name", "tenant_id", "tenant_name"}, createScanner(1, "Hello World", 2, "Demonstration"))+ Expect(u.ID).To(Equal(1))+ Expect(u.Name).To(Equal("Hello World"))+ Expect(u.Tenant.ID).To(Equal(2))+ Expect(u.Tenant.Name).To(Equal("Demonstration"))+}++func TestRowMapper_DeepNestedStruct(t *testing.T) {+ RegisterTestingT(t)+ mapper := dbx.NewRowMapper()+ u := userProvider{}+ mapper.Map(&u, []string{"provider", "user_id", "user_name", "user_tenant_id", "user_tenant_name"}, createScanner("google", 1, "Jon Snow", 2, "The Tenant"))+ Expect(u.Provider).To(Equal("google"))+ Expect(u.User.ID).To(Equal(1))+ Expect(u.User.Name).To(Equal("Jon Snow"))+ Expect(u.User.Tenant.ID).To(Equal(2))+ Expect(u.User.Tenant.Name).To(Equal("The Tenant"))+}diff --git a/app/pkg/dbx/scan_test.go b/app/pkg/dbx/scan_test.godeleted file mode 100644index 34f2db5ba..000000000--- a/app/pkg/dbx/scan_test.go+++ /dev/null@@ -1,67 +0,0 @@-package dbx--import (- "testing"- "time"-- . "github.com/onsi/gomega"-)--type tenantWithStringPointer struct {- ID int `db:"id"`- Name *string `db:"name"`- Time *time.Time `db:"time"`-}--type tenant struct {- ID int `db:"id"`- Name string `db:"name"`-}--type user struct {- ID int `db:"id"`- Name string `db:"name"`- Tenant *tenant `db:"tenant"`- IgnoreThis bool-}--type userProvider struct {- Name string `db:"provider"`- User *user `db:"user"`-}--func TestScan(t *testing.T) {- RegisterTestingT(t)-- u := user{}- result := scan("", &u)- Expect(len(result)).To(Equal(4))- Expect(result["id"]).To(Equal(&u.ID))- Expect(result["name"]).To(Equal(&u.Name))- Expect(result["tenant_id"]).To(Equal(&u.Tenant.ID))- Expect(result["tenant_name"]).To(Equal(&u.Tenant.Name))-}--func TestScan_WithStringPointer(t *testing.T) {- RegisterTestingT(t)-- u := tenantWithStringPointer{}- result := scan("", &u)- Expect(len(result)).To(Equal(3))- Expect(result["id"]).To(Equal(&u.ID))- Expect(result["name"]).To(Equal(u.Name))- Expect(result["time"]).To(Equal(u.Time))-}--func TestDeepScan(t *testing.T) {- RegisterTestingT(t)-- u := userProvider{}- result := scan("", &u)- Expect(len(result)).To(Equal(5))- Expect(result["provider"]).To(Equal(&u.Name))- Expect(result["user_id"]).To(Equal(&u.User.ID))- Expect(result["user_name"]).To(Equal(&u.User.Name))- Expect(result["user_tenant_id"]).To(Equal(&u.User.Tenant.ID))- Expect(result["user_tenant_name"]).To(Equal(&u.User.Tenant.Name))-}diff --git a/app/storage/postgres/tenant.go b/app/storage/postgres/tenant.goindex 27a3ba1ac..447756e7c 100644--- a/app/storage/postgres/tenant.go+++ b/app/storage/postgres/tenant.go@@ -20,6 +20,10 @@ type dbTenant struct {}func (t *dbTenant) toModel() *models.Tenant {+ if t == nil {+ return nil+ }+return &models.Tenant{ID: t.ID,Name: t.Name,diff --git a/public/pages/admin/AdminHomePage.tsx b/public/pages/admin/AdminHomePage.tsxindex 8f4698386..be4f4d50f 100644--- a/public/pages/admin/AdminHomePage.tsx+++ b/public/pages/admin/AdminHomePage.tsx@@ -1,7 +1,7 @@import * as React from 'react';import { AppSettings, CurrentUser, Tenant } from '@fider/models';-import { Button, Textarea, DisplayError } from '@fider/components/common';+import { Button, ButtonClickEvent, Textarea, DisplayError } from '@fider/components/common';import { actions, page, Failure } from '@fider/services';interface AdminHomePageProps {@@ -33,7 +33,7 @@ export class AdminHomePage extends React.Component<AdminHomePageProps, AdminHomepage.setTitle(`Administration · ${document.title}`);}- private async confirm() {+ private async confirm(e: ButtonClickEvent) {const result = await actions.updateTenantSettings(this.state.title,this.state.invitation,@@ -41,6 +41,7 @@ export class AdminHomePage extends React.Component<AdminHomePageProps, AdminHomethis.state.cname,);if (result.ok) {+ e.preventEnable();location.href = `/`;} else if (result.error) {this.setState({ error: result.error });@@ -136,7 +137,7 @@ export class AdminHomePage extends React.Component<AdminHomePageProps, AdminHome{this.props.user.isAdministrator &&<div className="field">- <Button className="positive" size="tiny" onClick={async () => await this.confirm()}>Confirm</Button>+ <Button className="positive" size="tiny" onClick={async (e) => await this.confirm(e)}>Confirm</Button></div>}</div>