A continuation of HomeBox the inventory and organization system built for the Home User
fix import bug and add ref support (#88)
Haydendbaaf4ad0a5e
diff --git a/backend/app/api/demo.go b/backend/app/api/demo.goindex adacfd80..a98f03db 100644--- a/backend/app/api/demo.go+++ b/backend/app/api/demo.go@@ -52,7 +52,7 @@ func (a *app) SetupDemo() {log.Fatal().Msg("Failed to setup demo")}- err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)+ _, err = a.services.Items.CsvImport(context.Background(), self.GroupID, records)if err != nil {log.Err(err).Msg("Failed to import CSV")log.Fatal().Msg("Failed to setup demo")diff --git a/backend/app/api/v1/v1_ctrl_items.go b/backend/app/api/v1/v1_ctrl_items.goindex 686842d5..b515be94 100644--- a/backend/app/api/v1/v1_ctrl_items.go+++ b/backend/app/api/v1/v1_ctrl_items.go@@ -220,7 +220,7 @@ func (ctrl *V1Controller) HandleItemsImport() http.HandlerFunc {user := services.UseUserCtx(r.Context())- err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)+ _, err = ctrl.svc.Items.CsvImport(r.Context(), user.GroupID, data)if err != nil {log.Err(err).Msg("failed to import items")server.RespondServerError(w)diff --git a/backend/internal/repo/repo_items.go b/backend/internal/repo/repo_items.goindex 64715ca4..a0aa6f90 100644--- a/backend/internal/repo/repo_items.go+++ b/backend/internal/repo/repo_items.go@@ -211,6 +211,11 @@ func (e *ItemsRepository) GetOne(ctx context.Context, id uuid.UUID) (ItemOut, erreturn e.getOne(ctx, item.ID(id))}+func (e *ItemsRepository) CheckRef(ctx context.Context, GID uuid.UUID, ref string) (bool, error) {+ q := e.db.Item.Query().Where(item.HasGroupWith(group.ID(GID)))+ return q.Where(item.ImportRef(ref)).Exist(ctx)+}+// GetOneByGroup returns a single item by ID. If the item does not exist, an error is returned.// GetOneByGroup ensures that the item belongs to a specific group.func (e *ItemsRepository) GetOneByGroup(ctx context.Context, gid, id uuid.UUID) (ItemOut, error) {@@ -287,6 +292,7 @@ func (e *ItemsRepository) GetAll(ctx context.Context, gid uuid.UUID) ([]ItemSummfunc (e *ItemsRepository) Create(ctx context.Context, gid uuid.UUID, data ItemCreate) (ItemOut, error) {q := e.db.Item.Create().+ SetImportRef(data.ImportRef).SetName(data.Name).SetDescription(data.Description).SetGroupID(gid).diff --git a/backend/internal/services/service_items.go b/backend/internal/services/service_items.goindex e944e14f..7650a978 100644--- a/backend/internal/services/service_items.go+++ b/backend/internal/services/service_items.go@@ -3,7 +3,6 @@ package servicesimport ("context""errors"- "fmt""github.com/google/uuid""github.com/hay-kot/homebox/backend/internal/repo"@@ -48,7 +47,7 @@ func (svc *ItemService) Update(ctx context.Context, gid uuid.UUID, data repo.Itereturn svc.repo.Items.UpdateByGroup(ctx, gid, data)}-func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) error {+func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]string) (int, error) {loaded := []csvRow{}// Skip first row@@ -59,18 +58,41 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s}if len(row) != NumOfCols {- return ErrInvalidCsv+ return 0, ErrInvalidCsv}r := newCsvRow(row)loaded = append(loaded, r)}+ // validate rows+ var errMap = map[int][]error{}+ var hasErr bool+ for i, r := range loaded {++ errs := r.validate()++ if len(errs) > 0 {+ hasErr = true+ lineNum := i + 2++ errMap[lineNum] = errs+ }+ }++ if hasErr {+ for lineNum, errs := range errMap {+ for _, err := range errs {+ log.Error().Err(err).Int("line", lineNum).Msg("csv import error")+ }+ }+ }+// Bootstrap the locations and labels so we can reuse the created IDs for the itemslocations := map[string]uuid.UUID{}existingLocation, err := svc.repo.Locations.GetAll(ctx, gid)if err != nil {- return err+ return 0, err}for _, loc := range existingLocation {locations[loc.Name] = loc.ID@@ -79,7 +101,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]slabels := map[string]uuid.UUID{}existingLabels, err := svc.repo.Labels.GetAll(ctx, gid)if err != nil {- return err+ return 0, err}for _, label := range existingLabels {labels[label.Name] = label.ID@@ -88,25 +110,21 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]sfor _, row := range loaded {// Locations- if _, ok := locations[row.Location]; ok {- continue- }-- fmt.Println("Creating Location: ", row.Location)-- result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{- Name: row.Location,- Description: "",- })- if err != nil {- return err+ if _, exists := locations[row.Location]; !exists {+ result, err := svc.repo.Locations.Create(ctx, gid, repo.LocationCreate{+ Name: row.Location,+ Description: "",+ })+ if err != nil {+ return 0, err+ }+ locations[row.Location] = result.ID}- locations[row.Location] = result.ID// Labelsfor _, label := range row.getLabels() {- if _, ok := labels[label]; ok {+ if _, exists := labels[label]; exists {continue}result, err := svc.repo.Labels.Create(ctx, gid, repo.LabelCreate{@@ -114,14 +132,26 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]sDescription: "",})if err != nil {- return err+ return 0, err}labels[label] = result.ID}}// Create the items+ var count intfor _, row := range loaded {+ // Check Import Ref+ if row.Item.ImportRef != "" {+ exists, err := svc.repo.Items.CheckRef(ctx, gid, row.Item.ImportRef)+ if exists {+ continue+ }+ if err != nil {+ log.Err(err).Msg("error checking import ref")+ }+ }+locationID := locations[row.Location]labelIDs := []uuid.UUID{}for _, label := range row.getLabels() {@@ -131,8 +161,6 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]slog.Info().Str("name", row.Item.Name).Str("location", row.Location).- Strs("labels", row.getLabels()).- Str("locationId", locationID.String()).Msgf("Creating Item: %s", row.Item.Name)result, err := svc.repo.Items.Create(ctx, gid, repo.ItemCreate{@@ -144,7 +172,7 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s})if err != nil {- return err+ return count, err}// Update the item with the rest of the data@@ -183,8 +211,10 @@ func (svc *ItemService) CsvImport(ctx context.Context, gid uuid.UUID, data [][]s})if err != nil {- return err+ return count, err}++ count++}- return nil+ return count, nil}diff --git a/backend/internal/services/service_items_csv.go b/backend/internal/services/service_items_csv.goindex 4ad1aed1..c5593869 100644--- a/backend/internal/services/service_items_csv.go+++ b/backend/internal/services/service_items_csv.go@@ -97,3 +97,22 @@ func (c csvRow) getLabels() []string {return split}++func (c csvRow) validate() []error {+ var errs []error++ add := func(err error) {+ errs = append(errs, err)+ }++ required := func(s string, name string) {+ if s == "" {+ add(errors.New(name + " is required"))+ }+ }++ required(c.Location, "Location")+ required(c.Item.Name, "Name")++ return errs+}diff --git a/backend/internal/services/service_items_csv_test.go b/backend/internal/services/service_items_csv_test.goindex fed6e313..8c453e89 100644--- a/backend/internal/services/service_items_csv_test.go+++ b/backend/internal/services/service_items_csv_test.go@@ -9,12 +9,12 @@ import (const CSV_DATA = `Import Ref,Location,Labels,Quantity,Name,Description,Insured,Serial Number,Mode Number,Manufacturer,Notes,Purchase From,Purchased Price,Purchased Time,Lifetime Warranty,Warranty Expires,Warranty Details,Sold To,Sold Price,Sold Time,Sold Notes-,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,-,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,-,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,-,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,-,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,-,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`+A,Garage,IOT;Home Assistant; Z-Wave,1,Zooz Universal Relay ZEN17,"Zooz 700 Series Z-Wave Universal Relay ZEN17 for Awnings, Garage Doors, Sprinklers, and More | 2 NO-C-NC Relays (20A, 10A) | Signal Repeater | Hub Required (Compatible with SmartThings and Hubitat)",,,ZEN17,Zooz,,Amazon,39.95,10/13/2021,,,,,,,+B,Living Room,IOT;Home Assistant; Z-Wave,1,Zooz Motion Sensor,"Zooz Z-Wave Plus S2 Motion Sensor ZSE18 with Magnetic Mount, Works with Vera and SmartThings",,,ZSE18,Zooz,,Amazon,29.95,10/15/2021,,,,,,,+C,Office,IOT;Home Assistant; Z-Wave,1,Zooz 110v Power Switch,"Zooz Z-Wave Plus Power Switch ZEN15 for 110V AC Units, Sump Pumps, Humidifiers, and More",,,ZEN15,Zooz,,Amazon,39.95,10/13/2021,,,,,,,+D,Downstairs,IOT;Home Assistant; Z-Wave,1,Ecolink Z-Wave PIR Motion Sensor,"Ecolink Z-Wave PIR Motion Detector Pet Immune, White (PIRZWAVE2.5-ECO)",,,PIRZWAVE2.5-ECO,Ecolink,,Amazon,35.58,10/21/2020,,,,,,,+E,Entry,IOT;Home Assistant; Z-Wave,1,Yale Security Touchscreen Deadbolt,"Yale Security YRD226-ZW2-619 YRD226ZW2619 Touchscreen Deadbolt, Satin Nickel",,,YRD226ZW2619,Yale,,Amazon,120.39,10/14/2020,,,,,,,+F,Kitchen,IOT;Home Assistant; Z-Wave,1,Smart Rocker Light Dimmer,"UltraPro Z-Wave Smart Rocker Light Dimmer with QuickFit and SimpleWire, 3-Way Ready, Compatible with Alexa, Google Assistant, ZWave Hub Required, Repeater/Range Extender, White Paddle Only, 39351",,,39351,Honeywell,,Amazon,65.98,09/30/0202,,,,,,,`func loadcsv() [][]string {reader := csv.NewReader(bytes.NewBuffer([]byte(CSV_DATA)))diff --git a/backend/internal/services/service_items_test.go b/backend/internal/services/service_items_test.goindex 15fe3ca3..018dbc16 100644--- a/backend/internal/services/service_items_test.go+++ b/backend/internal/services/service_items_test.go@@ -13,7 +13,13 @@ func TestItemService_CsvImport(t *testing.T) {svc := &ItemService{repo: tRepos,}- err := svc.CsvImport(context.Background(), tGroup.ID, data)+ count, err := svc.CsvImport(context.Background(), tGroup.ID, data)+ assert.Equal(t, 6, count)+ assert.NoError(t, err)++ // Check import refs are deduplicated+ count, err = svc.CsvImport(context.Background(), tGroup.ID, data)+ assert.Equal(t, 0, count)assert.NoError(t, err)items, err := svc.GetAll(context.Background(), tGroup.ID)diff --git a/docs/docs/import-csv.md b/docs/docs/import-csv.mdindex 9c5e5871..ea32b8f2 100644--- a/docs/docs/import-csv.md+++ b/docs/docs/import-csv.md@@ -23,29 +23,29 @@ Import RefLocation Labels Quantity Name Description Insured Serial Number Model## CSV Reference-| Column | Type | Description |-| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------- |-| ImportRef | String (100) | Future |-| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. |-| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new |-| Quantity | Integer | The quantity of items to create |-| Name | String | Name of the item |-| Description | String | Description of the item |-| Insured | Boolean | Whether or not the item is insured |-| Serial Number | String | Serial number of the item |-| Model Number | String | Model of the item |-| Manufacturer | String | Manufacturer of the item |-| Notes | String (1000) | General notes about the product |-| Purchase From | String | Name of the place the item was purchased from |-| Purchase Price | Float64 | |-| Purchase At | Date | Date the item was purchased |-| Lifetime Warranty | Boolean | true or false - case insensitive |-| Warranty Expires | Date | Date in the format |-| Warranty Details | String | Details about the warranty |-| Sold To | String | Name of the person the item was sold to |-| Sold At | Date | Date the item was sold |-| Sold Price | Float64 | |-| Sold Notes | String (1000) | |+| Column | Type | Description |+| ----------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| ImportRef | String (100) | Import Refs are unique strings that can be used to deduplicate imports. Before an item is imported, we check the database for a matching ref. If the ref exists, we skip that item. |+| Location | String | This is the location of the item that will be created. These are de-duplicated and won't create another instance when reused. |+| Labels | `;` Separated String | List of labels to apply to the item separated by a `;`, can be existing or new |+| Quantity | Integer | The quantity of items to create |+| Name | String | Name of the item |+| Description | String | Description of the item |+| Insured | Boolean | Whether or not the item is insured |+| Serial Number | String | Serial number of the item |+| Model Number | String | Model of the item |+| Manufacturer | String | Manufacturer of the item |+| Notes | String (1000) | General notes about the product |+| Purchase From | String | Name of the place the item was purchased from |+| Purchase Price | Float64 | |+| Purchase At | Date | Date the item was purchased |+| Lifetime Warranty | Boolean | true or false - case insensitive |+| Warranty Expires | Date | Date in the format |+| Warranty Details | String | Details about the warranty |+| Sold To | String | Name of the person the item was sold to |+| Sold At | Date | Date the item was sold |+| Sold Price | Float64 | |+| Sold Notes | String (1000) | |**Type Key**