Skip to content

A continuation of HomeBox the inventory and organization system built for the Home User

fix: backup memory usage (#1582)

Lukasz Piepiora0e97012391ed
diff --git a/backend/app/api/handlers/v1/controller.go b/backend/app/api/handlers/v1/controller.go
index bd7a6762..5e9ce30f 100644
--- a/backend/app/api/handlers/v1/controller.go
+++ b/backend/app/api/handlers/v1/controller.go
@@ -47,6 +47,12 @@ func WithMaxImportSize(maxImportSize int64) func(*V1Controller) {
}
}
+func WithMaxParseMemory(maxParseMemory int64) func(*V1Controller) {
+ return func(ctrl *V1Controller) {
+ ctrl.maxParseMemory = maxParseMemory
+ }
+}
+
func WithDemoStatus(demoStatus bool) func(*V1Controller) {
return func(ctrl *V1Controller) {
ctrl.isDemo = demoStatus
@@ -77,6 +83,7 @@ type V1Controller struct {
svc *services.AllServices
maxUploadSize int64
maxImportSize int64
+ maxParseMemory int64
isDemo bool
allowRegistration bool
bus *eventbus.EventBus
diff --git a/backend/app/api/handlers/v1/v1_ctrl_exports.go b/backend/app/api/handlers/v1/v1_ctrl_exports.go
index fb8912c7..bf62bd59 100644
--- a/backend/app/api/handlers/v1/v1_ctrl_exports.go
+++ b/backend/app/api/handlers/v1/v1_ctrl_exports.go
@@ -228,9 +228,9 @@ func (ctrl *V1Controller) HandleCollectionImport() errchain.HandlerFunc {
}
// maxImportSize is in MB and applies to the whole request body via the
- // path-aware middleware; here we pass it to ParseMultipartForm as the
- // memory-vs-disk threshold so larger archives spool gracefully.
- if err := r.ParseMultipartForm(ctrl.maxImportSize << 20); err != nil {
+ // path-aware middleware; here we pass `maxParseMemory` to ParseMultipartForm
+ // as the memory-vs-disk threshold so larger archives spool gracefully.
+ if err := r.ParseMultipartForm(ctrl.maxParseMemory << 20); err != nil {
log.Err(err).Msg("import: parse multipart")
return validate.NewRequestError(err, http.StatusBadRequest)
}
diff --git a/backend/app/api/routes.go b/backend/app/api/routes.go
index 21854675..d2d11ed2 100644
--- a/backend/app/api/routes.go
+++ b/backend/app/api/routes.go
@@ -69,6 +69,7 @@ func (a *app) mountRoutes(r *chi.Mux, chain *errchain.ErrChain, repos *repo.AllR
a.conf,
v1.WithMaxUploadSize(a.conf.Web.MaxUploadSize),
v1.WithMaxImportSize(a.conf.Web.MaxImportSize),
+ v1.WithMaxParseMemory(a.conf.Web.MaxParseMemory),
v1.WithRegistration(a.conf.Options.AllowRegistration),
v1.WithDemoStatus(a.conf.Demo), // Disable Password Change in Demo Mode
v1.WithURL(fmt.Sprintf("%s:%s", a.conf.Web.Host, a.conf.Web.Port)),
diff --git a/backend/internal/sys/config/conf.go b/backend/internal/sys/config/conf.go
index e947ccb4..c9e22daf 100644
--- a/backend/internal/sys/config/conf.go
+++ b/backend/internal/sys/config/conf.go
@@ -96,10 +96,14 @@ type WebConfig struct {
// (POST /v1/group/import). Set independently because a full collection
// backup including attachments can be much larger than a single asset
// upload. Defaults to 1 GB.
- MaxImportSize int64 `yaml:"max_import_upload" conf:"default:1024"`
- ReadTimeout time.Duration `yaml:"read_timeout" conf:"default:10s"`
- WriteTimeout time.Duration `yaml:"write_timeout" conf:"default:10s"`
- IdleTimeout time.Duration `yaml:"idle_timeout" conf:"default:30s"`
+ MaxImportSize int64 `yaml:"max_import_size" conf:"default:1024"`
+ // MaxParseMemory is the amount of memory used when parsing multipart form
+ // the data that does not fit into this memory will spil to temp files.
+ // Defaults to 64 MB.
+ MaxParseMemory int64 `yaml:"max_parse_memory" conf:"default:64"`
+ ReadTimeout time.Duration `yaml:"read_timeout" conf:"default:10s"`
+ WriteTimeout time.Duration `yaml:"write_timeout" conf:"default:10s"`
+ IdleTimeout time.Duration `yaml:"idle_timeout" conf:"default:30s"`
}
type LabelMakerConf struct {