A continuation of HomeBox the inventory and organization system built for the Home User
fix: #1653, #1590, #1641 - bug fixes
Matthew Kilgore575dd64ffc97
diff --git a/backend/app/api/handlers/v1/helpers.go b/backend/app/api/handlers/v1/helpers.goindex 68302524..24dbc91d 100644--- a/backend/app/api/handlers/v1/helpers.go+++ b/backend/app/api/handlers/v1/helpers.go@@ -4,11 +4,35 @@ import ("net/http""net/url""strings"+ "time""github.com/rs/zerolog/log""github.com/sysadminsmedia/homebox/backend/internal/sys/config")+// allowSlowResponse clears the server's write deadline for the current+// response.+//+// http.Server.WriteTimeout is an absolute deadline armed when the request is+// read, not an idle timeout — it does not extend as bytes are written. Any+// response that takes longer than Web.WriteTimeout (10s by default) is+// therefore severed mid-body, at whatever offset throughput happened to reach.+// For a streamed backup or attachment that means a silently truncated file+// rather than an error the user can act on.+//+// Handlers that stream a blob of user-controlled size must call this before+// writing the body. Failure is non-fatal and only logged: if some middleware in+// the chain wraps the ResponseWriter without an Unwrap method, the deadline+// stays in place and large downloads keep failing, so the log line is the+// breadcrumb back to this comment.+func allowSlowResponse(w http.ResponseWriter, r *http.Request) {+ if err := http.NewResponseController(w).SetWriteDeadline(time.Time{}); err != nil {+ log.Warn().Err(err).+ Str("path", r.URL.Path).+ Msg("could not clear write deadline; large downloads may be truncated by web.write_timeout")+ }+}+// GetHBURL determines the base URL of the Homebox instance using the following priority:// 1. Configured hostname from Options.Hostname// 2. X-Forwarded headers (if TrustProxy is enabled)diff --git a/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go b/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.goindex bbbd17e7..1d82b099 100644--- a/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go+++ b/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go@@ -280,6 +280,11 @@ func (ctrl *V1Controller) handleEntityAttachmentsHandler(w http.ResponseWriter,}}(bucket)+ // Attachments are user-supplied and can be arbitrarily large (videos,+ // scanned manuals). Without this the default 10s write timeout truncates+ // them mid-transfer on slow links.+ allowSlowResponse(w, r)+disposition := "attachment"if isSafeInlineType(doc.MimeType) {disposition = "inline"diff --git a/backend/app/api/handlers/v1/v1_ctrl_exports.go b/backend/app/api/handlers/v1/v1_ctrl_exports.goindex 29cc756a..2d118dd5 100644--- a/backend/app/api/handlers/v1/v1_ctrl_exports.go+++ b/backend/app/api/handlers/v1/v1_ctrl_exports.go@@ -130,6 +130,12 @@ func (ctrl *V1Controller) HandleExportDownload() errchain.HandlerFunc {}defer func() { _ = reader.Close() }()+ // Backups routinely run to hundreds of megabytes and take far longer than+ // the default 10s write timeout to stream. Content-Length is set from the+ // recorded artifact size, so a deadline cut mid-copy leaves the browser+ // with a short read it reports as a failed download.+ allowSlowResponse(w, r)+w.Header().Set("Content-Type", "application/zip")w.Header().Set("Content-Disposition",fmt.Sprintf(`attachment; filename="homebox-export-%s.zip"`, out.ID.String()))diff --git a/backend/internal/core/services/service_exports.go b/backend/internal/core/services/service_exports.goindex a5a91092..986b4dde 100644--- a/backend/internal/core/services/service_exports.go+++ b/backend/internal/core/services/service_exports.go@@ -10,6 +10,7 @@ import ("io""os""path"+ "strconv""strings""sync""time"@@ -1036,6 +1037,96 @@ type sqlExecer interface {ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)}+// sqlQuerier is the read-side counterpart of sqlExecer, used to introspect the+// destination schema before replaying rows into it.+type sqlQuerier interface {+ QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)+}++// boolColumns returns the columns of table that the destination database+// stores as booleans.+//+// Exports are portable across engines and sqlite has no native boolean type:+// it returns int64 0/1 for a BOOLEAN column, which marshals to JSON as 0/1 and+// unmarshals back into a float64. Handing that number to a postgres bool+// column fails outright ("unable to encode 0 into binary format for bool+// (OID 16)"), so a sqlite-sourced backup could never be restored into+// postgres. Normalizing on the import side rather than in the dump means the+// export zips users already hold are repaired too.+func boolColumns(ctx context.Context, db sqlQuerier, dialect, table string) (map[string]struct{}, error) {+ if !isValidSQLIdent(table) {+ return nil, fmt.Errorf("invalid table identifier %q", table)+ }++ query := `SELECT name FROM pragma_table_info(?) WHERE lower(type) IN ('bool', 'boolean')`+ if dialect == "postgres" {+ query = `SELECT column_name FROM information_schema.columns+ WHERE table_schema = current_schema() AND table_name = $1 AND data_type = 'boolean'`+ }++ rows, err := db.QueryContext(ctx, query, table)+ if err != nil {+ return nil, fmt.Errorf("introspect bool columns on %q: %w", table, err)+ }+ defer func() { _ = rows.Close() }()++ cols := make(map[string]struct{})+ for rows.Next() {+ var name string+ if err := rows.Scan(&name); err != nil {+ return nil, fmt.Errorf("introspect bool columns on %q: %w", table, err)+ }+ cols[name] = struct{}{}+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("introspect bool columns on %q: %w", table, err)+ }+ return cols, nil+}++// coerceBoolColumns rewrites the boolean-typed entries of row in place so the+// driver is handed a real bool regardless of how the source engine encoded it.+func coerceBoolColumns(row map[string]any, boolCols map[string]struct{}) error {+ for col := range boolCols {+ v, ok := row[col]+ if !ok || v == nil {+ continue+ }+ b, err := valueToBool(v)+ if err != nil {+ return fmt.Errorf("column %q: %w", col, err)+ }+ row[col] = b+ }+ return nil+}++// valueToBool converts a JSON-decoded export value into a bool. Numbers follow+// the sqlite convention (0 false, anything else true); strings cover engines+// and hand-edited exports that render booleans as "true"/"f"/"1".+func valueToBool(v any) (bool, error) {+ switch x := v.(type) {+ case bool:+ return x, nil+ case float64:+ return x != 0, nil+ case json.Number:+ f, err := x.Float64()+ if err != nil {+ return false, fmt.Errorf("cannot interpret %q as a boolean", x.String())+ }+ return f != 0, nil+ case string:+ b, err := strconv.ParseBool(strings.TrimSpace(x))+ if err != nil {+ return false, fmt.Errorf("cannot interpret %q as a boolean", x)+ }+ return b, nil+ default:+ return false, fmt.Errorf("cannot interpret %T as a boolean", v)+ }+}+// insertRow builds and runs an INSERT for one row's worth of column-value// pairs. Self-maintaining: every JSON key becomes a column.func insertRow(ctx context.Context, db sqlExecer, dialect, table string, row map[string]any) error {@@ -1210,7 +1301,19 @@ func (s *ExportService) replayImportRows(ctx context.Context, tx *sql.Tx, zr *ziif err != nil {return nil, fmt.Errorf("read %s.json: %w", spec.name, err)}+ if len(rows) == 0 {+ continue+ }+ // Introspected once per table, not per row: the shape is fixed for the+ // whole file and an inventory can carry tens of thousands of entities.+ boolCols, err := boolColumns(ctx, tx, s.dialect, spec.name)+ if err != nil {+ return nil, err+ }for _, row := range rows {+ if err := coerceBoolColumns(row, boolCols); err != nil {+ return nil, fmt.Errorf("insert %s: %w", spec.name, err)+ }newID, err := remapImportRow(row, spec, gid, userID, srcGroupID, remapFK, rememberID)if err != nil {return nil, errdiff --git a/backend/internal/core/services/service_exports_bools_test.go b/backend/internal/core/services/service_exports_bools_test.gonew file mode 100644index 00000000..f6ed6638--- /dev/null+++ b/backend/internal/core/services/service_exports_bools_test.go@@ -0,0 +1,118 @@+package services++import (+ "context"+ "encoding/json"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// Exports are portable across engines, and sqlite reports BOOLEAN columns as+// int64 0/1 — which lands in the export JSON as 0/1 and decodes back to a+// float64. Postgres rejects that outright ("unable to encode 0 into binary+// format for bool"), so without coercion a sqlite-sourced backup can never be+// restored into postgres. See issue #1641.+func TestValueToBool(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ in any+ want bool+ wantErr bool+ }{+ {name: "native false", in: false, want: false},+ {name: "native true", in: true, want: true},+ {name: "sqlite zero", in: float64(0), want: false},+ {name: "sqlite one", in: float64(1), want: true},+ {name: "json.Number zero", in: json.Number("0"), want: false},+ {name: "json.Number one", in: json.Number("1"), want: true},+ {name: "string true", in: "true", want: true},+ {name: "string false", in: "false", want: false},+ {name: "string one", in: "1", want: true},+ {name: "string padded", in: " t ", want: true},+ {name: "nonsense string", in: "banana", wantErr: true},+ {name: "unsupported type", in: []string{"nope"}, wantErr: true},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()+ got, err := valueToBool(tt.in)+ if tt.wantErr {+ require.Error(t, err)+ return+ }+ require.NoError(t, err)+ assert.Equal(t, tt.want, got)+ })+ }+}++func TestCoerceBoolColumns(t *testing.T) {+ t.Parallel()++ boolCols := map[string]struct{}{"archived": {}, "insured": {}, "missing": {}}+ row := map[string]any{+ "archived": float64(0),+ "insured": float64(1),+ "name": "Drill",+ "quantity": float64(2),+ "notes": nil,+ }++ require.NoError(t, coerceBoolColumns(row, boolCols))++ assert.Equal(t, false, row["archived"], "0 should become a real false")+ assert.Equal(t, true, row["insured"], "1 should become a real true")++ // Non-boolean columns must be left exactly as they were: quantity is a+ // float column and would be corrupted by a blanket numeric conversion.+ quantity, ok := row["quantity"].(float64)+ require.True(t, ok, "quantity must stay a float, not be coerced to a bool")+ assert.InDelta(t, 2, quantity, 0)+ assert.Equal(t, "Drill", row["name"])+ assert.Nil(t, row["notes"])++ // A bool column absent from the row stays absent rather than being+ // inserted as a spurious false.+ _, present := row["missing"]+ assert.False(t, present)+}++func TestCoerceBoolColumnsRejectsGarbage(t *testing.T) {+ t.Parallel()++ err := coerceBoolColumns(+ map[string]any{"archived": "not-a-bool"},+ map[string]struct{}{"archived": {}},+ )+ require.Error(t, err)+ assert.Contains(t, err.Error(), "archived")+}++// boolColumns drives a dialect-specific introspection query; this pins the+// sqlite one (pragma_table_info with a bound table name) against a real+// database so a driver change can't silently turn it into a no-op, which+// would quietly restore the broken behaviour.+func TestBoolColumnsSQLite(t *testing.T) {+ ctx := context.Background()+ db := tClient.Sql()++ cols, err := boolColumns(ctx, db, "sqlite3", "entities")+ require.NoError(t, err)++ assert.Contains(t, cols, "archived")+ assert.Contains(t, cols, "insured")+ assert.NotContains(t, cols, "name")+ assert.NotContains(t, cols, "quantity")+}++func TestBoolColumnsRejectsBadIdentifier(t *testing.T) {+ t.Parallel()++ _, err := boolColumns(context.Background(), tClient.Sql(), "sqlite3", "entities; DROP TABLE entities")+ require.Error(t, err)+}diff --git a/backend/internal/data/migrations/postgres/20250112202302_sync_children.go b/backend/internal/data/migrations/postgres/20250112202302_sync_children.goindex 100b1008..076fcf79 100644--- a/backend/internal/data/migrations/postgres/20250112202302_sync_children.go+++ b/backend/internal/data/migrations/postgres/20250112202302_sync_children.go@@ -14,10 +14,18 @@ func init() {}func Up20250112202302(ctx context.Context, tx *sql.Tx) error {+ // The table_schema filter is load-bearing. information_schema.columns spans+ // every schema in the database, so an unqualified table_name = 'items'+ // lookup also matches an unrelated items table living in another schema.+ // When that happened the guard concluded the column already existed, skipped+ // the ALTER, and left the real table without it — which only surfaced later+ // as a hard failure renaming the missing column in 20260416120000.query := `- SELECT column_name+ SELECT column_nameFROM information_schema.columns- WHERE table_name = 'items' AND column_name = 'sync_child_items_locations';+ WHERE table_schema = current_schema()+ AND table_name = 'items'+ AND column_name = 'sync_child_items_locations';`err := tx.QueryRowContext(ctx, query).Scan(new("sync_child_items_locations"))if err != nil {diff --git a/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go b/backend/internal/data/migrations/postgres/20260416120000_merge_entities.goindex 1c7f7801..21c67daf 100644--- a/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go+++ b/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go@@ -68,7 +68,32 @@ func Up20260402120000(ctx context.Context, tx *sql.Tx) error {return fmt.Errorf("step 4b: rename item_children: %w", err)}- _, err = tx.ExecContext(ctx, `ALTER TABLE "entities" RENAME COLUMN "sync_child_items_locations" TO "sync_child_entity_locations";`)+ // A plain rename here is not safe: databases migrated before the+ // 20250112202302 guard was schema-qualified can arrive with that migration+ // marked applied but the legacy column never actually added. Goose will not+ // re-run an applied migration, so those databases would fail this rename on+ // every start forever. Create the column outright in that case, and treat an+ // already-renamed column as done.+ _, err = tx.ExecContext(ctx, `+ DO $$+ BEGIN+ IF EXISTS (+ SELECT 1 FROM information_schema.columns+ WHERE table_schema = current_schema()+ AND table_name = 'entities'+ AND column_name = 'sync_child_items_locations'+ ) THEN+ ALTER TABLE "entities" RENAME COLUMN "sync_child_items_locations" TO "sync_child_entity_locations";+ ELSIF NOT EXISTS (+ SELECT 1 FROM information_schema.columns+ WHERE table_schema = current_schema()+ AND table_name = 'entities'+ AND column_name = 'sync_child_entity_locations'+ ) THEN+ ALTER TABLE "entities" ADD COLUMN "sync_child_entity_locations" boolean NOT NULL DEFAULT false;+ END IF;+ END $$;+ `)if err != nil {return fmt.Errorf("step 4c: rename sync_child_items_locations: %w", err)}diff --git a/backend/internal/web/mid/logger.go b/backend/internal/web/mid/logger.goindex a3c608cd..24a6371a 100644--- a/backend/internal/web/mid/logger.go+++ b/backend/internal/web/mid/logger.go@@ -20,6 +20,14 @@ func (s *spy) WriteHeader(status int) {s.ResponseWriter.WriteHeader(status)}+// Unwrap exposes the wrapped writer to http.ResponseController, which walks+// the Unwrap chain to reach the underlying connection. Without it a handler+// cannot clear the server's write deadline, and long streaming downloads get+// truncated mid-body by Web.WriteTimeout.+func (s *spy) Unwrap() http.ResponseWriter {+ return s.ResponseWriter+}+func (s *spy) Hijack() (net.Conn, *bufio.ReadWriter, error) {hj, ok := s.ResponseWriter.(http.Hijacker)if !ok {diff --git a/backend/internal/web/mid/logger_test.go b/backend/internal/web/mid/logger_test.gonew file mode 100644index 00000000..fe608578--- /dev/null+++ b/backend/internal/web/mid/logger_test.go@@ -0,0 +1,92 @@+package mid++import (+ "io"+ "net"+ "net/http"+ "strconv"+ "testing"+ "time"++ "github.com/go-chi/chi/v5/middleware"+ "github.com/rs/zerolog"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// serveSlowBody starts a server whose WriteTimeout is deliberately shorter than+// the time the handler takes to stream its body, and returns how many bytes the+// client managed to read. clearDeadline mirrors what a streaming download+// handler does before writing.+func serveSlowBody(t *testing.T, clearDeadline bool) (int64, error) {+ t.Helper()++ const (+ writeTimeout = 200 * time.Millisecond+ chunks = 20+ chunkSize = 32 * 1024+ chunkDelay = 30 * time.Millisecond // 20 * 30ms = 600ms > writeTimeout+ )++ ln, err := net.Listen("tcp", "127.0.0.1:0")+ require.NoError(t, err)++ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if clearDeadline {+ // The fix under test: reaching the underlying connection requires+ // every wrapper in the chain to expose Unwrap.+ assert.NoError(t, http.NewResponseController(w).SetWriteDeadline(time.Time{}))+ }+ w.Header().Set("Content-Length", strconv.Itoa(chunks*chunkSize))+ w.WriteHeader(http.StatusOK)+ chunk := make([]byte, chunkSize)+ for i := 0; i < chunks; i++ {+ if _, err := w.Write(chunk); err != nil {+ return+ }+ _ = http.NewResponseController(w).Flush()+ time.Sleep(chunkDelay)+ }+ })++ srv := &http.Server{+ WriteTimeout: writeTimeout,+ // Mirror the production order: RequestID then Logger, which is the+ // middleware that wraps the ResponseWriter.+ Handler: middleware.RequestID(Logger(zerolog.Nop())(handler)),+ }+ go func() { _ = srv.Serve(ln) }()+ t.Cleanup(func() { _ = srv.Close() })++ resp, err := http.Get("http://" + ln.Addr().String() + "/download")+ if err != nil {+ return 0, err+ }+ defer func() { _ = resp.Body.Close() }()++ return io.Copy(io.Discard, resp.Body)+}++// http.Server.WriteTimeout is an absolute deadline armed when the request is+// read, not an idle timeout: it does not extend as bytes are written. Streaming+// handlers therefore have to clear it, or long downloads (backups, large+// attachments) are severed mid-body and the user gets a truncated file. This is+// issue #1653.+func TestLoggerAllowsHandlerToClearWriteDeadline(t *testing.T) {+ const want = 20 * 32 * 1024++ n, err := serveSlowBody(t, true)+ require.NoError(t, err, "body should stream to completion once the deadline is cleared")+ assert.Equal(t, int64(want), n, "whole body should arrive")+}++// Control: without clearing the deadline the same response is truncated. If+// this ever stops failing, the test above has stopped proving anything.+func TestWriteTimeoutTruncatesWithoutClearing(t *testing.T) {+ const want = 20 * 32 * 1024++ n, err := serveSlowBody(t, false)+ if err == nil && n == int64(want) {+ t.Fatal("expected the response to be truncated by WriteTimeout, but it completed")+ }+}