Skip to content

A continuation of HomeBox the inventory and organization system built for the Home User

fix: #1653, #1590, #1641 - bug fixes

Matthew Kilgore575dd64ffc97
diff --git a/backend/app/api/handlers/v1/helpers.go b/backend/app/api/handlers/v1/helpers.go
index 68302524..24dbc91d 100644
--- a/backend/app/api/handlers/v1/helpers.go
+++ b/backend/app/api/handlers/v1/helpers.go
@@ -4,11 +4,35 @@ import (
"net/http"
"net/url"
"strings"
+ "time"
"github.com/rs/zerolog/log"
"github.com/sysadminsmedia/homebox/backend/internal/sys/config"
)
+// allowSlowResponse clears the server's write deadline for the current
+// response.
+//
+// http.Server.WriteTimeout is an absolute deadline armed when the request is
+// read, not an idle timeout — it does not extend as bytes are written. Any
+// response that takes longer than Web.WriteTimeout (10s by default) is
+// therefore severed mid-body, at whatever offset throughput happened to reach.
+// For a streamed backup or attachment that means a silently truncated file
+// rather than an error the user can act on.
+//
+// Handlers that stream a blob of user-controlled size must call this before
+// writing the body. Failure is non-fatal and only logged: if some middleware in
+// the chain wraps the ResponseWriter without an Unwrap method, the deadline
+// stays in place and large downloads keep failing, so the log line is the
+// breadcrumb back to this comment.
+func allowSlowResponse(w http.ResponseWriter, r *http.Request) {
+ if err := http.NewResponseController(w).SetWriteDeadline(time.Time{}); err != nil {
+ log.Warn().Err(err).
+ Str("path", r.URL.Path).
+ Msg("could not clear write deadline; large downloads may be truncated by web.write_timeout")
+ }
+}
+
// GetHBURL determines the base URL of the Homebox instance using the following priority:
// 1. Configured hostname from Options.Hostname
// 2. X-Forwarded headers (if TrustProxy is enabled)
diff --git a/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go b/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go
index bbbd17e7..1d82b099 100644
--- a/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go
+++ b/backend/app/api/handlers/v1/v1_ctrl_entities_attachments.go
@@ -280,6 +280,11 @@ func (ctrl *V1Controller) handleEntityAttachmentsHandler(w http.ResponseWriter,
}
}(bucket)
+ // Attachments are user-supplied and can be arbitrarily large (videos,
+ // scanned manuals). Without this the default 10s write timeout truncates
+ // them mid-transfer on slow links.
+ allowSlowResponse(w, r)
+
disposition := "attachment"
if isSafeInlineType(doc.MimeType) {
disposition = "inline"
diff --git a/backend/app/api/handlers/v1/v1_ctrl_exports.go b/backend/app/api/handlers/v1/v1_ctrl_exports.go
index 29cc756a..2d118dd5 100644
--- a/backend/app/api/handlers/v1/v1_ctrl_exports.go
+++ b/backend/app/api/handlers/v1/v1_ctrl_exports.go
@@ -130,6 +130,12 @@ func (ctrl *V1Controller) HandleExportDownload() errchain.HandlerFunc {
}
defer func() { _ = reader.Close() }()
+ // Backups routinely run to hundreds of megabytes and take far longer than
+ // the default 10s write timeout to stream. Content-Length is set from the
+ // recorded artifact size, so a deadline cut mid-copy leaves the browser
+ // with a short read it reports as a failed download.
+ allowSlowResponse(w, r)
+
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition",
fmt.Sprintf(`attachment; filename="homebox-export-%s.zip"`, out.ID.String()))
diff --git a/backend/internal/core/services/service_exports.go b/backend/internal/core/services/service_exports.go
index a5a91092..986b4dde 100644
--- a/backend/internal/core/services/service_exports.go
+++ b/backend/internal/core/services/service_exports.go
@@ -10,6 +10,7 @@ import (
"io"
"os"
"path"
+ "strconv"
"strings"
"sync"
"time"
@@ -1036,6 +1037,96 @@ type sqlExecer interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
}
+// sqlQuerier is the read-side counterpart of sqlExecer, used to introspect the
+// destination schema before replaying rows into it.
+type sqlQuerier interface {
+ QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
+}
+
+// boolColumns returns the columns of table that the destination database
+// stores as booleans.
+//
+// Exports are portable across engines and sqlite has no native boolean type:
+// it returns int64 0/1 for a BOOLEAN column, which marshals to JSON as 0/1 and
+// unmarshals back into a float64. Handing that number to a postgres bool
+// column fails outright ("unable to encode 0 into binary format for bool
+// (OID 16)"), so a sqlite-sourced backup could never be restored into
+// postgres. Normalizing on the import side rather than in the dump means the
+// export zips users already hold are repaired too.
+func boolColumns(ctx context.Context, db sqlQuerier, dialect, table string) (map[string]struct{}, error) {
+ if !isValidSQLIdent(table) {
+ return nil, fmt.Errorf("invalid table identifier %q", table)
+ }
+
+ query := `SELECT name FROM pragma_table_info(?) WHERE lower(type) IN ('bool', 'boolean')`
+ if dialect == "postgres" {
+ query = `SELECT column_name FROM information_schema.columns
+ WHERE table_schema = current_schema() AND table_name = $1 AND data_type = 'boolean'`
+ }
+
+ rows, err := db.QueryContext(ctx, query, table)
+ if err != nil {
+ return nil, fmt.Errorf("introspect bool columns on %q: %w", table, err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ cols := make(map[string]struct{})
+ for rows.Next() {
+ var name string
+ if err := rows.Scan(&name); err != nil {
+ return nil, fmt.Errorf("introspect bool columns on %q: %w", table, err)
+ }
+ cols[name] = struct{}{}
+ }
+ if err := rows.Err(); err != nil {
+ return nil, fmt.Errorf("introspect bool columns on %q: %w", table, err)
+ }
+ return cols, nil
+}
+
+// coerceBoolColumns rewrites the boolean-typed entries of row in place so the
+// driver is handed a real bool regardless of how the source engine encoded it.
+func coerceBoolColumns(row map[string]any, boolCols map[string]struct{}) error {
+ for col := range boolCols {
+ v, ok := row[col]
+ if !ok || v == nil {
+ continue
+ }
+ b, err := valueToBool(v)
+ if err != nil {
+ return fmt.Errorf("column %q: %w", col, err)
+ }
+ row[col] = b
+ }
+ return nil
+}
+
+// valueToBool converts a JSON-decoded export value into a bool. Numbers follow
+// the sqlite convention (0 false, anything else true); strings cover engines
+// and hand-edited exports that render booleans as "true"/"f"/"1".
+func valueToBool(v any) (bool, error) {
+ switch x := v.(type) {
+ case bool:
+ return x, nil
+ case float64:
+ return x != 0, nil
+ case json.Number:
+ f, err := x.Float64()
+ if err != nil {
+ return false, fmt.Errorf("cannot interpret %q as a boolean", x.String())
+ }
+ return f != 0, nil
+ case string:
+ b, err := strconv.ParseBool(strings.TrimSpace(x))
+ if err != nil {
+ return false, fmt.Errorf("cannot interpret %q as a boolean", x)
+ }
+ return b, nil
+ default:
+ return false, fmt.Errorf("cannot interpret %T as a boolean", v)
+ }
+}
+
// insertRow builds and runs an INSERT for one row's worth of column-value
// pairs. Self-maintaining: every JSON key becomes a column.
func insertRow(ctx context.Context, db sqlExecer, dialect, table string, row map[string]any) error {
@@ -1210,7 +1301,19 @@ func (s *ExportService) replayImportRows(ctx context.Context, tx *sql.Tx, zr *zi
if err != nil {
return nil, fmt.Errorf("read %s.json: %w", spec.name, err)
}
+ if len(rows) == 0 {
+ continue
+ }
+ // Introspected once per table, not per row: the shape is fixed for the
+ // whole file and an inventory can carry tens of thousands of entities.
+ boolCols, err := boolColumns(ctx, tx, s.dialect, spec.name)
+ if err != nil {
+ return nil, err
+ }
for _, row := range rows {
+ if err := coerceBoolColumns(row, boolCols); err != nil {
+ return nil, fmt.Errorf("insert %s: %w", spec.name, err)
+ }
newID, err := remapImportRow(row, spec, gid, userID, srcGroupID, remapFK, rememberID)
if err != nil {
return nil, err
diff --git a/backend/internal/core/services/service_exports_bools_test.go b/backend/internal/core/services/service_exports_bools_test.go
new file mode 100644
index 00000000..f6ed6638
--- /dev/null
+++ b/backend/internal/core/services/service_exports_bools_test.go
@@ -0,0 +1,118 @@
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// Exports are portable across engines, and sqlite reports BOOLEAN columns as
+// int64 0/1 — which lands in the export JSON as 0/1 and decodes back to a
+// float64. Postgres rejects that outright ("unable to encode 0 into binary
+// format for bool"), so without coercion a sqlite-sourced backup can never be
+// restored into postgres. See issue #1641.
+func TestValueToBool(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ in any
+ want bool
+ wantErr bool
+ }{
+ {name: "native false", in: false, want: false},
+ {name: "native true", in: true, want: true},
+ {name: "sqlite zero", in: float64(0), want: false},
+ {name: "sqlite one", in: float64(1), want: true},
+ {name: "json.Number zero", in: json.Number("0"), want: false},
+ {name: "json.Number one", in: json.Number("1"), want: true},
+ {name: "string true", in: "true", want: true},
+ {name: "string false", in: "false", want: false},
+ {name: "string one", in: "1", want: true},
+ {name: "string padded", in: " t ", want: true},
+ {name: "nonsense string", in: "banana", wantErr: true},
+ {name: "unsupported type", in: []string{"nope"}, wantErr: true},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ got, err := valueToBool(tt.in)
+ if tt.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
+
+func TestCoerceBoolColumns(t *testing.T) {
+ t.Parallel()
+
+ boolCols := map[string]struct{}{"archived": {}, "insured": {}, "missing": {}}
+ row := map[string]any{
+ "archived": float64(0),
+ "insured": float64(1),
+ "name": "Drill",
+ "quantity": float64(2),
+ "notes": nil,
+ }
+
+ require.NoError(t, coerceBoolColumns(row, boolCols))
+
+ assert.Equal(t, false, row["archived"], "0 should become a real false")
+ assert.Equal(t, true, row["insured"], "1 should become a real true")
+
+ // Non-boolean columns must be left exactly as they were: quantity is a
+ // float column and would be corrupted by a blanket numeric conversion.
+ quantity, ok := row["quantity"].(float64)
+ require.True(t, ok, "quantity must stay a float, not be coerced to a bool")
+ assert.InDelta(t, 2, quantity, 0)
+ assert.Equal(t, "Drill", row["name"])
+ assert.Nil(t, row["notes"])
+
+ // A bool column absent from the row stays absent rather than being
+ // inserted as a spurious false.
+ _, present := row["missing"]
+ assert.False(t, present)
+}
+
+func TestCoerceBoolColumnsRejectsGarbage(t *testing.T) {
+ t.Parallel()
+
+ err := coerceBoolColumns(
+ map[string]any{"archived": "not-a-bool"},
+ map[string]struct{}{"archived": {}},
+ )
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "archived")
+}
+
+// boolColumns drives a dialect-specific introspection query; this pins the
+// sqlite one (pragma_table_info with a bound table name) against a real
+// database so a driver change can't silently turn it into a no-op, which
+// would quietly restore the broken behaviour.
+func TestBoolColumnsSQLite(t *testing.T) {
+ ctx := context.Background()
+ db := tClient.Sql()
+
+ cols, err := boolColumns(ctx, db, "sqlite3", "entities")
+ require.NoError(t, err)
+
+ assert.Contains(t, cols, "archived")
+ assert.Contains(t, cols, "insured")
+ assert.NotContains(t, cols, "name")
+ assert.NotContains(t, cols, "quantity")
+}
+
+func TestBoolColumnsRejectsBadIdentifier(t *testing.T) {
+ t.Parallel()
+
+ _, err := boolColumns(context.Background(), tClient.Sql(), "sqlite3", "entities; DROP TABLE entities")
+ require.Error(t, err)
+}
diff --git a/backend/internal/data/migrations/postgres/20250112202302_sync_children.go b/backend/internal/data/migrations/postgres/20250112202302_sync_children.go
index 100b1008..076fcf79 100644
--- a/backend/internal/data/migrations/postgres/20250112202302_sync_children.go
+++ b/backend/internal/data/migrations/postgres/20250112202302_sync_children.go
@@ -14,10 +14,18 @@ func init() {
}
func Up20250112202302(ctx context.Context, tx *sql.Tx) error {
+ // The table_schema filter is load-bearing. information_schema.columns spans
+ // every schema in the database, so an unqualified table_name = 'items'
+ // lookup also matches an unrelated items table living in another schema.
+ // When that happened the guard concluded the column already existed, skipped
+ // the ALTER, and left the real table without it — which only surfaced later
+ // as a hard failure renaming the missing column in 20260416120000.
query := `
- SELECT column_name
+ SELECT column_name
FROM information_schema.columns
- WHERE table_name = 'items' AND column_name = 'sync_child_items_locations';
+ WHERE table_schema = current_schema()
+ AND table_name = 'items'
+ AND column_name = 'sync_child_items_locations';
`
err := tx.QueryRowContext(ctx, query).Scan(new("sync_child_items_locations"))
if err != nil {
diff --git a/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go b/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go
index 1c7f7801..21c67daf 100644
--- a/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go
+++ b/backend/internal/data/migrations/postgres/20260416120000_merge_entities.go
@@ -68,7 +68,32 @@ func Up20260402120000(ctx context.Context, tx *sql.Tx) error {
return fmt.Errorf("step 4b: rename item_children: %w", err)
}
- _, err = tx.ExecContext(ctx, `ALTER TABLE "entities" RENAME COLUMN "sync_child_items_locations" TO "sync_child_entity_locations";`)
+ // A plain rename here is not safe: databases migrated before the
+ // 20250112202302 guard was schema-qualified can arrive with that migration
+ // marked applied but the legacy column never actually added. Goose will not
+ // re-run an applied migration, so those databases would fail this rename on
+ // every start forever. Create the column outright in that case, and treat an
+ // already-renamed column as done.
+ _, err = tx.ExecContext(ctx, `
+ DO $$
+ BEGIN
+ IF EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_schema = current_schema()
+ AND table_name = 'entities'
+ AND column_name = 'sync_child_items_locations'
+ ) THEN
+ ALTER TABLE "entities" RENAME COLUMN "sync_child_items_locations" TO "sync_child_entity_locations";
+ ELSIF NOT EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_schema = current_schema()
+ AND table_name = 'entities'
+ AND column_name = 'sync_child_entity_locations'
+ ) THEN
+ ALTER TABLE "entities" ADD COLUMN "sync_child_entity_locations" boolean NOT NULL DEFAULT false;
+ END IF;
+ END $$;
+ `)
if err != nil {
return fmt.Errorf("step 4c: rename sync_child_items_locations: %w", err)
}
diff --git a/backend/internal/web/mid/logger.go b/backend/internal/web/mid/logger.go
index a3c608cd..24a6371a 100644
--- a/backend/internal/web/mid/logger.go
+++ b/backend/internal/web/mid/logger.go
@@ -20,6 +20,14 @@ func (s *spy) WriteHeader(status int) {
s.ResponseWriter.WriteHeader(status)
}
+// Unwrap exposes the wrapped writer to http.ResponseController, which walks
+// the Unwrap chain to reach the underlying connection. Without it a handler
+// cannot clear the server's write deadline, and long streaming downloads get
+// truncated mid-body by Web.WriteTimeout.
+func (s *spy) Unwrap() http.ResponseWriter {
+ return s.ResponseWriter
+}
+
func (s *spy) Hijack() (net.Conn, *bufio.ReadWriter, error) {
hj, ok := s.ResponseWriter.(http.Hijacker)
if !ok {
diff --git a/backend/internal/web/mid/logger_test.go b/backend/internal/web/mid/logger_test.go
new file mode 100644
index 00000000..fe608578
--- /dev/null
+++ b/backend/internal/web/mid/logger_test.go
@@ -0,0 +1,92 @@
+package mid
+
+import (
+ "io"
+ "net"
+ "net/http"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/go-chi/chi/v5/middleware"
+ "github.com/rs/zerolog"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// serveSlowBody starts a server whose WriteTimeout is deliberately shorter than
+// the time the handler takes to stream its body, and returns how many bytes the
+// client managed to read. clearDeadline mirrors what a streaming download
+// handler does before writing.
+func serveSlowBody(t *testing.T, clearDeadline bool) (int64, error) {
+ t.Helper()
+
+ const (
+ writeTimeout = 200 * time.Millisecond
+ chunks = 20
+ chunkSize = 32 * 1024
+ chunkDelay = 30 * time.Millisecond // 20 * 30ms = 600ms > writeTimeout
+ )
+
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if clearDeadline {
+ // The fix under test: reaching the underlying connection requires
+ // every wrapper in the chain to expose Unwrap.
+ assert.NoError(t, http.NewResponseController(w).SetWriteDeadline(time.Time{}))
+ }
+ w.Header().Set("Content-Length", strconv.Itoa(chunks*chunkSize))
+ w.WriteHeader(http.StatusOK)
+ chunk := make([]byte, chunkSize)
+ for i := 0; i < chunks; i++ {
+ if _, err := w.Write(chunk); err != nil {
+ return
+ }
+ _ = http.NewResponseController(w).Flush()
+ time.Sleep(chunkDelay)
+ }
+ })
+
+ srv := &http.Server{
+ WriteTimeout: writeTimeout,
+ // Mirror the production order: RequestID then Logger, which is the
+ // middleware that wraps the ResponseWriter.
+ Handler: middleware.RequestID(Logger(zerolog.Nop())(handler)),
+ }
+ go func() { _ = srv.Serve(ln) }()
+ t.Cleanup(func() { _ = srv.Close() })
+
+ resp, err := http.Get("http://" + ln.Addr().String() + "/download")
+ if err != nil {
+ return 0, err
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ return io.Copy(io.Discard, resp.Body)
+}
+
+// http.Server.WriteTimeout is an absolute deadline armed when the request is
+// read, not an idle timeout: it does not extend as bytes are written. Streaming
+// handlers therefore have to clear it, or long downloads (backups, large
+// attachments) are severed mid-body and the user gets a truncated file. This is
+// issue #1653.
+func TestLoggerAllowsHandlerToClearWriteDeadline(t *testing.T) {
+ const want = 20 * 32 * 1024
+
+ n, err := serveSlowBody(t, true)
+ require.NoError(t, err, "body should stream to completion once the deadline is cleared")
+ assert.Equal(t, int64(want), n, "whole body should arrive")
+}
+
+// Control: without clearing the deadline the same response is truncated. If
+// this ever stops failing, the test above has stopped proving anything.
+func TestWriteTimeoutTruncatesWithoutClearing(t *testing.T) {
+ const want = 20 * 32 * 1024
+
+ n, err := serveSlowBody(t, false)
+ if err == nil && n == int64(want) {
+ t.Fatal("expected the response to be truncated by WriteTimeout, but it completed")
+ }
+}