Skip to content

A continuation of HomeBox the inventory and organization system built for the Home User

Optimize RAM usage in attachment upload and thumbnail processing (#1243)

Copilot342497e85359
diff --git a/backend/internal/core/services/reporting/import.go b/backend/internal/core/services/reporting/import.go
index 6f01b1b1..9e5ca6e2 100644
--- a/backend/internal/core/services/reporting/import.go
+++ b/backend/internal/core/services/reporting/import.go
@@ -39,23 +39,32 @@ func determineSeparator(data []byte) (rune, error) {
}
}
+// separatorDetectionBufferSize is the buffer size for reading CSV headers
+// to detect the separator (comma vs tab)
+const separatorDetectionBufferSize = 4096
+
// readRawCsv reads a CSV file and returns the raw data as a 2D string array
// It determines the separator used in the CSV file and returns an error if
// it could not be determined
func readRawCsv(r io.Reader) ([][]string, error) {
- data, err := io.ReadAll(r)
- if err != nil {
+ // Buffer for reading the first line to detect separator
+ // We read up to 4KB which should be more than enough for any header row
+ firstLineBuffer := make([]byte, separatorDetectionBufferSize)
+ n, err := io.ReadFull(r, firstLineBuffer)
+ if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
return nil, err
}
+ firstLineBuffer = firstLineBuffer[:n]
- reader := csv.NewReader(bytes.NewReader(data))
-
- // Determine separator
- sep, err := determineSeparator(data)
+ // Determine separator from first line
+ sep, err := determineSeparator(firstLineBuffer)
if err != nil {
return nil, err
}
+ // Create a multi-reader combining what we already read and the rest
+ combinedReader := io.MultiReader(bytes.NewReader(firstLineBuffer), r)
+ reader := csv.NewReader(combinedReader)
reader.Comma = sep
return reader.ReadAll()
diff --git a/backend/internal/data/repo/repo_item_attachments.go b/backend/internal/data/repo/repo_item_attachments.go
index 0b4aa612..3ca5f72b 100644
--- a/backend/internal/data/repo/repo_item_attachments.go
+++ b/backend/internal/data/repo/repo_item_attachments.go
@@ -239,7 +239,7 @@ func (r *AttachmentRepo) Create(ctx context.Context, itemID uuid.UUID, doc ItemC
}
// Upload the file to the storage bucket
- path, err := r.UploadFile(ctx, itemGroup, doc)
+ uploadResult, err := r.UploadFile(ctx, itemGroup, doc)
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
return nil, rollbackErr
@@ -247,18 +247,8 @@ func (r *AttachmentRepo) Create(ctx context.Context, itemID uuid.UUID, doc ItemC
return nil, err
}
- limitedReader := io.LimitReader(doc.Content, 1024*128)
- file, err := io.ReadAll(limitedReader)
- if err != nil {
- log.Err(err).Msg("failed to read file content")
- err = tx.Rollback()
- if err != nil {
- return nil, err
- }
- return nil, err
- }
- bldr = bldr.SetMimeType(http.DetectContentType(file[:min(512, len(file))]))
- bldr = bldr.SetPath(path)
+ bldr = bldr.SetMimeType(uploadResult.ContentType)
+ bldr = bldr.SetPath(uploadResult.Path)
attachmentDb, err := bldr.Save(ctx)
if err != nil {
@@ -540,7 +530,10 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
log.Debug().Msg("reading original file content")
- contentBytes, err := io.ReadAll(origFile)
+ // Use LimitReader as a safety measure to prevent reading more than 100MB
+ // even if the stat size is incorrect or tampered
+ limitedReader := io.LimitReader(origFile, 100*1024*1024)
+ contentBytes, err := io.ReadAll(limitedReader)
if err != nil {
err := tx.Rollback()
if err != nil {
@@ -564,6 +557,17 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
}
+ // Pre-read orientation once for all image types that support it
+ // This avoids re-decoding the metadata for each image type
+ var orientation uint16 = 1 // Default orientation
+ if contentType != "image/avif" && contentType != "image/jxl" {
+ imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))
+ if err == nil {
+ orientation = uint16(imageMeta.Orientation)
+ }
+ // If error, just use default orientation (1)
+ }
+
switch {
case isImageFile(contentType):
log.Debug().Msg("creating thumbnail for image file")
@@ -577,18 +581,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- log.Debug().Msg("reading original file orientation")
- imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))
- if err != nil {
- log.Err(err).Msg("failed to decode original file content")
- err := tx.Rollback()
- if err != nil {
- return err
- }
- return err
- }
- orientation := uint16(imageMeta.Orientation)
- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)
+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)
if err != nil {
err := tx.Rollback()
if err != nil {
@@ -596,7 +589,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- att.SetPath(thumbnailPath)
+ att.SetPath(uploadResult.Path)
case contentType == "image/webp":
log.Debug().Msg("creating thumbnail for webp file")
img, err := webp.Decode(bytes.NewReader(contentBytes))
@@ -608,26 +601,15 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- log.Debug().Msg("reading original file orientation")
- imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))
+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)
if err != nil {
- log.Err(err).Msg("failed to decode original file content")
err := tx.Rollback()
if err != nil {
return err
}
return err
}
- orientation := uint16(imageMeta.Orientation)
- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)
- if err != nil {
- err := tx.Rollback()
- if err != nil {
- return err
- }
- return err
- }
- att.SetPath(thumbnailPath)
+ att.SetPath(uploadResult.Path)
case contentType == "image/avif":
log.Debug().Msg("creating thumbnail for avif file")
img, err := avif.Decode(bytes.NewReader(contentBytes))
@@ -639,7 +621,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))
+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))
if err != nil {
err := tx.Rollback()
if err != nil {
@@ -647,7 +629,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- att.SetPath(thumbnailPath)
+ att.SetPath(uploadResult.Path)
case contentType == "image/heic" || contentType == "image/heif":
log.Debug().Msg("creating thumbnail for heic file")
img, err := heic.Decode(bytes.NewReader(contentBytes))
@@ -659,18 +641,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- log.Debug().Msg("reading original file orientation")
- imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))
- if err != nil {
- log.Err(err).Msg("failed to decode original file content")
- err := tx.Rollback()
- if err != nil {
- return err
- }
- return err
- }
- orientation := uint16(imageMeta.Orientation)
- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)
+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)
if err != nil {
err := tx.Rollback()
if err != nil {
@@ -678,7 +649,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- att.SetPath(thumbnailPath)
+ att.SetPath(uploadResult.Path)
case contentType == "image/jxl":
log.Debug().Msg("creating thumbnail for jpegxl file")
img, err := jpegxl.Decode(bytes.NewReader(contentBytes))
@@ -690,7 +661,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))
+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))
if err != nil {
err := tx.Rollback()
if err != nil {
@@ -698,7 +669,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen
}
return err
}
- att.SetPath(thumbnailPath)
+ att.SetPath(uploadResult.Path)
default:
return fmt.Errorf("file type %s is not supported for thumbnail creation or document thumnails disabled", title)
}
@@ -773,30 +744,45 @@ func (r *AttachmentRepo) CreateMissingThumbnails(ctx context.Context, groupId uu
return count, nil
}
-func (r *AttachmentRepo) UploadFile(ctx context.Context, itemGroup *ent.Group, doc ItemCreateAttachment) (string, error) {
+// UploadResult contains the results of uploading a file
+type UploadResult struct {
+ Path string
+ ContentType string
+}
+
+func (r *AttachmentRepo) UploadFile(ctx context.Context, itemGroup *ent.Group, doc ItemCreateAttachment) (UploadResult, error) {
// Prepare for the hashing of the file contents
hashOut := make([]byte, 32)
- // Read all content into a buffer
+ // Use a buffer to store content for blake3 key derivation and storage
+ // While buffering, we compute MD5 and blake3 hashes in parallel for efficiency
buf := new(bytes.Buffer)
- _, err := io.Copy(buf, doc.Content)
+
+ // Create hash writers
+ blake3Hasher := blake3.New()
+ md5Hasher := md5.New()
+
+ // Use MultiWriter to write to buffer and both hashers simultaneously
+ multiWriter := io.MultiWriter(buf, blake3Hasher, md5Hasher)
+
+ _, err := io.Copy(multiWriter, doc.Content)
if err != nil {
log.Err(err).Msg("failed to read file content")
- return "", err
+ return UploadResult{}, err
}
- // Now the buffer contains all the data, use it for hashing
+
+ // Now the buffer contains all the data, and streaming hashes are computed
contentBytes := buf.Bytes()
- // We use blake3 to generate a hash of the file contents, the group ID is used as context to ensure unique hashes
- // for the same file across different groups to reduce the chance of collisions
- // additionally, the hash can be used to validate the file contents if needed
+ // Derive the blake3 key using the group ID as context
+ // Note: DeriveKey requires the full content buffer (not streaming)
blake3.DeriveKey(itemGroup.ID.String(), contentBytes, hashOut)
// Write the file to the blob storage bucket which might be a local file system or cloud storage
bucket, err := blob.OpenBucket(ctx, r.GetConnString())
if err != nil {
log.Err(err).Msg("failed to open bucket")
- return "", err
+ return UploadResult{}, err
}
defer func(bucket *blob.Bucket) {
err := bucket.Close()
@@ -804,26 +790,24 @@ func (r *AttachmentRepo) UploadFile(ctx context.Context, itemGroup *ent.Group, d
log.Err(err).Msg("failed to close bucket")
}
}(bucket)
- md5hash := md5.New()
- _, err = md5hash.Write(contentBytes)
- if err != nil {
- log.Err(err).Msg("failed to generate MD5 hash for storage")
- return "", err
- }
+
contentType := http.DetectContentType(contentBytes[:min(512, len(contentBytes))])
options := &blob.WriterOptions{
ContentType: contentType,
- ContentMD5: md5hash.Sum(nil),
+ ContentMD5: md5Hasher.Sum(nil),
}
relativePath := r.path(itemGroup.ID, fmt.Sprintf("%x", hashOut))
fullPath := r.fullPath(relativePath)
err = bucket.WriteAll(ctx, fullPath, contentBytes, options)
if err != nil {
log.Err(err).Msg("failed to write file to bucket")
- return "", err
+ return UploadResult{}, err
}
- return relativePath, nil
+ return UploadResult{
+ Path: relativePath,
+ ContentType: contentType,
+ }, nil
}
func isImageFile(mimetype string) bool {
@@ -864,7 +848,7 @@ func calculateThumbnailDimensions(origWidth, origHeight, maxWidth, maxHeight int
// processThumbnailFromImage handles the common thumbnail processing logic after image decoding
// Returns the thumbnail file path or an error
-func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId uuid.UUID, img image.Image, title string, orientation uint16) (string, error) {
+func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId uuid.UUID, img image.Image, title string, orientation uint16) (UploadResult, error) {
bounds := img.Bounds()
// Apply EXIF orientation if needed
if orientation > 1 {
@@ -878,7 +862,7 @@ func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId
buf := new(bytes.Buffer)
err := webp.Encode(buf, dst, webp.Options{Quality: 80, Lossless: false})
if err != nil {
- return "", err
+ return UploadResult{}, err
}
contentBytes := buf.Bytes()
log.Debug().Msg("uploading thumbnail file")
@@ -886,17 +870,17 @@ func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId
// Get the group for uploading the thumbnail
group, err := r.db.Group.Get(ctx, groupId)
if err != nil {
- return "", err
+ return UploadResult{}, err
}
- thumbnailFile, err := r.UploadFile(ctx, group, ItemCreateAttachment{
+ uploadResult, err := r.UploadFile(ctx, group, ItemCreateAttachment{
Title: fmt.Sprintf("%s-thumb", title),
Content: bytes.NewReader(contentBytes),
})
if err != nil {
log.Err(err).Msg("failed to upload thumbnail file")
- return "", err
+ return UploadResult{}, err
}
- return thumbnailFile, nil
+ return uploadResult, nil
}