A continuation of HomeBox the inventory and organization system built for the Home User
Optimize RAM usage in attachment upload and thumbnail processing (#1243)
Copilot342497e85359
diff --git a/backend/internal/core/services/reporting/import.go b/backend/internal/core/services/reporting/import.goindex 6f01b1b1..9e5ca6e2 100644--- a/backend/internal/core/services/reporting/import.go+++ b/backend/internal/core/services/reporting/import.go@@ -39,23 +39,32 @@ func determineSeparator(data []byte) (rune, error) {}}+// separatorDetectionBufferSize is the buffer size for reading CSV headers+// to detect the separator (comma vs tab)+const separatorDetectionBufferSize = 4096+// readRawCsv reads a CSV file and returns the raw data as a 2D string array// It determines the separator used in the CSV file and returns an error if// it could not be determinedfunc readRawCsv(r io.Reader) ([][]string, error) {- data, err := io.ReadAll(r)- if err != nil {+ // Buffer for reading the first line to detect separator+ // We read up to 4KB which should be more than enough for any header row+ firstLineBuffer := make([]byte, separatorDetectionBufferSize)+ n, err := io.ReadFull(r, firstLineBuffer)+ if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {return nil, err}+ firstLineBuffer = firstLineBuffer[:n]- reader := csv.NewReader(bytes.NewReader(data))-- // Determine separator- sep, err := determineSeparator(data)+ // Determine separator from first line+ sep, err := determineSeparator(firstLineBuffer)if err != nil {return nil, err}+ // Create a multi-reader combining what we already read and the rest+ combinedReader := io.MultiReader(bytes.NewReader(firstLineBuffer), r)+ reader := csv.NewReader(combinedReader)reader.Comma = sepreturn reader.ReadAll()diff --git a/backend/internal/data/repo/repo_item_attachments.go b/backend/internal/data/repo/repo_item_attachments.goindex 0b4aa612..3ca5f72b 100644--- a/backend/internal/data/repo/repo_item_attachments.go+++ b/backend/internal/data/repo/repo_item_attachments.go@@ -239,7 +239,7 @@ func (r *AttachmentRepo) Create(ctx context.Context, itemID uuid.UUID, doc ItemC}// Upload the file to the storage bucket- path, err := r.UploadFile(ctx, itemGroup, doc)+ uploadResult, err := r.UploadFile(ctx, itemGroup, doc)if err != nil {if rollbackErr := tx.Rollback(); rollbackErr != nil {return nil, rollbackErr@@ -247,18 +247,8 @@ func (r *AttachmentRepo) Create(ctx context.Context, itemID uuid.UUID, doc ItemCreturn nil, err}- limitedReader := io.LimitReader(doc.Content, 1024*128)- file, err := io.ReadAll(limitedReader)- if err != nil {- log.Err(err).Msg("failed to read file content")- err = tx.Rollback()- if err != nil {- return nil, err- }- return nil, err- }- bldr = bldr.SetMimeType(http.DetectContentType(file[:min(512, len(file))]))- bldr = bldr.SetPath(path)+ bldr = bldr.SetMimeType(uploadResult.ContentType)+ bldr = bldr.SetPath(uploadResult.Path)attachmentDb, err := bldr.Save(ctx)if err != nil {@@ -540,7 +530,10 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}log.Debug().Msg("reading original file content")- contentBytes, err := io.ReadAll(origFile)+ // Use LimitReader as a safety measure to prevent reading more than 100MB+ // even if the stat size is incorrect or tampered+ limitedReader := io.LimitReader(origFile, 100*1024*1024)+ contentBytes, err := io.ReadAll(limitedReader)if err != nil {err := tx.Rollback()if err != nil {@@ -564,6 +557,17 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}}+ // Pre-read orientation once for all image types that support it+ // This avoids re-decoding the metadata for each image type+ var orientation uint16 = 1 // Default orientation+ if contentType != "image/avif" && contentType != "image/jxl" {+ imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))+ if err == nil {+ orientation = uint16(imageMeta.Orientation)+ }+ // If error, just use default orientation (1)+ }+switch {case isImageFile(contentType):log.Debug().Msg("creating thumbnail for image file")@@ -577,18 +581,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- log.Debug().Msg("reading original file orientation")- imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))- if err != nil {- log.Err(err).Msg("failed to decode original file content")- err := tx.Rollback()- if err != nil {- return err- }- return err- }- orientation := uint16(imageMeta.Orientation)- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)if err != nil {err := tx.Rollback()if err != nil {@@ -596,7 +589,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- att.SetPath(thumbnailPath)+ att.SetPath(uploadResult.Path)case contentType == "image/webp":log.Debug().Msg("creating thumbnail for webp file")img, err := webp.Decode(bytes.NewReader(contentBytes))@@ -608,26 +601,15 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- log.Debug().Msg("reading original file orientation")- imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)if err != nil {- log.Err(err).Msg("failed to decode original file content")err := tx.Rollback()if err != nil {return err}return err}- orientation := uint16(imageMeta.Orientation)- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)- if err != nil {- err := tx.Rollback()- if err != nil {- return err- }- return err- }- att.SetPath(thumbnailPath)+ att.SetPath(uploadResult.Path)case contentType == "image/avif":log.Debug().Msg("creating thumbnail for avif file")img, err := avif.Decode(bytes.NewReader(contentBytes))@@ -639,7 +621,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))if err != nil {err := tx.Rollback()if err != nil {@@ -647,7 +629,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- att.SetPath(thumbnailPath)+ att.SetPath(uploadResult.Path)case contentType == "image/heic" || contentType == "image/heif":log.Debug().Msg("creating thumbnail for heic file")img, err := heic.Decode(bytes.NewReader(contentBytes))@@ -659,18 +641,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- log.Debug().Msg("reading original file orientation")- imageMeta, err := imagemeta.Decode(bytes.NewReader(contentBytes))- if err != nil {- log.Err(err).Msg("failed to decode original file content")- err := tx.Rollback()- if err != nil {- return err- }- return err- }- orientation := uint16(imageMeta.Orientation)- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, orientation)if err != nil {err := tx.Rollback()if err != nil {@@ -678,7 +649,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- att.SetPath(thumbnailPath)+ att.SetPath(uploadResult.Path)case contentType == "image/jxl":log.Debug().Msg("creating thumbnail for jpegxl file")img, err := jpegxl.Decode(bytes.NewReader(contentBytes))@@ -690,7 +661,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- thumbnailPath, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))+ uploadResult, err := r.processThumbnailFromImage(ctx, groupId, img, title, uint16(1))if err != nil {err := tx.Rollback()if err != nil {@@ -698,7 +669,7 @@ func (r *AttachmentRepo) CreateThumbnail(ctx context.Context, groupId, attachmen}return err}- att.SetPath(thumbnailPath)+ att.SetPath(uploadResult.Path)default:return fmt.Errorf("file type %s is not supported for thumbnail creation or document thumnails disabled", title)}@@ -773,30 +744,45 @@ func (r *AttachmentRepo) CreateMissingThumbnails(ctx context.Context, groupId uureturn count, nil}-func (r *AttachmentRepo) UploadFile(ctx context.Context, itemGroup *ent.Group, doc ItemCreateAttachment) (string, error) {+// UploadResult contains the results of uploading a file+type UploadResult struct {+ Path string+ ContentType string+}++func (r *AttachmentRepo) UploadFile(ctx context.Context, itemGroup *ent.Group, doc ItemCreateAttachment) (UploadResult, error) {// Prepare for the hashing of the file contentshashOut := make([]byte, 32)- // Read all content into a buffer+ // Use a buffer to store content for blake3 key derivation and storage+ // While buffering, we compute MD5 and blake3 hashes in parallel for efficiencybuf := new(bytes.Buffer)- _, err := io.Copy(buf, doc.Content)++ // Create hash writers+ blake3Hasher := blake3.New()+ md5Hasher := md5.New()++ // Use MultiWriter to write to buffer and both hashers simultaneously+ multiWriter := io.MultiWriter(buf, blake3Hasher, md5Hasher)++ _, err := io.Copy(multiWriter, doc.Content)if err != nil {log.Err(err).Msg("failed to read file content")- return "", err+ return UploadResult{}, err}- // Now the buffer contains all the data, use it for hashing++ // Now the buffer contains all the data, and streaming hashes are computedcontentBytes := buf.Bytes()- // We use blake3 to generate a hash of the file contents, the group ID is used as context to ensure unique hashes- // for the same file across different groups to reduce the chance of collisions- // additionally, the hash can be used to validate the file contents if needed+ // Derive the blake3 key using the group ID as context+ // Note: DeriveKey requires the full content buffer (not streaming)blake3.DeriveKey(itemGroup.ID.String(), contentBytes, hashOut)// Write the file to the blob storage bucket which might be a local file system or cloud storagebucket, err := blob.OpenBucket(ctx, r.GetConnString())if err != nil {log.Err(err).Msg("failed to open bucket")- return "", err+ return UploadResult{}, err}defer func(bucket *blob.Bucket) {err := bucket.Close()@@ -804,26 +790,24 @@ func (r *AttachmentRepo) UploadFile(ctx context.Context, itemGroup *ent.Group, dlog.Err(err).Msg("failed to close bucket")}}(bucket)- md5hash := md5.New()- _, err = md5hash.Write(contentBytes)- if err != nil {- log.Err(err).Msg("failed to generate MD5 hash for storage")- return "", err- }+contentType := http.DetectContentType(contentBytes[:min(512, len(contentBytes))])options := &blob.WriterOptions{ContentType: contentType,- ContentMD5: md5hash.Sum(nil),+ ContentMD5: md5Hasher.Sum(nil),}relativePath := r.path(itemGroup.ID, fmt.Sprintf("%x", hashOut))fullPath := r.fullPath(relativePath)err = bucket.WriteAll(ctx, fullPath, contentBytes, options)if err != nil {log.Err(err).Msg("failed to write file to bucket")- return "", err+ return UploadResult{}, err}- return relativePath, nil+ return UploadResult{+ Path: relativePath,+ ContentType: contentType,+ }, nil}func isImageFile(mimetype string) bool {@@ -864,7 +848,7 @@ func calculateThumbnailDimensions(origWidth, origHeight, maxWidth, maxHeight int// processThumbnailFromImage handles the common thumbnail processing logic after image decoding// Returns the thumbnail file path or an error-func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId uuid.UUID, img image.Image, title string, orientation uint16) (string, error) {+func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId uuid.UUID, img image.Image, title string, orientation uint16) (UploadResult, error) {bounds := img.Bounds()// Apply EXIF orientation if neededif orientation > 1 {@@ -878,7 +862,7 @@ func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupIdbuf := new(bytes.Buffer)err := webp.Encode(buf, dst, webp.Options{Quality: 80, Lossless: false})if err != nil {- return "", err+ return UploadResult{}, err}contentBytes := buf.Bytes()log.Debug().Msg("uploading thumbnail file")@@ -886,17 +870,17 @@ func (r *AttachmentRepo) processThumbnailFromImage(ctx context.Context, groupId// Get the group for uploading the thumbnailgroup, err := r.db.Group.Get(ctx, groupId)if err != nil {- return "", err+ return UploadResult{}, err}- thumbnailFile, err := r.UploadFile(ctx, group, ItemCreateAttachment{+ uploadResult, err := r.UploadFile(ctx, group, ItemCreateAttachment{Title: fmt.Sprintf("%s-thumb", title),Content: bytes.NewReader(contentBytes),})if err != nil {log.Err(err).Msg("failed to upload thumbnail file")- return "", err+ return UploadResult{}, err}- return thumbnailFile, nil+ return uploadResult, nil}