Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

Detect progressive JPEGs for a tighter decode budget (GHSA-7cw3-7xh9-529r)

Matt Robertse568b436d9d9
diff --git a/app/pkg/validate/image.go b/app/pkg/validate/image.go
index 7fcd8ddc1..987fa017e 100644
--- a/app/pkg/validate/image.go
+++ b/app/pkg/validate/image.go
@@ -23,7 +23,8 @@ import (
// compressed file is, so a tiny file can be a "decompression bomb". See
// EstimateDecodeBytes for the per-format estimates. With 256MB the largest images that
// can be decoded are roughly:
-// - JPEG (colour): ~14 megapixels (e.g. 12MP phone photos)
+// - JPEG (colour, baseline): ~38 megapixels
+// - JPEG (colour, progressive): ~14 megapixels
// - PNG (8-bit RGB/RGBA): ~22 megapixels
// - PNG (8-bit gray/paletted): ~44 megapixels
// - GIF: ~53 megapixels
@@ -49,9 +50,10 @@ var ErrImageTooLarge = errors.New("image is too large to be decoded")
// EstimateDecodeBytes estimates the worst case memory needed to decode an image with
// the given header info (format and color model as returned by image.DecodeConfig) and
-// to create one full-size intermediate copy of it.
+// to create one full-size intermediate copy of it. progressive is only used for JPEGs
+// (see isProgressiveJPEG).
// Returns -1 if the dimensions are invalid or beyond MaxImageSide.
-func EstimateDecodeBytes(width, height int, format string, model color.Model) int64 {
+func EstimateDecodeBytes(width, height int, format string, model color.Model, progressive bool) int64 {
if width <= 0 || height <= 0 || width > MaxImageSide || height > MaxImageSide {
return -1
}
@@ -59,18 +61,30 @@ func EstimateDecodeBytes(width, height int, format string, model color.Model) in
var decodeBytesPerPixel int64
switch format {
case "jpeg":
- // We can't tell from the header if the JPEG is progressive, so assume it is.
- // A progressive decode keeps all DCT coefficients (64 x int32 per 8x8 block,
- // i.e. 4 bytes per pixel per component) in addition to the decoded image.
+ // Go's jpeg decoder allocates whole MCUs (up to 32x32 pixels with 4x sampling factors)
+ width, height = roundUp(width, 32), roundUp(height, 32)
+
+ // Buffers allocated by image/jpeg, per pixel, assuming no chroma subsampling (worst case):
+ // - Gray: the image.Gray (1)
+ // - YCbCr: the image.YCbCr (3)
+ // - Adobe RGB (RGBAModel): the image.YCbCr (3), converted into an image.RGBA (4)
+ // - CMYK (or unknown): the image.YCbCr (3) + black channel (1), converted into an image.CMYK (4)
+ // A progressive decode additionally keeps all DCT coefficients until the end
+ // (64 x int32 per 8x8 block, i.e. 4 bytes per pixel per component).
+ var imageBytes, components int64
switch model {
case color.GrayModel:
- decodeBytesPerPixel = 4*1 + 1
+ imageBytes, components = 1, 1
case color.YCbCrModel:
- decodeBytesPerPixel = 4*3 + 3
- case color.RGBAModel: // Adobe RGB JPEG: decoded as YCbCr, then converted to RGBA
- decodeBytesPerPixel = 4*3 + 3 + 4
- default: // CMYK or unknown
- decodeBytesPerPixel = 4*4 + 4
+ imageBytes, components = 3, 3
+ case color.RGBAModel:
+ imageBytes, components = 3+4, 3
+ default:
+ imageBytes, components = 3+1+4, 4
+ }
+ decodeBytesPerPixel = imageBytes
+ if progressive {
+ decodeBytesPerPixel += 4 * components
}
case "png":
// Interlaced PNGs decode each pass into its own image before merging, so
@@ -83,10 +97,71 @@ func EstimateDecodeBytes(width, height int, format string, model color.Model) in
decodeBytesPerPixel = 2 * bytesPerPixel(model)
}
- // Can't overflow: at most 16384 * 16384 * (20 + 4)
+ // Can't overflow: at most 16416 * 16416 * (24 + 4)
return int64(width) * int64(height) * (decodeBytesPerPixel + fullSizeIntermediateBytesPerPixel)
}
+func roundUp(n, multiple int) int {
+ return (n + multiple - 1) / multiple * multiple
+}
+
+// maxJPEGHeaderScan limits how many bytes isProgressiveJPEG looks at to find the SOF marker.
+// Only metadata segments (EXIF, ICC profile, etc) come before it, each at most 64KB.
+const maxJPEGHeaderScan = 1 << 20
+
+// isProgressiveJPEG walks the JPEG segments up to the first Start Of Frame marker and
+// reports whether it's anything other than baseline/extended sequential (SOF0/SOF1).
+// Anything unexpected, malformed or truncated is reported as progressive, so that the
+// decode budget falls back to the worst case. When this returns false, image/jpeg follows
+// the exact same segments and sees the same SOF0/SOF1 marker (it rejects multiple SOFs).
+func isProgressiveJPEG(content []byte) bool {
+ data := content
+ if len(data) > maxJPEGHeaderScan {
+ data = data[:maxJPEGHeaderScan]
+ }
+ if len(data) < 2 || data[0] != 0xFF || data[1] != 0xD8 { // SOI
+ return true
+ }
+
+ i := 2
+ for {
+ if i >= len(data) || data[i] != 0xFF {
+ return true
+ }
+ // A marker may be preceded by any number of 0xFF fill bytes
+ for i < len(data) && data[i] == 0xFF {
+ i++
+ }
+ if i >= len(data) {
+ return true
+ }
+ marker := data[i]
+ i++
+
+ switch {
+ case marker == 0xC0 || marker == 0xC1: // SOF0 (baseline), SOF1 (extended sequential)
+ return false
+ case marker == 0xC4 || marker == 0xC8 || marker == 0xCC: // DHT, JPG, DAC: not frames
+ case marker >= 0xC2 && marker <= 0xCF: // SOF2 (progressive), SOF6/SOF10/SOF14 and others
+ return true
+ case marker == 0x01 || (marker >= 0xD0 && marker <= 0xD7): // TEM, RSTn: no length
+ continue
+ case marker == 0x00 || marker == 0xD8 || marker == 0xD9 || marker == 0xDA:
+ // Stuffed byte, SOI, EOI or SOS before any SOF: not a well formed header
+ return true
+ }
+
+ if i+2 > len(data) {
+ return true
+ }
+ length := int(data[i])<<8 | int(data[i+1])
+ if length < 2 {
+ return true
+ }
+ i += length
+ }
+}
+
func bytesPerPixel(model color.Model) int64 {
if _, ok := model.(color.Palette); ok {
return 1
@@ -122,7 +197,8 @@ func CheckDecodeBudget(content []byte) error {
return imagic.ErrNotSupported
}
- estimate := EstimateDecodeBytes(file.Width, file.Height, format, cfg.ColorModel)
+ progressive := format == "jpeg" && isProgressiveJPEG(content)
+ estimate := EstimateDecodeBytes(file.Width, file.Height, format, cfg.ColorModel, progressive)
if estimate < 0 || estimate > MaxDecodeBytes {
return ErrImageTooLarge
}
diff --git a/app/pkg/validate/image_internal_test.go b/app/pkg/validate/image_internal_test.go
index 4df2d9b97..89c0769c1 100644
--- a/app/pkg/validate/image_internal_test.go
+++ b/app/pkg/validate/image_internal_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"image"
+ "image/jpeg"
"image/png"
"testing"
@@ -33,3 +34,66 @@ func TestSafeApply_LimitsConcurrentDecodes(t *testing.T) {
Expect(err).Equals(context.Canceled)
Expect(result).IsNil()
}
+
+// encodeJPEG returns a small JPEG. Go's encoder always writes baseline (SOF0) JPEGs.
+func encodeJPEG() []byte {
+ var buf bytes.Buffer
+ Expect(jpeg.Encode(&buf, image.NewYCbCr(image.Rect(0, 0, 16, 16), image.YCbCrSubsampleRatio444), nil)).IsNil()
+ return buf.Bytes()
+}
+
+// withSOF returns a copy of the given JPEG with its SOF0 marker replaced by the given marker
+func withSOF(content []byte, marker byte) []byte {
+ content = append([]byte{}, content...)
+ sof := bytes.Index(content, []byte{0xFF, 0xC0})
+ Expect(sof > 0).IsTrue()
+ content[sof+1] = marker
+ return content
+}
+
+func TestIsProgressiveJPEG(t *testing.T) {
+ RegisterT(t)
+
+ baseline := encodeJPEG()
+ Expect(isProgressiveJPEG(baseline)).IsFalse()
+ Expect(isProgressiveJPEG(withSOF(baseline, 0xC1))).IsFalse() // extended sequential
+
+ // Progressive and other non-sequential SOF markers
+ for _, marker := range []byte{0xC2, 0xC3, 0xC6, 0xCA, 0xCE} {
+ Expect(isProgressiveJPEG(withSOF(baseline, marker))).IsTrue()
+ }
+
+ // Go reads the header of a patched SOF2 JPEG just fine
+ _, format, err := image.DecodeConfig(bytes.NewReader(withSOF(baseline, 0xC2)))
+ Expect(err).IsNil()
+ Expect(format).Equals("jpeg")
+
+ // Fill bytes, standalone markers (RST0, TEM) and APPn segments before the SOF are skipped
+ withExtras := []byte{0xFF, 0xD8, 0xFF, 0xFF, 0xD0, 0xFF, 0x01, 0xFF, 0xE1, 0x00, 0x06, 'E', 'x', 'i', 'f'}
+ Expect(isProgressiveJPEG(append(withExtras, baseline[2:]...))).IsFalse()
+ withExtras = []byte{0xFF, 0xD8, 0xFF, 0xFF, 0xFF}
+ Expect(isProgressiveJPEG(append(withExtras, withSOF(baseline, 0xC2)[2:]...))).IsTrue()
+
+ // Truncated, malformed or garbage input is treated as progressive (worst case)
+ sof := bytes.Index(baseline, []byte{0xFF, 0xC0})
+ for n := 0; n <= sof+1; n++ {
+ Expect(isProgressiveJPEG(baseline[:n])).IsTrue()
+ }
+ Expect(isProgressiveJPEG(nil)).IsTrue()
+ Expect(isProgressiveJPEG([]byte("not a jpeg at all"))).IsTrue()
+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0x00, 0xFF, 0xC0})).IsTrue() // data where a marker should be
+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0x00, 0xFF, 0xC0})).IsTrue() // stuffed zero
+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01, 0xFF, 0xC0})).IsTrue() // invalid segment length
+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xDA, 0x00, 0x02, 0xFF, 0xC0})).IsTrue() // SOS before SOF
+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xD9})).IsTrue() // EOI before SOF
+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xE0, 0xFF, 0xFF, 0xFF, 0xC0})).IsTrue() // segment runs past the end
+
+ // SOF beyond the scan limit
+ padded := []byte{0xFF, 0xD8}
+ for len(padded) < maxJPEGHeaderScan {
+ padded = append(padded, 0xFF, 0xE1, 0xFF, 0xFF)
+ padded = append(padded, make([]byte, 0xFFFF-2)...)
+ }
+ padded = append(padded, baseline[2:]...)
+ Expect(isProgressiveJPEG(padded)).IsTrue()
+}
diff --git a/app/pkg/validate/upload_test.go b/app/pkg/validate/upload_test.go
index 6b6b0df34..4f48501e6 100644
--- a/app/pkg/validate/upload_test.go
+++ b/app/pkg/validate/upload_test.go
@@ -206,7 +206,8 @@ func imageDimensions(t *testing.T, content []byte) (int, int) {
// jpegHeaderWithDimensions returns a JPEG whose header (SOF) claims the given dimensions.
// Only the header is valid, which is all image.DecodeConfig reads.
-func jpegHeaderWithDimensions(t *testing.T, width, height int) []byte {
+// Go always encodes baseline JPEGs (SOF0); progressive ones are made by patching it to SOF2.
+func jpegHeaderWithDimensions(t *testing.T, width, height int, progressive bool) []byte {
var buf bytes.Buffer
err := jpeg.Encode(&buf, image.NewYCbCr(image.Rect(0, 0, 16, 16), image.YCbCrSubsampleRatio444), nil)
Expect(err).IsNil()
@@ -216,6 +217,9 @@ func jpegHeaderWithDimensions(t *testing.T, width, height int) []byte {
// FF C0, length (2), precision (1), height (2), width (2)
binary.BigEndian.PutUint16(content[sof+5:], uint16(height))
binary.BigEndian.PutUint16(content[sof+7:], uint16(width))
+ if progressive {
+ content[sof+1] = 0xC2
+ }
return content
}
@@ -232,8 +236,10 @@ func TestValidateImageUpload_DecompressionBomb(t *testing.T) {
{"gif 40000x40000", mock.GIFHeader(40000, 40000)},
// Small pixel count, but extreme aspect ratio
{"png 100000x1", mock.UniformPNG(100000, 1)},
- // 4000x4000 colour JPEG is over the budget, as it might be progressive
- {"jpeg 4000x4000", jpegHeaderWithDimensions(t, 4000, 4000)},
+ // 4000x4000 progressive colour JPEG
+ {"jpeg 4000x4000", jpegHeaderWithDimensions(t, 4000, 4000, true)},
+ // 8000x8000 baseline colour JPEG
+ {"jpeg 8000x8000", jpegHeaderWithDimensions(t, 8000, 8000, false)},
}
for _, testCase := range testCases {
@@ -329,29 +335,39 @@ func TestEstimateDecodeBytes(t *testing.T) {
RegisterT(t)
// Invalid or over the per-side cap
- Expect(validate.EstimateDecodeBytes(0, 100, "png", color.GrayModel)).Equals(int64(-1))
- Expect(validate.EstimateDecodeBytes(100, 0, "png", color.GrayModel)).Equals(int64(-1))
- Expect(validate.EstimateDecodeBytes(-100, -100, "png", color.GrayModel)).Equals(int64(-1))
- Expect(validate.EstimateDecodeBytes(validate.MaxImageSide+1, 1, "png", color.GrayModel)).Equals(int64(-1))
- Expect(validate.EstimateDecodeBytes(1, validate.MaxImageSide+1, "png", color.GrayModel)).Equals(int64(-1))
- Expect(validate.EstimateDecodeBytes(40_000_000, 1, "gif", color.Palette{})).Equals(int64(-1))
- Expect(validate.EstimateDecodeBytes(math.MaxInt, math.MaxInt, "png", color.GrayModel)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(0, 100, "png", color.GrayModel, false)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(100, 0, "png", color.GrayModel, false)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(-100, -100, "png", color.GrayModel, false)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(validate.MaxImageSide+1, 1, "png", color.GrayModel, false)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(1, validate.MaxImageSide+1, "png", color.GrayModel, false)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(40_000_000, 1, "gif", color.Palette{}, false)).Equals(int64(-1))
+ Expect(validate.EstimateDecodeBytes(math.MaxInt, math.MaxInt, "png", color.GrayModel, false)).Equals(int64(-1))
// Bytes per pixel include 4 bytes for a full-size RGBA intermediate
const mp = 1000 * 1000
- Expect(validate.EstimateDecodeBytes(1000, 1000, "gif", color.Palette{})).Equals(int64(5 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Palette{})).Equals(int64(6 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.GrayModel)).Equals(int64(6 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Gray16Model)).Equals(int64(8 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBAModel)).Equals(int64(12 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBA64Model)).Equals(int64(20 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.GrayModel)).Equals(int64(9 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.YCbCrModel)).Equals(int64(19 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.RGBAModel)).Equals(int64(23 * mp))
- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.CMYKModel)).Equals(int64(24 * mp))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "gif", color.Palette{}, false)).Equals(int64(5 * mp))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Palette{}, false)).Equals(int64(6 * mp))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.GrayModel, false)).Equals(int64(6 * mp))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Gray16Model, false)).Equals(int64(8 * mp))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBAModel, false)).Equals(int64(12 * mp))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBA64Model, false)).Equals(int64(20 * mp))
+ // JPEG images are allocated in whole MCUs (up to 32x32), so 1000 is rounded up to 1024
+ const jpegPixels = 1024 * 1024
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.GrayModel, false)).Equals(int64(5 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.YCbCrModel, false)).Equals(int64(7 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.RGBAModel, false)).Equals(int64(11 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.CMYKModel, false)).Equals(int64(12 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.GrayModel, true)).Equals(int64(9 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.YCbCrModel, true)).Equals(int64(19 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.RGBAModel, true)).Equals(int64(23 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.CMYKModel, true)).Equals(int64(28 * jpegPixels))
+ Expect(validate.EstimateDecodeBytes(1024, 1024, "jpeg", color.YCbCrModel, false)).Equals(int64(7 * jpegPixels))
+
+ // progressive is ignored for other formats
+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBAModel, true)).Equals(int64(12 * mp))
// Worst case at the per-side cap doesn't overflow
- Expect(validate.EstimateDecodeBytes(validate.MaxImageSide, validate.MaxImageSide, "jpeg", color.CMYKModel) > 0).IsTrue()
+ Expect(validate.EstimateDecodeBytes(validate.MaxImageSide, validate.MaxImageSide, "jpeg", color.CMYKModel, true) > 0).IsTrue()
}
func TestCheckDecodeBudget(t *testing.T) {
@@ -366,10 +382,14 @@ func TestCheckDecodeBudget(t *testing.T) {
Expect(validate.CheckDecodeBudget(mock.UniformPNG(100000, 1))).Equals(validate.ErrImageTooLarge)
Expect(validate.CheckDecodeBudget(mock.GIFHeader(40000, 40000))).Equals(validate.ErrImageTooLarge)
- // 12MP phone photo: 12.2MP * 19 bytes = 232MB
- Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 4032, 3024))).IsNil()
- // 16MP: 304MB
- Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 4000, 4000))).Equals(validate.ErrImageTooLarge)
+ // 24MP baseline colour JPEG: 24MP * 7 bytes = 168MB
+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 6000, 4000, false))).IsNil()
+ // Same dimensions, but progressive: 24MP * 19 bytes = 457MB
+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 6000, 4000, true))).Equals(validate.ErrImageTooLarge)
+ // 12MP progressive phone photo: 12.2MP * 19 bytes = 233MB
+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 4032, 3024, true))).IsNil()
+ // 40MP baseline: 280MB
+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 8000, 5000, false))).Equals(validate.ErrImageTooLarge)
img, _ := os.ReadFile(env.Path("/app/pkg/web/testdata/logo2.jpg"))
Expect(validate.CheckDecodeBudget(img)).IsNil()