Open platform to collect and prioritize feedback
Detect progressive JPEGs for a tighter decode budget (GHSA-7cw3-7xh9-529r)
Matt Robertse568b436d9d9
diff --git a/app/pkg/validate/image.go b/app/pkg/validate/image.goindex 7fcd8ddc1..987fa017e 100644--- a/app/pkg/validate/image.go+++ b/app/pkg/validate/image.go@@ -23,7 +23,8 @@ import (// compressed file is, so a tiny file can be a "decompression bomb". See// EstimateDecodeBytes for the per-format estimates. With 256MB the largest images that// can be decoded are roughly:-// - JPEG (colour): ~14 megapixels (e.g. 12MP phone photos)+// - JPEG (colour, baseline): ~38 megapixels+// - JPEG (colour, progressive): ~14 megapixels// - PNG (8-bit RGB/RGBA): ~22 megapixels// - PNG (8-bit gray/paletted): ~44 megapixels// - GIF: ~53 megapixels@@ -49,9 +50,10 @@ var ErrImageTooLarge = errors.New("image is too large to be decoded")// EstimateDecodeBytes estimates the worst case memory needed to decode an image with// the given header info (format and color model as returned by image.DecodeConfig) and-// to create one full-size intermediate copy of it.+// to create one full-size intermediate copy of it. progressive is only used for JPEGs+// (see isProgressiveJPEG).// Returns -1 if the dimensions are invalid or beyond MaxImageSide.-func EstimateDecodeBytes(width, height int, format string, model color.Model) int64 {+func EstimateDecodeBytes(width, height int, format string, model color.Model, progressive bool) int64 {if width <= 0 || height <= 0 || width > MaxImageSide || height > MaxImageSide {return -1}@@ -59,18 +61,30 @@ func EstimateDecodeBytes(width, height int, format string, model color.Model) invar decodeBytesPerPixel int64switch format {case "jpeg":- // We can't tell from the header if the JPEG is progressive, so assume it is.- // A progressive decode keeps all DCT coefficients (64 x int32 per 8x8 block,- // i.e. 4 bytes per pixel per component) in addition to the decoded image.+ // Go's jpeg decoder allocates whole MCUs (up to 32x32 pixels with 4x sampling factors)+ width, height = roundUp(width, 32), roundUp(height, 32)++ // Buffers allocated by image/jpeg, per pixel, assuming no chroma subsampling (worst case):+ // - Gray: the image.Gray (1)+ // - YCbCr: the image.YCbCr (3)+ // - Adobe RGB (RGBAModel): the image.YCbCr (3), converted into an image.RGBA (4)+ // - CMYK (or unknown): the image.YCbCr (3) + black channel (1), converted into an image.CMYK (4)+ // A progressive decode additionally keeps all DCT coefficients until the end+ // (64 x int32 per 8x8 block, i.e. 4 bytes per pixel per component).+ var imageBytes, components int64switch model {case color.GrayModel:- decodeBytesPerPixel = 4*1 + 1+ imageBytes, components = 1, 1case color.YCbCrModel:- decodeBytesPerPixel = 4*3 + 3- case color.RGBAModel: // Adobe RGB JPEG: decoded as YCbCr, then converted to RGBA- decodeBytesPerPixel = 4*3 + 3 + 4- default: // CMYK or unknown- decodeBytesPerPixel = 4*4 + 4+ imageBytes, components = 3, 3+ case color.RGBAModel:+ imageBytes, components = 3+4, 3+ default:+ imageBytes, components = 3+1+4, 4+ }+ decodeBytesPerPixel = imageBytes+ if progressive {+ decodeBytesPerPixel += 4 * components}case "png":// Interlaced PNGs decode each pass into its own image before merging, so@@ -83,10 +97,71 @@ func EstimateDecodeBytes(width, height int, format string, model color.Model) indecodeBytesPerPixel = 2 * bytesPerPixel(model)}- // Can't overflow: at most 16384 * 16384 * (20 + 4)+ // Can't overflow: at most 16416 * 16416 * (24 + 4)return int64(width) * int64(height) * (decodeBytesPerPixel + fullSizeIntermediateBytesPerPixel)}+func roundUp(n, multiple int) int {+ return (n + multiple - 1) / multiple * multiple+}++// maxJPEGHeaderScan limits how many bytes isProgressiveJPEG looks at to find the SOF marker.+// Only metadata segments (EXIF, ICC profile, etc) come before it, each at most 64KB.+const maxJPEGHeaderScan = 1 << 20++// isProgressiveJPEG walks the JPEG segments up to the first Start Of Frame marker and+// reports whether it's anything other than baseline/extended sequential (SOF0/SOF1).+// Anything unexpected, malformed or truncated is reported as progressive, so that the+// decode budget falls back to the worst case. When this returns false, image/jpeg follows+// the exact same segments and sees the same SOF0/SOF1 marker (it rejects multiple SOFs).+func isProgressiveJPEG(content []byte) bool {+ data := content+ if len(data) > maxJPEGHeaderScan {+ data = data[:maxJPEGHeaderScan]+ }+ if len(data) < 2 || data[0] != 0xFF || data[1] != 0xD8 { // SOI+ return true+ }++ i := 2+ for {+ if i >= len(data) || data[i] != 0xFF {+ return true+ }+ // A marker may be preceded by any number of 0xFF fill bytes+ for i < len(data) && data[i] == 0xFF {+ i+++ }+ if i >= len(data) {+ return true+ }+ marker := data[i]+ i++++ switch {+ case marker == 0xC0 || marker == 0xC1: // SOF0 (baseline), SOF1 (extended sequential)+ return false+ case marker == 0xC4 || marker == 0xC8 || marker == 0xCC: // DHT, JPG, DAC: not frames+ case marker >= 0xC2 && marker <= 0xCF: // SOF2 (progressive), SOF6/SOF10/SOF14 and others+ return true+ case marker == 0x01 || (marker >= 0xD0 && marker <= 0xD7): // TEM, RSTn: no length+ continue+ case marker == 0x00 || marker == 0xD8 || marker == 0xD9 || marker == 0xDA:+ // Stuffed byte, SOI, EOI or SOS before any SOF: not a well formed header+ return true+ }++ if i+2 > len(data) {+ return true+ }+ length := int(data[i])<<8 | int(data[i+1])+ if length < 2 {+ return true+ }+ i += length+ }+}+func bytesPerPixel(model color.Model) int64 {if _, ok := model.(color.Palette); ok {return 1@@ -122,7 +197,8 @@ func CheckDecodeBudget(content []byte) error {return imagic.ErrNotSupported}- estimate := EstimateDecodeBytes(file.Width, file.Height, format, cfg.ColorModel)+ progressive := format == "jpeg" && isProgressiveJPEG(content)+ estimate := EstimateDecodeBytes(file.Width, file.Height, format, cfg.ColorModel, progressive)if estimate < 0 || estimate > MaxDecodeBytes {return ErrImageTooLarge}diff --git a/app/pkg/validate/image_internal_test.go b/app/pkg/validate/image_internal_test.goindex 4df2d9b97..89c0769c1 100644--- a/app/pkg/validate/image_internal_test.go+++ b/app/pkg/validate/image_internal_test.go@@ -4,6 +4,7 @@ import ("bytes""context""image"+ "image/jpeg""image/png""testing"@@ -33,3 +34,66 @@ func TestSafeApply_LimitsConcurrentDecodes(t *testing.T) {Expect(err).Equals(context.Canceled)Expect(result).IsNil()}++// encodeJPEG returns a small JPEG. Go's encoder always writes baseline (SOF0) JPEGs.+func encodeJPEG() []byte {+ var buf bytes.Buffer+ Expect(jpeg.Encode(&buf, image.NewYCbCr(image.Rect(0, 0, 16, 16), image.YCbCrSubsampleRatio444), nil)).IsNil()+ return buf.Bytes()+}++// withSOF returns a copy of the given JPEG with its SOF0 marker replaced by the given marker+func withSOF(content []byte, marker byte) []byte {+ content = append([]byte{}, content...)+ sof := bytes.Index(content, []byte{0xFF, 0xC0})+ Expect(sof > 0).IsTrue()+ content[sof+1] = marker+ return content+}++func TestIsProgressiveJPEG(t *testing.T) {+ RegisterT(t)++ baseline := encodeJPEG()+ Expect(isProgressiveJPEG(baseline)).IsFalse()+ Expect(isProgressiveJPEG(withSOF(baseline, 0xC1))).IsFalse() // extended sequential++ // Progressive and other non-sequential SOF markers+ for _, marker := range []byte{0xC2, 0xC3, 0xC6, 0xCA, 0xCE} {+ Expect(isProgressiveJPEG(withSOF(baseline, marker))).IsTrue()+ }++ // Go reads the header of a patched SOF2 JPEG just fine+ _, format, err := image.DecodeConfig(bytes.NewReader(withSOF(baseline, 0xC2)))+ Expect(err).IsNil()+ Expect(format).Equals("jpeg")++ // Fill bytes, standalone markers (RST0, TEM) and APPn segments before the SOF are skipped+ withExtras := []byte{0xFF, 0xD8, 0xFF, 0xFF, 0xD0, 0xFF, 0x01, 0xFF, 0xE1, 0x00, 0x06, 'E', 'x', 'i', 'f'}+ Expect(isProgressiveJPEG(append(withExtras, baseline[2:]...))).IsFalse()+ withExtras = []byte{0xFF, 0xD8, 0xFF, 0xFF, 0xFF}+ Expect(isProgressiveJPEG(append(withExtras, withSOF(baseline, 0xC2)[2:]...))).IsTrue()++ // Truncated, malformed or garbage input is treated as progressive (worst case)+ sof := bytes.Index(baseline, []byte{0xFF, 0xC0})+ for n := 0; n <= sof+1; n++ {+ Expect(isProgressiveJPEG(baseline[:n])).IsTrue()+ }+ Expect(isProgressiveJPEG(nil)).IsTrue()+ Expect(isProgressiveJPEG([]byte("not a jpeg at all"))).IsTrue()+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0x00, 0xFF, 0xC0})).IsTrue() // data where a marker should be+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0x00, 0xFF, 0xC0})).IsTrue() // stuffed zero+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01, 0xFF, 0xC0})).IsTrue() // invalid segment length+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xDA, 0x00, 0x02, 0xFF, 0xC0})).IsTrue() // SOS before SOF+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xD9})).IsTrue() // EOI before SOF+ Expect(isProgressiveJPEG([]byte{0xFF, 0xD8, 0xFF, 0xE0, 0xFF, 0xFF, 0xFF, 0xC0})).IsTrue() // segment runs past the end++ // SOF beyond the scan limit+ padded := []byte{0xFF, 0xD8}+ for len(padded) < maxJPEGHeaderScan {+ padded = append(padded, 0xFF, 0xE1, 0xFF, 0xFF)+ padded = append(padded, make([]byte, 0xFFFF-2)...)+ }+ padded = append(padded, baseline[2:]...)+ Expect(isProgressiveJPEG(padded)).IsTrue()+}diff --git a/app/pkg/validate/upload_test.go b/app/pkg/validate/upload_test.goindex 6b6b0df34..4f48501e6 100644--- a/app/pkg/validate/upload_test.go+++ b/app/pkg/validate/upload_test.go@@ -206,7 +206,8 @@ func imageDimensions(t *testing.T, content []byte) (int, int) {// jpegHeaderWithDimensions returns a JPEG whose header (SOF) claims the given dimensions.// Only the header is valid, which is all image.DecodeConfig reads.-func jpegHeaderWithDimensions(t *testing.T, width, height int) []byte {+// Go always encodes baseline JPEGs (SOF0); progressive ones are made by patching it to SOF2.+func jpegHeaderWithDimensions(t *testing.T, width, height int, progressive bool) []byte {var buf bytes.Buffererr := jpeg.Encode(&buf, image.NewYCbCr(image.Rect(0, 0, 16, 16), image.YCbCrSubsampleRatio444), nil)Expect(err).IsNil()@@ -216,6 +217,9 @@ func jpegHeaderWithDimensions(t *testing.T, width, height int) []byte {// FF C0, length (2), precision (1), height (2), width (2)binary.BigEndian.PutUint16(content[sof+5:], uint16(height))binary.BigEndian.PutUint16(content[sof+7:], uint16(width))+ if progressive {+ content[sof+1] = 0xC2+ }return content}@@ -232,8 +236,10 @@ func TestValidateImageUpload_DecompressionBomb(t *testing.T) {{"gif 40000x40000", mock.GIFHeader(40000, 40000)},// Small pixel count, but extreme aspect ratio{"png 100000x1", mock.UniformPNG(100000, 1)},- // 4000x4000 colour JPEG is over the budget, as it might be progressive- {"jpeg 4000x4000", jpegHeaderWithDimensions(t, 4000, 4000)},+ // 4000x4000 progressive colour JPEG+ {"jpeg 4000x4000", jpegHeaderWithDimensions(t, 4000, 4000, true)},+ // 8000x8000 baseline colour JPEG+ {"jpeg 8000x8000", jpegHeaderWithDimensions(t, 8000, 8000, false)},}for _, testCase := range testCases {@@ -329,29 +335,39 @@ func TestEstimateDecodeBytes(t *testing.T) {RegisterT(t)// Invalid or over the per-side cap- Expect(validate.EstimateDecodeBytes(0, 100, "png", color.GrayModel)).Equals(int64(-1))- Expect(validate.EstimateDecodeBytes(100, 0, "png", color.GrayModel)).Equals(int64(-1))- Expect(validate.EstimateDecodeBytes(-100, -100, "png", color.GrayModel)).Equals(int64(-1))- Expect(validate.EstimateDecodeBytes(validate.MaxImageSide+1, 1, "png", color.GrayModel)).Equals(int64(-1))- Expect(validate.EstimateDecodeBytes(1, validate.MaxImageSide+1, "png", color.GrayModel)).Equals(int64(-1))- Expect(validate.EstimateDecodeBytes(40_000_000, 1, "gif", color.Palette{})).Equals(int64(-1))- Expect(validate.EstimateDecodeBytes(math.MaxInt, math.MaxInt, "png", color.GrayModel)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(0, 100, "png", color.GrayModel, false)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(100, 0, "png", color.GrayModel, false)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(-100, -100, "png", color.GrayModel, false)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(validate.MaxImageSide+1, 1, "png", color.GrayModel, false)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(1, validate.MaxImageSide+1, "png", color.GrayModel, false)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(40_000_000, 1, "gif", color.Palette{}, false)).Equals(int64(-1))+ Expect(validate.EstimateDecodeBytes(math.MaxInt, math.MaxInt, "png", color.GrayModel, false)).Equals(int64(-1))// Bytes per pixel include 4 bytes for a full-size RGBA intermediateconst mp = 1000 * 1000- Expect(validate.EstimateDecodeBytes(1000, 1000, "gif", color.Palette{})).Equals(int64(5 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Palette{})).Equals(int64(6 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.GrayModel)).Equals(int64(6 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Gray16Model)).Equals(int64(8 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBAModel)).Equals(int64(12 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBA64Model)).Equals(int64(20 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.GrayModel)).Equals(int64(9 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.YCbCrModel)).Equals(int64(19 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.RGBAModel)).Equals(int64(23 * mp))- Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.CMYKModel)).Equals(int64(24 * mp))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "gif", color.Palette{}, false)).Equals(int64(5 * mp))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Palette{}, false)).Equals(int64(6 * mp))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.GrayModel, false)).Equals(int64(6 * mp))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.Gray16Model, false)).Equals(int64(8 * mp))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBAModel, false)).Equals(int64(12 * mp))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBA64Model, false)).Equals(int64(20 * mp))+ // JPEG images are allocated in whole MCUs (up to 32x32), so 1000 is rounded up to 1024+ const jpegPixels = 1024 * 1024+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.GrayModel, false)).Equals(int64(5 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.YCbCrModel, false)).Equals(int64(7 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.RGBAModel, false)).Equals(int64(11 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.CMYKModel, false)).Equals(int64(12 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.GrayModel, true)).Equals(int64(9 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.YCbCrModel, true)).Equals(int64(19 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.RGBAModel, true)).Equals(int64(23 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1000, 1000, "jpeg", color.CMYKModel, true)).Equals(int64(28 * jpegPixels))+ Expect(validate.EstimateDecodeBytes(1024, 1024, "jpeg", color.YCbCrModel, false)).Equals(int64(7 * jpegPixels))++ // progressive is ignored for other formats+ Expect(validate.EstimateDecodeBytes(1000, 1000, "png", color.NRGBAModel, true)).Equals(int64(12 * mp))// Worst case at the per-side cap doesn't overflow- Expect(validate.EstimateDecodeBytes(validate.MaxImageSide, validate.MaxImageSide, "jpeg", color.CMYKModel) > 0).IsTrue()+ Expect(validate.EstimateDecodeBytes(validate.MaxImageSide, validate.MaxImageSide, "jpeg", color.CMYKModel, true) > 0).IsTrue()}func TestCheckDecodeBudget(t *testing.T) {@@ -366,10 +382,14 @@ func TestCheckDecodeBudget(t *testing.T) {Expect(validate.CheckDecodeBudget(mock.UniformPNG(100000, 1))).Equals(validate.ErrImageTooLarge)Expect(validate.CheckDecodeBudget(mock.GIFHeader(40000, 40000))).Equals(validate.ErrImageTooLarge)- // 12MP phone photo: 12.2MP * 19 bytes = 232MB- Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 4032, 3024))).IsNil()- // 16MP: 304MB- Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 4000, 4000))).Equals(validate.ErrImageTooLarge)+ // 24MP baseline colour JPEG: 24MP * 7 bytes = 168MB+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 6000, 4000, false))).IsNil()+ // Same dimensions, but progressive: 24MP * 19 bytes = 457MB+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 6000, 4000, true))).Equals(validate.ErrImageTooLarge)+ // 12MP progressive phone photo: 12.2MP * 19 bytes = 233MB+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 4032, 3024, true))).IsNil()+ // 40MP baseline: 280MB+ Expect(validate.CheckDecodeBudget(jpegHeaderWithDimensions(t, 8000, 5000, false))).Equals(validate.ErrImageTooLarge)img, _ := os.ReadFile(env.Path("/app/pkg/web/testdata/logo2.jpg"))Expect(validate.CheckDecodeBudget(img)).IsNil()