From b844e0c1afe2a4e22fe66ebbca00a84863c2f7a6 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 30 Aug 2026 19:32:18 +0200 Subject: [PATCH 1/2] Walk a page's resources once, and charge for a picture before making it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit render.Images(d, 1) on openpdf's pdfsmartcopy_bec.pdf — 208 KB, three pages — was killed at 87 GB resident and still climbing. THE CAUSE. A page's resources are a GRAPH, not a tree, and imagesIn walked every path through it. In that file the page's /Resources names 37 form XObjects and every one of them names that same /Resources dictionary straight back, so the walk visits 37 dictionaries at the first level, 1 369 at the second, 50 653 at the third — 37 to the eighth at the depth limit of 8, which is three and a half million million — and decodes each of the page's 40 pictures once per visit, four bytes a pixel. Depth three alone is 156 GB of pixels. That is also the whole of the neighbouring finding. The three pictures of fr-cerfa/cerfa_10103.pdf were not decoded 511 times because they are DRAWN 511 times: that page's forms fan out by two, and the sum of two to the power nought through eight is 511. Measured here: 1 022 visits for 2 distinct pictures, over resource dictionaries counted 1, 2, 4, 8, 16, 32, 64, 128, 256 by depth. THE FIX. Enter each XObject once, by its reference. bec.pdf now hands back 74 pictures a page and cerfa_10103.pdf 3, immediately, and the same picture reached by two forms comes back once rather than as two identical entries under the same name. THE BOUND. Walked once, a document may still name more picture than the machine will hold, and it NAMES it rather than carrying it: decodeBase makes four bytes for every pixel of the declared /Width by /Height whatever the stream holds. So the declared size is charged against a budget BEFORE the picture is decoded, and a page past it is refused whole with ErrTooMuchToDecode naming the picture and the limit. Masks are charged too — a mask is a picture in its own right. WHERE THE LIMIT COMES FROM. Over the 2 268 real forms of the corpus, 10 659 pages, 2 111 of which draw a picture at all: the page naming the most comes to 31 814 093 pixels, 127 MB decoded; the median is 90 048; the 99.9th centile is the maximum. Not one page names as much as a sixth of the 256 Mi-pixel limit, which is a gigabyte of RGBA. Page is untouched and was never affected: it is bounded by maxOperations, and all three pages of bec.pdf draw. --- imagebudget_test.go | 257 ++++++++++++++++++++++++++++++++++++++++++++ images.go | 121 ++++++++++++++++++++- state.go | 10 ++ 3 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 imagebudget_test.go diff --git a/imagebudget_test.go b/imagebudget_test.go new file mode 100644 index 0000000..bc4f4b5 --- /dev/null +++ b/imagebudget_test.go @@ -0,0 +1,257 @@ +// Copyright (c) 2026, the go-pdfkit/render authors +// All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +package render + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/go-pdfkit/reader" +) + +// sharedResourcesPage builds the shape that took 87 GB: a page whose +// /Resources names a number of form XObjects, every one of which names that +// same /Resources dictionary straight back, and a number of pictures beside +// them. +// +// It is openpdf's pdfsmartcopy_bec.pdf reduced — 208 KB, three pages, 37 forms +// and 40 pictures per page, and every form pointing home. Nothing about the +// pictures matters to the blow-up; the fan-out does all of it. Walked as a +// tree, forms=4 reaches 4^8 = 65 536 dictionaries at the depth limit and +// decodes every picture in each of them, which for the real file is +// 37^8 ≈ 3.5e12 visits and 40 pictures apiece. +func sharedResourcesPage(t *testing.T, forms, pictures int, pic func(*reader.Writer) reader.Object) *reader.Document { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + resRef := w.Reserve() + xobj := reader.Dict{} + for i := 0; i < pictures; i++ { + xobj[reader.Name(fmt.Sprintf("Im%d", i))] = pic(w) + } + for i := 0; i < forms; i++ { + xobj[reader.Name(fmt.Sprintf("Tr%d", i))] = w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + // The whole of the defect is on this line: the form hands the walk + // back the dictionary the walk is already in. + "Resources": resRef, + }, Raw: []byte("")}) + } + w.Put(resRef, reader.Dict{"XObject": xobj}) + page := w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), reader.Integer(20), reader.Integer(20)}, + "Contents": w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte("")}), + "Resources": resRef, + }) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + out, err := w.Finish(reader.Dict{"Root": w.Add(reader.Dict{ + "Type": reader.Name("Catalog"), "Pages": pagesRef})}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + return d +} + +// hugePicture declares a picture of the largest size a single one may be, in a +// format nothing here reads, so that what is charged for it can be watched +// without a quarter of a gigabyte actually being made. +func hugePicture(w *reader.Writer) reader.Object { + return w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(8192), "Height": reader.Integer(8192), + "Filter": reader.Name("JPXDecode"), + }, Raw: []byte{0xff, 0x4f, 0xff, 0x51, 0, 1}}) +} + +func TestASharedResourceDictionaryIsWalkedOnce(t *testing.T) { + // The cause. A page's resources are a graph, not a tree, and walking every + // path through one visits fan-out to the power of the depth limit and + // decodes every picture that many times. It is why cerfa_10103.pdf decodes + // its two pictures 511 times — its forms fan out by two, and the sum of + // two to the power nought through eight is 511 — and why the three-page + // pdfsmartcopy_bec.pdf, 208 KB on disk, was still allocating at 87 GB. + d := sharedResourcesPage(t, 4, 3, greyImage) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + // Three pictures, once each. Walked as a tree it is 3 x (4^0 + ... + 4^8), + // which is 262 143. + if len(got) != 3 { + t.Fatalf("%d pictures came back for a page holding three", len(got)) + } + for i, im := range got { + if want := fmt.Sprintf("Im%d", i); im.Name != want { + t.Errorf("picture %d is named %q, want %q", i, im.Name, want) + } + if im.Pic.W != 2 || im.Pic.H != 1 { + t.Errorf("picture %d came back %dx%d", i, im.Pic.W, im.Pic.H) + } + } +} + +func TestImagesRefusesAPageNamingMorePictureThanItWillDecode(t *testing.T) { + // The bound. Walked once, a document may still name more picture than the + // machine will hold — and it names it rather than carrying it, since a + // picture costs four bytes for every pixel of its declared size whatever + // its stream turns out to hold. So the size is charged BEFORE the picture + // is decoded: five of the largest a single picture may be come to more + // than the gigabyte this will decode. + d := sharedResourcesPage(t, 4, 5, hugePicture) + got, err := Images(d, 1) + if !errors.Is(err, ErrTooMuchToDecode) { + t.Fatalf("a page naming %d pixels came back with err=%v and %d pictures", + 5*8192*8192, err, len(got)) + } + if got != nil { + t.Errorf("%d pictures came back with the refusal", len(got)) + } + // The refusal has to say what was exceeded, or it cannot be acted on. + for _, want := range []string{"8192", fmt.Sprint(maxImagesPixels)} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal %q does not name %s", err, want) + } + } +} + +func TestAPictureOfAnImpossibleSizeIsRefusedBeforeItIsMade(t *testing.T) { + // A file may declare a width that no arithmetic survives. It is refused on + // either side alone, before the two are multiplied. + for _, tc := range []struct { + name string + w, h int64 + panic bool + }{ + {name: "a width past the whole budget", w: 1 << 40, h: 1}, + {name: "a height past the whole budget", w: 1, h: 1 << 40}, + } { + t.Run(tc.name, func(t *testing.T) { + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"I": w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(tc.w), "Height": reader.Integer(tc.h), + "ColorSpace": reader.Name("DeviceGray"), "BitsPerComponent": reader.Integer(8), + }, Raw: []byte{0x00}})}} + }) + if _, err := Images(d, 1); !errors.Is(err, ErrTooMuchToDecode) { + t.Fatalf("a picture of %d by %d came back with %v", tc.w, tc.h, err) + } + }) + } +} + +func TestAMaskIsChargedForToo(t *testing.T) { + // A mask is a picture in its own right and is decoded like one, so it is + // charged like one. Refusing the picture and then making its mask anyway + // would leave the largest thing on the page unbounded. + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + mask := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(1 << 40), "Height": reader.Integer(1), + "ColorSpace": reader.Name("DeviceGray"), "BitsPerComponent": reader.Integer(8), + }, Raw: []byte{0x00}}) + return reader.Dict{"XObject": reader.Dict{"I": w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(2), "Height": reader.Integer(1), + "ColorSpace": reader.Name("DeviceGray"), "BitsPerComponent": reader.Integer(8), + "SMask": mask, + }, Raw: []byte{0x00, 0xff}})}} + }) + if _, err := Images(d, 1); !errors.Is(err, ErrTooMuchToDecode) { + t.Fatalf("a mask of a million million pixels came back with %v", err) + } +} + +func TestARefusalInsideAFormStopsTheWalk(t *testing.T) { + // The picture that breaks the budget may be several forms down, and the + // walk has to come back up rather than carry on with the next form. + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + inner := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"XObject": reader.Dict{"Deep": hugePicture(w)}}, + }, Raw: []byte("")}) + xo := reader.Dict{"F": inner} + // Four more of the largest a picture may be, named so they are walked + // before the form: the budget is gone by the time the form is reached. + for i := 0; i < 4; i++ { + xo[reader.Name(fmt.Sprintf("A%d", i))] = hugePicture(w) + } + return reader.Dict{"XObject": xo} + }) + if _, err := Images(d, 1); !errors.Is(err, ErrTooMuchToDecode) { + t.Fatalf("came back with %v", err) + } +} + +func TestAnOrdinaryPageIsNotRefused(t *testing.T) { + // A bound that refused real documents would be worse than none. Not one + // page of the 10 659 in the corpus names as much as a sixth of it. + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"I": greyImage(w)}} + }) + got, err := Images(d, 1) + if err != nil { + t.Fatalf("an ordinary page was refused: %v", err) + } + if len(got) != 1 { + t.Fatalf("%d pictures came back", len(got)) + } +} + +func TestFormsMayStillNestAsDeepAsTheyMay(t *testing.T) { + // Entering each XObject once is what stops the walk going round; the depth + // limit is what stops it going down. A chain of forms that are all + // different is not a cycle, and it still has to end somewhere. + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + // The deepest form holds the picture, and the chain is built upwards + // so that the picture sits one level past the limit. + deepest := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"XObject": reader.Dict{"TooDeep": greyImage(w)}}, + }, Raw: []byte("")}) + for i := 0; i < maxImageDepth; i++ { + deepest = w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"XObject": reader.Dict{"F": deepest}}, + }, Raw: []byte("")}) + } + return reader.Dict{"XObject": reader.Dict{"F": deepest}} + }) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("a picture %d forms down came back: %+v", maxImageDepth+1, got) + } +} + +func TestAStreamWrittenIntoTheDictionaryIsNotRemembered(t *testing.T) { + // Only an indirect reference can be reached twice; a stream written into + // the dictionary itself has one way in, and there is nothing to remember. + r := &renderer{seen: map[reader.Ref]bool{}} + if !r.firstVisit(reader.Integer(3)) { + t.Error("something that is not a reference was taken for one already seen") + } + if len(r.seen) != 0 { + t.Errorf("it was remembered anyway: %v", r.seen) + } + ref := reader.Ref{Num: 7} + if !r.firstVisit(ref) { + t.Error("a reference not seen before was called seen") + } + if r.firstVisit(ref) { + t.Error("a reference was entered twice") + } +} diff --git a/images.go b/images.go index 0a821d7..7b966c7 100644 --- a/images.go +++ b/images.go @@ -6,6 +6,8 @@ package render import ( + "errors" + "fmt" "sort" "github.com/go-gfx/gfx/raster" @@ -64,21 +66,53 @@ type Image struct { // which is the same answer [Page] gives by not drawing it. Inline images — // the ones written into the content stream — are not returned: they belong to // the stream that draws them rather than to the page's resources. +// +// Each picture comes back once, however many of the page's forms reach it, and +// a page that names more picture than [maxImagesPixels] is refused whole with +// [ErrTooMuchToDecode] rather than decoded until the machine gives out. func Images(d *reader.Document, i int) ([]Image, error) { page, err := d.Page(i) if err != nil { return nil, err } - r := &renderer{doc: d, fonts: map[int]*pdfFont{}, softMasks: map[softMaskKey][]uint8{}} + r := &renderer{ + doc: d, + fonts: map[int]*pdfFont{}, + softMasks: map[softMaskKey][]uint8{}, + seen: map[reader.Ref]bool{}, + budget: maxImagesPixels, + } res, _ := reader.ToDict(resolve(d, page.Get("Resources"))) - return r.imagesIn(res, 0), nil + out := r.imagesIn(res, 0) + if r.refused != nil { + return nil, r.refused + } + return out, nil } // maxImageDepth is how far a form XObject may nest before its pictures stop -// being counted. A form may hold a form, and a document may say it holds -// itself. +// being counted. A form may hold a form; this bounds how deep the walk goes, +// and firstVisit below is what stops it going round. const maxImageDepth = 8 +// maxImagesPixels is how many pixels one call to [Images] may decode between +// all the pictures a page reaches, mask included. A picture costs four bytes a +// pixel, so this is a gigabyte. +// +// # EXPERIMENT +// +// Over the 2 268 real forms of the corpus — 10 659 pages, of which 2 111 draw +// a picture at all — the page naming the most comes to 31 814 093 pixels, or +// 127 MB once decoded. The median is 90 048. Not one page in the corpus names +// as much as a sixth of this limit, and the document that started this named +// enough for 87 GB. +const maxImagesPixels = 256 << 20 + +// ErrTooMuchToDecode says a page names more picture than [Images] will decode +// at once. Nothing comes back with it: half the pictures of a page would be +// read as the whole of them by anything counting. +var ErrTooMuchToDecode = errors.New("render: the page names more picture than may be decoded at once") + // imagesIn collects the pictures one resource dictionary reaches, following // the forms it names. func (r *renderer) imagesIn(res reader.Dict, depth int) []Image { @@ -96,27 +130,101 @@ func (r *renderer) imagesIn(res reader.Dict, depth int) []Image { var out []Image for _, name := range names { - st, ok := reader.ToStream(resolve(r.doc, xo.Get(reader.Name(name)))) + entry := xo.Get(reader.Name(name)) + st, ok := reader.ToStream(resolve(r.doc, entry)) if !ok { continue } + if !r.firstVisit(entry) { + continue + } sub, _ := reader.ToName(resolve(r.doc, st.Dict.Get("Subtype"))) if sub == "Form" { inner, _ := reader.ToDict(resolve(r.doc, st.Dict.Get("Resources"))) out = append(out, r.imagesIn(inner, depth+1)...) + if r.refused != nil { + return out + } continue } if sub != "Image" { continue } out = append(out, r.decoded(name, st, res)...) + if r.refused != nil { + return out + } } return out } +// firstVisit reports whether an XObject has not been walked before, and +// remembers it if not. +// +// A page's resources are a GRAPH and not a tree. In openpdf's +// pdfsmartcopy_bec.pdf the page names 37 form XObjects and every one of them +// names the page's own /Resources dictionary straight back, so a walk that +// descends every path visits 37 dictionaries at the first level, 1 369 at the +// second, 50 653 at the third — 37 to the eighth at the depth limit, which is +// three and a half million million — and decodes each of the page's 40 +// pictures once per visit. That is where 87 GB came from, and it is the same +// arithmetic that decodes each of the two pictures of the French form +// cerfa_10103.pdf 511 times: that page's forms fan out by two, and the sum of +// two to the power nought through eight is 511. +// +// Entering each XObject once turns that back into one visit per object, which +// is what the file holds. It also means a picture two forms both reach comes +// back once rather than twice, which is the answer wanted anyway: the two +// entries would be the same stream decoded twice under the same name. +func (r *renderer) firstVisit(entry reader.Object) bool { + ref, ok := entry.(reader.Ref) + if !ok { + // A stream written into the dictionary rather than referred to cannot + // be reached a second way, so there is nothing to remember. + return true + } + if r.seen[ref] { + return false + } + r.seen[ref] = true + return true +} + +// afford takes the pixels a picture declares out of the budget, and reports +// whether there were enough. +// +// It is asked BEFORE the picture is decoded, because a limit noticed after the +// allocation has not helped: decodeBase makes four bytes for every pixel of +// the declared /Width by /Height whatever the stream turns out to hold, so a +// document names the memory rather than carrying it — a 208 KB file names 87 +// GB. What is charged is therefore what the file declares and not what the +// codec produced, and a picture charged for and then found undecodable is not +// refunded. That is the conservative direction and the only one that can be +// checked in time. +func (r *renderer) afford(dict reader.Dict) bool { + w := intOr(resolve(r.doc, dict.Get("Width")), 0) + h := intOr(resolve(r.doc, dict.Get("Height")), 0) + if w <= 0 || h <= 0 { + // decodeBase hands this one back as nothing without allocating. + return true + } + // Either side on its own past the whole budget is refused before the two + // are multiplied, since a file may declare a width that overflows. + if w > maxImagesPixels || h > maxImagesPixels || w*h > int64(r.budget) { + r.refused = fmt.Errorf("%w: a picture of %d by %d pixels, with %d of the %d pixels left", + ErrTooMuchToDecode, w, h, r.budget, maxImagesPixels) + return false + } + r.budget -= int(w * h) + return true +} + // decoded reads one image XObject and the mask it names, if any. func (r *renderer) decoded(name string, st *reader.Stream, res reader.Dict) []Image { var out []Image + if !r.afford(st.Dict) { + return nil + } if s := r.decodeBase(st.Dict, st.Raw, res); s != nil { stencil, _ := reader.ToBool(resolve(r.doc, st.Dict.Get("ImageMask"))) out = append(out, Image{ @@ -135,6 +243,9 @@ func (r *renderer) decoded(name string, st *reader.Stream, res reader.Dict) []Im if !ok { continue } + if !r.afford(ms.Dict) { + return out + } s := r.decodeBase(ms.Dict, ms.Raw, res) if s == nil { continue diff --git a/state.go b/state.go index 9a6cd6b..fdf46c3 100644 --- a/state.go +++ b/state.go @@ -119,6 +119,16 @@ type renderer struct { // is hidden. Marked content does not span streams, so both are saved and // restored around a nested one. mc, hideAt int + + // seen is the XObjects a walk of the resources has already entered, and + // budget is how many pixels it may still decode. Both belong to [Images], + // which walks the resources itself rather than running a content stream + // and so is bounded by neither maxOperations nor the size of the page. + seen map[reader.Ref]bool + budget int + // refused is why the walk stopped, and is set instead of decoding the + // picture that would have gone past the budget. + refused error } // timeCheckEvery is how many operations pass between looks at the clock. A From 0e70f2b9c776632282b77a0248dd27e9b6c92bec Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 30 Aug 2026 19:40:37 +0200 Subject: [PATCH 2/2] Read what a codec says it holds before handing it the bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The budget of the commit before this one charges a picture for the /Width and /Height its DICTIONARY declares. A codestream carries its own size and need not agree, and where they disagree this package already prefers the codestream's — so the budget had a hole a small file goes straight through. image/jpeg makes the whole picture the moment it reaches the start of scan, before it looks at any scan data. A valid 8x8 JPEG of 376 bytes whose frame header is altered to claim 65 535 by 65 535 therefore allocates FOUR GIGABYTES and only then reports the scan data missing. Measured: totalAlloc delta 4 096 MB for a 376-byte input. The dictionary may say 8 by 8 and be charged 64 pixels for it. Both codecs will say how large they are from the header alone and without allocating — image/jpeg through jpeg.DecodeConfig, the JPEG 2000 reader through its own — so that is read first, and the bytes are handed over only if what they claim can be afforded. A codestream past the ceiling on a single picture is refused outright; one larger than its dictionary pays the difference into the budget, so that a page of pictures each declaring one pixel and each holding a large codestream cannot come to as much as it likes. The ceiling applies to Page too, which is the only change to Page here: it keeps no picture, so it is not charged, but it should not allocate four gigabytes for a 376-byte file either. A header nothing can be read from is left alone: the decoder gives up on it long before it allocates. The fixture is that patched JPEG, built in the test rather than carried. What it asserts is not that nothing came back — nothing came back before this too, once the decoder had allocated and then failed — but that the decoder is NEVER REACHED, which is the whole of the claim that the check happens before the allocation. --- image.go | 79 +++++++++++++++++-- imagebudget_test.go | 186 ++++++++++++++++++++++++++++++++++++++++++++ images.go | 1 + state.go | 3 + 4 files changed, 264 insertions(+), 5 deletions(-) diff --git a/image.go b/image.go index 4c799ec..8bb61d7 100644 --- a/image.go +++ b/image.go @@ -2,9 +2,10 @@ package render import ( "bytes" + "fmt" "image" "image/color" - _ "image/jpeg" // the one image format a PDF may carry undecoded + "image/jpeg" // the one image format a PDF may carry undecoded "math" jpeg2000 "github.com/ajroetker/go-jpeg2000" @@ -170,9 +171,9 @@ func (r *renderer) decodeBase(dict reader.Dict, raw []byte, resources reader.Dic case "": out = r.samples(dict, data, w, h, resources) case "DCTDecode", "DCT": - out = decodeJPEG(data, w, h, r.decodeInverts(dict)) + out = r.decodeJPEG(data, w, h, r.decodeInverts(dict)) case "JPXDecode": - out = decodeJPX(data, w, h) + out = r.decodeJPX(data, w, h) } // No arm ran, or the one that ran could not read its bytes: the image is // not drawn rather than drawn wrong. Every filter the reader hands back @@ -297,7 +298,11 @@ func sampleAt(data []byte, rowStart, bitOffset, bpc int) uint32 { } // decodeJPEG reads the one compressed image format a PDF may carry whole. -func decodeJPEG(data []byte, w, h int, inverted bool) *sampled { +func (r *renderer) decodeJPEG(data []byte, w, h int, inverted bool) *sampled { + cw, ch := jpegSize(data) + if !r.affordDecoded(cw, ch, w*h) { + return nil + } img, err := jpegDecode(data) if err != nil { return nil @@ -321,7 +326,11 @@ func decodeJPEG(data []byte, w, h int, inverted bool) *sampled { // The size is taken from the picture rather than from the dictionary, as it is // for JPEG: a codestream carries its own, and where the two disagree the one // the pixels are actually in is the one that can be drawn. -func decodeJPX(data []byte, w, h int) *sampled { +func (r *renderer) decodeJPX(data []byte, w, h int) *sampled { + cw, ch := jpxSize(data) + if !r.affordDecoded(cw, ch, w*h) { + return nil + } img, err := jpxDecode(data) if err != nil || img == nil { return nil @@ -332,6 +341,66 @@ func decodeJPX(data []byte, w, h int) *sampled { return &sampled{w: w, h: h, pix: img.Pix} } +// affordDecoded reports whether a picture of cw by ch pixels may be made. +// +// A codec carries its own size and it need not be the one the dictionary +// declares, so the dictionary's is not enough to go on: image/jpeg makes the +// whole picture the moment it reaches the start of scan, so a 376-byte JPEG +// whose frame header claims 65 535 by 65 535 allocates four gigabytes and only +// then says the scan data is missing. Reading the header first costs nothing +// and allocates nothing, and it is the only place the question can be asked in +// time. +// +// charged is what the dictionary already paid for this picture, since [Images] +// charges the declared size before it gets here; a codestream claiming more +// than the dictionary pays the difference. [Page] keeps no picture and is not +// bounded that way, so its renderer spends nothing and only the ceiling on a +// single picture applies to it. +func (r *renderer) affordDecoded(cw, ch, charged int) bool { + if cw <= 0 || ch <= 0 { + // Nothing could be read from the header, so nothing will be made from + // the body either: the decoder gives up before it allocates. + return true + } + if cw > maxImagePixels || ch > maxImagePixels || cw*ch > maxImagePixels { + return false + } + if !r.bounded { + return true + } + extra := cw*ch - charged + if extra <= 0 { + return true + } + if extra > r.budget { + r.refused = fmt.Errorf("%w: a picture whose codestream holds %d by %d pixels, with %d of the %d pixels left", + ErrTooMuchToDecode, cw, ch, r.budget, maxImagesPixels) + return false + } + r.budget -= extra + return true +} + +// jpegSize is how large a JPEG says it is, read from its header alone. It is +// zero when nothing can be read, which is a decoder's problem and not a +// budget's. +var jpegSize = func(data []byte) (int, int) { + cfg, err := jpeg.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return 0, 0 + } + return cfg.Width, cfg.Height +} + +// jpxSize is the same question asked of a JPEG 2000 codestream. +var jpxSize = func(data []byte) (int, int) { + cfg, err := jpeg2000.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return 0, 0 + } + return cfg.Width, cfg.Height +} + // jpxDecode is a variable so a test can watch what happens when a decoder // refuses what it is given. var jpxDecode = func(data []byte) (*raster.Image, error) { diff --git a/imagebudget_test.go b/imagebudget_test.go index bc4f4b5..b1efbe9 100644 --- a/imagebudget_test.go +++ b/imagebudget_test.go @@ -6,11 +6,15 @@ package render import ( + "bytes" "errors" "fmt" + "image" + "image/jpeg" "strings" "testing" + "github.com/go-gfx/gfx/raster" "github.com/go-pdfkit/reader" ) @@ -255,3 +259,185 @@ func TestAStreamWrittenIntoTheDictionaryIsNotRemembered(t *testing.T) { t.Error("a reference was entered twice") } } + +// lyingJPEG is a real, valid JPEG of eight pixels by eight whose frame header +// has been altered to claim 65 535 by 65 535. +// +// It is 376 bytes and it makes image/jpeg allocate four gigabytes: the decoder +// makes the whole picture the moment it reaches the start of scan, and only +// then finds the scan data missing. Whatever /Width and /Height the PDF +// declares is beside the point — the codec believes its own header. +func lyingJPEG(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + if err := jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 8, 8)), nil); err != nil { + t.Fatal(err) + } + data := buf.Bytes() + for i := 0; i+8 < len(data); i++ { + if data[i] == 0xFF && data[i+1] == 0xC0 { + data[i+5], data[i+6] = 0xFF, 0xFF // height + data[i+7], data[i+8] = 0xFF, 0xFF // width + // The header has to say what it is going to make, or this fixture + // proves nothing. + cfg, err := jpeg.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width != 65535 || cfg.Height != 65535 { + t.Fatalf("the patched header reads back as %dx%d, %v", cfg.Width, cfg.Height, err) + } + return data + } + } + t.Fatal("no frame header to patch") + return nil +} + +func TestACodestreamIsNotBelievedBeforeItIsMeasured(t *testing.T) { + // The other way a small file names a great deal of memory, and the one + // that walks straight past a budget charged on the dictionary: the + // dictionary says eight pixels by eight and the codestream says 65 535 by + // 65 535, and it is the codestream the decoder allocates for. + data := lyingJPEG(t) + if len(data) > 1024 { + t.Fatalf("the fixture is %d bytes; it is meant to be tiny", len(data)) + } + d := jpegPage(t, data, nil) + + // The decoder must never be reached: it is the decoder that allocates, + // and it does so before it discovers there is no scan data. Nothing came + // back either way, both before this guard and after it, so what has to be + // asserted is that the four gigabytes were never asked for. + reached := false + was := jpegDecode + jpegDecode = func(b []byte) (image.Image, error) { + reached = true + return was(b) + } + defer func() { jpegDecode = was }() + + got, err := Images(d, 1) + if err != nil { + t.Errorf("unexpected error %v", err) + } + if len(got) != 0 { + t.Errorf("%d pictures came back from a codestream of 4 294 836 225 pixels", len(got)) + } + if reached { + t.Error("Images handed the bytes to the decoder anyway") + } + + // The page draws nothing rather than allocating four gigabytes for it. + // Page keeps no picture and so is not charged, but the ceiling on a single + // one applies to it all the same. + img, err := Page(d, 1, Options{Scale: 1}) + if err != nil { + t.Fatal(err) + } + if ink := inked(img); ink != 0 { + t.Errorf("%d pixels drawn from a codestream nothing may hold", ink) + } + if reached { + t.Error("Page handed the bytes to the decoder anyway") + } +} + +func TestWhatACodecSaysItHoldsIsPaidForToo(t *testing.T) { + // A ceiling on one picture is not a budget: a page of pictures each + // declaring one pixel and each holding a large codestream would come to + // as much as it liked. What the codec says it holds is charged for, less + // whatever the dictionary already paid. + const most = maxImagePixels + for _, tc := range []struct { + name string + cw, ch int + charged int + budget int + bounded bool + want bool + wantLeft int + wantRefusal bool + wantRefusalHas string + }{ + {name: "a header that says nothing is left to the decoder", + cw: 0, ch: 0, budget: 10, bounded: true, want: true, wantLeft: 10}, + {name: "a height that says nothing is left to the decoder", + cw: 4, ch: 0, budget: 10, bounded: true, want: true, wantLeft: 10}, + {name: "a codestream past the ceiling on one picture", + cw: 65535, ch: 65535, budget: most, bounded: true, want: false, wantLeft: most}, + {name: "a width alone past the ceiling", + cw: most + 1, ch: 1, budget: most, bounded: true, want: false, wantLeft: most}, + {name: "a height alone past the ceiling", + cw: 1, ch: most + 1, budget: most, bounded: true, want: false, wantLeft: most}, + {name: "a page keeps no picture and spends nothing", + cw: 100, ch: 100, budget: 0, bounded: false, want: true, wantLeft: 0}, + {name: "a codestream no larger than the dictionary is already paid for", + cw: 10, ch: 10, charged: 100, budget: 5, bounded: true, want: true, wantLeft: 5}, + {name: "a codestream larger than the dictionary pays the difference", + cw: 10, ch: 10, charged: 60, budget: 100, bounded: true, want: true, wantLeft: 60}, + {name: "and is refused when it cannot", + cw: 10, ch: 10, charged: 60, budget: 39, bounded: true, want: false, wantLeft: 39, + wantRefusal: true, wantRefusalHas: "39"}, + } { + t.Run(tc.name, func(t *testing.T) { + r := &renderer{budget: tc.budget, bounded: tc.bounded} + if got := r.affordDecoded(tc.cw, tc.ch, tc.charged); got != tc.want { + t.Errorf("a codestream of %d by %d with %d left: %v, want %v", + tc.cw, tc.ch, tc.budget, got, tc.want) + } + if r.budget != tc.wantLeft { + t.Errorf("%d pixels left, want %d", r.budget, tc.wantLeft) + } + switch { + case tc.wantRefusal && !errors.Is(r.refused, ErrTooMuchToDecode): + t.Errorf("refused with %v", r.refused) + case tc.wantRefusal && !strings.Contains(r.refused.Error(), tc.wantRefusalHas): + t.Errorf("the refusal %q does not name %s", r.refused, tc.wantRefusalHas) + case !tc.wantRefusal && r.refused != nil: + t.Errorf("refused with %v when it should not have", r.refused) + } + }) + } +} + +func TestACodestreamThatSaysNothingIsLeftToItsDecoder(t *testing.T) { + // A header nothing can be read from is a decoder's problem, not a + // budget's: it gives up long before it allocates. Both codecs are asked + // the same way and both are asked something they cannot read. + if w, h := jpegSize([]byte("not a JPEG")); w != 0 || h != 0 { + t.Errorf("a JPEG header read out of nothing as %dx%d", w, h) + } + if w, h := jpxSize([]byte("not a codestream")); w != 0 || h != 0 { + t.Errorf("a JPEG 2000 header read out of nothing as %dx%d", w, h) + } + // And a real one is read. + if w, h := jpxSize(jpxImage(t, 6, 4)); w != 6 || h != 4 { + t.Errorf("a real codestream of 6x4 read as %dx%d", w, h) + } +} + +func TestAJPXCodestreamIsMeasuredBeforeItIsDecoded(t *testing.T) { + // The same guard on the other codec. jpxSize is a variable so that a + // header claiming more than may be held can be put behind it without + // having to encode four gigabytes of picture to say so. + wasSize := jpxSize + jpxSize = func([]byte) (int, int) { return 100000, 100000 } + defer func() { jpxSize = wasSize }() + reached := false + wasDecode := jpxDecode + jpxDecode = func(b []byte) (*raster.Image, error) { + reached = true + return wasDecode(b) + } + defer func() { jpxDecode = wasDecode }() + + d := jpxPage(t, jpxImage(t, 6, 4), 6, 4) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("%d pictures came back from a codestream of ten thousand million pixels", len(got)) + } + if reached { + t.Error("the bytes were handed to the decoder anyway") + } +} diff --git a/images.go b/images.go index 7b966c7..df1dddf 100644 --- a/images.go +++ b/images.go @@ -81,6 +81,7 @@ func Images(d *reader.Document, i int) ([]Image, error) { softMasks: map[softMaskKey][]uint8{}, seen: map[reader.Ref]bool{}, budget: maxImagesPixels, + bounded: true, } res, _ := reader.ToDict(resolve(d, page.Get("Resources"))) out := r.imagesIn(res, 0) diff --git a/state.go b/state.go index fdf46c3..02b8401 100644 --- a/state.go +++ b/state.go @@ -126,6 +126,9 @@ type renderer struct { // and so is bounded by neither maxOperations nor the size of the page. seen map[reader.Ref]bool budget int + // bounded says a budget is being kept at all. Page keeps no picture — it + // draws each one and lets it go — so only Images spends. + bounded bool // refused is why the walk stopped, and is set instead of decoding the // picture that would have gone past the budget. refused error