From 1a200d12b748fc4ff7634ac7ab07edb24f5b21b4 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 28 Aug 2026 22:28:42 +0200 Subject: [PATCH 1/2] Take the JBIG2 decoder that reads the last seven streams go-gfx/gfx v0.15.0 takes its JBIG2 decoder from a fork carrying one fix, offered upstream as dkrisman/gobig2#2: the per-symbol pixel cap defaulted to 4 megapixels, below what real scanned documents contain, and a library could not raise it because the limits are process-global variables. Of 403 JBIG2 streams taken from the /Mask and /SMask entries of public Internet Archive scans, 7 were refused at that default. With the fix all 403 decode, every one bit-exact with poppler's own decoder. The dependency also stops being untagged. --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a130ab1..8c2d1a2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/go-pdfkit/render go 1.26.4 require ( - github.com/go-gfx/gfx v0.14.0 + github.com/go-gfx/gfx v0.15.0 github.com/go-opentype/fonts v0.9.0 github.com/go-opentype/opentype v0.10.0 github.com/go-pdfkit/reader v0.6.0 @@ -16,9 +16,9 @@ require ( require ( github.com/ajroetker/go-highway v0.0.4 // indirect - github.com/dkrisman/gobig2 v0.0.0-20260513123937-51e39052fde6 // indirect github.com/sergeymakinen/go-bmp v1.0.0 // indirect github.com/sergeymakinen/go-ico v1.0.0 // indirect + github.com/tannevaled/gobig2 v0.1.0 // indirect golang.org/x/image v0.45.0 // indirect golang.org/x/sys v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index d3e626e..425587a 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,8 @@ github.com/ajroetker/go-highway v0.0.4 h1:RDQo+9OhTXI6BFctLo+5gYpHNbb92VYJ0ObnR4 github.com/ajroetker/go-highway v0.0.4/go.mod h1:C/zYPNSSpOaraejY89FUTZTyQNEhi5+rEbU0LjlqJeU= github.com/ajroetker/go-jpeg2000 v0.0.2 h1:ni8brffZrci4Kacx3nM5d92ipmTDfak84KgHYi6IxFw= github.com/ajroetker/go-jpeg2000 v0.0.2/go.mod h1:7ld88W47lZy0x8gRQesRGAonDPOpr6ev8rckjCAfbzE= -github.com/dkrisman/gobig2 v0.0.0-20260513123937-51e39052fde6 h1:hHPgbpPdcaSXtpLb9NMEUJPxsZ0cxGtGHwmvKGI36NA= -github.com/dkrisman/gobig2 v0.0.0-20260513123937-51e39052fde6/go.mod h1:2Ij0rpAVy1tZ+PQ9FM1jCpnVo/OOCMCD6rLtSp4JkEI= -github.com/go-gfx/gfx v0.14.0 h1:G6A0ZIirv9XjlxVFOl3HLscLRJOdhe9e1HqiCK5CHz0= -github.com/go-gfx/gfx v0.14.0/go.mod h1:8KcjwJmHW2ImXcjeMEIQ1bY+7Q2DNcMYoQCl270Isr8= +github.com/go-gfx/gfx v0.15.0 h1:ol+4uqUTq8s23cGSjfbEHciTxkrZ7kTrpUuAPPz4XuY= +github.com/go-gfx/gfx v0.15.0/go.mod h1:VAK6hgCgkhT3j3ek2K7G8zDfkCPD3RCF45UbcRlCV8Q= github.com/go-opentype/fonts v0.9.0 h1:slB6OB3riLyUPrOxqXe0s6/AzdenF1TDvCN8N87hhQk= github.com/go-opentype/fonts v0.9.0/go.mod h1:C6yQL2apHItfEZ5hztpsHF0S5mlX/hklLlq/Z5fRG/g= github.com/go-opentype/opentype v0.10.0 h1:cZVMZ3RVkcijXmxlmpqyVkjlNc33aSB2ugKVWYvoLUg= @@ -18,6 +16,8 @@ github.com/sergeymakinen/go-bmp v1.0.0 h1:SdGTzp9WvCV0A1V0mBeaS7kQAwNLdVJbmHlqNW github.com/sergeymakinen/go-bmp v1.0.0/go.mod h1:/mxlAQZRLxSvJFNIEGGLBE/m40f3ZnUifpgVDlcUIEY= github.com/sergeymakinen/go-ico v1.0.0 h1:uL3khgvKkY6WfAetA+RqsguClBuu7HpvBB/nq/Jvr80= github.com/sergeymakinen/go-ico v1.0.0/go.mod h1:wQ47mTczswBO5F0NoDt7O0IXgnV4Xy3ojrroMQzyhUk= +github.com/tannevaled/gobig2 v0.1.0 h1:9PdMvmnmYQURlUF40zt8t35Wnrz3KhZrwfxgCMvQc04= +github.com/tannevaled/gobig2 v0.1.0/go.mod h1:X0S1H+N35kg6zYgVYRqGMsGj3C35zs+iA2r1MXyPcV4= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= From c0c364308fbe82edc32bc01ab15b035d701dac00 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sat, 29 Aug 2026 11:13:32 +0200 Subject: [PATCH 2/2] Hand back the pictures a page draws, one by one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This package could draw a page and nothing else. A page is the wrong unit for two jobs. Getting the pictures out of a document is one, and it is what pdfimages is for. Finding out whether a codec is right is the other, and it is why this is here now. A page is a composition: one image decoded wrongly is averaged away by everything drawn over and around it, so a per-page difference of a few percent says nothing about WHICH image was wrong. That is not hypothetical. v0.12.0 went out drawing scanned pages dark because what was measured was "ink appeared" rather than "the right ink", and a per-page number is what let it through. Images returns each picture with the name the page draws it by, the image filter it was stored in — which the pixels cannot tell you and a codec comparison must know — and whether it is a stencil, whose shape comes back without a colour because the colour is not the image's to know. Forms are followed, as pdfimages follows them: a picture reachable only through one is still a picture the page draws. A document may say a form holds itself, so the walk stops after eight levels. The names are walked in order. A map hands its keys back differently every run, and a list of pictures that reorders itself is not a measurement — that has already cost this repository a day, in a filter table that answered five different ways in eight runs. 100% statement coverage, go vet and -race clean, nine cross-compile targets. --- images.go | 111 +++++++++++++++++++++++++ images_test.go | 222 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 333 insertions(+) create mode 100644 images.go create mode 100644 images_test.go diff --git a/images.go b/images.go new file mode 100644 index 0000000..982c464 --- /dev/null +++ b/images.go @@ -0,0 +1,111 @@ +// Copyright (c) 2026, the go-pdfkit/render authors +// All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +package render + +import ( + "sort" + + "github.com/go-gfx/gfx/raster" + "github.com/go-pdfkit/reader" +) + +// Image is one picture a page draws, decoded. +type Image struct { + // Name is the resource name the page draws it by. + Name string + // Filter is the image format the picture was stored in — DCTDecode, + // JPXDecode, JBIG2Decode, CCITTFaxDecode — and is empty when the bytes + // were samples that no image codec had to read. + Filter string + // Stencil says the picture is a one-bit mask, which carries no colours of + // its own: it paints whatever colour is in force through its own shape. + // Its Pic has that shape in the alpha channel and black everywhere else, + // because the colour is not the image's to know. + Stencil bool + // Pic is the decoded picture. + Pic *raster.Image +} + +// Images decodes the pictures the i'th page draws, counting from one, in the +// order of the names it draws them by. +// +// It exists because a page is the wrong unit for two jobs. Getting the +// pictures out of a document is one. Finding out whether a codec is right is +// the other: a page is a composition, so one image decoded wrongly can be +// averaged away by everything drawn over and around it, and a per-page +// difference of a few percent says nothing about which image was wrong. That +// mattered here — a release went out drawing scanned pages dark because "ink +// appeared" was measured instead of "the right ink". +// +// A picture nothing here can decode is left out rather than returned empty, +// which is the same answer [Page] gives by not drawing it. Inline images — +// the ones written into the content stream — are not returned: they belong to +// the stream that draws them rather than to the page's resources. +func Images(d *reader.Document, i int) ([]Image, error) { + page, err := d.Page(i) + if err != nil { + return nil, err + } + r := &renderer{doc: d, fonts: map[int]*pdfFont{}, softMasks: map[softMaskKey][]uint8{}} + res, _ := reader.ToDict(resolve(d, page.Get("Resources"))) + return r.imagesIn(res, 0), nil +} + +// maxImageDepth is how far a form XObject may nest before its pictures stop +// being counted. A form may hold a form, and a document may say it holds +// itself. +const maxImageDepth = 8 + +// imagesIn collects the pictures one resource dictionary reaches, following +// the forms it names. +func (r *renderer) imagesIn(res reader.Dict, depth int) []Image { + if depth > maxImageDepth { + return nil + } + xo, _ := reader.ToDict(resolve(r.doc, res.Get("XObject"))) + names := make([]string, 0, len(xo)) + for name := range xo { + names = append(names, string(name)) + } + // A map hands its keys back in a different order every run, and a list of + // pictures that reorders itself is not a measurement. + sort.Strings(names) + + var out []Image + for _, name := range names { + st, ok := reader.ToStream(resolve(r.doc, xo.Get(reader.Name(name)))) + if !ok { + continue + } + sub, _ := reader.ToName(resolve(r.doc, st.Dict.Get("Subtype"))) + if sub == "Form" { + inner, _ := reader.ToDict(resolve(r.doc, st.Dict.Get("Resources"))) + out = append(out, r.imagesIn(inner, depth+1)...) + continue + } + if sub != "Image" { + continue + } + s := r.decodeImage(st.Dict, st.Raw, res) + if s == nil { + continue + } + stencil, _ := reader.ToBool(resolve(r.doc, st.Dict.Get("ImageMask"))) + out = append(out, Image{ + Name: name, + Filter: imageFilterOf(r.doc, st), + Stencil: bool(stencil), + Pic: &raster.Image{W: s.w, H: s.h, Pix: s.pix}, + }) + } + return out +} + +// imageFilterOf names the image format a stream's filter chain stopped at, and +// is empty when the chain ran to samples. +func imageFilterOf(d *reader.Document, st *reader.Stream) string { + return string(reader.DecodeRecovering(st.Dict, st.Raw, d.Resolver()).Image) +} diff --git a/images_test.go b/images_test.go new file mode 100644 index 0000000..7901ac5 --- /dev/null +++ b/images_test.go @@ -0,0 +1,222 @@ +// Copyright (c) 2026, the go-pdfkit/render authors +// All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +package render + +import ( + "testing" + + "github.com/go-pdfkit/reader" +) + +// pageWithResources builds a page whose resource dictionary is whatever the +// caller gives, so a test can put forms, nonsense and pictures side by side. +func pageWithResources(t *testing.T, build func(w *reader.Writer) reader.Dict) *reader.Document { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + res := build(w) + page := w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), reader.Integer(20), reader.Integer(20)}, + "Contents": w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte("")}), + "Resources": res, + }) + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": reader.Array{page}, "Count": reader.Integer(1)}) + out, err := w.Finish(reader.Dict{"Root": w.Add(reader.Dict{ + "Type": reader.Name("Catalog"), "Pages": pagesRef})}) + if err != nil { + t.Fatal(err) + } + d, err := reader.Open(out) + if err != nil { + t.Fatal(err) + } + return d +} + +// greyImage is a picture stored as plain samples, dark then light. +func greyImage(w *reader.Writer) reader.Object { + return w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(2), "Height": reader.Integer(1), + "ColorSpace": reader.Name("DeviceGray"), "BitsPerComponent": reader.Integer(8), + }, Raw: []byte{0x00, 0xff}}) +} + +func TestThePicturesAPageDrawsComeBackDecoded(t *testing.T) { + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"I": greyImage(w)}} + }) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("%d pictures, want 1", len(got)) + } + im := got[0] + if im.Name != "I" || im.Filter != "" || im.Stencil { + t.Errorf("name %q, filter %q, stencil %v", im.Name, im.Filter, im.Stencil) + } + if im.Pic.W != 2 || im.Pic.H != 1 { + t.Fatalf("decoded %dx%d, want 2x1", im.Pic.W, im.Pic.H) + } + // Dark then light, so a decoder handing back a flat or mirrored picture is + // caught and not merely one handing back nothing. + if im.Pic.Pix[0] >= 128 { + t.Errorf("the dark pixel came back at %d", im.Pic.Pix[0]) + } + if im.Pic.Pix[4] < 128 { + t.Errorf("the light pixel came back at %d", im.Pic.Pix[4]) + } +} + +func TestAPictureIsNamedByTheFilterItWasStoredIn(t *testing.T) { + // Which codec read a picture is the question a conformance run asks, and + // it cannot be answered from the pixels. + d := pageWithImage(t, reader.Dict{ + "Width": reader.Integer(16), "Height": reader.Integer(8), + "ColorSpace": reader.Name("DeviceGray"), "BitsPerComponent": reader.Integer(1), + "Filter": reader.Name("JBIG2Decode"), + }, jbig2Ink, "") + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("%d pictures, want 1", len(got)) + } + if got[0].Filter != "JBIG2Decode" { + t.Errorf("filter %q", got[0].Filter) + } +} + +func TestAStencilSaysSoAndCarriesOnlyItsShape(t *testing.T) { + // A stencil has no colours of its own: what comes back is the shape, and + // the colour belongs to whoever draws it. + d := pageWithImage(t, reader.Dict{ + "Width": reader.Integer(2), "Height": reader.Integer(1), + "ImageMask": reader.Bool(true), + }, []byte{0b01000000}, "") + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || !got[0].Stencil { + t.Fatalf("%d pictures, stencil %v", len(got), len(got) == 1 && got[0].Stencil) + } + if got[0].Pic.Pix[3] != 255 { + t.Errorf("the painted pixel is transparent") + } + if got[0].Pic.Pix[7] != 0 { + t.Errorf("the unpainted pixel is opaque") + } +} + +func TestThePicturesInsideAFormAreFound(t *testing.T) { + // A form is a page inside a page. pdfimages follows them, and a picture + // that is only reachable through one is still a picture the page draws. + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + inner := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"XObject": reader.Dict{"Deep": greyImage(w)}}, + }, Raw: []byte("")}) + return reader.Dict{"XObject": reader.Dict{"F": inner, "A": greyImage(w)}} + }) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("%d pictures, want 2", len(got)) + } + // A map hands its keys back in a different order every run. The names are + // walked in order, so "A" comes before the form named "F" whose picture is + // "Deep" — and the answer is the same every time, which a measurement + // needs and a map does not give. + if got[0].Name != "A" || got[1].Name != "Deep" { + t.Errorf("came back as %q then %q", got[0].Name, got[1].Name) + } +} + +func TestAFormThatHoldsItselfStops(t *testing.T) { + // A document may say a form holds itself, and following that is a way of + // never coming back. + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { + ref := w.Reserve() + w.Put(ref, &reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Form"), + "Resources": reader.Dict{"XObject": reader.Dict{ + "Loop": ref, "Pic": greyImage(w)}}, + }, Raw: []byte("")}) + return reader.Dict{"XObject": reader.Dict{"F": ref}} + }) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + // One picture per level it went down, and then it stopped. + if len(got) == 0 { + t.Fatal("the picture inside the form was not found at all") + } + if len(got) > maxImageDepth+1 { + t.Errorf("it went down %d levels", len(got)) + } +} + +func TestWhatIsNotAPictureIsNotReturned(t *testing.T) { + for _, tc := range []struct { + name string + res func(w *reader.Writer) reader.Dict + }{ + {"a page with no resources", func(*reader.Writer) reader.Dict { return nil }}, + {"resources naming no XObjects", func(*reader.Writer) reader.Dict { + return reader.Dict{"Font": reader.Dict{}} + }}, + {"an XObject entry that is not a dictionary", func(*reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Integer(3)} + }}, + {"an entry that is not a stream", func(*reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"X": reader.Integer(3)}} + }}, + {"a stream that is neither picture nor form", func(w *reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"X": w.Add(&reader.Stream{ + Dict: reader.Dict{"Subtype": reader.Name("PS")}, Raw: []byte("")})}} + }}, + {"a picture nothing here can decode", func(w *reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"X": w.Add(&reader.Stream{ + Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(4), "Height": reader.Integer(4), + "Filter": reader.Name("JPXDecode"), + }, Raw: []byte{0xff, 0x4f, 0xff, 0x51, 0, 1}})}} + }}, + {"a form with nothing in it", func(w *reader.Writer) reader.Dict { + return reader.Dict{"XObject": reader.Dict{"F": w.Add(&reader.Stream{ + Dict: reader.Dict{"Type": reader.Name("XObject"), + "Subtype": reader.Name("Form")}, Raw: []byte("")})}} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + d := pageWithResources(t, tc.res) + got, err := Images(d, 1) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("%d pictures came back", len(got)) + } + }) + } +} + +func TestAPageThatIsNotThereHasNoPictures(t *testing.T) { + d := pageWithResources(t, func(w *reader.Writer) reader.Dict { return nil }) + if _, err := Images(d, 9); err == nil { + t.Error("page nine of a one-page document came back without complaint") + } +}