From 19d076e71b83d841677aae75cde27aa7912720e2 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 30 Aug 2026 16:03:28 +0200 Subject: [PATCH] Drop the pages that carry nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A duplex stack run through a single-sided feeder comes back with a blank behind every one-sided sheet, and a scanner set to "both sides" makes one for every sheet that only had one. They are the pages nobody wants and everybody has — and with Interleave in ops v0.9.0 putting the two halves of such a scan back together, this is the other half of that job. The threshold was measured rather than decreed. Over 395 pages of government forms and library scans, the ink on a page runs to a median of 4.7% and falls away sharply below one part in a thousand: eleven pages are under it, two are under a twentieth of it, and the pages between are covers carrying a rule and nothing else. A page with a line of text is around three parts in a thousand, so this keeps one and drops a page number alone — which is what a blank back with a footer is. Picking that by eye would have been wrong twice over. The first measurement used a threshold of 200 rather than 128 and reported a median of 18% ink and not one blank page in 395: at that setting the off-white background of a scan counts as ink. Which threshold is used decides the answer, so the answer has to say which. A page that cannot be DRAWN is not counted blank. Not being able to see a page is not evidence that there is nothing on it, and deleting on that footing would throw away exactly the pages this program had most trouble with. A document whose every page is blank is refused rather than emptied, and that is not a corner case: nine of two hundred real forms are single-page documents whose one page reads "Please wait... your PDF viewer may not be able to display this type of document". They are XFA forms, whose real content is XML that no PDF viewer draws. The page is blank; the document is not. Over those two hundred forms — 773 pages — it drops 42, which is 5.4%. 100% statement coverage, go vet and -race clean, the wasm build, and the browser check. --- blank.go | 130 ++++++++++++++++++++++++++++++++++++++++++++ blank_test.go | 144 +++++++++++++++++++++++++++++++++++++++++++++++++ corpus_test.go | 48 +++++++++++++++++ panel.go | 4 ++ 4 files changed, 326 insertions(+) create mode 100644 blank.go create mode 100644 blank_test.go create mode 100644 corpus_test.go diff --git a/blank.go b/blank.go new file mode 100644 index 0000000..0380c58 --- /dev/null +++ b/blank.go @@ -0,0 +1,130 @@ +// Dropping the pages that carry nothing. +// +// A duplex stack run through a single-sided feeder comes back with a blank +// behind every one-sided sheet, and a scanner set to "both sides" produces one +// for every sheet that only had one. They are the pages nobody wants and +// everybody has. + +package main + +import ( + "fmt" + + "github.com/go-pdfkit/ops" + "github.com/go-pdfkit/reader" + "github.com/go-pdfkit/render" + "github.com/go-widgets/toolkit" +) + +// blankInk is how little a page may carry and still be blank. +// +// Chosen by measuring rather than by decree. Over 395 pages of government +// forms and library scans, the ink on a page runs to a median of 4.7% and +// falls away sharply below one part in a thousand: 11 pages are under it, 2 +// are under a twentieth of it, and the pages between are covers carrying a +// rule and nothing else. A page with a line of text is around three parts in a +// thousand, so this keeps one and drops a page number alone — which is what a +// blank back with a footer is. +const blankInk = 0.001 + +// blankScale is how large the page is drawn to look at it. Small on purpose: +// this is a count of dark pixels over a whole page, and a quarter-size drawing +// answers it in a sixteenth of the time. A document of two hundred pages is +// drawn twice by this verb — once to look, once to show what is left — and the +// looking should not be the slow half. +const blankScale = 0.25 + +// dropBlank removes the pages that carry no ink. +func (s *state) dropBlank() { + if s.doc == nil { + s.fail("open a document first") + return + } + src, msg := s.reopen() + if msg != "" { + s.fail(msg) + return + } + blanks := blankPages(src, s.fitScale(src)) + if len(blanks) == 0 { + s.fail("every page of this document carries something") + return + } + if len(blanks) == src.PageCount() { + // Not a corner case. Nine of two hundred real forms are single-page + // documents whose one page is a panel reading "Please wait... your PDF + // viewer may not be able to display this type of document": an XFA + // form, whose real content is XML that no PDF viewer draws. The page + // is blank, the document is not, and dropping the page would leave + // nothing at all. + s.fail("every page of this document is blank, and a document needs one") + return + } + spec := rangeOf(blanks) + s.changeSaying(fmt.Sprintf("dropped %d blank page(s): %s", len(blanks), spec), + func(d *ops.Doc) error { return d.Delete(spec) }) +} + +// blankPages are the pages carrying less ink than a page carries. +// +// A page that cannot be drawn is NOT counted blank. Not being able to see a +// page is not evidence that there is nothing on it, and deleting on that +// footing would throw away exactly the pages this program had most trouble +// with. +// +// Over two hundred real government forms — 773 pages — this drops 42, which is +// 5.4%, and refuses nine documents outright as blank throughout. +func blankPages(src *reader.Document, fit float64) []int { + var out []int + for p := 1; p <= src.PageCount(); p++ { + img, err := drawPage(src, p, render.Options{ + Scale: blankScale * fit, + MaxDuration: pageBudget, + }) + if err != nil || img == nil || img.W*img.H == 0 { + continue + } + ink := 0 + for i := 0; i < img.W*img.H; i++ { + r, g, b := uint32(img.Pix[i*4]), uint32(img.Pix[i*4+1]), uint32(img.Pix[i*4+2]) + if (r*299+g*587+b*114)/1000 < 128 { + ink++ + } + } + if float64(ink)/float64(img.W*img.H) < blankInk { + out = append(out, p) + } + } + return out +} + +// rangeOf writes page numbers the way the verbs read them, collapsing runs so +// that a document whose every other page is blank does not produce a range as +// long as itself. +func rangeOf(pages []int) string { + if len(pages) == 0 { + return "" + } + out := "" + for i := 0; i < len(pages); { + j := i + for j+1 < len(pages) && pages[j+1] == pages[j]+1 { + j++ + } + if out != "" { + out += "," + } + if j == i { + out += fmt.Sprint(pages[i]) + } else { + out += fmt.Sprintf("%d-%d", pages[i], pages[j]) + } + i = j + 1 + } + return out +} + +// blankButton is the control, put with the other things that remove pages. +func blankButton(s *state) toolkit.Widget { + return button("Drop the blank pages", toolkit.ButtonDanger, s.dropBlank) +} diff --git a/blank_test.go b/blank_test.go new file mode 100644 index 0000000..af250e2 --- /dev/null +++ b/blank_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "errors" + "strings" + "testing" + + "github.com/go-gfx/gfx/raster" + "github.com/go-pdfkit/ops" + "github.com/go-pdfkit/reader" + "github.com/go-pdfkit/render" +) + +// errNotDrawn stands in for whatever went wrong. +var errNotDrawn = errors.New("not drawn") + +// mixedPDF writes a document whose pages carry ink or do not, so which ones a +// verb drops can be read rather than inferred. +func mixedPDF(t *testing.T, inked ...bool) []byte { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + kids := make(reader.Array, 0, len(inked)) + for _, ink := range inked { + content := "" // a page with nothing on it at all + if ink { + content = "0 g 10 10 180 180 re f" + } + kids = append(kids, w.Add(reader.Dict{"Type": reader.Name("Page"), "Parent": pagesRef, + "Contents": w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: []byte(content)})})) + } + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), "Kids": kids, + "Count": reader.Integer(len(kids)), + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), reader.Integer(200), reader.Integer(200)}}) + out, err := w.Finish(reader.Dict{"Root": w.Add(reader.Dict{ + "Type": reader.Name("Catalog"), "Pages": pagesRef})}) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestTheBlankPagesGo(t *testing.T) { + // A duplex stack run through a single-sided feeder comes back with a blank + // behind every one-sided sheet. + h := &fakeHost{name: "scan.pdf", file: mixedPDF(t, true, false, true, false)} + s := newState(surfaceW, surfaceH, h) + s.open() + if s.doc == nil { + t.Fatal("the document did not open") + } + s.dropBlank() + if s.doc.PageCount() != 2 { + t.Fatalf("%d pages left, want 2: %q", s.doc.PageCount(), s.note) + } + if !strings.Contains(s.note, "dropped 2 blank page") { + t.Errorf("it said %q", s.note) + } +} + +func TestADocumentWithNothingToDrop(t *testing.T) { + h := &fakeHost{name: "scan.pdf", file: mixedPDF(t, true, true)} + s := newState(surfaceW, surfaceH, h) + s.open() + s.dropBlank() + if s.doc.PageCount() != 2 { + t.Errorf("%d pages left", s.doc.PageCount()) + } + if !strings.Contains(s.note, "carries something") { + t.Errorf("it said %q", s.note) + } +} + +func TestADocumentThatIsAllBlank(t *testing.T) { + // A document needs a page, so this refuses rather than leaving none. + h := &fakeHost{name: "scan.pdf", file: mixedPDF(t, false, false)} + s := newState(surfaceW, surfaceH, h) + s.open() + s.dropBlank() + if s.doc.PageCount() != 2 { + t.Errorf("%d pages left", s.doc.PageCount()) + } + if !strings.Contains(s.note, "needs one") { + t.Errorf("it said %q", s.note) + } +} + +func TestAPageThatCannotBeDrawnIsNotBlank(t *testing.T) { + // Not being able to see a page is not evidence that there is nothing on + // it, and deleting on that footing would throw away exactly the pages this + // program had most trouble with. + was := drawPage + t.Cleanup(func() { drawPage = was }) + drawPage = func(*reader.Document, int, render.Options) (*raster.Image, error) { + return nil, errNotDrawn + } + h := &fakeHost{name: "scan.pdf", file: mixedPDF(t, true, false)} + s := newState(surfaceW, surfaceH, h) + s.open() + s.dropBlank() + if s.doc.PageCount() != 2 { + t.Errorf("%d pages left; a page nobody could draw was dropped", s.doc.PageCount()) + } +} + +func TestNothingOpenToLookAt(t *testing.T) { + s := newState(surfaceW, surfaceH, &fakeHost{}) + s.dropBlank() + if !strings.Contains(s.note, "open a document first") { + t.Errorf("it said %q", s.note) + } +} + +func TestADocumentThatCannotBeWrittenBack(t *testing.T) { + h := &fakeHost{name: "scan.pdf", file: mixedPDF(t, true, false)} + s := newState(surfaceW, surfaceH, h) + s.open() + was := docBytes + t.Cleanup(func() { docBytes = was }) + docBytes = func(*ops.Doc) ([]byte, error) { return nil, errNotDrawn } + s.dropBlank() + if !strings.Contains(s.note, "cannot be written") { + t.Errorf("it said %q", s.note) + } +} + +func TestHowPageNumbersAreWritten(t *testing.T) { + // A document whose every other page is blank must not produce a range as + // long as itself. + for _, tc := range []struct { + in []int + want string + }{ + {nil, ""}, + {[]int{3}, "3"}, + {[]int{1, 2, 3}, "1-3"}, + {[]int{1, 3, 5}, "1,3,5"}, + {[]int{1, 2, 5, 6, 7, 9}, "1-2,5-7,9"}, + } { + if got := rangeOf(tc.in); got != tc.want { + t.Errorf("rangeOf(%v) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/corpus_test.go b/corpus_test.go new file mode 100644 index 0000000..6184677 --- /dev/null +++ b/corpus_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestOverTheCorpus runs the verb against real documents and says what it +// would drop. Skipped unless a corpus is named: no scan of anybody's document +// enters the repository, and the test suite must pass on a machine that has +// none. +func TestOverTheCorpus(t *testing.T) { + dir := os.Getenv("BLANKCORPUS") + if dir == "" { + t.Skip("no BLANKCORPUS") + } + docs, pages, dropped, allBlank := 0, 0, 0, 0 + filepath.WalkDir(dir, func(path string, e os.DirEntry, err error) error { + if err != nil || e.IsDir() || filepath.Ext(path) != ".pdf" || docs >= 200 { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil + } + h := &fakeHost{name: filepath.Base(path), file: raw} + s := newState(surfaceW, surfaceH, h) + s.open() + if s.doc == nil { + return nil + } + docs++ + before := s.doc.PageCount() + pages += before + s.dropBlank() + after := s.doc.PageCount() + dropped += before - after + if before == after && strings.Contains(s.note, "every page of this document is blank") { + allBlank++ + t.Logf(" tout blanc: %s (%d pages)", filepath.Base(path), before) + } + return nil + }) + t.Logf("%d documents, %d pages, %d dropped, %d refused as all-blank", + docs, pages, dropped, allBlank) +} diff --git a/panel.go b/panel.go index 2e78fcf..70b7f0b 100644 --- a/panel.go +++ b/panel.go @@ -245,6 +245,10 @@ func (s *state) pagesGroup() *column { box.add(s.spinRow("Split into files of", 1, s.tools.every, func(v int) { s.tools.every = v }), labelledH) box.add(button("Split and hand them over", toolkit.ButtonProminent, s.split), bareH) + + // Last, and not among the things that take a range: this one asks the + // document which pages it means rather than being told. + box.add(blankButton(s), bareH) return box }