From 191f45bf47ff54a9417055b2749d1d8a6773613e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20O=2E=20S=C3=B8rensen?= Date: Wed, 5 Aug 2026 17:50:17 +0200 Subject: [PATCH] build: Compile and pack on images chosen for the job dotnet appears exactly once in the native build, to pack an already-compiled binary into a .nupkg. Everything else is gcc and clang. Building C on a dotnet/sdk image therefore meant apt-getting a toolchain onto an image picked for something else, and pulling in packages the build never uses - which is how a 404 on linux-libc-dev, a dependency of build-essential, once failed CI. Now: gcc-build ubuntu:noble installs the cross toolchains llvm-build mstorsjo/llvm-mingw installs nothing pack dotnet/sdk:10.0-noble installs nothing win-arm64 comes from the image llvm-mingw's own author publishes, pinned to a dated release, rather than downloading and checksumming the tarball by hand. That image already has make, m4, curl and the toolchain on PATH, so it needs no apt at all, and it removes an 82MB download from every uncached build along with a pair of values that had to be bumped together. The toolchain images have no SDK, so compiling and packing had to split: SKIP_PACK leaves binaries staged, and pack_runtime_packages.sh packs whatever is staged rather than a list kept alongside it, which cannot then fall out of step with the RIDs the build stages produce. Finding nothing staged is an error, since a stage that copied no binaries would otherwise look like a success. The pack stage moves to the .NET 10 SDK. Which SDK does the packing barely affects the output - netstandard2.0 metadata around an already-compiled binary - but .NET 8 goes out of support in November 2026. The Dockerfile is excluded from the build context. It is not needed inside any image, and excluding it means editing the build definition no longer invalidates every COPY . /source and forces a full recompile. All six RIDs build and verify unchanged. Co-Authored-By: Claude Opus 5 --- .dockerignore | 5 ++ .gitignore | 3 + Dockerfile | 140 ++++++++++++++++++-------------- PACKAGING.md | 31 +++++-- build/build_runtime_packages.sh | 6 +- build/pack_runtime_packages.sh | 37 +++++++++ 6 files changed, 153 insertions(+), 69 deletions(-) create mode 100755 build/pack_runtime_packages.sh diff --git a/.dockerignore b/.dockerignore index 09b3588..26b0a5e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,8 @@ packages packages-docker runtime.*.liblouis/runtimes LibLouis.NET.Tables/tables + +# Not needed inside the image, and excluding them means editing the build definition does not +# invalidate every COPY . /source and force a full recompile of every target. +Dockerfile +.dockerignore diff --git a/.gitignore b/.gitignore index 86c063c..cedb7b8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ runtime.*.liblouis/runtimes/ # Rider / ReSharper per-user settings *.DotSettings.user + +# Worktrees created by spawned Claude Code sessions +.claude/worktrees/ diff --git a/Dockerfile b/Dockerfile index 916d491..a2d614b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# The pinned platform below is deliberate, see the comment on the FROM line. +# The pinned platforms below are deliberate, see the comment on the first FROM. # check=skip=FromPlatformFlagConstDisallowed # Builds the native liblouis binaries for the Linux and Windows runtime identifiers, packs one @@ -8,85 +8,101 @@ # native-macos CI job. The managed packages are not built here either, because building them # resolves runtime.liblouis, which depends on the macOS packages this container cannot produce. # -# The gcc and llvm-mingw targets are separate stages on purpose. llvm-mingw also ships -# i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, so having it installed alongside the Ubuntu cross -# compilers means it can displace them and silently change which toolchain builds win-x86 and -# win-x64. Keeping it out of that stage entirely makes the mistake impossible rather than merely -# documented. BuildKit also builds the independent stages concurrently, so the wall clock is the -# slowest stage rather than the sum. +# Compiling and packing are separate stages, each on an image chosen for the job. +# +# The compilers do not need a .NET SDK: dotnet appears exactly once in the native build, to pack an +# already-compiled binary into a .nupkg. Building C on a dotnet/sdk image meant apt-get installing a +# toolchain onto an image picked for something else, and pulling in packages the build never uses - +# which is how a 404 on linux-libc-dev, a dependency of build-essential, once failed CI. +# +# So: toolchain images compile and stage binaries, and the SDK image packs whatever it finds. The +# SDK stage installs nothing at all. # # --platform is pinned because the cross toolchain package names below only exist for amd64. On an # Apple Silicon machine this runs under emulation: slower, but it works. -FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:8.0-jammy AS base -LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ - -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - m4 \ - xz-utils \ - && rm -rf /var/lib/apt/lists/* - -ENV PACKAGE_OUTPUT_DIR=/packages -WORKDIR /source # The five targets Ubuntu has cross compilers for. -FROM base AS gcc-targets - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc-i686-linux-gnu \ - gcc-aarch64-linux-gnu \ - gcc-mingw-w64-i686 \ - gcc-mingw-w64-x86-64 \ - # The cross gcc packages only Recommend their target libc, so with - # --no-install-recommends they install a compiler that cannot link. Name them explicitly - # rather than dropping the flag, so the requirement is visible. - libc6-dev-i386-cross \ - libc6-dev-arm64-cross \ - && rm -rf /var/lib/apt/lists/* +FROM --platform=linux/amd64 ubuntu:noble AS gcc-build +# Retried, because a single apt-get run is a coin flip against archive.ubuntu.com: the index and +# the pool are not updated atomically, so a package version can be listed after it has been removed +# and the fetch 404s. Each attempt refreshes the index first, since a newer index is usually what +# resolves it. The explicit ok check matters: without it a loop that never succeeds still falls +# through and the layer builds with nothing installed. +RUN set -eu; \ + ok=0; \ + for attempt in 1 2 3; do \ + if apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + m4 \ + gcc-i686-linux-gnu \ + gcc-aarch64-linux-gnu \ + gcc-mingw-w64-i686 \ + gcc-mingw-w64-x86-64 \ + libc6-dev-i386-cross \ + libc6-dev-arm64-cross; then \ + ok=1; break; \ + fi; \ + echo "apt attempt $attempt failed, retrying" >&2; \ + sleep 10; \ + done; \ + [ "$ok" = 1 ] || exit 1; \ + rm -rf /var/lib/apt/lists/* +# The cross gcc packages above only Recommend their target libc, so with --no-install-recommends +# they would install a compiler that cannot link. libc6-dev-*-cross are named explicitly rather +# than dropping the flag, so the requirement is visible. + +ENV SKIP_PACK=1 +WORKDIR /source COPY . /source RUN sh ./build/build_runtime_packages.sh gcc -# win-arm64. Ubuntu has no aarch64 mingw-w64 cross compiler, so this stage uses the prebuilt -# llvm-mingw toolchain, pinned by digest: an unpinned toolchain would silently change what the -# published binaries were built with. Bump both values together when moving to a newer release. -FROM base AS llvm-targets - -ARG LLVM_MINGW_VERSION=20260616 -ARG LLVM_MINGW_SHA256=534b92e067b22a6b4441f48ae9240a3341b17825d04d577eab0cf85c44b4deda -RUN set -eu; \ - archive="llvm-mingw-${LLVM_MINGW_VERSION}-ucrt-ubuntu-22.04-x86_64.tar.xz"; \ - curl -fL -o "/tmp/$archive" \ - "https://github.com/mstorsjo/llvm-mingw/releases/download/${LLVM_MINGW_VERSION}/$archive"; \ - echo "${LLVM_MINGW_SHA256} /tmp/$archive" | sha256sum --check; \ - mkdir -p /opt/llvm-mingw; \ - tar xf "/tmp/$archive" -C /opt/llvm-mingw --strip-components=1; \ - rm "/tmp/$archive" - -# The build script prepends this to PATH for the targets that need it. Left off PATH here so there -# is exactly one mechanism selecting the toolchain, in the script, where it is visible. -ENV LLVM_MINGW_BIN=/opt/llvm-mingw/bin - +# win-arm64. Ubuntu has no aarch64 mingw-w64 cross compiler, so this uses llvm-mingw, taken from +# the image its own author publishes and pinned to a dated release rather than downloaded and +# checksummed by hand. The image already carries make, m4, curl and the toolchain on PATH, so this +# stage installs nothing. +# +# It is a separate stage from the gcc targets, and that separation is load bearing: llvm-mingw also +# ships i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, so having it on PATH alongside the Ubuntu +# cross compilers silently takes over the win-x86 and win-x64 builds. That is not hypothetical - it +# happened, and it broke win-x86, because clang treats the -Wincompatible-pointer-types that gnulib +# trips on mingw as an error where gcc only warns. +FROM --platform=linux/amd64 mstorsjo/llvm-mingw:20260616 AS llvm-build + +ENV SKIP_PACK=1 +WORKDIR /source COPY . /source RUN sh ./build/build_runtime_packages.sh llvm -# The metapackage is pure metadata and needs no toolchain at all. -FROM base AS metapackage +# Packs what the toolchain stages produced, and the metapackage, which needs no native binary at +# all. Installs nothing: dotnet pack is the only thing this stage does. +# +# The SDK version barely affects the output here - these packages are netstandard2.0 metadata around +# an already-compiled binary, with IncludeBuildOutput off - but .NET 8 goes out of support in +# November 2026, and there is no reason for the build to be the thing still on it. +FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble AS pack +LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ +ENV PACKAGE_OUTPUT_DIR=/packages +WORKDIR /source COPY . /source -RUN sh ./build/build_metapackage.sh + +COPY --from=gcc-build /source/runtime.linux-x86.liblouis/runtimes /source/runtime.linux-x86.liblouis/runtimes +COPY --from=gcc-build /source/runtime.linux-x64.liblouis/runtimes /source/runtime.linux-x64.liblouis/runtimes +COPY --from=gcc-build /source/runtime.linux-arm64.liblouis/runtimes /source/runtime.linux-arm64.liblouis/runtimes +COPY --from=gcc-build /source/runtime.win-x86.liblouis/runtimes /source/runtime.win-x86.liblouis/runtimes +COPY --from=gcc-build /source/runtime.win-x64.liblouis/runtimes /source/runtime.win-x64.liblouis/runtimes +COPY --from=llvm-build /source/runtime.win-arm64.liblouis/runtimes /source/runtime.win-arm64.liblouis/runtimes + +RUN sh ./build/pack_runtime_packages.sh && \ + sh ./build/build_metapackage.sh # `docker build --output=packages .` exports just the .nupkg files into ./packages. FROM scratch -COPY --from=gcc-targets /packages/* / -COPY --from=llvm-targets /packages/* / -COPY --from=metapackage /packages/* / +COPY --from=pack /packages/* / diff --git a/PACKAGING.md b/PACKAGING.md index a1ae783..6ff7250 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -94,13 +94,32 @@ NUGET_LOCAL_FEED=$PWD/packages sh build/build_managed_packages.sh ### Cross-compilation Linux and Windows binaries are cross-compiled in the container defined by `Dockerfile`. The -toolchain package names there only exist for amd64, so the image is pinned to `linux/amd64` and -runs under emulation on Apple Silicon. +toolchain package names there only exist for amd64, so the images are pinned to `linux/amd64` and +run under emulation on Apple Silicon. -`win-arm64` is the exception: Ubuntu has no aarch64 mingw-w64 gcc, so the image installs the -[llvm-mingw](https://github.com/mstorsjo/llvm-mingw) toolchain, pinned by SHA-256. It provides -`aarch64-w64-mingw32-gcc` driver wrappers, so the build script treats it like the other Windows -targets. Bump `LLVM_MINGW_VERSION` and `LLVM_MINGW_SHA256` together. +Compiling and packing are separate stages, on images chosen for the job: + +| Stage | Image | Installs | +| --- | --- | --- | +| `gcc-build` | `ubuntu:noble` | the cross toolchains | +| `llvm-build` | `mstorsjo/llvm-mingw` | nothing | +| `pack` | `dotnet/sdk` | nothing | + +dotnet appears exactly once in the native build, to pack an already-compiled binary. Compiling C on +a `dotnet/sdk` image meant apt-getting a toolchain onto an image chosen for something else, and +pulling in packages the build never uses — which is how a 404 on `linux-libc-dev`, a dependency of +`build-essential`, once failed CI. The toolchain stages now leave binaries staged under +`runtime..liblouis/runtimes/`, and `build/pack_runtime_packages.sh` packs whatever it finds +rather than a list that could fall out of step. Finding nothing is an error. + +`win-arm64` is the exception among the Windows targets: Ubuntu has no aarch64 mingw-w64 gcc, so it +uses [llvm-mingw](https://github.com/mstorsjo/llvm-mingw), taken from the image its own author +publishes and pinned to a dated release. That image already carries `make`, `m4`, `curl` and the +toolchain on `PATH`. + +The `Dockerfile` is excluded from the build context. It is not needed inside any image, and +excluding it means editing the build definition does not invalidate every `COPY . /source` and +force a full recompile of every target. llvm-mingw lives in its own container stage, and that separation is load bearing. It also ships `i686-w64-mingw32-gcc` and `x86_64-w64-mingw32-gcc`, so merely having it on `PATH` alongside the diff --git a/build/build_runtime_packages.sh b/build/build_runtime_packages.sh index 5d616b9..557bea9 100755 --- a/build/build_runtime_packages.sh +++ b/build/build_runtime_packages.sh @@ -89,7 +89,11 @@ build_runtime_nuget() { make distclean ) - pack_runtime_package "$rid" + # SKIP_PACK leaves the binary staged without packing it, for a container stage that has a + # toolchain but no .NET SDK. A later stage packs what this one produced. + if [ -z "${SKIP_PACK:-}" ]; then + pack_runtime_package "$rid" + fi } if [ "$group" = "gcc" ] || [ "$group" = "all" ]; then diff --git a/build/pack_runtime_packages.sh b/build/pack_runtime_packages.sh new file mode 100755 index 0000000..2c6650a --- /dev/null +++ b/build/pack_runtime_packages.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Packs a runtime package for every RID that has a native binary staged. +# +# Compiling and packing happen in different container stages: the toolchain images have no .NET SDK +# and the SDK image has no cross compilers. The build stages leave binaries staged under +# runtime..liblouis/runtimes/, and this packs whatever it finds. +# +# Packing whatever is present, rather than a list kept here, means the list cannot fall out of step +# with the RIDs the build stages actually produce. Finding nothing is an error: a stage that copied +# no binaries would otherwise produce an empty set of packages and look like a success. + +. "$(dirname "$0")/common.sh" + +packed=0 + +for project in "$REPO_ROOT"/runtime.*.liblouis; do + rid=$(basename "$project" | sed 's/^runtime\.\(.*\)\.liblouis$/\1/') + + # The metapackage has no RID of its own and no native payload. + if [ "$rid" = "liblouis" ]; then + continue + fi + + if [ -z "$(find "$project/runtimes" -type f 2>/dev/null | head -n 1)" ]; then + continue + fi + + pack_runtime_package "$rid" + packed=$((packed + 1)) +done + +if [ "$packed" -eq 0 ]; then + echo "pack_runtime_packages: no staged native binaries found under runtime.*.liblouis/runtimes/" >&2 + exit 1 +fi + +echo "==> Packed $packed runtime package(s) into $PACKAGE_OUTPUT_DIR"