From 81b9eb526859deb3383749a9aa001a57ca273059 Mon Sep 17 00:00:00 2001 From: John Cater Date: Tue, 7 Jul 2026 17:11:01 -0400 Subject: [PATCH 1/9] Refactor example CI setup. This change: - Adds many more `build_test` targets to ensure non-tests will still be built and checked. - Some of these are not working and are currently disabled. - Splits each top-level package into a separate CI action so that it's easier to spot failures and ensure other tests continue - Adds a new tag-based way to disable tests: - `no-ci` will skip the test in CI entirely - `no-remote-ci` and `no-local-ci` will skip based on remote execution - `no-linux-ci` and `no-macos-ci` will skip based on the OS --- .github/workflows/main.yml | 73 +++++++----- cpp/BUILD | 8 ++ csharp/BUILD | 8 ++ docker/BUILD | 0 docker/network/BUILD | 4 +- docker/sandbox/BUILD | 9 ++ docker/sysbox/dind_test/BUILD | 5 +- genrules/BUILD | 13 +- go/BUILD | 8 ++ infra/test-all.py | 111 +++++++++++++++--- ios/{BUILD.bazel => BUILD} | 14 ++- java/BUILD | 0 java/com/engflow/binaryinput/BUILD | 8 ++ java/com/engflow/notificationqueue/BUILD | 8 ++ .../notificationqueue/demoserver/BUILD | 8 ++ kotlin/BUILD | 8 ++ perl/BUILD | 8 ++ python/BUILD | 8 ++ swift/BUILD | 6 +- 19 files changed, 252 insertions(+), 55 deletions(-) create mode 100644 docker/BUILD mode change 100644 => 100755 infra/test-all.py rename ios/{BUILD.bazel => BUILD} (81%) create mode 100644 java/BUILD diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b85ea4d1..af00af24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,32 +21,32 @@ env: OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }} jobs: - test-matrix: + macos-test-matrix: runs-on: - self-hosted - - "os=${{ matrix.os }}" - - "arch=${{ matrix.arch }}" - - "os_distribution=${{ matrix.os_distribution }}" - - "os_version=${{ matrix.os_version }}" - - "revision=${{ matrix.revision }}" + - "os=macos" + - "arch=arm64" + - "os_distribution=sonoma" + - "os_version=14" + - "revision=4edc10aa8f8ca239699aaf1078832208fb3b7efe" timeout-minutes: 10 strategy: fail-fast: false matrix: - include: - - os: macos - arch: arm64 - os_distribution: sonoma - os_version: "14" - revision: 4edc10aa8f8ca239699aaf1078832208fb3b7efe - remote_execution: 'false' - - - os: macos - arch: arm64 - os_distribution: sonoma - os_version: "14" - revision: 4edc10aa8f8ca239699aaf1078832208fb3b7efe - remote_execution: 'true' + test_package: + - cpp + - csharp + - docker + - genrules + - go + #- ios # TODO: fix ios builds + - java + - kotlin + - perl + - scala + - swift + - typescript + remote_execution: [true, false] steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" @@ -61,12 +61,12 @@ jobs: - name: Build and test env: - ARCH: ${{ matrix.arch }} - OS: ${{ matrix.os }} + ARCH: arm64 + OS: macos REMOTE_EXECUTION: ${{ matrix.remote_execution }} - run: python3 infra/test-all.py + run: python3 infra/test-all.py --package=${{ matrix.test_package }} - ci-runners-test-matrix: + linux-test-matrix: runs-on: - self-hosted - "os=linux" @@ -79,17 +79,32 @@ jobs: - "engflow-run-id=${{ github.run_id }}" # To avoid a race condition, among all the jobs that use the same Bento in the same repo, only # one should specify "engflow-bento-allow-save=true". + # In this case, it is linux-test-matrix(cpp, true, true) - "engflow-bento-allow-save=${{ matrix.allow_bento_snapshot }}" timeout-minutes: 10 strategy: fail-fast: false matrix: + test_package: + - cpp + - csharp + - docker + - genrules + - go + #- ios # TODO: fix ios builds + - java + - kotlin + - perl + - scala + - swift + - typescript + remote_execution: [true, false] + allow_bento_snapshot: [false] include: - - remote_execution: 'false' - allow_bento_snapshot: 'false' - - - remote_execution: 'true' + - test_package: 'cpp' + remote_execution: 'true' + # Only one job from the matrix should set allow_bento_snapshot. allow_bento_snapshot: 'true' steps: @@ -108,7 +123,7 @@ jobs: ARCH: x64 OS: linux REMOTE_EXECUTION: ${{ matrix.remote_execution }} - run: python3 infra/test-all.py + run: python3 infra/test-all.py --package=${{ matrix.test_package }} buck2-test: runs-on: diff --git a/cpp/BUILD b/cpp/BUILD index eab1ab18..d392faa4 100644 --- a/cpp/BUILD +++ b/cpp/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_test.bzl", "cc_test") @@ -16,6 +17,13 @@ cc_library( ], ) +build_test( + name = "cpp_build_test", + targets = [ + ":cpp", + ], +) + cc_test( name = "cpp_test", size = "small", diff --git a/csharp/BUILD b/csharp/BUILD index 668f17bd..94036417 100644 --- a/csharp/BUILD +++ b/csharp/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_dotnet//dotnet:defs.bzl", "csharp_binary") csharp_binary( @@ -9,3 +10,10 @@ csharp_binary( "net8.0", ], ) + +build_test( + name = "csharp_test", + targets = [ + ":csharp.exe", + ], +) diff --git a/docker/BUILD b/docker/BUILD new file mode 100644 index 00000000..e69de29b diff --git a/docker/network/BUILD b/docker/network/BUILD index 520957a0..de443058 100644 --- a/docker/network/BUILD +++ b/docker/network/BUILD @@ -22,7 +22,9 @@ DOCKER_JDK_PSQL = "docker://gcr.io/YOUR-IMAGE-NAME@sha256:b6ed8f68abd6e23409ee7a java_test( name = "docker-network-test", srcs = ["DockerNetworkTest.java"], - tags = ["manual"], + tags = [ + "no-ci", # Consistent NPE + ], test_class = "docker.network.DockerNetworkTest", ) diff --git a/docker/sandbox/BUILD b/docker/sandbox/BUILD index 167df89f..fc3f7afe 100644 --- a/docker/sandbox/BUILD +++ b/docker/sandbox/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_java//java:java_binary.bzl", "java_binary") @@ -25,3 +26,11 @@ cc_binary( name = "hello-cc", srcs = ["hello.cc"], ) + +build_test( + name = "hello_test", + tags = ["no-macos-ci"], + targets = [ + ":hello", + ], +) diff --git a/docker/sysbox/dind_test/BUILD b/docker/sysbox/dind_test/BUILD index 58b2b1d2..c91fa965 100644 --- a/docker/sysbox/dind_test/BUILD +++ b/docker/sysbox/dind_test/BUILD @@ -30,5 +30,8 @@ sh_test( # DO NOT ENABLE!!! "dockerPrivileged": "True", # sysbox containers will refuse to start }, - tags = ["manual"], + tags = [ + "no-local-ci", + "no-macos-ci", + ], ) diff --git a/genrules/BUILD b/genrules/BUILD index 1a462fa4..964aa9d8 100644 --- a/genrules/BUILD +++ b/genrules/BUILD @@ -16,6 +16,8 @@ # on the user's input. # This can be used for stress testing your build infrastructure. # Change the upper bound of the range and enjoy! +load("@bazel_skylib//rules:build_test.bzl", "build_test") + UPPER_BOUND = 10 [genrule( @@ -31,7 +33,6 @@ UPPER_BOUND = 10 name = "gnu-manifesto_PDF_76KB" + str(x), outs = ["gnu-manifesto" + str(x) + ".pdf"], cmd = "cp $(location gnu-manifesto.pdf) $@", - tags = ["manual"], tools = ["gnu-manifesto.pdf"], ) for x in range(1, UPPER_BOUND)] @@ -41,7 +42,6 @@ UPPER_BOUND = 10 name = "emacs_TARGZ_68MB" + str(x), outs = ["emacs" + str(x) + ".tar.gz"], cmd = "cp $(location @emacs//file) $@", - tags = ["manual"], tools = ["@emacs//file"], ) for x in range(1, UPPER_BOUND)] @@ -52,6 +52,13 @@ UPPER_BOUND = 10 name = "ubuntu_20.04_1.3GB" + str(x), outs = ["ubuntu_20.04_1.3GB" + str(x) + ".iso"], cmd = "cp $(location @ubuntu_20.04_1.3GB//file) $@", - tags = ["manual"], tools = ["@ubuntu_20.04_1.3GB//file"], ) for x in range(1, 2)] + +# Only test a few to ensure they build, without creating undue load. +build_test( + name = "foo_TXT_test", + targets = [ + ":foo_TXT_4KB_1", + ], +) diff --git a/go/BUILD b/go/BUILD index e87cfd8b..19bbd0f9 100644 --- a/go/BUILD +++ b/go/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_go//go:def.bzl", "go_binary", "go_library") go_binary( @@ -12,3 +13,10 @@ go_library( importpath = "github.com/EngFlow/example/go", visibility = ["//visibility:private"], ) + +build_test( + name = "go_build_test", + targets = [ + ":go", + ], +) diff --git a/infra/test-all.py b/infra/test-all.py old mode 100644 new mode 100755 index 7a62415c..ea9e0631 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -1,21 +1,66 @@ #!/usr/bin/env python3 +import argparse import os import subprocess import sys -def main(): - # All targets that can run in any environment, except ios examples. - targets = [ - "//...", - # TODO: Add testing configuration for ios tests. - "-//ios/...", - ] - - for key in ("ARCH", "OPAL_RPC_CREDENTIALS", "OS", "REMOTE_EXECUTION"): +ALL_PACKAGES = [ + "cpp", + "csharp", + "docker", + "genrules", + "go", + "ios", + "java", + "kotlin", + "perl", + "scala", + "swift", + "typescript", +] + +EXTRA_FLAGS = { + "ios": ["--config=ios"], + "swift": ["--config=clang"], +} + + +def report_error(msg): + print(msg, file=sys.stderr) + sys.exit(1) + + +def validate_env(): + for key in ("ARCH", "OS", "REMOTE_EXECUTION", "OPAL_RPC_CREDENTIALS"): if not os.getenv(key): - sys.stderr.write(f"{key} not set\n") - sys.exit(1) + report_error(f"{key} not set") + + +def find_tests(package): + os_name = os.getenv("OS") + remote = "remote" if os.getenv("REMOTE_EXECUTION") == "true" else "local" + # Remove no-ci tests and no-{os_name}-ci tests + query = f""" + let t = tests(//{package}/...) in + $t - + attr(tags, no-ci, $t) - + attr(tags, no-{os_name}-ci, $t) - + attr(tags, no-{remote}-ci, $t) + """.strip() + print(f"Executing query to find test targets:\n{query}") + args = ["bazel", "query", "--output=label", "--", query] + result = subprocess.run(args, capture_output=True) + output = result.stdout.decode() + return [t for t in output.split("\n") if t] + + +def run_tests(package): + targets = find_tests(package) + if not targets: + print(f"No targets found for {package}, skipping.") + return 0 + print(f"Executing tests {targets}") os_arch = os.getenv("OS") + "_" + os.getenv("ARCH") flags = ["--config=ci"] @@ -29,13 +74,47 @@ def main(): "--config=opal_bes", "--config=opal_auth", ] - # The //docker/sysbox/... targets should only run in linux + remote - if os.getenv("REMOTE_EXECUTION") == "true" and os.getenv("OS") == "linux": - targets += ["//docker/sysbox/dind_test:check_docker",] + + if package in EXTRA_FLAGS: + flags += EXTRA_FLAGS[package] + + print(f"---------------\nTesting {package}...") + sys.stdout.flush() + args = ["bazel", "test"] + flags + ["--"] + targets + result = subprocess.run(args, check=False) + sys.stdout.flush() + sys.stderr.flush() + return result.returncode + + +def main(): + parser = argparse.ArgumentParser( + description="Runs example test suites", fromfile_prefix_chars="@" + ) + parser.add_argument( + "--package", + required=False, + help="Package to test.", + ) + parser.add_argument( + "--validate-env", + action=argparse.BooleanOptionalAction, + help="Whether to validate environment variables", + ) + opts = parser.parse_args() + + if opts.validate_env: + validate_env() + + package = opts.package + + if opts.package not in ALL_PACKAGES: + report_error(f"Unknown package {opts.package}") - result = subprocess.run(args) - sys.exit(result.returncode) + returncode = run_tests(opts.package) + if returncode != 0: + sys.exit(returncode) if __name__ == "__main__": diff --git a/ios/BUILD.bazel b/ios/BUILD similarity index 81% rename from ios/BUILD.bazel rename to ios/BUILD index ab1fdf9b..87e22895 100644 --- a/ios/BUILD.bazel +++ b/ios/BUILD @@ -1,10 +1,10 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_apple//apple:ios.bzl", "ios_application", "ios_ui_test", "ios_unit_test") load("@rules_swift//swift:swift.bzl", "swift_library") swift_library( name = "HelloAppLibrary", srcs = ["HelloApp.swift"], - tags = ["manual"], ) ios_application( @@ -18,11 +18,17 @@ ios_application( deps = [":HelloAppLibrary"], ) +build_test( + name = "HelloApp_build_test", + targets = [ + ":HelloApp", + ], +) + swift_library( name = "HelloTestLib", testonly = True, srcs = ["HelloTest.swift"], - tags = ["manual"], ) ios_unit_test( @@ -38,13 +44,15 @@ swift_library( "HelloAppUITest.swift", ], module_name = "HelloAppUITest", - tags = ["manual"], ) ios_ui_test( name = "HelloAppUITest", minimum_os_version = "26.0", runner = "@rules_apple//apple/testing/default_runner:ios_xctestrun_ordered_runner", + tags = [ + "no-remote-ci", # Investigating permissions error. + ], test_host = ":HelloApp", deps = [":HelloAppUITestLib"], ) diff --git a/java/BUILD b/java/BUILD new file mode 100644 index 00000000..e69de29b diff --git a/java/com/engflow/binaryinput/BUILD b/java/com/engflow/binaryinput/BUILD index 44a15c24..4402ecd2 100644 --- a/java/com/engflow/binaryinput/BUILD +++ b/java/com/engflow/binaryinput/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_java//java:defs.bzl", "java_binary", "java_library") NUM_FILES = 100 @@ -43,3 +44,10 @@ java_binary( ) ], ) + +build_test( + name = "main_test", + targets = [ + ":main", + ], +) diff --git a/java/com/engflow/notificationqueue/BUILD b/java/com/engflow/notificationqueue/BUILD index e9b5586f..b117334b 100644 --- a/java/com/engflow/notificationqueue/BUILD +++ b/java/com/engflow/notificationqueue/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@grpc-java//:java_grpc_library.bzl", "java_grpc_library") load("@rules_java//java:java_binary.bzl", "java_binary") @@ -28,3 +29,10 @@ java_binary( "@protobuf//java/core", ], ) + +build_test( + name = "client_test", + targets = [ + ":client", + ], +) diff --git a/java/com/engflow/notificationqueue/demoserver/BUILD b/java/com/engflow/notificationqueue/demoserver/BUILD index 3d51a2ef..9cca5105 100644 --- a/java/com/engflow/notificationqueue/demoserver/BUILD +++ b/java/com/engflow/notificationqueue/demoserver/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@grpc-java//:java_grpc_library.bzl", "java_grpc_library") load("@protobuf//bazel:java_proto_library.bzl", "java_proto_library") load("@protobuf//bazel:proto_library.bzl", "proto_library") @@ -40,3 +41,10 @@ java_binary( "@grpc-java//stub", ], ) + +build_test( + name = "server_test", + targets = [ + ":server", + ], +) diff --git a/kotlin/BUILD b/kotlin/BUILD index 05163505..330bb776 100644 --- a/kotlin/BUILD +++ b/kotlin/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_binary") kt_jvm_binary( @@ -5,3 +6,10 @@ kt_jvm_binary( srcs = ["Main.kt"], main_class = "example.MainKt", ) + +build_test( + name = "kotlin_test", + targets = [ + ":kotlin", + ], +) diff --git a/perl/BUILD b/perl/BUILD index 18e7c3b9..8c0f6c76 100644 --- a/perl/BUILD +++ b/perl/BUILD @@ -1,6 +1,14 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_perl//perl:perl.bzl", "perl_binary") perl_binary( name = "perl", srcs = ["hello_world.pl"], ) + +build_test( + name = "perl_test", + targets = [ + ":perl", + ], +) diff --git a/python/BUILD b/python/BUILD index 2903d1b3..8db53f9c 100644 --- a/python/BUILD +++ b/python/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("//python/pytest:pytest.bzl", "pytest_test") @@ -57,3 +58,10 @@ compile_pip_requirements( requirements_txt = "requirements_lock.txt", tags = ["manual"], ) + +build_test( + name = "python_test", + targets = [ + ":python", + ], +) diff --git a/swift/BUILD b/swift/BUILD index c5d89e1c..3b3147e9 100644 --- a/swift/BUILD +++ b/swift/BUILD @@ -1,11 +1,13 @@ load("@rules_swift//swift:swift.bzl", "swift_test") swift_test( - name = "tests", + name = "swift_test", size = "small", srcs = [ "example.swift", "main.swift", ], - tags = ["manual"], + tags = [ + "no-ci", # Broken toolchain setup + ], ) From 0a0ffd607ad3b2f14a049077cd68fb4975687623 Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:26:34 -0400 Subject: [PATCH 2/9] Re-add manual tags --- docker/network/BUILD | 1 + docker/sysbox/dind_test/BUILD | 1 + genrules/BUILD | 3 +++ swift/BUILD | 1 + 4 files changed, 6 insertions(+) diff --git a/docker/network/BUILD b/docker/network/BUILD index de443058..f9e67ff9 100644 --- a/docker/network/BUILD +++ b/docker/network/BUILD @@ -23,6 +23,7 @@ java_test( name = "docker-network-test", srcs = ["DockerNetworkTest.java"], tags = [ + "manual", "no-ci", # Consistent NPE ], test_class = "docker.network.DockerNetworkTest", diff --git a/docker/sysbox/dind_test/BUILD b/docker/sysbox/dind_test/BUILD index c91fa965..c05874c9 100644 --- a/docker/sysbox/dind_test/BUILD +++ b/docker/sysbox/dind_test/BUILD @@ -31,6 +31,7 @@ sh_test( # sysbox containers will refuse to start }, tags = [ + "manual", "no-local-ci", "no-macos-ci", ], diff --git a/genrules/BUILD b/genrules/BUILD index 964aa9d8..c5ca8dd3 100644 --- a/genrules/BUILD +++ b/genrules/BUILD @@ -33,6 +33,7 @@ UPPER_BOUND = 10 name = "gnu-manifesto_PDF_76KB" + str(x), outs = ["gnu-manifesto" + str(x) + ".pdf"], cmd = "cp $(location gnu-manifesto.pdf) $@", + tags = ["manual"], tools = ["gnu-manifesto.pdf"], ) for x in range(1, UPPER_BOUND)] @@ -42,6 +43,7 @@ UPPER_BOUND = 10 name = "emacs_TARGZ_68MB" + str(x), outs = ["emacs" + str(x) + ".tar.gz"], cmd = "cp $(location @emacs//file) $@", + tags = ["manual"], tools = ["@emacs//file"], ) for x in range(1, UPPER_BOUND)] @@ -52,6 +54,7 @@ UPPER_BOUND = 10 name = "ubuntu_20.04_1.3GB" + str(x), outs = ["ubuntu_20.04_1.3GB" + str(x) + ".iso"], cmd = "cp $(location @ubuntu_20.04_1.3GB//file) $@", + tags = ["manual"], tools = ["@ubuntu_20.04_1.3GB//file"], ) for x in range(1, 2)] diff --git a/swift/BUILD b/swift/BUILD index 3b3147e9..f728cc33 100644 --- a/swift/BUILD +++ b/swift/BUILD @@ -8,6 +8,7 @@ swift_test( "main.swift", ], tags = [ + "manual", "no-ci", # Broken toolchain setup ], ) From b97826d7403bb077c0e6c226867cea4634e8bb50 Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:35:06 -0400 Subject: [PATCH 3/9] Rename REMOTE_EXECUTION to EXECUTION_TYPE --- .github/workflows/main.yml | 14 +++++++------- infra/test-all.py | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af00af24..5d5f42fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: - scala - swift - typescript - remote_execution: [true, false] + execution_type: [remote, local] steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" @@ -63,7 +63,7 @@ jobs: env: ARCH: arm64 OS: macos - REMOTE_EXECUTION: ${{ matrix.remote_execution }} + EXECUTION_TYPE: ${{ matrix.execution_type }} run: python3 infra/test-all.py --package=${{ matrix.test_package }} linux-test-matrix: @@ -99,13 +99,13 @@ jobs: - scala - swift - typescript - remote_execution: [true, false] + execution_type: [remote, local] allow_bento_snapshot: [false] include: - - test_package: 'cpp' - remote_execution: 'true' + - test_package: cpp + execution_type: remote # Only one job from the matrix should set allow_bento_snapshot. - allow_bento_snapshot: 'true' + allow_bento_snapshot: true steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" @@ -122,7 +122,7 @@ jobs: env: ARCH: x64 OS: linux - REMOTE_EXECUTION: ${{ matrix.remote_execution }} + EXECUTION_TYPE: ${{ matrix.execution_type }} run: python3 infra/test-all.py --package=${{ matrix.test_package }} buck2-test: diff --git a/infra/test-all.py b/infra/test-all.py index ea9e0631..34260d14 100755 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -32,15 +32,15 @@ def report_error(msg): def validate_env(): - for key in ("ARCH", "OS", "REMOTE_EXECUTION", "OPAL_RPC_CREDENTIALS"): + for key in ("ARCH", "OS", "EXECUTION_TYPE", "OPAL_RPC_CREDENTIALS"): if not os.getenv(key): report_error(f"{key} not set") def find_tests(package): os_name = os.getenv("OS") - remote = "remote" if os.getenv("REMOTE_EXECUTION") == "true" else "local" - # Remove no-ci tests and no-{os_name}-ci tests + remote = os.getenv("EXECUTION_type") + # Remove tests based on tags and environment query = f""" let t = tests(//{package}/...) in $t - @@ -64,7 +64,7 @@ def run_tests(package): os_arch = os.getenv("OS") + "_" + os.getenv("ARCH") flags = ["--config=ci"] - if os.getenv("REMOTE_EXECUTION") == "true": + if os.getenv("EXECUTION_TYPE") == "remote": flags += [ "--config=remote_" + os_arch, "--config=opal", From 5ba935e3b359ec0644420601df529c2f3080619f Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:38:28 -0400 Subject: [PATCH 4/9] Split out bento save job --- .github/workflows/main.yml | 53 +++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d5f42fb..0c4a54f5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,16 +71,12 @@ jobs: - self-hosted - "os=linux" - "arch=x64" - - "remote_execution=${{ matrix.remote_execution }}" - "engflow-bento-name=example-gh-x64" - "engflow-cluster=glass" - - "engflow-job-name=ci-runners-test-matrix-${{matrix.remote_execution}}_${{matrix.allow_bento_snapshot}}" + - "engflow-job-name=linux-test-matrix-${{matrix.execution_type}}" - "engflow-job-type=${{github.event_name}}-${{github.ref_name}}" - "engflow-run-id=${{ github.run_id }}" - # To avoid a race condition, among all the jobs that use the same Bento in the same repo, only - # one should specify "engflow-bento-allow-save=true". - # In this case, it is linux-test-matrix(cpp, true, true) - - "engflow-bento-allow-save=${{ matrix.allow_bento_snapshot }}" + - "engflow-bento-allow-save=false" timeout-minutes: 10 strategy: @@ -100,12 +96,45 @@ jobs: - swift - typescript execution_type: [remote, local] - allow_bento_snapshot: [false] - include: - - test_package: cpp - execution_type: remote - # Only one job from the matrix should set allow_bento_snapshot. - allow_bento_snapshot: true + + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" + + - name: Set up authentication + shell: bash + run: cp infra/bazelrc .bazelrc.user + + - name: Generate python requirements lock + shell: bash + run: bazel run //python:requirements.update + + - name: Build and test + env: + ARCH: x64 + OS: linux + EXECUTION_TYPE: ${{ matrix.execution_type }} + run: python3 infra/test-all.py --package=${{ matrix.test_package }} + + bento-snapshot: + runs-on: + - self-hosted + - "os=linux" + - "arch=x64" + - "remote_execution=true" + - "engflow-bento-name=example-gh-x64" + - "engflow-cluster=glass" + - "engflow-job-name=bento-snapshot" + - "engflow-job-type=${{github.event_name}}-${{github.ref_name}}" + - "engflow-run-id=${{ github.run_id }}" + # To avoid a race condition, among all the jobs that use the same Bento in the same repo, only + # one should specify "engflow-bento-allow-save=true". + - "engflow-bento-allow-save=true" + + timeout-minutes: 10 + strategy: + fail-fast: false + test_package: cpp + execution_type: remote steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" From 6499177ac85efdfc7ed26f013cb7fa2ae5ce031e Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:41:53 -0400 Subject: [PATCH 5/9] Add whitespace. --- infra/test-all.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra/test-all.py b/infra/test-all.py index 34260d14..b011dc6c 100755 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -40,6 +40,7 @@ def validate_env(): def find_tests(package): os_name = os.getenv("OS") remote = os.getenv("EXECUTION_type") + # Remove tests based on tags and environment query = f""" let t = tests(//{package}/...) in @@ -48,6 +49,7 @@ def find_tests(package): attr(tags, no-{os_name}-ci, $t) - attr(tags, no-{remote}-ci, $t) """.strip() + print(f"Executing query to find test targets:\n{query}") args = ["bazel", "query", "--output=label", "--", query] result = subprocess.run(args, capture_output=True) From 162bbf026a724096b446806af21a837fbf6173d0 Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:42:17 -0400 Subject: [PATCH 6/9] Remove ALL_PACKAGES --- infra/test-all.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/infra/test-all.py b/infra/test-all.py index b011dc6c..a6f39f44 100755 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -5,21 +5,6 @@ import subprocess import sys -ALL_PACKAGES = [ - "cpp", - "csharp", - "docker", - "genrules", - "go", - "ios", - "java", - "kotlin", - "perl", - "scala", - "swift", - "typescript", -] - EXTRA_FLAGS = { "ios": ["--config=ios"], "swift": ["--config=clang"], @@ -111,9 +96,6 @@ def main(): package = opts.package - if opts.package not in ALL_PACKAGES: - report_error(f"Unknown package {opts.package}") - returncode = run_tests(opts.package) if returncode != 0: sys.exit(returncode) From 287ad6bb64c4d407a273f39a480701337e3c78f9 Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:43:01 -0400 Subject: [PATCH 7/9] Fix syntax --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0c4a54f5..440262bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -133,8 +133,9 @@ jobs: timeout-minutes: 10 strategy: fail-fast: false - test_package: cpp - execution_type: remote + matrix: + test_package: [cpp] + execution_type: [remote] steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" From 41ce82db0e04a36ffb8d66e20f5d160c2e65d464 Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:53:46 -0400 Subject: [PATCH 8/9] Fix typo --- infra/test-all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/test-all.py b/infra/test-all.py index a6f39f44..23c262ed 100755 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -24,7 +24,7 @@ def validate_env(): def find_tests(package): os_name = os.getenv("OS") - remote = os.getenv("EXECUTION_type") + remote = os.getenv("EXECUTION_TYPE") # Remove tests based on tags and environment query = f""" From e7d4ffd1fac918f1836fb2038468ae62a59abf19 Mon Sep 17 00:00:00 2001 From: John Cater Date: Wed, 8 Jul 2026 10:59:37 -0400 Subject: [PATCH 9/9] Reformatted query --- infra/test-all.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/test-all.py b/infra/test-all.py index 23c262ed..8b9e4f17 100755 --- a/infra/test-all.py +++ b/infra/test-all.py @@ -29,10 +29,10 @@ def find_tests(package): # Remove tests based on tags and environment query = f""" let t = tests(//{package}/...) in - $t - - attr(tags, no-ci, $t) - - attr(tags, no-{os_name}-ci, $t) - - attr(tags, no-{remote}-ci, $t) + $t + - attr(tags, no-ci, $t) + - attr(tags, no-{os_name}-ci, $t) + - attr(tags, no-{remote}-ci, $t) """.strip() print(f"Executing query to find test targets:\n{query}")