Fix tests

.github/workflows/main.yml

@@ -21,32 +21,32 @@ env:
   OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }}
 
 jobs:
-  test-matrix:
+  macos-test-matrix:
     runs-on:
       - self-hosted
-      - "os=${{ matrix.os }}"
-      - "arch=${{ matrix.arch }}"
-      - "os_distribution=${{ matrix.os_distribution }}"
-      - "os_version=${{ matrix.os_version }}"
-      - "revision=${{ matrix.revision }}"
+      - "os=macos"
+      - "arch=arm64"
+      - "os_distribution=sonoma"
+      - "os_version=14"
+      - "revision=4edc10aa8f8ca239699aaf1078832208fb3b7efe"
     timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: macos
-            arch: arm64
-            os_distribution: sonoma
-            os_version: "14"
-            revision: 4edc10aa8f8ca239699aaf1078832208fb3b7efe
-            remote_execution: 'false'
-
-          - os: macos
-            arch: arm64
-            os_distribution: sonoma
-            os_version: "14"
-            revision: 4edc10aa8f8ca239699aaf1078832208fb3b7efe
-            remote_execution: 'true'
+        test_package:
+          - cpp
+          - csharp
+          - docker
+          - genrules
+          - go
+          #- ios # TODO: fix ios builds
+          - java
+          - kotlin
+          - perl
+          - scala
+          - swift
+          - typescript
+        remote_execution: [true, false]
 
     steps:
       - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"
@@ -61,12 +61,12 @@ jobs:
 
       - name: Build and test
         env:
-          ARCH: ${{ matrix.arch }}
-          OS: ${{ matrix.os }}
+          ARCH: arm64
+          OS: macos
           REMOTE_EXECUTION: ${{ matrix.remote_execution }}
-        run: python3 infra/test-all.py
+        run: python3 infra/test-all.py --package=${{ matrix.test_package }}
 
-  ci-runners-test-matrix:
+  linux-test-matrix:
     runs-on:
       - self-hosted
       - "os=linux"
@@ -79,17 +79,32 @@ jobs:
       - "engflow-run-id=${{ github.run_id }}"
       # To avoid a race condition, among all the jobs that use the same Bento in the same repo, only
       # one should specify "engflow-bento-allow-save=true".
+      # In this case, it is linux-test-matrix(cpp, true, true)
       - "engflow-bento-allow-save=${{ matrix.allow_bento_snapshot }}"
 
     timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
+        test_package:
+          - cpp
+          - csharp
+          - docker
+          - genrules
+          - go
+          #- ios # TODO: fix ios builds
+          - java
+          - kotlin
+          - perl
+          - scala
+          - swift
+          - typescript
+        remote_execution: [true, false]
+        allow_bento_snapshot: [false]
         include:
-          - remote_execution: 'false'
-            allow_bento_snapshot: 'false'
-
-          - remote_execution: 'true'
+          - test_package: 'cpp'
+            remote_execution: 'true'
+            # Only one job from the matrix should set allow_bento_snapshot.
             allow_bento_snapshot: 'true'
 
     steps:
@@ -108,7 +123,7 @@ jobs:
           ARCH: x64
           OS: linux
           REMOTE_EXECUTION: ${{ matrix.remote_execution }}
-        run: python3 infra/test-all.py
+        run: python3 infra/test-all.py --package=${{ matrix.test_package }}
 
   buck2-test:
     runs-on:

cpp/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
 load("@rules_cc//cc:cc_library.bzl", "cc_library")
 load("@rules_cc//cc:cc_test.bzl", "cc_test")
@@ -16,6 +17,13 @@ cc_library(
     ],
 )
 
+build_test(
+    name = "cpp_build_test",
+    targets = [
+        ":cpp",
+    ],
+)
+
 cc_test(
     name = "cpp_test",
     size = "small",

csharp/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_dotnet//dotnet:defs.bzl", "csharp_binary")
 
 csharp_binary(
@@ -9,3 +10,10 @@ csharp_binary(
         "net8.0",
     ],
 )
+
+build_test(
+    name = "csharp_test",
+    targets = [
+        ":csharp.exe",
+    ],
+)

docker/network/BUILD

@@ -22,7 +22,9 @@ DOCKER_JDK_PSQL = "docker://gcr.io/YOUR-IMAGE-NAME@sha256:b6ed8f68abd6e23409ee7a
 java_test(
     name = "docker-network-test",
     srcs = ["DockerNetworkTest.java"],
-    tags = ["manual"],
+    tags = [
+        "no-ci",  # Consistent NPE
+    ],
     test_class = "docker.network.DockerNetworkTest",
 )
 

docker/sandbox/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
 load("@rules_java//java:java_binary.bzl", "java_binary")
 
@@ -25,3 +26,11 @@ cc_binary(
     name = "hello-cc",
     srcs = ["hello.cc"],
 )
+
+build_test(
+    name = "hello_test",
+    tags = ["no-macos-ci"],
+    targets = [
+        ":hello",
+    ],
+)

docker/sysbox/dind_test/BUILD

@@ -30,5 +30,8 @@ sh_test(
         # DO NOT ENABLE!!! "dockerPrivileged": "True",
         # sysbox containers will refuse to start
     },
-    tags = ["manual"],
+    tags = [
+        "no-local-ci",
+        "no-macos-ci",
+    ],
 )

genrules/BUILD

@@ -16,6 +16,8 @@
 # on the user's input.
 # This can be used for stress testing your build infrastructure.
 # Change the upper bound of the range and enjoy!
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+
 UPPER_BOUND = 10
 
 [genrule(
@@ -31,7 +33,6 @@ UPPER_BOUND = 10
     name = "gnu-manifesto_PDF_76KB" + str(x),
     outs = ["gnu-manifesto" + str(x) + ".pdf"],
     cmd = "cp $(location gnu-manifesto.pdf) $@",
-    tags = ["manual"],
     tools = ["gnu-manifesto.pdf"],
 ) for x in range(1, UPPER_BOUND)]
 
@@ -41,7 +42,6 @@ UPPER_BOUND = 10
     name = "emacs_TARGZ_68MB" + str(x),
     outs = ["emacs" + str(x) + ".tar.gz"],
     cmd = "cp $(location @emacs//file) $@",
-    tags = ["manual"],
     tools = ["@emacs//file"],
 ) for x in range(1, UPPER_BOUND)]
 
@@ -52,6 +52,13 @@ UPPER_BOUND = 10
     name = "ubuntu_20.04_1.3GB" + str(x),
     outs = ["ubuntu_20.04_1.3GB" + str(x) + ".iso"],
     cmd = "cp $(location @ubuntu_20.04_1.3GB//file) $@",
-    tags = ["manual"],
     tools = ["@ubuntu_20.04_1.3GB//file"],
 ) for x in range(1, 2)]
+
+# Only test a few to ensure they build, without creating undue load.
+build_test(
+    name = "foo_TXT_test",
+    targets = [
+        ":foo_TXT_4KB_1",
+    ],
+)

go/BUILD

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@rules_go//go:def.bzl", "go_binary", "go_library")
 
 go_binary(
@@ -12,3 +13,10 @@ go_library(
     importpath = "github.com/EngFlow/example/go",
     visibility = ["//visibility:private"],
 )
+
+build_test(
+    name = "go_build_test",
+    targets = [
+        ":go",
+    ],
+)

infra/test-all.py

@@ -1,21 +1,70 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 import subprocess
 import sys
 
-def main():
-    # All targets that can run in any environment, except ios examples.
-    targets = [
-        "//...",
-        # TODO: Add testing configuration for ios tests.
-        "-//ios/...",
-    ]
-
-    for key in ("ARCH", "OPAL_RPC_CREDENTIALS", "OS", "REMOTE_EXECUTION"):
+ALL_PACKAGES = [
+    "cpp",
+    "csharp",
+    "docker",
+    "genrules",
+    "go",
+    "ios",
+    "java",
+    "kotlin",
+    "perl",
+    "scala",
+    "swift",
+    "typescript",
+]
+
+EXTRA_FLAGS = {
+    "ios": ["--config=ios"],
+    "swift": ["--config=clang"],
+}
+
+
+def report_error(msg):
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def validate_env():
+    for key in ("ARCH", "OS", "REMOTE_EXECUTION", "OPAL_RPC_CREDENTIALS"):
         if not os.getenv(key):
-            sys.stderr.write(f"{key} not set\n")
-            sys.exit(1)
+            report_error(f"{key} not set")
+
+
+def determine_packages(packages, skip=[]):
+    return [p for p in packages if p not in skip]
+
+
+def find_tests(package):
+    os_name = os.getenv("OS")
+    remote = "remote" if os.getenv("REMOTE_EXECUTION") == "true" else "local"
+    # Remove no-ci tests and no-{os_name}-ci tests
+    query = f"""
+    let t = tests(//{package}/...) in
+    $t - 
+    attr(tags, no-ci, $t) - 
+    attr(tags, no-{os_name}-ci, $t) - 
+    attr(tags, no-{remote}-ci, $t)
+    """.strip()
+    print(f"Executing query to find test targets:\n{query}")
+    args = ["bazel", "query", "--output=label", "--", query]
+    result = subprocess.run(args, capture_output=True)
+    output = result.stdout.decode()
+    return [t for t in output.split("\n") if t]
+
+
+def run_tests(package):
+    targets = find_tests(package)
+    if not targets:
+        print(f"No targets found for {package}, skipping.")
+        return 0
+    print(f"Executing tests {targets}")
 
     os_arch = os.getenv("OS") + "_" + os.getenv("ARCH")
     flags = ["--config=ci"]
@@ -29,13 +78,62 @@ def main():
             "--config=opal_bes",
             "--config=opal_auth",
         ]
-    # The //docker/sysbox/... targets should only run in linux + remote
-    if os.getenv("REMOTE_EXECUTION") == "true" and os.getenv("OS") == "linux":
-        targets += ["//docker/sysbox/dind_test:check_docker",]
+
+    if package in EXTRA_FLAGS:
+        flags += EXTRA_FLAGS[package]
+
+    print(f"---------------\nTesting {package}...")
+    sys.stdout.flush()
+
     args = ["bazel", "test"] + flags + ["--"] + targets
+    result = subprocess.run(args, check=False)
+    sys.stdout.flush()
+    sys.stderr.flush()
+    return result.returncode
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Runs example test suites", fromfile_prefix_chars="@"
+    )
+    parser.add_argument(
+        "--package",
+        required=False,
+        help="Package to test.",
+    )
+    parser.add_argument(
+        "--skip-packages",
+        required=False,
+        default=[],
+        type=lambda arg: arg.split(","),
+        help="Packages to skip testing.",
+    )
+    parser.add_argument(
+        "--validate-env",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to validate environment variables",
+    )
+    opts = parser.parse_args()
+
+    if opts.validate_env:
+        validate_env()
+
+    packages = []
+    if opts.package:
+        packages = [opts.package]
+    else:
+        packages = determine_packages(ALL_PACKAGES, skip=opts.skip_packages)
+
+    # Make sure all packages are valid
+    for package in packages:
+        if package not in ALL_PACKAGES:
+            report_error(f"Unknown package {package}")
 
-    result = subprocess.run(args)
-    sys.exit(result.returncode)
+    for package in packages:
+        returncode = run_tests(package)
+        # TODO: run all tests and then report any failures?
+        if returncode != 0:
+            sys.exit(returncode)
 
 
 if __name__ == "__main__":