From 55a2d89f15a027a4e2bedf3a840053947ccc4258 Mon Sep 17 00:00:00 2001 From: lichuyang Date: Wed, 26 Aug 2026 16:49:11 +0800 Subject: [PATCH 1/2] ci: add self-hosted runner availability probe Dispatch a job against linux-aarch64-a2-2 and linux-aarch64-a3-2 so we can see whether those labels can pick up work for this repo. --- .github/actionlint.yaml | 4 + .../workflows/probe-self-hosted-runners.yml | 100 ++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/probe-self-hosted-runners.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..884972df --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - linux-aarch64-a2-2 + - linux-aarch64-a3-2 diff --git a/.github/workflows/probe-self-hosted-runners.yml b/.github/workflows/probe-self-hosted-runners.yml new file mode 100644 index 00000000..1bfdaabe --- /dev/null +++ b/.github/workflows/probe-self-hosted-runners.yml @@ -0,0 +1,100 @@ +# Manual probe: can this repo schedule a job on the two NPU runner +# labels, pull an AscendHub CANN image, and see the cards? +# +# How to read a run: +# - Job stays queued: this repo has no matching scale set, or no idle capacity. +# - Job starts then fails on image pull / container create: runner is up; the +# container path is broken (wrong tag, hook, or device plugin). +# - Job is green and npu-smi lists devices: that label is usable. +# +# schedule stays commented. Do not enable until someone decides this should +# be a standing health check. +name: Probe self-hosted runners + +on: + workflow_dispatch: + # schedule: + # - cron: '17 */6 * * *' + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + probe: + name: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + - runner: linux-aarch64-a3-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-a3-ubuntu22.04-py3.12 + defaults: + run: + shell: bash + env: + ASCEND_RT_VISIBLE_DEVICES: "0,1" + # container.image is required on these ARC runners; without it the job + # may start but NPU is not allocated. Do not use ${{ env.* }} here — + # that context is not available for image. + container: + image: ${{ matrix.image }} + steps: + - name: Probe host and NPU + env: + RUNNER_LABEL: ${{ matrix.runner }} + IMAGE: ${{ matrix.image }} + run: | + set -euo pipefail + export PATH="/usr/local/sbin:/usr/local/bin:${PATH}" + if [ -f /usr/local/Ascend/ascend-toolkit/set_env.sh ]; then + # shellcheck disable=SC1091 + source /usr/local/Ascend/ascend-toolkit/set_env.sh + fi + + echo "=== runner ===" + echo "RUNNER_NAME=${RUNNER_NAME:-}" + echo "RUNNER_OS=${RUNNER_OS:-}" + echo "RUNNER_ARCH=${RUNNER_ARCH:-}" + echo "RUNNER_LABEL=${RUNNER_LABEL}" + echo "IMAGE=${IMAGE}" + hostname || true + uname -a + + echo "=== devices ===" + ls -l /dev/davinci* /dev/davinci_manager /dev/devmm_svm /dev/hisi_hdc 2>&1 || true + + echo "=== npu-smi ===" + npu-smi info + + npu_nodes="$(find /dev -maxdepth 1 -type c -name 'davinci[0-9]*' | wc -l | tr -d ' ')" + echo "davinci_nodes=${npu_nodes}" + if [ "${npu_nodes}" -lt 2 ]; then + echo "expected at least 2 NPU device nodes on a *-2 runner, got ${npu_nodes}" >&2 + exit 1 + fi + + summarize: + name: summarize + needs: probe + if: always() + runs-on: ubuntu-latest + steps: + - name: Report probe result + env: + PROBE_RESULT: ${{ needs.probe.result }} + run: | + set -euo pipefail + echo "probe matrix result: ${PROBE_RESULT}" + if [ "${PROBE_RESULT}" != "success" ]; then + echo "one or both runner labels failed or did not finish" >&2 + exit 1 + fi + echo "both linux-aarch64-a2-2 and linux-aarch64-a3-2 picked up a job and saw NPU" From de820c402aca576ae85a76d3bb013df8db5da967 Mon Sep 17 00:00:00 2001 From: lichuyang Date: Wed, 26 Aug 2026 17:01:30 +0800 Subject: [PATCH 2/2] ci: probe all a2/a3 runner card counts Cover linux-aarch64-{a2,a3}-{1,2,4,8,16} and check that each label exposes the number of NPU devices in its suffix. --- .github/actionlint.yaml | 8 +++ .../workflows/probe-self-hosted-runners.yml | 51 ++++++++++++------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 884972df..981dc725 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,4 +1,12 @@ self-hosted-runner: labels: + - linux-aarch64-a2-1 - linux-aarch64-a2-2 + - linux-aarch64-a2-4 + - linux-aarch64-a2-8 + - linux-aarch64-a2-16 + - linux-aarch64-a3-1 - linux-aarch64-a3-2 + - linux-aarch64-a3-4 + - linux-aarch64-a3-8 + - linux-aarch64-a3-16 diff --git a/.github/workflows/probe-self-hosted-runners.yml b/.github/workflows/probe-self-hosted-runners.yml index 1bfdaabe..a2fe13b6 100644 --- a/.github/workflows/probe-self-hosted-runners.yml +++ b/.github/workflows/probe-self-hosted-runners.yml @@ -1,11 +1,16 @@ -# Manual probe: can this repo schedule a job on the two NPU runner -# labels, pull an AscendHub CANN image, and see the cards? +# Manual probe: can this repo schedule a job on each NPU runner label, +# pull the matching AscendHub CANN image, and see that many cards? +# +# Label shape: linux-aarch64-- +# chip: a2 (910B) or a3 (910C) +# npu: available card count for that scale set (1 / 2 / 4 / 8 / 16) # # How to read a run: # - Job stays queued: this repo has no matching scale set, or no idle capacity. # - Job starts then fails on image pull / container create: runner is up; the # container path is broken (wrong tag, hook, or device plugin). -# - Job is green and npu-smi lists devices: that label is usable. +# - Job is green and npu-smi lists the expected number of devices: that +# label is usable. # # schedule stays commented. Do not enable until someone decides this should # be a standing health check. @@ -25,32 +30,28 @@ permissions: jobs: probe: - name: ${{ matrix.runner }} - runs-on: ${{ matrix.runner }} + name: linux-aarch64-${{ matrix.chip }}-${{ matrix.npu }} + runs-on: linux-aarch64-${{ matrix.chip }}-${{ matrix.npu }} timeout-minutes: 20 strategy: fail-fast: false matrix: - include: - - runner: linux-aarch64-a2-2 - image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 - - runner: linux-aarch64-a3-2 - image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-a3-ubuntu22.04-py3.12 + chip: [a2, a3] + npu: [1, 2, 4, 8, 16] defaults: run: shell: bash - env: - ASCEND_RT_VISIBLE_DEVICES: "0,1" # container.image is required on these ARC runners; without it the job # may start but NPU is not allocated. Do not use ${{ env.* }} here — # that context is not available for image. container: - image: ${{ matrix.image }} + image: ${{ matrix.chip == 'a2' && 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12' || 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-a3-ubuntu22.04-py3.12' }} steps: - name: Probe host and NPU env: - RUNNER_LABEL: ${{ matrix.runner }} - IMAGE: ${{ matrix.image }} + RUNNER_LABEL: linux-aarch64-${{ matrix.chip }}-${{ matrix.npu }} + IMAGE: ${{ matrix.chip == 'a2' && 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12' || 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-a3-ubuntu22.04-py3.12' }} + NPU_COUNT: ${{ matrix.npu }} run: | set -euo pipefail export PATH="/usr/local/sbin:/usr/local/bin:${PATH}" @@ -59,12 +60,24 @@ jobs: source /usr/local/Ascend/ascend-toolkit/set_env.sh fi + ids=() + i=0 + while [ "${i}" -lt "${NPU_COUNT}" ]; do + ids+=("${i}") + i=$((i + 1)) + done + IFS=, + export ASCEND_RT_VISIBLE_DEVICES="${ids[*]}" + unset IFS + echo "=== runner ===" echo "RUNNER_NAME=${RUNNER_NAME:-}" echo "RUNNER_OS=${RUNNER_OS:-}" echo "RUNNER_ARCH=${RUNNER_ARCH:-}" echo "RUNNER_LABEL=${RUNNER_LABEL}" echo "IMAGE=${IMAGE}" + echo "NPU_COUNT=${NPU_COUNT}" + echo "ASCEND_RT_VISIBLE_DEVICES=${ASCEND_RT_VISIBLE_DEVICES}" hostname || true uname -a @@ -76,8 +89,8 @@ jobs: npu_nodes="$(find /dev -maxdepth 1 -type c -name 'davinci[0-9]*' | wc -l | tr -d ' ')" echo "davinci_nodes=${npu_nodes}" - if [ "${npu_nodes}" -lt 2 ]; then - echo "expected at least 2 NPU device nodes on a *-2 runner, got ${npu_nodes}" >&2 + if [ "${npu_nodes}" -lt "${NPU_COUNT}" ]; then + echo "expected at least ${NPU_COUNT} NPU device nodes on ${RUNNER_LABEL}, got ${npu_nodes}" >&2 exit 1 fi @@ -94,7 +107,7 @@ jobs: set -euo pipefail echo "probe matrix result: ${PROBE_RESULT}" if [ "${PROBE_RESULT}" != "success" ]; then - echo "one or both runner labels failed or did not finish" >&2 + echo "one or more runner labels failed or did not finish" >&2 exit 1 fi - echo "both linux-aarch64-a2-2 and linux-aarch64-a3-2 picked up a job and saw NPU" + echo "all linux-aarch64-a2/a3 labels (1/2/4/8/16 cards) picked up a job and saw NPU"