diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..981dc725 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +self-hosted-runner: + labels: + - linux-aarch64-a2-1 + - linux-aarch64-a2-2 + - linux-aarch64-a2-4 + - linux-aarch64-a2-8 + - linux-aarch64-a2-16 + - linux-aarch64-a3-1 + - linux-aarch64-a3-2 + - linux-aarch64-a3-4 + - linux-aarch64-a3-8 + - linux-aarch64-a3-16 diff --git a/.github/workflows/probe-self-hosted-runners.yml b/.github/workflows/probe-self-hosted-runners.yml new file mode 100644 index 00000000..a2fe13b6 --- /dev/null +++ b/.github/workflows/probe-self-hosted-runners.yml @@ -0,0 +1,113 @@ +# Manual probe: can this repo schedule a job on each NPU runner label, +# pull the matching AscendHub CANN image, and see that many cards? +# +# Label shape: linux-aarch64-- +# chip: a2 (910B) or a3 (910C) +# npu: available card count for that scale set (1 / 2 / 4 / 8 / 16) +# +# How to read a run: +# - Job stays queued: this repo has no matching scale set, or no idle capacity. +# - Job starts then fails on image pull / container create: runner is up; the +# container path is broken (wrong tag, hook, or device plugin). +# - Job is green and npu-smi lists the expected number of devices: that +# label is usable. +# +# schedule stays commented. Do not enable until someone decides this should +# be a standing health check. +name: Probe self-hosted runners + +on: + workflow_dispatch: + # schedule: + # - cron: '17 */6 * * *' + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + probe: + name: linux-aarch64-${{ matrix.chip }}-${{ matrix.npu }} + runs-on: linux-aarch64-${{ matrix.chip }}-${{ matrix.npu }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + chip: [a2, a3] + npu: [1, 2, 4, 8, 16] + defaults: + run: + shell: bash + # container.image is required on these ARC runners; without it the job + # may start but NPU is not allocated. Do not use ${{ env.* }} here — + # that context is not available for image. + container: + image: ${{ matrix.chip == 'a2' && 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12' || 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-a3-ubuntu22.04-py3.12' }} + steps: + - name: Probe host and NPU + env: + RUNNER_LABEL: linux-aarch64-${{ matrix.chip }}-${{ matrix.npu }} + IMAGE: ${{ matrix.chip == 'a2' && 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12' || 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-a3-ubuntu22.04-py3.12' }} + NPU_COUNT: ${{ matrix.npu }} + run: | + set -euo pipefail + export PATH="/usr/local/sbin:/usr/local/bin:${PATH}" + if [ -f /usr/local/Ascend/ascend-toolkit/set_env.sh ]; then + # shellcheck disable=SC1091 + source /usr/local/Ascend/ascend-toolkit/set_env.sh + fi + + ids=() + i=0 + while [ "${i}" -lt "${NPU_COUNT}" ]; do + ids+=("${i}") + i=$((i + 1)) + done + IFS=, + export ASCEND_RT_VISIBLE_DEVICES="${ids[*]}" + unset IFS + + echo "=== runner ===" + echo "RUNNER_NAME=${RUNNER_NAME:-}" + echo "RUNNER_OS=${RUNNER_OS:-}" + echo "RUNNER_ARCH=${RUNNER_ARCH:-}" + echo "RUNNER_LABEL=${RUNNER_LABEL}" + echo "IMAGE=${IMAGE}" + echo "NPU_COUNT=${NPU_COUNT}" + echo "ASCEND_RT_VISIBLE_DEVICES=${ASCEND_RT_VISIBLE_DEVICES}" + hostname || true + uname -a + + echo "=== devices ===" + ls -l /dev/davinci* /dev/davinci_manager /dev/devmm_svm /dev/hisi_hdc 2>&1 || true + + echo "=== npu-smi ===" + npu-smi info + + npu_nodes="$(find /dev -maxdepth 1 -type c -name 'davinci[0-9]*' | wc -l | tr -d ' ')" + echo "davinci_nodes=${npu_nodes}" + if [ "${npu_nodes}" -lt "${NPU_COUNT}" ]; then + echo "expected at least ${NPU_COUNT} NPU device nodes on ${RUNNER_LABEL}, got ${npu_nodes}" >&2 + exit 1 + fi + + summarize: + name: summarize + needs: probe + if: always() + runs-on: ubuntu-latest + steps: + - name: Report probe result + env: + PROBE_RESULT: ${{ needs.probe.result }} + run: | + set -euo pipefail + echo "probe matrix result: ${PROBE_RESULT}" + if [ "${PROBE_RESULT}" != "success" ]; then + echo "one or more runner labels failed or did not finish" >&2 + exit 1 + fi + echo "all linux-aarch64-a2/a3 labels (1/2/4/8/16 cards) picked up a job and saw NPU"