benchmark.yml

# =============================================================================
# Benchmark — decoupled, post-release, semi-manual. Runs the ENTIRE NumSharp-vs-
# NumPy comparison via benchmark/run_benchmark.py: the API-complete op/dtype/N matrix
# (BenchmarkDotNet + NumPy across applicable bounded tiers), the NDIter iterator benchmark
# (benchmark/nditer, + README cards), and the complementary subsystems that fill the
# axes/routes the op matrix omits — layout, operand, cast, fusion, and isolated
# backend profiles (Managed C# + OpenBLAS/LAPACK) — merged per exact benchmark cell.
#
# Trigger is the GitHub Release being published (the "after a successful release"
# signal — 'Build and Release' creates it on a v* tag) or a manual dispatch. It
# is a SEPARATE workflow from the release pipeline on purpose: a slow/failed
# benchmark must never gate a release.
#
# It publishes the refreshed benchmark snapshot (report + subsystem sheets + cards) to the
# orphan `data` branch (tools/dashboard_data/publish.py) — the single home of this data — then
# triggers a docs redeploy so the site rebuilds from it via the refs/data submodule. NOTHING is
# committed to master (master is code-only). GITHUB_TOKEN + contents:write — no PAT.
#
# NOTE on numbers: GitHub runners are shared, variable hardware. ABSOLUTE ms are
# not comparable run-to-run; every ratio is NumPy / NumSharp on the SAME runner,
# which stays meaningful. numpy is pinned. The NDIter harness IGNORES NumSharp's
# known intermittent AccessViolation: a crashing section is reported NA.
# =============================================================================
name: Benchmark

on:
  release:
    types: [published]
  workflow_dispatch: {}

permissions:
  contents: write
  actions: write   # dispatch the Deploy Docs workflow after committing refreshed report pages

concurrency:
  group: benchmark
  cancel-in-progress: false

jobs:
  benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - name: Checkout master
        uses: actions/checkout@v7
        with:
          ref: master
          fetch-depth: 0

      - name: Setup .NET (8 + 10)
        uses: actions/setup-dotnet@v6
        with:
          dotnet-version: |
            8.0.x
            10.0.x

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.12'

      - name: Install Python deps (pin numpy for cross-release comparability)
        run: pip install "numpy==2.4.2" matplotlib tabulate

      - name: Stage pinned OpenBLAS runtime assets
        run: python src/NumSharp.Interop.OpenBLAS/tools/fetch_openblas.py

      - name: Run full benchmark (official op suites + NDIter)
        # run_benchmark.py builds the CSharp bench project + Core, runs the official suites, then
        # the NDIter sheet (section-isolated, AV-ignored→NA) + cards, and appends
        # the NDIter section to benchmark-report.md.
        run: python benchmark/run_benchmark.py

      # Publish the fresh benchmark snapshot to the orphan `data` branch as a new <date>_<sha>
      # folder + a refreshed real `latest/` directory (the whole benchmark/history/latest set:
      # report JSON/CSV/MD, subsystem sheets, cards). This branch is the ONLY home of the data;
      # the docs site reads it via the refs/data submodule (no master-side copy).
      - name: Publish benchmark data to the data branch
        run: |
          set -euo pipefail
          git config user.name  "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git fetch origin data --depth=1
          git worktree add -B data /tmp/data origin/data
          python tools/dashboard_data/publish.py --type benchmark \
            --from benchmark/history/latest --branch-worktree /tmp/data --sha "${GITHUB_SHA}" --commit
          git -C /tmp/data push origin data

      # The benchmark report + subsystem sheets + cards now live ONLY on the `data` branch
      # (published above); NOTHING is committed to master. The docs site reads them via the
      # refs/data submodule and renders the matrix/iterator pages at docs-build, so all that
      # remains is to trigger a docs rebuild. workflow_dispatch ignores docs.yml's path filter.
      - name: Redeploy docs (rebuild from the new data-branch snapshot)
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh workflow run docs.yml --ref master