From 35aa2a6eee22a65f026780e1f54e579132d1bb05 Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan Date: Wed, 16 Sep 2026 10:52:09 -0700 Subject: [PATCH] docs(examples): add managed GPU inference codelab Add a guided, repeatable AKS example for validating managed GPU nodes, staging a model once on shared storage, and serving it privately with two vLLM replicas. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4393b19f-b77a-4d03-aa95-cfb25e91da35 --- .../managed-gpu-inference-service/README.md | 195 ++++++++++++++++++ .../manifests/gpu-smoke-test.yaml | 47 +++++ .../manifests/model-stage-job.yaml | 109 ++++++++++ .../manifests/model-storage.yaml | 33 +++ .../manifests/namespace.yaml | 7 + .../manifests/vllm-serving.yaml | 173 ++++++++++++++++ .../modules/00-prerequisites.md | 83 ++++++++ .../modules/01-cluster.md | 56 +++++ .../modules/02-managed-gpu-nodepool.md | 73 +++++++ .../modules/03-verify.md | 56 +++++ .../modules/04-model-storage.md | 58 ++++++ .../modules/05-inference-service.md | 64 ++++++ .../modules/06-observability.md | 65 ++++++ .../modules/07-cleanup.md | 62 ++++++ .../scripts/00-preflight.sh | 182 ++++++++++++++++ .../scripts/10-create-cluster.sh | 113 ++++++++++ .../scripts/20-create-managed-gpu-pool.sh | 92 +++++++++ .../scripts/30-verify-gpu.sh | 188 +++++++++++++++++ .../scripts/40-stage-model.sh | 89 ++++++++ .../scripts/50-deploy-vllm.sh | 131 ++++++++++++ .../scripts/90-cleanup.sh | 103 +++++++++ .../scripts/lib.sh | 129 ++++++++++++ 22 files changed, 2108 insertions(+) create mode 100644 examples/managed-gpu-inference-service/README.md create mode 100644 examples/managed-gpu-inference-service/manifests/gpu-smoke-test.yaml create mode 100644 examples/managed-gpu-inference-service/manifests/model-stage-job.yaml create mode 100644 examples/managed-gpu-inference-service/manifests/model-storage.yaml create mode 100644 examples/managed-gpu-inference-service/manifests/namespace.yaml create mode 100644 examples/managed-gpu-inference-service/manifests/vllm-serving.yaml create mode 100644 examples/managed-gpu-inference-service/modules/00-prerequisites.md create mode 100644 examples/managed-gpu-inference-service/modules/01-cluster.md create mode 100644 examples/managed-gpu-inference-service/modules/02-managed-gpu-nodepool.md create mode 100644 examples/managed-gpu-inference-service/modules/03-verify.md create mode 100644 examples/managed-gpu-inference-service/modules/04-model-storage.md create mode 100644 examples/managed-gpu-inference-service/modules/05-inference-service.md create mode 100644 examples/managed-gpu-inference-service/modules/06-observability.md create mode 100644 examples/managed-gpu-inference-service/modules/07-cleanup.md create mode 100755 examples/managed-gpu-inference-service/scripts/00-preflight.sh create mode 100755 examples/managed-gpu-inference-service/scripts/10-create-cluster.sh create mode 100755 examples/managed-gpu-inference-service/scripts/20-create-managed-gpu-pool.sh create mode 100755 examples/managed-gpu-inference-service/scripts/30-verify-gpu.sh create mode 100755 examples/managed-gpu-inference-service/scripts/40-stage-model.sh create mode 100755 examples/managed-gpu-inference-service/scripts/50-deploy-vllm.sh create mode 100755 examples/managed-gpu-inference-service/scripts/90-cleanup.sh create mode 100755 examples/managed-gpu-inference-service/scripts/lib.sh diff --git a/examples/managed-gpu-inference-service/README.md b/examples/managed-gpu-inference-service/README.md new file mode 100644 index 000000000..1b031f71e --- /dev/null +++ b/examples/managed-gpu-inference-service/README.md @@ -0,0 +1,195 @@ +# Serve an LLM with managed GPUs on AKS + +> [!IMPORTANT] +> Fully managed GPU nodes are a preview feature. Preview features are provided +> as is and as available, are excluded from service-level agreements and +> limited warranty, and aren't intended for production use. For details, see +> [AKS support policies](https://learn.microsoft.com/azure/aks/support-policies). + +Build a two-replica vLLM service on Azure Kubernetes Service (AKS) with fully +managed NVIDIA GPU nodes. The example verifies GPU access before deployment, +stages model weights once on shared storage, and keeps the service private +behind a `ClusterIP` Service. + +```mermaid +flowchart LR + client["Local client"] -->|"kubectl port-forward"| service["ClusterIP Service
vllm:8000"] + + subgraph cluster["AKS cluster"] + system["CPU system node pool"] + service --> replica1["vLLM replica"] + service --> replica2["vLLM replica"] + replica1 --> gpu1["A100 GPU node 1"] + replica2 --> gpu2["A100 GPU node 2"] + storage[("Azure Blob NFS
shared model weights")] + storage --> replica1 + storage --> replica2 + end + + managed["AKS-managed NVIDIA driver,
device plugin, and DCGM exporter"] + managed -.-> gpu1 + managed -.-> gpu2 +``` + +## What you build + +- Two managed GPU nodes that expose `nvidia.com/gpu` without a separately + installed device plugin or GPU Operator. +- A shared `ReadWriteMany` volume that stores + `Qwen/Qwen2.5-7B-Instruct`. +- Two vLLM replicas spread across separate GPU nodes. +- Startup, readiness, and liveness probes that account for model load time. +- A PodDisruptionBudget and a zero-surge rollout strategy for scarce GPU + capacity. +- Direct access to NVIDIA Data Center GPU Manager (DCGM) metrics on each node. + +The example does not create a public endpoint. Use `kubectl port-forward` to +test the service from your computer. + +## Time and cost + +Allow 45–75 minutes for the full example. Most of that time is cluster +provisioning, model transfer, and the first vLLM startup. + +The default configuration creates: + +- Two `Standard_D4s_v5` system nodes. +- Two `Standard_NC24ads_A100_v4` GPU nodes. +- A 200-GiB Premium Azure Blob volume. + +GPU nodes account for most of the cost. Run the cleanup module as soon as you +finish. Pricing varies by region and agreement; estimate the current cost with +the [Azure pricing calculator](https://azure.microsoft.com/pricing/calculator/). +If a long-running step fails and you don't plan to retry immediately, run +`./scripts/90-cleanup.sh` to remove the GPU pool. + +## Prerequisites + +You need: + +- An Azure subscription where you can create resource groups, AKS clusters, + node pools, and storage. +- Regional quota for two `Standard_NC24ads_A100_v4` nodes and two + `Standard_D4s_v5` nodes. +- Azure CLI 2.85.0 or later. +- `aks-preview` extension 19.0.0b29 or later. +- `kubectl` 1.34 or later, Python 3, and Bash. +- Outbound HTTPS access to Docker Hub, PyPI, and Hugging Face. + +The default region is `westus2`. Set `LAB_LOCATION` before running the scripts +if your quota is in another region. + +## Modules + +| # | Module | Outcome | +| --- | --- | --- | +| 0 | [Check prerequisites](modules/00-prerequisites.md) | Confirm tools, Azure access, feature registration, SKU availability, and quota | +| 1 | [Create the cluster](modules/01-cluster.md) | Create a CPU-only AKS cluster with the Blob CSI driver | +| 2 | [Create the managed GPU pool](modules/02-managed-gpu-nodepool.md) | Add two A100 nodes with the AKS-managed NVIDIA stack | +| 3 | [Verify GPU access](modules/03-verify.md) | Check the managed profile, schedulable GPU resources, CUDA access, and DCGM metrics | +| 4 | [Stage the model](modules/04-model-storage.md) | Download and verify model weights once on shared storage | +| 5 | [Deploy the inference service](modules/05-inference-service.md) | Start two vLLM replicas and send an OpenAI-compatible request | +| 6 | [Observe the service](modules/06-observability.md) | Compare device-level and service-level metrics | +| 7 | [Clean up](modules/07-cleanup.md) | Remove billable GPU capacity or delete the complete example | + +Run the modules in order. Each module includes an expected result and focused +troubleshooting steps. + +## Fast path + +Run these commands from this directory: + +```bash +./scripts/00-preflight.sh +./scripts/10-create-cluster.sh +./scripts/20-create-managed-gpu-pool.sh +./scripts/30-verify-gpu.sh +./scripts/40-stage-model.sh +./scripts/50-deploy-vllm.sh + +kubectl port-forward -n managed-gpu-inference service/vllm 8000:8000 +``` + +In another terminal, send a request: + +```bash +curl --fail-with-body http://127.0.0.1:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "qwen", + "messages": [ + {"role": "user", "content": "Reply with exactly: AKS GPU service online."} + ], + "max_tokens": 16, + "temperature": 0 + }' +``` + +When you finish, stop the port-forward and delete the example: + +```bash +./scripts/90-cleanup.sh --all +``` + +## Why the example uses these patterns + +**Separate system and GPU pools.** System components remain on CPU nodes, so +you can remove GPU capacity without disrupting the control-plane add-ons that +run in the cluster. + +**Shared model storage.** Each serving pod mounts the same model checkpoint. +Replacing a pod remounts the volume instead of downloading the model again. + +**Two replicas with no rollout surge.** Each replica requests one GPU. A +default rolling update can request a third GPU that does not exist and stall. +The manifest updates one replica at a time instead. + +**Private access by default.** An unauthenticated public model endpoint can +consume expensive GPU capacity. Add authentication, transport security, and +rate limiting before exposing the service outside the cluster. + +**Guarded destructive operations.** The scripts label or tag the resources +they own, use a subscription-specific kubeconfig context, and refuse to clean +up resources that don't have the expected ownership marker. + +## Validated configuration + +This configuration was validated end to end in September 2026 with: + +- AKS 1.35. +- Two `Standard_NC24ads_A100_v4` nodes. +- `vllm/vllm-openai:v0.28.0`. +- `Qwen/Qwen2.5-7B-Instruct`. +- Two ready replicas on separate nodes. +- Shared model weights mounted read-only from Azure Blob NFS. +- Successful OpenAI-compatible completion requests. + +## Production considerations + +This example demonstrates a serving baseline, not a complete production +platform. + +| Gap | Next step | +| --- | --- | +| Public access | Add an authenticated gateway with TLS and rate limiting | +| Secret management | Use workload identity and Azure Key Vault for protected model access | +| Node autoscaling | Managed GPU node pools do not support cluster autoscaler during preview; scale the pool manually | +| Model rollout | Add versioned model paths and controlled traffic shifting | +| Multi-region availability | Deploy independent regional stacks and route between them | +| Restricted egress | Import the vLLM image into Azure Container Registry and use a prebuilt staging image instead of installing from PyPI at runtime | +| Cluster isolation | Add a NetworkPolicy so only approved clients can reach the private Service | + +## Layout + +```text +manifests/ Kubernetes resources for storage, validation, and serving +modules/ Guided steps with expected results and troubleshooting +scripts/ Repeatable setup, validation, deployment, and cleanup commands +``` + +## Learn more + +- [Fully managed GPU nodes on AKS](https://learn.microsoft.com/azure/aks/aks-managed-gpu-nodes) +- [Use Azure Blob storage with AKS](https://learn.microsoft.com/azure/aks/azure-blob-csi) +- [Monitor GPU metrics on AKS](https://learn.microsoft.com/azure/aks/monitor-gpu-metrics) +- [vLLM documentation](https://docs.vllm.ai) diff --git a/examples/managed-gpu-inference-service/manifests/gpu-smoke-test.yaml b/examples/managed-gpu-inference-service/manifests/gpu-smoke-test.yaml new file mode 100644 index 000000000..0a122e996 --- /dev/null +++ b/examples/managed-gpu-inference-service/manifests/gpu-smoke-test.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-validation-node + namespace: managed-gpu-inference + labels: + app.kubernetes.io/name: gpu-validation + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + nodeSelector: + kubernetes.io/hostname: NODE_NAME + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cuda-check + image: vllm/vllm-openai:v0.28.0 + command: + - python3 + - -c + - | + import torch + tensor = torch.zeros(8, device="cuda") + print(f"CUDA_OK device={torch.cuda.get_device_name(0)!r} values={tensor.tolist()}") + resources: + requests: + cpu: 250m + memory: 1Gi + nvidia.com/gpu: "1" + limits: + cpu: "2" + memory: 4Gi + nvidia.com/gpu: "1" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/examples/managed-gpu-inference-service/manifests/model-stage-job.yaml b/examples/managed-gpu-inference-service/manifests/model-stage-job.yaml new file mode 100644 index 000000000..ef0210251 --- /dev/null +++ b/examples/managed-gpu-inference-service/manifests/model-stage-job.yaml @@ -0,0 +1,109 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: stage-model + namespace: managed-gpu-inference + labels: + app.kubernetes.io/name: stage-model + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +spec: + backoffLimit: 2 + activeDeadlineSeconds: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: stage-model + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service + spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: stage + image: python:3.12.11-slim-bookworm + command: + - /bin/bash + - -lc + args: + - | + set -euo pipefail + pip install --no-cache-dir --quiet huggingface_hub==0.34.4 + python3 - <<'PY' + import json + import os + import shutil + import sys + from pathlib import Path + + from huggingface_hub import snapshot_download + + model = os.environ["MODEL_ID"] + destination = Path("/models") / model.split("/")[-1] + + snapshot_download( + repo_id=model, + local_dir=str(destination), + allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt"], + max_workers=4, + ) + + index = destination / "model.safetensors.index.json" + if index.exists(): + shards = sorted(set(json.loads(index.read_text())["weight_map"].values())) + else: + shards = [path.name for path in destination.glob("*.safetensors")] + if not shards: + sys.exit("No safetensors files or model index were downloaded.") + + missing = [ + shard + for shard in shards + if not (destination / shard).is_file() + or (destination / shard).stat().st_size == 0 + ] + if missing: + sys.exit(f"Incomplete model download: {missing}") + + weight_bytes = sum((destination / shard).stat().st_size for shard in shards) + print( + f"MODEL_READY shards={len(shards)} " + f"weight_gib={weight_bytes / 1024**3:.1f} path={destination}", + flush=True, + ) + + cache = destination / ".cache" + if cache.is_dir(): + shutil.rmtree(cache) + print("Removed the transfer cache after verification.", flush=True) + PY + env: + - name: MODEL_ID + value: Qwen/Qwen2.5-7B-Instruct + - name: HF_XET_HIGH_PERFORMANCE + value: "1" + - name: HF_HUB_DISABLE_TELEMETRY + value: "1" + resources: + requests: + cpu: 250m + memory: 2Gi + limits: + cpu: "2" + memory: 10Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: models + mountPath: /models + volumes: + - name: models + persistentVolumeClaim: + claimName: model-weights diff --git a/examples/managed-gpu-inference-service/manifests/model-storage.yaml b/examples/managed-gpu-inference-service/manifests/model-storage.yaml new file mode 100644 index 000000000..fad6325b8 --- /dev/null +++ b/examples/managed-gpu-inference-service/manifests/model-storage.yaml @@ -0,0 +1,33 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: managed-gpu-model-blob + labels: + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +provisioner: blob.csi.azure.com +parameters: + protocol: nfs + skuName: Premium_LRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +mountOptions: + - nconnect=4 + - noresvport + - actimeo=120 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-weights + namespace: managed-gpu-inference + labels: + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +spec: + accessModes: + - ReadWriteMany + storageClassName: managed-gpu-model-blob + resources: + requests: + storage: 200Gi diff --git a/examples/managed-gpu-inference-service/manifests/namespace.yaml b/examples/managed-gpu-inference-service/manifests/namespace.yaml new file mode 100644 index 000000000..34f98be26 --- /dev/null +++ b/examples/managed-gpu-inference-service/manifests/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: managed-gpu-inference + labels: + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service diff --git a/examples/managed-gpu-inference-service/manifests/vllm-serving.yaml b/examples/managed-gpu-inference-service/manifests/vllm-serving.yaml new file mode 100644 index 000000000..740820e01 --- /dev/null +++ b/examples/managed-gpu-inference-service/manifests/vllm-serving.yaml @@ -0,0 +1,173 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vllm + namespace: managed-gpu-inference + labels: + app.kubernetes.io/name: vllm + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +automountServiceAccountToken: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm + namespace: managed-gpu-inference + labels: + app.kubernetes.io/name: vllm + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +spec: + replicas: 2 + progressDeadlineSeconds: 3600 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: vllm + template: + metadata: + labels: + app.kubernetes.io/name: vllm + app.kubernetes.io/part-of: managed-gpu-inference-service + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8000" + prometheus.io/path: /metrics + spec: + serviceAccountName: vllm + automountServiceAccountToken: false + enableServiceLinks: false + nodeSelector: + agentpool: a100np + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule + topologySpreadConstraints: + - maxSkew: 1 + minDomains: 2 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: vllm + terminationGracePeriodSeconds: 120 + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: vllm + image: vllm/vllm-openai:v0.28.0 + args: + - /models/Qwen2.5-7B-Instruct + - --served-model-name=qwen + - --max-model-len=16384 + - --gpu-memory-utilization=0.90 + - --max-num-seqs=64 + - --host=0.0.0.0 + - --port=8000 + env: + - name: HF_HUB_OFFLINE + value: "1" + - name: PYTHONUNBUFFERED + value: "1" + ports: + - name: http + containerPort: 8000 + resources: + requests: + cpu: "8" + memory: 32Gi + nvidia.com/gpu: "1" + limits: + cpu: "20" + memory: 160Gi + nvidia.com/gpu: "1" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - sleep 15 + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 120 + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 6 + volumeMounts: + - name: models + mountPath: /models + readOnly: true + - name: dshm + mountPath: /dev/shm + volumes: + - name: models + persistentVolumeClaim: + claimName: model-weights + readOnly: true + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: vllm + namespace: managed-gpu-inference + labels: + app.kubernetes.io/name: vllm + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: vllm + ports: + - name: http + port: 8000 + targetPort: http +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: vllm + namespace: managed-gpu-inference + labels: + app.kubernetes.io/name: vllm + app.kubernetes.io/part-of: managed-gpu-inference-service + aks.azure.com/example: managed-gpu-inference-service +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: vllm diff --git a/examples/managed-gpu-inference-service/modules/00-prerequisites.md b/examples/managed-gpu-inference-service/modules/00-prerequisites.md new file mode 100644 index 000000000..dc2d1320c --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/00-prerequisites.md @@ -0,0 +1,83 @@ +# Module 0: Check prerequisites + +This module takes about 5 minutes and creates no Azure resources. + +## Run the preflight check + +```bash +./scripts/00-preflight.sh +``` + +The script checks: + +- Azure CLI, `kubectl`, Python 3, and Bash. +- Azure sign-in and the active subscription. +- Required Azure resource providers. +- The `ManagedGPUExperiencePreview` feature registration. +- Provider re-registration that propagates the preview feature. +- Kubernetes 1.35 availability in the selected region. +- Availability and quota for the system and GPU virtual machine (VM) sizes. +- Total regional virtual CPU quota for the complete example. + +The default configuration uses `westus2`. If your quota is in another region, +set the location before running the script: + +```bash +export LAB_LOCATION=eastus +./scripts/00-preflight.sh +``` + +The scripts also support custom resource group and cluster names: + +```bash +export LAB_RESOURCE_GROUP=my-managed-gpu-example +export LAB_CLUSTER=my-managed-gpu-example +``` + +The validated Kubernetes version is 1.35. If that version isn't available in +your selected region, set another version supported by the managed GPU preview: + +```bash +export LAB_KUBERNETES_VERSION=1.34 +``` + +The VM sizes are fixed because the model, replica count, memory requests, and +validation results depend on their exact shape. + +## Register the preview feature + +If preflight reports that the feature is not registered, run: + +```bash +az feature register \ + --namespace Microsoft.ContainerService \ + --name ManagedGPUExperiencePreview +``` + +Registration can take several minutes. Rerun preflight after the state changes +to `Registered`. Preflight then re-registers `Microsoft.ContainerService` and +waits for the preview feature to propagate. + +## Checkpoint + +The final output should be: + +```output +PASS Preflight passed. Continue with modules/01-cluster.md. +``` + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| Azure CLI is too old | Upgrade to Azure CLI 2.85.0 or later | +| `aks-preview` is missing | Run `az extension add --name aks-preview`, then update it | +| `kubectl` is too old | Upgrade to kubectl 1.34 or later | +| Kubernetes 1.35 isn't available | Set `LAB_KUBERNETES_VERSION` to a version supported by the managed GPU preview in that region | +| A provider is not registered | Run the registration command printed by preflight | +| The GPU VM size is restricted | Select a region where the subscription can use `Standard_NC24ads_A100_v4` | +| Family or regional quota is insufficient | Request quota for the reported family and **Total Regional vCPUs** | + +## Next step + +[Module 1: Create the cluster](01-cluster.md) diff --git a/examples/managed-gpu-inference-service/modules/01-cluster.md b/examples/managed-gpu-inference-service/modules/01-cluster.md new file mode 100644 index 000000000..d9d20a79d --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/01-cluster.md @@ -0,0 +1,56 @@ +# Module 1: Create the cluster + +Create an AKS cluster with a CPU-only system node pool. GPU capacity is added +separately in Module 2 so you can remove the expensive nodes without removing +the cluster. + +## Create the cluster + +```bash +./scripts/10-create-cluster.sh +``` + +The script: + +1. Creates a dedicated resource group with an ownership tag. +2. Creates two `Standard_D4s_v5` system nodes. +3. Enables the Azure Blob Container Storage Interface (CSI) driver. +4. Writes the cluster credentials to your kubeconfig. +5. Selects the new cluster as the active `kubectl` context. + +The ownership tag prevents cleanup from deleting a resource group that this +example did not create. + +## Checkpoint + +```bash +kubectl get nodes -l agentpool=system +az aks show \ + --resource-group "${LAB_RESOURCE_GROUP:-aks-managed-gpu-inference}" \ + --name "${LAB_CLUSTER:-aks-managed-gpu-inference}" \ + --query 'storageProfile.blobCsiDriver.enabled' +``` + +Expect two `Ready` system nodes and `true` for the Blob CSI driver. + +No node should advertise GPU capacity yet: + +```bash +kubectl get nodes \ + -o custom-columns='NODE:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu' +``` + +The GPU column should be empty. + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| The resource group already exists without the ownership tag | Set a different `LAB_RESOURCE_GROUP` | +| Cluster creation reports quota limits | Resolve the system VM family or total regional virtual CPU quota reported by preflight | +| `kubectl` points to another cluster | Rerun `./scripts/10-create-cluster.sh` | +| Blob CSI reports `false` | Wait for the cluster operation to finish, then inspect `az aks show` and rerun the script | + +## Next step + +[Module 2: Create the managed GPU pool](02-managed-gpu-nodepool.md) diff --git a/examples/managed-gpu-inference-service/modules/02-managed-gpu-nodepool.md b/examples/managed-gpu-inference-service/modules/02-managed-gpu-nodepool.md new file mode 100644 index 000000000..07e97a543 --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/02-managed-gpu-nodepool.md @@ -0,0 +1,73 @@ +# Module 2: Create the managed GPU pool + +> [!IMPORTANT] +> Fully managed GPU nodes are a preview feature and aren't intended for +> production use. Preview CLI flags and API fields can change before general +> availability. + +Add two A100 nodes with the AKS-managed NVIDIA stack. + +## Create the pool + +```bash +./scripts/20-create-managed-gpu-pool.sh +``` + +The script creates a Linux user node pool with: + +- Two `Standard_NC24ads_A100_v4` nodes. +- `--enable-managed-gpu=true`. +- The `sku=gpu:NoSchedule` taint. +- A managed operating system disk. + +With managed GPU enabled, AKS installs and maintains the NVIDIA driver, +Kubernetes device plugin, Data Center GPU Manager (DCGM) exporter, and GPU +health integration described in the +[managed GPU documentation](https://learn.microsoft.com/azure/aks/aks-managed-gpu-nodes). + +The taint keeps general workloads off the GPU nodes. Every GPU workload in this +example includes the matching toleration. + +## Understand the immutable profile + +AKS stores the configuration under `gpuProfile`. The driver, management mode, +and Multi-Instance GPU strategy cannot be changed after pool creation. To +change them, create another node pool. + +The script validates an existing pool before reusing it. It stops if the VM +size, node count, operating system, taint, or managed GPU profile differs from +the expected configuration. + +Managed GPU node pools do not support cluster autoscaler during preview. This +example uses a fixed two-node pool and removes it during cleanup. + +## Checkpoint + +```bash +az aks nodepool show \ + --resource-group "${LAB_RESOURCE_GROUP:-aks-managed-gpu-inference}" \ + --cluster-name "${LAB_CLUSTER:-aks-managed-gpu-inference}" \ + --name a100np \ + --query '{count:count,vmSize:vmSize,osType:osType,gpuProfile:gpuProfile,tags:tags}' +``` + +Expect: + +- `count` is `2`. +- `vmSize` is `Standard_NC24ads_A100_v4`. +- `osType` is `Linux`. +- `gpuProfile.nvidia.managementMode` is `Managed`. +- `tags.aks-example` is `managed-gpu-inference-service`. + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| `AllocationFailed` | The region has no capacity for the VM size; try another region where preflight passes | +| `InsufficientVCPUQuota` | Request quota for the A100 family or total regional virtual CPUs | +| An existing pool does not match | Delete the example-owned pool and rerun the script | +| `gpuProfile.nvidia` is `null` | Recreate the pool with `--enable-managed-gpu=true` | + +## Next step + +[Module 3: Verify GPU access](03-verify.md) diff --git a/examples/managed-gpu-inference-service/modules/03-verify.md b/examples/managed-gpu-inference-service/modules/03-verify.md new file mode 100644 index 000000000..c7cfd3ce0 --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/03-verify.md @@ -0,0 +1,56 @@ +# Module 3: Verify GPU access + +Verify every GPU node before deploying the model server. + +## Run the checks + +```bash +./scripts/30-verify-gpu.sh +``` + +The script checks: + +1. The node pool reports `nvidia.managementMode=Managed`. +2. Both nodes advertise `nvidia.com/gpu`. +3. Both nodes carry the DCGM exporter label. +4. A container pinned to each node creates a CUDA tensor. +5. The DCGM endpoint on each node publishes GPU metrics. + +The DCGM probe uses the host network to reach the node-local metrics endpoint +on port 19400. A cluster that enforces the baseline Pod Security Standard can +block this diagnostic pod. + +The CUDA allocation is deliberate. `nvidia-smi` confirms that the NVIDIA +Management Library can enumerate the device, but it does not prove that a +container can initialize CUDA and allocate GPU memory. + +## Checkpoint + +The final output should name two nodes and end with: + +```output +PASS Managed GPU access verified on 2 node(s) +``` + +You can also inspect the schedulable capacity directly: + +```bash +kubectl get nodes -l agentpool=a100np \ + -o custom-columns='NODE:.metadata.name,READY:.status.conditions[?(@.type=="Ready")].status,GPU:.status.allocatable.nvidia\.com/gpu,DCGM:.metadata.labels.kubernetes\.azure\.com/dcgm-exporter' +``` + +Each node should report `True`, `1`, and `enabled`. + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| No nodes match `agentpool=a100np` | Confirm the pool finished provisioning | +| `nvidia.com/gpu` is empty | Inspect `gpuProfile`; the pool might not use the managed profile | +| The CUDA pod remains `Pending` | Confirm the pod has the GPU taint toleration and the node has free GPU capacity | +| CUDA allocation fails | Delete the validation pod and rerun after the node finishes initializing; inspect node and container events if it fails again | +| DCGM metrics are unavailable | Confirm the node label is `enabled`, then inspect the managed GPU node health | + +## Next step + +[Module 4: Stage the model](04-model-storage.md) diff --git a/examples/managed-gpu-inference-service/modules/04-model-storage.md b/examples/managed-gpu-inference-service/modules/04-model-storage.md new file mode 100644 index 000000000..23ae6fd7d --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/04-model-storage.md @@ -0,0 +1,58 @@ +# Module 4: Stage the model + +Download the model once to an Azure Blob NFS volume that both replicas can +mount. + +## Create storage and stage the weights + +```bash +./scripts/40-stage-model.sh +``` + +The script creates: + +- A dedicated `managed-gpu-inference` namespace. +- A Premium Azure Blob NFS `StorageClass`. +- A 200-GiB `ReadWriteMany` persistent volume claim. +- A Job that downloads and verifies `Qwen/Qwen2.5-7B-Instruct`. + +The Job verifies every safetensors shard listed in the model index before it +reports success. It also removes the Hugging Face transfer cache after +verification because NFS stores the cache as a second physical copy. + +This example downloads +[`Qwen/Qwen2.5-7B-Instruct`](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) +from Hugging Face. Review the model license and terms before use. + +## Why use a separate Job + +An init container would run once per replica. Two replicas could download the +same files concurrently, duplicate network transfer, and write to the same +paths. A separate Job makes staging an explicit prerequisite for serving. + +The serving pods set `HF_HUB_OFFLINE=1`. If staging is incomplete, deployment +fails instead of silently downloading another copy inside a GPU pod. + +## Checkpoint + +```bash +kubectl get pvc -n managed-gpu-inference model-weights +kubectl logs -n managed-gpu-inference job/stage-model +``` + +Expect the claim to report `Bound` with `RWX` access. The final Job output +should report four verified shards and approximately 14 GiB of weights. + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| The claim remains `Pending` | Confirm the Blob CSI driver is enabled and inspect PVC events | +| A pod remains in `ContainerCreating` | Inspect mount events and confirm the StorageClass uses the NFS mount options from the manifest | +| The Job is `OOMKilled` | Confirm the staging container retains its 10-GiB memory limit | +| Model transfer fails | Inspect Job logs, then rerun the script; `snapshot_download` resumes partial files | +| Cleanup refuses an existing object | Choose a clean cluster or remove the conflicting object after confirming its owner | + +## Next step + +[Module 5: Deploy the inference service](05-inference-service.md) diff --git a/examples/managed-gpu-inference-service/modules/05-inference-service.md b/examples/managed-gpu-inference-service/modules/05-inference-service.md new file mode 100644 index 000000000..eaacd26d3 --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/05-inference-service.md @@ -0,0 +1,64 @@ +# Module 5: Deploy the inference service + +Deploy two vLLM replicas that mount the staged model read-only. + +## Deploy and test vLLM + +```bash +./scripts/50-deploy-vllm.sh +``` + +The first rollout usually takes 10–25 minutes while both GPU nodes pull the +image and load the model. The script waits up to 60 minutes. If the rollout +hasn't progressed after about 30 minutes, inspect events and logs from another +terminal. + +The manifest includes: + +- One GPU request per replica. +- Topology spread across host names. +- A PodDisruptionBudget with one replica always available. +- `maxSurge: 0` so an update does not require a third GPU. +- A 20-minute startup probe for model initialization. +- Separate readiness and liveness thresholds. +- An 8-GiB in-memory `/dev/shm` volume. +- A read-only model mount and offline Hugging Face mode. + +## Checkpoint + +```bash +kubectl get pods -n managed-gpu-inference \ + -l app.kubernetes.io/name=vllm \ + -o wide +``` + +Expect two ready pods on different nodes. + +The deployment script also sends an OpenAI-compatible request through the +cluster-internal Service. A successful run ends with: + +```output +PASS Inference request completed +``` + +To test from your computer: + +```bash +kubectl port-forward -n managed-gpu-inference service/vllm 8000:8000 +``` + +Then send the request from the example README. + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| A pod is `Pending` | Check for free `nvidia.com/gpu` capacity and the `sku=gpu:NoSchedule` toleration | +| A pod reports `No CUDA GPUs are available` | Delete the failed pod after the new node finishes initializing so Kubernetes makes a fresh allocation | +| The model path is missing | Rerun `./scripts/40-stage-model.sh` and inspect the staging Job | +| The process exits after loading | Confirm `enableServiceLinks: false`; a Service named `vllm` otherwise injects a conflicting `VLLM_PORT` variable | +| Rollout hasn't progressed after 30 minutes | Inspect pod events and logs with the commands printed by the script; the script times out after 60 minutes | + +## Next step + +[Module 6: Observe the service](06-observability.md) diff --git a/examples/managed-gpu-inference-service/modules/06-observability.md b/examples/managed-gpu-inference-service/modules/06-observability.md new file mode 100644 index 000000000..8ae7f3729 --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/06-observability.md @@ -0,0 +1,65 @@ +# Module 6: Observe the service + +Use device metrics and server metrics together. They answer different +questions. + +| Source | Endpoint | Use it to measure | +| --- | --- | --- | +| DCGM exporter | Node port `19400` | GPU activity, memory, temperature, power, and hardware errors | +| vLLM | Pod port `8000/metrics` | Running requests, queued requests, token throughput, and cache pressure | + +## Read vLLM metrics + +Forward the Service: + +```bash +kubectl port-forward -n managed-gpu-inference service/vllm 8000:8000 +``` + +In another terminal: + +```bash +curl --fail http://127.0.0.1:8000/metrics \ + | grep -E 'vllm:(num_requests_running|num_requests_waiting|kv_cache_usage_perc)' +``` + +`vllm:num_requests_waiting` is the clearest signal that requests are arriving +faster than the replicas can serve them. + +## Read DCGM metrics + +The verification script reads the node-local endpoint from a pod pinned to +each GPU node: + +```bash +./scripts/30-verify-gpu.sh --dcgm-only +``` + +Use `--dcgm-only` after deployment because the two vLLM replicas already hold +all available GPUs. This mode checks the exporter without requesting another +GPU. + +Focus on these metrics: + +| Metric | Meaning | +| --- | --- | +| `DCGM_FI_DEV_GPU_UTIL` | Percentage of time work is active on the GPU | +| `DCGM_FI_DEV_FB_USED` | Framebuffer memory in use | +| `DCGM_FI_DEV_GPU_TEMP` | GPU temperature | +| `DCGM_FI_DEV_POWER_USAGE` | Current power draw | +| `DCGM_FI_DEV_XID_ERRORS` | Driver-reported hardware error count | + +Do not interpret GPU utilization by itself as request throughput. Compare DCGM +with vLLM queue depth and latency before deciding whether the service needs +more capacity or different batching settings. + +## Production monitoring + +This example reads metrics directly to keep the deployment focused. For +central collection and alerting, follow +[Monitor GPU metrics on AKS](https://learn.microsoft.com/azure/aks/monitor-gpu-metrics) +and scrape the managed DCGM exporter into Azure Managed Prometheus. + +## Next step + +[Module 7: Clean up](07-cleanup.md) diff --git a/examples/managed-gpu-inference-service/modules/07-cleanup.md b/examples/managed-gpu-inference-service/modules/07-cleanup.md new file mode 100644 index 000000000..905107dd3 --- /dev/null +++ b/examples/managed-gpu-inference-service/modules/07-cleanup.md @@ -0,0 +1,62 @@ +# Module 7: Clean up + +Remove GPU capacity as soon as you finish. + +## Remove the workload and GPU pool + +```bash +./scripts/90-cleanup.sh +``` + +This path: + +1. Verifies that `kubectl` points to the example cluster. +2. Refuses to delete Kubernetes objects without the example ownership label. +3. Deletes the vLLM workload, staging Job, persistent volume claim, namespace, + and StorageClass. +4. Deletes the managed GPU node pool. +5. Leaves the AKS cluster and CPU system nodes running. + +Use this option when you want to inspect the cluster after completing the +example. + +## Delete every example resource + +```bash +./scripts/90-cleanup.sh --all +``` + +The script deletes the complete resource group only when it has the exact +ownership tag created in Module 1. Azure continues resource group deletion in +the background. + +## Checkpoint + +For standard cleanup: + +```bash +az aks nodepool show \ + --resource-group "${LAB_RESOURCE_GROUP:-aks-managed-gpu-inference}" \ + --cluster-name "${LAB_CLUSTER:-aks-managed-gpu-inference}" \ + --name a100np +``` + +Expect `ResourceNotFound`. + +For complete cleanup: + +```bash +az group show \ + --name "${LAB_RESOURCE_GROUP:-aks-managed-gpu-inference}" +``` + +Expect `ResourceGroupNotFound` after Azure finishes deleting the group. + +## Troubleshoot + +| Problem | Action | +| --- | --- | +| The active context does not match | Rerun `./scripts/10-create-cluster.sh` before standard cleanup | +| An ownership check fails | Inspect the object before deleting it manually; the script will not remove shared resources | +| The persistent volume remains | Check the PVC and storage account deletion state; the StorageClass uses `reclaimPolicy: Delete` | +| Resource group deletion is still running | Query `az group show` until Azure reports that the group no longer exists | diff --git a/examples/managed-gpu-inference-service/scripts/00-preflight.sh b/examples/managed-gpu-inference-service/scripts/00-preflight.sh new file mode 100755 index 000000000..7cb81771e --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/00-preflight.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Check local tools, Azure access, feature registration, SKU availability, and quota. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" + +step "Checking tools" +require_command az "Install the Azure CLI: https://learn.microsoft.com/cli/azure/install-azure-cli" +require_command kubectl "Run: az aks install-cli" +require_command python3 "Install Python 3 from https://python.org/downloads/" + +AZ_VERSION=$(az version --query '"azure-cli"' -o tsv 2>/dev/null || true) +[[ -n "$AZ_VERSION" ]] || fail "Azure CLI did not report a version." +version_ge "$AZ_VERSION" "$MIN_AZ_VERSION" || + fail "Azure CLI $AZ_VERSION is older than $MIN_AZ_VERSION." + +AKS_PREVIEW_VERSION=$(az extension show --name aks-preview --query version -o tsv 2>/dev/null || true) +[[ -n "$AKS_PREVIEW_VERSION" ]] || + fail "aks-preview isn't installed. Run: az extension add --name aks-preview" +version_ge "$AKS_PREVIEW_VERSION" "$MIN_AKS_PREVIEW_VERSION" || + fail "aks-preview $AKS_PREVIEW_VERSION is older than $MIN_AKS_PREVIEW_VERSION." +KUBECTL_VERSION=$(kubectl version --client -o json 2>/dev/null | + python3 -c 'import json,sys; print(json.load(sys.stdin)["clientVersion"]["gitVersion"])' || + true) +[[ -n "$KUBECTL_VERSION" ]] || fail "kubectl did not report a client version." +version_ge "$KUBECTL_VERSION" "$MIN_KUBECTL_VERSION" || + fail "kubectl $KUBECTL_VERSION is older than $MIN_KUBECTL_VERSION." +pass "Azure CLI, aks-preview, kubectl, and Python 3 are ready" + +step "Checking Azure sign-in" +SUBSCRIPTION_ID=$(az account show --query id -o tsv 2>/dev/null || true) +[[ -n "$SUBSCRIPTION_ID" ]] || + fail "Sign in with 'az login', then select a subscription." +SUBSCRIPTION_NAME=$(az account show --query name -o tsv) +pass "Using $SUBSCRIPTION_NAME ($SUBSCRIPTION_ID)" + +step "Checking required resource providers" +for provider in Microsoft.Compute Microsoft.ContainerService Microsoft.Network Microsoft.ManagedIdentity Microsoft.Storage; do + state=$(az provider show \ + --namespace "$provider" \ + --query registrationState \ + -o tsv 2>/dev/null || true) + [[ "$state" == "Registered" ]] || + fail "$provider is ${state:-not registered}. Run: az provider register --namespace $provider" + pass "$provider is registered" +done + +step "Checking managed GPU feature registration" +FEATURE_STATE=$(az feature show \ + --namespace "$FEATURE_NAMESPACE" \ + --name "$FEATURE_NAME" \ + --query properties.state \ + -o tsv 2>/dev/null || true) +[[ "$FEATURE_STATE" == "Registered" ]] || + fail "$FEATURE_NAME is ${FEATURE_STATE:-not registered}. Run 'az feature register --namespace $FEATURE_NAMESPACE --name $FEATURE_NAME', wait for Registered, then rerun preflight." +pass "$FEATURE_NAME is registered" + +step "Propagating the managed GPU feature" +az provider register \ + --namespace "$FEATURE_NAMESPACE" \ + --wait \ + -o none +pass "$FEATURE_NAMESPACE registration is current" + +step "Checking Kubernetes version availability" +VERSION_COUNT=$(az aks get-versions \ + --location "$LAB_LOCATION" \ + --query "length(values[?version=='$LAB_KUBERNETES_VERSION'])" \ + -o tsv) +[[ "$VERSION_COUNT" != "0" ]] || + fail "Kubernetes $LAB_KUBERNETES_VERSION isn't available in $LAB_LOCATION." +pass "Kubernetes $LAB_KUBERNETES_VERSION is available in $LAB_LOCATION" + +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +az vm list-usage --location "$LAB_LOCATION" -o json >"$TMP_DIR/usage.json" + +check_sku() { + local sku=$1 + local count=$2 + local require_gpu=$3 + local role=$4 + local output + local sku_file="$TMP_DIR/${role// /-}.json" + + step "Checking $role SKU $sku" + az vm list-skus \ + --location "$LAB_LOCATION" \ + --size "$sku" \ + --all \ + -o json >"$sku_file" + + output=$(python3 - "$sku_file" "$TMP_DIR/usage.json" "$sku" "$count" "$require_gpu" 2>&1 <<'PY' +import json +import re +import sys + +sku_file, usage_file, expected_name, count, require_gpu = sys.argv[1:] +count = int(count) +require_gpu = require_gpu == "true" +matches = [item for item in json.load(open(sku_file)) if item.get("name") == expected_name] +if not matches: + raise SystemExit(f"{expected_name} isn't listed in this region") +sku = matches[0] +if any(item.get("type") == "Location" for item in sku.get("restrictions", [])): + raise SystemExit(f"{expected_name} isn't available for this subscription in this region") +capabilities = {item["name"]: item["value"] for item in sku.get("capabilities", [])} +try: + vcpus = int(capabilities["vCPUs"]) +except (KeyError, TypeError, ValueError): + raise SystemExit(f"{expected_name} doesn't report a numeric vCPUs capability") +try: + gpus = int(capabilities.get("GPUs", "0")) +except (TypeError, ValueError): + gpus = 0 +if require_gpu and gpus != 1: + raise SystemExit(f"{expected_name} advertises {gpus} GPUs per node; this example requires exactly one") + +def normalize(value): + return re.sub(r"[ _]", "", value).lower() + +family = sku.get("family", "") +quota = next( + ( + item + for item in json.load(open(usage_file)) + if normalize(item.get("name", {}).get("value", "")) == normalize(family) + ), + None, +) +if quota is None: + raise SystemExit(f"quota family {family!r} isn't present in the regional usage list") +used = int(quota["currentValue"]) +limit = int(quota["limit"]) +required = vcpus * count +if limit - used < required: + raise SystemExit(f"{family} has {limit-used} free virtual CPUs; {count} {expected_name} nodes need {required}") +print(f"{expected_name}: {vcpus} virtual CPUs, {gpus} GPU(s), quota {used}/{limit}; room for {count} node(s)") +PY + ) || fail "$output" + + pass "$output" +} + +check_sku "$LAB_SYSTEM_SKU" 2 false "system-node" +check_sku "$LAB_GPU_SKU" "$LAB_GPU_NODE_COUNT" true "GPU-node" + +step "Checking total regional virtual CPU quota" +TOTAL_OUTPUT=$(python3 - "$TMP_DIR/system-node.json" "$TMP_DIR/GPU-node.json" \ + "$TMP_DIR/usage.json" "$LAB_GPU_NODE_COUNT" 2>&1 <<'PY' +import json +import sys + +system_file, gpu_file, usage_file, gpu_count = sys.argv[1:] + +def vcpus(path): + sku = json.load(open(path))[0] + values = {item["name"]: item["value"] for item in sku.get("capabilities", [])} + return int(values["vCPUs"]) + +required = 2 * vcpus(system_file) + int(gpu_count) * vcpus(gpu_file) +quota = next( + ( + item + for item in json.load(open(usage_file)) + if item.get("name", {}).get("value") == "cores" + ), + None, +) +if quota is None: + raise SystemExit("Total Regional vCPUs quota isn't present in the usage list") +used = int(quota["currentValue"]) +limit = int(quota["limit"]) +if limit - used < required: + raise SystemExit(f"Total Regional vCPUs has {limit-used} free; the example needs {required}") +print(f"Total Regional vCPUs quota is {used}/{limit}; the example needs {required}") +PY +) || fail "$TOTAL_OUTPUT" +pass "$TOTAL_OUTPUT" + +step "Result" +pass "Preflight passed. Continue with modules/01-cluster.md." diff --git a/examples/managed-gpu-inference-service/scripts/10-create-cluster.sh b/examples/managed-gpu-inference-service/scripts/10-create-cluster.sh new file mode 100755 index 000000000..83fad7ed0 --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/10-create-cluster.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Create the AKS cluster with a CPU-only system node pool and Blob CSI driver. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" + +step "Creating the resource group" +if az group show --name "$LAB_RESOURCE_GROUP" >/dev/null 2>&1; then + resource_group_is_owned || + fail "$LAB_RESOURCE_GROUP already exists without the $LAB_OWNER_TAG=$LAB_OWNER_VALUE ownership tag." + pass "$LAB_RESOURCE_GROUP already exists and is owned by this example" +else + az group create \ + --name "$LAB_RESOURCE_GROUP" \ + --location "$LAB_LOCATION" \ + --tags "$LAB_OWNER_TAG=$LAB_OWNER_VALUE" \ + -o none + pass "Created and tagged $LAB_RESOURCE_GROUP" +fi + +step "Creating the AKS cluster" +if cluster_exists; then + CLUSTER_JSON=$(az aks show \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --name "$LAB_CLUSTER" \ + -o json) + CLUSTER_RESULT=$(python3 - "$LAB_LOCATION" "$LAB_KUBERNETES_VERSION" "$CLUSTER_JSON" 2>&1 <<'PY' +import json +import sys + +expected_location, expected_version, cluster_json = sys.argv[1:] +cluster = json.loads(cluster_json) +errors = [] +if cluster.get("location") != expected_location: + errors.append( + f"location is {cluster.get('location')}, expected {expected_location}" + ) +actual_version = cluster.get("kubernetesVersion", "") +expected_parts = expected_version.split(".") +actual_parts = actual_version.split(".") +if actual_parts[: len(expected_parts)] != expected_parts: + errors.append( + "Kubernetes version is " + f"{actual_version}, expected {expected_version}" + ) +if errors: + raise SystemExit("; ".join(errors)) +print("compatible") +PY + ) || fail "Existing cluster is incompatible: $CLUSTER_RESULT" + SYSTEM_POOL=$(az aks nodepool show \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --cluster-name "$LAB_CLUSTER" \ + --name system \ + -o json) + SYSTEM_RESULT=$(python3 - "$LAB_SYSTEM_SKU" "$SYSTEM_POOL" 2>&1 <<'PY' +import json +import sys + +expected_sku, pool_json = sys.argv[1:] +pool = json.loads(pool_json) +errors = [] +if pool.get("vmSize") != expected_sku: + errors.append(f"system VM size is {pool.get('vmSize')}, expected {expected_sku}") +if int(pool.get("count", -1)) != 2: + errors.append(f"system node count is {pool.get('count')}, expected 2") +if pool.get("mode") != "System": + errors.append(f"system pool mode is {pool.get('mode')}, expected System") +if pool.get("osType") != "Linux": + errors.append(f"system pool OS type is {pool.get('osType')}, expected Linux") +if errors: + raise SystemExit("; ".join(errors)) +print("compatible") +PY + ) || fail "Existing cluster is incompatible: $SYSTEM_RESULT" + pass "$LAB_CLUSTER already exists in the example-owned resource group" +else + az aks create \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --name "$LAB_CLUSTER" \ + --location "$LAB_LOCATION" \ + --kubernetes-version "$LAB_KUBERNETES_VERSION" \ + --nodepool-name system \ + --node-count 2 \ + --node-vm-size "$LAB_SYSTEM_SKU" \ + --enable-managed-identity \ + --enable-blob-driver \ + --generate-ssh-keys \ + -o none + pass "Created $LAB_CLUSTER" +fi + +LAB_CONTEXT=$(lab_context_name) +az aks get-credentials \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --name "$LAB_CLUSTER" \ + --context "$LAB_CONTEXT" \ + --overwrite-existing \ + -o none + +require_lab_context + +BLOB_DRIVER=$(az aks show \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --name "$LAB_CLUSTER" \ + --query storageProfile.blobCsiDriver.enabled \ + -o tsv) +[[ "$BLOB_DRIVER" == "true" ]] || fail "The Blob CSI driver isn't enabled." + +READY_SYSTEM_NODES=$(ready_node_count "agentpool=system") +[[ "$READY_SYSTEM_NODES" == "2" ]] || + fail "Expected 2 ready system nodes, found $READY_SYSTEM_NODES." + +pass "Connected to $LAB_CLUSTER with 2 ready system nodes and Blob CSI enabled" diff --git a/examples/managed-gpu-inference-service/scripts/20-create-managed-gpu-pool.sh b/examples/managed-gpu-inference-service/scripts/20-create-managed-gpu-pool.sh new file mode 100755 index 000000000..c36e02d92 --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/20-create-managed-gpu-pool.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Create or validate the two-node managed A100 pool used by the service. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" + +require_lab_context + +step "Creating the managed GPU node pool" +POOL_LIST=$(az aks nodepool list \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --cluster-name "$LAB_CLUSTER" \ + -o json) +POOL_JSON=$(python3 - "$LAB_GPU_POOL" "$POOL_LIST" <<'PY' +import json +import sys + +name, pool_list = sys.argv[1:] +pool = next((item for item in json.loads(pool_list) if item.get("name") == name), None) +if pool is not None: + print(json.dumps(pool)) +PY +) + +if [[ -n "$POOL_JSON" ]]; then + VALIDATION=$(python3 - "$LAB_GPU_SKU" "$LAB_GPU_NODE_COUNT" "$LAB_GPU_TAINT" \ + "$LAB_OWNER_TAG" "$LAB_OWNER_VALUE" "$POOL_JSON" 2>&1 <<'PY' +import json +import sys + +( + expected_sku, + expected_count, + expected_taint, + owner_tag, + owner_value, + pool_json, +) = sys.argv[1:] +pool = json.loads(pool_json) +errors = [] +if pool.get("vmSize") != expected_sku: + errors.append(f"VM size is {pool.get('vmSize')}, expected {expected_sku}") +if int(pool.get("count", -1)) != int(expected_count): + errors.append(f"node count is {pool.get('count')}, expected {expected_count}") +if pool.get("osType") != "Linux": + errors.append(f"OS type is {pool.get('osType')}, expected Linux") +if pool.get("mode") != "User": + errors.append(f"mode is {pool.get('mode')}, expected User") +if pool.get("osDiskType") != "Managed": + errors.append(f"OS disk type is {pool.get('osDiskType')}, expected Managed") +if pool.get("enableAutoScaling"): + errors.append("cluster autoscaler is enabled") +if expected_taint not in (pool.get("nodeTaints") or []): + errors.append(f"taint {expected_taint} is missing") +if (pool.get("tags") or {}).get(owner_tag) != owner_value: + errors.append(f"ownership tag {owner_tag}={owner_value} is missing") +profile = pool.get("gpuProfile") or {} +nvidia = profile.get("nvidia") or {} +if profile.get("driver") != "Install": + errors.append(f"GPU driver profile is {profile.get('driver')}, expected Install") +if nvidia.get("managementMode") != "Managed": + errors.append(f"management mode is {nvidia.get('managementMode')}, expected Managed") +if errors: + raise SystemExit("; ".join(errors)) +print("compatible") +PY + ) || fail "Existing pool $LAB_GPU_POOL is incompatible: $VALIDATION" + pass "$LAB_GPU_POOL already exists with the expected configuration" +else + az aks nodepool add \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --cluster-name "$LAB_CLUSTER" \ + --name "$LAB_GPU_POOL" \ + --mode User \ + --node-count "$LAB_GPU_NODE_COUNT" \ + --node-vm-size "$LAB_GPU_SKU" \ + --node-osdisk-type Managed \ + --node-taints "$LAB_GPU_TAINT" \ + --enable-managed-gpu=true \ + --tags "$LAB_OWNER_TAG=$LAB_OWNER_VALUE" \ + -o none + pass "Created $LAB_GPU_POOL" +fi + +step "Waiting for the GPU nodes" +for attempt in $(seq 1 90); do + ready=$(ready_node_count "agentpool=$LAB_GPU_POOL") + [[ "$ready" == "$LAB_GPU_NODE_COUNT" ]] && break + [[ "$attempt" != "90" ]] || fail "The GPU nodes didn't become ready within 30 minutes." + sleep 20 +done + +pass "$LAB_GPU_NODE_COUNT GPU nodes are ready" diff --git a/examples/managed-gpu-inference-service/scripts/30-verify-gpu.sh b/examples/managed-gpu-inference-service/scripts/30-verify-gpu.sh new file mode 100755 index 000000000..ef964968e --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/30-verify-gpu.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Verify the managed profile, CUDA access, and DCGM metrics on every GPU node. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" +ROOT=$(cd "$(dirname "$0")/.." && pwd) + +DCGM_ONLY=false +case "${1:-}" in + "") ;; + --dcgm-only) DCGM_ONLY=true ;; + *) fail "Usage: $0 [--dcgm-only]" ;; +esac + +require_lab_context + +step "Checking the managed GPU profile" +PROFILE=$(az aks nodepool show \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --cluster-name "$LAB_CLUSTER" \ + --name "$LAB_GPU_POOL" \ + --query gpuProfile \ + -o json) + +PROFILE_RESULT=$(python3 - "$PROFILE" 2>&1 <<'PY' +import json +import sys + +profile = json.loads(sys.argv[1]) or {} +nvidia = profile.get("nvidia") or {} +if profile.get("driver") != "Install": + raise SystemExit(f"driver is {profile.get('driver')}, expected Install") +if nvidia.get("managementMode") != "Managed": + raise SystemExit( + f"management mode is {nvidia.get('managementMode')}, expected Managed" + ) +print("driver=Install, nvidia.managementMode=Managed") +PY +) || fail "$PROFILE_RESULT" +pass "$PROFILE_RESULT" + +GPU_NODES=$(kubectl get nodes \ + -l "agentpool=$LAB_GPU_POOL" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') +GPU_NODE_COUNT=$(printf '%s\n' "$GPU_NODES" | grep -c . || true) +[[ "$GPU_NODE_COUNT" == "$LAB_GPU_NODE_COUNT" ]] || + fail "Expected $LAB_GPU_NODE_COUNT GPU nodes, found $GPU_NODE_COUNT." + +step "Creating the validation namespace" +require_owned_object_or_absent namespace "$LAB_NAMESPACE" +kubectl apply -f "$ROOT/manifests/namespace.yaml" >/dev/null + +for node in $GPU_NODES; do + step "Checking $node" + + READY=$(kubectl get node "$node" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') + [[ "$READY" == "True" ]] || fail "$node isn't Ready." + + ALLOCATABLE=$(kubectl get node "$node" \ + -o jsonpath='{.status.allocatable.nvidia\.com/gpu}') + [[ "$ALLOCATABLE" == "1" ]] || + fail "$node advertises ${ALLOCATABLE:-no} GPU, expected 1." + + DCGM_LABEL=$(kubectl get node "$node" \ + -o jsonpath='{.metadata.labels.kubernetes\.azure\.com/dcgm-exporter}') + [[ "$DCGM_LABEL" == "enabled" ]] || + fail "$node doesn't have the managed DCGM exporter label." + + if [[ "$DCGM_ONLY" == "false" ]]; then + VALIDATION_NAME="gpu-validation-${node##*-}" + require_owned_object_or_absent pod "$VALIDATION_NAME" "$LAB_NAMESPACE" + kubectl delete pod "$VALIDATION_NAME" \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true \ + >/dev/null + + sed \ + -e "s/name: gpu-validation-node/name: $VALIDATION_NAME/" \ + -e "s/NODE_NAME/$node/" \ + "$ROOT/manifests/gpu-smoke-test.yaml" | + kubectl apply -f - >/dev/null + + if ! kubectl wait \ + --namespace "$LAB_NAMESPACE" \ + --for=jsonpath='{.status.phase}'=Succeeded \ + "pod/$VALIDATION_NAME" \ + --timeout=900s \ + >/dev/null 2>&1; then + kubectl describe pod "$VALIDATION_NAME" --namespace "$LAB_NAMESPACE" + kubectl logs "$VALIDATION_NAME" --namespace "$LAB_NAMESPACE" || true + fail "CUDA validation failed on $node." + fi + + kubectl logs "$VALIDATION_NAME" --namespace "$LAB_NAMESPACE" | + grep -q "CUDA_OK" || fail "CUDA validation didn't report success on $node." + pass "A container allocated CUDA memory on $node" + fi + + DCGM_NAME="dcgm-validation-${node##*-}" + require_owned_object_or_absent pod "$DCGM_NAME" "$LAB_NAMESPACE" + kubectl delete pod "$DCGM_NAME" \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true \ + >/dev/null + + cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: $DCGM_NAME + namespace: $LAB_NAMESPACE + labels: + app.kubernetes.io/name: dcgm-validation + app.kubernetes.io/part-of: managed-gpu-inference-service + $LAB_OWNER_LABEL: $LAB_OWNER_VALUE +spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + hostNetwork: true + nodeSelector: + kubernetes.io/hostname: $node + tolerations: + - key: sku + operator: Equal + value: gpu + effect: NoSchedule + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: probe + image: mcr.microsoft.com/azurelinux/base/core:3.0 + command: + - /bin/sh + - -c + - curl --fail --silent --max-time 15 http://localhost:19400/metrics + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +EOF + + if ! kubectl wait \ + --namespace "$LAB_NAMESPACE" \ + --for=jsonpath='{.status.phase}'=Succeeded \ + "pod/$DCGM_NAME" \ + --timeout=180s \ + >/dev/null 2>&1; then + kubectl describe pod "$DCGM_NAME" --namespace "$LAB_NAMESPACE" + fail "DCGM validation failed on $node." + fi + + kubectl logs "$DCGM_NAME" --namespace "$LAB_NAMESPACE" | + grep -q "DCGM_FI_DEV_GPU_UTIL" || + fail "DCGM metrics on $node don't include DCGM_FI_DEV_GPU_UTIL." + pass "DCGM metrics are available on $node" + + if [[ "$DCGM_ONLY" == "true" ]]; then + kubectl delete pod "$DCGM_NAME" \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true \ + >/dev/null + else + kubectl delete pod "$VALIDATION_NAME" "$DCGM_NAME" \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true \ + >/dev/null + fi +done + +if [[ "$DCGM_ONLY" == "true" ]]; then + pass "Managed DCGM metrics verified on $GPU_NODE_COUNT node(s)" +else + pass "Managed GPU access verified on $GPU_NODE_COUNT node(s)" +fi diff --git a/examples/managed-gpu-inference-service/scripts/40-stage-model.sh b/examples/managed-gpu-inference-service/scripts/40-stage-model.sh new file mode 100755 index 000000000..1dfb38bef --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/40-stage-model.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Create shared model storage and stage the verified model checkpoint. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" +ROOT=$(cd "$(dirname "$0")/.." && pwd) + +require_lab_context + +get_succeeded_stage_pod() { + kubectl get pods \ + --namespace "$LAB_NAMESPACE" \ + -l job-name=stage-model \ + -o json | + python3 -c ' +import json +import sys + +pods = json.load(sys.stdin).get("items", []) +succeeded = [ + pod + for pod in pods + if pod.get("status", {}).get("phase") == "Succeeded" +] +if succeeded: + succeeded.sort( + key=lambda pod: pod.get("status", {}).get("startTime", ""), + reverse=True, + ) + print(succeeded[0]["metadata"]["name"]) +' +} + +step "Checking object ownership" +require_owned_object_or_absent namespace "$LAB_NAMESPACE" +require_owned_object_or_absent storageclass "$LAB_STORAGE_CLASS" +require_owned_object_or_absent persistentvolumeclaim model-weights "$LAB_NAMESPACE" +require_owned_object_or_absent job stage-model "$LAB_NAMESPACE" + +step "Creating shared model storage" +kubectl apply -f "$ROOT/manifests/namespace.yaml" >/dev/null +kubectl apply -f "$ROOT/manifests/model-storage.yaml" >/dev/null + +if ! kubectl wait \ + --namespace "$LAB_NAMESPACE" \ + --for=jsonpath='{.status.phase}'=Bound \ + persistentvolumeclaim/model-weights \ + --timeout=300s \ + >/dev/null 2>&1; then + kubectl describe persistentvolumeclaim model-weights --namespace "$LAB_NAMESPACE" + fail "The model volume didn't bind within 5 minutes." +fi +pass "The shared model volume is bound" + +step "Staging $LAB_MODEL_ID" +if kubectl get job stage-model --namespace "$LAB_NAMESPACE" >/dev/null 2>&1; then + if [[ "$(kubectl get job stage-model --namespace "$LAB_NAMESPACE" \ + -o jsonpath='{.status.succeeded}' 2>/dev/null || true)" == "1" ]]; then + SUCCEEDED_POD=$(get_succeeded_stage_pod) + [[ -n "$SUCCEEDED_POD" ]] && + kubectl logs "$SUCCEEDED_POD" --namespace "$LAB_NAMESPACE" | + grep -q "MODEL_READY" && + pass "The model is already staged and verified" && + exit 0 + fi + kubectl delete job stage-model \ + --namespace "$LAB_NAMESPACE" \ + --wait=true \ + >/dev/null +fi + +kubectl apply -f "$ROOT/manifests/model-stage-job.yaml" >/dev/null +if ! kubectl wait \ + --namespace "$LAB_NAMESPACE" \ + --for=condition=complete \ + job/stage-model \ + --timeout=3600s \ + >/dev/null 2>&1; then + kubectl describe job stage-model --namespace "$LAB_NAMESPACE" + kubectl logs job/stage-model --namespace "$LAB_NAMESPACE" || true + fail "Model staging didn't complete within 60 minutes." +fi + +SUCCEEDED_POD=$(get_succeeded_stage_pod) +[[ -n "$SUCCEEDED_POD" ]] || + fail "The staging Job completed without a succeeded pod." +kubectl logs "$SUCCEEDED_POD" --namespace "$LAB_NAMESPACE" | + grep -q "MODEL_READY" || + fail "The staging Job completed without the model verification marker." +pass "The model is staged and verified" diff --git a/examples/managed-gpu-inference-service/scripts/50-deploy-vllm.sh b/examples/managed-gpu-inference-service/scripts/50-deploy-vllm.sh new file mode 100755 index 000000000..e611a219b --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/50-deploy-vllm.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Deploy two vLLM replicas and validate one OpenAI-compatible request. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" +ROOT=$(cd "$(dirname "$0")/.." && pwd) + +require_lab_context + +step "Checking object ownership" +require_owned_object_or_absent namespace "$LAB_NAMESPACE" +for kind_name in \ + "serviceaccount vllm" \ + "deployment vllm" \ + "service vllm" \ + "poddisruptionbudget vllm"; do + read -r kind name <<<"$kind_name" + require_owned_object_or_absent "$kind" "$name" "$LAB_NAMESPACE" +done + +[[ "$(kubectl get job stage-model --namespace "$LAB_NAMESPACE" \ + -o jsonpath='{.status.succeeded}' 2>/dev/null || true)" == "1" ]] || + fail "The model staging Job isn't complete. Run 40-stage-model.sh first." + +step "Deploying vLLM" +kubectl apply -f "$ROOT/manifests/vllm-serving.yaml" >/dev/null + +if ! kubectl rollout status \ + deployment/vllm \ + --namespace "$LAB_NAMESPACE" \ + --timeout=3600s; then + kubectl get pods --namespace "$LAB_NAMESPACE" -o wide + kubectl describe pod \ + --namespace "$LAB_NAMESPACE" \ + -l app.kubernetes.io/name=vllm + kubectl logs \ + --namespace "$LAB_NAMESPACE" \ + -l app.kubernetes.io/name=vllm \ + --all-containers \ + --tail=100 || true + fail "The vLLM rollout didn't complete within 60 minutes." +fi + +NODES=$(kubectl get pods \ + --namespace "$LAB_NAMESPACE" \ + -l app.kubernetes.io/name=vllm \ + -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | + sort -u | + grep -c . || true) +[[ "$NODES" == "2" ]] || fail "The two replicas aren't running on separate nodes." +pass "Two vLLM replicas are ready on separate GPU nodes" + +step "Sending an inference request" +PROBE_NAME="vllm-request" +require_owned_object_or_absent pod "$PROBE_NAME" "$LAB_NAMESPACE" +kubectl delete pod "$PROBE_NAME" \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true \ + >/dev/null + +cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: $PROBE_NAME + namespace: $LAB_NAMESPACE + labels: + app.kubernetes.io/name: vllm-request + app.kubernetes.io/part-of: managed-gpu-inference-service + $LAB_OWNER_LABEL: $LAB_OWNER_VALUE +spec: + restartPolicy: Never + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: request + image: mcr.microsoft.com/azurelinux/base/core:3.0 + command: + - /bin/sh + - -c + - | + curl --fail-with-body --silent --max-time 120 \ + http://vllm:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"qwen","messages":[{"role":"user","content":"Reply with exactly: AKS GPU service online."}],"max_tokens":16,"temperature":0}' + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +EOF + +if ! kubectl wait \ + --namespace "$LAB_NAMESPACE" \ + --for=jsonpath='{.status.phase}'=Succeeded \ + "pod/$PROBE_NAME" \ + --timeout=180s \ + >/dev/null 2>&1; then + kubectl describe pod "$PROBE_NAME" --namespace "$LAB_NAMESPACE" + kubectl logs "$PROBE_NAME" --namespace "$LAB_NAMESPACE" || true + fail "The inference request failed." +fi + +RESPONSE=$(kubectl logs "$PROBE_NAME" --namespace "$LAB_NAMESPACE") +RESPONSE_TEXT=$(python3 - "$RESPONSE" 2>&1 <<'PY' +import json +import sys + +response = json.loads(sys.argv[1]) +text = response["choices"][0]["message"]["content"] +if "AKS GPU service online" not in text: + raise SystemExit(f"Unexpected response: {text!r}") +print(text) +PY +) || fail "Inference response validation failed: $RESPONSE_TEXT" +printf '%s\n' "$RESPONSE_TEXT" +kubectl delete pod "$PROBE_NAME" \ + --namespace "$LAB_NAMESPACE" \ + --wait=true \ + >/dev/null +pass "Inference request completed" diff --git a/examples/managed-gpu-inference-service/scripts/90-cleanup.sh b/examples/managed-gpu-inference-service/scripts/90-cleanup.sh new file mode 100755 index 000000000..f8224a469 --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/90-cleanup.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Remove the inference workload and GPU pool, or delete the complete resource group. + +. "$(cd "$(dirname "$0")" && pwd)/lib.sh" +ROOT=$(cd "$(dirname "$0")/.." && pwd) + +[[ "$#" -le 1 ]] || fail "Usage: $0 [--all]" +case "${1:-}" in + "" | --all) ;; + *) fail "Usage: $0 [--all]" ;; +esac + +if [[ "${1:-}" == "--all" ]]; then + step "Deleting resource group $LAB_RESOURCE_GROUP" + resource_group_is_owned || + fail "Refusing to delete $LAB_RESOURCE_GROUP: it isn't tagged $LAB_OWNER_TAG=$LAB_OWNER_VALUE." + az group delete \ + --name "$LAB_RESOURCE_GROUP" \ + --yes \ + --no-wait + pass "Resource group deletion started" + exit 0 +fi + +require_lab_context + +step "Checking Kubernetes object ownership" +objects=( + "serviceaccount vllm $LAB_NAMESPACE" + "deployment vllm $LAB_NAMESPACE" + "service vllm $LAB_NAMESPACE" + "poddisruptionbudget vllm $LAB_NAMESPACE" + "job stage-model $LAB_NAMESPACE" + "persistentvolumeclaim model-weights $LAB_NAMESPACE" + "namespace $LAB_NAMESPACE" + "storageclass $LAB_STORAGE_CLASS" +) +for object in "${objects[@]}"; do + read -r kind name namespace <<<"$object" + require_owned_object_or_absent "$kind" "$name" "${namespace:-}" +done + +step "Deleting the inference workload" +if kubectl get namespace "$LAB_NAMESPACE" >/dev/null 2>&1; then + kubectl delete -f "$ROOT/manifests/vllm-serving.yaml" \ + --ignore-not-found \ + --wait=true + kubectl delete job stage-model \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true + kubectl delete persistentvolumeclaim model-weights \ + --namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true + kubectl delete namespace "$LAB_NAMESPACE" \ + --ignore-not-found \ + --wait=true +fi +kubectl delete storageclass "$LAB_STORAGE_CLASS" \ + --ignore-not-found \ + --wait=true +pass "Deleted the example-owned Kubernetes resources" + +step "Deleting the managed GPU node pool" +POOL_LIST=$(az aks nodepool list \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --cluster-name "$LAB_CLUSTER" \ + -o json) +POOL_JSON=$(python3 - "$LAB_GPU_POOL" "$POOL_LIST" <<'PY' +import json +import sys + +name, pool_list = sys.argv[1:] +pool = next((item for item in json.loads(pool_list) if item.get("name") == name), None) +if pool is not None: + print(json.dumps(pool)) +PY +) + +if [[ -n "$POOL_JSON" ]]; then + POOL_OWNER=$(python3 - "$LAB_OWNER_TAG" "$POOL_JSON" <<'PY' +import json +import sys + +owner_tag, pool_json = sys.argv[1:] +print((json.loads(pool_json).get("tags") or {}).get(owner_tag, "")) +PY +) + [[ "$POOL_OWNER" == "$LAB_OWNER_VALUE" ]] || + fail "Refusing to delete $LAB_GPU_POOL: it isn't tagged $LAB_OWNER_TAG=$LAB_OWNER_VALUE." + az aks nodepool delete \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --cluster-name "$LAB_CLUSTER" \ + --name "$LAB_GPU_POOL" \ + -o none + pass "Deleted $LAB_GPU_POOL" +else + pass "$LAB_GPU_POOL is already absent" +fi + +warn "The AKS cluster and its two CPU system nodes still incur charges." +warn "Run '$ROOT/scripts/90-cleanup.sh --all' to delete the complete resource group." diff --git a/examples/managed-gpu-inference-service/scripts/lib.sh b/examples/managed-gpu-inference-service/scripts/lib.sh new file mode 100755 index 000000000..3a2948b17 --- /dev/null +++ b/examples/managed-gpu-inference-service/scripts/lib.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Shared configuration, ownership checks, and output helpers. +# shellcheck disable=SC2034 + +set -euo pipefail + +: "${LAB_LOCATION:=westus2}" +: "${LAB_RESOURCE_GROUP:=aks-managed-gpu-inference}" +: "${LAB_CLUSTER:=aks-managed-gpu-inference}" +: "${LAB_KUBERNETES_VERSION:=1.35}" + +readonly LAB_SYSTEM_SKU="Standard_D4s_v5" +readonly LAB_GPU_SKU="Standard_NC24ads_A100_v4" +readonly LAB_GPU_POOL="a100np" +readonly LAB_GPU_NODE_COUNT=2 +readonly LAB_GPU_TAINT="sku=gpu:NoSchedule" +readonly LAB_NAMESPACE="managed-gpu-inference" +readonly LAB_STORAGE_CLASS="managed-gpu-model-blob" +readonly LAB_OWNER_LABEL="aks.azure.com/example" +readonly LAB_OWNER_TAG="aks-example" +readonly LAB_OWNER_VALUE="managed-gpu-inference-service" +readonly LAB_MODEL_ID="Qwen/Qwen2.5-7B-Instruct" +readonly MIN_AZ_VERSION="2.85.0" +readonly MIN_AKS_PREVIEW_VERSION="19.0.0b29" +readonly MIN_KUBECTL_VERSION="1.34.0" +readonly FEATURE_NAMESPACE="Microsoft.ContainerService" +readonly FEATURE_NAME="ManagedGPUExperiencePreview" + +if [[ -t 1 ]]; then + readonly GREEN=$'\033[32m' + readonly YELLOW=$'\033[33m' + readonly RED=$'\033[31m' + readonly RESET=$'\033[0m' +else + readonly GREEN="" YELLOW="" RED="" RESET="" +fi + +step() { printf '\n=== %s ===\n' "$1"; } +pass() { printf '%sPASS%s %s\n' "$GREEN" "$RESET" "$1"; } +warn() { printf '%sWARN%s %s\n' "$YELLOW" "$RESET" "$1"; } +fail() { printf '%sFAIL%s %s\n' "$RED" "$RESET" "$1" >&2; exit 1; } + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "$1 isn't installed. $2" +} + +version_ge() { + python3 - "$1" "$2" <<'PY' +import re +import sys + +def parts(value): + return tuple(int(part) for part in re.findall(r"\d+", value)) + +raise SystemExit(0 if parts(sys.argv[1]) >= parts(sys.argv[2]) else 1) +PY +} + +cluster_exists() { + az aks show \ + --resource-group "$LAB_RESOURCE_GROUP" \ + --name "$LAB_CLUSTER" \ + >/dev/null 2>&1 +} + +resource_group_is_owned() { + [[ "$(az group show --name "$LAB_RESOURCE_GROUP" \ + --query "tags.\"$LAB_OWNER_TAG\"" -o tsv 2>/dev/null || true)" == "$LAB_OWNER_VALUE" ]] +} + +lab_context_name() { + local subscription_id + subscription_id=$(az account show --query id -o tsv 2>/dev/null) || + fail "Azure CLI couldn't read the active subscription." + [[ -n "$subscription_id" ]] || + fail "Select an Azure subscription before accessing the cluster." + printf '%s-%s-%s\n' "$LAB_CLUSTER" "$LAB_RESOURCE_GROUP" "$subscription_id" +} + +require_lab_context() { + local context expected + expected=$(lab_context_name) + context=$(kubectl config current-context 2>/dev/null || true) + [[ "$context" == "$expected" ]] || + fail "kubectl context is '$context', expected '$expected'. Run 10-create-cluster.sh." +} + +ready_node_count() { + local selector=$1 + kubectl get nodes -l "$selector" -o json | + python3 -c ' +import json +import sys + +nodes = json.load(sys.stdin).get("items", []) +print( + sum( + any( + condition.get("type") == "Ready" + and condition.get("status") == "True" + for condition in node.get("status", {}).get("conditions", []) + ) + for node in nodes + ) +) +' +} + +object_is_owned() { + local kind=$1 + local name=$2 + local namespace=${3:-} + local args=(get "$kind" "$name") + [[ -z "$namespace" ]] || args+=(--namespace "$namespace") + [[ "$(kubectl "${args[@]}" -o json 2>/dev/null | + python3 -c "import json,sys; print((json.load(sys.stdin).get('metadata', {}).get('labels', {}) or {}).get('$LAB_OWNER_LABEL', ''))" 2>/dev/null || true)" == "$LAB_OWNER_VALUE" ]] +} + +require_owned_object_or_absent() { + local kind=$1 + local name=$2 + local namespace=${3:-} + local args=(get "$kind" "$name") + [[ -z "$namespace" ]] || args+=(--namespace "$namespace") + if kubectl "${args[@]}" >/dev/null 2>&1; then + object_is_owned "$kind" "$name" "$namespace" || + fail "$kind/$name already exists without the $LAB_OWNER_LABEL=$LAB_OWNER_VALUE label." + fi +}