RHEL에서 GPU 가속을 사용한 vLLM 추론 서버 프로파일링

# Install matching kernel development packages, which are required for building the driver modules
# https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/index.html#rhel-installation-network
dnf install -y kernel-devel-matched kernel-headers
# Enable the CodeReady Builder (CRB) repository to access additional development packages
dnf config-manager --set-enabled crb
# Define variables for the distribution and architecture
distro=rhel9
arch=x86_64
# Add the NVIDIA CUDA repository to the system's package manager
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/$distro/$arch/cuda-$distro.repo
# Clear the package manager cache
dnf clean expire-cache
# Enable the open-source, DKMS-based NVIDIA driver module
dnf module enable -y nvidia-driver:open-dkms
# As an alternative, you could use: dnf module enable nvidia-driver:latest-dkms
# Install the NVIDIA open-source driver package
dnf install -y nvidia-open
# To uninstall, you would run: dnf remove -y nvidia-open
# For proprietary drivers, you might install: dnf install cuda-drivers

# Add the NVIDIA Container Toolkit repository
# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | \
  sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
# Install a specific version of the toolkit components for stability
export NVIDIA_CONTAINER_TOOLKIT_VERSION=1.17.8-1
sudo dnf install -y \
      nvidia-container-toolkit-${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      nvidia-container-toolkit-base-${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container-tools-${NVIDIA_CONTAINER_TOOLKIT_VERSION} \
      libnvidia-container1-${NVIDIA_CONTAINER_TOOLKIT_VERSION}
# Generate the CDI specification file, which allows container runtimes to discover the GPU devices
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
# List the available CDI devices to confirm the setup
nvidia-ctk cdi list
# Expected output:
# INFO[0000] Found 3 CDI devices
# nvidia.com/gpu=0
# nvidia.com/gpu=GPU-7e22d269-01f7-77c4-be2c-18969b4eec98
# nvidia.com/gpu=all

# Allow containers to use devices by setting the appropriate SELinux boolean
sudo setsebool -P container_use_devices 1
# Run a CUDA-based container and execute nvidia-smi to verify GPU access
podman run --rm -it \
--security-opt=label=disable \
--device nvidia.com/gpu=all \
nvcr.io/nvidia/cuda:12.4.1-base-ubi9 \
nvidia-smi
# The output should display the GPU status, confirming the setup is correct.
# Tue Sep 16 15:40:13 2025
# +-----------------------------------------------------------------------------------------+
# | NVIDIA-SMI 580.82.07              Driver Version: 580.82.07      CUDA Version: 13.0     |
# +-----------------------------------------+------------------------+----------------------+
# | GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
# | Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
# |                                         |                        |               MIG M. |
# |=========================================+========================+======================|
# |   0  NVIDIA L4                      Off |   00000000:31:00.0 Off |                    0 |
# | N/A   29C    P8             12W /   72W |       0MiB /  23034MiB |      0%      Default |
# |                                         |                        |                  N/A |
# +-----------------------------------------+------------------------+----------------------+
# ...

# Navigate to a directory for storing container data
mkdir -p rhaiis-cache
chmod g+rwX rhaiis-cache
# Create an environment file for your Hugging Face token
echo "export HF_TOKEN=<your_HF_token>" > private.env
source private.env
# Run the vLLM container, mapping ports and volumes.
podman run --rm -it \
--device nvidia.com/gpu=all \
--security-opt=label=disable \
--shm-size=8g -p 8000:8000 \
--userns=keep-id:uid=1001 \
--env "HUGGING_FACE_HUB_TOKEN=$HF_TOKEN" \
--env "HF_HUB_OFFLINE=0" \
--env "VLLM_NO_USAGE_STATS=1" \
-v ./rhaiis-cache:/home/vllm:Z \
registry.redhat.io/rhaiis/vllm-cuda-rhel9:3.2.1 \
--model RedHatAI/Qwen3-8B-FP8-dynamic \
--tensor-parallel-size 1

# In a new terminal, send a test request to the server.
# We limit the output to 50 tokens for this initial test.
curl -X POST -H "Content-Type: application/json" -d '{
"prompt": "What is the capital of France?",
"max_tokens": 50
}' http://127.0.0.1:8000/v1/completions | jq
# A successful response confirms the server is working.
# {
#   "id": "cmpl-206df592b55f47faaa28e4183f6f9f55",
#   ...
#   "usage": {
#     "prompt_tokens": 7,
#     "total_tokens": 57,
#     "completion_tokens": 50
#   }
# }

# Run the container again, this time setting the VLLM_TORCH_PROFILER_DIR environment variable.
# This instructs vLLM to save the resulting trace files to the specified directory inside the container.
# For more granular analysis, vLLM supports additional profiler variables to capture shapes, memory usage, stack traces, and FLOPs.
# --env "VLLM_TORCH_PROFILER_RECORD_SHAPES=1"
# --env "VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY=1"
# --env "VLLM_TORCH_PROFILER_WITH_STACK=1"
# --env "VLLM_TORCH_PROFILER_WITH_FLOPS=1"
podman run --rm -it \
--device nvidia.com/gpu=all \
--security-opt=label=disable \
--shm-size=8g -p 8000:8000 \
--userns=keep-id:uid=1001 \
--env "HUGGING_FACE_HUB_TOKEN=$HF_TOKEN" \
--env "HF_HUB_OFFLINE=0" \
--env "VLLM_NO_USAGE_STATS=1" \
--env "VLLM_TORCH_PROFILER_DIR=/home/vllm" \
-v ./rhaiis-cache:/home/vllm:Z \
registry.redhat.io/rhaiis/vllm-cuda-rhel9:3.2.1 \
--model RedHatAI/Qwen3-8B-FP8-dynamic \
--tensor-parallel-size 1

# In Terminal 2, send the command to start profiling.
curl -X POST http://localhost:8000/start_profile

# Send an inference request. We use a small number of tokens (10)
# to make the resulting trace easier to analyze.
curl -X POST -H "Content-Type: application/json" -d '{
"prompt": "What is the capital of France?",
"max_tokens": 10
}' http://127.0.0.1:8000/v1/completions | jq

# send the command to stop profiling.
curl -X POST http://localhost:8000/stop_profile

# Check the cache directory for the generated trace file.
ls -hla rhaiis-cache
# total 5.5M
# drwxrwxr-x. 4 root root   92 Sep 17 03:59 .
# ...
# -rw-r--r--. 1 2000 root 5.5M Sep 17 03:59 299a41dbe2d9_111.1758081594179965003.pt.trace.json.gz

class Qwen3Model(Qwen2Model):
    ...

# ...
        for idx, layer in enumerate(
                islice(self.layers, self.start_layer, self.end_layer)):
            # ...
            hidden_states, residual = layer(positions, hidden_states, residual)
# ...

# ...
    def forward(
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
        residual: Optional[torch.Tensor],
    ) -> tuple[torch.Tensor, torch.Tensor]:
        # Self Attention
        # ...
        hidden_states = self.self_attn(...)
        # Fully Connected
        # ...
        hidden_states = self.mlp(hidden_states)
        return hidden_states, residual
# ...

# nsys.dockerfile
FROM registry.redhat.io/rhaiis/vllm-cuda-rhel9:3.2.1
USER root
# Add NVIDIA developer repository for Nsight Systems
RUN rpm --import https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
RUN microdnf install -y dnf && microdnf clean all
RUN microdnf install -y dnf-plugins-core && microdnf clean all
RUN dnf config-manager --add-repo "https://developer.download.nvidia.com/devtools/repos/rhel$(source /etc/os-release; echo ${VERSION_ID%%.*})/$(rpm --eval '%{_arch}' | sed s/aarch/arm/)/"
# Install the Nsight Systems CLI
RUN dnf install -y nsight-systems-cli
USER 2000
# Set the entrypoint to automatically wrap the vLLM server with nsys profile
ENTRYPOINT ["nsys", "profile", "-o", "/home/vllm/report.nsys-rep", "--trace-fork-before-exec=true", "--cuda-graph-trace=node", "python3", "-m", "vllm.entrypoints.openai.api_server"]

podman build -t quay.io/wangzheng422/qimgs:vllm-cuda-rhel9-3.2.1-nsys-2025.09.17-v01 -f nsys.dockerfile .
podman push quay.io/wangzheng422/qimgs:vllm-cuda-rhel9-3.2.1-nsys-2025.09.17-v01

# Launch the custom container. The ENTRYPOINT is configured to wrap the vLLM
# server command with `nsys profile`, so profiling starts automatically.
podman run --rm -it \
--device nvidia.com/gpu=all \
--security-opt=label=disable \
--shm-size=8g -p 8000:8000 \
--userns=keep-id:uid=1001 \
--env "HUGGING_FACE_HUB_TOKEN=$HF_TOKEN" \
--env "HF_HUB_OFFLINE=0" \
--env "VLLM_NO_USAGE_STATS=1" \
-v ./rhaiis-cache:/home/vllm:Z \
quay.io/wangzheng422/qimgs:vllm-cuda-rhel9-3.2.1-nsys-2025.09.17-v01 \
--model RedHatAI/Qwen3-8B-FP8-dynamic \
--tensor-parallel-size 1

# In a new terminal, send an inference request to trigger the profiled code path.
# This ensures that the Nsight trace captures the GPU activity during model execution.
curl -X POST -H "Content-Type: application/json" -d '{
"prompt": "What is the capital of France?",
"max_tokens": 10
}' http://127.0.0.1:8000/v1/completions | jq

# Press Ctrl+C in the container's terminal. Nsight will gracefully shut down
# and save the profiling report.
# ^CINFO 09-17 04:14:28 [launcher.py:101] Shutting down FastAPI HTTP server.
# ...
# Generating '/tmp/nsys-report-b99f.qdstrm'
# [1/1] [========================100%] report.nsys-rep
# Generated:
#         /home/vllm/report.nsys-rep

# Verify that the report file was created in the host directory.
ls -hla rhaiis-cache/
# total 17M
# ...
# -rw-rw-r--. 1 2000 root 17M Sep 17 04:14 report.nsys-rep