OpenShift Virtualization에서 자동 CPU 확장을 위한 Ansible 이벤트 기반 자동화 활용

---
- name: Process Alertmanager events to scale KubeVirt VM sockets
  hosts: all
  sources:
    - ansible.eda.alertmanager:
        host: 0.0.0.0
        port: 5001
  rules:
    - name: Trigger AAP Job Template Launch on Alert
      condition: event.payload is defined and event.payload.status == "firing"
      actions:
        - run_job_template:
            name: "KubeVirt Increase Socket for a running Virtual Machine"
            organization: Default
            job_args:
              extra_vars:
                vm_name: "{{ event.payload.alerts[0].labels.name }}"
                vm_namespace: "{{ event.payload.alerts[0].labels.namespace }}"

 ---
- name: Increment KubeVirt VM CPU Sockets
  hosts: localhost         # Run on the machine where Ansible is executed
  connection: local       # Interact with the k8s API locally, not via SSH
  gather_facts: false     # No need to gather facts about the localhost

  # Define the target VM details here
  vars:
    vm_name: "my-kubevirt-vm"  # <--- We need to override THIS to our VM name
    vm_namespace: "default"    # <--- We need to override THIS to our VM namespace

  tasks:
    - name: 1. Get current KubeVirt VM object
      kubernetes.core.k8s_info:
        api_version: kubevirt.io/v1
        kind: VirtualMachine
        name: "{{ vm_name }}"
        namespace: "{{ vm_namespace }}"
      register: vm_info
      # Handle potential connection errors or if the VM doesn't exist
      failed_when: "vm_info.resources | length == 0"
      vars:
        # Custom error message if VM not found
        ansible_failed_result:
          msg: "Failed to find KubeVirt VirtualMachine '{{ vm_name }}' in namespace '{{ vm_namespace }}'."

    - name: 2. Extract current socket count and calculate new value
      ansible.builtin.set_fact:
        # Extract the current value. Use default(1) if 'sockets' is not defined in the spec.
        # Ensure it's treated as an integer.
        current_sockets: "{{ (vm_info.resources[0].spec.domain.cpu.sockets | default(1)) | int }}"
        # Calculate the new value
        new_sockets: "{{ ((vm_info.resources[0].spec.domain.cpu.sockets | default(1)) | int) + 1 }}"

    - name: Display socket update plan
      ansible.builtin.debug:
        msg: "VM '{{ vm_name }}' in ns '{{ vm_namespace }}': Planning to update sockets from {{ current_sockets }} to {{ new_sockets }}"

    - name: 3. Patch VM using kubectl patch command
      ansible.builtin.command:
        cmd: >
          kubectl patch virtualmachine {{ vm_name | quote }} -n {{ vm_namespace | quote }}
          --server={{ lookup('env', 'K8S_AUTH_HOST') | quote }}
          --token={{ lookup('env', 'K8S_AUTH_API_KEY')  | quote }}
          --insecure-skip-tls-verify=true
          --type='json'
          -p='[{"op": "replace", "path": "/spec/template/spec/domain/cpu/sockets", "value": {{ new_sockets }} }]'
      register: kubectl_patch_result
      # Check the output of kubectl to see if a change was made
      changed_when: "'patched' in kubectl_patch_result.stdout"

    - name: Show Kubectl Patch Result (if changed)
      ansible.builtin.debug:
        var: kubectl_patch_result.stdout  # Show kubectl output if changed
      when: kubectl_patch_result is defined and kubectl_patch_result.changed

- name: 3. Patch VM with the incremented socket count using JSON Patch
  kubernetes.core.k8s_json_patch:
    api_version: kubevirt.io/v1  # <--- Added api_version
    kind: VirtualMachine
    namespace: "{{ vm_namespace }}"
    name: "{{ vm_name }}"
    patch:
      - op: replace
        path: /spec/template/spec/domain/cpu/sockets
        value: "{{ new_sockets | int }}"
  register: patch_result
  when: new_sockets != current_sockets

Failed to patch object: 
{
   "kind": "Status",
   "apiVersion": "v1",
   "metadata": {},
   "status": "Failure",
   "message": "admission webhook 'mutatevirtualmachines.kubemacpool.io' denied the request: json: cannot unmarshal string into Go struct field VirtualMachineInstanceTemplateSpec.spec.template.spec of type uint32",
   "code": 400
}

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: vm-cpu-load-alert
  namespace: openshift-monitoring
spec:
  groups:
  - name: vmCpuLoad
    rules:
    - alert: cpuVmUsageCritical
      expr: >
        rate(kubevirt_vmi_cpu_usage_seconds_total[1m]) > 1
      for: 1m
      labels:
        severity: 'critical'
        vertautoscale: 'true'
      annotations:
        description: 'Total CPU usage of the VirtualMachine {{ $labels.name }} is more than 100% for the last 1 minute'
        summary: 'Total CPU usage of the VirtualMachine {{ $labels.name }} is more than 100% for the last 1 minute'

global:
  http_config:
    proxy_from_environment: true
inhibit_rules:
  - equal:
      - namespace
      - alertname
    source_matchers:
      - severity = critical
    target_matchers:
      - severity =~ warning|info
  - equal:
      - namespace
      - alertname
    source_matchers:
      - severity = warning
    target_matchers:
      - severity = info
receivers:
  - name: Critical
  - name: Default
  - name: Watchdog
  - name: AnsibleEDA
    webhook_configs:
      - url: >-
          https://aapinternal-aap.apps.ocpcluster.mydomain.local/eda-event-streams/api/eda/v1/external_event_stream/d68a64ed-d547-4021-b384-340df7443929/post/
        http_config:
          basic_auth:
            username: openshift
            password: P4ssw0rd1
        tls_config:
          insecure_skip_verify: true
route:
  group_by:
    - namespace
  group_interval: 5m
  group_wait: 30s
  receiver: Default
  repeat_interval: 12h
  routes:
    - matchers:
        - vertautoscale = true
      receiver: AnsibleEDA
    - matchers:
        - alertname = Watchdog
      receiver: Watchdog
    - receiver: Critical
      matchers:
        - severity = critical

$ stress-ng -c 1 -l 100 -t 7m &