infra: introduce terraform/opentofu for proxmox management

- move ansible project within infra
- introduce terraform/opentofu for proxmox VM management

infra/ansible/.gitignore

@@ -0,0 +1 @@
+secrets/

infra/ansible/README.md

@@ -0,0 +1,110 @@
+# Ansible Playbook for Proxmox VM Management
+
+This Ansible playbook automates the creation, deletion, and configuration of
+virtual machines (VMs) on a Proxmox server.
+
+## Prerequisites
+
+- Ansible installed on the local machine
+- Ansible community.general.proxmox_kvm module
+- Access to a Proxmox server with API access enabled
+- Python `proxmoxer` library installed (`pip install proxmoxer`)
+
+## Setup
+
+1. Clone this repository:
+    ```sh
+    git clone https://github.com/TheTaqiTahmid/proxmox_ansible_automation
+    ```
+
+2. Update the `inventory` file with your Proxmox server details:
+    ```yaml
+    all:
+      hosts:
+        proxmox:
+          ansible_host: your_proxmox_ip
+          ansible_user: your_proxmox_user
+          ansible_password: your_proxmox_password
+    ```
+    In the current example implementation in `inventories/hosts.yaml`, there are
+    multiple groups depending on the types of hosts.
+
+3. Add group-related variables to the group file under the `group_vars` directory
+   and individual host-related variables to the files under the `host_vars`
+  directory. Ansible will automatically pick up these variables.
+
+4. Add the following secrets to the ansible-vault:
+   - proxmox_api_token_id
+   - proxmox_api_token
+   - ansible_proxmox_user
+   - ansible_vm_user
+   - proxmox_user
+   - ansible_ssh_private_key_file
+   - ciuser
+   - cipassword
+
+  One can create the secret file using the following command:
+  ```sh
+  ansible-vault create secrets/vault.yml
+  ```
+
+  To encrypt and decrypt the file, use the following commands:
+  ```sh
+  ansible-vault encrypt secrets/vault.yml
+  ansible-vault decrypt secrets/vault.yml
+  ```
+  The password for vault file can be stored in a file or can be provided during
+  the encryption/decryption process. The password file location can be specified
+  in the `ansible.cfg` file.
+
+## Playbooks
+
+### Create VM
+
+To create the VMs, run the following command:
+```sh
+ansible-playbook playbooks/create-vms.yaml
+```
+The playbook can be run against specific Proxmox instance using:
+```sh
+ansible-playbook playbooks/create-vms.yaml --limit proxmox1
+```
+
+### Delete VM
+
+To delete existing VMs, run the following command:
+```sh
+ansible-playbook playbooks/destroy-vms.yaml
+```
+
+Similarly the destory playbook can be run against specific Proxmox instance using:
+```sh
+ansible-playbook playbooks/destroy-vms.yaml --limit proxmox1
+```
+
+### Configure VM
+
+To configure an existing VM, run the following command:
+```sh
+ansible-playbook playbooks/configure-vms.yaml
+```
+
+The configuration can be limited to individual VMs using limits:
+```sh
+ansible-playbook playbooks/configure-vms.yaml --limit vm6
+```
+
+## Variables
+
+The playbooks use the following variables, which can be customized in the
+`group_vars/proxmox.yml` file:
+
+- `vm_id`: The ID of the VM
+- `vm_name`: The name of the VM
+- `vm_memory`: The amount of memory for the VM
+- `vm_cores`: The number of CPU cores for the VM
+- `vm_disk_size`: The size of the VM disk
+
+## Author
+
+- Taqi Tahmid (mdtaqitahmid@gmail.com)

infra/ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+inventory = ./inventory/hosts.yaml
+roles_path = ./roles
+host_key_checking = False
+vault_password_file = ~/.ansible_vault_pass

infra/ansible/inventory/group_vars/all.yaml

@@ -0,0 +1,11 @@
+# Proxmox access related variables
+proxmox_api_url: "192.168.1.121"
+
+# Cloud-init image related variables
+image_url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+image_dest: "/tmp/cloud-image.img"
+image_format: "qcow2"
+storage_name: "local"
+
+# ansible venv
+ansible_venv: "/home/taqi/.venv/ansible/bin/python"

infra/ansible/inventory/group_vars/vms.yaml

@@ -0,0 +1,22 @@
+apt_packages:
+  - curl
+  - vim
+  - htop
+
+# Kubernetes k0sctl configuration vars
+master1_ip: "192.168.1.151"
+master1_hostname: "vm6"
+master2_ip: "192.168.1.161"
+master2_hostname: "vm8"
+worker1_ip: "192.168.1.152"
+worker1_hostname: "vm7"
+worker2_ip: "192.168.1.162"
+worker2_hostname: "vm9"
+
+pod_CIDR: "10.244.0.0/16"
+service_CIDR: "10.96.0.0/12"
+metallb_ip_range: "192.168.1.201-192.168.1.220"
+
+k0s_version: "v1.33.2+k0s.0"
+metallb_version: "0.15.2"
+traefik_version: "36.2.0"

infra/ansible/inventory/host_vars/proxmox1.yaml

@@ -0,0 +1,29 @@
+# VM related variables
+vm_list:
+  - id: 106
+    name: "vm6"
+    memory: 4096
+    cores: 2
+    disk_size: 30G
+    ip: "192.168.1.151/24"
+    gateway: "192.168.1.1"
+    nameserver1: "192.168.1.145"
+    nameserver2: "1.1.1.1"
+  - id: 107
+    name: "vm7"
+    memory: 4096
+    cores: 2
+    disk_size: 30G
+    ip: "192.168.1.152/24"
+    gateway: "192.168.1.1"
+    nameserver1: "192.168.1.145"
+    nameserver2: "1.1.1.1"
+
+# cloud-init variables
+node: "homeserver1"
+net0: "virtio,bridge=vmbr0"
+# disk_name: "local:1000/vm-1000-disk-0.raw,discard=on"
+disk_path: "/var/lib/vz/images/1000"
+ide2: "local:cloudinit,format=qcow2"
+boot_order: "order=scsi0"
+scsi_hw: "virtio-scsi-pci"

infra/ansible/inventory/host_vars/proxmox2.yaml

@@ -0,0 +1,29 @@
+# VM related variables
+vm_list:
+  - id: 206
+    name: "vm8"
+    memory: 4096
+    cores: 2
+    disk_size: 30G
+    ip: "192.168.1.161/24"
+    gateway: "192.168.1.1"
+    nameserver1: "192.168.1.145"
+    nameserver2: "1.1.1.1"
+  - id: 207
+    name: "vm9"
+    memory: 4096
+    cores: 2
+    disk_size: 30G
+    ip: "192.168.1.162/24"
+    gateway: "192.168.1.1"
+    nameserver1: "192.168.1.145"
+    nameserver2: "1.1.1.1"
+
+# cloud-init template variables
+node: "homeserver2"
+net0: "virtio,bridge=vmbr0"
+# disk_name: "local:2000/vm-2000-disk-0.raw,discard=on"
+disk_path: "/var/lib/vz/images/2000"
+ide2: "local:cloudinit,format=qcow2"
+boot_order: "order=scsi0"
+scsi_hw: "virtio-scsi-pci"

infra/ansible/inventory/hosts.yaml

@@ -0,0 +1,51 @@
+all:
+  children:
+    hypervisors:
+    vms:
+
+hypervisors:
+  children:
+    server1:
+    server2:
+
+server1:
+  hosts:
+    proxmox1:
+      ansible_host: 192.168.1.121
+      ansible_user:  "{{ ansible_proxmox_user }}"
+      ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file }}"
+
+server2:
+  hosts:
+    proxmox2:
+      ansible_host: 192.168.1.122
+      ansible_user:  "{{ ansible_proxmox_user }}"
+      ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file }}"
+
+vms:
+  children:
+    vm_group_1:
+    vm_group_2:
+
+vm_group_1:
+  hosts:
+    vm6:
+      ansible_host: 192.168.1.151
+      ansible_user: "{{ ansible_vm_user }}"
+      ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file }}"
+    vm7:
+      ansible_host: 192.168.1.152
+      ansible_user: "{{ ansible_vm_user }}"
+      ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file }}"
+
+vm_group_2:
+  hosts:
+    vm8:
+      ansible_host: 192.168.1.161
+      ansible_user: "{{ ansible_vm_user }}"
+      ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file }}"
+    vm9:
+      ansible_host: 192.168.1.162
+      ansible_user: "{{ ansible_vm_user }}"
+      ansible_ssh_private_key_file: "{{ ansible_ssh_private_key_file }}"
+

infra/ansible/playbooks/configure-vms.yaml

@@ -0,0 +1,6 @@
+- name: Configure Proxmox VMs
+  hosts: vms
+  vars_files:
+    - ../secrets/vault.yaml # Load the encrypted vault file
+  roles:
+    - configure-vms

infra/ansible/playbooks/create-kubernetes-cluster.yaml

@@ -0,0 +1,6 @@
+- name: Create Kubernetes Cluster
+  hosts: vms
+  vars_files:
+    - ../secrets/vault.yaml
+  roles:
+    - create-kubernetes-cluster

infra/ansible/playbooks/create-vms.yaml

@@ -0,0 +1,6 @@
+- name: Create Proxmox VMs
+  hosts: hypervisors
+  vars_files:
+    - ../secrets/vault.yaml  # Load the encrypted vault file
+  roles:
+    - create-vms

infra/ansible/playbooks/destroy-vms.yaml

@@ -0,0 +1,6 @@
+- name: Destroy Proxmox VMs
+  hosts: hypervisors
+  vars_files:
+    - ../secrets/vault.yaml  # Load the encrypted vault file
+  roles:
+    - destroy-vms

infra/ansible/roles/configure-vms/tasks/main.yaml

@@ -0,0 +1,11 @@
+---
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: yes
+  become: true
+
+- name: Install necessary packages
+  ansible.builtin.apt:
+    name: "{{ apt_packages }}"
+    state: present
+  become: true

infra/ansible/roles/create-kubernetes-cluster/tasks/main.yaml

@@ -0,0 +1,97 @@
+- name: Remove known_hosts file if it exists
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.file:
+    path: /home/taqi/.ssh/known_hosts
+    state: absent
+
+- name: Remove k0ctl lock file if it exists
+  ansible.builtin.file:
+    path: /run/lock/k0sctl
+    state: absent
+  become: true
+
+- name: Install k0sctl on host
+  delegate_to: localhost
+  ansible.builtin.command:
+    cmd: "go install github.com/k0sproject/k0sctl@latest"
+
+- name: Ensure k0sctl is installed on host
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.command:
+    cmd: "k0sctl version"
+  register: k0sctl_version
+  changed_when: false
+
+- name: Generate k0sctl configuration file
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.template:
+    src: k0sctl.yaml.j2
+    dest: /tmp/k0sctl.yaml
+  when: k0sctl_version is defined
+  tags:
+    - generate-k0sctl-config
+
+- name: Generate MetalLB IP Address Pool configuration file
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.template:
+    src: ipAddressPool.yaml.j2
+    dest: /tmp/ipAddressPool.yaml
+  when: k0sctl_version is defined
+  tags:
+    - generatemetallb-ippool
+    - metallb-ippool
+
+- name: Create Cluster using k0sctl from host
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.command:
+    cmd: "k0sctl apply --config /tmp/k0sctl.yaml"
+  when: k0sctl_version is defined
+
+- name: Save kubeconfig file on host
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.shell:
+    cmd: "cd /tmp && k0sctl kubeconfig > /home/taqi/.kube/k0s_config.yaml"
+  register: kubeconfig_result
+  retries: 3
+  delay: 5
+  until: kubeconfig_result.rc == 0
+  when: k0sctl_version is defined
+  tags:
+    - generate-kubeconfig
+
+- name: Apply IP Pool for MetalLB from host
+  delegate_to: localhost
+  run_once: true
+  ansible.builtin.shell:
+    cmd: "kubectl apply -f /tmp/ipAddressPool.yaml --kubeconfig /home/taqi/.kube/k0s_config.yaml"
+  register: metallb_ippool_result
+  retries: 3
+  delay: 5
+  until: metallb_ippool_result.rc == 0
+  when: k0sctl_version is defined
+  tags:
+    - metallb-ippool
+
+- name: Cleanup temporary files
+  delegate_to: localhost
+  run_once: true
+  block:
+    - name: Remove k0sctl.yaml temporary file
+      ansible.builtin.file:
+        path: /tmp/k0sctl.yaml
+        state: absent
+
+    - name: Remove ipAddressPool.yaml temporary file
+      ansible.builtin.shell:
+        cmd: "rm -f /tmp/ipAddressPool.yaml"
+  delegate_to: localhost
+  run_once: true
+  tags:
+    - cleanup
+  when: k0sctl_version is defined

infra/ansible/roles/create-kubernetes-cluster/templates/ipAddressPool.yaml.j2

@@ -0,0 +1,8 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: first-pool
+  namespace: metallb-system
+spec:
+  addresses:
+  - "{{ metallb_ip_range }}"

infra/ansible/roles/create-kubernetes-cluster/templates/k0sctl.yaml.j2

@@ -0,0 +1,70 @@
+apiVersion: k0sctl.k0sproject.io/v1beta1
+kind: Cluster
+metadata:
+  name: k0s-cluster
+spec:
+  hosts:
+    - ssh:
+        address: "{{ master1_ip }}"
+        user: "{{ ansible_vm_user }}"
+        keyPath: "{{ ansible_ssh_private_key_file }}"
+      role: controller+worker
+      hostname: "{{ master1_hostname }}"
+      noTaints: true
+    - ssh:
+        address: "{{ master2_ip }}"
+        user: "{{ ansible_vm_user }}"
+        keyPath: "{{ ansible_ssh_private_key_file }}"
+      role: controller+worker
+      hostname: "{{ master2_hostname }}"
+      noTaints: true
+    - ssh:
+        address: "{{ worker1_ip }}"
+        user: "{{ ansible_vm_user }}"
+        keyPath: "{{ ansible_ssh_private_key_file }}"
+      role: worker
+      hostname: "{{ worker1_hostname }}"
+    - ssh:
+        address: "{{ worker2_ip }}"
+        user: "{{ ansible_vm_user }}"
+        keyPath: "{{ ansible_ssh_private_key_file }}"
+      role: worker
+      hostname: "{{ worker2_hostname }}"
+  k0s:
+    version: "{{ k0s_version }}"
+    config:
+      spec:
+        api:
+          address: "{{ master1_ip }}"
+          port: 6443
+          k0sApiPort: 9443
+          sans:
+            - "{{ master1_ip }}"
+            - "{{ master2_ip }}"
+            - k8s.local
+            - api.k8s.local
+        network:
+          kubeProxy:
+            mode: iptables
+          kuberouter:
+            disabled: false
+          podCIDR: "{{ pod_CIDR }}"
+          serviceCIDR: "{{ service_CIDR }}"
+          provider: kuberouter
+        extensions:
+          helm:
+            concurrencyLevel: 5
+            repositories:
+              - name: metallb
+                url: https://metallb.github.io/metallb
+              - name: traefik
+                url: https://traefik.github.io/charts
+            charts:
+              - name: metallb
+                chartname: metallb/metallb
+                version: "{{ metallb_version }}"
+                namespace: metallb-system
+              - name: traefik
+                chartname: traefik/traefik
+                version: "{{ traefik_version }}"
+                namespace: traefik-system

infra/ansible/roles/create-vms/tasks/main.yaml

@@ -0,0 +1,70 @@
+---
+- name: Download cloud image
+  get_url:
+    url: "{{ image_url }}"
+    dest: "{{ image_dest }}"
+    use_netrc: yes
+
+- name: create VMs
+  delegate_to: localhost
+  vars:
+    ansible_python_interpreter: /home/taqi/.venv/ansible/bin/python
+  community.general.proxmox_kvm:
+    api_host: "{{ proxmox_api_url }}"
+    api_user: "{{ proxmox_user }}"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token }}"
+    node: "{{ node }}"
+    vmid: "{{ item.id }}"
+    name: "{{ item.name }}"
+    memory: "{{ item.memory }}"
+    cores: "{{ item.cores }}"
+    scsihw: "{{ scsi_hw }}"
+    boot: "{{ boot_order }}"
+    net:
+      net0: "{{ net0 }}"
+    ipconfig:
+      ipconfig0: "ip={{ item.ip }},gw={{ item.gateway }}"
+    ide:
+      ide2: "{{ ide2 }}"
+    nameservers: "{{ item.nameserver1 }},{{ item.nameserver2 }}"
+    ciuser: "{{ ciuser }}"
+    cipassword: "{{ cipassword }}"
+    sshkeys: "{{ lookup('file', '/home/taqi/.ssh/homeserver.pub') }}"
+  loop: "{{ vm_list }}"
+
+- name: Import disk image
+  ansible.builtin.shell: |
+    qm importdisk "{{ item.id }}" "{{ image_dest }}" "{{ storage_name }}" --format "{{ image_format }}"
+  loop: "{{ vm_list }}"
+
+- name: Attach disk to VM
+  ansible.builtin.shell: |
+    qm set "{{ item.id }}" --scsi0 "{{ storage_name }}:{{ item.id }}/vm-{{ item.id }}-disk-0.{{ image_format }},discard=on"
+  loop: "{{ vm_list }}"
+
+- name: Resize disk
+  ansible.builtin.shell: |
+    qm resize {{ item.id }} scsi0 {{ item.disk_size }}
+  loop: "{{ vm_list }}"
+
+- name: Start VMs
+  delegate_to: localhost
+  vars:
+    ansible_python_interpreter: /home/taqi/.venv/ansible/bin/python
+  community.general.proxmox_kvm:
+    api_host: "{{ proxmox_api_url }}"
+    api_user: "{{ proxmox_user }}"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token }}"
+    node: "{{ node }}"
+    name: "{{ item.name }}"
+    state: started
+  loop: "{{ vm_list }}"
+  tags:
+    - start_vms
+
+- name: Clean up downloaded image
+  file:
+    path: "{{ image_dest }}"
+    state: absent

infra/ansible/roles/destroy-vms/tasks/main.yaml

@@ -0,0 +1,72 @@
+- name: Get VM current state
+  delegate_to: localhost
+  vars:
+    ansible_python_interpreter: "{{ ansible_venv }}"
+  community.general.proxmox_kvm:
+    api_host: "{{ proxmox_api_url }}"
+    api_user: "{{ proxmox_user }}"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token }}"
+    name: "{{ item.name }}"
+    node: "{{ node }}"
+    state: current
+  register: vm_state
+  ignore_errors: yes
+  loop: "{{ vm_list }}"
+  loop_control:
+    index_var: vm_index
+  tags:
+    - vm_delete
+
+- name: Debug VM state
+  debug:
+    msg: "VM {{ item.name }} state: {{ vm_state.results[vm_index].status }}"
+  when: vm_state.results[vm_index] is defined and vm_state.results[vm_index] is succeeded
+  loop: "{{ vm_list }}"
+  loop_control:
+    index_var: vm_index
+
+- name: Stop VM
+  delegate_to: localhost
+  vars:
+    ansible_python_interpreter: "{{ ansible_venv }}"
+  community.general.proxmox_kvm:
+    api_host: "{{ proxmox_api_url }}"
+    api_user: "{{ proxmox_user }}"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token }}"
+    name: "{{ item.name }}"
+    node: "{{ node }}"
+    state: stopped
+    force: true
+  when: >
+    vm_state.results[vm_index] is defined and
+    vm_state.results[vm_index] is succeeded and
+    vm_state.results[vm_index].status != 'absent'
+  loop: "{{ vm_list }}"
+  loop_control:
+    index_var: vm_index
+  tags:
+    - vm_delete
+
+- name: Delete VM
+  delegate_to: localhost
+  vars:
+    ansible_python_interpreter: "{{ ansible_venv }}"
+  community.general.proxmox_kvm:
+    api_host: "{{ proxmox_api_url }}"
+    api_user: "{{ proxmox_user }}"
+    api_token_id: "{{ proxmox_api_token_id }}"
+    api_token_secret: "{{ proxmox_api_token }}"
+    name: "{{ item.name }}"
+    node: "{{ node }}"
+    state: absent
+  when: >
+    vm_state.results[vm_index] is defined and
+    vm_state.results[vm_index] is succeeded and
+    vm_state.results[vm_index].status != 'absent'
+  loop: "{{ vm_list }}"
+  loop_control:
+    index_var: vm_index
+  tags:
+    - vm_delete