Add hawser role and playbook for stack upgrades

.gitignore

@@ -1,5 +1,5 @@
 hosts
 proxmox.*
 .vscode
-inventory
-.vault_pass
+.vault_pass
+logs

ansible.cfg

@@ -2,9 +2,6 @@
 host_key_checking = False
 roles_path = ./roles:/etc/ansible/roles
 group_vars = ./group_vars
-forks = 20
-strategy = free
 
 [ssh_connection]
-ssh_args = -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlMaster=auto -o ControlPersist=1800s
-pipelining = True
+ssh_args = -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null

bootstrap.yml

@@ -1,6 +1,6 @@
 - hosts: all
   become: true
-  remote_user: "{{ user|default('admin') }}"
+  user: admin
   tasks:
     - name: Verify if system is Debian
       debug:

docker/Dockerfile

@@ -1,34 +1,35 @@
-FROM debian:trixie-slim
-ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=Europe/Berlin
-ENV ANSIBLE_VERSION=2.20.6
-ENV PIP_BREAK_SYSTEM_PACKAGES=1
-
-RUN apt update && apt install -y --no-install-recommends \
-    nano \
-    git \
-    wget \
-    unzip \
-    curl \
-    python3-pip \
-    tzdata \
-    openssh-client \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /etc/ssh/ssh_config.d \
-    && printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > /etc/ssh/ssh_config.d/99-no-hostkey-check.conf
-
-RUN pip3 install --no-cache-dir --upgrade \
-    ansible-core~=${ANSIBLE_VERSION}
-
-RUN mkdir -p /root/.bashrc.d
-COPY ansible-functs.sh /root/.bashrc.d/ansible-functs.sh
-RUN chmod +x /root/.bashrc.d/ansible-functs.sh \
-    && printf '\n[ -f /root/.bashrc.d/ansible-functs.sh ] && . /root/.bashrc.d/ansible-functs.sh\n' >> /root/.bashrc
-
-COPY requirements.yml /ansible/requirements.yml
-RUN ansible-galaxy collection install -r /ansible/requirements.yml
-
-WORKDIR /ansible
-
+FROM debian:trixie-slim
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Berlin
+ENV ANSIBLE_VERSION=2.20.6
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN apt update && apt install -y --no-install-recommends \
+    nano \
+    git \
+    wget \
+    unzip \
+    curl \
+    python3-pip \
+    tzdata \
+    openssh-client \
+    iputils-ping \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /etc/ssh/ssh_config.d \
+    && printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > /etc/ssh/ssh_config.d/99-no-hostkey-check.conf
+
+RUN pip3 install --no-cache-dir --upgrade \
+    ansible-core~=${ANSIBLE_VERSION}
+
+RUN mkdir -p /root/.bashrc.d
+COPY ansible-functs.sh /root/.bashrc.d/ansible-functs.sh
+RUN chmod +x /root/.bashrc.d/ansible-functs.sh \
+    && printf '\n[ -f /root/.bashrc.d/ansible-functs.sh ] && . /root/.bashrc.d/ansible-functs.sh\n' >> /root/.bashrc
+
+COPY requirements.yml /ansible/requirements.yml
+RUN ansible-galaxy collection install -r /ansible/requirements.yml
+
+WORKDIR /ansible
+
 CMD ["ansible-playbook", "--version"]

docker/requirements.yml

@@ -1,7 +1,7 @@
-collections:
-  - name: ansible.posix
-    version: 2.2.0
-  - name: community.general
-    version: 13.0.1
-  - name: community.docker
+collections:
+  - name: ansible.posix
+    version: 2.2.0
+  - name: community.general
+    version: 13.0.1
+  - name: community.docker
     version: 5.2.0

inventory/dmc12.yml

@@ -0,0 +1,37 @@
+debian:
+  hosts:
+    rp:
+      ansible_host: 192.168.212.50
+    bitwarden:
+      ansible_host: 192.168.212.20
+    pdf:
+      ansible_host: 192.168.212.65
+    mailcow:
+      ansible_host: 192.168.212.70
+    plex:
+      ansible_host: 192.168.212.62
+    cloud:
+      ansible_host: 192.168.212.60
+    romm: 
+      ansible_host: 192.168.212.64
+    gitea:
+      ansible_host: 192.168.212.30
+      ssh_service_name: ssh
+    ipam:
+      ansible_host: 192.168.212.56
+      ssh_service_name: ssh
+    teamspeck6:
+      ansible_host: 192.168.212.75
+    wiki:
+      ansible_host: 192.168.212.55
+    patchmon:
+      ansible_host: 192.168.212.80
+    einstein-dns:
+      ansible_host: 192.168.212.53
+
+proxmox:
+  hosts:
+    dmc12:
+      ansible_host: 192.168.212.10
+    pbs: 
+      ansible_host: 192.168.212.11

inventory/external.yml

@@ -0,0 +1,10 @@
+debian:
+  hosts:
+    healthcheck:
+      ansible_host: 2a01:4f8:1c1e:62dd::1
+    netbird:
+      ansible_host: 2a01:4f8:1c1c:8c8::1
+    teleport:
+      ansible_host: 2a01:4f8:1c19:c7d::1
+    podcast-linkstack:
+      ansible_host: 2a01:4f8:c17:de13::1

inventory/group_vars/debian.yml

@@ -0,0 +1,8 @@
+# Standardwerte, die überschrieben werden können
+os_update_auto_upgrade: true
+os_also_update_mirror: false
+os_update_mirrors:
+  # Role needs two mirros to use for the sources.list.j2 Template
+  - "http://deb.debian.org/debian" # Enter a main mirror here (not security)
+  - "http://security.debian.org/debian-security" # Enter a security mirror here
+os_update_version_codename: "trixie" # KEEP UNTOUCHED!! | Used for jinja2 Template fill in as it determines the current codename of system where ansible is run on

inventory/group_vars/external.yml

@@ -0,0 +1,11 @@
+---
+# SSH authorized keys for admin user
+# Format: key and optional comment
+admin_authorized_keys:
+  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL/XwF0Na+YH7lRqGtwEcyIMVGTQZetNDrC9sZ8ofjC5 niklas@Linkman-PC"
+    comment: "Niklas - Linkman-PC"
+  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+EvtGavPlXfv7b00jSYsXX2+IEsqFWupEs6Rzf5z9q root@ansible"
+    comment: "Generic Ansible Key"
+
+# Admin user password (hashed with SHA-512)
+admin_password: "$6$NmIxM3ZbJDsbC1E6$BO8ag1ZyBqbhELjk0ppKV0CLnYXhHDp9oZc.jmkc62N9hjwbXihF1FmvsYgMKKINVcwaE73u2dRO8pnE6yEGH/"

inventory/group_vars/home.yml

@@ -0,0 +1 @@
+monitoring_zabbix_passive_servers: ["10.13.37.1"]

inventory/home.yml

@@ -0,0 +1,20 @@
+home:
+  children:
+    debian:
+      hosts:
+        rp:
+          ansible_host: 10.13.37.11
+        paperless:
+          ansible_host: 10.13.37.30
+        homeassistant:
+          ansible_host: 10.13.37.15
+        teleport-jumper:
+          ansible_host: 10.13.37.50
+        idris-dns:
+          ansible_host: 10.13.37.53
+        it-tools:
+          ansible_host: 10.13.37.16
+    proxmox:
+      hosts:
+        tardis:
+          ansible_host: 10.13.37.5

playbooks/monitoring.yml

@@ -1,18 +1,18 @@
-- hosts: all
-  user: admin
-  become: true
-  tasks:
-    - name: Verify if system is Debian
-      debug:
-        msg: "This playbook is running on a Debian system."
-      when: ansible_facts['os_family'] == "Debian"
-
-    - name: Stop playbook if system is not Debian
-      fail:
-        msg: "This playbook only supports Debian."
-      when: ansible_facts['os_family'] != "Debian"
-
-    - name: Include monitoring role
-      import_role:
-        name: monitoring
+- hosts: all
+  user: admin
+  become: true
+  tasks:
+    - name: Verify if system is Debian
+      debug:
+        msg: "This playbook is running on a Debian system."
+      when: ansible_facts['os_family'] == "Debian"
+
+    - name: Stop playbook if system is not Debian
+      fail:
+        msg: "This playbook only supports Debian."
+      when: ansible_facts['os_family'] != "Debian"
+
+    - name: Include monitoring role
+      import_role:
+        name: monitoring
       when: ansible_facts['os_family'] == "Debian"

playbooks/os-updates-deb.yml

@@ -15,4 +15,9 @@
     - name: Include OS update role
       include_role:
         name: os-updates
+      when: ansible_facts['os_family'] == "Debian"
+
+    - name: Include healthcheck role (post-update quality gate)
+      include_role:
+        name: healthcheck
       when: ansible_facts['os_family'] == "Debian"

playbooks/upgrade-hawser.yml

@@ -0,0 +1,18 @@
+- hosts: all
+  user: admin
+  become: true
+  tasks:
+    - name: Verify if system is Debian
+      debug:
+        msg: "This playbook is running on a Debian system."
+      when: ansible_facts['os_family'] == "Debian"
+
+    - name: Stop playbook if system is not Debian
+      fail:
+        msg: "This playbook only supports Debian."
+      when: ansible_facts['os_family'] != "Debian"
+
+    - name: Include hawser role
+      import_role:
+        name: hawser
+      when: ansible_facts['os_family'] == "Debian"

roles/bootstrap/tasks/configure-sysctl.yml

@@ -1,9 +1,9 @@
----
-- name: Configure swappiness to reduce swap usage
-  become: yes
-  ansible.posix.sysctl:
-    name: vm.swappiness
-    value: 10
-    state: present
-    reload: yes
+---
+- name: Configure swappiness to reduce swap usage
+  become: yes
+  ansible.posix.sysctl:
+    name: vm.swappiness
+    value: 10
+    state: present
+    reload: yes
   when: ansible_facts['virtualization_type'] != "lxc"

roles/bootstrap/tasks/main.yml

@@ -21,4 +21,12 @@
   tags: bashrc
 
 - import_tasks: configure-sysctl.yml
-  tags: sysctl
+  tags: sysctl
+
+- import_role:
+    name: docker
+  when: not skip_docker | default(false)
+
+- import_role:
+    name: monitoring
+  when: not skip_monitoring | default(false)

roles/docker/defaults/main.yml

@@ -1,5 +1,5 @@
 docker_mirror: https://download.docker.com/linux/debian
-os_version_codename: "{{ ansible_facts['distribution_release'] | lower }}"
+os_version_codename: "{{ ansible_lsb.codename }}"
 docker_packages:
   - docker-ce
   - docker-ce-cli

roles/docker/tasks/main.yml

@@ -1,6 +1,2 @@
 - name: Include Docker installation tasks
   ansible.builtin.include_tasks: install-docker.yml
-  when: docker_check is failed
-
-- name: Include Docker APT migration tasks
-  ansible.builtin.include_tasks: migrate-apt.yml

roles/docker/tasks/migrate-apt.yml

@@ -1,23 +0,0 @@
-- name: Remove old Docker APT repo list files (legacy deb format)
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: absent
-  loop:
-    - /etc/apt/sources.list.d/docker.list
-    - /etc/apt/sources.list.d/docker-ce.list
-    - /etc/apt/sources.list.d/download_docker_com_linux_ubuntu.list
-  register: docker_apt_sources_cleanup
-  failed_when: false
-
-- name: Remove legacy Docker repo entries from /etc/apt/sources.list
-  ansible.builtin.lineinfile:
-    path: /etc/apt/sources.list
-    regexp: '^deb\s+.*download\.docker\.com/linux/.*$'
-    state: absent
-  register: docker_sources_list_cleanup
-  failed_when: false
-
-- name: Update apt cache after legacy Docker repo cleanup
-  ansible.builtin.apt:
-    update_cache: true
-  when: docker_apt_sources_cleanup is changed or docker_sources_list_cleanup is changed

roles/hawser/defaults/main.yml

@@ -0,0 +1,15 @@
+# Name des Docker Compose Projekts
+hawser_project_name: hawser
+
+# Verzeichnis, in dem die Compose-Datei auf den Zielsystemen liegt
+hawser_compose_dir: /opt/hawser
+
+# Freigegebener Port des Hawser-Agents
+hawser_port: "2376"
+
+# Name des externen Docker Volumes für die Stack-Dateien
+hawser_stacks_volume: hawser_stacks
+
+# ALLOW_INSECURE_NO_AUTH=true erlaubt Standard-Mode ohne Token auf nicht-loopback Adressen.
+# Nur setzen, wenn das Netzwerk durch andere Maßnahmen abgesichert ist.
+hawser_allow_insecure_no_auth: true

roles/hawser/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# Keine Handler erforderlich – docker compose up -d ersetzt laufende Container eigenständig

roles/hawser/tasks/main.yml

@@ -0,0 +1,2 @@
+- name: Update Hawser Docker stack
+  ansible.builtin.import_tasks: update-stack.yml

roles/hawser/tasks/update-stack.yml

@@ -0,0 +1,31 @@
+- name: Gather Docker host information
+  community.docker.docker_host_info:
+  register: hawser_docker_info
+  ignore_errors: true
+
+- name: Check if Hawser compose file is present
+  ansible.builtin.stat:
+    path: "{{ hawser_compose_dir }}/docker-compose.yml"
+  register: hawser_compose_file
+  when: hawser_docker_info is not failed
+
+- name: Deploy Hawser compose file from template
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ hawser_compose_dir }}/docker-compose.yml"
+    owner: root
+    group: root
+    mode: "0640"
+  when:
+    - hawser_docker_info is not failed
+    - hawser_compose_file.stat.exists
+
+- name: Pull latest images and recreate Hawser stack
+  community.docker.docker_compose_v2:
+    project_src: "{{ hawser_compose_dir }}"
+    project_name: "{{ hawser_project_name }}"
+    state: present
+    pull: always
+  when:
+    - hawser_docker_info is not failed
+    - hawser_compose_file.stat.exists

roles/hawser/templates/docker-compose.yml.j2

@@ -0,0 +1,17 @@
+services:
+  hawser:
+    container_name: hawser
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - {{ hawser_stacks_volume }}:/data/stacks
+    ports:
+      - "{{ hawser_port }}:2376"
+    environment:
+      - ALLOW_INSECURE_NO_AUTH={{ hawser_allow_insecure_no_auth | string | lower }}
+    image: ghcr.io/finsys/hawser:latest
+    restart: always
+
+volumes:
+  {{ hawser_stacks_volume }}:
+    external: true
+    name: {{ hawser_stacks_volume }}

roles/healthcheck/defaults/main.yml

@@ -0,0 +1,11 @@
+# Post-update health check (quality gate) defaults.
+# The role verifies that every Docker container is running after an OS update
+# and records the result in the same update log used by the os-updates role.
+
+healthcheck_logging_enabled: "{{ os_update_logging_enabled | default(true) }}"
+healthcheck_log_dir: "{{ os_update_log_dir | default('/ansible/logs') }}"
+healthcheck_log_inventory: "{{ inventory_file | basename | splitext | first }}"
+healthcheck_log_file: "{{ healthcheck_log_dir }}/{{ healthcheck_log_inventory }}/{{ ansible_facts['hostname'] }}/update.log"
+
+# When true, the role fails the playbook if any container is not running.
+healthcheck_fail_on_unhealthy: false

roles/healthcheck/tasks/main.yml

@@ -0,0 +1,85 @@
+# Post-update health check (quality gate).
+# Detects whether Docker is installed and, if so, verifies that every
+# container is in the running state. Results are appended to the update
+# log as a "quality_gate" section, tied to the current update run via the
+# os-updates preflight start timestamp.
+
+- name: Healthcheck - Detect Docker binary
+  ansible.builtin.command: which docker
+  register: healthcheck_docker_bin
+  changed_when: false
+  failed_when: false
+
+- name: Healthcheck - Record Docker presence
+  ansible.builtin.set_fact:
+    healthcheck_docker_installed: "{{ healthcheck_docker_bin.rc == 0 }}"
+
+- name: Healthcheck - List all container names
+  ansible.builtin.command: "docker ps -a --format {{ '{{' }}.Names{{ '}}' }}"
+  register: healthcheck_all_containers
+  changed_when: false
+  when: healthcheck_docker_installed | bool
+
+- name: Healthcheck - List running container names
+  ansible.builtin.command: "docker ps --filter status=running --format {{ '{{' }}.Names{{ '}}' }}"
+  register: healthcheck_running_containers
+  changed_when: false
+  when: healthcheck_docker_installed | bool
+
+- name: Healthcheck - Evaluate container health
+  ansible.builtin.set_fact:
+    healthcheck_total_containers: "{{ healthcheck_all_containers.stdout_lines | length }}"
+    healthcheck_running_count: "{{ healthcheck_running_containers.stdout_lines | length }}"
+    healthcheck_non_running_containers: "{{ (healthcheck_all_containers.stdout_lines | difference(healthcheck_running_containers.stdout_lines)) | list }}"
+  when: healthcheck_docker_installed | bool
+
+- name: Healthcheck - Determine overall result
+  ansible.builtin.set_fact:
+    healthcheck_all_running: "{{ (healthcheck_non_running_containers | length) == 0 }}"
+    healthcheck_passed: "{{ (healthcheck_non_running_containers | length) == 0 }}"
+  when: healthcheck_docker_installed | bool
+
+- name: Healthcheck - Default result when Docker is not installed
+  ansible.builtin.set_fact:
+    healthcheck_total_containers: 0
+    healthcheck_running_count: 0
+    healthcheck_non_running_containers: []
+    healthcheck_all_running: true
+    healthcheck_passed: true
+  when: not (healthcheck_docker_installed | bool)
+
+- name: Healthcheck - Ensure log directory exists
+  ansible.builtin.file:
+    path: "{{ healthcheck_log_file | dirname }}"
+    state: directory
+  delegate_to: localhost
+  when: healthcheck_logging_enabled | bool
+
+- name: Healthcheck - Write quality gate section to update log
+  ansible.builtin.blockinfile:
+    path: "{{ healthcheck_log_file }}"
+    create: yes
+    marker: "# {mark} ANSIBLE-HEALTHCHECK"
+    block: |
+      quality_gate:
+        docker_installed: {{ healthcheck_docker_installed }}
+        total_containers: {{ healthcheck_total_containers }}
+        running_containers: {{ healthcheck_running_count }}
+        all_running: {{ healthcheck_all_running }}
+        non_running_containers:
+      {% for c in healthcheck_non_running_containers %}
+        - {{ c }}
+      {% endfor %}
+        healthcheck_passed: {{ healthcheck_passed }}
+  delegate_to: localhost
+  when: healthcheck_logging_enabled | bool
+
+- name: Healthcheck - Fail when containers are unhealthy (quality gate)
+  ansible.builtin.fail:
+    msg: >-
+      Quality gate failed: the following containers are not running:
+      {{ healthcheck_non_running_containers | join(', ') }}
+  when:
+    - healthcheck_fail_on_unhealthy | bool
+    - healthcheck_docker_installed | bool
+    - not (healthcheck_all_running | bool)

roles/monitoring/defaults/main.yml

@@ -1,15 +1,15 @@
-monitoring_zabbix_version: "7.0"
-monitoring_zabbix_release_package: "zabbix-release_latest_{{ monitoring_zabbix_version }}+debian{{ ansible_distribution_major_version }}_all.deb"
-monitoring_zabbix_release_url: "https://repo.zabbix.com/zabbix/{{ monitoring_zabbix_version }}/debian/pool/main/z/zabbix-release/{{ monitoring_zabbix_release_package }}"
-monitoring_zabbix_release_path: "/tmp/{{ monitoring_zabbix_release_package }}"
-monitoring_zabbix_agent_package: zabbix-agent2
-monitoring_zabbix_agent_service: zabbix-agent2
-monitoring_zabbix_config_file: /etc/zabbix/zabbix_agent2.conf
-monitoring_zabbix_agent_user: zabbix
-monitoring_zabbix_passive_servers: ["192.168.212.1"]
-monitoring_zabbix_active_servers: ""
-monitoring_zabbix_listen_port: 10050
-monitoring_zabbix_docker_group: docker
-monitoring_zabbix_docker_socket_path: /var/run/docker.sock
-monitoring_zabbix_docker_plugin_config_file: /etc/zabbix/zabbix_agent2.d/plugins.d/docker.conf
-monitoring_zabbix_docker_endpoint: "unix://{{ monitoring_zabbix_docker_socket_path }}"
+monitoring_zabbix_version: "7.0"
+monitoring_zabbix_release_package: "zabbix-release_latest_{{ monitoring_zabbix_version }}+debian{{ ansible_distribution_major_version }}_all.deb"
+monitoring_zabbix_release_url: "https://repo.zabbix.com/zabbix/{{ monitoring_zabbix_version }}/debian/pool/main/z/zabbix-release/{{ monitoring_zabbix_release_package }}"
+monitoring_zabbix_release_path: "/tmp/{{ monitoring_zabbix_release_package }}"
+monitoring_zabbix_agent_package: zabbix-agent2
+monitoring_zabbix_agent_service: zabbix-agent2
+monitoring_zabbix_config_file: /etc/zabbix/zabbix_agent2.conf
+monitoring_zabbix_agent_user: zabbix
+monitoring_zabbix_passive_servers: ["192.168.212.1"]
+monitoring_zabbix_active_servers: ""
+monitoring_zabbix_listen_port: 10050
+monitoring_zabbix_docker_group: docker
+monitoring_zabbix_docker_socket_path: /var/run/docker.sock
+monitoring_zabbix_docker_plugin_config_file: /etc/zabbix/zabbix_agent2.d/plugins.d/docker.conf
+monitoring_zabbix_docker_endpoint: "unix://{{ monitoring_zabbix_docker_socket_path }}"

roles/monitoring/handlers/main.yml

@@ -1,6 +1,6 @@
----
-- name: Restart Zabbix Agent
-  ansible.builtin.systemd:
-    name: "{{ monitoring_zabbix_agent_service }}"
-    state: restarted
-    daemon_reload: yes
+---
+- name: Restart Zabbix Agent
+  ansible.builtin.systemd:
+    name: "{{ monitoring_zabbix_agent_service }}"
+    state: restarted
+    daemon_reload: yes

roles/monitoring/tasks/configure-agent.yml

@@ -1,9 +1,9 @@
----
-- name: Configure Zabbix agent
-  ansible.builtin.template:
-    src: zabbix_agent2.conf.j2
-    dest: "{{ monitoring_zabbix_config_file }}"
-    owner: root
-    group: root
-    mode: '0644'
-  notify: Restart Zabbix Agent
+---
+- name: Configure Zabbix agent
+  ansible.builtin.template:
+    src: zabbix_agent2.conf.j2
+    dest: "{{ monitoring_zabbix_config_file }}"
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Restart Zabbix Agent

roles/monitoring/tasks/configure-docker.yml

@@ -1,47 +1,47 @@
----
-- name: Gather Docker group entries
-  ansible.builtin.getent:
-    database: group
-    key: "{{ monitoring_zabbix_docker_group }}"
-  failed_when: false
-
-- name: Check whether Docker socket exists
-  ansible.builtin.stat:
-    path: "{{ monitoring_zabbix_docker_socket_path }}"
-  register: monitoring_docker_socket
-
-- name: Ensure Zabbix agent can access Docker socket
-  ansible.builtin.user:
-    name: "{{ monitoring_zabbix_agent_user }}"
-    groups: "{{ monitoring_zabbix_docker_group }}"
-    append: yes
-  when:
-    - monitoring_docker_socket.stat.exists
-    - monitoring_zabbix_docker_group in (ansible_facts.getent_group | default({}))
-  notify: Restart Zabbix Agent
-
-- name: Ensure Docker plugin configuration directory exists
-  ansible.builtin.file:
-    path: "{{ monitoring_zabbix_docker_plugin_config_file | dirname }}"
-    state: directory
-    owner: root
-    group: root
-    mode: '0755'
-  when: monitoring_docker_socket.stat.exists
-
-- name: Configure Docker plugin endpoint
-  ansible.builtin.template:
-    src: docker.conf.j2
-    dest: "{{ monitoring_zabbix_docker_plugin_config_file }}"
-    owner: root
-    group: root
-    mode: '0644'
-  when: monitoring_docker_socket.stat.exists
-  notify: Restart Zabbix Agent
-
-- name: Remove Docker plugin configuration when Docker socket is absent
-  ansible.builtin.file:
-    path: "{{ monitoring_zabbix_docker_plugin_config_file }}"
-    state: absent
-  when: not monitoring_docker_socket.stat.exists
+---
+- name: Gather Docker group entries
+  ansible.builtin.getent:
+    database: group
+    key: "{{ monitoring_zabbix_docker_group }}"
+  failed_when: false
+
+- name: Check whether Docker socket exists
+  ansible.builtin.stat:
+    path: "{{ monitoring_zabbix_docker_socket_path }}"
+  register: monitoring_docker_socket
+
+- name: Ensure Zabbix agent can access Docker socket
+  ansible.builtin.user:
+    name: "{{ monitoring_zabbix_agent_user }}"
+    groups: "{{ monitoring_zabbix_docker_group }}"
+    append: yes
+  when:
+    - monitoring_docker_socket.stat.exists
+    - monitoring_zabbix_docker_group in (ansible_facts.getent_group | default({}))
+  notify: Restart Zabbix Agent
+
+- name: Ensure Docker plugin configuration directory exists
+  ansible.builtin.file:
+    path: "{{ monitoring_zabbix_docker_plugin_config_file | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+  when: monitoring_docker_socket.stat.exists
+
+- name: Configure Docker plugin endpoint
+  ansible.builtin.template:
+    src: docker.conf.j2
+    dest: "{{ monitoring_zabbix_docker_plugin_config_file }}"
+    owner: root
+    group: root
+    mode: '0644'
+  when: monitoring_docker_socket.stat.exists
+  notify: Restart Zabbix Agent
+
+- name: Remove Docker plugin configuration when Docker socket is absent
+  ansible.builtin.file:
+    path: "{{ monitoring_zabbix_docker_plugin_config_file }}"
+    state: absent
+  when: not monitoring_docker_socket.stat.exists
   notify: Restart Zabbix Agent

roles/monitoring/tasks/install-agent.yml

@@ -1,11 +1,11 @@
----
-- name: Install Zabbix agent package
-  ansible.builtin.apt:
-    name: "{{ monitoring_zabbix_agent_package }}"
-    state: present
-
-- name: Ensure Zabbix agent service is enabled and started
-  ansible.builtin.systemd:
-    name: "{{ monitoring_zabbix_agent_service }}"
-    state: started
-    enabled: yes
+---
+- name: Install Zabbix agent package
+  ansible.builtin.apt:
+    name: "{{ monitoring_zabbix_agent_package }}"
+    state: present
+
+- name: Ensure Zabbix agent service is enabled and started
+  ansible.builtin.systemd:
+    name: "{{ monitoring_zabbix_agent_service }}"
+    state: started
+    enabled: yes

roles/monitoring/tasks/install-repository.yml

@@ -1,22 +1,22 @@
----
-- name: Install repository prerequisites
-  ansible.builtin.apt:
-    name:
-      - ca-certificates
-    state: present
-    update_cache: yes
-    cache_valid_time: 3600
-
-- name: Download Zabbix release package
-  ansible.builtin.get_url:
-    url: "{{ monitoring_zabbix_release_url }}"
-    dest: "{{ monitoring_zabbix_release_path }}"
-    mode: '0644'
-
-- name: Install Zabbix release package
-  ansible.builtin.apt:
-    deb: "{{ monitoring_zabbix_release_path }}"
-
-- name: Update apt cache after Zabbix repository setup
-  ansible.builtin.apt:
-    update_cache: yes
+---
+- name: Install repository prerequisites
+  ansible.builtin.apt:
+    name:
+      - ca-certificates
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Download Zabbix release package
+  ansible.builtin.get_url:
+    url: "{{ monitoring_zabbix_release_url }}"
+    dest: "{{ monitoring_zabbix_release_path }}"
+    mode: '0644'
+
+- name: Install Zabbix release package
+  ansible.builtin.apt:
+    deb: "{{ monitoring_zabbix_release_path }}"
+
+- name: Update apt cache after Zabbix repository setup
+  ansible.builtin.apt:
+    update_cache: yes

roles/monitoring/tasks/main.yml

@@ -1,14 +1,14 @@
-- import_tasks: validate.yml
-  tags: monitoring
-
-- import_tasks: install-repository.yml
-  tags: monitoring
-
-- import_tasks: install-agent.yml
-  tags: monitoring
-
-- import_tasks: configure-docker.yml
-  tags: monitoring
-
-- import_tasks: configure-agent.yml
-  tags: monitoring
+- import_tasks: validate.yml
+  tags: monitoring
+
+- import_tasks: install-repository.yml
+  tags: monitoring
+
+- import_tasks: install-agent.yml
+  tags: monitoring
+
+- import_tasks: configure-docker.yml
+  tags: monitoring
+
+- import_tasks: configure-agent.yml
+  tags: monitoring

roles/monitoring/tasks/validate.yml

@@ -1,11 +1,11 @@
----
-- name: Stop playbook if system is not Debian
-  ansible.builtin.fail:
-    msg: "This role only supports Debian."
-  when: ansible_facts['os_family'] != "Debian"
-
-- name: Ensure Zabbix passive servers are defined
-  ansible.builtin.assert:
-    that:
-      - monitoring_zabbix_passive_servers | length > 0
-    fail_msg: "Set monitoring_zabbix_passive_servers to the IPs, CIDRs or DNS names allowed to query the agent."
+---
+- name: Stop playbook if system is not Debian
+  ansible.builtin.fail:
+    msg: "This role only supports Debian."
+  when: ansible_facts['os_family'] != "Debian"
+
+- name: Ensure Zabbix passive servers are defined
+  ansible.builtin.assert:
+    that:
+      - monitoring_zabbix_passive_servers | length > 0
+    fail_msg: "Set monitoring_zabbix_passive_servers to the IPs, CIDRs or DNS names allowed to query the agent."

roles/monitoring/templates/docker.conf.j2

@@ -1,2 +1,2 @@
-# {{ ansible_managed }}
+# {{ ansible_managed }}
 Plugins.Docker.Endpoint={{ monitoring_zabbix_docker_endpoint }}

roles/monitoring/templates/zabbix_agent2.conf.j2

@@ -1,11 +1,11 @@
-# {{ ansible_managed }}
-PidFile=/run/zabbix/zabbix_agent2.pid
-LogFile=/var/log/zabbix/zabbix_agent2.log
-LogFileSize=0
-Server={{ monitoring_zabbix_passive_servers | join(',') }}
-{% if monitoring_zabbix_active_servers | length > 0 %}
-ServerActive={{ monitoring_zabbix_active_servers | join(',') }}
-{% endif %}
-HostnameItem=system.hostname
-ListenPort={{ monitoring_zabbix_listen_port }}
-Include=/etc/zabbix/zabbix_agent2.d/*.conf
+# {{ ansible_managed }}
+PidFile=/run/zabbix/zabbix_agent2.pid
+LogFile=/var/log/zabbix/zabbix_agent2.log
+LogFileSize=0
+Server={{ monitoring_zabbix_passive_servers | join(',') }}
+{% if monitoring_zabbix_active_servers | length > 0 %}
+ServerActive={{ monitoring_zabbix_active_servers | join(',') }}
+{% endif %}
+HostnameItem=system.hostname
+ListenPort={{ monitoring_zabbix_listen_port }}
+Include=/etc/zabbix/zabbix_agent2.d/*.conf

roles/os-updates/defaults/main.yml

@@ -1,11 +1,17 @@
 # Standardwerte, die überschrieben werden können
 os_update_auto_upgrade: true
 os_also_update_mirror: false
+
+# Logging of OS update runs (preflight package count/list, updated packages, duration)
+os_update_logging_enabled: true
+os_update_log_dir: /ansible/logs
+os_update_log_inventory: "{{ inventory_file | basename | splitext | first }}"
+os_update_log_file: "{{ os_update_log_dir }}/{{ os_update_log_inventory }}/{{ ansible_facts['hostname'] }}/update.log"
 os_update_mirrors:
   # Role needs two mirros to use for the sources.list.j2 Template
   - "http://deb.debian.org/debian" # Enter a main mirror here (not security)
   - "http://security.debian.org/debian-security" # Enter a security mirror here
-os_update_version_codename: "{{ ansible_distribution_release }}" # KEEP UNTOUCHED!! | Used for jinja2 Template fill in as it determines the current codename of system where ansible is run on
+os_update_version_codename: "{{ ansible_facts['distribution_release'] }}" # KEEP UNTOUCHED!! | Used for jinja2 Template fill in as it determines the current codename of system where ansible is run on
 os_update_debian_codenames:
   # Only these suites are considered Debian codenames and will be rewritten in sources.list.d
   - trixie

roles/os-updates/handlers/main.yml

@@ -3,9 +3,6 @@
     clean: yes
     autoclean: yes
 
-- name: Reboot system
-  command: /sbin/reboot
-  async: 1
-  poll: 0
-  ignore_errors: true
-  when: reboot_required == "yes"
+- name: apt autoremove
+  apt:
+    autoremove: yes

roles/os-updates/tasks/logging_postflight.yml

@@ -0,0 +1,46 @@
+# Postflight logging: which packages were updated and how long it took.
+# Run logging steps only when os_update_logging_enabled is true.
+
+- name: Logging - Record end epoch
+  command: date +%s
+  register: os_update_log_end_ts
+  changed_when: false
+  when: os_update_logging_enabled | bool
+
+- name: Logging - Compute upgrade duration
+  set_fact:
+    os_update_duration_seconds: "{{ (os_update_log_end_ts.stdout | int) - (os_update_log_start_ts.stdout | int) }}"
+  when: os_update_logging_enabled | bool
+
+- name: Logging - Ensure log directory exists
+  file:
+    path: "{{ os_update_log_file | dirname }}"
+    state: directory
+  delegate_to: localhost
+  when: os_update_logging_enabled | bool
+
+- name: Logging - Write update log entry
+  blockinfile:
+    path: "{{ os_update_log_file }}"
+    create: yes
+    marker: "# {mark} ANSIBLE-OS-UPDATE"
+    block: |
+      host: {{ ansible_facts['hostname'] }}
+      started: {{ os_update_log_start_iso.stdout }}
+      duration_seconds: {{ os_update_duration_seconds }}
+      upgradable_packages_count: {{ os_update_upgradable_count }}
+      upgrade_changed: {{ os_update_upgrade_result.changed | default(false) }}
+      upgrade_failed: {{ os_update_upgrade_result.failed | default(false) }}
+      upgradable_packages:
+      {% for ver in os_update_upgradable_versions %}
+        - {{ ver.split()[0] }} ({{ ver.split()[1] }} -> {{ ver.split()[2] }})
+      {% endfor %}
+      reboot_required: {{ os_update_reboot_required }}
+      reboot_reason: {{ os_update_reboot_reason }}
+      reboot_triggered: {{ os_update_reboot_triggered }}
+      reboot_started: {{ os_update_reboot_start_iso }}
+      reboot_downtime_seconds: {{ os_update_reboot_downtime_seconds }}
+      kernel_before: {{ os_update_running_kernel.stdout | default('n/a') }}
+      kernel_after: {{ os_update_running_kernel_after }}
+  delegate_to: localhost
+  when: os_update_logging_enabled | bool

roles/os-updates/tasks/logging_preflight.yml

@@ -0,0 +1,48 @@
+# Preflight logging: how many and which packages need to be updated.
+# Run logging steps only when os_update_logging_enabled is true.
+
+- name: Logging - Refresh apt cache for accurate preflight
+  apt:
+    update_cache: yes
+  changed_when: false
+  when: os_update_logging_enabled | bool
+
+- name: Logging - Record start epoch
+  command: date +%s
+  register: os_update_log_start_ts
+  changed_when: false
+  when: os_update_logging_enabled | bool
+
+- name: Logging - Record start ISO time
+  command: date -Iseconds
+  register: os_update_log_start_iso
+  changed_when: false
+  when: os_update_logging_enabled | bool
+
+- name: Preflight - Gather upgradable packages
+  command: apt list --upgradable
+  register: os_update_upgradable
+  changed_when: false
+  when: os_update_logging_enabled | bool
+
+- name: Preflight - Build list of upgradable package names, versions and count
+  set_fact:
+    os_update_upgradable_names: >-
+      {{
+        os_update_upgradable.stdout_lines
+        | select('search', '\[upgradable')
+        | map('regex_replace', '^([^/]+)/.*$', '\1')
+        | list
+      }}
+    os_update_upgradable_versions: >-
+      {{
+        os_update_upgradable.stdout_lines
+        | select('search', '\[upgradable')
+        | map('regex_replace', '^([^/]+)/\S+\s+(\S+)\s+\S+\s+\[upgradable from: (\S+)\]$', '\1 \3 \2')
+        | list
+      }}
+    os_update_upgradable_count: >-
+      {{
+        (os_update_upgradable.stdout_lines | select('search', '\[upgradable') | list | length) | int
+      }}
+  when: os_update_logging_enabled | bool

roles/os-updates/tasks/reboot.yml

@@ -0,0 +1,83 @@
+# Reboot decision and execution.
+# Determines whether a reboot is needed because a new kernel was installed,
+# performs the reboot asynchronously, waits for the host to come back, and
+# records timing facts for the update log.
+
+- name: Gather running kernel
+  command: uname -r
+  register: os_update_running_kernel
+  changed_when: false
+  when: ansible_facts['virtualization_type'] != 'lxc'
+
+- name: Gather latest installed kernel
+  shell: |
+    dpkg -l | grep -E '^ii' | grep 'linux-image-[0-9]' | awk '{print $2}' | sort | tail -n 1
+  register: os_update_latest_kernel
+  changed_when: false
+  failed_when: false
+  when: ansible_facts['virtualization_type'] != 'lxc'
+
+- name: Determine reboot decision
+  set_fact:
+    os_update_reboot_required: true
+    os_update_reboot_reason: "new kernel installed ({{ os_update_running_kernel.stdout }} -> {{ os_update_latest_kernel.stdout | regex_replace('^linux-image-', '') }})"
+  when:
+    - ansible_facts['virtualization_type'] != 'lxc'
+    - os_update_latest_kernel.stdout | length > 0
+    - os_update_running_kernel.stdout not in os_update_latest_kernel.stdout
+
+- name: Default reboot decision to false
+  set_fact:
+    os_update_reboot_required: "{{ os_update_reboot_required | default(false) }}"
+    os_update_reboot_reason: "{{ os_update_reboot_reason | default('none') }}"
+
+- name: Default reboot timing facts
+  set_fact:
+    os_update_reboot_triggered: false
+    os_update_reboot_start_iso: "n/a"
+    os_update_reboot_downtime_seconds: 0
+    os_update_running_kernel_after: "{{ os_update_running_kernel.stdout | default('n/a') }}"
+
+- block:
+    - name: Reboot - Record start time
+      command: date -Iseconds
+      register: os_update_reboot_start_iso_raw
+      changed_when: false
+
+    - name: Reboot - Record start epoch
+      command: date +%s
+      register: os_update_reboot_start_ts
+      changed_when: false
+
+    - name: Reboot - Reboot system
+      command: /sbin/reboot
+      async: 1
+      poll: 0
+      ignore_errors: true
+
+    - name: Reboot - Wait for host to come back online
+      wait_for_connection:
+        delay: 10
+        timeout: 600
+      register: os_update_reboot_wait
+
+    - name: Reboot - Record end epoch
+      command: date +%s
+      register: os_update_reboot_end_ts
+      changed_when: false
+
+    - name: Reboot - Compute downtime
+      set_fact:
+        os_update_reboot_downtime_seconds: "{{ (os_update_reboot_end_ts.stdout | int) - (os_update_reboot_start_ts.stdout | int) }}"
+
+    - name: Reboot - Gather running kernel after reboot
+      command: uname -r
+      register: os_update_running_kernel_after_raw
+      changed_when: false
+
+    - name: Reboot - Record reboot facts
+      set_fact:
+        os_update_reboot_triggered: true
+        os_update_reboot_start_iso: "{{ os_update_reboot_start_iso_raw.stdout }}"
+        os_update_running_kernel_after: "{{ os_update_running_kernel_after_raw.stdout }}"
+  when: os_update_reboot_required | bool

roles/os-updates/tasks/upgrade_packages.yml

@@ -1,30 +1,17 @@
+- name: Run preflight logging
+  include_tasks: logging_preflight.yml
+
 - name: Upgrade all installed packages
   apt:
     upgrade: full
     update_cache: yes
-  notify: 
-    - apt cleanup
-
-- name: Check if a kernel update is available
-  shell: |
-    dpkg -l | grep -E '^ii' | grep 'linux-image-[0-9]' | awk '{print $2}' | sort | tail -n 1
-  register: latest_kernel
-  when: ansible_virtualization_type != 'lxc'
-
-- name: Check if running kernel matches the latest installed kernel
-  shell: |
-    echo "{{ latest_kernel.stdout }}" | grep -c $(uname -r)
-  register: kernel_match
-  changed_when: false
-  failed_when: false
-  when: ansible_facts['virtualization_type'] != 'lxc'
-
-- name: Mark reboot required if a new kernel is installed
-  set_fact:
-    reboot_required: "yes"
-  changed_when: true
+  register: os_update_upgrade_result
   notify:
-    - Reboot system
-  when: 
-    - ansible_facts['virtualization_type'] != 'lxc'
-    - (kernel_match.stdout | int) == 0
+    - apt cleanup
+    - apt autoremove
+
+- name: Run reboot decision and reboot if required
+  include_tasks: reboot.yml
+
+- name: Run postflight logging
+  include_tasks: logging_postflight.yml

setenv.sh

@@ -1,5 +0,0 @@
-export PROXMOX_URL=https://192.168.212.10:8006/
-export PROXMOX_USERNAME=ansible@pve
-export PROXMOX_PASSWORD=u0H1GQMsAuSYZJ
-export PROXMOX_INVALID_CERT=False
-export EXCLUDE_LIST="docker.*;br-*;veth*"