Fix playbook bugs from review, add Telegram alerts to health and service checks

Review fixes:
- Availability plays: "ping_result is succeeded" is true for unreachable
  hosts, so offline hosts were added to reachable_hosts and failed the
  next play. Test reachability explicitly (7 playbooks).
- bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next
  to EPEL, add the root key without replacing authorized_keys, validate
  sshd_config edits and restart sshd only on change.
- backup_docker: the command module does not expand $(...); fix rsync
  filter order and add --exclude=*; restart only the containers that were
  stopped, even when rsync fails; read the storage address from the
  inventory; keep semaphore and semaphore-db running.
- new_lxc: community.general.proxmox no longer exists, use
  community.proxmox; node is mini-pc; valid hostname; password or token
  auth; let the module pick the VMID; longer timeout.
- proxmox-backup: surface the vzdump exit code, let vzdump prune with
  keep-last, drop the rename/prune steps that globbed *.zstd.
- time-region: chrony regex no longer comments out its own server line;
  /etc/localtime as a symlink; reuse facts across plays.
- update_pihole_blocklists: FQCN, become: true, changed_when.
- ansible.cfg gathering = smart; update_system only pings ubuntu/rocky.

Telegram alerts:
- health_check: filesystems above disk_warn_pct (default 80), excluding
  efivarfs, and offline hosts.
- service_check: services not running.
- One message per run from localhost, credentials from Semaphore
  (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task,
  prints the alert text when Telegram is not configured.

Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq

playbooks/auto_updates.yml

@@ -18,12 +18,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: Auto Updates - Ubuntu

playbooks/backup_docker.yml

@@ -1,62 +1,120 @@
 ---
-- name: Backup Docker containers and volumes to storage
+# Backs up the docker host to the storage container: compose files and
+# configs from compose_dir while everything runs, then the Docker volumes
+# with the containers stopped so the copy is consistent.
+#
+# Containers in backup_skip_containers are never stopped. Semaphore runs on
+# this host, so it and its database stay up (their volumes are copied live).
+# root on the docker host needs an SSH key that is authorized on backup_host.
+#
+# Extra vars (optional):
+#   backup_host            - inventory name of the destination (default storage)
+#   backup_root            - directory on backup_host (default /mnt/storage/Docker_backups)
+#   backup_skip_containers - default [semaphore, semaphore-db]
+
+- name: Backup Docker compose files and volumes to storage
   hosts: docker
   become: true
+  gather_facts: false
 
   vars:
-    backup_dest: "192.168.2.115:/mnt/storage/Docker_backups"
-    compose_dir: "/home/harry"
-    volumes_dir: "/var/lib/docker/volumes"
+    backup_host: storage
+    backup_root: /mnt/storage/Docker_backups
+    backup_addr: "{{ hostvars[backup_host].ansible_host }}"
+    backup_skip_containers:
+      - semaphore
+      - semaphore-db
+    compose_dir: /home/harry
+    volumes_dir: /var/lib/docker/volumes
+    rsync_ssh: ssh -o StrictHostKeyChecking=accept-new
 
   tasks:
     - name: Ensure backup destination directories exist
-      ansible.builtin.command:
-        cmd: ssh -o StrictHostKeyChecking=no root@192.168.2.115 "mkdir -p /mnt/storage/Docker_backups/compose /mnt/storage/Docker_backups/volumes"
-      changed_when: false
+      ansible.builtin.file:
+        path: "{{ backup_root }}/{{ item }}"
+        state: directory
+        mode: '0755'
+      loop: [compose, volumes]
+      delegate_to: "{{ backup_host }}"
 
+    # rsync applies the first matching rule: excludes first, then the rule
+    # that lets it descend into directories, then the wanted files, then
+    # everything else is dropped.
     - name: Rsync compose files and configs (safe while running)
-      ansible.builtin.shell: |
-        rsync -az --delete \
-          --include='*/' \
-          --include='docker-compose.yml' \
-          --include='compose.yml' \
-          --include='.env' \
-          --include='*.conf' \
-          --include='*.yml' \
-          --include='*.yaml' \
-          --include='*.json' \
-          --exclude='node_modules/' \
-          --exclude='.git/' \
-          --exclude='__pycache__/' \
-          {{ compose_dir }}/ root@{{ backup_dest }}/compose/
-      register: compose_sync
+      ansible.builtin.command:
+        argv:
+          - rsync
+          - -az
+          - --delete
+          - --prune-empty-dirs
+          - -e
+          - "{{ rsync_ssh }}"
+          - --exclude=node_modules/
+          - --exclude=.git/
+          - --exclude=__pycache__/
+          - --include=*/
+          - --include=docker-compose.yml
+          - --include=compose.yml
+          - --include=.env
+          - --include=*.conf
+          - --include=*.yml
+          - --include=*.yaml
+          - --include=*.json
+          - --exclude=*
+          - "{{ compose_dir }}/"
+          - "root@{{ backup_addr }}:{{ backup_root }}/compose/"
+      changed_when: true
 
-    - name: Get list of running containers
-      ansible.builtin.command: docker ps -q
+    - name: List running containers
+      ansible.builtin.command:
+        argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }}"]
       register: running_containers
       changed_when: false
 
-    - name: Stop all containers
-      ansible.builtin.command: docker stop $(docker ps -q)
-      when: running_containers.stdout | length > 0
+    - name: Work out which containers to stop
+      ansible.builtin.set_fact:
+        containers_to_stop: "{{ running_containers.stdout_lines | difference(backup_skip_containers) }}"
 
-    - name: Rsync Docker volumes to storage
-      ansible.builtin.shell: |
-        rsync -az --delete \
-          {{ volumes_dir }}/ root@{{ backup_dest }}/volumes/
-      register: volume_sync
+    - name: Copy the volumes with the containers stopped
+      block:
+        - name: Stop containers
+          ansible.builtin.command:
+            argv: "{{ ['docker', 'stop'] + containers_to_stop }}"
+          when: containers_to_stop | length > 0
 
-    - name: Start all containers back up
-      ansible.builtin.command: docker start $(docker ps -aq)
+        - name: Rsync Docker volumes to storage
+          ansible.builtin.command:
+            argv:
+              - rsync
+              - -az
+              - --delete
+              - -e
+              - "{{ rsync_ssh }}"
+              - "{{ volumes_dir }}/"
+              - "root@{{ backup_addr }}:{{ backup_root }}/volumes/"
+          changed_when: true
+
+      always:
+        - name: Start the containers that were running before
+          ansible.builtin.command:
+            argv: "{{ ['docker', 'start'] + containers_to_stop }}"
+          when: containers_to_stop | length > 0
 
     - name: Verify containers are running
-      ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}"
+      ansible.builtin.command:
+        argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}"]
       register: container_status
       changed_when: false
 
     - name: Backup summary
       ansible.builtin.debug:
         msg:
-          - "Compose sync: {{ 'OK' if compose_sync.rc == 0 else 'FAILED' }}"
-          - "Volume sync: {{ 'OK' if volume_sync.rc == 0 else 'FAILED' }}"
-          - "Containers running: {{ container_status.stdout_lines | length }}"
+          - "Destination: {{ backup_host }} ({{ backup_addr }}):{{ backup_root }}"
+          - "Stopped for the volume copy: {{ containers_to_stop | join(', ') if containers_to_stop else 'none' }}"
+          - "Kept running: {{ backup_skip_containers | join(', ') }}"
+          - "Containers running: {{ container_status.stdout_lines | length }} now, {{ running_containers.stdout_lines | length }} before"
+
+    - name: Warn when fewer containers are running than before
+      ansible.builtin.debug:
+        msg: "WARNING: {{ item }} did not come back up on {{ inventory_hostname }}"
+      loop: "{{ running_containers.stdout_lines | difference(container_status.stdout_lines | map('regex_replace', ' - .*$', '')) }}"

playbooks/bootstrap_lxc.yml

@@ -1,4 +1,9 @@
 ---
+# Base packages, the Ansible root key and sshd basics for every Ubuntu/Rocky
+# guest. Run it once after a container is created (and adopted, see
+# discover_guests.yml). ssh_harden.yml then tightens sshd further through
+# /etc/ssh/sshd_config.d/00-ansible.conf.
+
 - name: Bootstrap - Install packages on Ubuntu
   hosts: ubuntu
   become: true
@@ -20,7 +25,9 @@
           - zip
           - git
           - sudo
+          - tmux
           - tree
+          - bash-completion
           - traceroute
           - dnsutils
           - ca-certificates
@@ -28,36 +35,46 @@
           - python3-pip
           - software-properties-common
           - rsync
-          - apt-transport-https
           - sysstat
           - screen
-          - mtr
+          - mtr-tiny             # plain "mtr" pulls in GTK on Ubuntu
           - tar
         state: present
         update_cache: true
+        cache_valid_time: 3600
 
 - name: Bootstrap - Install packages on Rocky Linux
   hosts: rocky
   become: true
 
   tasks:
-    - name: Ensure EPEL repo is enabled
+    # epel-release ships in the "extras" repo on Rocky/Alma/CentOS Stream.
+    # dnf-plugins-core provides "dnf config-manager" for the next task.
+    - name: Ensure EPEL release package and dnf plugins are installed
       ansible.builtin.dnf:
-        name: epel-release
+        name:
+          - epel-release
+          - dnf-plugins-core
         state: present
 
+    # EPEL packages may depend on CRB (CodeReady Builder). EPEL's install
+    # guide enables it together with epel-release.
+    - name: Enable the CRB repo
+      community.general.dnf_config_manager:
+        name: crb
+        state: enabled
+
     - name: Install packages via dnf
       ansible.builtin.dnf:
         name:
+          # BaseOS / AppStream
           - tar
-          - vim
-          - htop
+          - vim-enhanced         # "vim" is only a virtual provide on RHEL
           - nmap
           - curl
           - mtr
           - wget
           - iperf3
-          - fail2ban
           - net-tools
           - lsof
           - unzip
@@ -74,6 +91,9 @@
           - python3-pip
           - rsync
           - sysstat
+          # EPEL
+          - htop
+          - fail2ban
           - screen
         state: present
         update_cache: true
@@ -81,64 +101,38 @@
 - name: Bootstrap - Configure SSH access
   hosts: ubuntu:rocky
   become: true
+  gather_facts: false
+
+  vars:
+    bootstrap_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key"
+    sshd_service: "{{ 'ssh' if inventory_hostname in groups['ubuntu'] else 'sshd' }}"
 
   tasks:
-    - name: Ensure root's .ssh directory exists
-      ansible.builtin.file:
-        path: /root/.ssh
-        state: directory
-        owner: root
-        group: root
-        mode: '0700'
+    - name: Add the Ansible key to root's authorized_keys
+      ansible.posix.authorized_key:
+        user: root
+        key: "{{ bootstrap_ssh_key }}"
+        state: present
 
-    - name: Set authorized_keys for root
-      ansible.builtin.copy:
-        dest: /root/.ssh/authorized_keys
-        content: |
-          ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key
-        owner: root
-        group: root
-        mode: '0600'
-
-    - name: Permit root login via public key
+    # PasswordAuthentication stays on so a fresh container is still reachable
+    # by password; ssh_harden.yml turns it off once keys are in place.
+    - name: Allow key-based root login
       ansible.builtin.lineinfile:
         path: /etc/ssh/sshd_config
-        regexp: '^#?PermitRootLogin\s+'
-        line: 'PermitRootLogin prohibit-password'
-        state: present
+        regexp: '^#?{{ item.option }}\s'
+        line: "{{ item.option }} {{ item.value }}"
+        validate: /usr/sbin/sshd -t -f %s
         backup: true
+      loop:
+        - { option: PermitRootLogin, value: prohibit-password }
+        - { option: PubkeyAuthentication, value: "yes" }
+        - { option: PasswordAuthentication, value: "yes" }
+      loop_control:
+        label: "{{ item.option }} {{ item.value }}"
+      notify: Restart sshd
 
-    - name: Ensure PubkeyAuthentication is enabled
-      ansible.builtin.lineinfile:
-        path: /etc/ssh/sshd_config
-        regexp: '^#?PubkeyAuthentication\s+'
-        line: 'PubkeyAuthentication yes'
-        state: present
-
-    # NOTE: playbooks/ssh_harden.yml overrides this via /etc/ssh/sshd_config.d/00-ansible.conf
-    - name: Ensure PasswordAuthentication is enabled
-      ansible.builtin.lineinfile:
-        path: /etc/ssh/sshd_config
-        regexp: '^#?PasswordAuthentication\s+'
-        line: 'PasswordAuthentication yes'
-        state: present
-
-- name: Bootstrap - Restart SSH on Ubuntu
-  hosts: ubuntu
-  become: true
-
-  tasks:
-    - name: Restart ssh service
+  handlers:
+    - name: Restart sshd
       ansible.builtin.service:
-        name: ssh
-        state: restarted
-
-- name: Bootstrap - Restart SSH on Rocky Linux
-  hosts: rocky
-  become: true
-
-  tasks:
-    - name: Restart sshd service
-      ansible.builtin.service:
-        name: sshd
+        name: "{{ sshd_service }}"
         state: restarted

playbooks/health_check.yml

@@ -1,4 +1,14 @@
 ---
+# Disk and RAM report for every reachable host. Filesystems above
+# disk_warn_pct and hosts that did not answer are printed as warnings and,
+# when Telegram credentials are present, sent as one Telegram message per run.
+#
+# Extra vars (optional):
+#   disk_warn_pct      - default 80
+#   telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN
+#   telegram_chat_id   - or environment variable TELEGRAM_CHAT_ID
+# Without both Telegram values the playbook only prints the report.
+
 - name: Check Host Availability
   hosts: all
   gather_facts: false
@@ -14,12 +24,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: Disk and RAM Usage Report
@@ -27,9 +37,12 @@
   become: true
   gather_facts: true
 
+  vars:
+    disk_warn_pct: 80
+
   tasks:
     - name: Get disk usage (excluding snap, overlay, tmpfs)
-      ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay | tail -n +2
+      ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay -x efivarfs | tail -n +2
       register: disk_usage
       changed_when: false
 
@@ -55,8 +68,75 @@
           {{ disk_usage.stdout }}
           ======================================================
 
-    - name: Warn on high disk usage (>85%)
-      ansible.builtin.debug:
-        msg: "WARNING: {{ item.split()[-1] }} usage on {{ item.split()[0] }} of {{ inventory_hostname }}"
+    - name: Collect filesystems above {{ disk_warn_pct }}%
+      ansible.builtin.set_fact:
+        disk_alerts: "{{ disk_alerts | default([]) + [item.split()[0] ~ ' ' ~ item.split()[-1]] }}"
       loop: "{{ disk_usage.stdout_lines }}"
-      when: (item.split()[-1] | replace('%', '') | int) > 85
+      when: (item.split()[-1] | replace('%', '') | int) > disk_warn_pct | int
+
+    - name: Warn on high disk usage
+      ansible.builtin.debug:
+        msg: "WARNING: {{ item }} used on {{ inventory_hostname }} (threshold {{ disk_warn_pct }}%)"
+      loop: "{{ disk_alerts | default([]) }}"
+
+- name: Telegram alert
+  hosts: localhost
+  connection: local
+  gather_facts: false
+
+  vars:
+    disk_warn_pct: 80
+    telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}"
+    telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}"
+    telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}"
+    offline_hosts: "{{ groups['all'] | difference(groups['reachable_hosts'] | default([])) }}"
+
+  tasks:
+    - name: Build alert text
+      ansible.builtin.set_fact:
+        alert_text: "{{ alert_body | default('', true) | trim }}"   # an empty template renders as None
+      vars:
+        alert_body: |-
+          {% for h in groups['reachable_hosts'] | default([]) %}
+          {% for fs in hostvars[h].disk_alerts | default([]) %}
+          DISK {{ h }}: {{ fs }} used (threshold {{ disk_warn_pct }}%)
+          {% endfor %}
+          {% endfor %}
+          {% for h in offline_hosts %}
+          OFFLINE {{ h }} did not answer
+          {% endfor %}
+
+    - name: Nothing to alert
+      ansible.builtin.debug:
+        msg: "Every host reachable and all filesystems at or below {{ disk_warn_pct }}% - no alert"
+      when: alert_text | length == 0
+
+    - name: Print alerts when Telegram is not configured
+      ansible.builtin.debug:
+        msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}"
+      when:
+        - alert_text | length > 0
+        - not telegram_configured
+
+    - name: Send Telegram alert
+      ansible.builtin.uri:
+        url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
+        method: POST
+        body_format: json
+        body:
+          chat_id: "{{ telegram_chat_id }}"
+          text: "Homelab health check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
+        status_code: [200]
+      register: telegram_result
+      failed_when: false
+      no_log: true          # the bot token is part of the URL
+      when:
+        - alert_text | length > 0
+        - telegram_configured
+
+    - name: Telegram delivery status
+      ansible.builtin.assert:
+        that: telegram_result.status == 200
+        fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
+        success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
+      when: telegram_result is not skipped

playbooks/new_lxc.yml

@@ -1,73 +1,74 @@
 ---
-- name: Build Dynamic Rocky Linux LXC Container
+# Creates a Rocky Linux 9 container through the Proxmox API and starts it.
+# Runs on the Semaphore host (localhost); proxmoxer must be importable by the
+# Python that runs Ansible there.
+#
+# Extra vars (Semaphore environment / variable group):
+#   proxmox_api_user      - e.g. root@pam            (required)
+#   proxmox_api_password  - that user's password     (this, or a token)
+#   proxmox_api_token_id / proxmox_api_token_secret  (API token instead of a password)
+#   proxmox_api_host      - defaults to the pve entry in the inventory
+#
+# Optional: lxc_hostname, lxc_template, lxc_cores, lxc_memory, lxc_storage,
+# lxc_disk_size, lxc_bridge, lxc_dns, target_node.
+
+- name: Build a Rocky Linux LXC container
   hosts: localhost
+  connection: local
   gather_facts: false
 
   vars:
-    # --- Container Specifications ---
-    lxc_hostname: "new_lxc"
-    lxc_template: "local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz" 
+    proxmox_api_host: "{{ hostvars['pve'].ansible_host }}"
+    target_node: mini-pc
+    lxc_hostname: new-lxc              # underscores are not valid in a hostname
+    lxc_template: local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz
     lxc_cores: 2
     lxc_memory: 1024
-    lxc_storage: "TB"
+    lxc_storage: TB
     lxc_disk_size: 10
-    lxc_bridge: "vmbr0"
-    lxc_dns: "192.168.2.2"
-    target_node: "pve" 
-    
-    # --- Security & Access ---
+    lxc_bridge: vmbr0
+    lxc_dns: "{{ hostvars['pihole'].ansible_host }}"
     lxc_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key"
 
+  module_defaults:
+    community.proxmox.proxmox:
+      api_host: "{{ proxmox_api_host }}"
+      api_user: "{{ proxmox_api_user }}"
+      api_password: "{{ proxmox_api_password | default(omit) }}"
+      api_token_id: "{{ proxmox_api_token_id | default(omit) }}"
+      api_token_secret: "{{ proxmox_api_token_secret | default(omit) }}"
+      validate_certs: false
+      timeout: 300               # template extraction takes longer than the 30 s default
+
   tasks:
-    # ========================================================================
-    # NEW FIX: Auto-install missing dependencies into Semaphore's isolated venv
-    # ========================================================================
-    - name: Ensure Proxmox API dependencies are installed in Semaphore
-      ansible.builtin.shell: "{{ ansible_playbook_python }} -m pip install proxmoxer requests"
-      changed_when: false
-    # ========================================================================
+    - name: Ensure proxmoxer is available to the controller's Python
+      ansible.builtin.command: "{{ ansible_playbook_python }} -m pip install proxmoxer requests"
+      register: pip_install
+      changed_when: "'Successfully installed' in pip_install.stdout"
 
-    - name: Get the next available VMID from the Proxmox cluster
-      ansible.builtin.uri:
-        url: "https://{{ proxmox_api_host }}:8006/api2/json/cluster/nextid"
-        method: GET
-        headers:
-          Authorization: "PVEAPIToken {{ proxmox_api_user }}={{ proxmox_api_password }}"
-        validate_certs: false
-        return_content: true
-      register: nextid_response
-
-    - name: Extract the dynamic VMID from the API response
-      ansible.builtin.set_fact:
-        dynamic_vmid: "{{ nextid_response.json.data }}"
-
-    - name: Create the Rocky Linux LXC container
-      community.general.proxmox:
-        api_host: "{{ proxmox_api_host }}"
-        api_user: "{{ proxmox_api_user }}"
-        api_password: "{{ proxmox_api_password }}"
-        
+    - name: Create the container (Proxmox assigns the next free VMID)
+      community.proxmox.proxmox:
         node: "{{ target_node }}"
-        vmid: "{{ dynamic_vmid }}"
         hostname: "{{ lxc_hostname }}"
         ostemplate: "{{ lxc_template }}"
         cores: "{{ lxc_cores }}"
         memory: "{{ lxc_memory }}"
         disk: "{{ lxc_storage }}:{{ lxc_disk_size }}"
-        netif: '{"net0":"name=eth0,bridge={{ lxc_bridge }},ip=dhcp"}'
+        netif:
+          net0: "name=eth0,bridge={{ lxc_bridge }},ip=dhcp"
         nameserver: "{{ lxc_dns }}"
         pubkey: "{{ lxc_ssh_key }}"
         unprivileged: true
         state: present
+      register: lxc_create
 
-    - name: Ensure the new LXC container is powered on
-      community.general.proxmox:
-        api_host: "{{ proxmox_api_host }}"
-        api_user: "{{ proxmox_api_user }}"
-        api_password: "{{ proxmox_api_password }}"
-        vmid: "{{ dynamic_vmid }}"
+    - name: Start the container
+      community.proxmox.proxmox:
+        vmid: "{{ lxc_create.vmid }}"
         state: started
-        
-    - name: Output the assigned VMID to the Semaphore UI logs
+
+    - name: Report
       ansible.builtin.debug:
-        msg: "Successfully built {{ lxc_hostname }} (Rocky Linux 9) with dynamic ID: {{ dynamic_vmid }}"
+        msg: >-
+          Built {{ lxc_hostname }} (Rocky Linux 9) as VMID {{ lxc_create.vmid }} on {{ target_node }}.
+          Next: run discover_guests.yml with adopt=true, add it to inventories/all.yml, then bootstrap_lxc.yml.

playbooks/proxmox-backup.yml

@@ -1,60 +1,81 @@
 ---
+# vzdump of selected guests on the Proxmox host. vzdump prunes old dumps of
+# each guest itself (keep-last), so nothing else touches the dump directory.
+# Every run is appended to {{ backup_dir }}/proxmox_backup.log on the host.
+#
+# backup_mode "stop" shuts the guest down for the whole backup. Never list
+# the guest that runs Semaphore with mode stop: the job would stop itself
+# half way through. Use "snapshot" for that one (falls back to suspend on
+# storage without snapshot support).
+#
+# Extra vars (optional):
+#   guests       - list of { id, type, name }
+#   keep_last    - dumps kept per guest (default 3)
+#   backup_mode  - stop | snapshot | suspend (default stop)
+#   storage      - target PVE storage id (default local)
+#   compression  - zstd | gzip | lzo (default zstd)
+
 - name: Backup selected Proxmox VMs and LXCs
   hosts: proxmox
   become: true
+  gather_facts: false
+
   vars:
-    backup_dir: "/var/lib/vz/dump"
-    storage: "local"
-    compression: "zstd"
-    maxfiles: 3
+    backup_dir: /var/lib/vz/dump
+    storage: local
+    compression: zstd
+    keep_last: 3
+    backup_mode: stop
     guests:
-      #- { id: 104, type: "lxc", name: "jellyfin" }
-      - { id: 108, type: "lxc", name: "docker" }
-      #- { id: 101, type: "qemu", name: "pnetlab" }
+      # - { id: 104, type: lxc, name: courses }
+      - { id: 108, type: lxc, name: docker }
+      # - { id: 101, type: qemu, name: pnetlab }
 
   tasks:
-    - name: Ensure backup directory exists
-      file:
+    - name: Ensure the log directory exists
+      ansible.builtin.file:
         path: "{{ backup_dir }}"
         state: directory
         mode: '0755'
 
-    - name: Run vzdump backup with logging
-      shell: |
+    - name: Run vzdump
+      ansible.builtin.shell: |
+        set -o pipefail
         LOGFILE="{{ backup_dir }}/proxmox_backup.log"
         START=$(date +%s)
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup of {{ item.name }} ({{ item.type }}) [ID: {{ item.id }}]" >> "$LOGFILE"
-
-        OUTPUT=$(vzdump {{ item.id }} --mode stop --compress {{ compression }} --storage {{ storage }} --remove 0 2>&1)
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] START   {{ item.name }} ({{ item.type }} {{ item.id }}) mode={{ backup_mode }}" >> "$LOGFILE"
+        OUTPUT=$(vzdump {{ item.id }} --mode {{ backup_mode }} --compress {{ compression }} \
+                 --storage {{ storage }} --prune-backups keep-last={{ keep_last }} 2>&1)
         STATUS=$?
-
-        END=$(date +%s)
-        DURATION=$((END - START))
-
-        if [ $STATUS -eq 0 ]; then
-          echo "[$(date '+%Y-%m-%d %H:%M:%S')] ✅ SUCCESS - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE"
+        DURATION=$(( $(date +%s) - START ))
+        if [ "$STATUS" -eq 0 ]; then
+          echo "[$(date '+%Y-%m-%d %H:%M:%S')] SUCCESS {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE"
+          echo "$OUTPUT" | grep -E 'archive file size|pruned|removing backup' || true
         else
-          echo "[$(date '+%Y-%m-%d %H:%M:%S')] ❌ FAILURE - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE"
-          echo "---- ERROR OUTPUT ----" >> "$LOGFILE"
-          echo "$OUTPUT" >> "$LOGFILE"
-          echo "----------------------" >> "$LOGFILE"
+          echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAILURE {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE"
+          printf -- '---- vzdump output ----\n%s\n-----------------------\n' "$OUTPUT" >> "$LOGFILE"
+          echo "$OUTPUT" | tail -n 20
         fi
+        exit "$STATUS"
       args:
         executable: /bin/bash
       loop: "{{ guests }}"
+      loop_control:
+        label: "{{ item.name }} ({{ item.type }} {{ item.id }})"
+      changed_when: true
 
-    - name: Rename backup files to include guest name
-      shell: |
-        latest_file=$(ls -t {{ backup_dir }}/vzdump-{{ item.type }}-{{ item.id }}-*.{{ compression }} | head -n 1)
-        new_file="{{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-{{ '%Y%m%d-%H%M%S' | strftime(ansible_date_time.epoch | int) }}.{{ compression }}"
-        cp "$latest_file" "$new_file"
-      args:
-        executable: /bin/bash
+    - name: List the dumps kept for each guest
+      ansible.builtin.command: pvesm list {{ storage }} --content backup --vmid {{ item.id }}
       loop: "{{ guests }}"
+      loop_control:
+        label: "{{ item.name }} ({{ item.id }})"
+      register: dump_lists
+      changed_when: false
 
-    - name: Prune old backups (keep last {{ maxfiles }})
-      shell: |
-        ls -tp {{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-*{{ compression }} | grep -v '/$' | tail -n +{{ maxfiles + 1 }} | xargs -r rm --
-      args:
-        executable: /bin/bash
-      loop: "{{ guests }}"
+    - name: Backup summary
+      ansible.builtin.debug:
+        msg: |
+          {% for r in dump_lists.results %}
+          {{ r.item.name }} ({{ r.item.type }} {{ r.item.id }}) - mode {{ backup_mode }}, keeping the last {{ keep_last }}:
+          {{ r.stdout }}
+          {% endfor %}

playbooks/reboot_check.yml

@@ -22,12 +22,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: Reboot Check - Ubuntu

playbooks/service_check.yml

@@ -1,9 +1,13 @@
 ---
 # Verifies the critical services listed per host in the inventory
-# (critical_services) and optionally starts any that are stopped.
+# (critical_services) and optionally starts any that are stopped. Services
+# found not running are sent as one Telegram message per run when Telegram
+# credentials are present.
 #
 # Extra vars (optional):
-#   restart_stopped - default false; when true, start services found stopped
+#   restart_stopped    - default false; when true, start services found stopped
+#   telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN
+#   telegram_chat_id   - or environment variable TELEGRAM_CHAT_ID
 
 - name: Check Host Availability
   hosts: ubuntu:rocky
@@ -20,12 +24,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: Service Check
@@ -80,3 +84,62 @@
       when:
         - restart_stopped | bool
         - item.state not in ['running', 'not-found']
+
+- name: Telegram alert
+  hosts: localhost
+  connection: local
+  gather_facts: false
+
+  vars:
+    restart_stopped: false
+    telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}"
+    telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}"
+    telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}"
+
+  tasks:
+    - name: Build alert text
+      ansible.builtin.set_fact:
+        alert_text: "{{ alert_body | default('', true) | trim }}"   # an empty template renders as None
+      vars:
+        alert_body: |-
+          {% for h in groups['reachable_hosts'] | default([]) %}
+          {% for svc in hostvars[h].service_states | default([]) if svc.state != 'running' %}
+          {% set note = ' - start attempted' if (restart_stopped | bool and svc.state != 'not-found') else '' %}
+          SERVICE {{ h }}: {{ svc.name }} {{ 'not installed' if svc.state == 'not-found' else svc.state }}{{ note }}
+          {% endfor %}
+          {% endfor %}
+
+    - name: Nothing to alert
+      ansible.builtin.debug:
+        msg: "All critical services running - no alert"
+      when: alert_text | length == 0
+
+    - name: Print alerts when Telegram is not configured
+      ansible.builtin.debug:
+        msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}"
+      when:
+        - alert_text | length > 0
+        - not telegram_configured
+
+    - name: Send Telegram alert
+      ansible.builtin.uri:
+        url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
+        method: POST
+        body_format: json
+        body:
+          chat_id: "{{ telegram_chat_id }}"
+          text: "Homelab service check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
+        status_code: [200]
+      register: telegram_result
+      failed_when: false
+      no_log: true          # the bot token is part of the URL
+      when:
+        - alert_text | length > 0
+        - telegram_configured
+
+    - name: Telegram delivery status
+      ansible.builtin.assert:
+        that: telegram_result.status == 200
+        fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
+        success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
+      when: telegram_result is not skipped

playbooks/ssh_harden.yml

@@ -26,12 +26,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: SSH Hardening - Install fail2ban on Ubuntu

playbooks/time-region.yml

@@ -1,30 +1,46 @@
 ---
-- name: Configure timezone on all hosts
+# Sets the timezone on every Ubuntu/Rocky guest. LXC containers share the
+# host's clock, so NTP is disabled inside them; VMs sync against ntp_server.
+#
+# Extra vars (optional):
+#   timezone_region - default America/New_York
+#   ntp_server      - default time.google.com
+
+- name: Configure timezone
   hosts: ubuntu:rocky
   become: true
-  gather_facts: true
 
   vars:
-    timezone_region: "America/New_York"
+    timezone_region: America/New_York
 
   tasks:
-    - name: Copy timezone file
-      ansible.builtin.copy:
+    - name: Link /etc/localtime to the zoneinfo file
+      ansible.builtin.file:
         src: "/usr/share/zoneinfo/{{ timezone_region }}"
         dest: /etc/localtime
-        remote_src: true
+        state: link
+        force: true
 
     - name: Write /etc/timezone
       ansible.builtin.copy:
         dest: /etc/timezone
         content: "{{ timezone_region }}\n"
+        owner: root
+        group: root
+        mode: '0644'
 
-- name: Disable NTP on Ubuntu LXC containers
+# The plays below reuse the facts gathered above instead of collecting them again.
+
+- name: NTP on Ubuntu
   hosts: ubuntu
   become: true
+  gather_facts: false
+
+  vars:
+    ntp_server: time.google.com
 
   tasks:
-    - name: Stop and mask timesyncd on LXC
+    - name: Stop and mask timesyncd in LXC containers
       ansible.builtin.systemd:
         name: systemd-timesyncd
         state: stopped
@@ -32,33 +48,11 @@
         masked: true
       when: ansible_virtualization_type == "lxc"
 
-- name: Disable NTP on Rocky Linux LXC containers
-  hosts: rocky
-  become: true
-
-  tasks:
-    - name: Stop and mask chronyd on LXC
-      ansible.builtin.systemd:
-        name: chronyd
-        state: stopped
-        enabled: false
-        masked: true
-      when: ansible_virtualization_type == "lxc"
-
-- name: Configure NTP on Ubuntu VMs
-  hosts: ubuntu
-  become: true
-
-  vars:
-    ntp_server: "time.google.com"
-
-  tasks:
-    - name: Configure timesyncd
+    - name: Point timesyncd at ntp_server on VMs
       ansible.builtin.lineinfile:
         path: /etc/systemd/timesyncd.conf
         regexp: '^#?NTP='
         line: "NTP={{ ntp_server }}"
-        state: present
       when: ansible_virtualization_type != "lxc"
       notify: Restart timesyncd
 
@@ -69,27 +63,38 @@
         state: restarted
         enabled: true
 
-- name: Configure NTP on Rocky Linux VMs
+- name: NTP on Rocky Linux
   hosts: rocky
   become: true
+  gather_facts: false
 
   vars:
-    ntp_server: "time.google.com"
+    ntp_server: time.google.com
 
   tasks:
-    - name: Disable default chrony pools
+    - name: Stop and mask chronyd in LXC containers
+      ansible.builtin.systemd:
+        name: chronyd
+        state: stopped
+        enabled: false
+        masked: true
+      when: ansible_virtualization_type == "lxc"
+
+    # The lookahead keeps the server line added below from being commented
+    # out again on the next run.
+    - name: Disable the default chrony pools on VMs
       ansible.builtin.replace:
         path: /etc/chrony.conf
-        regexp: '^(pool|server) .*'
-        replace: '# \1 removed by Ansible'
+        regexp: '^(pool|server)\s+(?!{{ ntp_server | regex_escape }}(\s|$)).*'
+        replace: '# \g<0>  (disabled by Ansible)'
       when: ansible_virtualization_type != "lxc"
+      notify: Restart chronyd
 
-    - name: Configure chrony NTP server
+    - name: Point chrony at ntp_server on VMs
       ansible.builtin.lineinfile:
         path: /etc/chrony.conf
+        regexp: '^server\s+{{ ntp_server | regex_escape }}(\s|$)'
         line: "server {{ ntp_server }} iburst"
-        insertafter: EOF
-        state: present
       when: ansible_virtualization_type != "lxc"
       notify: Restart chronyd
 
@@ -103,6 +108,7 @@
 - name: Verify time configuration
   hosts: ubuntu:rocky
   become: true
+  gather_facts: false
 
   tasks:
     - name: Check current system time

playbooks/update_pihole_blocklists.yml

@@ -1,18 +1,22 @@
 ---
 - name: Update Pi-hole core and blocklists
   hosts: pihole
-  become: yes
+  become: true
+  gather_facts: false
+
   tasks:
     - name: Update Pi-hole core components
-      command: pihole -up
+      ansible.builtin.command: pihole -up
       register: update_out
+      changed_when: "'Everything is up to date' not in update_out.stdout"
 
     - name: Update Pi-hole blocklists
-      command: pihole -g
+      ansible.builtin.command: pihole -g
       register: gravity_out
+      changed_when: true
 
     - name: Display update summary
-      debug:
+      ansible.builtin.debug:
         msg:
           - "Core update: {{ update_out.stdout }}"
           - "Gravity update: {{ gravity_out.stdout }}"

playbooks/update_system.yml

@@ -1,6 +1,6 @@
 ---
 - name: Check Host Availability
-  hosts: all
+  hosts: ubuntu:rocky
   gather_facts: false
   ignore_unreachable: true
 
@@ -14,12 +14,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "Host {{ inventory_hostname }} is OFFLINE and will be skipped"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: Update and Upgrade - Ubuntu

playbooks/user_setup.yml

@@ -24,12 +24,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is succeeded
+      when: ping_result is reachable and ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is not succeeded
+      when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
 - name: User Setup - Create admin user on Ubuntu