Fix playbook bugs from review, add Telegram alerts to health and service checks

Review fixes:
- Availability plays: "ping_result is succeeded" is true for unreachable
  hosts, so offline hosts were added to reachable_hosts and failed the
  next play. Test reachability explicitly (7 playbooks).
- bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next
  to EPEL, add the root key without replacing authorized_keys, validate
  sshd_config edits and restart sshd only on change.
- backup_docker: the command module does not expand $(...); fix rsync
  filter order and add --exclude=*; restart only the containers that were
  stopped, even when rsync fails; read the storage address from the
  inventory; keep semaphore and semaphore-db running.
- new_lxc: community.general.proxmox no longer exists, use
  community.proxmox; node is mini-pc; valid hostname; password or token
  auth; let the module pick the VMID; longer timeout.
- proxmox-backup: surface the vzdump exit code, let vzdump prune with
  keep-last, drop the rename/prune steps that globbed *.zstd.
- time-region: chrony regex no longer comments out its own server line;
  /etc/localtime as a symlink; reuse facts across plays.
- update_pihole_blocklists: FQCN, become: true, changed_when.
- ansible.cfg gathering = smart; update_system only pings ubuntu/rocky.

Telegram alerts:
- health_check: filesystems above disk_warn_pct (default 80), excluding
  efivarfs, and offline hosts.
- service_check: services not running.
- One message per run from localhost, credentials from Semaphore
  (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task,
  prints the alert text when Telegram is not configured.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq

playbooks/proxmox-backup.yml

@@ -1,60 +1,81 @@
 ---
+# vzdump of selected guests on the Proxmox host. vzdump prunes old dumps of
+# each guest itself (keep-last), so nothing else touches the dump directory.
+# Every run is appended to {{ backup_dir }}/proxmox_backup.log on the host.
+#
+# backup_mode "stop" shuts the guest down for the whole backup. Never list
+# the guest that runs Semaphore with mode stop: the job would stop itself
+# half way through. Use "snapshot" for that one (falls back to suspend on
+# storage without snapshot support).
+#
+# Extra vars (optional):
+#   guests       - list of { id, type, name }
+#   keep_last    - dumps kept per guest (default 3)
+#   backup_mode  - stop | snapshot | suspend (default stop)
+#   storage      - target PVE storage id (default local)
+#   compression  - zstd | gzip | lzo (default zstd)
+
 - name: Backup selected Proxmox VMs and LXCs
   hosts: proxmox
   become: true
+  gather_facts: false
+
   vars:
-    backup_dir: "/var/lib/vz/dump"
-    storage: "local"
-    compression: "zstd"
-    maxfiles: 3
+    backup_dir: /var/lib/vz/dump
+    storage: local
+    compression: zstd
+    keep_last: 3
+    backup_mode: stop
     guests:
-      #- { id: 104, type: "lxc", name: "jellyfin" }
-      - { id: 108, type: "lxc", name: "docker" }
-      #- { id: 101, type: "qemu", name: "pnetlab" }
+      # - { id: 104, type: lxc, name: courses }
+      - { id: 108, type: lxc, name: docker }
+      # - { id: 101, type: qemu, name: pnetlab }
 
   tasks:
-    - name: Ensure backup directory exists
-      file:
+    - name: Ensure the log directory exists
+      ansible.builtin.file:
         path: "{{ backup_dir }}"
         state: directory
         mode: '0755'
 
-    - name: Run vzdump backup with logging
-      shell: |
+    - name: Run vzdump
+      ansible.builtin.shell: |
+        set -o pipefail
         LOGFILE="{{ backup_dir }}/proxmox_backup.log"
         START=$(date +%s)
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup of {{ item.name }} ({{ item.type }}) [ID: {{ item.id }}]" >> "$LOGFILE"
-
-        OUTPUT=$(vzdump {{ item.id }} --mode stop --compress {{ compression }} --storage {{ storage }} --remove 0 2>&1)
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] START   {{ item.name }} ({{ item.type }} {{ item.id }}) mode={{ backup_mode }}" >> "$LOGFILE"
+        OUTPUT=$(vzdump {{ item.id }} --mode {{ backup_mode }} --compress {{ compression }} \
+                 --storage {{ storage }} --prune-backups keep-last={{ keep_last }} 2>&1)
         STATUS=$?
-
-        END=$(date +%s)
-        DURATION=$((END - START))
-
-        if [ $STATUS -eq 0 ]; then
-          echo "[$(date '+%Y-%m-%d %H:%M:%S')] ✅ SUCCESS - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE"
+        DURATION=$(( $(date +%s) - START ))
+        if [ "$STATUS" -eq 0 ]; then
+          echo "[$(date '+%Y-%m-%d %H:%M:%S')] SUCCESS {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE"
+          echo "$OUTPUT" | grep -E 'archive file size|pruned|removing backup' || true
         else
-          echo "[$(date '+%Y-%m-%d %H:%M:%S')] ❌ FAILURE - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE"
-          echo "---- ERROR OUTPUT ----" >> "$LOGFILE"
-          echo "$OUTPUT" >> "$LOGFILE"
-          echo "----------------------" >> "$LOGFILE"
+          echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAILURE {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE"
+          printf -- '---- vzdump output ----\n%s\n-----------------------\n' "$OUTPUT" >> "$LOGFILE"
+          echo "$OUTPUT" | tail -n 20
         fi
+        exit "$STATUS"
       args:
         executable: /bin/bash
       loop: "{{ guests }}"
+      loop_control:
+        label: "{{ item.name }} ({{ item.type }} {{ item.id }})"
+      changed_when: true
 
-    - name: Rename backup files to include guest name
-      shell: |
-        latest_file=$(ls -t {{ backup_dir }}/vzdump-{{ item.type }}-{{ item.id }}-*.{{ compression }} | head -n 1)
-        new_file="{{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-{{ '%Y%m%d-%H%M%S' | strftime(ansible_date_time.epoch | int) }}.{{ compression }}"
-        cp "$latest_file" "$new_file"
-      args:
-        executable: /bin/bash
+    - name: List the dumps kept for each guest
+      ansible.builtin.command: pvesm list {{ storage }} --content backup --vmid {{ item.id }}
       loop: "{{ guests }}"
+      loop_control:
+        label: "{{ item.name }} ({{ item.id }})"
+      register: dump_lists
+      changed_when: false
 
-    - name: Prune old backups (keep last {{ maxfiles }})
-      shell: |
-        ls -tp {{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-*{{ compression }} | grep -v '/$' | tail -n +{{ maxfiles + 1 }} | xargs -r rm --
-      args:
-        executable: /bin/bash
-      loop: "{{ guests }}"
+    - name: Backup summary
+      ansible.builtin.debug:
+        msg: |
+          {% for r in dump_lists.results %}
+          {{ r.item.name }} ({{ r.item.type }} {{ r.item.id }}) - mode {{ backup_mode }}, keeping the last {{ keep_last }}:
+          {{ r.stdout }}
+          {% endfor %}