Fix playbook bugs from review, add Telegram alerts to health and service checks

Review fixes:
- Availability plays: "ping_result is succeeded" is true for unreachable
  hosts, so offline hosts were added to reachable_hosts and failed the
  next play. Test reachability explicitly (7 playbooks).
- bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next
  to EPEL, add the root key without replacing authorized_keys, validate
  sshd_config edits and restart sshd only on change.
- backup_docker: the command module does not expand $(...); fix rsync
  filter order and add --exclude=*; restart only the containers that were
  stopped, even when rsync fails; read the storage address from the
  inventory; keep semaphore and semaphore-db running.
- new_lxc: community.general.proxmox no longer exists, use
  community.proxmox; node is mini-pc; valid hostname; password or token
  auth; let the module pick the VMID; longer timeout.
- proxmox-backup: surface the vzdump exit code, let vzdump prune with
  keep-last, drop the rename/prune steps that globbed *.zstd.
- time-region: chrony regex no longer comments out its own server line;
  /etc/localtime as a symlink; reuse facts across plays.
- update_pihole_blocklists: FQCN, become: true, changed_when.
- ansible.cfg gathering = smart; update_system only pings ubuntu/rocky.

Telegram alerts:
- health_check: filesystems above disk_warn_pct (default 80), excluding
  efivarfs, and offline hosts.
- service_check: services not running.
- One message per run from localhost, credentials from Semaphore
  (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task,
  prints the alert text when Telegram is not configured.

Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq

playbooks/new_lxc.yml

@@ -1,73 +1,74 @@
 ---
-- name: Build Dynamic Rocky Linux LXC Container
+# Creates a Rocky Linux 9 container through the Proxmox API and starts it.
+# Runs on the Semaphore host (localhost); proxmoxer must be importable by the
+# Python that runs Ansible there.
+#
+# Extra vars (Semaphore environment / variable group):
+#   proxmox_api_user      - e.g. root@pam            (required)
+#   proxmox_api_password  - that user's password     (this, or a token)
+#   proxmox_api_token_id / proxmox_api_token_secret  (API token instead of a password)
+#   proxmox_api_host      - defaults to the pve entry in the inventory
+#
+# Optional: lxc_hostname, lxc_template, lxc_cores, lxc_memory, lxc_storage,
+# lxc_disk_size, lxc_bridge, lxc_dns, target_node.
+
+- name: Build a Rocky Linux LXC container
   hosts: localhost
+  connection: local
   gather_facts: false
 
   vars:
-    # --- Container Specifications ---
-    lxc_hostname: "new_lxc"
-    lxc_template: "local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz" 
+    proxmox_api_host: "{{ hostvars['pve'].ansible_host }}"
+    target_node: mini-pc
+    lxc_hostname: new-lxc              # underscores are not valid in a hostname
+    lxc_template: local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz
     lxc_cores: 2
     lxc_memory: 1024
-    lxc_storage: "TB"
+    lxc_storage: TB
     lxc_disk_size: 10
-    lxc_bridge: "vmbr0"
-    lxc_dns: "192.168.2.2"
-    target_node: "pve" 
-    
-    # --- Security & Access ---
+    lxc_bridge: vmbr0
+    lxc_dns: "{{ hostvars['pihole'].ansible_host }}"
     lxc_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key"
 
+  module_defaults:
+    community.proxmox.proxmox:
+      api_host: "{{ proxmox_api_host }}"
+      api_user: "{{ proxmox_api_user }}"
+      api_password: "{{ proxmox_api_password | default(omit) }}"
+      api_token_id: "{{ proxmox_api_token_id | default(omit) }}"
+      api_token_secret: "{{ proxmox_api_token_secret | default(omit) }}"
+      validate_certs: false
+      timeout: 300               # template extraction takes longer than the 30 s default
+
   tasks:
-    # ========================================================================
-    # NEW FIX: Auto-install missing dependencies into Semaphore's isolated venv
-    # ========================================================================
-    - name: Ensure Proxmox API dependencies are installed in Semaphore
-      ansible.builtin.shell: "{{ ansible_playbook_python }} -m pip install proxmoxer requests"
-      changed_when: false
-    # ========================================================================
+    - name: Ensure proxmoxer is available to the controller's Python
+      ansible.builtin.command: "{{ ansible_playbook_python }} -m pip install proxmoxer requests"
+      register: pip_install
+      changed_when: "'Successfully installed' in pip_install.stdout"
 
-    - name: Get the next available VMID from the Proxmox cluster
-      ansible.builtin.uri:
-        url: "https://{{ proxmox_api_host }}:8006/api2/json/cluster/nextid"
-        method: GET
-        headers:
-          Authorization: "PVEAPIToken {{ proxmox_api_user }}={{ proxmox_api_password }}"
-        validate_certs: false
-        return_content: true
-      register: nextid_response
-
-    - name: Extract the dynamic VMID from the API response
-      ansible.builtin.set_fact:
-        dynamic_vmid: "{{ nextid_response.json.data }}"
-
-    - name: Create the Rocky Linux LXC container
-      community.general.proxmox:
-        api_host: "{{ proxmox_api_host }}"
-        api_user: "{{ proxmox_api_user }}"
-        api_password: "{{ proxmox_api_password }}"
-        
+    - name: Create the container (Proxmox assigns the next free VMID)
+      community.proxmox.proxmox:
         node: "{{ target_node }}"
-        vmid: "{{ dynamic_vmid }}"
         hostname: "{{ lxc_hostname }}"
         ostemplate: "{{ lxc_template }}"
         cores: "{{ lxc_cores }}"
         memory: "{{ lxc_memory }}"
         disk: "{{ lxc_storage }}:{{ lxc_disk_size }}"
-        netif: '{"net0":"name=eth0,bridge={{ lxc_bridge }},ip=dhcp"}'
+        netif:
+          net0: "name=eth0,bridge={{ lxc_bridge }},ip=dhcp"
         nameserver: "{{ lxc_dns }}"
         pubkey: "{{ lxc_ssh_key }}"
         unprivileged: true
         state: present
+      register: lxc_create
 
-    - name: Ensure the new LXC container is powered on
-      community.general.proxmox:
-        api_host: "{{ proxmox_api_host }}"
-        api_user: "{{ proxmox_api_user }}"
-        api_password: "{{ proxmox_api_password }}"
-        vmid: "{{ dynamic_vmid }}"
+    - name: Start the container
+      community.proxmox.proxmox:
+        vmid: "{{ lxc_create.vmid }}"
         state: started
-        
-    - name: Output the assigned VMID to the Semaphore UI logs
+
+    - name: Report
       ansible.builtin.debug:
-        msg: "Successfully built {{ lxc_hostname }} (Rocky Linux 9) with dynamic ID: {{ dynamic_vmid }}"
+        msg: >-
+          Built {{ lxc_hostname }} (Rocky Linux 9) as VMID {{ lxc_create.vmid }} on {{ target_node }}.
+          Next: run discover_guests.yml with adopt=true, add it to inventories/all.yml, then bootstrap_lxc.yml.