Fix playbook bugs from review, add Telegram alerts to health and service checks

Review fixes:
- Availability plays: "ping_result is succeeded" is true for unreachable
  hosts, so offline hosts were added to reachable_hosts and failed the
  next play. Test reachability explicitly (7 playbooks).
- bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next
  to EPEL, add the root key without replacing authorized_keys, validate
  sshd_config edits and restart sshd only on change.
- backup_docker: the command module does not expand $(...); fix rsync
  filter order and add --exclude=*; restart only the containers that were
  stopped, even when rsync fails; read the storage address from the
  inventory; keep semaphore and semaphore-db running.
- new_lxc: community.general.proxmox no longer exists, use
  community.proxmox; node is mini-pc; valid hostname; password or token
  auth; let the module pick the VMID; longer timeout.
- proxmox-backup: surface the vzdump exit code, let vzdump prune with
  keep-last, drop the rename/prune steps that globbed *.zstd.
- time-region: chrony regex no longer comments out its own server line;
  /etc/localtime as a symlink; reuse facts across plays.
- update_pihole_blocklists: FQCN, become: true, changed_when.
- ansible.cfg gathering = smart; update_system only pings ubuntu/rocky.

Telegram alerts:
- health_check: filesystems above disk_warn_pct (default 80), excluding
  efivarfs, and offline hosts.
- service_check: services not running.
- One message per run from localhost, credentials from Semaphore
  (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task,
  prints the alert text when Telegram is not configured.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq
This commit is contained in:
Harwinder Sekhon
2026-09-06 23:06:18 -04:00
parent 40eb95def5
commit 006103760f
21 changed files with 515 additions and 259 deletions
+67 -4
View File
@@ -1,9 +1,13 @@
---
# Verifies the critical services listed per host in the inventory
# (critical_services) and optionally starts any that are stopped.
# (critical_services) and optionally starts any that are stopped. Services
# found not running are sent as one Telegram message per run when Telegram
# credentials are present.
#
# Extra vars (optional):
# restart_stopped - default false; when true, start services found stopped
# restart_stopped - default false; when true, start services found stopped
# telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN
# telegram_chat_id - or environment variable TELEGRAM_CHAT_ID
- name: Check Host Availability
hosts: ubuntu:rocky
@@ -20,12 +24,12 @@
- name: Add reachable hosts to group
ansible.builtin.group_by:
key: reachable_hosts
when: ping_result is succeeded
when: ping_result is reachable and ping_result is succeeded
- name: Report offline hosts
ansible.builtin.debug:
msg: "{{ inventory_hostname }} - OFFLINE"
when: ping_result is not succeeded
when: ping_result is unreachable or ping_result is failed
delegate_to: localhost
- name: Service Check
@@ -80,3 +84,62 @@
when:
- restart_stopped | bool
- item.state not in ['running', 'not-found']
- name: Telegram alert
hosts: localhost
connection: local
gather_facts: false
vars:
restart_stopped: false
telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}"
telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}"
telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}"
tasks:
- name: Build alert text
ansible.builtin.set_fact:
alert_text: "{{ alert_body | default('', true) | trim }}" # an empty template renders as None
vars:
alert_body: |-
{% for h in groups['reachable_hosts'] | default([]) %}
{% for svc in hostvars[h].service_states | default([]) if svc.state != 'running' %}
{% set note = ' - start attempted' if (restart_stopped | bool and svc.state != 'not-found') else '' %}
SERVICE {{ h }}: {{ svc.name }} {{ 'not installed' if svc.state == 'not-found' else svc.state }}{{ note }}
{% endfor %}
{% endfor %}
- name: Nothing to alert
ansible.builtin.debug:
msg: "All critical services running - no alert"
when: alert_text | length == 0
- name: Print alerts when Telegram is not configured
ansible.builtin.debug:
msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}"
when:
- alert_text | length > 0
- not telegram_configured
- name: Send Telegram alert
ansible.builtin.uri:
url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
method: POST
body_format: json
body:
chat_id: "{{ telegram_chat_id }}"
text: "Homelab service check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
status_code: [200]
register: telegram_result
failed_when: false
no_log: true # the bot token is part of the URL
when:
- alert_text | length > 0
- telegram_configured
- name: Telegram delivery status
ansible.builtin.assert:
that: telegram_result.status == 200
fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
when: telegram_result is not skipped