Fix playbook bugs from review, add Telegram alerts to health and service checks
Review fixes: - Availability plays: "ping_result is succeeded" is true for unreachable hosts, so offline hosts were added to reachable_hosts and failed the next play. Test reachability explicitly (7 playbooks). - bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next to EPEL, add the root key without replacing authorized_keys, validate sshd_config edits and restart sshd only on change. - backup_docker: the command module does not expand $(...); fix rsync filter order and add --exclude=*; restart only the containers that were stopped, even when rsync fails; read the storage address from the inventory; keep semaphore and semaphore-db running. - new_lxc: community.general.proxmox no longer exists, use community.proxmox; node is mini-pc; valid hostname; password or token auth; let the module pick the VMID; longer timeout. - proxmox-backup: surface the vzdump exit code, let vzdump prune with keep-last, drop the rename/prune steps that globbed *.zstd. - time-region: chrony regex no longer comments out its own server line; /etc/localtime as a symlink; reuse facts across plays. - update_pihole_blocklists: FQCN, become: true, changed_when. - ansible.cfg gathering = smart; update_system only pings ubuntu/rocky. Telegram alerts: - health_check: filesystems above disk_warn_pct (default 80), excluding efivarfs, and offline hosts. - service_check: services not running. - One message per run from localhost, credentials from Semaphore (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task, prints the alert text when Telegram is not configured. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq
This commit is contained in:
@@ -1,9 +1,13 @@
|
||||
---
|
||||
# Verifies the critical services listed per host in the inventory
|
||||
# (critical_services) and optionally starts any that are stopped.
|
||||
# (critical_services) and optionally starts any that are stopped. Services
|
||||
# found not running are sent as one Telegram message per run when Telegram
|
||||
# credentials are present.
|
||||
#
|
||||
# Extra vars (optional):
|
||||
# restart_stopped - default false; when true, start services found stopped
|
||||
# restart_stopped - default false; when true, start services found stopped
|
||||
# telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN
|
||||
# telegram_chat_id - or environment variable TELEGRAM_CHAT_ID
|
||||
|
||||
- name: Check Host Availability
|
||||
hosts: ubuntu:rocky
|
||||
@@ -20,12 +24,12 @@
|
||||
- name: Add reachable hosts to group
|
||||
ansible.builtin.group_by:
|
||||
key: reachable_hosts
|
||||
when: ping_result is succeeded
|
||||
when: ping_result is reachable and ping_result is succeeded
|
||||
|
||||
- name: Report offline hosts
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ inventory_hostname }} - OFFLINE"
|
||||
when: ping_result is not succeeded
|
||||
when: ping_result is unreachable or ping_result is failed
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Service Check
|
||||
@@ -80,3 +84,62 @@
|
||||
when:
|
||||
- restart_stopped | bool
|
||||
- item.state not in ['running', 'not-found']
|
||||
|
||||
- name: Telegram alert
|
||||
hosts: localhost
|
||||
connection: local
|
||||
gather_facts: false
|
||||
|
||||
vars:
|
||||
restart_stopped: false
|
||||
telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}"
|
||||
telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}"
|
||||
telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}"
|
||||
|
||||
tasks:
|
||||
- name: Build alert text
|
||||
ansible.builtin.set_fact:
|
||||
alert_text: "{{ alert_body | default('', true) | trim }}" # an empty template renders as None
|
||||
vars:
|
||||
alert_body: |-
|
||||
{% for h in groups['reachable_hosts'] | default([]) %}
|
||||
{% for svc in hostvars[h].service_states | default([]) if svc.state != 'running' %}
|
||||
{% set note = ' - start attempted' if (restart_stopped | bool and svc.state != 'not-found') else '' %}
|
||||
SERVICE {{ h }}: {{ svc.name }} {{ 'not installed' if svc.state == 'not-found' else svc.state }}{{ note }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
- name: Nothing to alert
|
||||
ansible.builtin.debug:
|
||||
msg: "All critical services running - no alert"
|
||||
when: alert_text | length == 0
|
||||
|
||||
- name: Print alerts when Telegram is not configured
|
||||
ansible.builtin.debug:
|
||||
msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}"
|
||||
when:
|
||||
- alert_text | length > 0
|
||||
- not telegram_configured
|
||||
|
||||
- name: Send Telegram alert
|
||||
ansible.builtin.uri:
|
||||
url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
|
||||
method: POST
|
||||
body_format: json
|
||||
body:
|
||||
chat_id: "{{ telegram_chat_id }}"
|
||||
text: "Homelab service check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
|
||||
status_code: [200]
|
||||
register: telegram_result
|
||||
failed_when: false
|
||||
no_log: true # the bot token is part of the URL
|
||||
when:
|
||||
- alert_text | length > 0
|
||||
- telegram_configured
|
||||
|
||||
- name: Telegram delivery status
|
||||
ansible.builtin.assert:
|
||||
that: telegram_result.status == 200
|
||||
fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
|
||||
success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
|
||||
when: telegram_result is not skipped
|
||||
|
||||
Reference in New Issue
Block a user