Files
ansible/playbooks/health_check.yml
T
Harwinder Sekhon 006103760f Fix playbook bugs from review, add Telegram alerts to health and service checks
Review fixes:
- Availability plays: "ping_result is succeeded" is true for unreachable
  hosts, so offline hosts were added to reachable_hosts and failed the
  next play. Test reachability explicitly (7 playbooks).
- bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next
  to EPEL, add the root key without replacing authorized_keys, validate
  sshd_config edits and restart sshd only on change.
- backup_docker: the command module does not expand $(...); fix rsync
  filter order and add --exclude=*; restart only the containers that were
  stopped, even when rsync fails; read the storage address from the
  inventory; keep semaphore and semaphore-db running.
- new_lxc: community.general.proxmox no longer exists, use
  community.proxmox; node is mini-pc; valid hostname; password or token
  auth; let the module pick the VMID; longer timeout.
- proxmox-backup: surface the vzdump exit code, let vzdump prune with
  keep-last, drop the rename/prune steps that globbed *.zstd.
- time-region: chrony regex no longer comments out its own server line;
  /etc/localtime as a symlink; reuse facts across plays.
- update_pihole_blocklists: FQCN, become: true, changed_when.
- ansible.cfg gathering = smart; update_system only pings ubuntu/rocky.

Telegram alerts:
- health_check: filesystems above disk_warn_pct (default 80), excluding
  efivarfs, and offline hosts.
- service_check: services not running.
- One message per run from localhost, credentials from Semaphore
  (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task,
  prints the alert text when Telegram is not configured.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq
2026-09-06 23:06:18 -04:00

143 lines
5.4 KiB
YAML

---
# Disk and RAM report for every reachable host. Filesystems above
# disk_warn_pct and hosts that did not answer are printed as warnings and,
# when Telegram credentials are present, sent as one Telegram message per run.
#
# Extra vars (optional):
# disk_warn_pct - default 80
# telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN
# telegram_chat_id - or environment variable TELEGRAM_CHAT_ID
# Without both Telegram values the playbook only prints the report.
- name: Check Host Availability
hosts: all
gather_facts: false
ignore_unreachable: true
tasks:
- name: Ping hosts to check availability
ansible.builtin.ping:
register: ping_result
ignore_errors: true
ignore_unreachable: true
- name: Add reachable hosts to group
ansible.builtin.group_by:
key: reachable_hosts
when: ping_result is reachable and ping_result is succeeded
- name: Report offline hosts
ansible.builtin.debug:
msg: "{{ inventory_hostname }} - OFFLINE"
when: ping_result is unreachable or ping_result is failed
delegate_to: localhost
- name: Disk and RAM Usage Report
hosts: reachable_hosts
become: true
gather_facts: true
vars:
disk_warn_pct: 80
tasks:
- name: Get disk usage (excluding snap, overlay, tmpfs)
ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay -x efivarfs | tail -n +2
register: disk_usage
changed_when: false
- name: Calculate memory usage
ansible.builtin.set_fact:
mem_total_mb: "{{ ansible_memtotal_mb }}"
mem_used_mb: "{{ ansible_memory_mb.real.used }}"
mem_pct: "{{ ((ansible_memory_mb.real.used / ansible_memtotal_mb) * 100) | round(1) }}"
swap_total_mb: "{{ ansible_swaptotal_mb }}"
swap_used_mb: "{{ ansible_swaptotal_mb - ansible_swapfree_mb }}"
- name: Print report
ansible.builtin.debug:
msg: |
======================================================
HOST: {{ inventory_hostname }} ({{ ansible_host }})
OS: {{ ansible_distribution }} {{ ansible_distribution_version }}
======================================================
RAM: {{ mem_used_mb }}MB / {{ mem_total_mb }}MB ({{ mem_pct }}%)
SWAP: {{ swap_used_mb }}MB / {{ swap_total_mb }}MB
------------------------------------------------------
DISK:
{{ disk_usage.stdout }}
======================================================
- name: Collect filesystems above {{ disk_warn_pct }}%
ansible.builtin.set_fact:
disk_alerts: "{{ disk_alerts | default([]) + [item.split()[0] ~ ' ' ~ item.split()[-1]] }}"
loop: "{{ disk_usage.stdout_lines }}"
when: (item.split()[-1] | replace('%', '') | int) > disk_warn_pct | int
- name: Warn on high disk usage
ansible.builtin.debug:
msg: "WARNING: {{ item }} used on {{ inventory_hostname }} (threshold {{ disk_warn_pct }}%)"
loop: "{{ disk_alerts | default([]) }}"
- name: Telegram alert
hosts: localhost
connection: local
gather_facts: false
vars:
disk_warn_pct: 80
telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}"
telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}"
telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}"
offline_hosts: "{{ groups['all'] | difference(groups['reachable_hosts'] | default([])) }}"
tasks:
- name: Build alert text
ansible.builtin.set_fact:
alert_text: "{{ alert_body | default('', true) | trim }}" # an empty template renders as None
vars:
alert_body: |-
{% for h in groups['reachable_hosts'] | default([]) %}
{% for fs in hostvars[h].disk_alerts | default([]) %}
DISK {{ h }}: {{ fs }} used (threshold {{ disk_warn_pct }}%)
{% endfor %}
{% endfor %}
{% for h in offline_hosts %}
OFFLINE {{ h }} did not answer
{% endfor %}
- name: Nothing to alert
ansible.builtin.debug:
msg: "Every host reachable and all filesystems at or below {{ disk_warn_pct }}% - no alert"
when: alert_text | length == 0
- name: Print alerts when Telegram is not configured
ansible.builtin.debug:
msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}"
when:
- alert_text | length > 0
- not telegram_configured
- name: Send Telegram alert
ansible.builtin.uri:
url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
method: POST
body_format: json
body:
chat_id: "{{ telegram_chat_id }}"
text: "Homelab health check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
status_code: [200]
register: telegram_result
failed_when: false
no_log: true # the bot token is part of the URL
when:
- alert_text | length > 0
- telegram_configured
- name: Telegram delivery status
ansible.builtin.assert:
that: telegram_result.status == 200
fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
when: telegram_result is not skipped