From 006103760f3649d3becfebfd9e96fb846970b21b Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Sun, 6 Sep 2026 23:06:18 -0400 Subject: [PATCH] Fix playbook bugs from review, add Telegram alerts to health and service checks Review fixes: - Availability plays: "ping_result is succeeded" is true for unreachable hosts, so offline hosts were added to reachable_hosts and failed the next play. Test reachability explicitly (7 playbooks). - bootstrap_lxc: vim-enhanced and mtr-tiny package names, enable CRB next to EPEL, add the root key without replacing authorized_keys, validate sshd_config edits and restart sshd only on change. - backup_docker: the command module does not expand $(...); fix rsync filter order and add --exclude=*; restart only the containers that were stopped, even when rsync fails; read the storage address from the inventory; keep semaphore and semaphore-db running. - new_lxc: community.general.proxmox no longer exists, use community.proxmox; node is mini-pc; valid hostname; password or token auth; let the module pick the VMID; longer timeout. - proxmox-backup: surface the vzdump exit code, let vzdump prune with keep-last, drop the rename/prune steps that globbed *.zstd. - time-region: chrony regex no longer comments out its own server line; /etc/localtime as a symlink; reuse facts across plays. - update_pihole_blocklists: FQCN, become: true, changed_when. - ansible.cfg gathering = smart; update_system only pings ubuntu/rocky. Telegram alerts: - health_check: filesystems above disk_warn_pct (default 80), excluding efivarfs, and offline hosts. - service_check: services not running. - One message per run from localhost, credentials from Semaphore (TELEGRAM_BOT_TOKEN / TELEGRAM_CHAT_ID), no_log on the send task, prints the alert text when Telegram is not configured. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XEizHawueRXUcxQELRZ8kq --- .ansible-lint | 1 + .gitignore | 1 + .yamllint | 1 + README.md | 42 ++++++-- ansible.cfg | 2 + collections/requirements.yml | 1 + inventories/all.yml | 2 +- inventories/new-lxc.yml | 3 +- playbooks/auto_updates.yml | 4 +- playbooks/backup_docker.yml | 134 ++++++++++++++++++------- playbooks/bootstrap_lxc.yml | 112 ++++++++++----------- playbooks/health_check.yml | 94 +++++++++++++++-- playbooks/new_lxc.yml | 97 +++++++++--------- playbooks/proxmox-backup.yml | 95 +++++++++++------- playbooks/reboot_check.yml | 4 +- playbooks/service_check.yml | 71 ++++++++++++- playbooks/ssh_harden.yml | 4 +- playbooks/time-region.yml | 84 +++++++++------- playbooks/update_pihole_blocklists.yml | 12 ++- playbooks/update_system.yml | 6 +- playbooks/user_setup.yml | 4 +- 21 files changed, 515 insertions(+), 259 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index cd6a48a..f6087bb 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -3,6 +3,7 @@ profile: basic exclude_paths: - .claude/ + - .venv/ - collections/ansible_collections/ skip_list: diff --git a/.gitignore b/.gitignore index 57b7d7a..ff576ff 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ .ansible/ collections/ansible_collections/ .claude/ +.venv/ diff --git a/.yamllint b/.yamllint index 46ca2cc..eb83038 100644 --- a/.yamllint +++ b/.yamllint @@ -3,6 +3,7 @@ extends: default ignore: | .claude/ + .venv/ collections/ansible_collections/ rules: diff --git a/README.md b/README.md index f0459ee..2732a91 100644 --- a/README.md +++ b/README.md @@ -42,20 +42,20 @@ address, the first Ubuntu VM, and the service lists for `webserver`, `bot` and | Playbook | Hosts | Purpose / extra vars | |----------|-------|----------------------| | `ping.yml` | all | connectivity check | -| `health_check.yml` | all | disk and RAM report, warns above 85 % disk | +| `health_check.yml` | all | disk and RAM report; Telegram alert for filesystems above `disk_warn_pct` (default 80) and offline hosts | | `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update | -| `bootstrap_lxc.yml` | ubuntu, rocky | base packages, root SSH key, sshd basics | +| `bootstrap_lxc.yml` | ubuntu, rocky | base packages (EPEL and CRB enabled on Rocky), Ansible root key, sshd basics | | `add_root_key.yml` | all | add a personal public key to root | -| `time-region.yml` | ubuntu, rocky | timezone, NTP on VMs, NTP disabled in LXC | -| `new_lxc.yml` | localhost | create a Rocky container via the Proxmox API | -| `backup_docker.yml` | docker | rsync compose files and volumes to storage | -| `proxmox-backup.yml` | proxmox | vzdump selected guests, keep last N | +| `time-region.yml` | ubuntu, rocky | timezone, NTP on VMs, NTP disabled in LXC. `timezone_region`, `ntp_server` | +| `new_lxc.yml` | localhost | create a Rocky container via the Proxmox API. `proxmox_api_user` plus `proxmox_api_password` or `proxmox_api_token_id`/`proxmox_api_token_secret`; `lxc_hostname` | +| `backup_docker.yml` | docker | rsync compose files and volumes to storage; containers in `backup_skip_containers` (default semaphore, semaphore-db) stay up | +| `proxmox-backup.yml` | proxmox | vzdump selected guests, vzdump keeps the last `keep_last`. `guests`, `backup_mode` | | `discover_guests.yml` | proxmox | list every guest on the host and flag those missing from the inventory; `adopt=true` pushes the Ansible key into unmanaged running containers | | `update_pihole_blocklists.yml` | pihole | `pihole -up` and gravity update | | `ssh_harden.yml` | ubuntu, rocky | sshd drop-in (key-only root login) and fail2ban sshd jail. `ssh_disable_password_auth`, `ssh_max_auth_tries` | | `reboot_check.yml` | ubuntu, rocky | report reboot-required; reboot when `do_reboot=true`, skipping `reboot_exclude` (default `[docker]`) | | `auto_updates.yml` | ubuntu, rocky | unattended-upgrades / dnf-automatic for security updates, no auto reboot | -| `service_check.yml` | ubuntu, rocky | check `critical_services` per host; `restart_stopped=true` starts stopped ones | +| `service_check.yml` | ubuntu, rocky | check `critical_services` per host; `restart_stopped=true` starts stopped ones; Telegram alert for services not running | | `vm_guest_agent.yml` | vms | install and enable qemu-guest-agent | | `user_setup.yml` | ubuntu, rocky | non-root admin with sudo and SSH keys. `admin_user`, `admin_sudo_nopasswd`, `admin_ssh_keys` | @@ -76,6 +76,26 @@ as the first play of anything that touches many hosts, ASCII-only reports. - `auto_updates.yml` and `update_system.yml` overlap. Use the first for hands-off security patches, the second for full upgrades on demand. +## Telegram alerts + +`health_check.yml` and `service_check.yml` send one Telegram message per run +when there is something to report (filesystem above the threshold, host +offline, service not running). Without credentials they print the same text +in the run log instead, so nothing breaks before the bot exists. + +1. Create a bot with `@BotFather` and copy the token. +2. Send the bot a message (or add it to a group), then open + `https://api.telegram.org/bot/getUpdates` and read `chat.id`. +3. In Semaphore, create a variable group with the two values as secret + environment variables `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` (extra + vars `telegram_bot_token` / `telegram_chat_id` work too) and attach it to + the Health check and Service check templates. + +The send task runs with `no_log`, so the token never appears in the run log; +a rejected call fails the run with the Telegram error text only. To add the +alert to another playbook, copy the final `Telegram alert` play from +`health_check.yml` and change the lines the template builds. + ## Semaphore schedules One template per row. Cron times are in Semaphore's own timezone (UTC inside @@ -112,9 +132,11 @@ Monthly (drift check, also run after adding a host): On demand only: `bootstrap_lxc.yml`, `add_root_key.yml`, `time-region.yml`, `new_lxc.yml`, `user_setup.yml`, `vm_guest_agent.yml`, `ping.yml`. -Before enabling: `backup_docker.yml` stops every container on the docker host -while it copies volumes. If Semaphore itself runs as a container there, move -it or exclude it from the stop step first. +`backup_docker.yml` stops the containers on the docker host while it copies +volumes, except those in `backup_skip_containers` (default `semaphore` and +`semaphore-db`, because Semaphore runs there; their volumes are copied live). +`proxmox-backup.yml` has the same trap: never list the guest that runs +Semaphore with `backup_mode: stop`, use `snapshot` for it. ## Adding a new container diff --git a/ansible.cfg b/ansible.cfg index d4c6169..870449d 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -5,6 +5,8 @@ retry_files_enabled = False interpreter_python = auto_silent callback_result_format = yaml forks = 10 +# Gather facts once per host per run; later plays reuse them. +gathering = smart [ssh_connection] pipelining = True diff --git a/collections/requirements.yml b/collections/requirements.yml index a8dd8b4..e47f99a 100644 --- a/collections/requirements.yml +++ b/collections/requirements.yml @@ -4,4 +4,5 @@ # Unpinned on purpose until the Semaphore ansible-core version is known. collections: - name: community.general + - name: community.proxmox # the Proxmox modules left community.general in 12.0.0 - name: ansible.posix diff --git a/inventories/all.yml b/inventories/all.yml index d9a9710..384c0eb 100644 --- a/inventories/all.yml +++ b/inventories/all.yml @@ -53,7 +53,7 @@ all: ansible_host: 192.168.0.254 # PLACEHOLDER: verify lxc: hosts: - #webserver: + # webserver: docker: pihole: bot: diff --git a/inventories/new-lxc.yml b/inventories/new-lxc.yml index 72583d5..caac6a0 100644 --- a/inventories/new-lxc.yml +++ b/inventories/new-lxc.yml @@ -1,7 +1,8 @@ +--- all: children: newlxcs: hosts: new: ansible_host: 192.168.2.185 - ansible_user: root \ No newline at end of file + ansible_user: root diff --git a/playbooks/auto_updates.yml b/playbooks/auto_updates.yml index 669a29e..f57f4f0 100644 --- a/playbooks/auto_updates.yml +++ b/playbooks/auto_updates.yml @@ -18,12 +18,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "{{ inventory_hostname }} - OFFLINE" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: Auto Updates - Ubuntu diff --git a/playbooks/backup_docker.yml b/playbooks/backup_docker.yml index e59690f..436654d 100644 --- a/playbooks/backup_docker.yml +++ b/playbooks/backup_docker.yml @@ -1,62 +1,120 @@ --- -- name: Backup Docker containers and volumes to storage +# Backs up the docker host to the storage container: compose files and +# configs from compose_dir while everything runs, then the Docker volumes +# with the containers stopped so the copy is consistent. +# +# Containers in backup_skip_containers are never stopped. Semaphore runs on +# this host, so it and its database stay up (their volumes are copied live). +# root on the docker host needs an SSH key that is authorized on backup_host. +# +# Extra vars (optional): +# backup_host - inventory name of the destination (default storage) +# backup_root - directory on backup_host (default /mnt/storage/Docker_backups) +# backup_skip_containers - default [semaphore, semaphore-db] + +- name: Backup Docker compose files and volumes to storage hosts: docker become: true + gather_facts: false vars: - backup_dest: "192.168.2.115:/mnt/storage/Docker_backups" - compose_dir: "/home/harry" - volumes_dir: "/var/lib/docker/volumes" + backup_host: storage + backup_root: /mnt/storage/Docker_backups + backup_addr: "{{ hostvars[backup_host].ansible_host }}" + backup_skip_containers: + - semaphore + - semaphore-db + compose_dir: /home/harry + volumes_dir: /var/lib/docker/volumes + rsync_ssh: ssh -o StrictHostKeyChecking=accept-new tasks: - name: Ensure backup destination directories exist - ansible.builtin.command: - cmd: ssh -o StrictHostKeyChecking=no root@192.168.2.115 "mkdir -p /mnt/storage/Docker_backups/compose /mnt/storage/Docker_backups/volumes" - changed_when: false + ansible.builtin.file: + path: "{{ backup_root }}/{{ item }}" + state: directory + mode: '0755' + loop: [compose, volumes] + delegate_to: "{{ backup_host }}" + # rsync applies the first matching rule: excludes first, then the rule + # that lets it descend into directories, then the wanted files, then + # everything else is dropped. - name: Rsync compose files and configs (safe while running) - ansible.builtin.shell: | - rsync -az --delete \ - --include='*/' \ - --include='docker-compose.yml' \ - --include='compose.yml' \ - --include='.env' \ - --include='*.conf' \ - --include='*.yml' \ - --include='*.yaml' \ - --include='*.json' \ - --exclude='node_modules/' \ - --exclude='.git/' \ - --exclude='__pycache__/' \ - {{ compose_dir }}/ root@{{ backup_dest }}/compose/ - register: compose_sync + ansible.builtin.command: + argv: + - rsync + - -az + - --delete + - --prune-empty-dirs + - -e + - "{{ rsync_ssh }}" + - --exclude=node_modules/ + - --exclude=.git/ + - --exclude=__pycache__/ + - --include=*/ + - --include=docker-compose.yml + - --include=compose.yml + - --include=.env + - --include=*.conf + - --include=*.yml + - --include=*.yaml + - --include=*.json + - --exclude=* + - "{{ compose_dir }}/" + - "root@{{ backup_addr }}:{{ backup_root }}/compose/" + changed_when: true - - name: Get list of running containers - ansible.builtin.command: docker ps -q + - name: List running containers + ansible.builtin.command: + argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }}"] register: running_containers changed_when: false - - name: Stop all containers - ansible.builtin.command: docker stop $(docker ps -q) - when: running_containers.stdout | length > 0 + - name: Work out which containers to stop + ansible.builtin.set_fact: + containers_to_stop: "{{ running_containers.stdout_lines | difference(backup_skip_containers) }}" - - name: Rsync Docker volumes to storage - ansible.builtin.shell: | - rsync -az --delete \ - {{ volumes_dir }}/ root@{{ backup_dest }}/volumes/ - register: volume_sync + - name: Copy the volumes with the containers stopped + block: + - name: Stop containers + ansible.builtin.command: + argv: "{{ ['docker', 'stop'] + containers_to_stop }}" + when: containers_to_stop | length > 0 - - name: Start all containers back up - ansible.builtin.command: docker start $(docker ps -aq) + - name: Rsync Docker volumes to storage + ansible.builtin.command: + argv: + - rsync + - -az + - --delete + - -e + - "{{ rsync_ssh }}" + - "{{ volumes_dir }}/" + - "root@{{ backup_addr }}:{{ backup_root }}/volumes/" + changed_when: true + + always: + - name: Start the containers that were running before + ansible.builtin.command: + argv: "{{ ['docker', 'start'] + containers_to_stop }}" + when: containers_to_stop | length > 0 - name: Verify containers are running - ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}" + ansible.builtin.command: + argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}"] register: container_status changed_when: false - name: Backup summary ansible.builtin.debug: msg: - - "Compose sync: {{ 'OK' if compose_sync.rc == 0 else 'FAILED' }}" - - "Volume sync: {{ 'OK' if volume_sync.rc == 0 else 'FAILED' }}" - - "Containers running: {{ container_status.stdout_lines | length }}" + - "Destination: {{ backup_host }} ({{ backup_addr }}):{{ backup_root }}" + - "Stopped for the volume copy: {{ containers_to_stop | join(', ') if containers_to_stop else 'none' }}" + - "Kept running: {{ backup_skip_containers | join(', ') }}" + - "Containers running: {{ container_status.stdout_lines | length }} now, {{ running_containers.stdout_lines | length }} before" + + - name: Warn when fewer containers are running than before + ansible.builtin.debug: + msg: "WARNING: {{ item }} did not come back up on {{ inventory_hostname }}" + loop: "{{ running_containers.stdout_lines | difference(container_status.stdout_lines | map('regex_replace', ' - .*$', '')) }}" diff --git a/playbooks/bootstrap_lxc.yml b/playbooks/bootstrap_lxc.yml index a069b99..cf70dfc 100644 --- a/playbooks/bootstrap_lxc.yml +++ b/playbooks/bootstrap_lxc.yml @@ -1,4 +1,9 @@ --- +# Base packages, the Ansible root key and sshd basics for every Ubuntu/Rocky +# guest. Run it once after a container is created (and adopted, see +# discover_guests.yml). ssh_harden.yml then tightens sshd further through +# /etc/ssh/sshd_config.d/00-ansible.conf. + - name: Bootstrap - Install packages on Ubuntu hosts: ubuntu become: true @@ -20,7 +25,9 @@ - zip - git - sudo + - tmux - tree + - bash-completion - traceroute - dnsutils - ca-certificates @@ -28,36 +35,46 @@ - python3-pip - software-properties-common - rsync - - apt-transport-https - sysstat - screen - - mtr + - mtr-tiny # plain "mtr" pulls in GTK on Ubuntu - tar state: present update_cache: true + cache_valid_time: 3600 - name: Bootstrap - Install packages on Rocky Linux hosts: rocky become: true tasks: - - name: Ensure EPEL repo is enabled + # epel-release ships in the "extras" repo on Rocky/Alma/CentOS Stream. + # dnf-plugins-core provides "dnf config-manager" for the next task. + - name: Ensure EPEL release package and dnf plugins are installed ansible.builtin.dnf: - name: epel-release + name: + - epel-release + - dnf-plugins-core state: present + # EPEL packages may depend on CRB (CodeReady Builder). EPEL's install + # guide enables it together with epel-release. + - name: Enable the CRB repo + community.general.dnf_config_manager: + name: crb + state: enabled + - name: Install packages via dnf ansible.builtin.dnf: name: + # BaseOS / AppStream - tar - - vim - - htop + - vim-enhanced # "vim" is only a virtual provide on RHEL - nmap - curl - mtr - wget - iperf3 - - fail2ban - net-tools - lsof - unzip @@ -74,6 +91,9 @@ - python3-pip - rsync - sysstat + # EPEL + - htop + - fail2ban - screen state: present update_cache: true @@ -81,64 +101,38 @@ - name: Bootstrap - Configure SSH access hosts: ubuntu:rocky become: true + gather_facts: false + + vars: + bootstrap_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key" + sshd_service: "{{ 'ssh' if inventory_hostname in groups['ubuntu'] else 'sshd' }}" tasks: - - name: Ensure root's .ssh directory exists - ansible.builtin.file: - path: /root/.ssh - state: directory - owner: root - group: root - mode: '0700' + - name: Add the Ansible key to root's authorized_keys + ansible.posix.authorized_key: + user: root + key: "{{ bootstrap_ssh_key }}" + state: present - - name: Set authorized_keys for root - ansible.builtin.copy: - dest: /root/.ssh/authorized_keys - content: | - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key - owner: root - group: root - mode: '0600' - - - name: Permit root login via public key + # PasswordAuthentication stays on so a fresh container is still reachable + # by password; ssh_harden.yml turns it off once keys are in place. + - name: Allow key-based root login ansible.builtin.lineinfile: path: /etc/ssh/sshd_config - regexp: '^#?PermitRootLogin\s+' - line: 'PermitRootLogin prohibit-password' - state: present + regexp: '^#?{{ item.option }}\s' + line: "{{ item.option }} {{ item.value }}" + validate: /usr/sbin/sshd -t -f %s backup: true + loop: + - { option: PermitRootLogin, value: prohibit-password } + - { option: PubkeyAuthentication, value: "yes" } + - { option: PasswordAuthentication, value: "yes" } + loop_control: + label: "{{ item.option }} {{ item.value }}" + notify: Restart sshd - - name: Ensure PubkeyAuthentication is enabled - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?PubkeyAuthentication\s+' - line: 'PubkeyAuthentication yes' - state: present - - # NOTE: playbooks/ssh_harden.yml overrides this via /etc/ssh/sshd_config.d/00-ansible.conf - - name: Ensure PasswordAuthentication is enabled - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?PasswordAuthentication\s+' - line: 'PasswordAuthentication yes' - state: present - -- name: Bootstrap - Restart SSH on Ubuntu - hosts: ubuntu - become: true - - tasks: - - name: Restart ssh service + handlers: + - name: Restart sshd ansible.builtin.service: - name: ssh - state: restarted - -- name: Bootstrap - Restart SSH on Rocky Linux - hosts: rocky - become: true - - tasks: - - name: Restart sshd service - ansible.builtin.service: - name: sshd + name: "{{ sshd_service }}" state: restarted diff --git a/playbooks/health_check.yml b/playbooks/health_check.yml index f050921..bfa93e7 100644 --- a/playbooks/health_check.yml +++ b/playbooks/health_check.yml @@ -1,4 +1,14 @@ --- +# Disk and RAM report for every reachable host. Filesystems above +# disk_warn_pct and hosts that did not answer are printed as warnings and, +# when Telegram credentials are present, sent as one Telegram message per run. +# +# Extra vars (optional): +# disk_warn_pct - default 80 +# telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN +# telegram_chat_id - or environment variable TELEGRAM_CHAT_ID +# Without both Telegram values the playbook only prints the report. + - name: Check Host Availability hosts: all gather_facts: false @@ -14,12 +24,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "{{ inventory_hostname }} - OFFLINE" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: Disk and RAM Usage Report @@ -27,9 +37,12 @@ become: true gather_facts: true + vars: + disk_warn_pct: 80 + tasks: - name: Get disk usage (excluding snap, overlay, tmpfs) - ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay | tail -n +2 + ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay -x efivarfs | tail -n +2 register: disk_usage changed_when: false @@ -55,8 +68,75 @@ {{ disk_usage.stdout }} ====================================================== - - name: Warn on high disk usage (>85%) - ansible.builtin.debug: - msg: "WARNING: {{ item.split()[-1] }} usage on {{ item.split()[0] }} of {{ inventory_hostname }}" + - name: Collect filesystems above {{ disk_warn_pct }}% + ansible.builtin.set_fact: + disk_alerts: "{{ disk_alerts | default([]) + [item.split()[0] ~ ' ' ~ item.split()[-1]] }}" loop: "{{ disk_usage.stdout_lines }}" - when: (item.split()[-1] | replace('%', '') | int) > 85 + when: (item.split()[-1] | replace('%', '') | int) > disk_warn_pct | int + + - name: Warn on high disk usage + ansible.builtin.debug: + msg: "WARNING: {{ item }} used on {{ inventory_hostname }} (threshold {{ disk_warn_pct }}%)" + loop: "{{ disk_alerts | default([]) }}" + +- name: Telegram alert + hosts: localhost + connection: local + gather_facts: false + + vars: + disk_warn_pct: 80 + telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}" + telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}" + telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}" + offline_hosts: "{{ groups['all'] | difference(groups['reachable_hosts'] | default([])) }}" + + tasks: + - name: Build alert text + ansible.builtin.set_fact: + alert_text: "{{ alert_body | default('', true) | trim }}" # an empty template renders as None + vars: + alert_body: |- + {% for h in groups['reachable_hosts'] | default([]) %} + {% for fs in hostvars[h].disk_alerts | default([]) %} + DISK {{ h }}: {{ fs }} used (threshold {{ disk_warn_pct }}%) + {% endfor %} + {% endfor %} + {% for h in offline_hosts %} + OFFLINE {{ h }} did not answer + {% endfor %} + + - name: Nothing to alert + ansible.builtin.debug: + msg: "Every host reachable and all filesystems at or below {{ disk_warn_pct }}% - no alert" + when: alert_text | length == 0 + + - name: Print alerts when Telegram is not configured + ansible.builtin.debug: + msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}" + when: + - alert_text | length > 0 + - not telegram_configured + + - name: Send Telegram alert + ansible.builtin.uri: + url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage" + method: POST + body_format: json + body: + chat_id: "{{ telegram_chat_id }}" + text: "Homelab health check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}" + status_code: [200] + register: telegram_result + failed_when: false + no_log: true # the bot token is part of the URL + when: + - alert_text | length > 0 + - telegram_configured + + - name: Telegram delivery status + ansible.builtin.assert: + that: telegram_result.status == 200 + fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}" + success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))" + when: telegram_result is not skipped diff --git a/playbooks/new_lxc.yml b/playbooks/new_lxc.yml index 73026c2..7cc1edb 100644 --- a/playbooks/new_lxc.yml +++ b/playbooks/new_lxc.yml @@ -1,73 +1,74 @@ --- -- name: Build Dynamic Rocky Linux LXC Container +# Creates a Rocky Linux 9 container through the Proxmox API and starts it. +# Runs on the Semaphore host (localhost); proxmoxer must be importable by the +# Python that runs Ansible there. +# +# Extra vars (Semaphore environment / variable group): +# proxmox_api_user - e.g. root@pam (required) +# proxmox_api_password - that user's password (this, or a token) +# proxmox_api_token_id / proxmox_api_token_secret (API token instead of a password) +# proxmox_api_host - defaults to the pve entry in the inventory +# +# Optional: lxc_hostname, lxc_template, lxc_cores, lxc_memory, lxc_storage, +# lxc_disk_size, lxc_bridge, lxc_dns, target_node. + +- name: Build a Rocky Linux LXC container hosts: localhost + connection: local gather_facts: false vars: - # --- Container Specifications --- - lxc_hostname: "new_lxc" - lxc_template: "local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz" + proxmox_api_host: "{{ hostvars['pve'].ansible_host }}" + target_node: mini-pc + lxc_hostname: new-lxc # underscores are not valid in a hostname + lxc_template: local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz lxc_cores: 2 lxc_memory: 1024 - lxc_storage: "TB" + lxc_storage: TB lxc_disk_size: 10 - lxc_bridge: "vmbr0" - lxc_dns: "192.168.2.2" - target_node: "pve" - - # --- Security & Access --- + lxc_bridge: vmbr0 + lxc_dns: "{{ hostvars['pihole'].ansible_host }}" lxc_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key" + module_defaults: + community.proxmox.proxmox: + api_host: "{{ proxmox_api_host }}" + api_user: "{{ proxmox_api_user }}" + api_password: "{{ proxmox_api_password | default(omit) }}" + api_token_id: "{{ proxmox_api_token_id | default(omit) }}" + api_token_secret: "{{ proxmox_api_token_secret | default(omit) }}" + validate_certs: false + timeout: 300 # template extraction takes longer than the 30 s default + tasks: - # ======================================================================== - # NEW FIX: Auto-install missing dependencies into Semaphore's isolated venv - # ======================================================================== - - name: Ensure Proxmox API dependencies are installed in Semaphore - ansible.builtin.shell: "{{ ansible_playbook_python }} -m pip install proxmoxer requests" - changed_when: false - # ======================================================================== + - name: Ensure proxmoxer is available to the controller's Python + ansible.builtin.command: "{{ ansible_playbook_python }} -m pip install proxmoxer requests" + register: pip_install + changed_when: "'Successfully installed' in pip_install.stdout" - - name: Get the next available VMID from the Proxmox cluster - ansible.builtin.uri: - url: "https://{{ proxmox_api_host }}:8006/api2/json/cluster/nextid" - method: GET - headers: - Authorization: "PVEAPIToken {{ proxmox_api_user }}={{ proxmox_api_password }}" - validate_certs: false - return_content: true - register: nextid_response - - - name: Extract the dynamic VMID from the API response - ansible.builtin.set_fact: - dynamic_vmid: "{{ nextid_response.json.data }}" - - - name: Create the Rocky Linux LXC container - community.general.proxmox: - api_host: "{{ proxmox_api_host }}" - api_user: "{{ proxmox_api_user }}" - api_password: "{{ proxmox_api_password }}" - + - name: Create the container (Proxmox assigns the next free VMID) + community.proxmox.proxmox: node: "{{ target_node }}" - vmid: "{{ dynamic_vmid }}" hostname: "{{ lxc_hostname }}" ostemplate: "{{ lxc_template }}" cores: "{{ lxc_cores }}" memory: "{{ lxc_memory }}" disk: "{{ lxc_storage }}:{{ lxc_disk_size }}" - netif: '{"net0":"name=eth0,bridge={{ lxc_bridge }},ip=dhcp"}' + netif: + net0: "name=eth0,bridge={{ lxc_bridge }},ip=dhcp" nameserver: "{{ lxc_dns }}" pubkey: "{{ lxc_ssh_key }}" unprivileged: true state: present + register: lxc_create - - name: Ensure the new LXC container is powered on - community.general.proxmox: - api_host: "{{ proxmox_api_host }}" - api_user: "{{ proxmox_api_user }}" - api_password: "{{ proxmox_api_password }}" - vmid: "{{ dynamic_vmid }}" + - name: Start the container + community.proxmox.proxmox: + vmid: "{{ lxc_create.vmid }}" state: started - - - name: Output the assigned VMID to the Semaphore UI logs + + - name: Report ansible.builtin.debug: - msg: "Successfully built {{ lxc_hostname }} (Rocky Linux 9) with dynamic ID: {{ dynamic_vmid }}" \ No newline at end of file + msg: >- + Built {{ lxc_hostname }} (Rocky Linux 9) as VMID {{ lxc_create.vmid }} on {{ target_node }}. + Next: run discover_guests.yml with adopt=true, add it to inventories/all.yml, then bootstrap_lxc.yml. diff --git a/playbooks/proxmox-backup.yml b/playbooks/proxmox-backup.yml index 3e8884a..00e265d 100644 --- a/playbooks/proxmox-backup.yml +++ b/playbooks/proxmox-backup.yml @@ -1,60 +1,81 @@ --- +# vzdump of selected guests on the Proxmox host. vzdump prunes old dumps of +# each guest itself (keep-last), so nothing else touches the dump directory. +# Every run is appended to {{ backup_dir }}/proxmox_backup.log on the host. +# +# backup_mode "stop" shuts the guest down for the whole backup. Never list +# the guest that runs Semaphore with mode stop: the job would stop itself +# half way through. Use "snapshot" for that one (falls back to suspend on +# storage without snapshot support). +# +# Extra vars (optional): +# guests - list of { id, type, name } +# keep_last - dumps kept per guest (default 3) +# backup_mode - stop | snapshot | suspend (default stop) +# storage - target PVE storage id (default local) +# compression - zstd | gzip | lzo (default zstd) + - name: Backup selected Proxmox VMs and LXCs hosts: proxmox become: true + gather_facts: false + vars: - backup_dir: "/var/lib/vz/dump" - storage: "local" - compression: "zstd" - maxfiles: 3 + backup_dir: /var/lib/vz/dump + storage: local + compression: zstd + keep_last: 3 + backup_mode: stop guests: - #- { id: 104, type: "lxc", name: "jellyfin" } - - { id: 108, type: "lxc", name: "docker" } - #- { id: 101, type: "qemu", name: "pnetlab" } + # - { id: 104, type: lxc, name: courses } + - { id: 108, type: lxc, name: docker } + # - { id: 101, type: qemu, name: pnetlab } tasks: - - name: Ensure backup directory exists - file: + - name: Ensure the log directory exists + ansible.builtin.file: path: "{{ backup_dir }}" state: directory mode: '0755' - - name: Run vzdump backup with logging - shell: | + - name: Run vzdump + ansible.builtin.shell: | + set -o pipefail LOGFILE="{{ backup_dir }}/proxmox_backup.log" START=$(date +%s) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup of {{ item.name }} ({{ item.type }}) [ID: {{ item.id }}]" >> "$LOGFILE" - - OUTPUT=$(vzdump {{ item.id }} --mode stop --compress {{ compression }} --storage {{ storage }} --remove 0 2>&1) + echo "[$(date '+%Y-%m-%d %H:%M:%S')] START {{ item.name }} ({{ item.type }} {{ item.id }}) mode={{ backup_mode }}" >> "$LOGFILE" + OUTPUT=$(vzdump {{ item.id }} --mode {{ backup_mode }} --compress {{ compression }} \ + --storage {{ storage }} --prune-backups keep-last={{ keep_last }} 2>&1) STATUS=$? - - END=$(date +%s) - DURATION=$((END - START)) - - if [ $STATUS -eq 0 ]; then - echo "[$(date '+%Y-%m-%d %H:%M:%S')] ✅ SUCCESS - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE" + DURATION=$(( $(date +%s) - START )) + if [ "$STATUS" -eq 0 ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] SUCCESS {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE" + echo "$OUTPUT" | grep -E 'archive file size|pruned|removing backup' || true else - echo "[$(date '+%Y-%m-%d %H:%M:%S')] ❌ FAILURE - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE" - echo "---- ERROR OUTPUT ----" >> "$LOGFILE" - echo "$OUTPUT" >> "$LOGFILE" - echo "----------------------" >> "$LOGFILE" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAILURE {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE" + printf -- '---- vzdump output ----\n%s\n-----------------------\n' "$OUTPUT" >> "$LOGFILE" + echo "$OUTPUT" | tail -n 20 fi + exit "$STATUS" args: executable: /bin/bash loop: "{{ guests }}" + loop_control: + label: "{{ item.name }} ({{ item.type }} {{ item.id }})" + changed_when: true - - name: Rename backup files to include guest name - shell: | - latest_file=$(ls -t {{ backup_dir }}/vzdump-{{ item.type }}-{{ item.id }}-*.{{ compression }} | head -n 1) - new_file="{{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-{{ '%Y%m%d-%H%M%S' | strftime(ansible_date_time.epoch | int) }}.{{ compression }}" - cp "$latest_file" "$new_file" - args: - executable: /bin/bash + - name: List the dumps kept for each guest + ansible.builtin.command: pvesm list {{ storage }} --content backup --vmid {{ item.id }} loop: "{{ guests }}" + loop_control: + label: "{{ item.name }} ({{ item.id }})" + register: dump_lists + changed_when: false - - name: Prune old backups (keep last {{ maxfiles }}) - shell: | - ls -tp {{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-*{{ compression }} | grep -v '/$' | tail -n +{{ maxfiles + 1 }} | xargs -r rm -- - args: - executable: /bin/bash - loop: "{{ guests }}" + - name: Backup summary + ansible.builtin.debug: + msg: | + {% for r in dump_lists.results %} + {{ r.item.name }} ({{ r.item.type }} {{ r.item.id }}) - mode {{ backup_mode }}, keeping the last {{ keep_last }}: + {{ r.stdout }} + {% endfor %} diff --git a/playbooks/reboot_check.yml b/playbooks/reboot_check.yml index 3052a8c..543e60a 100644 --- a/playbooks/reboot_check.yml +++ b/playbooks/reboot_check.yml @@ -22,12 +22,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "{{ inventory_hostname }} - OFFLINE" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: Reboot Check - Ubuntu diff --git a/playbooks/service_check.yml b/playbooks/service_check.yml index 6d17533..8fdb302 100644 --- a/playbooks/service_check.yml +++ b/playbooks/service_check.yml @@ -1,9 +1,13 @@ --- # Verifies the critical services listed per host in the inventory -# (critical_services) and optionally starts any that are stopped. +# (critical_services) and optionally starts any that are stopped. Services +# found not running are sent as one Telegram message per run when Telegram +# credentials are present. # # Extra vars (optional): -# restart_stopped - default false; when true, start services found stopped +# restart_stopped - default false; when true, start services found stopped +# telegram_bot_token - or environment variable TELEGRAM_BOT_TOKEN +# telegram_chat_id - or environment variable TELEGRAM_CHAT_ID - name: Check Host Availability hosts: ubuntu:rocky @@ -20,12 +24,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "{{ inventory_hostname }} - OFFLINE" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: Service Check @@ -80,3 +84,62 @@ when: - restart_stopped | bool - item.state not in ['running', 'not-found'] + +- name: Telegram alert + hosts: localhost + connection: local + gather_facts: false + + vars: + restart_stopped: false + telegram_bot_token: "{{ lookup('env', 'TELEGRAM_BOT_TOKEN') }}" + telegram_chat_id: "{{ lookup('env', 'TELEGRAM_CHAT_ID') }}" + telegram_configured: "{{ telegram_bot_token | length > 0 and telegram_chat_id | length > 0 }}" + + tasks: + - name: Build alert text + ansible.builtin.set_fact: + alert_text: "{{ alert_body | default('', true) | trim }}" # an empty template renders as None + vars: + alert_body: |- + {% for h in groups['reachable_hosts'] | default([]) %} + {% for svc in hostvars[h].service_states | default([]) if svc.state != 'running' %} + {% set note = ' - start attempted' if (restart_stopped | bool and svc.state != 'not-found') else '' %} + SERVICE {{ h }}: {{ svc.name }} {{ 'not installed' if svc.state == 'not-found' else svc.state }}{{ note }} + {% endfor %} + {% endfor %} + + - name: Nothing to alert + ansible.builtin.debug: + msg: "All critical services running - no alert" + when: alert_text | length == 0 + + - name: Print alerts when Telegram is not configured + ansible.builtin.debug: + msg: "ALERTS (Telegram not configured - set telegram_bot_token and telegram_chat_id):\n{{ alert_text }}" + when: + - alert_text | length > 0 + - not telegram_configured + + - name: Send Telegram alert + ansible.builtin.uri: + url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage" + method: POST + body_format: json + body: + chat_id: "{{ telegram_chat_id }}" + text: "Homelab service check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}" + status_code: [200] + register: telegram_result + failed_when: false + no_log: true # the bot token is part of the URL + when: + - alert_text | length > 0 + - telegram_configured + + - name: Telegram delivery status + ansible.builtin.assert: + that: telegram_result.status == 200 + fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}" + success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))" + when: telegram_result is not skipped diff --git a/playbooks/ssh_harden.yml b/playbooks/ssh_harden.yml index 5912447..2dac854 100644 --- a/playbooks/ssh_harden.yml +++ b/playbooks/ssh_harden.yml @@ -26,12 +26,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "{{ inventory_hostname }} - OFFLINE" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: SSH Hardening - Install fail2ban on Ubuntu diff --git a/playbooks/time-region.yml b/playbooks/time-region.yml index 4565ed1..88f7116 100644 --- a/playbooks/time-region.yml +++ b/playbooks/time-region.yml @@ -1,30 +1,46 @@ --- -- name: Configure timezone on all hosts +# Sets the timezone on every Ubuntu/Rocky guest. LXC containers share the +# host's clock, so NTP is disabled inside them; VMs sync against ntp_server. +# +# Extra vars (optional): +# timezone_region - default America/New_York +# ntp_server - default time.google.com + +- name: Configure timezone hosts: ubuntu:rocky become: true - gather_facts: true vars: - timezone_region: "America/New_York" + timezone_region: America/New_York tasks: - - name: Copy timezone file - ansible.builtin.copy: + - name: Link /etc/localtime to the zoneinfo file + ansible.builtin.file: src: "/usr/share/zoneinfo/{{ timezone_region }}" dest: /etc/localtime - remote_src: true + state: link + force: true - name: Write /etc/timezone ansible.builtin.copy: dest: /etc/timezone content: "{{ timezone_region }}\n" + owner: root + group: root + mode: '0644' -- name: Disable NTP on Ubuntu LXC containers +# The plays below reuse the facts gathered above instead of collecting them again. + +- name: NTP on Ubuntu hosts: ubuntu become: true + gather_facts: false + + vars: + ntp_server: time.google.com tasks: - - name: Stop and mask timesyncd on LXC + - name: Stop and mask timesyncd in LXC containers ansible.builtin.systemd: name: systemd-timesyncd state: stopped @@ -32,33 +48,11 @@ masked: true when: ansible_virtualization_type == "lxc" -- name: Disable NTP on Rocky Linux LXC containers - hosts: rocky - become: true - - tasks: - - name: Stop and mask chronyd on LXC - ansible.builtin.systemd: - name: chronyd - state: stopped - enabled: false - masked: true - when: ansible_virtualization_type == "lxc" - -- name: Configure NTP on Ubuntu VMs - hosts: ubuntu - become: true - - vars: - ntp_server: "time.google.com" - - tasks: - - name: Configure timesyncd + - name: Point timesyncd at ntp_server on VMs ansible.builtin.lineinfile: path: /etc/systemd/timesyncd.conf regexp: '^#?NTP=' line: "NTP={{ ntp_server }}" - state: present when: ansible_virtualization_type != "lxc" notify: Restart timesyncd @@ -69,27 +63,38 @@ state: restarted enabled: true -- name: Configure NTP on Rocky Linux VMs +- name: NTP on Rocky Linux hosts: rocky become: true + gather_facts: false vars: - ntp_server: "time.google.com" + ntp_server: time.google.com tasks: - - name: Disable default chrony pools + - name: Stop and mask chronyd in LXC containers + ansible.builtin.systemd: + name: chronyd + state: stopped + enabled: false + masked: true + when: ansible_virtualization_type == "lxc" + + # The lookahead keeps the server line added below from being commented + # out again on the next run. + - name: Disable the default chrony pools on VMs ansible.builtin.replace: path: /etc/chrony.conf - regexp: '^(pool|server) .*' - replace: '# \1 removed by Ansible' + regexp: '^(pool|server)\s+(?!{{ ntp_server | regex_escape }}(\s|$)).*' + replace: '# \g<0> (disabled by Ansible)' when: ansible_virtualization_type != "lxc" + notify: Restart chronyd - - name: Configure chrony NTP server + - name: Point chrony at ntp_server on VMs ansible.builtin.lineinfile: path: /etc/chrony.conf + regexp: '^server\s+{{ ntp_server | regex_escape }}(\s|$)' line: "server {{ ntp_server }} iburst" - insertafter: EOF - state: present when: ansible_virtualization_type != "lxc" notify: Restart chronyd @@ -103,6 +108,7 @@ - name: Verify time configuration hosts: ubuntu:rocky become: true + gather_facts: false tasks: - name: Check current system time diff --git a/playbooks/update_pihole_blocklists.yml b/playbooks/update_pihole_blocklists.yml index f858470..4830e44 100644 --- a/playbooks/update_pihole_blocklists.yml +++ b/playbooks/update_pihole_blocklists.yml @@ -1,18 +1,22 @@ --- - name: Update Pi-hole core and blocklists hosts: pihole - become: yes + become: true + gather_facts: false + tasks: - name: Update Pi-hole core components - command: pihole -up + ansible.builtin.command: pihole -up register: update_out + changed_when: "'Everything is up to date' not in update_out.stdout" - name: Update Pi-hole blocklists - command: pihole -g + ansible.builtin.command: pihole -g register: gravity_out + changed_when: true - name: Display update summary - debug: + ansible.builtin.debug: msg: - "Core update: {{ update_out.stdout }}" - "Gravity update: {{ gravity_out.stdout }}" diff --git a/playbooks/update_system.yml b/playbooks/update_system.yml index cecd4a9..f278d64 100644 --- a/playbooks/update_system.yml +++ b/playbooks/update_system.yml @@ -1,6 +1,6 @@ --- - name: Check Host Availability - hosts: all + hosts: ubuntu:rocky gather_facts: false ignore_unreachable: true @@ -14,12 +14,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "Host {{ inventory_hostname }} is OFFLINE and will be skipped" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: Update and Upgrade - Ubuntu diff --git a/playbooks/user_setup.yml b/playbooks/user_setup.yml index 156015f..a729bd3 100644 --- a/playbooks/user_setup.yml +++ b/playbooks/user_setup.yml @@ -24,12 +24,12 @@ - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts - when: ping_result is succeeded + when: ping_result is reachable and ping_result is succeeded - name: Report offline hosts ansible.builtin.debug: msg: "{{ inventory_hostname }} - OFFLINE" - when: ping_result is not succeeded + when: ping_result is unreachable or ping_result is failed delegate_to: localhost - name: User Setup - Create admin user on Ubuntu