discover_guests: treat any 192.168.x address as LAN, the lab now spans several subnets

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JEczyvRjfakkiDNkbZA7PR

.ansible-lint

@@ -3,7 +3,6 @@ profile: basic
 
 exclude_paths:
   - .claude/
-  - .venv/
   - collections/ansible_collections/
 
 skip_list:

.gitignore

@@ -5,4 +5,3 @@ __pycache__/
 .ansible/
 collections/ansible_collections/
 .claude/
-.venv/

.yamllint

@@ -3,7 +3,6 @@ extends: default
 
 ignore: |
   .claude/
-  .venv/
   collections/ansible_collections/
 
 rules:

README.md

@@ -42,20 +42,20 @@ address, the first Ubuntu VM, and the service lists for `webserver`, `bot` and
 | Playbook | Hosts | Purpose / extra vars |
 |----------|-------|----------------------|
 | `ping.yml` | all | connectivity check |
-| `health_check.yml` | all | disk and RAM report; Telegram alert for filesystems above `disk_warn_pct` (default 80) and offline hosts |
+| `health_check.yml` | all | disk and RAM report, warns above 85 % disk |
-| `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update; reports per host whether it was updated (with the package list) or already up to date, plus a final summary. Supports `--check` as a dry run |
+| `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update |
-| `bootstrap_lxc.yml` | ubuntu, rocky | base packages (EPEL and CRB enabled on Rocky), Ansible root key, sshd basics |
+| `bootstrap_lxc.yml` | ubuntu, rocky | base packages, root SSH key, sshd basics |
 | `add_root_key.yml` | all | add a personal public key to root |
-| `time-region.yml` | ubuntu, rocky | timezone, NTP on VMs, NTP disabled in LXC. `timezone_region`, `ntp_server` |
+| `time-region.yml` | ubuntu, rocky | timezone, NTP on VMs, NTP disabled in LXC |
-| `new_lxc.yml` | localhost | create a Rocky container via the Proxmox API. `proxmox_api_user` plus `proxmox_api_password` or `proxmox_api_token_id`/`proxmox_api_token_secret`; `lxc_hostname` |
+| `new_lxc.yml` | localhost | create a Rocky container via the Proxmox API |
-| `backup_docker.yml` | docker | rsync compose files and volumes to storage; containers in `backup_skip_containers` (default semaphore, semaphore-db) stay up |
+| `backup_docker.yml` | docker | rsync compose files and volumes to storage |
-| `proxmox-backup.yml` | proxmox | vzdump selected guests, vzdump keeps the last `keep_last`. `guests`, `backup_mode` |
+| `proxmox-backup.yml` | proxmox | vzdump selected guests, keep last N |
 | `discover_guests.yml` | proxmox | list every guest on the host and flag those missing from the inventory; `adopt=true` pushes the Ansible key into unmanaged running containers |
 | `update_pihole_blocklists.yml` | pihole | `pihole -up` and gravity update |
 | `ssh_harden.yml` | ubuntu, rocky | sshd drop-in (key-only root login) and fail2ban sshd jail. `ssh_disable_password_auth`, `ssh_max_auth_tries` |
 | `reboot_check.yml` | ubuntu, rocky | report reboot-required; reboot when `do_reboot=true`, skipping `reboot_exclude` (default `[docker]`) |
 | `auto_updates.yml` | ubuntu, rocky | unattended-upgrades / dnf-automatic for security updates, no auto reboot |
-| `service_check.yml` | ubuntu, rocky | check `critical_services` per host; `restart_stopped=true` starts stopped ones; Telegram alert for services not running |
+| `service_check.yml` | ubuntu, rocky | check `critical_services` per host; `restart_stopped=true` starts stopped ones |
 | `vm_guest_agent.yml` | vms | install and enable qemu-guest-agent |
 | `user_setup.yml` | ubuntu, rocky | non-root admin with sudo and SSH keys. `admin_user`, `admin_sudo_nopasswd`, `admin_ssh_keys` |
 
@@ -76,26 +76,6 @@ as the first play of anything that touches many hosts, ASCII-only reports.
 - `auto_updates.yml` and `update_system.yml` overlap. Use the first for
   hands-off security patches, the second for full upgrades on demand.
 
-## Telegram alerts
-
-`health_check.yml` and `service_check.yml` send one Telegram message per run
-when there is something to report (filesystem above the threshold, host
-offline, service not running). Without credentials they print the same text
-in the run log instead, so nothing breaks before the bot exists.
-
-1. Create a bot with `@BotFather` and copy the token.
-2. Send the bot a message (or add it to a group), then open
-   `https://api.telegram.org/bot<TOKEN>/getUpdates` and read `chat.id`.
-3. In Semaphore, create a variable group with the two values as secret
-   environment variables `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` (extra
-   vars `telegram_bot_token` / `telegram_chat_id` work too) and attach it to
-   the Health check and Service check templates.
-
-The send task runs with `no_log`, so the token never appears in the run log;
-a rejected call fails the run with the Telegram error text only. To add the
-alert to another playbook, copy the final `Telegram alert` play from
-`health_check.yml` and change the lines the template builds.
-
 ## Semaphore schedules
 
 One template per row. Cron times are in Semaphore's own timezone (UTC inside
@@ -132,11 +112,9 @@ Monthly (drift check, also run after adding a host):
 On demand only: `bootstrap_lxc.yml`, `add_root_key.yml`, `time-region.yml`,
 `new_lxc.yml`, `user_setup.yml`, `vm_guest_agent.yml`, `ping.yml`.
 
-`backup_docker.yml` stops the containers on the docker host while it copies
+Before enabling: `backup_docker.yml` stops every container on the docker host
-volumes, except those in `backup_skip_containers` (default `semaphore` and
+while it copies volumes. If Semaphore itself runs as a container there, move
-`semaphore-db`, because Semaphore runs there; their volumes are copied live).
+it or exclude it from the stop step first.
-`proxmox-backup.yml` has the same trap: never list the guest that runs
-Semaphore with `backup_mode: stop`, use `snapshot` for it.
 
 ## Adding a new container
 

ansible.cfg

@@ -5,8 +5,6 @@ retry_files_enabled = False
 interpreter_python = auto_silent
 callback_result_format = yaml
 forks = 10
-# Gather facts once per host per run; later plays reuse them.
-gathering = smart
 
 [ssh_connection]
 pipelining = True

collections/requirements.yml

@@ -4,5 +4,4 @@
 # Unpinned on purpose until the Semaphore ansible-core version is known.
 collections:
   - name: community.general
-  - name: community.proxmox   # the Proxmox modules left community.general in 12.0.0
   - name: ansible.posix

inventories/all.yml

@@ -18,12 +18,12 @@ all:
   children:
     ubuntu:
       hosts:
-        # webserver:
+        webserver:
-        #   ansible_host: 192.168.55.10
+          ansible_host: 192.168.55.10
-        #   critical_services:            # PLACEHOLDER: verify service names
+          critical_services:            # PLACEHOLDER: verify service names
-        #     - nginx
+            - nginx
         docker:
-          ansible_host: 192.168.0.11
+          ansible_host: 192.168.55.11
           critical_services:
             - docker
             - containerd
@@ -32,28 +32,23 @@ all:
         #   critical_services: []
     rocky:
       hosts:
-        courses-lxc:
-          ansible_host: 192.168.0.31
-          critical_services:
-            - httpd
-            - mariadb
         pihole:
-          ansible_host: 192.168.0.2
+          ansible_host: 192.168.2.2
           critical_services:
             - pihole-FTL
         bot:
-          ansible_host: 192.168.0.174
+          ansible_host: 192.168.2.174
-          critical_services: [myapp]         # PLACEHOLDER: add the bot's service name
+          critical_services: []         # PLACEHOLDER: add the bot's service name
         storage:
-          ansible_host: 192.168.0.115
+          ansible_host: 192.168.2.115
-          critical_services: [smb]         # PLACEHOLDER: e.g. nfs-server, smb
+          critical_services: []         # PLACEHOLDER: e.g. nfs-server, smb
     proxmox:
       hosts:
         pve:
-          ansible_host: 192.168.0.254   # PLACEHOLDER: verify
+          ansible_host: 192.168.2.254   # PLACEHOLDER: verify
     lxc:
       hosts:
-        # webserver:
+        webserver:
         docker:
         pihole:
         bot:

inventories/new-lxc.yml

@@ -1,8 +1,7 @@
----
 all:
   children:
     newlxcs:
       hosts:
         new:
           ansible_host: 192.168.2.185
-          ansible_user: root
+          ansible_user: root

playbooks/auto_updates.yml

@@ -3,46 +3,35 @@
 # dnf-automatic on Rocky Linux. Neither reboots automatically; use
 # reboot_check.yml for that.
 
-# Play 1: probe every ubuntu/rocky host and sort them into a "reachable_hosts"
-# group so the two update plays below can skip anything that's offline.
 - name: Check Host Availability
   hosts: ubuntu:rocky
   gather_facts: false
   ignore_unreachable: true
 
   tasks:
-    # Ping each host; failures/unreachables are swallowed so the play keeps
-    # going instead of aborting the whole run.
     - name: Ping hosts to check availability
       ansible.builtin.ping:
       register: ping_result
       ignore_errors: true
       ignore_unreachable: true
 
-    # Dynamically creates/populates the "reachable_hosts" group, used by the
-    # Ubuntu and Rocky plays' host patterns (ubuntu:&reachable_hosts etc.).
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
-    # Print a message for any host that failed the ping, so offline hosts
-    # are visible in the run output.
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
-# Play 2: install and configure unattended-upgrades on Ubuntu hosts, then
-# report its status. Does not enable automatic reboots.
 - name: Auto Updates - Ubuntu
   hosts: ubuntu:&reachable_hosts
   become: true
   gather_facts: false
 
   tasks:
-    # Make sure the unattended-upgrades package is present.
     - name: Install unattended-upgrades
       ansible.builtin.apt:
         name: unattended-upgrades
@@ -50,8 +39,6 @@
         update_cache: true
         cache_valid_time: 3600
 
-    # Turn on apt's periodic package-list refresh, unattended-upgrade run,
-    # and weekly autoclean.
     - name: Enable periodic package list refresh and upgrades
       ansible.builtin.copy:
         dest: /etc/apt/apt.conf.d/20auto-upgrades
@@ -64,8 +51,6 @@
         group: root
         mode: '0644'
 
-    # Local overrides layered on top of the stock 50unattended-upgrades
-    # config: clean up unused deps, but never reboot automatically.
     - name: Configure unattended-upgrades behaviour
       ansible.builtin.copy:
         dest: /etc/apt/apt.conf.d/52unattended-upgrades-local
@@ -78,8 +63,6 @@
         group: root
         mode: '0644'
 
-    # Enable and start the systemd timers that actually trigger apt's
-    # periodic update/upgrade cycle.
     - name: Enable apt timers
       ansible.builtin.systemd:
         name: "{{ item }}"
@@ -89,32 +72,17 @@
         - apt-daily.timer
         - apt-daily-upgrade.timer
 
-    # Simulate an unattended-upgrade run (no changes made) so the report
-    # below shows what would actually be upgraded.
     - name: Dry-run unattended-upgrade
       ansible.builtin.command: unattended-upgrade --dry-run -v
       register: uu_dry_run
       changed_when: false
       failed_when: false
 
-    # Pull the package list out of the dry-run's "Packages that will be
-    # upgraded: ..." line, used for the per-host and final summary reports.
-    - name: Extract pending package list
-      ansible.builtin.set_fact:
-        pending_update_packages: "{{ (uu_dry_run.stdout | regex_findall('Packages that will be upgraded:\\s*(.*)') | first | default('')).split() }}"
-
-    - name: Compute pending update count
-      ansible.builtin.set_fact:
-        pending_update_count: "{{ pending_update_packages | length }}"
-
-    # Grab the current state of the apt-daily timers for the report.
     - name: List apt timers  # noqa: command-instead-of-module (read-only listing)
       ansible.builtin.command: systemctl list-timers apt-daily* --no-pager --all
       register: apt_timers
       changed_when: false
 
-    # Summarize timer status, pending update count, and the tail of the
-    # dry-run output per host.
     - name: Print report
       ansible.builtin.debug:
         msg: |
@@ -123,30 +91,22 @@
           ======================================================
            TIMERS:
           {{ apt_timers.stdout }}
-          ------------------------------------------------------
-           PENDING UPDATES: {{ pending_update_count }}
-          {{ pending_update_packages | join(', ') }}
           ------------------------------------------------------
            DRY RUN (last lines):
           {{ (uu_dry_run.stdout_lines + uu_dry_run.stderr_lines)[-6:] | join('\n') }}
           ======================================================
 
-# Play 3: install and configure dnf-automatic on Rocky Linux hosts (security
-# updates only), then report its status.
 - name: Auto Updates - Rocky Linux
   hosts: rocky:&reachable_hosts
   become: true
   gather_facts: false
 
   tasks:
-    # Make sure the dnf-automatic package is present.
     - name: Install dnf-automatic
       ansible.builtin.dnf:
         name: dnf-automatic
         state: present
 
-    # Set dnf-automatic to download and apply security updates only, and
-    # log output to stdout (captured by the systemd journal).
     - name: Configure dnf-automatic for security updates
       community.general.ini_file:
         path: /etc/dnf/automatic.conf
@@ -162,38 +122,17 @@
       loop_control:
         label: "{{ item.section }}/{{ item.option }} = {{ item.value }}"
 
-    # Enable and start the systemd timer that triggers dnf-automatic runs.
     - name: Enable dnf-automatic timer
       ansible.builtin.systemd:
         name: dnf-automatic.timer
         enabled: true
         state: started
 
-    # Grab the current state of the dnf-automatic timer for the report.
     - name: List dnf-automatic timers  # noqa: command-instead-of-module (read-only listing)
       ansible.builtin.command: systemctl list-timers dnf-automatic* --no-pager --all
       register: dnf_timers
       changed_when: false
 
-    # Check what security updates are pending. rc 100 means updates are
-    # available, rc 0 means none, so only other codes count as a failure.
-    - name: Check for pending security updates
-      ansible.builtin.command: dnf check-update --security
-      register: dnf_check
-      changed_when: false
-      failed_when: dnf_check.rc not in [0, 100]
-
-    # Pull the "name.arch  version  repo" lines out of the check-update
-    # output, used for the per-host and final summary reports.
-    - name: Extract pending package list
-      ansible.builtin.set_fact:
-        pending_update_packages: "{{ dnf_check.stdout_lines | select('match', '^\\S+\\.\\S+\\s+\\S+\\s+\\S+$') | map('regex_replace', '^(\\S+)\\s.*$', '\\1') | list }}"
-
-    - name: Compute pending update count
-      ansible.builtin.set_fact:
-        pending_update_count: "{{ pending_update_packages | length }}"
-
-    # Summarize timer status and pending update count per host.
     - name: Print report
       ansible.builtin.debug:
         msg: |
@@ -202,37 +141,4 @@
           ======================================================
            TIMERS:
           {{ dnf_timers.stdout }}
-          ------------------------------------------------------
-           PENDING UPDATES: {{ pending_update_count }}
-          {{ pending_update_packages | join(', ') }}
           ======================================================
-
-# Play 4: after both OS-specific plays have run, print one consolidated
-# summary of which hosts have pending updates and how many packages.
-- name: Summary - Pending Updates
-  hosts: localhost
-  gather_facts: false
-
-  tasks:
-    - name: Print hosts with pending updates
-      ansible.builtin.debug:
-        msg: "{{ item }} - {{ hostvars[item].pending_update_count }} package(s) pending: {{ hostvars[item].pending_update_packages | join(', ') }}"
-      loop: "{{ groups['reachable_hosts'] | default([]) }}"
-      when: hostvars[item].pending_update_count | default(0) | int > 0
-
-    - name: Print hosts fully up to date
-      ansible.builtin.debug:
-        msg: "{{ item }} - up to date"
-      loop: "{{ groups['reachable_hosts'] | default([]) }}"
-      when: hostvars[item].pending_update_count is defined and hostvars[item].pending_update_count | int == 0
-
-    - name: Print totals
-      ansible.builtin.debug:
-        msg: >-
-          {{ groups['reachable_hosts'] | default([]) | length }} host(s) reachable,
-          {{ groups['reachable_hosts'] | default([])
-             | map('extract', hostvars, 'pending_update_count') | select('defined') | map('int') | select('>', 0) | list | length }}
-          with pending updates,
-          {{ groups['reachable_hosts'] | default([])
-             | map('extract', hostvars, 'pending_update_count') | select('defined') | map('int') | sum }}
-          package(s) total pending.

playbooks/backup_docker.yml

@@ -1,120 +1,62 @@
 ---
-# Backs up the docker host to the storage container: compose files and
+- name: Backup Docker containers and volumes to storage
-# configs from compose_dir while everything runs, then the Docker volumes
-# with the containers stopped so the copy is consistent.
-#
-# Containers in backup_skip_containers are never stopped. Semaphore runs on
-# this host, so it and its database stay up (their volumes are copied live).
-# root on the docker host needs an SSH key that is authorized on backup_host.
-#
-# Extra vars (optional):
-#   backup_host            - inventory name of the destination (default storage)
-#   backup_root            - directory on backup_host (default /mnt/storage/Docker_backups)
-#   backup_skip_containers - default [semaphore, semaphore-db]
-
-- name: Backup Docker compose files and volumes to storage
   hosts: docker
   become: true
-  gather_facts: false
 
   vars:
-    backup_host: storage
+    backup_dest: "192.168.2.115:/mnt/storage/Docker_backups"
-    backup_root: /mnt/storage/Docker_backups
+    compose_dir: "/home/harry"
-    backup_addr: "{{ hostvars[backup_host].ansible_host }}"
+    volumes_dir: "/var/lib/docker/volumes"
-    backup_skip_containers:
-      - semaphore
-      - semaphore-db
-    compose_dir: /home/harry
-    volumes_dir: /var/lib/docker/volumes
-    rsync_ssh: ssh -o StrictHostKeyChecking=accept-new
 
   tasks:
     - name: Ensure backup destination directories exist
-      ansible.builtin.file:
+      ansible.builtin.command:
-        path: "{{ backup_root }}/{{ item }}"
+        cmd: ssh -o StrictHostKeyChecking=no root@192.168.2.115 "mkdir -p /mnt/storage/Docker_backups/compose /mnt/storage/Docker_backups/volumes"
-        state: directory
+      changed_when: false
-        mode: '0755'
-      loop: [compose, volumes]
-      delegate_to: "{{ backup_host }}"
 
-    # rsync applies the first matching rule: excludes first, then the rule
-    # that lets it descend into directories, then the wanted files, then
-    # everything else is dropped.
     - name: Rsync compose files and configs (safe while running)
-      ansible.builtin.command:
+      ansible.builtin.shell: |
-        argv:
+        rsync -az --delete \
-          - rsync
+          --include='*/' \
-          - -az
+          --include='docker-compose.yml' \
-          - --delete
+          --include='compose.yml' \
-          - --prune-empty-dirs
+          --include='.env' \
-          - -e
+          --include='*.conf' \
-          - "{{ rsync_ssh }}"
+          --include='*.yml' \
-          - --exclude=node_modules/
+          --include='*.yaml' \
-          - --exclude=.git/
+          --include='*.json' \
-          - --exclude=__pycache__/
+          --exclude='node_modules/' \
-          - --include=*/
+          --exclude='.git/' \
-          - --include=docker-compose.yml
+          --exclude='__pycache__/' \
-          - --include=compose.yml
+          {{ compose_dir }}/ root@{{ backup_dest }}/compose/
-          - --include=.env
+      register: compose_sync
-          - --include=*.conf
-          - --include=*.yml
-          - --include=*.yaml
-          - --include=*.json
-          - --exclude=*
-          - "{{ compose_dir }}/"
-          - "root@{{ backup_addr }}:{{ backup_root }}/compose/"
-      changed_when: true
 
-    - name: List running containers
+    - name: Get list of running containers
-      ansible.builtin.command:
+      ansible.builtin.command: docker ps -q
-        argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }}"]
       register: running_containers
       changed_when: false
 
-    - name: Work out which containers to stop
+    - name: Stop all containers
-      ansible.builtin.set_fact:
+      ansible.builtin.command: docker stop $(docker ps -q)
-        containers_to_stop: "{{ running_containers.stdout_lines | difference(backup_skip_containers) }}"
+      when: running_containers.stdout | length > 0
 
-    - name: Copy the volumes with the containers stopped
+    - name: Rsync Docker volumes to storage
-      block:
+      ansible.builtin.shell: |
-        - name: Stop containers
+        rsync -az --delete \
-          ansible.builtin.command:
+          {{ volumes_dir }}/ root@{{ backup_dest }}/volumes/
-            argv: "{{ ['docker', 'stop'] + containers_to_stop }}"
+      register: volume_sync
-          when: containers_to_stop | length > 0
 
-        - name: Rsync Docker volumes to storage
+    - name: Start all containers back up
-          ansible.builtin.command:
+      ansible.builtin.command: docker start $(docker ps -aq)
-            argv:
-              - rsync
-              - -az
-              - --delete
-              - -e
-              - "{{ rsync_ssh }}"
-              - "{{ volumes_dir }}/"
-              - "root@{{ backup_addr }}:{{ backup_root }}/volumes/"
-          changed_when: true
-
-      always:
-        - name: Start the containers that were running before
-          ansible.builtin.command:
-            argv: "{{ ['docker', 'start'] + containers_to_stop }}"
-          when: containers_to_stop | length > 0
 
     - name: Verify containers are running
-      ansible.builtin.command:
+      ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}"
-        argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}"]
       register: container_status
       changed_when: false
 
     - name: Backup summary
       ansible.builtin.debug:
         msg:
-          - "Destination: {{ backup_host }} ({{ backup_addr }}):{{ backup_root }}"
+          - "Compose sync: {{ 'OK' if compose_sync.rc == 0 else 'FAILED' }}"
-          - "Stopped for the volume copy: {{ containers_to_stop | join(', ') if containers_to_stop else 'none' }}"
+          - "Volume sync: {{ 'OK' if volume_sync.rc == 0 else 'FAILED' }}"
-          - "Kept running: {{ backup_skip_containers | join(', ') }}"
+          - "Containers running: {{ container_status.stdout_lines | length }}"
-          - "Containers running: {{ container_status.stdout_lines | length }} now, {{ running_containers.stdout_lines | length }} before"
-
-    - name: Warn when fewer containers are running than before
-      ansible.builtin.debug:
-        msg: "WARNING: {{ item }} did not come back up on {{ inventory_hostname }}"
-      loop: "{{ running_containers.stdout_lines | difference(container_status.stdout_lines | map('regex_replace', ' - .*$', '')) }}"

playbooks/bootstrap_lxc.yml

@@ -1,9 +1,4 @@
 ---
-# Base packages, the Ansible root key and sshd basics for every Ubuntu/Rocky
-# guest. Run it once after a container is created (and adopted, see
-# discover_guests.yml). ssh_harden.yml then tightens sshd further through
-# /etc/ssh/sshd_config.d/00-ansible.conf.
-
 - name: Bootstrap - Install packages on Ubuntu
   hosts: ubuntu
   become: true
@@ -25,9 +20,7 @@
           - zip
           - git
           - sudo
-          - tmux
           - tree
-          - bash-completion
           - traceroute
           - dnsutils
           - ca-certificates
@@ -35,46 +28,36 @@
           - python3-pip
           - software-properties-common
           - rsync
+          - apt-transport-https
           - sysstat
           - screen
-          - mtr-tiny             # plain "mtr" pulls in GTK on Ubuntu
+          - mtr
           - tar
         state: present
         update_cache: true
-        cache_valid_time: 3600
 
 - name: Bootstrap - Install packages on Rocky Linux
   hosts: rocky
   become: true
 
   tasks:
-    # epel-release ships in the "extras" repo on Rocky/Alma/CentOS Stream.
+    - name: Ensure EPEL repo is enabled
-    # dnf-plugins-core provides "dnf config-manager" for the next task.
-    - name: Ensure EPEL release package and dnf plugins are installed
       ansible.builtin.dnf:
-        name:
+        name: epel-release
-          - epel-release
-          - dnf-plugins-core
         state: present
 
-    # EPEL packages may depend on CRB (CodeReady Builder). EPEL's install
-    # guide enables it together with epel-release.
-    - name: Enable the CRB repo
-      community.general.dnf_config_manager:
-        name: crb
-        state: enabled
-
     - name: Install packages via dnf
       ansible.builtin.dnf:
         name:
-          # BaseOS / AppStream
           - tar
-          - vim-enhanced         # "vim" is only a virtual provide on RHEL
+          - vim
+          - htop
           - nmap
           - curl
           - mtr
           - wget
           - iperf3
+          - fail2ban
           - net-tools
           - lsof
           - unzip
@@ -91,9 +74,6 @@
           - python3-pip
           - rsync
           - sysstat
-          # EPEL
-          - htop
-          - fail2ban
           - screen
         state: present
         update_cache: true
@@ -101,38 +81,64 @@
 - name: Bootstrap - Configure SSH access
   hosts: ubuntu:rocky
   become: true
-  gather_facts: false
-
-  vars:
-    bootstrap_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key"
-    sshd_service: "{{ 'ssh' if inventory_hostname in groups['ubuntu'] else 'sshd' }}"
 
   tasks:
-    - name: Add the Ansible key to root's authorized_keys
+    - name: Ensure root's .ssh directory exists
-      ansible.posix.authorized_key:
+      ansible.builtin.file:
-        user: root
+        path: /root/.ssh
-        key: "{{ bootstrap_ssh_key }}"
+        state: directory
-        state: present
+        owner: root
+        group: root
+        mode: '0700'
 
-    # PasswordAuthentication stays on so a fresh container is still reachable
+    - name: Set authorized_keys for root
-    # by password; ssh_harden.yml turns it off once keys are in place.
+      ansible.builtin.copy:
-    - name: Allow key-based root login
+        dest: /root/.ssh/authorized_keys
+        content: |
+          ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key
+        owner: root
+        group: root
+        mode: '0600'
+
+    - name: Permit root login via public key
       ansible.builtin.lineinfile:
         path: /etc/ssh/sshd_config
-        regexp: '^#?{{ item.option }}\s'
+        regexp: '^#?PermitRootLogin\s+'
-        line: "{{ item.option }} {{ item.value }}"
+        line: 'PermitRootLogin prohibit-password'
-        validate: /usr/sbin/sshd -t -f %s
+        state: present
         backup: true
-      loop:
-        - { option: PermitRootLogin, value: prohibit-password }
-        - { option: PubkeyAuthentication, value: "yes" }
-        - { option: PasswordAuthentication, value: "yes" }
-      loop_control:
-        label: "{{ item.option }} {{ item.value }}"
-      notify: Restart sshd
 
-  handlers:
+    - name: Ensure PubkeyAuthentication is enabled
-    - name: Restart sshd
+      ansible.builtin.lineinfile:
+        path: /etc/ssh/sshd_config
+        regexp: '^#?PubkeyAuthentication\s+'
+        line: 'PubkeyAuthentication yes'
+        state: present
+
+    # NOTE: playbooks/ssh_harden.yml overrides this via /etc/ssh/sshd_config.d/00-ansible.conf
+    - name: Ensure PasswordAuthentication is enabled
+      ansible.builtin.lineinfile:
+        path: /etc/ssh/sshd_config
+        regexp: '^#?PasswordAuthentication\s+'
+        line: 'PasswordAuthentication yes'
+        state: present
+
+- name: Bootstrap - Restart SSH on Ubuntu
+  hosts: ubuntu
+  become: true
+
+  tasks:
+    - name: Restart ssh service
       ansible.builtin.service:
-        name: "{{ sshd_service }}"
+        name: ssh
+        state: restarted
+
+- name: Bootstrap - Restart SSH on Rocky Linux
+  hosts: rocky
+  become: true
+
+  tasks:
+    - name: Restart sshd service
+      ansible.builtin.service:
+        name: sshd
         state: restarted

playbooks/health_check.yml

@@ -1,14 +1,4 @@
 ---
-# Disk and RAM report for every reachable host. Filesystems above
-# disk_warn_pct and hosts that did not answer are printed as warnings and,
-# when Telegram credentials are present, sent as one Telegram message per run.
-#
-# Extra vars (optional):
-#   disk_warn_pct      - default 80
-#   telegram_bot_token / TELEGRAM_BOT_TOKEN - extra var or environment variable
-#   telegram_chat_id / TELEGRAM_CHAT_ID     - extra var or environment variable
-# Without both Telegram values the playbook only prints the report.
-
 - name: Check Host Availability
   hosts: all
   gather_facts: false
@@ -24,12 +14,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
 - name: Disk and RAM Usage Report
@@ -37,12 +27,9 @@
   become: true
   gather_facts: true
 
-  vars:
-    disk_warn_pct: 80
-
   tasks:
     - name: Get disk usage (excluding snap, overlay, tmpfs)
-      ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay -x efivarfs | tail -n +2
+      ansible.builtin.shell: df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs -x overlay | tail -n +2
       register: disk_usage
       changed_when: false
 
@@ -68,81 +55,8 @@
           {{ disk_usage.stdout }}
           ======================================================
 
-    - name: Collect filesystems above {{ disk_warn_pct }}%
+    - name: Warn on high disk usage (>85%)
-      ansible.builtin.set_fact:
+      ansible.builtin.debug:
-        disk_alerts: "{{ disk_alerts | default([]) + [item.split()[0] ~ ' ' ~ item.split()[-1]] }}"
+        msg: "WARNING: {{ item.split()[-1] }} usage on {{ item.split()[0] }} of {{ inventory_hostname }}"
       loop: "{{ disk_usage.stdout_lines }}"
-      when: (item.split()[-1] | replace('%', '') | int) > disk_warn_pct | int
+      when: (item.split()[-1] | replace('%', '') | int) > 85
-
-    - name: Warn on high disk usage
-      ansible.builtin.debug:
-        msg: "WARNING: {{ item }} used on {{ inventory_hostname }} (threshold {{ disk_warn_pct }}%)"
-      loop: "{{ disk_alerts | default([]) }}"
-
-- name: Telegram alert
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  vars:
-    disk_warn_pct: 80
-    # Accepted from a Semaphore variable group as extra vars (telegram_bot_token
-    # or TELEGRAM_BOT_TOKEN) or as environment variables (TELEGRAM_BOT_TOKEN).
-    telegram_bot_token: "{{ TELEGRAM_BOT_TOKEN | default(lookup('env', 'TELEGRAM_BOT_TOKEN'), true) | string }}"
-    telegram_chat_id: "{{ TELEGRAM_CHAT_ID | default(lookup('env', 'TELEGRAM_CHAT_ID'), true) | string }}"
-    telegram_configured: "{{ telegram_bot_token | string | length > 0 and telegram_chat_id | string | length > 0 }}"
-    offline_hosts: "{{ groups['all'] | difference(groups['reachable_hosts'] | default([])) }}"
-
-  tasks:
-    - name: Build alert text
-      ansible.builtin.set_fact:
-        alert_text: "{{ alert_body | default('', true) | trim }}"   # an empty template renders as None
-      vars:
-        alert_body: |-
-          {% for h in groups['reachable_hosts'] | default([]) %}
-          {% for fs in hostvars[h].disk_alerts | default([]) %}
-          DISK {{ h }}: {{ fs }} used (threshold {{ disk_warn_pct }}%)
-          {% endfor %}
-          {% endfor %}
-          {% for h in offline_hosts %}
-          OFFLINE {{ h }} did not answer
-          {% endfor %}
-
-    - name: Nothing to alert
-      ansible.builtin.debug:
-        msg: "Every host reachable and all filesystems at or below {{ disk_warn_pct }}% - no alert"
-      when: alert_text | length == 0
-
-    - name: Print alerts when Telegram is not configured
-      ansible.builtin.debug:
-        msg: |-
-          ALERTS (Telegram not configured - bot token {{ 'set' if telegram_bot_token | string | length > 0 else 'MISSING' }},
-          chat id {{ 'set' if telegram_chat_id | string | length > 0 else 'MISSING' }};
-          pass TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID through the variable group attached to this template)
-          {{ alert_text }}
-      when:
-        - alert_text | length > 0
-        - not telegram_configured
-
-    - name: Send Telegram alert
-      ansible.builtin.uri:
-        url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
-        method: POST
-        body_format: json
-        body:
-          chat_id: "{{ telegram_chat_id }}"
-          text: "Homelab health check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
-        status_code: [200]
-      register: telegram_result
-      failed_when: false
-      no_log: true          # the bot token is part of the URL
-      when:
-        - alert_text | length > 0
-        - telegram_configured
-
-    - name: Telegram delivery status
-      ansible.builtin.assert:
-        that: telegram_result.status == 200
-        fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
-        success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
-      when: telegram_result is not skipped

playbooks/new_lxc.yml

@@ -1,74 +1,73 @@
 ---
-# Creates a Rocky Linux 9 container through the Proxmox API and starts it.
+- name: Build Dynamic Rocky Linux LXC Container
-# Runs on the Semaphore host (localhost); proxmoxer must be importable by the
-# Python that runs Ansible there.
-#
-# Extra vars (Semaphore environment / variable group):
-#   proxmox_api_user      - e.g. root@pam            (required)
-#   proxmox_api_password  - that user's password     (this, or a token)
-#   proxmox_api_token_id / proxmox_api_token_secret  (API token instead of a password)
-#   proxmox_api_host      - defaults to the pve entry in the inventory
-#
-# Optional: lxc_hostname, lxc_template, lxc_cores, lxc_memory, lxc_storage,
-# lxc_disk_size, lxc_bridge, lxc_dns, target_node.
-
-- name: Build a Rocky Linux LXC container
   hosts: localhost
-  connection: local
   gather_facts: false
 
   vars:
-    proxmox_api_host: "{{ hostvars['pve'].ansible_host }}"
+    # --- Container Specifications ---
-    target_node: mini-pc
+    lxc_hostname: "new_lxc"
-    lxc_hostname: new-lxc              # underscores are not valid in a hostname
+    lxc_template: "local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz" 
-    lxc_template: local:vztmpl/rockylinux-9-default_20240912_amd64.tar.xz
     lxc_cores: 2
     lxc_memory: 1024
-    lxc_storage: TB
+    lxc_storage: "TB"
     lxc_disk_size: 10
-    lxc_bridge: vmbr0
+    lxc_bridge: "vmbr0"
-    lxc_dns: "{{ hostvars['pihole'].ansible_host }}"
+    lxc_dns: "192.168.2.2"
+    target_node: "pve" 
+    
+    # --- Security & Access ---
     lxc_ssh_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINegitHaQG8i+s412h3wTWf/KvpeX1CEpYFva9MhHR+7 ansible-key"
 
-  module_defaults:
-    community.proxmox.proxmox:
-      api_host: "{{ proxmox_api_host }}"
-      api_user: "{{ proxmox_api_user }}"
-      api_password: "{{ proxmox_api_password | default(omit) }}"
-      api_token_id: "{{ proxmox_api_token_id | default(omit) }}"
-      api_token_secret: "{{ proxmox_api_token_secret | default(omit) }}"
-      validate_certs: false
-      timeout: 300               # template extraction takes longer than the 30 s default
-
   tasks:
-    - name: Ensure proxmoxer is available to the controller's Python
+    # ========================================================================
-      ansible.builtin.command: "{{ ansible_playbook_python }} -m pip install proxmoxer requests"
+    # NEW FIX: Auto-install missing dependencies into Semaphore's isolated venv
-      register: pip_install
+    # ========================================================================
-      changed_when: "'Successfully installed' in pip_install.stdout"
+    - name: Ensure Proxmox API dependencies are installed in Semaphore
+      ansible.builtin.shell: "{{ ansible_playbook_python }} -m pip install proxmoxer requests"
+      changed_when: false
+    # ========================================================================
 
-    - name: Create the container (Proxmox assigns the next free VMID)
+    - name: Get the next available VMID from the Proxmox cluster
-      community.proxmox.proxmox:
+      ansible.builtin.uri:
+        url: "https://{{ proxmox_api_host }}:8006/api2/json/cluster/nextid"
+        method: GET
+        headers:
+          Authorization: "PVEAPIToken {{ proxmox_api_user }}={{ proxmox_api_password }}"
+        validate_certs: false
+        return_content: true
+      register: nextid_response
+
+    - name: Extract the dynamic VMID from the API response
+      ansible.builtin.set_fact:
+        dynamic_vmid: "{{ nextid_response.json.data }}"
+
+    - name: Create the Rocky Linux LXC container
+      community.general.proxmox:
+        api_host: "{{ proxmox_api_host }}"
+        api_user: "{{ proxmox_api_user }}"
+        api_password: "{{ proxmox_api_password }}"
+        
         node: "{{ target_node }}"
+        vmid: "{{ dynamic_vmid }}"
         hostname: "{{ lxc_hostname }}"
         ostemplate: "{{ lxc_template }}"
         cores: "{{ lxc_cores }}"
         memory: "{{ lxc_memory }}"
         disk: "{{ lxc_storage }}:{{ lxc_disk_size }}"
-        netif:
+        netif: '{"net0":"name=eth0,bridge={{ lxc_bridge }},ip=dhcp"}'
-          net0: "name=eth0,bridge={{ lxc_bridge }},ip=dhcp"
         nameserver: "{{ lxc_dns }}"
         pubkey: "{{ lxc_ssh_key }}"
         unprivileged: true
         state: present
-      register: lxc_create
 
-    - name: Start the container
+    - name: Ensure the new LXC container is powered on
-      community.proxmox.proxmox:
+      community.general.proxmox:
-        vmid: "{{ lxc_create.vmid }}"
+        api_host: "{{ proxmox_api_host }}"
+        api_user: "{{ proxmox_api_user }}"
+        api_password: "{{ proxmox_api_password }}"
+        vmid: "{{ dynamic_vmid }}"
         state: started
-
+        
-    - name: Report
+    - name: Output the assigned VMID to the Semaphore UI logs
       ansible.builtin.debug:
-        msg: >-
+        msg: "Successfully built {{ lxc_hostname }} (Rocky Linux 9) with dynamic ID: {{ dynamic_vmid }}"
-          Built {{ lxc_hostname }} (Rocky Linux 9) as VMID {{ lxc_create.vmid }} on {{ target_node }}.
-          Next: run discover_guests.yml with adopt=true, add it to inventories/all.yml, then bootstrap_lxc.yml.

playbooks/proxmox-backup.yml

@@ -1,81 +1,60 @@
 ---
-# vzdump of selected guests on the Proxmox host. vzdump prunes old dumps of
-# each guest itself (keep-last), so nothing else touches the dump directory.
-# Every run is appended to {{ backup_dir }}/proxmox_backup.log on the host.
-#
-# backup_mode "stop" shuts the guest down for the whole backup. Never list
-# the guest that runs Semaphore with mode stop: the job would stop itself
-# half way through. Use "snapshot" for that one (falls back to suspend on
-# storage without snapshot support).
-#
-# Extra vars (optional):
-#   guests       - list of { id, type, name }
-#   keep_last    - dumps kept per guest (default 3)
-#   backup_mode  - stop | snapshot | suspend (default stop)
-#   storage      - target PVE storage id (default local)
-#   compression  - zstd | gzip | lzo (default zstd)
-
 - name: Backup selected Proxmox VMs and LXCs
   hosts: proxmox
   become: true
-  gather_facts: false
-
   vars:
-    backup_dir: /var/lib/vz/dump
+    backup_dir: "/var/lib/vz/dump"
-    storage: local
+    storage: "local"
-    compression: zstd
+    compression: "zstd"
-    keep_last: 3
+    maxfiles: 3
-    backup_mode: stop
     guests:
-      # - { id: 104, type: lxc, name: courses }
+      #- { id: 104, type: "lxc", name: "jellyfin" }
-      - { id: 108, type: lxc, name: docker }
+      - { id: 108, type: "lxc", name: "docker" }
-      # - { id: 101, type: qemu, name: pnetlab }
+      #- { id: 101, type: "qemu", name: "pnetlab" }
 
   tasks:
-    - name: Ensure the log directory exists
+    - name: Ensure backup directory exists
-      ansible.builtin.file:
+      file:
         path: "{{ backup_dir }}"
         state: directory
         mode: '0755'
 
-    - name: Run vzdump
+    - name: Run vzdump backup with logging
-      ansible.builtin.shell: |
+      shell: |
-        set -o pipefail
         LOGFILE="{{ backup_dir }}/proxmox_backup.log"
         START=$(date +%s)
-        echo "[$(date '+%Y-%m-%d %H:%M:%S')] START   {{ item.name }} ({{ item.type }} {{ item.id }}) mode={{ backup_mode }}" >> "$LOGFILE"
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup of {{ item.name }} ({{ item.type }}) [ID: {{ item.id }}]" >> "$LOGFILE"
-        OUTPUT=$(vzdump {{ item.id }} --mode {{ backup_mode }} --compress {{ compression }} \
+
-                 --storage {{ storage }} --prune-backups keep-last={{ keep_last }} 2>&1)
+        OUTPUT=$(vzdump {{ item.id }} --mode stop --compress {{ compression }} --storage {{ storage }} --remove 0 2>&1)
         STATUS=$?
-        DURATION=$(( $(date +%s) - START ))
+
-        if [ "$STATUS" -eq 0 ]; then
+        END=$(date +%s)
-          echo "[$(date '+%Y-%m-%d %H:%M:%S')] SUCCESS {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE"
+        DURATION=$((END - START))
-          echo "$OUTPUT" | grep -E 'archive file size|pruned|removing backup' || true
+
+        if [ $STATUS -eq 0 ]; then
+          echo "[$(date '+%Y-%m-%d %H:%M:%S')] ✅ SUCCESS - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE"
         else
-          echo "[$(date '+%Y-%m-%d %H:%M:%S')] FAILURE {{ item.name }} ({{ item.id }}) ${DURATION}s" >> "$LOGFILE"
+          echo "[$(date '+%Y-%m-%d %H:%M:%S')] ❌ FAILURE - {{ item.name }} [{{ item.id }}] - Duration: ${DURATION}s" >> "$LOGFILE"
-          printf -- '---- vzdump output ----\n%s\n-----------------------\n' "$OUTPUT" >> "$LOGFILE"
+          echo "---- ERROR OUTPUT ----" >> "$LOGFILE"
-          echo "$OUTPUT" | tail -n 20
+          echo "$OUTPUT" >> "$LOGFILE"
+          echo "----------------------" >> "$LOGFILE"
         fi
-        exit "$STATUS"
       args:
         executable: /bin/bash
       loop: "{{ guests }}"
-      loop_control:
-        label: "{{ item.name }} ({{ item.type }} {{ item.id }})"
-      changed_when: true
 
-    - name: List the dumps kept for each guest
+    - name: Rename backup files to include guest name
-      ansible.builtin.command: pvesm list {{ storage }} --content backup --vmid {{ item.id }}
+      shell: |
+        latest_file=$(ls -t {{ backup_dir }}/vzdump-{{ item.type }}-{{ item.id }}-*.{{ compression }} | head -n 1)
+        new_file="{{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-{{ '%Y%m%d-%H%M%S' | strftime(ansible_date_time.epoch | int) }}.{{ compression }}"
+        cp "$latest_file" "$new_file"
+      args:
+        executable: /bin/bash
       loop: "{{ guests }}"
-      loop_control:
-        label: "{{ item.name }} ({{ item.id }})"
-      register: dump_lists
-      changed_when: false
 
-    - name: Backup summary
+    - name: Prune old backups (keep last {{ maxfiles }})
-      ansible.builtin.debug:
+      shell: |
-        msg: |
+        ls -tp {{ backup_dir }}/{{ item.name }}-{{ item.type }}-{{ item.id }}-*{{ compression }} | grep -v '/$' | tail -n +{{ maxfiles + 1 }} | xargs -r rm --
-          {% for r in dump_lists.results %}
+      args:
-          {{ r.item.name }} ({{ r.item.type }} {{ r.item.id }}) - mode {{ backup_mode }}, keeping the last {{ keep_last }}:
+        executable: /bin/bash
-          {{ r.stdout }}
+      loop: "{{ guests }}"
-          {% endfor %}

playbooks/reboot_check.yml

@@ -22,12 +22,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
 - name: Reboot Check - Ubuntu

playbooks/service_check.yml

@@ -1,13 +1,9 @@
 ---
 # Verifies the critical services listed per host in the inventory
-# (critical_services) and optionally starts any that are stopped. Services
+# (critical_services) and optionally starts any that are stopped.
-# found not running are sent as one Telegram message per run when Telegram
-# credentials are present.
 #
 # Extra vars (optional):
-#   restart_stopped    - default false; when true, start services found stopped
+#   restart_stopped - default false; when true, start services found stopped
-#   telegram_bot_token / TELEGRAM_BOT_TOKEN - extra var or environment variable
-#   telegram_chat_id / TELEGRAM_CHAT_ID     - extra var or environment variable
 
 - name: Check Host Availability
   hosts: ubuntu:rocky
@@ -24,12 +20,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
 - name: Service Check
@@ -84,68 +80,3 @@
       when:
         - restart_stopped | bool
         - item.state not in ['running', 'not-found']
-
-- name: Telegram alert
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  vars:
-    restart_stopped: false
-    # Accepted from a Semaphore variable group as extra vars (telegram_bot_token
-    # or TELEGRAM_BOT_TOKEN) or as environment variables (TELEGRAM_BOT_TOKEN).
-    telegram_bot_token: "{{ TELEGRAM_BOT_TOKEN | default(lookup('env', 'TELEGRAM_BOT_TOKEN'), true) | string }}"
-    telegram_chat_id: "{{ TELEGRAM_CHAT_ID | default(lookup('env', 'TELEGRAM_CHAT_ID'), true) | string }}"
-    telegram_configured: "{{ telegram_bot_token | string | length > 0 and telegram_chat_id | string | length > 0 }}"
-
-  tasks:
-    - name: Build alert text
-      ansible.builtin.set_fact:
-        alert_text: "{{ alert_body | default('', true) | trim }}"   # an empty template renders as None
-      vars:
-        alert_body: |-
-          {% for h in groups['reachable_hosts'] | default([]) %}
-          {% for svc in hostvars[h].service_states | default([]) if svc.state != 'running' %}
-          {% set note = ' - start attempted' if (restart_stopped | bool and svc.state != 'not-found') else '' %}
-          SERVICE {{ h }}: {{ svc.name }} {{ 'not installed' if svc.state == 'not-found' else svc.state }}{{ note }}
-          {% endfor %}
-          {% endfor %}
-
-    - name: Nothing to alert
-      ansible.builtin.debug:
-        msg: "All critical services running - no alert"
-      when: alert_text | length == 0
-
-    - name: Print alerts when Telegram is not configured
-      ansible.builtin.debug:
-        msg: |-
-          ALERTS (Telegram not configured - bot token {{ 'set' if telegram_bot_token | string | length > 0 else 'MISSING' }},
-          chat id {{ 'set' if telegram_chat_id | string | length > 0 else 'MISSING' }};
-          pass TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID through the variable group attached to this template)
-          {{ alert_text }}
-      when:
-        - alert_text | length > 0
-        - not telegram_configured
-
-    - name: Send Telegram alert
-      ansible.builtin.uri:
-        url: "https://api.telegram.org/bot{{ telegram_bot_token }}/sendMessage"
-        method: POST
-        body_format: json
-        body:
-          chat_id: "{{ telegram_chat_id }}"
-          text: "Homelab service check {{ now().strftime('%Y-%m-%d %H:%M') }}\n{{ alert_text }}"
-        status_code: [200]
-      register: telegram_result
-      failed_when: false
-      no_log: true          # the bot token is part of the URL
-      when:
-        - alert_text | length > 0
-        - telegram_configured
-
-    - name: Telegram delivery status
-      ansible.builtin.assert:
-        that: telegram_result.status == 200
-        fail_msg: "Telegram API returned {{ telegram_result.status }}: {{ telegram_result.json.description | default('no response') }}"
-        success_msg: "Telegram alert sent ({{ alert_text.split('\n') | length }} line(s))"
-      when: telegram_result is not skipped

playbooks/ssh_harden.yml

@@ -26,12 +26,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
 - name: SSH Hardening - Install fail2ban on Ubuntu

playbooks/time-region.yml

@@ -1,46 +1,30 @@
 ---
-# Sets the timezone on every Ubuntu/Rocky guest. LXC containers share the
+- name: Configure timezone on all hosts
-# host's clock, so NTP is disabled inside them; VMs sync against ntp_server.
-#
-# Extra vars (optional):
-#   timezone_region - default America/New_York
-#   ntp_server      - default time.google.com
-
-- name: Configure timezone
   hosts: ubuntu:rocky
   become: true
+  gather_facts: true
 
   vars:
-    timezone_region: America/New_York
+    timezone_region: "America/New_York"
 
   tasks:
-    - name: Link /etc/localtime to the zoneinfo file
+    - name: Copy timezone file
-      ansible.builtin.file:
+      ansible.builtin.copy:
         src: "/usr/share/zoneinfo/{{ timezone_region }}"
         dest: /etc/localtime
-        state: link
+        remote_src: true
-        force: true
 
     - name: Write /etc/timezone
       ansible.builtin.copy:
         dest: /etc/timezone
         content: "{{ timezone_region }}\n"
-        owner: root
-        group: root
-        mode: '0644'
 
-# The plays below reuse the facts gathered above instead of collecting them again.
+- name: Disable NTP on Ubuntu LXC containers
-
-- name: NTP on Ubuntu
   hosts: ubuntu
   become: true
-  gather_facts: false
-
-  vars:
-    ntp_server: time.google.com
 
   tasks:
-    - name: Stop and mask timesyncd in LXC containers
+    - name: Stop and mask timesyncd on LXC
       ansible.builtin.systemd:
         name: systemd-timesyncd
         state: stopped
@@ -48,11 +32,33 @@
         masked: true
       when: ansible_virtualization_type == "lxc"
 
-    - name: Point timesyncd at ntp_server on VMs
+- name: Disable NTP on Rocky Linux LXC containers
+  hosts: rocky
+  become: true
+
+  tasks:
+    - name: Stop and mask chronyd on LXC
+      ansible.builtin.systemd:
+        name: chronyd
+        state: stopped
+        enabled: false
+        masked: true
+      when: ansible_virtualization_type == "lxc"
+
+- name: Configure NTP on Ubuntu VMs
+  hosts: ubuntu
+  become: true
+
+  vars:
+    ntp_server: "time.google.com"
+
+  tasks:
+    - name: Configure timesyncd
       ansible.builtin.lineinfile:
         path: /etc/systemd/timesyncd.conf
         regexp: '^#?NTP='
         line: "NTP={{ ntp_server }}"
+        state: present
       when: ansible_virtualization_type != "lxc"
       notify: Restart timesyncd
 
@@ -63,38 +69,27 @@
         state: restarted
         enabled: true
 
-- name: NTP on Rocky Linux
+- name: Configure NTP on Rocky Linux VMs
   hosts: rocky
   become: true
-  gather_facts: false
 
   vars:
-    ntp_server: time.google.com
+    ntp_server: "time.google.com"
 
   tasks:
-    - name: Stop and mask chronyd in LXC containers
+    - name: Disable default chrony pools
-      ansible.builtin.systemd:
-        name: chronyd
-        state: stopped
-        enabled: false
-        masked: true
-      when: ansible_virtualization_type == "lxc"
-
-    # The lookahead keeps the server line added below from being commented
-    # out again on the next run.
-    - name: Disable the default chrony pools on VMs
       ansible.builtin.replace:
         path: /etc/chrony.conf
-        regexp: '^(pool|server)\s+(?!{{ ntp_server | regex_escape }}(\s|$)).*'
+        regexp: '^(pool|server) .*'
-        replace: '# \g<0>  (disabled by Ansible)'
+        replace: '# \1 removed by Ansible'
       when: ansible_virtualization_type != "lxc"
-      notify: Restart chronyd
 
-    - name: Point chrony at ntp_server on VMs
+    - name: Configure chrony NTP server
       ansible.builtin.lineinfile:
         path: /etc/chrony.conf
-        regexp: '^server\s+{{ ntp_server | regex_escape }}(\s|$)'
         line: "server {{ ntp_server }} iburst"
+        insertafter: EOF
+        state: present
       when: ansible_virtualization_type != "lxc"
       notify: Restart chronyd
 
@@ -108,7 +103,6 @@
 - name: Verify time configuration
   hosts: ubuntu:rocky
   become: true
-  gather_facts: false
 
   tasks:
     - name: Check current system time

playbooks/update_pihole_blocklists.yml

@@ -1,22 +1,18 @@
 ---
 - name: Update Pi-hole core and blocklists
   hosts: pihole
-  become: true
+  become: yes
-  gather_facts: false
-
   tasks:
     - name: Update Pi-hole core components
-      ansible.builtin.command: pihole -up
+      command: pihole -up
       register: update_out
-      changed_when: "'Everything is up to date' not in update_out.stdout"
 
     - name: Update Pi-hole blocklists
-      ansible.builtin.command: pihole -g
+      command: pihole -g
       register: gravity_out
-      changed_when: true
 
     - name: Display update summary
-      ansible.builtin.debug:
+      debug:
         msg:
           - "Core update: {{ update_out.stdout }}"
           - "Gravity update: {{ gravity_out.stdout }}"

playbooks/update_system.yml

@@ -1,257 +1,47 @@
 ---
-# Full package upgrade for every reachable host: apt dist-upgrade on Ubuntu,
-# dnf update on Rocky Linux. Each host is checked for pending packages before
-# the upgrade runs, so the per-host and final reports show which systems were
-# actually updated (and with what) and which were already up to date.
-#
-# Nothing reboots here; use reboot_check.yml for that.
-
-# Play 1: probe every ubuntu/rocky host and sort them into a "reachable_hosts"
-# group so the two upgrade plays below can skip anything that's offline.
 - name: Check Host Availability
-  hosts: ubuntu:rocky
+  hosts: all
   gather_facts: false
   ignore_unreachable: true
 
   tasks:
-    # Ping each host; failures/unreachables are swallowed so the play keeps
-    # going instead of aborting the whole run.
     - name: Ping hosts to check availability
       ansible.builtin.ping:
       register: ping_result
       ignore_errors: true
       ignore_unreachable: true
 
-    # Dynamically creates/populates the "reachable_hosts" group, used by the
-    # upgrade plays' host patterns (ubuntu:&reachable_hosts etc.).
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
-    # Print a message for any host that failed the ping, so offline hosts
-    # are visible in the run output and in the final summary.
     - name: Report offline hosts
       ansible.builtin.debug:
-        msg: "{{ inventory_hostname }} - OFFLINE and will be skipped"
+        msg: "Host {{ inventory_hostname }} is OFFLINE and will be skipped"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
-# Play 2: upgrade Ubuntu hosts and record what each one needed.
 - name: Update and Upgrade - Ubuntu
   hosts: ubuntu:&reachable_hosts
   become: true
   gather_facts: false
 
   tasks:
-    # Refresh the package lists first so the pending check below is accurate.
+    - name: Update & Upgrade via apt
-    # A cache refreshed within the last hour is reused.
-    - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 3600
-      changed_when: false   # refreshing the lists is not a change worth reporting
+        upgrade: dist
-      check_mode: false
 
-    # Simulate the upgrade (no changes made) to learn which packages are
-    # pending. Each candidate shows up as an "Inst <name> ..." line.
-    - name: Check which packages are pending  # noqa: command-instead-of-module (dry run, the apt module cannot simulate)
-      ansible.builtin.command: apt-get --just-print dist-upgrade
-      register: apt_simulate
-      changed_when: false
-      check_mode: false
-      environment:
-        DEBIAN_FRONTEND: noninteractive
-
-    - name: Record pending package list
-      ansible.builtin.set_fact:
-        updated_packages: "{{ apt_simulate.stdout | regex_findall('(?m)^Inst (\\S+)') }}"
-
-    # The actual upgrade. Hosts with nothing pending come back unchanged.
-    # A failure is caught so this host still shows up in the reports below;
-    # the final play turns any failure back into a non-zero exit.
-    - name: Update & Upgrade via apt
-      block:
-        - name: Run apt dist-upgrade
-          ansible.builtin.apt:
-            update_cache: false
-            upgrade: dist
-          register: upgrade_result
-      rescue:
-        - name: Record the upgrade error
-          ansible.builtin.set_fact:
-            update_error: "{{ upgrade_result.msg | default('see the task output above') }}"
-
-    # update_status drives both the per-host report and the final summary:
-    #   updated     - packages were installed
-    #   up to date  - nothing was pending
-    #   failed      - the upgrade task itself errored out
-    - name: Record update status
-      ansible.builtin.set_fact:
-        update_status: >-
-          {{ 'failed' if upgrade_result is failed
-             else ('updated' if upgrade_result is changed else 'up to date') }}
-
-    - name: Print report
-      ansible.builtin.debug:
-        msg: |
-          ======================================================
-           HOST: {{ inventory_hostname }} ({{ ansible_host }}) - Ubuntu
-          ======================================================
-           STATUS: {{ update_status | upper }}{{ ' (CHECK MODE - nothing was applied)' if ansible_check_mode else '' }}
-          {% if update_status == 'updated' %}
-          ------------------------------------------------------
-           PACKAGES UPDATED: {{ updated_packages | length }}
-          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
-          {% elif update_status == 'failed' %}
-          ------------------------------------------------------
-           ERROR: {{ update_error | default('see the task output above') }}
-           PACKAGES STILL PENDING: {{ updated_packages | length }}
-          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
-          {% endif %}
-          ======================================================
-
-# Play 3: upgrade Rocky Linux hosts and record what each one needed.
 - name: Update and Upgrade - Rocky Linux
   hosts: rocky:&reachable_hosts
   become: true
   gather_facts: false
 
   tasks:
-    # Refresh the metadata first so the pending check below is accurate.
-    - name: Update dnf cache
-      ansible.builtin.dnf:
-        update_cache: true
-      changed_when: false   # refreshing the metadata is not a change worth reporting
-      check_mode: false
-
-    # rc 100 means updates are available, rc 0 means none, so only other
-    # codes count as a failure.
-    - name: Check which packages are pending
-      ansible.builtin.command: dnf check-update
-      register: dnf_check
-      changed_when: false
-      check_mode: false
-      failed_when: dnf_check.rc not in [0, 100]
-
-    # Pull the "name.arch  version  repo" lines out of the check-update
-    # output. Anything under "Obsoleting Packages" is a duplicate listing,
-    # so the output is truncated there first.
-    - name: Record pending package list
-      ansible.builtin.set_fact:
-        updated_packages: >-
-          {{ dnf_check.stdout.split('Obsoleting Packages')[0].splitlines()
-             | select('match', '^\S+\.\S+\s+\S+\s+\S+$')
-             | map('regex_replace', '^(\S+)\s.*$', '\1') | list }}
-
-    # The actual upgrade. Hosts with nothing pending come back unchanged.
-    # A failure is caught so this host still shows up in the reports below;
-    # the final play turns any failure back into a non-zero exit.
     - name: Update & Upgrade via dnf
-      block:
+      ansible.builtin.dnf:
-        - name: Run dnf update
+        name: "*"
-          ansible.builtin.dnf:
+        state: latest
-            name: "*"
+        update_cache: true
-            state: latest
-            update_cache: false
-          register: upgrade_result
-      rescue:
-        - name: Record the upgrade error
-          ansible.builtin.set_fact:
-            update_error: "{{ upgrade_result.msg | default('see the task output above') }}"
-
-    # Same three states as the Ubuntu play above.
-    - name: Record update status
-      ansible.builtin.set_fact:
-        update_status: >-
-          {{ 'failed' if upgrade_result is failed
-             else ('updated' if upgrade_result is changed else 'up to date') }}
-
-    - name: Print report
-      ansible.builtin.debug:
-        msg: |
-          ======================================================
-           HOST: {{ inventory_hostname }} ({{ ansible_host }}) - Rocky Linux
-          ======================================================
-           STATUS: {{ update_status | upper }}{{ ' (CHECK MODE - nothing was applied)' if ansible_check_mode else '' }}
-          {% if update_status == 'updated' %}
-          ------------------------------------------------------
-           PACKAGES UPDATED: {{ updated_packages | length }}
-          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
-          {% elif update_status == 'failed' %}
-          ------------------------------------------------------
-           ERROR: {{ update_error | default('see the task output above') }}
-           PACKAGES STILL PENDING: {{ updated_packages | length }}
-          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
-          {% endif %}
-          ======================================================
-
-# Play 4: after both upgrade plays have run, print one consolidated summary of
-# which hosts were updated, which were already current, and which were missed.
-- name: Summary - Updated Hosts
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  vars:
-    all_targets: "{{ groups['ubuntu'] | default([]) | union(groups['rocky'] | default([])) }}"
-    reachable: "{{ groups['reachable_hosts'] | default([]) }}"
-    # update_status of every host that got far enough to set it.
-    statuses: "{{ reachable | map('extract', hostvars, 'update_status') | select('defined') | list }}"
-    status_counts:
-      "updated": "{{ statuses | select('equalto', 'updated') | list | length }}"
-      "up to date": "{{ statuses | select('equalto', 'up to date') | list | length }}"
-      "failed": "{{ statuses | select('equalto', 'failed') | list | length }}"
-    failed_hosts: "{{ reachable | map('extract', hostvars) | selectattr('update_status', 'defined')
-                      | selectattr('update_status', 'equalto', 'failed')
-                      | map(attribute='inventory_hostname') | list }}"
-
-  tasks:
-    - name: Print hosts that were updated
-      ansible.builtin.debug:
-        msg: >-
-          UPDATED      {{ item }} -
-          {{ hostvars[item].updated_packages | default([]) | length }} package(s):
-          {{ hostvars[item].updated_packages | default([]) | join(', ') | default('(list unavailable)', true) }}
-      loop: "{{ reachable }}"
-      when: hostvars[item].update_status | default('') == 'updated'
-
-    - name: Print hosts that were already up to date
-      ansible.builtin.debug:
-        msg: "UP TO DATE   {{ item }} - no packages pending"
-      loop: "{{ reachable }}"
-      when: hostvars[item].update_status | default('') == 'up to date'
-
-    - name: Print hosts whose upgrade failed
-      ansible.builtin.debug:
-        msg: "FAILED       {{ item }} - the upgrade task reported an error, see the play output above"
-      loop: "{{ reachable }}"
-      when: hostvars[item].update_status | default('') == 'failed'
-
-    # Covers both hosts that failed the initial ping and any host that became
-    # unreachable part way through an upgrade play.
-    - name: Print hosts that were skipped
-      ansible.builtin.debug:
-        msg: "SKIPPED      {{ item }} - offline or unreachable, not upgraded"
-      loop: "{{ all_targets }}"
-      when: hostvars[item].update_status is not defined
-
-    - name: Print totals
-      ansible.builtin.debug:
-        msg: >-
-          {{ all_targets | length }} host(s) targeted,
-          {{ status_counts['updated'] }} updated,
-          {{ status_counts['up to date'] }} already up to date,
-          {{ status_counts['failed'] }} failed,
-          {{ all_targets | reject('in', reachable) | list | length }} skipped;
-          {{ reachable | map('extract', hostvars, 'updated_packages')
-             | select('defined') | map('length') | sum }}
-          package(s) updated in total.
-          {{ '(CHECK MODE - nothing was applied.)' if ansible_check_mode else '' }}
-
-    # Keep a failed upgrade visible to Semaphore: every report above has been
-    # printed by now, so it is safe to end the run non-zero here.
-    - name: Fail the run if any host could not be upgraded
-      ansible.builtin.fail:
-        msg: "Upgrade failed on: {{ failed_hosts | join(', ') }}"
-      when: failed_hosts | length > 0

playbooks/user_setup.yml

@@ -24,12 +24,12 @@
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
-      when: ping_result is reachable and ping_result is succeeded
+      when: ping_result is succeeded
 
     - name: Report offline hosts
       ansible.builtin.debug:
         msg: "{{ inventory_hostname }} - OFFLINE"
-      when: ping_result is unreachable or ping_result is failed
+      when: ping_result is not succeeded
       delegate_to: localhost
 
 - name: User Setup - Create admin user on Ubuntu