Report updated vs already up-to-date hosts in update_system.yml

Check each host for pending packages before upgrading, then print a
per-host status (updated with the package list, up to date, or failed)
and one consolidated summary play with totals.

Failed upgrades are caught per host so they are reported instead of
dropping out of the run, and a final fail task keeps the run non-zero.
The read-only cache refresh and pending checks run with check_mode:
false, so --check works as a dry run.

README.md

@@ -43,7 +43,7 @@ address, the first Ubuntu VM, and the service lists for `webserver`, `bot` and
 |----------|-------|----------------------|
 | `ping.yml` | all | connectivity check |
 | `health_check.yml` | all | disk and RAM report; Telegram alert for filesystems above `disk_warn_pct` (default 80) and offline hosts |
-| `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update |
+| `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update; reports per host whether it was updated (with the package list) or already up to date, plus a final summary. Supports `--check` as a dry run |
 | `bootstrap_lxc.yml` | ubuntu, rocky | base packages (EPEL and CRB enabled on Rocky), Ansible root key, sshd basics |
 | `add_root_key.yml` | all | add a personal public key to root |
 | `time-region.yml` | ubuntu, rocky | timezone, NTP on VMs, NTP disabled in LXC. `timezone_region`, `ntp_server` |

playbooks/update_system.yml

@@ -1,47 +1,257 @@
 ---
+# Full package upgrade for every reachable host: apt dist-upgrade on Ubuntu,
+# dnf update on Rocky Linux. Each host is checked for pending packages before
+# the upgrade runs, so the per-host and final reports show which systems were
+# actually updated (and with what) and which were already up to date.
+#
+# Nothing reboots here; use reboot_check.yml for that.
+
+# Play 1: probe every ubuntu/rocky host and sort them into a "reachable_hosts"
+# group so the two upgrade plays below can skip anything that's offline.
 - name: Check Host Availability
   hosts: ubuntu:rocky
   gather_facts: false
   ignore_unreachable: true
 
   tasks:
+    # Ping each host; failures/unreachables are swallowed so the play keeps
+    # going instead of aborting the whole run.
     - name: Ping hosts to check availability
       ansible.builtin.ping:
       register: ping_result
       ignore_errors: true
       ignore_unreachable: true
 
+    # Dynamically creates/populates the "reachable_hosts" group, used by the
+    # upgrade plays' host patterns (ubuntu:&reachable_hosts etc.).
     - name: Add reachable hosts to group
       ansible.builtin.group_by:
         key: reachable_hosts
       when: ping_result is reachable and ping_result is succeeded
 
+    # Print a message for any host that failed the ping, so offline hosts
+    # are visible in the run output and in the final summary.
     - name: Report offline hosts
       ansible.builtin.debug:
-        msg: "Host {{ inventory_hostname }} is OFFLINE and will be skipped"
+        msg: "{{ inventory_hostname }} - OFFLINE and will be skipped"
       when: ping_result is unreachable or ping_result is failed
       delegate_to: localhost
 
+# Play 2: upgrade Ubuntu hosts and record what each one needed.
 - name: Update and Upgrade - Ubuntu
   hosts: ubuntu:&reachable_hosts
   become: true
   gather_facts: false
 
   tasks:
-    - name: Update & Upgrade via apt
+    # Refresh the package lists first so the pending check below is accurate.
+    # A cache refreshed within the last hour is reused.
+    - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 3600
-        upgrade: dist
+      changed_when: false   # refreshing the lists is not a change worth reporting
+      check_mode: false
 
+    # Simulate the upgrade (no changes made) to learn which packages are
+    # pending. Each candidate shows up as an "Inst <name> ..." line.
+    - name: Check which packages are pending  # noqa: command-instead-of-module (dry run, the apt module cannot simulate)
+      ansible.builtin.command: apt-get --just-print dist-upgrade
+      register: apt_simulate
+      changed_when: false
+      check_mode: false
+      environment:
+        DEBIAN_FRONTEND: noninteractive
+
+    - name: Record pending package list
+      ansible.builtin.set_fact:
+        updated_packages: "{{ apt_simulate.stdout | regex_findall('(?m)^Inst (\\S+)') }}"
+
+    # The actual upgrade. Hosts with nothing pending come back unchanged.
+    # A failure is caught so this host still shows up in the reports below;
+    # the final play turns any failure back into a non-zero exit.
+    - name: Update & Upgrade via apt
+      block:
+        - name: Run apt dist-upgrade
+          ansible.builtin.apt:
+            update_cache: false
+            upgrade: dist
+          register: upgrade_result
+      rescue:
+        - name: Record the upgrade error
+          ansible.builtin.set_fact:
+            update_error: "{{ upgrade_result.msg | default('see the task output above') }}"
+
+    # update_status drives both the per-host report and the final summary:
+    #   updated     - packages were installed
+    #   up to date  - nothing was pending
+    #   failed      - the upgrade task itself errored out
+    - name: Record update status
+      ansible.builtin.set_fact:
+        update_status: >-
+          {{ 'failed' if upgrade_result is failed
+             else ('updated' if upgrade_result is changed else 'up to date') }}
+
+    - name: Print report
+      ansible.builtin.debug:
+        msg: |
+          ======================================================
+           HOST: {{ inventory_hostname }} ({{ ansible_host }}) - Ubuntu
+          ======================================================
+           STATUS: {{ update_status | upper }}{{ ' (CHECK MODE - nothing was applied)' if ansible_check_mode else '' }}
+          {% if update_status == 'updated' %}
+          ------------------------------------------------------
+           PACKAGES UPDATED: {{ updated_packages | length }}
+          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
+          {% elif update_status == 'failed' %}
+          ------------------------------------------------------
+           ERROR: {{ update_error | default('see the task output above') }}
+           PACKAGES STILL PENDING: {{ updated_packages | length }}
+          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
+          {% endif %}
+          ======================================================
+
+# Play 3: upgrade Rocky Linux hosts and record what each one needed.
 - name: Update and Upgrade - Rocky Linux
   hosts: rocky:&reachable_hosts
   become: true
   gather_facts: false
 
   tasks:
-    - name: Update & Upgrade via dnf
+    # Refresh the metadata first so the pending check below is accurate.
+    - name: Update dnf cache
       ansible.builtin.dnf:
-        name: "*"
-        state: latest
         update_cache: true
+      changed_when: false   # refreshing the metadata is not a change worth reporting
+      check_mode: false
+
+    # rc 100 means updates are available, rc 0 means none, so only other
+    # codes count as a failure.
+    - name: Check which packages are pending
+      ansible.builtin.command: dnf check-update
+      register: dnf_check
+      changed_when: false
+      check_mode: false
+      failed_when: dnf_check.rc not in [0, 100]
+
+    # Pull the "name.arch  version  repo" lines out of the check-update
+    # output. Anything under "Obsoleting Packages" is a duplicate listing,
+    # so the output is truncated there first.
+    - name: Record pending package list
+      ansible.builtin.set_fact:
+        updated_packages: >-
+          {{ dnf_check.stdout.split('Obsoleting Packages')[0].splitlines()
+             | select('match', '^\S+\.\S+\s+\S+\s+\S+$')
+             | map('regex_replace', '^(\S+)\s.*$', '\1') | list }}
+
+    # The actual upgrade. Hosts with nothing pending come back unchanged.
+    # A failure is caught so this host still shows up in the reports below;
+    # the final play turns any failure back into a non-zero exit.
+    - name: Update & Upgrade via dnf
+      block:
+        - name: Run dnf update
+          ansible.builtin.dnf:
+            name: "*"
+            state: latest
+            update_cache: false
+          register: upgrade_result
+      rescue:
+        - name: Record the upgrade error
+          ansible.builtin.set_fact:
+            update_error: "{{ upgrade_result.msg | default('see the task output above') }}"
+
+    # Same three states as the Ubuntu play above.
+    - name: Record update status
+      ansible.builtin.set_fact:
+        update_status: >-
+          {{ 'failed' if upgrade_result is failed
+             else ('updated' if upgrade_result is changed else 'up to date') }}
+
+    - name: Print report
+      ansible.builtin.debug:
+        msg: |
+          ======================================================
+           HOST: {{ inventory_hostname }} ({{ ansible_host }}) - Rocky Linux
+          ======================================================
+           STATUS: {{ update_status | upper }}{{ ' (CHECK MODE - nothing was applied)' if ansible_check_mode else '' }}
+          {% if update_status == 'updated' %}
+          ------------------------------------------------------
+           PACKAGES UPDATED: {{ updated_packages | length }}
+          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
+          {% elif update_status == 'failed' %}
+          ------------------------------------------------------
+           ERROR: {{ update_error | default('see the task output above') }}
+           PACKAGES STILL PENDING: {{ updated_packages | length }}
+          {{ updated_packages | join(', ') | default('(list unavailable)', true) }}
+          {% endif %}
+          ======================================================
+
+# Play 4: after both upgrade plays have run, print one consolidated summary of
+# which hosts were updated, which were already current, and which were missed.
+- name: Summary - Updated Hosts
+  hosts: localhost
+  connection: local
+  gather_facts: false
+
+  vars:
+    all_targets: "{{ groups['ubuntu'] | default([]) | union(groups['rocky'] | default([])) }}"
+    reachable: "{{ groups['reachable_hosts'] | default([]) }}"
+    # update_status of every host that got far enough to set it.
+    statuses: "{{ reachable | map('extract', hostvars, 'update_status') | select('defined') | list }}"
+    status_counts:
+      "updated": "{{ statuses | select('equalto', 'updated') | list | length }}"
+      "up to date": "{{ statuses | select('equalto', 'up to date') | list | length }}"
+      "failed": "{{ statuses | select('equalto', 'failed') | list | length }}"
+    failed_hosts: "{{ reachable | map('extract', hostvars) | selectattr('update_status', 'defined')
+                      | selectattr('update_status', 'equalto', 'failed')
+                      | map(attribute='inventory_hostname') | list }}"
+
+  tasks:
+    - name: Print hosts that were updated
+      ansible.builtin.debug:
+        msg: >-
+          UPDATED      {{ item }} -
+          {{ hostvars[item].updated_packages | default([]) | length }} package(s):
+          {{ hostvars[item].updated_packages | default([]) | join(', ') | default('(list unavailable)', true) }}
+      loop: "{{ reachable }}"
+      when: hostvars[item].update_status | default('') == 'updated'
+
+    - name: Print hosts that were already up to date
+      ansible.builtin.debug:
+        msg: "UP TO DATE   {{ item }} - no packages pending"
+      loop: "{{ reachable }}"
+      when: hostvars[item].update_status | default('') == 'up to date'
+
+    - name: Print hosts whose upgrade failed
+      ansible.builtin.debug:
+        msg: "FAILED       {{ item }} - the upgrade task reported an error, see the play output above"
+      loop: "{{ reachable }}"
+      when: hostvars[item].update_status | default('') == 'failed'
+
+    # Covers both hosts that failed the initial ping and any host that became
+    # unreachable part way through an upgrade play.
+    - name: Print hosts that were skipped
+      ansible.builtin.debug:
+        msg: "SKIPPED      {{ item }} - offline or unreachable, not upgraded"
+      loop: "{{ all_targets }}"
+      when: hostvars[item].update_status is not defined
+
+    - name: Print totals
+      ansible.builtin.debug:
+        msg: >-
+          {{ all_targets | length }} host(s) targeted,
+          {{ status_counts['updated'] }} updated,
+          {{ status_counts['up to date'] }} already up to date,
+          {{ status_counts['failed'] }} failed,
+          {{ all_targets | reject('in', reachable) | list | length }} skipped;
+          {{ reachable | map('extract', hostvars, 'updated_packages')
+             | select('defined') | map('length') | sum }}
+          package(s) updated in total.
+          {{ '(CHECK MODE - nothing was applied.)' if ansible_check_mode else '' }}
+
+    # Keep a failed upgrade visible to Semaphore: every report above has been
+    # printed by now, so it is safe to end the run non-zero here.
+    - name: Fail the run if any host could not be upgraded
+      ansible.builtin.fail:
+        msg: "Upgrade failed on: {{ failed_hosts | join(', ') }}"
+      when: failed_hosts | length > 0