diff --git a/README.md b/README.md index 2732a91..14ce398 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ address, the first Ubuntu VM, and the service lists for `webserver`, `bot` and |----------|-------|----------------------| | `ping.yml` | all | connectivity check | | `health_check.yml` | all | disk and RAM report; Telegram alert for filesystems above `disk_warn_pct` (default 80) and offline hosts | -| `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update | +| `update_system.yml` | ubuntu, rocky | apt dist-upgrade / dnf update; reports per host whether it was updated (with the package list) or already up to date, plus a final summary. Supports `--check` as a dry run | | `bootstrap_lxc.yml` | ubuntu, rocky | base packages (EPEL and CRB enabled on Rocky), Ansible root key, sshd basics | | `add_root_key.yml` | all | add a personal public key to root | | `time-region.yml` | ubuntu, rocky | timezone, NTP on VMs, NTP disabled in LXC. `timezone_region`, `ntp_server` | diff --git a/playbooks/update_system.yml b/playbooks/update_system.yml index f278d64..4056f72 100644 --- a/playbooks/update_system.yml +++ b/playbooks/update_system.yml @@ -1,47 +1,257 @@ --- +# Full package upgrade for every reachable host: apt dist-upgrade on Ubuntu, +# dnf update on Rocky Linux. Each host is checked for pending packages before +# the upgrade runs, so the per-host and final reports show which systems were +# actually updated (and with what) and which were already up to date. +# +# Nothing reboots here; use reboot_check.yml for that. + +# Play 1: probe every ubuntu/rocky host and sort them into a "reachable_hosts" +# group so the two upgrade plays below can skip anything that's offline. - name: Check Host Availability hosts: ubuntu:rocky gather_facts: false ignore_unreachable: true tasks: + # Ping each host; failures/unreachables are swallowed so the play keeps + # going instead of aborting the whole run. - name: Ping hosts to check availability ansible.builtin.ping: register: ping_result ignore_errors: true ignore_unreachable: true + # Dynamically creates/populates the "reachable_hosts" group, used by the + # upgrade plays' host patterns (ubuntu:&reachable_hosts etc.). - name: Add reachable hosts to group ansible.builtin.group_by: key: reachable_hosts when: ping_result is reachable and ping_result is succeeded + # Print a message for any host that failed the ping, so offline hosts + # are visible in the run output and in the final summary. - name: Report offline hosts ansible.builtin.debug: - msg: "Host {{ inventory_hostname }} is OFFLINE and will be skipped" + msg: "{{ inventory_hostname }} - OFFLINE and will be skipped" when: ping_result is unreachable or ping_result is failed delegate_to: localhost +# Play 2: upgrade Ubuntu hosts and record what each one needed. - name: Update and Upgrade - Ubuntu hosts: ubuntu:&reachable_hosts become: true gather_facts: false tasks: - - name: Update & Upgrade via apt + # Refresh the package lists first so the pending check below is accurate. + # A cache refreshed within the last hour is reused. + - name: Update apt cache ansible.builtin.apt: update_cache: true cache_valid_time: 3600 - upgrade: dist + changed_when: false # refreshing the lists is not a change worth reporting + check_mode: false + # Simulate the upgrade (no changes made) to learn which packages are + # pending. Each candidate shows up as an "Inst ..." line. + - name: Check which packages are pending # noqa: command-instead-of-module (dry run, the apt module cannot simulate) + ansible.builtin.command: apt-get --just-print dist-upgrade + register: apt_simulate + changed_when: false + check_mode: false + environment: + DEBIAN_FRONTEND: noninteractive + + - name: Record pending package list + ansible.builtin.set_fact: + updated_packages: "{{ apt_simulate.stdout | regex_findall('(?m)^Inst (\\S+)') }}" + + # The actual upgrade. Hosts with nothing pending come back unchanged. + # A failure is caught so this host still shows up in the reports below; + # the final play turns any failure back into a non-zero exit. + - name: Update & Upgrade via apt + block: + - name: Run apt dist-upgrade + ansible.builtin.apt: + update_cache: false + upgrade: dist + register: upgrade_result + rescue: + - name: Record the upgrade error + ansible.builtin.set_fact: + update_error: "{{ upgrade_result.msg | default('see the task output above') }}" + + # update_status drives both the per-host report and the final summary: + # updated - packages were installed + # up to date - nothing was pending + # failed - the upgrade task itself errored out + - name: Record update status + ansible.builtin.set_fact: + update_status: >- + {{ 'failed' if upgrade_result is failed + else ('updated' if upgrade_result is changed else 'up to date') }} + + - name: Print report + ansible.builtin.debug: + msg: | + ====================================================== + HOST: {{ inventory_hostname }} ({{ ansible_host }}) - Ubuntu + ====================================================== + STATUS: {{ update_status | upper }}{{ ' (CHECK MODE - nothing was applied)' if ansible_check_mode else '' }} + {% if update_status == 'updated' %} + ------------------------------------------------------ + PACKAGES UPDATED: {{ updated_packages | length }} + {{ updated_packages | join(', ') | default('(list unavailable)', true) }} + {% elif update_status == 'failed' %} + ------------------------------------------------------ + ERROR: {{ update_error | default('see the task output above') }} + PACKAGES STILL PENDING: {{ updated_packages | length }} + {{ updated_packages | join(', ') | default('(list unavailable)', true) }} + {% endif %} + ====================================================== + +# Play 3: upgrade Rocky Linux hosts and record what each one needed. - name: Update and Upgrade - Rocky Linux hosts: rocky:&reachable_hosts become: true gather_facts: false tasks: - - name: Update & Upgrade via dnf + # Refresh the metadata first so the pending check below is accurate. + - name: Update dnf cache ansible.builtin.dnf: - name: "*" - state: latest update_cache: true + changed_when: false # refreshing the metadata is not a change worth reporting + check_mode: false + + # rc 100 means updates are available, rc 0 means none, so only other + # codes count as a failure. + - name: Check which packages are pending + ansible.builtin.command: dnf check-update + register: dnf_check + changed_when: false + check_mode: false + failed_when: dnf_check.rc not in [0, 100] + + # Pull the "name.arch version repo" lines out of the check-update + # output. Anything under "Obsoleting Packages" is a duplicate listing, + # so the output is truncated there first. + - name: Record pending package list + ansible.builtin.set_fact: + updated_packages: >- + {{ dnf_check.stdout.split('Obsoleting Packages')[0].splitlines() + | select('match', '^\S+\.\S+\s+\S+\s+\S+$') + | map('regex_replace', '^(\S+)\s.*$', '\1') | list }} + + # The actual upgrade. Hosts with nothing pending come back unchanged. + # A failure is caught so this host still shows up in the reports below; + # the final play turns any failure back into a non-zero exit. + - name: Update & Upgrade via dnf + block: + - name: Run dnf update + ansible.builtin.dnf: + name: "*" + state: latest + update_cache: false + register: upgrade_result + rescue: + - name: Record the upgrade error + ansible.builtin.set_fact: + update_error: "{{ upgrade_result.msg | default('see the task output above') }}" + + # Same three states as the Ubuntu play above. + - name: Record update status + ansible.builtin.set_fact: + update_status: >- + {{ 'failed' if upgrade_result is failed + else ('updated' if upgrade_result is changed else 'up to date') }} + + - name: Print report + ansible.builtin.debug: + msg: | + ====================================================== + HOST: {{ inventory_hostname }} ({{ ansible_host }}) - Rocky Linux + ====================================================== + STATUS: {{ update_status | upper }}{{ ' (CHECK MODE - nothing was applied)' if ansible_check_mode else '' }} + {% if update_status == 'updated' %} + ------------------------------------------------------ + PACKAGES UPDATED: {{ updated_packages | length }} + {{ updated_packages | join(', ') | default('(list unavailable)', true) }} + {% elif update_status == 'failed' %} + ------------------------------------------------------ + ERROR: {{ update_error | default('see the task output above') }} + PACKAGES STILL PENDING: {{ updated_packages | length }} + {{ updated_packages | join(', ') | default('(list unavailable)', true) }} + {% endif %} + ====================================================== + +# Play 4: after both upgrade plays have run, print one consolidated summary of +# which hosts were updated, which were already current, and which were missed. +- name: Summary - Updated Hosts + hosts: localhost + connection: local + gather_facts: false + + vars: + all_targets: "{{ groups['ubuntu'] | default([]) | union(groups['rocky'] | default([])) }}" + reachable: "{{ groups['reachable_hosts'] | default([]) }}" + # update_status of every host that got far enough to set it. + statuses: "{{ reachable | map('extract', hostvars, 'update_status') | select('defined') | list }}" + status_counts: + "updated": "{{ statuses | select('equalto', 'updated') | list | length }}" + "up to date": "{{ statuses | select('equalto', 'up to date') | list | length }}" + "failed": "{{ statuses | select('equalto', 'failed') | list | length }}" + failed_hosts: "{{ reachable | map('extract', hostvars) | selectattr('update_status', 'defined') + | selectattr('update_status', 'equalto', 'failed') + | map(attribute='inventory_hostname') | list }}" + + tasks: + - name: Print hosts that were updated + ansible.builtin.debug: + msg: >- + UPDATED {{ item }} - + {{ hostvars[item].updated_packages | default([]) | length }} package(s): + {{ hostvars[item].updated_packages | default([]) | join(', ') | default('(list unavailable)', true) }} + loop: "{{ reachable }}" + when: hostvars[item].update_status | default('') == 'updated' + + - name: Print hosts that were already up to date + ansible.builtin.debug: + msg: "UP TO DATE {{ item }} - no packages pending" + loop: "{{ reachable }}" + when: hostvars[item].update_status | default('') == 'up to date' + + - name: Print hosts whose upgrade failed + ansible.builtin.debug: + msg: "FAILED {{ item }} - the upgrade task reported an error, see the play output above" + loop: "{{ reachable }}" + when: hostvars[item].update_status | default('') == 'failed' + + # Covers both hosts that failed the initial ping and any host that became + # unreachable part way through an upgrade play. + - name: Print hosts that were skipped + ansible.builtin.debug: + msg: "SKIPPED {{ item }} - offline or unreachable, not upgraded" + loop: "{{ all_targets }}" + when: hostvars[item].update_status is not defined + + - name: Print totals + ansible.builtin.debug: + msg: >- + {{ all_targets | length }} host(s) targeted, + {{ status_counts['updated'] }} updated, + {{ status_counts['up to date'] }} already up to date, + {{ status_counts['failed'] }} failed, + {{ all_targets | reject('in', reachable) | list | length }} skipped; + {{ reachable | map('extract', hostvars, 'updated_packages') + | select('defined') | map('length') | sum }} + package(s) updated in total. + {{ '(CHECK MODE - nothing was applied.)' if ansible_check_mode else '' }} + + # Keep a failed upgrade visible to Semaphore: every report above has been + # printed by now, so it is safe to end the run non-zero here. + - name: Fail the run if any host could not be upgraded + ansible.builtin.fail: + msg: "Upgrade failed on: {{ failed_hosts | join(', ') }}" + when: failed_hosts | length > 0