ansible/playbooks/backup_docker.yml

---
# Backs up the docker host to the storage container: compose files and
# configs from compose_dir while everything runs, then the Docker volumes
# with the containers stopped so the copy is consistent.
#
# Containers in backup_skip_containers are never stopped. Semaphore runs on
# this host, so it and its database stay up (their volumes are copied live).
# root on the docker host needs an SSH key that is authorized on backup_host.
#
# Extra vars (optional):
#   backup_host            - inventory name of the destination (default storage)
#   backup_root            - directory on backup_host (default /mnt/storage/Docker_backups)
#   backup_skip_containers - default [semaphore, semaphore-db]

- name: Backup Docker compose files and volumes to storage
  hosts: docker
  become: true
  gather_facts: false

  vars:
    backup_host: storage
    backup_root: /mnt/storage/Docker_backups
    backup_addr: "{{ hostvars[backup_host].ansible_host }}"
    backup_skip_containers:
      - semaphore
      - semaphore-db
    compose_dir: /home/harry
    volumes_dir: /var/lib/docker/volumes
    rsync_ssh: ssh -o StrictHostKeyChecking=accept-new

  tasks:
    - name: Ensure backup destination directories exist
      ansible.builtin.file:
        path: "{{ backup_root }}/{{ item }}"
        state: directory
        mode: '0755'
      loop: [compose, volumes]
      delegate_to: "{{ backup_host }}"

    # rsync applies the first matching rule: excludes first, then the rule
    # that lets it descend into directories, then the wanted files, then
    # everything else is dropped.
    - name: Rsync compose files and configs (safe while running)
      ansible.builtin.command:
        argv:
          - rsync
          - -az
          - --delete
          - --prune-empty-dirs
          - -e
          - "{{ rsync_ssh }}"
          - --exclude=node_modules/
          - --exclude=.git/
          - --exclude=__pycache__/
          - --include=*/
          - --include=docker-compose.yml
          - --include=compose.yml
          - --include=.env
          - --include=*.conf
          - --include=*.yml
          - --include=*.yaml
          - --include=*.json
          - --exclude=*
          - "{{ compose_dir }}/"
          - "root@{{ backup_addr }}:{{ backup_root }}/compose/"
      changed_when: true

    - name: List running containers
      ansible.builtin.command:
        argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }}"]
      register: running_containers
      changed_when: false

    - name: Work out which containers to stop
      ansible.builtin.set_fact:
        containers_to_stop: "{{ running_containers.stdout_lines | difference(backup_skip_containers) }}"

    - name: Copy the volumes with the containers stopped
      block:
        - name: Stop containers
          ansible.builtin.command:
            argv: "{{ ['docker', 'stop'] + containers_to_stop }}"
          when: containers_to_stop | length > 0

        - name: Rsync Docker volumes to storage
          ansible.builtin.command:
            argv:
              - rsync
              - -az
              - --delete
              - -e
              - "{{ rsync_ssh }}"
              - "{{ volumes_dir }}/"
              - "root@{{ backup_addr }}:{{ backup_root }}/volumes/"
          changed_when: true

      always:
        - name: Start the containers that were running before
          ansible.builtin.command:
            argv: "{{ ['docker', 'start'] + containers_to_stop }}"
          when: containers_to_stop | length > 0

    - name: Verify containers are running
      ansible.builtin.command:
        argv: [docker, ps, --format, "{{ '{{' }}.Names{{ '}}' }} - {{ '{{' }}.Status{{ '}}' }}"]
      register: container_status
      changed_when: false

    - name: Backup summary
      ansible.builtin.debug:
        msg:
          - "Destination: {{ backup_host }} ({{ backup_addr }}):{{ backup_root }}"
          - "Stopped for the volume copy: {{ containers_to_stop | join(', ') if containers_to_stop else 'none' }}"
          - "Kept running: {{ backup_skip_containers | join(', ') }}"
          - "Containers running: {{ container_status.stdout_lines | length }} now, {{ running_containers.stdout_lines | length }} before"

    - name: Warn when fewer containers are running than before
      ansible.builtin.debug:
        msg: "WARNING: {{ item }} did not come back up on {{ inventory_hostname }}"
      loop: "{{ running_containers.stdout_lines | difference(container_status.stdout_lines | map('regex_replace', ' - .*$', '')) }}"