refactor(script): List snapshots (#1)

README.md

@@ -1,3 +1,80 @@
 # zfs-pacman-hook
 
-Arch Linux pacman hook for automatic snapshots
+Arch Linux pacman hook for automatic ZFS snapshots
+
+# Setup
+
+Get started like so:
+
+1. Install dependency `jq`
+1. Clone repo into arbitrary path `<repo>`
+1. Make `pacman-zfs-snapshot.sh` executable
+    ```
+    chmod +x <repo>/pacman-zfs-snapshot.sh
+    ```
+1. Symlink to files, for example
+    ```
+    sudo ln -s <repo>/pacman-zfs-snapshot.sh /usr/local/bin/pacman-zfs-snapshot
+    sudo ln -s <repo>/pacman-zfs-snapshot-install.hook /usr/share/libalpm/hooks/pacman-zfs-snapshot-install.hook
+    sudo ln -s <repo>/pacman-zfs-snapshot-remove.hook /usr/share/libalpm/hooks/pacman-zfs-snapshot-remove.hook
+    sudo ln -s <repo>/pacman-zfs-snapshot-upgrade.hook /usr/share/libalpm/hooks/pacman-zfs-snapshot-upgrade.hook
+    sudo ln -s <repo>/pacman-zfs-snapshot.conf /etc/pacman-zfs-snapshot.conf
+    ```
+    Note that while you may choose arbitrary locations for symlinks the `pacman-zfs-snapshot-*.hook` files reference `/usr/local/bin/pacman-zfs-snapshot`. Change that accordingly if you need to.
+1. For datasets you want auto-snapshotted add property `space.quico:auto-snapshot=true`
+    ```
+    zfs set space.quico:auto-snapshot=true zpool/root/archlinux
+    ```
+    With any other property and any other value datasets will not be auto-snapshotted.
+1. Adjust `pacman-zfs-snapshot.conf` to your liking. You may want to set `do_dry_run='true'` for a start and just reinstall a benign package to get a feel for what this hook would do.
+
+# What's it do?
+
+In `pacman` on every `PreTransaction`, meaning right before any actual operation on a package begins, we trigger a ZFS snapshot. By default we identify the active system dataset by doing `findmnt / --noheadings --output source`. If exactly one source returns that is the exact name of a ZFS dataset in an imported zpool we create a snapshot on it. If no source returns we silently exit. If more than one source returns we raise an error and halt the `pacman` transaction.
+
+We retain two different snapshot chains, one for `pacman` transactions that only affect what we are calling _trivial_ packages and a separate chain for _important_ packages. By default only the exact regular expression package name match `^(linux|systemd|zfs-(dkms|utils))$` is considered important. Whenever an important package is affected by a transaction a snapshot goes into the corresponding chain. In all other cases - when an important package is not affected - snapshots go into the trivial chain.
+
+The _trivial_ snapshot chain by default keeps 15 snapshots, the _important_ chain keeps 5. The thought process here is that you will likely not futz around with a kernel every day whereas you may very well install arbitrary packages multiple times a day. Snapshots should keep you safe for a couple of days hence the defaults of 5 and 15 snapshots, respectively.
+
+Snapshots may look like so:
+```
+$ zfs list -o name -t all
+NAME                                                     ┌─── Important because systemd
+zpool                 snap_date_format='%F-%H%M'         |    is on our list of
+zpool/root                         ▼                     |    important packages
+zpool/root/archlinux        ┌─────────────┐             ▼▼▼
+zpool/root/archlinux@pacman_2023-03-07-0113_op:upgr_sev:imp_pkgs:systemd:bind:enchant:grep
+zpool/root/archlinux@pacman_2023-03-07-0113_op:upgr_sev:trv_pkgs:jdk17-temurin
+zpool/root/archlinux@pacman_2023-03-07-0115_op:upgr_sev:trv_pkgs:proton-ge-custom-bin
+                                               ▲▲▲▲     ▲▲▲      └────────────────────────────┘
+                                                 |       |         Max. 30 characters per our
+Pacman operation that triggered this snapshot ───┘       |          pacman-zfs-snapshot.conf
+                                                         |       setting 'pkgs_list_max_length'
+Severity based on affected packages, here trivial ───────┘
+```
+
+Have a look at `pacman-zfs-snapshot.conf` as well, its comments should be clear enough to get you going.
+
+# Development
+
+## Conventional commits
+
+This project uses [Conventional Commits](https://www.conventionalcommits.org/) for its commit messages.
+
+### Commit types
+
+Commit _types_ besides `fix` and `feat` are:
+
+- `build`: Project structure, directory layout, build instructions for roll-out
+- `refactor`: Keeping functionality while streamlining or otherwise improving function flow
+- `test`: Working on test coverage
+- `docs`: Documentation for project or components
+
+### Commit scopes
+
+The following _scopes_ are known for this project. A Conventional Commits commit message may optionally use one of the following scopes or none:
+
+- `conf`: How we deal with script config
+- `script`: Any other script work that doesn't specifically fall into the above scopes
+- `hook`: Configuring the hook(s)
+- `meta`: Affects the project's repo layout, readme content, file names etc.

pacman-zfs-snapshot-install.hook

@@ -0,0 +1,12 @@
+[Trigger]
+Operation = Install
+Type = Package
+Target = *
+
+[Action]
+Description = Create ZFS snapshot(s)
+When = PreTransaction
+Exec = /bin/sh -c 'while read -r f; do echo "$f"; done | /usr/local/bin/pacman-zfs-snapshot install'
+Depends = jq
+AbortOnFail
+NeedsTargets

pacman-zfs-snapshot-remove.hook

@@ -0,0 +1,12 @@
+[Trigger]
+Operation = Remove
+Type = Package
+Target = *
+
+[Action]
+Description = Create ZFS snapshot(s)
+When = PreTransaction
+Exec = /bin/sh -c 'while read -r f; do echo "$f"; done | /usr/local/bin/pacman-zfs-snapshot remove'
+Depends = jq
+AbortOnFail
+NeedsTargets

pacman-zfs-snapshot-upgrade.hook

@@ -0,0 +1,12 @@
+[Trigger]
+Operation = Upgrade
+Type = Package
+Target = *
+
+[Action]
+Description = Create ZFS snapshot(s)
+When = PreTransaction
+Exec = /bin/sh -c 'while read -r f; do echo "$f"; done | /usr/local/bin/pacman-zfs-snapshot upgrade'
+Depends = jq
+AbortOnFail
+NeedsTargets

pacman-zfs-snapshot.conf

@@ -0,0 +1,50 @@
+# Set to 'true' to do nothing and just print messages during pacman
+# operations. Helpful to get a feel for what these hooks do. This defaults
+# to 'false' so if you set this to an empty string or remove or uncomment it
+# in this conf file it'll equal 'false'.
+do_dry_run='false'
+
+# Pipe-separated list of package names we consider important. Will be
+# matched against regular expression ^(this_var_here)$. Snapshots taken
+# before a pacman transaction on an important package have a separate
+# retention from snapshots for trivial packages. Lends itself to keeping
+# high-risk updates separate from everything else.
+important_names='linux|systemd|zfs-(dkms|utils)'
+
+# Number snapshots to keep
+snaps_trivial_keep='15'
+snaps_important_keep='5'
+
+# Which suffix to use in snapshot names to identify snapshots before a
+# trivial pacman operation and before important pacman operations.
+snaps_trivial_suffix='trv'
+snaps_important_suffix='imp'
+
+# Snapshot name will contain list of affected packages trimmed to this many
+# max characters.
+pkgs_list_max_length='30'
+
+# Hook will by default snapshot all datasets that have the property
+# 'space.quico:auto-snapshot=true' set, even the ones that are not currently
+# mounted and may belong to unrelated operating systems. Set
+# snap_only_local_datasets='true' to limit snapshots to only those datasets
+# that have aforementioned property and at the same time are currently
+# mounted in your running OS. Currently mounted is defined as:
+# findmnt --json --list --output 'fstype,source,target' | \
+# jq --raw-output '.[][] | select(.fstype=="zfs") | .source'
+snap_only_local_datasets='true'
+
+# Which characters do we want to use to separate snapshot name fields
+snap_field_separator='_'
+# Prefix all our snapshots with this string to keep them separate from
+# snapshots done by any other means
+snap_name_prefix='pacman'
+# We do "$(date +<whatever>)" to put a timestamp into snapshot names.
+# Defaults to "$(date +'%F-%H%M')" which returns '2023-03-07-0050'.
+snap_date_format='%F-%H%M'
+
+# Which strings do we want to diffferentiate pacman operations Install,
+# Remove, Upgrade
+snap_op_installation_suffix='inst'
+snap_op_remove_suffix='rmvl'
+snap_op_upgrade_suffix='upgr'

pacman-zfs-snapshot.sh

@@ -0,0 +1,323 @@
+#!/bin/bash
+
+declare -a pkgs
+while read pkg; do
+    pkgs+=("${pkg}")
+done
+
+declare conf_file
+conf_file='/etc/pacman-zfs-snapshot.conf'
+
+declare important_names snaps_trivial_keep snaps_important_keep snaps_trivial_suffix snaps_important_suffix
+if [[ -r "${conf_file}" ]]; then
+    source "${conf_file}"
+fi
+
+# User-defined
+do_dry_run="${do_dry_run:-false}"
+important_names="${important_names:-linux|systemd|zfs-(dkms|utils)}"
+snaps_trivial_keep="${snaps_trivial_keep:-15}"
+snaps_important_keep="${snaps_important_keep:-5}"
+snaps_trivial_suffix="${snaps_trivial_suffix:-trv}"
+snaps_important_suffix="${snaps_important_suffix:-imp}"
+pkgs_list_max_length="${pkgs_list_max_length:-30}"
+snap_only_local_datasets="${snap_only_local_datasets:-true}"
+snap_field_separator="${snap_field_separator:-_}"
+snap_name_prefix="${snap_name_prefix:-pacman}"
+snap_date_format="${snap_date_format:-%F-%H%M}"
+snap_op_installation_suffix="${snap_op_installation_suffix:-inst}"
+snap_op_remove_suffix="${snap_op_remove_suffix:-rmvl}"
+snap_op_upgrade_suffix="${snap_op_upgrade_suffix:-upgr}"
+
+# Internal
+declare pkg_separator max_zfs_snapshot_name_length color_reset color_lyellow color_red
+pkg_separator=':'
+max_zfs_snapshot_name_length='255'
+color_reset='\e[0m'
+color_lyellow='\e[93m'
+color_red='\e[31m'
+
+declare operation conf_op_suffix
+operation="${1}"
+case "${operation}" in
+    install)
+        conf_op_suffix="${snap_op_installation_suffix}"
+        ;;
+    remove)
+        conf_op_suffix="${snap_op_remove_suffix}"
+        ;;
+    upgrade)
+        conf_op_suffix="${snap_op_upgrade_suffix}"
+        ;;
+esac
+
+function pprint () {
+    local style msg exit_code
+    style="${1:?}"
+    msg="${2:?}"
+    exit_code="${3}"
+
+    case "${style}" in
+        warn)
+            printf -- "${color_lyellow}"'[WARN]'"${color_reset}"' %s\n' "${msg}"
+            ;;
+        err)
+            printf -- "${color_red}"'[ERR]'"${color_reset}"' %s\n' "${msg}"
+            ;;
+        info)
+            printf -- '[INFO] %s\n' "${msg}"
+            ;;
+    esac
+
+    [[ "${exit_code}" ]] && exit "${exit_code}"
+}
+
+function split_pkgs_by_importance () {
+    local pkgs_in_transaction
+    pkgs_in_transaction=("${@}")
+    for pkg in "${pkgs_in_transaction[@]}"; do
+        if grep -Piq -- '^('"${important_names}"')$' <<<"${pkg}"; then
+            important_pkgs_in_transaction+=("${pkg}")
+        else
+            trivial_pkgs_in_transaction+=("${pkg}")
+        fi
+    done
+}
+
+function set_severity () {
+    if [[ "${#important_pkgs_in_transaction[@]}" -ge '1' ]]; then
+        severity="${snaps_important_suffix}"
+    else
+        severity="${snaps_trivial_suffix}"
+    fi
+}
+
+function get_globally_snappable_datasets () {
+    local datasets_list
+    # For all datasets show their 'space.quico:auto-snapshot' property; only
+    # print dataset name in column 1 and property value in column 2. In awk
+    # limit this list to datasets where tab-delimited column 2 has exact
+    # string '^true$' then further limit output by eliminating snapshots
+    # from list, i.e. dataset names that contain an '@' character.
+    datasets_list="$(zfs get -H -o 'name,value' 'space.quico:auto-snapshot' | \
+                     awk -F'\t' '{if($2 ~ /^true$/ && $1 !~ /@/) print $1}')"
+    while IFS= read -u10 -r dataset; do
+        globally_snappable_datasets+=("${dataset}")
+    done 10<<<"${datasets_list}"
+}
+
+function get_local_snappable_datasets () {
+    local datasets_list
+    datasets_list="$(findmnt --json --list --output 'fstype,source,target' | \
+                     jq --raw-output '.[][] | select(.fstype=="zfs") | .source')"
+    while IFS= read -u10 -r dataset; do
+        local_snappable_datasets+=("${dataset}")
+    done 10<<<"${datasets_list}"
+}
+
+function trim_globally_snappable_datasets () {
+    for global_dataset in "${globally_snappable_datasets[@]}"; do
+        for local_dataset in "${local_snappable_datasets[@]}"; do
+            if grep -Piq -- '^'"${local_dataset}"'$' <<<"${global_dataset}"; then
+                snappable_datasets+=("${global_dataset}")
+            fi
+        done
+    done
+}
+
+function write_pkg_list_oneline () {
+    if [[ "${severity}" == 'imp' ]]; then
+        for pkg in "${important_pkgs_in_transaction[@]}"; do
+            if [[ "${unabridged_pkg_list_oneline}" ]]; then
+                unabridged_pkg_list_oneline="${unabridged_pkg_list_oneline}${pkg_separator}${pkg}"
+            else
+                unabridged_pkg_list_oneline="${pkg}"
+            fi
+        done
+    fi
+    if [[ "${#trivial_pkgs_in_transaction[@]}" -ge '1' ]]; then
+        for pkg in "${trivial_pkgs_in_transaction[@]}"; do
+            if [[ "${unabridged_pkg_list_oneline}" ]]; then
+                unabridged_pkg_list_oneline="${unabridged_pkg_list_oneline}${pkg_separator}${pkg}"
+            else
+                unabridged_pkg_list_oneline="${pkg}"
+            fi
+        done
+    fi
+}
+
+function find_max_dataset_name_length () {
+    local longest_op_suffix op_suffix_string
+    longest_op_suffix='0'
+    for op_suffix in "${snap_op_installation_suffix}" "${snap_op_remove_suffix}" "${snap_op_upgrade_suffix}"; do
+        if [[ "${#op_suffix}" -gt "${longest_op_suffix}" ]]; then
+            longest_op_suffix="${#op_suffix}"
+        fi
+    done
+    op_suffix_string="$(head -c "${longest_op_suffix}" '/dev/zero' | tr '\0' '_')"
+
+    local longest_sev_suffix sev_suffix_string
+    longest_sev_suffix='0'
+    for sev_suffix in "${snaps_trivial_suffix}" "${snaps_important_suffix}"; do
+        if [[ "${#sev_suffix}" -gt "${longest_sev_suffix}" ]]; then
+            longest_sev_suffix="${#sev_suffix}"
+        fi
+    done
+    sev_suffix_string="$(head -c "${longest_sev_suffix}" '/dev/zero' | tr '\0' '_')"
+
+    local dataset_name_no_pkgs
+    max_dataset_name_length='0'
+    for dataset in "${snappable_datasets[@]}"; do
+        dataset_name_no_pkgs="${dataset}"'@'"${snap_name_prefix}${snap_field_separator}${date_string}${snap_field_separator}"'op:'"${op_suffix_string}${snap_field_separator}"'sev:'"${sev_suffix_string}${snap_field_separator}"'pkgs:'
+        if [[ "${#dataset_name_no_pkgs}" -gt "${max_dataset_name_length}" ]]; then
+            max_dataset_name_length="${#dataset_name_no_pkgs}"
+        fi
+    done
+
+    if [[ "${max_dataset_name_length}" -gt "${max_zfs_snapshot_name_length}" ]]; then
+        pprint 'warn' 'Snapshot name would exceed ZFS '"${max_zfs_snapshot_name_length}"' chars limit. Skipping snapshots ...' '0'
+    fi
+}
+
+function trim_pkg_list_oneline () {
+    local available_pkg_list_length
+    available_pkg_list_length="$((${max_zfs_snapshot_name_length} - ${max_dataset_name_length}))"
+    if [[ "${available_pkg_list_length}" -lt "${pkgs_list_max_length}" ]]; then
+        # If we have fewer characters available than the user wants limit
+        # package list length
+        pkgs_list_max_length="${available_pkg_list_length}"
+    fi
+
+    local shorter_pkg_list
+    shorter_pkg_list="${unabridged_pkg_list_oneline}"
+    while [[ "${#shorter_pkg_list}" -gt "${pkgs_list_max_length}" ]]; do
+        shorter_pkg_list="${shorter_pkg_list%${pkg_separator}*}"
+        if ! grep -Piq "${pkg_separator}" <<<"${shorter_pkg_list}"; then
+            # Only one package remains in package list, no need to continue
+            break
+        fi
+    done
+    if [[ "${#shorter_pkg_list}" -gt "${pkgs_list_max_length}" ]]; then
+        # If this is still too long we empty the package list
+        shorter_pkg_list=''
+    fi
+    trimmed_pkg_list_oneline="${shorter_pkg_list}"
+}
+
+function do_snaps () {
+    local snap_name snap_return_code
+    local -a planned_snaps
+    for snappable_dataset_id in "${!snappable_datasets[@]}"; do
+        snap_name="${snappable_datasets[${snappable_dataset_id}]}"'@'"${snap_name_prefix}${snap_field_separator}${date_string}${snap_field_separator}"'op:'"${conf_op_suffix}${snap_field_separator}"'sev:'"${severity}${snap_field_separator}"'pkgs:'"${trimmed_pkg_list_oneline}"
+        planned_snaps["${snappable_dataset_id}"]="${snap_name}"
+    done
+    if [[ "${do_dry_run}" == 'true' ]]; then
+        pprint 'info' 'Dry-run, pretending to atomically do zfs snapshot:'
+        for planned_snap in "${planned_snaps[@]}"; do
+            pprint 'info' '  '"${planned_snap}"
+        done
+    else
+        zfs snapshot "${planned_snaps[@]}"
+        snap_return_code="${?}"
+        if [[ "${snap_return_code}" -eq '0' ]]; then
+            successfully_snapped_datasets=("${snappable_datasets[@]}")
+            pprint 'info' 'zfs snapshot atomically done:'
+            for planned_snap in "${planned_snaps[@]}"; do
+                pprint 'info' '  '"${planned_snap}"
+            done
+        else
+            pprint 'warn' 'zfs snapshot failed:'
+            for planned_snap in "${planned_snaps[@]}"; do
+                pprint 'warn' '  '"${planned_snap}"
+            done
+        fi
+    fi
+}
+
+function get_snaps_in_cur_sev () {
+    local dataset_to_query
+    dataset_to_query="${1:?}"
+    snap_list="$(zfs list -H -o 'name' -t snapshot "${dataset_to_query}")"
+    snaps_done_by_us="$(grep -Pi -- '@'"${snap_name_prefix}${snap_field_separator}" <<<"${snap_list}")"
+    snaps_in_cur_sev="$(grep -Pi -- "${snap_field_separator}"'sev:'"${severity}${snap_field_separator}" <<<"${snaps_done_by_us}")"
+    printf -- '%s\n' "${snaps_in_cur_sev}"
+}
+
+function do_retention () {
+    local snap_list snaps_done_by_us snaps_in_cur_sev snaps_limit oldest_snap snap_return_code
+    local -a destroyed_snaps failed_to_destroy_snaps
+    if [[ "${do_dry_run}" == 'true' ]]; then
+        pprint 'info' 'Dry-run, skipping potential zfs destroy operations ...'
+    else
+        for successfully_snapped_dataset in "${successfully_snapped_datasets[@]}"; do
+            snaps_in_cur_sev="$(get_snaps_in_cur_sev "${successfully_snapped_dataset}")"
+            if [[ "${severity}" == "${snaps_important_suffix}" ]]; then
+                snaps_limit="${snaps_important_keep}"
+            else
+                snaps_limit="${snaps_trivial_keep}"
+            fi
+            while [[ "$(get_snaps_in_cur_sev "${successfully_snapped_dataset}" | wc -l)" -gt "${snaps_limit}" ]]; do
+                oldest_snap="$(get_snaps_in_cur_sev "${successfully_snapped_dataset}" | head -n1)"
+                zfs destroy "${oldest_snap}"
+                snap_return_code="${?}"
+                if [[ "${snap_return_code}" -eq '0' ]]; then
+                    destroyed_snaps+=("${oldest_snap}")
+                else
+                    failed_to_destroy_snaps+=("${oldest_snap}")
+                fi
+            done
+            if [[ "${#destroyed_snaps[@]}" -gt '0' ]]; then
+                pprint 'info' 'Oldest ZFS snapshot'"$([[ "${#failed_to_destroy_snaps[@]}" -gt '1' ]] && printf -- '%s' 's')"' in chain '"'"'sev:'"${severity}"''"'"' destroyed:'
+                for destroyed_snap in "${destroyed_snaps[@]}"; do
+                    pprint 'info' '  '"${destroyed_snap}"
+                done
+            fi
+            if [[ "${#failed_to_destroy_snaps[@]}" -gt '0' ]]; then
+                pprint 'warn' 'Failed to prune ZFS snapshot'"$([[ "${#failed_to_destroy_snaps[@]}" -gt '1' ]] && printf -- '%s' 's')"' in chain '"'"'sev:'"${severity}"''"'"':'
+                for failed_to_destroy_snap in "${failed_to_destroy_snaps[@]}"; do
+                    pprint 'warn' '  '"${failed_to_destroy_snap}"
+                done
+            fi
+        done
+    fi
+    return 0
+}
+
+function main () {
+    local pkgs_in_transaction
+    pkgs_in_transaction=("${@}")
+
+    local -a important_pkgs_in_transaction trivial_pkgs_in_transaction
+    split_pkgs_by_importance "${pkgs_in_transaction[@]}"
+
+    local severity
+    set_severity
+
+    local -a globally_snappable_datasets
+    get_globally_snappable_datasets
+
+    local -a snappable_datasets
+    if [[ "${snap_only_local_datasets}" == 'true' ]]; then
+        local local_snappable_datasets
+        get_local_snappable_datasets
+        trim_globally_snappable_datasets
+    else
+        snappable_datasets=("${globally_snappable_datasets}")
+    fi
+
+    local unabridged_pkg_list_oneline
+    write_pkg_list_oneline
+
+    local date_string max_dataset_name_length
+    date_string="$(date +"${snap_date_format}")"
+    find_max_dataset_name_length
+
+    local trimmed_pkg_list_oneline
+    trim_pkg_list_oneline
+
+    local -a successfully_snapped_datasets
+    do_snaps
+    do_retention
+}
+
+main "${pkgs[@]}"