feat(os): Replace manually building ZBM UEFI image with AUR package (#3)

Testing shows 100 MiB is a bit tight with a backup
UEFI image and future growth.

README.md

@@ -15,10 +15,10 @@ We expect minimal prep on your end. Please make sure that before execution the f
 
 On a blank example disk `/dev/sda` you can fulfill the requirements (One `EF00` partition with a file system plus one `BF00` partition) for example like so:
 ```
-sgdisk --new '1::+100M' --new '2' --typecode '1:EF00' --typecode '2:BF00' /dev/sda
+sgdisk --new '1::+512M' --new '2' --typecode '1:EF00' --typecode '2:BF00' /dev/sda
 mkfs.vfat /dev/sda1
 ```
-> `--new '1::+100M'`: Create partition number `1`. The field separator `:` separates the partition number from start sector. In this case start sector is unspecified so start sector sits at whatever the system's default is for this operation. On a blank disk on an Arch Linux live CD ISO image this will default to sector `2048`. Partition ends at whatever the beginning is `+100M` meaning plus 100 Mebibytes.
+> `--new '1::+512M'`: Create partition number `1`. The field separator `:` separates the partition number from start sector. In this case start sector is unspecified so start sector sits at whatever the system's default is for this operation. On a blank disk on an Arch Linux live CD ISO image this will default to sector `2048`. Partition ends at whatever the beginning is `+512M` meaning plus 512 Mebibytes.
 >
 > `--new '2'`: Create partition number `2`. Both field number 2, the start sector, and field number 3, the end sector, are unspecified, there's no field separator `:`. Field number 2 will be the first free sector - in this case right after partition 1 - and field number 3 will be end of disk. Thus partition `2` will fill the remaining free disk space.
 >
@@ -33,7 +33,7 @@ NAME         MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS
 /dev/loop0     7:0    0 688.5M  1 loop /run/archiso/airootfs
 /dev/sr0      11:0    1 810.3M  0 rom  /run/archiso/bootmnt
 /dev/sda     202:0    0    10G  0 disk 
-├─/dev/sda1  202:1    0   100M  0 part
+├─/dev/sda1  202:1    0   512M  0 part
 └─/dev/sda2  202:2    0   9.9G  0 part 
 ```
 
@@ -109,7 +109,7 @@ After installation you're going to want to at least touch these points in your n
 - Passwords
     - ZFS: The password for all datasets underneath `zpool` is `password`.
     - Local `root` account: The local `root` account's password is `password`.
-- Arch User Repository (AUR) helper: We installed [paru](https://github.com/Morganamilo/paru) as our AUR helper, we installed from GitHub via `makepkg -si`.
+- Arch User Repository (AUR) helper: We installed [paru](https://github.com/Morganamilo/paru) as our AUR helper, we installed from GitHub via `makepkg -si` then replaced itself with its [paru-bin](https://aur.archlinux.org/packages/paru-bin) version from AUR.
 - In `/etc/systemd/network/50-wired.network` instead of a DHCP-based network config you can get a static one. The DHCP-based one for reference looks like:
     ```
     ...
@@ -426,7 +426,7 @@ The following _scopes_ are known for this project. A Conventional Commits commit
 - `iso`: Changing Arch Linux ISO CD
 - `zbm`: Adjusting ZFSBootMenu's behavior
 - `zfs`: A change to how ZFS interacts with the system, either a pool or a dataset
-- `os`: Getting an perating system set up to correctly work in a ZFS boot environment
+- `os`: Getting an operating system set up to correctly work in a ZFS boot environment
 - `meta`: Affects the project's repo layout, readme content, file names etc.
 
 # Credits

setup.sh

@@ -13,10 +13,6 @@ trap '[ "$?" -ne 77 ] || exit 77' ERR
 
 declare zpool_drive efi_drive
 
-function set_ntp () {
-    timedatectl set-ntp true
-}
-
 function we_are_changerooted () {
     if [ "$(stat -c '%d:%i' '/')" != "$(stat -c '%d:%i' '/proc/1/root/.')" ]; then
         return 0
@@ -25,7 +21,13 @@ function we_are_changerooted () {
     fi
 }
 
+function set_ntp () {
+    #1.1
+    timedatectl set-ntp true
+}
+
 function update_pacman_db () {
+    #1.2
     printf -- '%s\n' 'Refreshing mirror list ...'
     systemctl start reflector
     # In an ISO and for the minimal number of packages we need we do not
@@ -34,11 +36,62 @@ function update_pacman_db () {
 }
 
 function install_pkgs () {
+    #1.3
     printf -- '%s\n' 'Installing packages ...'
     pacman -S --needed --noconfirm "${@}"
 }
 
+function prepare_zfsbootmenu_efi_bin_pkg () {
+    #1.4
+    # Our Arch Linux will use prebuilt ZFSBootMenu UEFI image files for ease
+    # of use. We'd like to install them from within our chroot but the AUR
+    # package 'zfsbootmenu-efi-bin' that we're using currently (Friday,
+    # October 20, 2023) identifies its target EFI system partition (ESP) by
+    # doing 'lsblk' and checking the partition type name. Since within
+    # arch-chroot there's no '/run/udev' but only an empty '/run' tmpfs
+    # 'lsblk' cannot see partition type names and some other pieces of info.
+    #
+    # See also
+    # https://gitlab.archlinux.org/archlinux/arch-install-scripts/-/issues/24
+    #
+    # Thus within arch-chroot the installation of 'zfsbootmenu-efi-bin' and
+    # even 'makepkg -s' fails. We build the package in our Arch Linux live
+    # CD ISO image and copy the resulting file into our chroot.
+    #
+    # We circle back to the package file when we chroot into Arch and
+    # install_zbm_image().
+    pacman_cache_dir='/var/cache/pacman/pkg'
+    chroot_pacman_cache_dir='/mnt'"${pacman_cache_dir}"
+    git_zbm_efi_bin_dir='/tmp/zfsbootmenu-efi-bin'
+    sudo -u 'arch' git -C '/tmp' clone 'https://aur.archlinux.org/zfsbootmenu-efi-bin.git'
+
+    # We briefly bind-mount our ESP into '/efi' before we build the package.
+    # Its PKGBUILD will then find the ESP at the correct location and write
+    # a package file with that internal location ('/efi').
+    mkdir -p '/efi'
+    mount --bind '/mnt/efi' '/efi'
+    pushd "${git_zbm_efi_bin_dir}"
+    sudo -u 'arch' makepkg -s || {
+        printf -- '%s\n' 'Failed building zfsbootmenu-efi-bin package. Exiting ...'
+        exit 77
+    }
+    popd
+    umount '/efi'
+    rmdir '/efi'
+
+    package_file_abs="$(find "${git_zbm_efi_bin_dir}" -type f -iname '*pkg.tar.zst')"
+    package_file="$(basename "${package_file_abs}")"
+    mkdir -p "${chroot_pacman_cache_dir}"
+    rsync -av "${package_file_abs}" "${chroot_pacman_cache_dir}"'/'"${package_file}" || {
+        printf -- '%s\n' 'Failed rsyncing zfsbootmenu-efi-bin package file into chroot. Exiting ...'
+        exit 77
+    }
+    export ZFSBOOTMENU_EFI_BIN_PKG_PATH="${pacman_cache_dir}"'/'"${package_file}"
+    rm -rf "${git_zbm_efi_bin_dir}"
+}
+
 function install_zfs () {
+    #1.5
     declare reset_colors='\033[0m'
     curl -s 'https://raw.githubusercontent.com/eoli3n/archiso-zfs/master/init' | bash
     printf -- "${reset_colors}"
@@ -209,6 +262,7 @@ function export_pool () {
 }
 
 function setup_zpool () {
+    #1.6
     local drive_by_id
     zpool_drive="$(select_part 'zfs')"
     drive_by_id="$(get_drive_id "${zpool_drive}")"
@@ -228,6 +282,7 @@ function setup_zpool () {
 }
 
 function mount_system () {
+    #1.7
     zfs mount "${zpool_name}"'/root/'"${zfs_arch_dataset_name}"
     zfs mount -a
 
@@ -240,12 +295,16 @@ function mount_system () {
 }
 
 function copy_zpool_cache () {
+    #1.8
     mkdir -p '/mnt/etc/zfs'
     zpool set 'cachefile=/etc/zfs/'"${zpool_name}"'.cache' "${zpool_name}"
 }
 
 function pacman_dl_parallel () {
-    sed -ri -e 's'$'\x1''^.*?(ParallelDownloads)[^\r\n\f]*'$'\x1''\1 = 5'$'\x1''g' '/etc/pacman.conf'
+    #2.4
+    # We're setting this in Arch Linux ISO CD while we install proper Arch.
+    # No need to revert this later as it is ephemeral anyway.
+    sed -ri -e 's'$'\x1''^.*?(ParallelDownloads)[^\r\n\f]*'$'\x1''\1 = 20'$'\x1''g' '/etc/pacman.conf'
 }
 
 function pacman_dont_check_space () {
@@ -256,10 +315,14 @@ function pacman_dont_check_space () {
     # overestimates required disk space. We instead assume an installation
     # gets done with at least a 10 GiB drive which is plenty. Skip pacman's
     # size check.
+    #
+    # We're setting this in Arch Linux ISO CD while we install proper Arch.
+    # No need to revert this later as it is ephemeral anyway.
     sed -ri -e 's'$'\x1''^.*?(CheckSpace)([^\r\n\f]*)'$'\x1''#\1\2'$'\x1''g' '/etc/pacman.conf'
 }
 
 function install_archlinux () {
+    #1.9
     pacman_dl_parallel
     pacman_dont_check_space
     pacstrap /mnt              \
@@ -286,6 +349,7 @@ function install_archlinux () {
 }
 
 function gen_fstab () {
+    #1.10
     genfstab -U /mnt | grep -v "${zpool_name}" | tr -s '\n' | sed -r -e 's/\/mnt//' -e '/./,$!d' > '/mnt/etc/fstab'
 }
 
@@ -298,6 +362,7 @@ EOF
 }
 
 function set_hostname () {
+    #1.11
     declare new_hostname
     install_pkgs 'pwgen'
     new_hostname="$(pwgen --no-numerals --no-capitalize --ambiguous 8)"
@@ -306,6 +371,7 @@ function set_hostname () {
 }
 
 function set_locale () {
+    #1.12
     printf -- '%s\n' \
         'KEYMAP=de-latin1' \
         'FONT=Lat2-Terminus16' \
@@ -316,6 +382,7 @@ function set_locale () {
 }
 
 function add_zfs_hook_to_initramfs () {
+    #1.13
     # Add zfs hook, remove fsck hook from initramfs.
     sed -ri \
         -e 's'$'\x1''(HOOKS=)(.*?[\(| ])(filesystems)([\)| ][^\r\n\f]*)'$'\x1''\1\2zfs \3\4'$'\x1''g' \
@@ -330,6 +397,7 @@ function add_zfs_hook_to_initramfs () {
 }
 
 function set_initramfs_build_list () {
+    #1.14
     # No need to build fallback initramfs, our new fallback is ZFS snapshots
     sed -ri \
         -e '/^#/d' \
@@ -343,12 +411,14 @@ function set_initramfs_build_list () {
 }
 
 function add_zfs_files_to_new_os () {
+    #1.15
     for zfs_file in '/etc/hostid' '/etc/zfs/zpool.cache' $([[ ! "${ARCHZBM_ZFSPROPS_NO_ENCRYPTION}" ]] && printf -- '%s' '/etc/zfs/'"${zpool_name}"'.key'); do
         rsync -av --itemize-changes {'','/mnt'}"${zfs_file}"
     done
 }
 
 function enter_chroot () {
+    #2.1
     arch-chroot /mnt /bin/bash -xe <<EOF
 curl --silent '${this_script_url}' | bash
 EOF
@@ -414,6 +484,7 @@ function create_unpriv_user () {
 }
 
 function unleash_makepkg () {
+    #2.5
     local path_prefix
     path_prefix='/mnt'
     if we_are_changerooted; then
@@ -425,6 +496,7 @@ function unleash_makepkg () {
 }
 
 function get_aur_helper () {
+    #2.7
     create_unpriv_user 'build'
     usermod --append --groups 'wheel' 'build'
     printf -- '%s\n' '%wheel ALL=(ALL:ALL) NOPASSWD: ALL' > '/etc/sudoers.d/10-wheel-group-no-passwd-prompt'
@@ -440,10 +512,60 @@ function get_aur_helper () {
 }
 
 function paru_install () {
-    sudo --user build paru -S --noconfirm "${@}"
+    declare -a paru_install_packages
+    [[ "${1}" ]] && while :; do
+        case "${1}" in
+            -[[:alnum:]]*)
+                printf -- '%s\n' \
+                    'Short-form argument '"'${1}'"' not supported for function '"'${FUNCNAME[0]}()'"'. Only known accepted argument' \
+                    'is '"'"'--replace-conflicting'"'"' without a value given. Exiting ...'
+                exit 77
+                ;;
+            --replace-conflicting)
+                pacman_force_yes='true'
+                shift
+                continue
+                ;;
+            --*)
+                printf -- '%s\n' \
+                    'Long-form argument '"'${1}'"' not supported for function '"'${FUNCNAME[0]}()'"'. Only known accepted argument' \
+                    'is '"'"'--replace-conflicting'"'"' without a value given. Exiting ...'
+                exit 77
+                ;;
+            '')
+                # All arguments processed
+                break
+                ;;
+            *)
+                paru_install_packages+=("${1}")
+                shift
+                ;;
+            esac
+    done || {
+        printf -- '%s\n' \
+            'No argument '"'${1}'"' given for function '"'${FUNCNAME[0]}'"'. Exiting ...'
+        exit 77
+    }
+    if [[ "${pacman_force_yes}" ]]; then
+        yes 'y' | sudo --user build paru -S "${paru_install_packages[@]}"
+        unset -v pacman_force_yes
+    else
+        sudo --user build paru -S --noconfirm "${paru_install_packages[@]}"
+    fi
+}
+
+function install_zbm_image () {
+    #2.8
+    # This takes image files written earlier in our Arch Linux live CD ISO
+    # image at prepare_zfsbootmenu_efi_bin_pkg() and installs them via their
+    # locally built package file. When done we delete the manually built
+    # package file from pacman's cache dir.
+    pacman -U "${ZFSBOOTMENU_EFI_BIN_PKG_PATH}" --noconfirm
+    rm "${ZFSBOOTMENU_EFI_BIN_PKG_PATH}"
 }
 
 function keep_initiramfs_root_only_rw () {
+    #2.3
     declare systemd_local_admin_override_path unit_name
     systemd_local_admin_override_path='/etc/systemd/system'
     unit_name='chmod-initramfs'
@@ -475,6 +597,7 @@ EOF
 }
 
 function add_motd_getting_started_msg () {
+    #2.6
     cat > '/etc/motd' <<"EOF"
 
 ####################
@@ -490,6 +613,7 @@ EOF
 }
 
 function install_os_in_chroot () {
+    #2.2
     ### Reinit keyring
     # As keyring is initialized at boot, and copied to the install dir with pacstrap, and ntp is running
     # Time changed after keyring initialization, it leads to malfunction
@@ -499,33 +623,29 @@ function install_os_in_chroot () {
     pacman-key --populate archlinux
     pacman -S archlinux-keyring --noconfirm
 
-    keep_initiramfs_root_only_rw
+    keep_initiramfs_root_only_rw                #2.3
-    pacman_dl_parallel
+    pacman_dl_parallel                          #2.4
-    unleash_makepkg
+    unleash_makepkg                             #2.5
-    add_motd_getting_started_msg
+    add_motd_getting_started_msg                #2.6
-    get_aur_helper
+    get_aur_helper                              #2.7
-    paru_install 'zfs-dkms' 'zfs-utils'
+    paru_install --replace-conflicting 'paru-bin'
+    paru_install 'zfs-dkms' 'zfs-utils' 'rsync'
     hwclock --systohc
     locale-gen
     source /etc/locale.conf
     mkinitcpio -P
 
-    # Install ZFSBootMenu and deps
+    # Install ZFSBootMenu image
-    git clone --depth=1 https://github.com/zbm-dev/zfsbootmenu/ '/tmp/zfsbootmenu'
+    install_zbm_image                           #2.8
-    paru_install 'cpanminus' 'kexec-tools' 'fzf' 'util-linux'
-    pushd '/tmp/zfsbootmenu'
-    make
-    make install
-    cpanm --notest --installdeps .
-    popd
-    rm -rf '/tmp/zfsbootmenu'
 }
 
 function set_root_pw () {
-    printf -- '%s\n' 'root:password' | chpasswd --root '/mnt'
+    #3.2
+    printf -- '%s\n' 'root:password' | chpasswd --crypt-method 'SHA512' --root '/mnt'
 }
 
 function configure_networking () {
+    #3.3
     cat > '/mnt/etc/systemd/network/50-wired.network' <<"EOF"
 [Match]
 Name=en*
@@ -543,6 +663,7 @@ EOF
 }
 
 function configure_dns () {
+    #3.4
     rm '/mnt/etc/resolv.conf'
     ln -s '/run/systemd/resolve/stub-resolv.conf' '/mnt/etc/resolv.conf'
 
@@ -553,14 +674,17 @@ function configure_dns () {
 }
 
 function configure_reflector () {
-    systemctl enable 'reflector' --root='/mnt'
+    #3.5
+    systemctl enable 'reflector.service' 'reflector.timer' --root='/mnt'
 }
 
 function configure_zfs () {
+    #3.6
     systemctl enable 'zfs-import-cache' 'zfs-mount' 'zfs-import.target' 'zfs.target' --root='/mnt'
 }
 
 function configure_zfs_mount_gen () {
+    #3.7
     mkdir -p '/mnt/etc/zfs/zfs-list.cache'
     touch '/mnt/etc/zfs/zfs-list.cache/'"${zpool_name}"
     zfs list -H -o name,mountpoint,canmount,atime,relatime,devices,exec,readonly,setuid,nbmand | sed 's/\/mnt//' > '/mnt/etc/zfs/zfs-list.cache/'"${zpool_name}"
@@ -568,36 +692,10 @@ function configure_zfs_mount_gen () {
 }
 
 function configure_zfsbootmenu () {
-    curl -s 'https://raw.githubusercontent.com/zbm-dev/zfsbootmenu/master/etc/zfsbootmenu/mkinitcpio.conf' | sed -r -e '/^#/d' -e '/^$/d' > '/mnt/etc/zfsbootmenu/mkinitcpio.conf'
+    #3.8
-    cat > '/mnt/etc/zfsbootmenu/config.yaml' <<EOF
-Global:
-  ManageImages: true
-  BootMountPoint: /efi
-  InitCPIO: true
-
-Components:
-  Enabled: false
-EFI:
-  ImageDir: /efi/EFI/ZBM
-  Versions: false
-  Enabled: true
-Kernel:
-  CommandLine: ro loglevel=0 zbm.import_policy=hostid
-  Prefix: vmlinuz
-EOF
-# Up here maybe 'ro quiet' instead of 'ro'
     zfs set org.zfsbootmenu:commandline='rw nowatchdog rd.vconsole.keymap=de-latin1' "${zpool_name}"'/root/'"${zfs_arch_dataset_name}"
 }
 
-function gen_zfsbootmenu () {
-    arch-chroot /mnt /bin/bash -xe <<"EOF"
-source /etc/locale.conf
-mkdir -p '/efi/EFI/ZBM'
-find /efi/EFI/ZBM -type f -delete
-generate-zbm
-EOF
-}
-
 function get_disks_with_one_efipart () {
     local disks_with_one_efipart
     # Find disks that have exactly one EFI partition and where that EFI
@@ -612,6 +710,7 @@ function get_disks_with_one_efipart () {
 }
 
 function add_zbm_to_efi () {
+    #3.9
     if ! efibootmgr | grep -Piq -- 'ZFSBootMenu'; then
         local efi_disks_list
         efi_disks_list="$(get_disks_with_one_efipart)"
@@ -627,45 +726,47 @@ function add_zbm_to_efi () {
 }
 
 function umount_all () {
+    #3.10
     umount '/mnt/efi'
     zfs umount -a
     zpool export "${zpool_name}"
 }
 
 function finalize_os_setup () {
-    set_root_pw
+    #3.1
-    configure_networking
+    set_root_pw                                 #3.2
-    configure_dns
+    configure_networking                        #3.3
-    configure_reflector
+    configure_dns                               #3.4
-    configure_zfs
+    configure_reflector                         #3.5
-    configure_zfs_mount_gen
+    configure_zfs                               #3.6
-    configure_zfsbootmenu
+    configure_zfs_mount_gen                     #3.7
-    gen_zfsbootmenu
+    configure_zfsbootmenu                       #3.8
-    add_zbm_to_efi
+    add_zbm_to_efi                              #3.9
-    umount_all
+    umount_all                                  #3.11
 }
 
 function main () {
     if we_are_changerooted; then
-        install_os_in_chroot
+        install_os_in_chroot                    #2.2
     else
-        set_ntp
+        set_ntp                                 #1.1
-        update_pacman_db
+        update_pacman_db                        #1.2
-        install_pkgs 'jq'
+        install_pkgs 'base-devel' 'git' 'jq'    #1.3
-        install_zfs
+        prepare_zfsbootmenu_efi_bin_pkg         #1.4
-        setup_zpool
+        install_zfs                             #1.5
-        mount_system
+        setup_zpool                             #1.6
-        copy_zpool_cache
+        mount_system                            #1.7
-        install_archlinux
+        copy_zpool_cache                        #1.8
-        gen_fstab
+        install_archlinux                       #1.9
-        set_hostname
+        gen_fstab                               #1.10
-        set_locale
+        set_hostname                            #1.11
-        add_zfs_hook_to_initramfs
+        set_locale                              #1.12
-        set_initramfs_build_list
+        add_zfs_hook_to_initramfs               #1.13
-        add_zfs_files_to_new_os
+        set_initramfs_build_list                #1.14
-        enter_chroot
+        add_zfs_files_to_new_os                 #1.15
+        enter_chroot                            #2.1
         # We're done in chroot
-        finalize_os_setup
+        finalize_os_setup                       #3.1
     fi
 }