fix(build): Keep own files out of example repo

.gitignore

@@ -2,6 +2,9 @@
 *.retry
 ansible/hosts.yml
 ansible/group_vars/all/*
+ansible/roles/10-include-50-miniconda/defaults/*
+ansible/roles/10-include-50-miniconda/files/etc/systemd/system/*
+ansible/roles/10-include-50-miniconda/files/opt/python/update-firewall-source/master/config.ini
 ansible/roles/20-common-20-ssh/defaults/*
 ansible/roles/20-common-20-ssh/files/root/.ssh/*
 !*.example

README.md

@@ -2,7 +2,7 @@
 
 Set up infrastructure for all things e-mail
 
-## Prep
+# Prep
 
 On your Ansible controller make sure the `sshpass` binary exists if Ansible has to connect to target machines via SSH username-password authentication instead of SSH key authentication. The binary usually comes with a package of the same name.
 
@@ -11,12 +11,12 @@ We're assuming that you're running Ansible as a Python package inside a virtual
 pip install ansible
 ```
 
-We're also assuming that secrets are stored in a HashiCorp Vault instance to which you have access. These role access Vault via `hvac`, the HashiCorp Vault API client for Python 3.x, see [github.com/hvac/hvac](https://github.com/hvac/hvac) for reference. Install it like so:
+We're also assuming that secrets are stored in a HashiCorp Vault instance to which you have access. These roles access Vault via `hvac`, the HashiCorp Vault API client for Python 3.x, see [github.com/hvac/hvac](https://github.com/hvac/hvac) for reference. Install it like so:
 ```
 pip install hvac
 ```
 
-## Vars
+# Vars
 
 * For default variables copy [ansible/roles/20-common-20-ssh/defaults/main.yml.example](ansible/roles/20-common-20-ssh/defaults/main.yml.example) to a proper `ansible/roles/20-common-20-ssh/defaults/main.yml`, adjust as needed.
 
@@ -26,7 +26,7 @@ pip install hvac
 
 * In [ansible/roles/20-common-20-ssh/files/root/.ssh](ansible/roles/20-common-20-ssh/files/root/.ssh) copy both [authorized_keys.example](ansible/roles/20-common-20-ssh/files/root/.ssh/authorized_keys.example) and [known_hosts.example](ansible/roles/20-common-20-ssh/files/root/.ssh/known_hosts.example) to proper files. They contain SSH authorized_keys and public SSH host keys you want installed on target machines.
 
-## Vault structure
+# Vault structure
 
 In Vault we're assuming that every host in your inventory has a secret stored that contains at least the following keys:
 
@@ -36,7 +36,7 @@ In Vault we're assuming that every host in your inventory has a secret stored th
 
 For an example server `fully.qualified.domain.name` and example user `root` Vault secrets are located at `name/domain/qualified/fully/os/root/creds`. Per [ansible/group_vars/all/vars.yml.example](ansible/group_vars/all/vars.yml.example) the default secrets engine mount point is `kv` where this playbook expects a kv secrets engine.
 
-## Run it
+# Run it
 
 On first run execute it like so:
 ```
@@ -49,3 +49,24 @@ On subsequent runs like so:
 ```
 ansible-playbook --inventory hosts.yml playbook.yml
 ```
+
+## first_run
+
+To make sure a `first_run` correctly runs all regular tasks plus `first_run`-specific tasks [ansible/roles/20-common-20-ssh/tasks/main.yml](ansible/roles/20-common-20-ssh/tasks/main.yml) has its `import_tasks` tagged as `first_run`.
+
+```
+- import_tasks: '20-ssh.yml'
+  tags:
+    - 'first_run'
+```
+
+As a result all tasks in that import inherit `first_run`. Two of them, however, must only run during `first_run` and never during normal operation. They get:
+
+```
+- name: '...'
+  tags:
+    - 'first_run'
+    - 'never'
+```
+
+This overrides the single `first_run` inherited tag. Ansible special tag `never` ensures that these particular tasks are _**never**_ executed unless `first_run` is called. Tasks outside of `20-common-20-ssh` role are not tagged with `first_run` at all so are ignored during a `first_run`.

ansible/group_vars/all/vars.yml.example

@@ -1,6 +1,21 @@
+# Most roles
 ansible_user: 'root'
 ansible_hashi_vault_auth_method: 'token'
 ansible_hashi_vault_token: 'hvs.xxxxxxxxxx'
 ansible_hashi_vault_engine_mount_point: 'kv'
 ansible_hashi_vault_token_validate: 'false'
 ansible_hashi_vault_url: 'http://localhost:8200/'
+
+# 10-include-50-miniconda
+python_git_packages:
+ - repo_url: 'https://f.q.d.n/repo/my-project.git'
+   path: '{{ python_git_install_path }}/my-project'
+   branch: 'master'
+   config_file: 'opt/python/my-project/master/config.ini'
+   systemd:
+      service: 'etc/systemd/system/my-project.service'
+      timer: 'etc/systemd/system/my-project.timer'
+python_pip_packages:
+ - { env_name: 'cookiecutter', packages: ['cookiecutter', 'pip-tools'] }
+python_pip_requirements:
+ - { env_name: 'my-project', requirements: '{{ python_git_install_path }}/my-project/master/requirements.txt' }

ansible/roles/10-include-50-miniconda/defaults/main.yml.example

@@ -0,0 +1,13 @@
+miniconda_install_dir: '/opt/miniconda3'
+miniconda_bin_dir: '{{ miniconda_install_dir }}/bin'
+miniconda_envs_dir: '{{ miniconda_install_dir }}/envs'
+miniconda_conda_binary: '{{ miniconda_bin_dir }}/conda'
+miniconda_installer_dl_dir: '{{ miniconda_install_dir }}/_downloaded-installer'
+miniconda_dl_uri_base: 'https://repo.anaconda.com/miniconda/'
+miniconda_dl_uri_file: 'Miniconda3-latest-Linux-x86_64.sh'
+miniconda_dl_uri: '{{ miniconda_dl_uri_base }}{{ miniconda_dl_uri_file }}'
+miniconda_dl_local_abs: '{{ miniconda_installer_dl_dir }}/{{ miniconda_dl_uri_file }}'
+conda_envs_template_dir: '_conda_envs_dir_'
+conda_python_version: '3'
+conda_package_spec: 'pip'
+python_git_install_path: '/opt/python'

ansible/roles/10-include-50-miniconda/files/_current_user_home_/.condarc

@@ -0,0 +1 @@
+auto_activate_base: false

ansible/roles/10-include-50-miniconda/tasks/40-install-miniconda.yml

@@ -0,0 +1,77 @@
+- name: 'Create Miniconda base dir'
+  loop_control:
+    loop_var: 'conda_dir'
+  loop:
+    - { path: '{{ miniconda_install_dir }}', state: 'directory' }
+    - { path: '{{ miniconda_installer_dl_dir }}', state: 'directory' }
+  ansible.builtin.file:
+    path: '{{ conda_dir.path }}'
+    state: '{{ conda_dir.state }}'
+
+
+
+- name: 'Download latest Miniconda installer'
+  ansible.builtin.get_url:
+    url: '{{ miniconda_dl_uri }}'
+    dest: '{{ miniconda_dl_local_abs }}'
+
+
+
+- name: 'Set execute bit on Miniconda installer'
+  ansible.builtin.file:
+    path: '{{ miniconda_dl_local_abs }}'
+    mode: 'a+x'
+
+
+
+- name: 'Get Miniconda installer help output'
+  register: 'rv_miniconda_installer_help'
+  changed_when: 'rv_miniconda_installer_help.rc != 2'
+  failed_when: 'rv_miniconda_installer_help.rc != 2'
+  ansible.builtin.shell: '{{ miniconda_dl_local_abs }} -h'
+
+
+
+- name: 'Get Miniconda installer version from help output'
+  when: 'rv_miniconda_installer_help.rc == 2'
+  set_fact:
+    miniconda_installer_version: '{{ rv_miniconda_installer_help.stdout | regex_search(''Installs Miniconda3 (py.*?_)(.*?)(\n)'', ''\2'') }}'
+
+
+
+- name: 'Get conda version output'
+  register: 'rv_conda_version_output'
+  changed_when: 'rv_conda_version_output.rc != 0'
+  failed_when: 'false'
+  ansible.builtin.shell: 'conda --version 2> ''/dev/null'''
+
+
+
+- name: 'Get conda version from version output'
+  register: 'rv_conda_version'
+  changed_when: '(not conda_version) or (rv_conda_version_output.rc != 0)'
+  failed_when: 'false'
+  set_fact:
+    conda_version: '{{ rv_conda_version_output.stdout | regex_search(''conda ([^\r\n\f]+)'', ''\1'') }}'
+
+
+
+- name: 'If Miniconda old or missing: install it'
+  when: '(miniconda_installer_version | first is version(''4.11.9'', ''>'')) or (not conda_version)'
+  register: 'rv_miniconda_install'
+  ansible.builtin.shell: '{{ miniconda_dl_local_abs }} -b -p ''{{ miniconda_install_dir }}'' -u'
+
+
+
+- name: 'If Miniconda was installed: Initialize'
+  when: 'rv_miniconda_install.changed'
+  register: 'rv_conda_init'
+  ansible.builtin.shell: '{{ miniconda_conda_binary }} init'
+
+
+
+- name: 'Set ~/.condarc file'
+  ansible.builtin.copy:
+    src: '_current_user_home_/.condarc'
+    dest: '{{ ansible_env.HOME }}/.condarc'
+    mode: '0644'

ansible/roles/10-include-50-miniconda/tasks/50-setup-miniconda-env.yml

@@ -0,0 +1,30 @@
+- name: 'Set conda env file basename'
+  set_fact:
+    conda_env_file_base: '{{ conda_env_file | basename }}'
+
+
+
+- name: 'Set conda env file absolute path'
+  set_fact:
+    conda_env_file_abs: '{{ miniconda_envs_dir }}/{{ conda_env_file_base | regex_search(''.*?\.yml'', ''\0'') | first }}'
+
+
+
+- name: 'Set conda env name'
+  set_fact:
+    conda_env_name: '{{ conda_env_file_base | regex_search(''(.*?)(_conda_environment.yml)'', ''\1'') | first }}'
+
+
+
+- name: 'Store conda environment.yml file'
+  ansible.builtin.template:
+    src: '{{ conda_env_file }}'
+    dest: '{{ conda_env_file_abs }}'
+
+
+
+- name: 'Set up conda env ''{{ conda_env_name }}'''
+  register: 'rv_conda_env_install'
+  changed_when: 'not rv_conda_env_install.stderr'
+  failed_when: 'false'
+  ansible.builtin.shell: '{{ miniconda_conda_binary }} env create --quiet --file ''{{ conda_env_file_abs }}'''

ansible/roles/10-include-50-miniconda/tasks/55-setup-git-repo.yml

@@ -0,0 +1,45 @@
+- name: 'Clone Python package Git repo ''{{ git.repo_url }}'''
+  ansible.builtin.git:
+    repo: '{{ git.repo_url }}'
+    dest: '{{ git.path }}/{{ git.branch }}'
+
+
+
+- name: 'If Git repo has config: Add config'
+  when: 'git.config_file'
+  ansible.builtin.copy:
+    src: '{{ git.config_file }}'
+    dest: '/{{ git.config_file }}'
+    mode: '0644'
+
+
+
+- name: 'If Git repo wants systemd: Copy systemd unit file'
+  when: '(git.systemd.service) and (git.systemd.timer)'
+  register: 'rv_copy_systemd_unit_files'
+  loop_control:
+    loop_var: 'unit'
+    label: '{{ unit.destination }}'
+  loop:
+    - { source: '{{ git.systemd.service }}', destination: '/{{ git.systemd.service }}', mode: '0644' }
+    - { source: '{{ git.systemd.timer }}', destination: '/{{ git.systemd.timer }}', mode: '0644' }
+  ansible.builtin.copy:
+    src: '{{ unit.source }}'
+    dest: '{{ unit.destination }}'
+    mode: '{{ unit.mode }}'
+
+
+
+- name: 'If systemd copy changed: systemctl daemon-reload'
+  when: '(rv_copy_systemd_unit_files.changed)'
+  ansible.builtin.systemd:
+    daemon_reload: 'yes'
+
+
+
+- name: 'If systemd copy succeeded: systemctl enable --now timer unit'
+  when: '(rv_copy_systemd_unit_files is success)'
+  ansible.builtin.systemd:
+    enabled: 'yes'
+    state: 'started'
+    name: '{{ git.systemd.timer | basename }}'

ansible/roles/10-include-50-miniconda/tasks/60-install-packages.yml

@@ -0,0 +1,21 @@
+- name: 'When venv wants packages: Install packages'
+  when: 'python_pip_packages'
+  loop_control:
+    loop_var: 'pip'
+    label: 'Into ''{{ pip.env_name }}'' venv install {{ pip.packages }}'
+  loop: '{{ python_pip_packages }}'
+  ansible.builtin.pip:
+    name: '{{ pip.packages }}'
+    executable: '{{ miniconda_envs_dir }}/{{ pip.env_name }}/bin/pip'
+
+
+
+- name: 'When venv wants requirements: Install requirements'
+  when: 'python_pip_requirements'
+  loop_control:
+    loop_var: 'pip'
+    label: 'Into ''{{ pip.env_name }}'' venv install requirements text file'
+  loop: '{{ python_pip_requirements }}'
+  ansible.builtin.pip:
+    requirements: '{{ pip.requirements }}'
+    executable: '{{ miniconda_envs_dir }}/{{ pip.env_name }}/bin/pip'

ansible/roles/10-include-50-miniconda/tasks/main.yml

@@ -0,0 +1,18 @@
+- import_tasks: '40-install-miniconda.yml'
+- name: 'Set up conda environment'
+  loop_control:
+    loop_var: 'conda_env'
+    label: '{{ conda_env | regex_search(''(.*?)/(.*?)(_conda_environment.yml)'', ''\2'') | first }}'
+  loop: '{{ lookup(''ansible.builtin.fileglob'', ''{{ role_path }}/templates/{{ conda_envs_template_dir }}/*'', wantlist=True) | map(''regex_search'', conda_envs_template_dir + ''/.*?_conda_environment.yml.j2'') }}'
+  vars:
+    conda_env_file: '{{ conda_env }}'
+  include_tasks: '50-setup-miniconda-env.yml'
+- name: 'Set up Git repos'
+  loop_control:
+    loop_var: 'git_package'
+    label: 'From ''{{ git_package.repo_url }}'' clone ''{{ git_package.branch }}'' branch'
+  loop: '{{ python_git_packages }}'
+  vars:
+    git: '{{ git_package }}'
+  include_tasks: '55-setup-git-repo.yml'
+- import_tasks: '60-install-packages.yml'

ansible/roles/10-include-50-miniconda/templates/_conda_envs_dir_/cookiecutter_conda_environment.yml.j2

@@ -0,0 +1,7 @@
+name: cookiecutter
+channels:
+  - defaults
+dependencies:
+  - python=3
+  - pip
+prefix: {{ miniconda_envs_dir }}/cookiecutter

ansible/roles/10-include-50-miniconda/templates/_conda_envs_dir_/update-firewall-source_conda_environment.yml.j2

@@ -0,0 +1,7 @@
+name: update-firewall-source
+channels:
+  - defaults
+dependencies:
+  - python=3
+  - pip
+prefix: {{ miniconda_envs_dir }}/update-firewall-source

ansible/roles/50-service-mailcow/tasks/40-install-mailcow.yml

@@ -15,26 +15,47 @@
 - name: 'Install Docker engine basics'
   ansible.builtin.dnf:
     name:
-      - 'yum-utils'
-      - 'docker-ce'
-      - 'docker-ce-cli'
-      - 'containerd.io'
-      - 'docker-compose-plugin'
-      - 'git'
+      - 'docker-ce'             # Docker
+      - 'docker-ce-cli'         #
+      - 'containerd.io'         #
+      - 'docker-compose-plugin' #
+      - 'git'                   # git clone Mailcow repo
     state: 'latest'
 
 
 
-- name: 'Populate service facts'
+- name: 'Install Miniconda'
   tags:
     - 'testing'
+  ansible.builtin.include_role:
+    name: '10-include-50-miniconda'
+    # tasks_from: '40-install-miniconda'
+
+
+
+#- name: 'Set up Miniconda env'
+#  tags:
+#    - 'testing'
+#  loop_control:
+#    loop_var: 'conda'
+#  loop:
+#    - { env_name: 'cookiecutter', python_version: '3', package_spec: 'pip' }
+#    - { env_name: 'update-firewall-source', python_version: '3', package_spec: 'pip' }
+#  vars:
+#    env_name: '{{ conda.env_name }}'
+#    python_version: '{{ conda.python_version }}'
+#    package_spec: '{{ conda.package_spec }}'
+#  ansible.builtin.include_role:
+#    name: '10-include-50-miniconda'
+#
+
+
+- name: 'Populate service facts'
   ansible.builtin.service_facts:
 
 
 
 - name: 'Store Docker daemon.json'
-  tags:
-    - 'testing'
   register: 'rv_upload_daemonjson'
   ansible.builtin.copy:
     src: 'etc/docker/daemon.json'
@@ -48,8 +69,6 @@
 
 
 - name: 'Start systemd docker.service'
-  tags:
-    - 'testing'
   when: '(ansible_facts.services[''docker.service''].state != ''running'')'
   ansible.builtin.systemd:
     name: 'docker.service'