diff --git a/.gitignore b/.gitignore index 0cead078d..895407e25 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .direnv/ .envrc.local /var/ +/run/ # ignore roles pulled by ansible-galaxy /roles/galaxy/* diff --git a/bin/install-roles.py b/bin/install-roles.py new file mode 100755 index 000000000..2f4b02e4b --- /dev/null +++ b/bin/install-roles.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 MASH project contributors +# SPDX-FileCopyrightText: 2026 Slavi Pantaleev +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Installs the Ansible roles pinned in a requirements file, without throwing away local changes. +# +# agru replaces an installed role whenever its version differs from the pinned one (and ansible-galaxy with `--force` +# always does), losing any changes made to it. This script decides which roles really need to be (re)installed +# and only passes those to the installer: +# +# - missing roles are installed +# - roles at another version than the pinned one are replaced, unless they have local changes +# (detected by comparing their contents with a checksum recorded when this script installed them) +# - roles which are symlinks or contain a `.git` directory are managed by a developer and never touched +# +# With `--all-requirements-path` (a requirements file listing all roles the playbook knows about), +# installed roles which are listed there, but not in the requirements file, are deleted, unless they have local changes. +# Roles which are not listed in either file (e.g. a new role being developed) are never touched. + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys +import tempfile + +try: + import yaml +except ImportError: + sys.exit( + "Installing roles requires the PyYAML Python library. " + "Install it with your package manager (e.g. `python3-yaml` or `python-yaml`) or pip." + ) + +parser = argparse.ArgumentParser( + description="Installs the Ansible roles pinned in a requirements file, without throwing away local changes" +) +parser.add_argument( + "--requirements-path", + help="Path to the requirements file with the roles to install", + required=True, +) +parser.add_argument( + "--all-requirements-path", + help="Path to a requirements file with all known roles. Installed roles listed there, but not in --requirements-path, are deleted", +) +parser.add_argument( + "--roles-path", help="Path to the directory to install roles into", required=True +) +parser.add_argument( + "--state-path", + help="Path to the file recording the checksums of installed roles", + required=True, +) +parser.add_argument( + "--installer", + choices=("auto", "agru", "ansible-galaxy"), + default="auto", + help="Tool to install roles with (auto: agru if available, otherwise ansible-galaxy)", +) +parser.add_argument( + "--force", + action="store_true", + help="Also replace and delete roles with local changes", +) +parser.add_argument( + "--quiet", + action="store_true", + help="Do not report anything when all roles are already up to date", +) + +args = parser.parse_args() + +# Written by agru and ansible-galaxy into each role they install +INSTALL_INFO_RELATIVE_PATH = os.path.join("meta", ".galaxy_install_info") + + +def load_yaml_file(path): + with open(path, "r") as file: + return yaml.safe_load(file) or [] + + +def load_state(path): + if not os.path.isfile(path): + return {} + with open(path, "r") as file: + return (yaml.safe_load(file) or {}).get("roles", {}) + + +def save_state(path, roles): + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + temporary_path = path + ".tmp" + with open(temporary_path, "w") as file: + yaml.safe_dump({"roles": roles}, file, sort_keys=True) + os.replace(temporary_path, path) + + +def is_managed_by_developer(role_path): + return os.path.islink(role_path) or os.path.exists(os.path.join(role_path, ".git")) + + +def installed_version(role_path): + """Returns the version recorded by the installer, or None if the role was not installed by one.""" + info_path = os.path.join(role_path, INSTALL_INFO_RELATIVE_PATH) + if not os.path.isfile(info_path): + return None + with open(info_path, "r") as file: + info = yaml.safe_load(file) or {} + version = info.get("version") + return None if version is None else str(version) + + +def checksum(role_path): + """Returns a checksum of the role's files: their paths, contents, executable bits and symlink targets.""" + digest = hashlib.sha256() + for directory_path, directory_names, file_names in os.walk(role_path): + directory_names.sort() + for name in sorted(file_names): + path = os.path.join(directory_path, name) + relative_path = os.path.relpath(path, role_path) + digest.update(relative_path.encode() + b"\0") + if os.path.islink(path): + digest.update(b"link\0" + os.readlink(path).encode() + b"\0") + continue + digest.update(b"x\0" if os.stat(path).st_mode & 0o111 else b"-\0") + with open(path, "rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + return digest.hexdigest() + + +def has_local_changes(name, role_path, state): + """Tells whether the role was changed since this script installed it. + + Roles without a recorded checksum (installed before this script, or by another tool) are considered unchanged. + """ + recorded = state.get(name) + if recorded is None or recorded.get("version") != installed_version(role_path): + return False + return recorded.get("checksum") != checksum(role_path) + + +warnings_count = 0 + + +def warn(message): + global warnings_count + warnings_count += 1 + print("[WARNING] " + message, file=sys.stderr, flush=True) + + +def install(role_definitions, roles_path, installer): + """Installs the given roles, which must not be present in roles_path.""" + if installer == "auto": + installer = "agru" if shutil.which("agru") else "ansible-galaxy" + + with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False) as file: + yaml.safe_dump(role_definitions, file) + requirements_path = file.name + try: + if installer == "agru": + command = ["agru", "-r", requirements_path, "-p", roles_path, "-no-tui"] + else: + print( + "[NOTE] You are using the standard ansible-galaxy tool to install roles, which is slow and lacks other features. " + "We recommend installing the 'agru' tool to speed up the process: https://github.com/etkecc/agru#where-to-get", + flush=True, + ) + command = ["ansible-galaxy", "install", "-r", requirements_path, "-p", roles_path, "--force"] + result = subprocess.run(command) + if result.returncode != 0: + sys.exit("Installing roles failed ({0} exited with code {1})".format(installer, result.returncode)) + finally: + os.unlink(requirements_path) + + +role_definitions = load_yaml_file(args.requirements_path) +known_role_names = {definition["name"] for definition in role_definitions} +if args.all_requirements_path is not None: + known_role_names |= {definition["name"] for definition in load_yaml_file(args.all_requirements_path)} + +state = load_state(args.state_path) +os.makedirs(args.roles_path, exist_ok=True) + +to_install = [] +to_delete = [] +for definition in role_definitions: + name = definition["name"] + role_path = os.path.join(args.roles_path, name) + pinned_version = str(definition["version"]) + + if not os.path.lexists(role_path): + to_install.append(definition) + continue + if is_managed_by_developer(role_path): + continue + + version = installed_version(role_path) + if version == pinned_version: + continue + + if version is None: + reason = "was not installed by agru or ansible-galaxy" + elif has_local_changes(name, role_path, state): + reason = "has local changes" + else: + reason = None + + if reason is not None and not args.force: + warn( + "{0} {1}, so it was not replaced with {2}. Run `just roles --force` to replace it.".format( + role_path, reason, pinned_version + ) + ) + continue + + to_delete.append(role_path) + to_install.append(definition) + +required_role_names = {definition["name"] for definition in role_definitions} +for name in sorted(known_role_names - required_role_names): + role_path = os.path.join(args.roles_path, name) + if not os.path.lexists(role_path) or is_managed_by_developer(role_path): + continue + if installed_version(role_path) is None: + # Most likely put there by hand, so leave it alone + continue + if has_local_changes(name, role_path, state) and not args.force: + warn( + "{0} is no longer used, but has local changes, so it was not deleted. Run `just roles --force` to delete it.".format( + role_path + ) + ) + continue + to_delete.append(role_path) + print("Deleting {0}, which is no longer used".format(role_path), flush=True) + +for role_path in to_delete: + shutil.rmtree(role_path) + state.pop(os.path.basename(role_path), None) + +if len(to_install) > 0: + print( + "Installing {0} role(s): {1}".format( + len(to_install), ", ".join(definition["name"] for definition in to_install) + ), + flush=True, + ) + install(to_install, args.roles_path, args.installer) + for definition in to_install: + role_path = os.path.join(args.roles_path, definition["name"]) + state[definition["name"]] = { + "version": installed_version(role_path), + "checksum": checksum(role_path), + } +elif len(to_delete) == 0 and warnings_count == 0 and not args.quiet: + print("All roles are up to date.") + +# Forget roles which are gone +state = {name: recorded for name, recorded in state.items() if os.path.isdir(os.path.join(args.roles_path, name))} +save_state(args.state_path, state) diff --git a/justfile b/justfile index db668f283..a6767027c 100644 --- a/justfile +++ b/justfile @@ -12,6 +12,9 @@ mise_data_dir := env("MISE_DATA_DIR", justfile_directory() / "var/mise") mise_trusted_config_paths := justfile_directory() / "mise.toml" prek_home := env("PREK_HOME", justfile_directory() / "var/prek") +# Versions and checksums of the roles installed into roles/galaxy, for detecting local changes to them +installed_roles_file_path := justfile_directory() / "run/installed-roles.yml" + # Shows help default: @{{ just_executable() }} --list --justfile "{{ justfile() }}" @@ -20,10 +23,10 @@ default: add-inventory-host domain server_address: @{{ justfile_directory() }}/bin/add-inventory-host.sh {{ quote(domain) }} {{ quote(server_address) }} -# Pulls external Ansible roles -roles: +# Pulls external Ansible roles (--force also replaces roles with local changes) +roles *flags: @echo "[NOTE] This command just updates the roles, but if you want to update everything at once (playbook, roles, etc.) - use 'just update'" - @{{ just_executable() }} --justfile "{{ justfile() }}" _install-roles + @{{ just_executable() }} --justfile "{{ justfile() }}" _install-roles {{ flags }} # Updates the playbook and installs the necessary Ansible roles pinned in requirements.yml. If a -u flag is passed, also updates the requirements.yml file with new role versions (if available) update *flags: @@ -41,12 +44,11 @@ update *flags: {{ just_executable() }} --justfile "{{ justfile() }}" update-playbook-only if [ "{{ flags }}" = "-u" ]; then - # This updates the versions pinned in requirements.yml and installs the roles - echo "Updating roles and pinning new versions in requirements.yml…" - agru -p roles/galaxy/ -no-tui -u - else - {{ just_executable() }} --justfile "{{ justfile() }}" _install-roles + # Only updates the versions pinned in requirements.yml. The roles are installed below. + echo "Pinning new role versions in requirements.yml…" + agru -p roles/galaxy/ -no-tui -i=false -u fi + {{ just_executable() }} --justfile "{{ justfile() }}" _install-roles # Updates the playbook without installing/updating Ansible roles update-playbook-only: @@ -150,19 +152,13 @@ molecule *args: molecule-clean *args: @{{ justfile_directory() }}/bin/molecule-clean.sh {{ args }} -# Internal - installs the Ansible roles pinned in requirements.yml -_install-roles: - #!/usr/bin/env sh - set -eu - if [ -x "$(command -v agru)" ]; then - echo "Installing roles pinned in requirements.yml…" - agru -p roles/galaxy/ -no-tui - else - echo "[NOTE] You are using the standard ansible-galaxy tool to install roles, which is slow and lacks other features. We recommend installing the 'agru' tool to speed up the process: https://github.com/etkecc/agru#where-to-get" - echo "Installing roles…" - rm -rf roles/galaxy - ansible-galaxy install -r requirements.yml -p roles/galaxy/ --force - fi +# Internal - installs the roles in requirements.yml, keeping roles with local changes +_install-roles *flags: + @{{ justfile_directory() }}/bin/ansible-python.sh {{ justfile_directory() }}/bin/install-roles.py \ + --requirements-path={{ justfile_directory() }}/requirements.yml \ + --roles-path={{ justfile_directory() }}/roles/galaxy \ + --state-path={{ quote(installed_roles_file_path) }} \ + {{ flags }} # Internal - ensures var/mise and var/prek directories exist _ensure_mise_data_directory: